def take_action(self, parsed_args):
try:
cfg = config.getConfig()
if parsed_args.user in cfg.auth.keys():
self.app.stdout.write(
'Add auth fail, user %s has exists!\n' %
parsed_args.user)
log.error(
'Add auth fail, user %s has exists!' %
parsed_args.user)
return 1
ret, errMsg = cfg.AddAuth(
user=parsed_args.user,
auth_type=parsed_args.auth_type,
obs_endpoint=parsed_args.obs_endpoint,
ak=parsed_args.ak,
sk=parsed_args.sk,
perm=parsed_args.perm)
if not ret:
for msg in errMsg:
self.app.stderr.write(msg)
log.error(msg)
return 1
else:
self.app.stdout.write('Add auth successfully!\n')
log.info('Add auth successfully!')
return 0
except Exception as e:
log.exception(e)
def getLogger(name: str = 'obssftp') -> logging.Logger:
""" 运行日志初始化 """
logger = logging.getLogger(name)
if logger.hasHandlers():
return logger
try:
cfg = config.getConfig()
formatter = UTCFormatter(
'%(asctime)s UTC ' \
'[%(levelname)-8s] ' \
'[%(filename)-11s:%(lineno)-3d] ' \
'[Proc: %(process)d, %(processName)s] ' \
'[Thread: %(thread)d, %(threadName)s] ' \
'%(message)s')
logger.setLevel(VALID_LEVELS[cfg.log['level']])
logpath = cfg.log['logpath']
logdir = os.path.dirname(logpath)
logrotate_num = int(cfg.log['logrotate_num'])
rotate_value, rotate_unit = re.search(
r'^(\d+)\s*(?:([MmKk])?)?$', cfg.log['logrotate_size']).groups()
logrotate_size = int(rotate_value) * const.UNIT_DICT[rotate_unit.upper()]
if not os.path.exists(logdir):
os.makedirs(logdir, mode=0o750)
fh = ObsSftpFileRotatingFileHandler(
logpath, maxBytes=logrotate_size, backupCount=logrotate_num)
fh.setFormatter(formatter)
logger.addHandler(fh)
return logger
except Exception as e:
import traceback
traceback.print_exc()
def get_buffer_size():
"""receive client data larger than 10MB, will sync upload"""
cfg = config.getConfig()
buffer_value, buffer_unit = re.search(
r'^(\d+)\s*(?:([gGMmKk])?)?$', cfg.auth.get('buffer_size')).groups()
buffer_size = int(buffer_value.strip()) * const.UNIT_DICT.get(buffer_unit.upper())
return buffer_size
def main():
cfg = config.getConfig()
listen_address = cfg.state.get('listen_address')
listen_port = int(cfg.state.get('listen_port'))
keypath = const.HOST_PRIVATE_KEY_PATH
client_num = int(cfg.state.get('client_num'))
startServer(listen_address, listen_port, keypath, 'INFO', client_num)
def get_parser(self, prog_name):
try:
parser = super(ChangeState, self).get_parser(prog_name)
cfg = config.getConfig()
for key in cfg.state.keys():
cmd_name = '--{}'.format(key)
parser.add_argument(
cmd_name, dest=key, nargs='?', default=None)
return parser
except Exception as e:
log.exception(e)
def get_parser(self, prog_name):
try:
parser = super(ChangeAuth, self).get_parser(prog_name)
cfg = config.getConfig()
parser.add_argument('--user', dest='user', nargs='?', default=None)
for key in list(cfg.auth.values())[0].keys():
cmd_name = '--{}'.format(key)
parser.add_argument(
cmd_name, dest=key, nargs='?', default=None)
return parser
except Exception as e:
log.exception(e)
def take_action(self, parsed_args):
try:
cfg = config.getConfig()
user = parsed_args.user
if not user or user not in cfg.auth.keys():
self.app.stderr.write('%s is invalid!\n' % user)
log.error(('%s is invalid!' % user))
return 1
cfg.delAuth(user)
return 0
except Exception as e:
log.exception(e)
def getAuditLogger():
auditLogger = AuditLogger('obssftp_audit')
auditFormatter = AuditFormatter()
audditlog_path = config.getConfig().log['auditlogpath']
auditlog_dir = os.path.dirname(audditlog_path)
if not os.path.exists(auditlog_dir):
os.makedirs(auditlog_dir)
fh_audit = TimeRotatingCSVHandler(audditlog_path, when='H', interval=1, backupCount=48)
fh_audit.setLevel(logging.DEBUG)
fh_audit.setFormatter(auditFormatter)
auditLogger.addHandler(fh_audit)
return auditLogger
def open(self, path, flags, attr):
""" open just for file
:param path: file path
:param flags: SSH_FXF_READ|SSH_FXF_WRITE|SSH_FXF_APPEND|SSH_FXF_CREAT|SSH_FXF_TRUNC|SSH_FXF_EXCL
:param attr: not useful for obs
:return: file handler
"""
perm = config.getConfig().auth.get(self._userName).get('perm')
# put
if flags & os.O_WRONLY:
if 'put' not in perm.split(','):
self._logClient.log(logging.ERROR,
'user [%s] not supported put operation' , self._userName)
return SFTP_OP_UNSUPPORTED
elif flags == os.O_RDONLY:
if 'get' not in perm.split(','):
self._logClient.log(logging.ERROR,
'user [%s] not supported get operation' , self._userName)
return SFTP_OP_UNSUPPORTED
else:
# TODO read and write and append
raise Exception('Read and Write| Append operation is not support. flags [%d]' % flags)
_path = self._realpath(path)
bucketName, key = ObsSftpUtil.getBucketAndKey(_path)
auditDict = {'obs_start_time': ObsSftpUtil.utcFormater(time.time()), 'obs_bucket': bucketName, 'obs_key': key}
obsAdapter = obsadpter.ObsAdapter(client=self._client, bucketName=bucketName, key=key,
logClient=self._logClient, ar=self._ar, connHost=self._connHost)
if not self._obsAdapters:
self._obsAdapters = []
self._obsAdapters.append(obsAdapter)
self._ar.update(auditDict)
_key = ObsSftpUtil.maybeAddTrailingSlash(key)
# TODO just successfully for object bucket. file bucket need mode for jungle folder
if obsAdapter.dirInfo(bucketName, _key):
self._logClient.log(logging.WARNING,
'Open key [%s] of bucket [%s] failed. error message [Client want to open a dir].', _key,
bucketName)
return SFTP_FAILURE
try:
fobj = ObsSftpFileHandle(flags=flags, obsAdapter=obsAdapter, bucketName=bucketName, key=key,
logClient=self._logClient, sftpServer=self)
return fobj
except Exception as e:
self._logClient.log(logging.ERROR, 'Open key [%s] of bucket [%s] failed. error message [%s] - %s.', key,
bucketName, str(e), traceback.format_exc())
if 'not support' in str(e):
return SFTP_OP_UNSUPPORTED
return SFTP_FAILURE
def take_action(self, parsed_args):
try:
cfg = config.getConfig()
ret, errMsg = cfg.setState(parsed_args.__dict__)
if not ret:
for msg in errMsg:
self.app.stderr.write(msg)
log.error(msg)
return 1
else:
self.app.stdout.write('Change state successfully!\n')
log.info('Change state successfully!')
return 0
except Exception as e:
log.exception(e)
def __init__(self, server, *largs, **kwargs):
""" OBS SFTP server initiate method when auth successfully. includes some step:
1. obtain ak\sk from config;
2. initiate obs client and obs client log;
3. initiate a thread to close idle time greater than idle timeout;
4. if login user has not ak\sk setting, will close resource.
:param server: auth object
:param largs:
:param kwargs:
"""
super(SFTPServerInterface, self).__init__(*largs, **kwargs)
# read ak sk and build client
self._userName = server.username
self._session = server.session
self._transport = server.transport
try:
cfg = config.getConfig()
self._timeout = int(cfg.state.get('timeout'))
self._connHost = '%s_%s' % (server.connIp, server.connPort)
self._endpoint = cfg.auth.get(self._userName).get('obs_endpoint')
ak = cfg.auth.get(self._userName).get('ak')
sk = cfg.auth.get(self._userName).get('sk')
listen_address = cfg.state.get('listen_address')
listen_port = int(cfg.state.get('listen_port'))
self._closeLock = threading.Lock()
self._closed = False
self._ak = ak
self._listenHost = '%s:%s' % (listen_address, listen_port)
self._client = ObsClient(access_key_id=ak, secret_access_key=sk, server=self._endpoint, path_style=True)
self._client.initLog(LogConf(OBS_SDK_LOG_CONFIG), '%s[%s]' % (OBS_SDK_LOG_NAME_PREFIX, self._userName))
self._logClient = logClient
self._logClient.log(logging.INFO, 'Initiate a sftp server for [%s] - ak [%s]', self._userName, ak)
self._ar = {}
self._activeTime = time.time()
self._cleanThread = threading.Thread(target=self._clean, name='trans_%s' % self._connHost).start()
self._obsAdapters = None
except Exception as e:
logClient.log(logging.ERROR, 'Initialize sftp server failed. error message [%s] - %s.', str(e),
traceback.format_exc())
try:
self._transport.close()
if self._cleanThread:
self._cleanThread.close()
except Exception as e:
# ignore
self._logClient.log(logging.WARNING, 'Ignore close error. error message [%s] - %s', str(e),
traceback.format_exc())
def take_action(self, parsed_args):
try:
cfg = config.getConfig()
if parsed_args.type == 'all':
return (
tuple(cfg.auth.keys()) +
tuple(cfg.log.keys()) +
tuple(cfg.state.keys()),
tuple(cfg.auth.values()) +
tuple(cfg.log.values()) +
tuple(cfg.state.values()))
if parsed_args.type == 'auth':
return (cfg.auth.keys(), cfg.auth.values())
if parsed_args.type == 'log':
return (cfg.log.keys(), cfg.log.values())
if parsed_args.type == 'state':
return (cfg.state.keys(), cfg.state.values())
return False
except Exception as e:
log.exception(e)
def get_parser(self, prog_name):
parser = super(Change, self).get_parser(prog_name)
#parser.add_argument('section', nargs='?', default='')
parser.add_argument(
'--type', '-t',
choices=['auth', 'log', 'state'],
dest='type',
nargs='?',
default='all')
cfg = config.getConfig()
for key in list(cfg.auth.values())[0].keys():
cmd_name = '--{}'.format(key)
parser.add_argument(cmd_name, dest=key, nargs='?', default='')
for key in cfg.log.keys():
cmd_name = '--{}'.format(key)
parser.add_argument(cmd_name, dest=key, nargs='?', default='')
for key in cfg.state.keys():
cmd_name = '--{}'.format(key)
parser.add_argument(cmd_name, dest=key, nargs='?', default='')
return parser
def __init__(self, filename, mode='a', when='h', interval=1, backupCount=20, encoding=None):
filename = os.fspath(filename)
self.csv_stream = None
self._rotateFailed = False
# init will call _open
self.baseFilename = filename
self._open_lockfile()
super(TimeRotatingCSVHandler, self).__init__(filename, when, interval, backupCount, utc=True)
self._chmod(self.baseFilename, mode=const.log_file_mode)
self._backupPool = ThreadPoolExecutor(max_workers=1) # just a work
try:
cfg = config.getConfig()
self._bucket = str(cfg.log.get('log_obs_bucket')).strip()
self._endpoint = str(cfg.log.get('log_obs_endpoint')).strip()
self._ak = cfg.log.get('log_ak')
self._sk = cfg.log.get('log_sk')
self._auditPath, self._auditFileName = os.path.split(filename)
except Exception as e:
log.log(logging.ERROR, 'Audit backup must disappeared. error message [%s] - %s.', str(e),
traceback.format_exc())
self.backupLock = Lock('%s_backup.lock' % self.baseFilename, fcntl.LOCK_EX)
def take_action(self, parsed_args):
try:
cfg = config.getConfig()
if parsed_args.user not in cfg.auth.keys():
self.app.stdout.write(
'user [%s] not exists! \n' %
parsed_args.user)
log.error(('user [%s] not exists!' % parsed_args.user))
return 1
ret, errMsg = cfg.setAuth(
parsed_args.__dict__.pop('user'),
parsed_args.__dict__)
if not ret:
for msg in errMsg:
self.app.stderr.write(msg)
log.error(msg)
return 1
else:
self.app.stdout.write('Change auth successfully!\n')
log.info('Change auth successfully!')
return 0
except Exception as e:
log.exception(e)
def take_action(self, parsed_args):
try:
cfg = config.getConfig()
return (cfg.state.keys(), cfg.state.values())
except Exception as e:
log.exception(e)