def clean_session(bank, session):
dserv = DownloadService(config_file, rabbitmq=False)
biomaj_file_info = downmessage_pb2.DownloadFile()
biomaj_file_info.bank = bank
biomaj_file_info.session = session
dserv.clean(biomaj_file_info)
return jsonify({'msg': 'session cleared'})
def list_result(bank, session):
'''
Get file listing for bank and session, using FileList protobuf serialized string
'''
dserv = DownloadService(config_file, rabbitmq=False)
biomaj_file_info = downmessage_pb2.DownloadFile()
biomaj_file_info.bank = bank
biomaj_file_info.session = session
biomaj_file_info.local_dir = '/tmp'
list_elts = dserv.list_result(biomaj_file_info, protobuf_decode=False)
return jsonify({'files': list_elts})
def download_error(bank, session):
'''
Get errors info for bank and session
'''
dserv = DownloadService(config_file, rabbitmq=False)
biomaj_file_info = downmessage_pb2.DownloadFile()
biomaj_file_info.bank = bank
biomaj_file_info.session = session
biomaj_file_info.local_dir = '/tmp'
errors = dserv.download_errors(biomaj_file_info)
return jsonify({'error': errors})
def download_status(bank, session):
'''
Get number of downloads and errors for bank and session. Progress includes successful download and errored downloads.
'''
dserv = DownloadService(config_file, rabbitmq=False)
biomaj_file_info = downmessage_pb2.DownloadFile()
biomaj_file_info.bank = bank
biomaj_file_info.session = session
biomaj_file_info.local_dir = '/tmp'
(progress, errors) = dserv.download_status(biomaj_file_info)
return jsonify({'progress': progress, 'errors': errors})
def list_status(bank, session):
'''
Check if listing request is over
'''
dserv = DownloadService(config_file, rabbitmq=False)
biomaj_file_info = downmessage_pb2.DownloadFile()
biomaj_file_info.bank = bank
biomaj_file_info.session = session
biomaj_file_info.local_dir = '/tmp'
status = dserv.list_status(biomaj_file_info)
return jsonify({'status': status})
def download_remote_files(self, cf, downloaders, offline_dir):
'''
cf = Config
downloaders = list of downloader
offline_dir = base dir to download files
'''
for downloader in downloaders:
for file_to_download in downloader.files_to_download:
operation = downmessage_pb2.Operation()
operation.type = 1
message = downmessage_pb2.DownloadFile()
message.bank = self.bank
message.session = self.session
message.local_dir = offline_dir
remote_file = downmessage_pb2.DownloadFile.RemoteFile()
protocol = downloader.protocol
remote_file.protocol = downmessage_pb2.DownloadFile.Protocol.Value(
protocol.upper())
remote_file.server = downloader.server
if cf.get('remote.dir'):
remote_file.remote_dir = cf.get('remote.dir')
else:
remote_file.remote_dir = ''
remote_file.credentials = downloader.credentials
biomaj_file = remote_file.files.add()
biomaj_file.name = file_to_download['name']
if 'root' in file_to_download and file_to_download['root']:
biomaj_file.root = file_to_download['root']
if 'param' in file_to_download and file_to_download['param']:
for key in list(file_to_download['param'].keys()):
param = remote_file.param.add()
param.name = key
param.value = file_to_download['param'][key]
if 'save_as' in file_to_download and file_to_download[
'save_as']:
biomaj_file.save_as = file_to_download['save_as']
if 'url' in file_to_download and file_to_download['url']:
biomaj_file.url = file_to_download['url']
if 'permissions' in file_to_download and file_to_download[
'permissions']:
biomaj_file.metadata.permissions = file_to_download[
'permissions']
if 'size' in file_to_download and file_to_download['size']:
biomaj_file.metadata.size = file_to_download['size']
if 'year' in file_to_download and file_to_download['year']:
biomaj_file.metadata.year = file_to_download['year']
if 'month' in file_to_download and file_to_download['month']:
biomaj_file.metadata.month = file_to_download['month']
if 'day' in file_to_download and file_to_download['day']:
biomaj_file.metadata.day = file_to_download['day']
if 'hash' in file_to_download and file_to_download['hash']:
biomaj_file.metadata.hash = file_to_download['hash']
if 'md5' in file_to_download and file_to_download['md5']:
biomaj_file.metadata.md5 = file_to_download['md5']
message.http_method = downmessage_pb2.DownloadFile.HTTP_METHOD.Value(
downloader.method.upper())
timeout_download = cf.get('timeout.download', None)
if timeout_download:
try:
message.timeout_download = int(timeout_download)
except Exception:
logging.error(
'Invalid timeout value, not an integer, skipping')
message.remote_file.MergeFrom(remote_file)
operation.download.MergeFrom(message)
self.download_remote_file(operation)