def __get_analysis_by_run_id(self, run_id):
request_url = self.request_url.format(run_id=run_id)
try:
single_run_meta = self.runs.get_single_run(run_id)
if not single_run_meta:
raise CosmosidException('Response from service is empty for '
'run id %s' % run_id)
if not single_run_meta['status']:
raise NotFoundException(single_run_meta['message'])
results = requests.get(request_url, headers=self.header)
if results.status_code == 403:
raise AuthenticationFailed('Authentication Failed. '
'Wrong API Key.')
if results.status_code == 404:
result_data = results.json()
result_data.update({'status': 0})
result_data.update({'run_meta': single_run_meta})
return result_data
if requests.codes.ok:
result_data = results.json()
result_data.update({'status': 1})
result_data.update({'run_meta': single_run_meta})
return result_data
results.raise_for_status()
except AuthenticationFailed:
self.logger.error('Authentication Failed')
except NotFoundException:
self.logger.error('Not Found')
except CosmosidException:
self.logger.error('Got Analysis data exception.')
except requests.exceptions.RequestException:
self.logger.debug('Debug', exc_info=True)
self.logger.error('Error occured during request')
self.logger.error('Response Status Code: %s', results.status_code)
def get_runs_list(self, file_id=None):
sample_runs_url = "{}{}".format(
self.base_url, self.__resource_path.format(file_id=file_id))
results = {}
try:
file_metadata = self.get_file(file_id)
if not file_metadata:
raise CosmosidException('Response from service is empty '
'for file id {}'.format(file_id))
if not file_metadata['status']:
raise NotFoundException(file_metadata['message'])
results = requests.get(sample_runs_url, headers=self.auth_header)
if results.status_code == 400:
if json.loads(results.text)['error_code'] == 'NotUUID':
raise NotFound('Wrong ID.')
if results.status_code == 404:
results = results.json()
results.update({'status': 0})
results.update({'file_name': file_metadata['name']})
return results
if results.status_code == 403:
raise AuthenticationFailed('Authentication Failed. '
'Wrong API Key.')
results.raise_for_status()
if requests.codes.ok:
results = results.json()
results.update({'status': 1})
results.update({'file_name': file_metadata['name']})
return results
except AuthenticationFailed as err:
utils.log_traceback(err)
except NotFound as err:
utils.log_traceback(err)
except requests.exceptions.RequestException as err:
self.logger.error('Error occured during request')
self.logger.error('Response Status Code: %s', results.status_code)
utils.log_traceback(err)
except NotFoundException as err:
utils.log_traceback(err)
except CosmosidException as err:
self.logger.error('Got runs list exception')
utils.log_traceback(err)
except Exception as err:
self.logger.error('Client exception occured')
utils.log_traceback(err)
def directory_list(self, parent):
""""get listing of appropriate directory."""
file_obj = Files(base_url=self.base_url, api_key=self.api_key)
try:
res = file_obj.get_list(parent_id=parent)
if res:
if res['status']:
return res
else:
raise NotFoundException(res['message'])
else:
raise CosmosidException('Response from service is empty '
'for directory {}'.format(parent))
except NotFoundException as err:
utils.log_traceback(err)
except CosmosidException as err:
self.logger.error('Get directory list exception')
utils.log_traceback(err)
except Exception as err:
self.logger.error("Failed to get listing of directory %s", parent)
utils.log_traceback(err)
def get_last_run_for_file(self, file_id):
try:
runs_list = self.get_runs_list(file_id=file_id)
if not runs_list:
raise CosmosidException('Error occurred on get list of runs '
'for a File: {}'.format(file_id))
if not runs_list['status']:
raise NotFoundException(runs_list['message'])
sorted_runs_list = sorted(runs_list['runs'],
key=itemgetter('created'),
reverse=True)
return sorted_runs_list[0]
except NotFoundException as err:
self.logger.error('NotFound')
utils.log_traceback(err)
except CosmosidException:
self.logger.error('Runs list exception.')
return None
except Exception as err:
self.logger.error('Client exception occured')
utils.log_traceback(err)
def sample_run_list(self, file_id):
"""Get list of runs for a given file id."""
sample_runs = Runs(base_url=self.base_url, api_key=self.api_key)
try:
sample_run_list = sample_runs.get_runs_list(file_id=file_id)
if sample_run_list:
if sample_run_list['status']:
return sample_run_list
else:
raise NotFoundException(sample_run_list['message'])
else:
raise CosmosidException('Error occurred on get list of runs '
'for a File: %s' % file_id)
except NotFoundException as err:
self.logger.error('NotFound')
utils.log_traceback(err)
except CosmosidException as err:
self.logger.error('Get runs list exception')
utils.log_traceback(err)
except Exception as err:
self.logger.error('Client exception occured')
utils.log_traceback(err)
def analysis_list(self, file_id=None, run_id=None):
"""Get list of analysis for a given file id."""
analysis = Analysis(base_url=self.base_url, api_key=self.api_key)
try:
analysis_list = analysis.get_list(file_id=file_id, run_id=run_id)
if analysis_list:
if analysis_list['status']:
return analysis_list
else:
raise NotFoundException(analysis_list['message'])
else:
raise CosmosidException('Error occurred on get list of '
'analysis for a File: %s'
% file_id)
except NotFoundException as err:
self.logger.error('NotFound')
utils.log_traceback(err)
except CosmosidException as err:
self.logger.error('Get analysis list exception')
utils.log_traceback(err)
except Exception as err:
self.logger.error('Client exception occured')
utils.log_traceback(err)
def upload_file(**kwargs):
"""Upload manager."""
filename = kwargs.pop('file')
parent_id = kwargs.pop('parent_id', None)
multipart_chunksize = file_size = os.stat(filename)[
6] #get size of file in bytes
client = kwargs['client']
if file_size > MULTIPART_THRESHOLD: #bigger that 1GB
multipart_chunksize = min(int(file_size / 10), int(MAX_CHUNK_SIZE))
multipart_chunksize = max(multipart_chunksize, int(MIN_CHUNK_SIZE))
LOGGER.info('File size: %s MB', file_size / MB)
LOGGER.info('Chunk size: %s MB', int(multipart_chunksize / MB))
config = TransferConfig(multipart_threshold=MULTIPART_THRESHOLD,
max_concurrency=MAX_CONCURRENCY,
multipart_chunksize=multipart_chunksize)
osutil = OSUtilsWithCallbacks()
# Check if given parent folder exists
if parent_id:
fl_obj = Files(base_url=kwargs['base_url'], api_key=kwargs['api_key'])
res = fl_obj.get_list(parent_id=parent_id)
if not res['status']:
raise NotFoundException('Parent folder for upload does '
'not exists.')
transfer_manager = TransferManager(client, config=config, osutil=osutil)
subscribers = [
ProgressSubscriber(filename),
]
_, file_name = os.path.split(filename)
try:
init_url = client.base_url + urls.UPLOAD_INIT_URL
response = requests_retry_session().put(init_url,
json=dict(file_name=file_name),
headers=client.header)
if response.status_code == 402:
raise NotEnoughCredits('Insufficient credits for upload.')
if response.status_code == 403:
raise AuthenticationFailed('Authentication Failed. Wrong API Key.')
if response.status_code == requests.codes.ok:
sources = response.json()
future = transfer_manager.upload(filename,
sources['upload_source'],
sources['upload_key'],
extra_args=None,
subscribers=subscribers)
else:
LOGGER.error(
'File upload inititalisation Failed. '
'Response code: %s', response.status_code)
raise UploadException('File upload inititalisation Failed. '
'Response code: %s' % response.status_code)
try:
future.result()
except KeyboardInterrupt:
do_not_retry_event.set()
return
return sources['upload_key']
# If a client error was raised, add the backwards compatibility layer
# that raises a S3UploadFailedError. These specific errors were only
# ever thrown for upload_parts but now can be thrown for any related
# client error.
except ClientError as error:
raise S3UploadFailedError("Failed to upload {} to {}: {}".format(
filename,
'/'.join([sources['upload_source'],
sources['upload_key']]), error))
def save_report(self, out_file=None, out_dir=None, block_size=8192):
"""Save file for given url to disk."""
def _download_helper(response, out_f):
if block_size is None:
buffer = response.read()
out_f.write(buffer)
else:
file_size_dl = 0
while True:
buffer = response.read(block_size)
if not buffer:
break
file_size_dl += len(buffer)
out_f.write(buffer)
try:
if out_dir:
out_dir = expanduser(normpath(out_dir))
if not isdir(out_dir):
raise NotFoundException('Destination directory does '
'not exist: {}'.format(out_dir))
else:
out_dir = os.getcwd()
report_data = self.get_report_url()
if not report_data['url']:
raise Exception('Empty report Url received')
if report_data['url'] == 1:
raise NotFoundException(
'Report can not be generated for file with NON SUCCESS status.'
)
parsed_url = urlparse(report_data['url'])
_, file_name = split(parsed_url.path)
if out_file:
out_file = expanduser(normpath(out_file))
out_dir, file_name = split(out_file)
_, extension = splitext(file_name)
file_name = (file_name if extension == '.zip' else join(file_name +
'.zip'))
out_file = join(out_dir, file_name)
if isfile(out_file):
raise FileExistsException('Destination File exists: %s' %
out_file)
with open(out_file, "wb") as output:
with urllib.request.urlopen(report_data['url']) as response:
_download_helper(response, output)
return ({'status': 1, 'saved_report': out_file})
except AuthenticationFailed as auth_error:
self.logger.error('Authentication Failed')
return {'status': 0, 'message': auth_error}
except NotFoundException as nfe:
self.logger.error('Not Found')
return {'status': 0, 'message': nfe}
except FileExistsException as file_exists_error:
self.logger.error('File Exists')
return {'status': 0, 'message': file_exists_error}
except Exception as error:
self.logger.error('Could not save report')
return {'status': 0, 'message': error}