def _call_init_upload(file_name, file_size, metadata, tags, project, samples_resource): """Call init_upload at the One Codex API and return data used to upload the file. Parameters ---------- file_name : `string` The file_name you wish to associate this fastx file with at One Codex. file_size : `integer` Accurate size of file to be uploaded, in bytes. metadata : `dict`, optional tags : `list`, optional project : `string`, optional UUID of project to associate this sample with. samples_resource : `onecodex.models.Samples` Wrapped potion-client object exposing `init_upload` and `confirm_upload` routes to mainline. Returns ------- `dict` Contains, at a minimum, 'upload_url' and 'sample_id'. Should also contain various additional data used to upload the file to fastx-proxy, a user's S3 bucket, or an intermediate bucket. """ upload_args = { "filename": file_name, "size": file_size, "upload_type": "standard", # this is multipart form data } if metadata: # format metadata keys as snake case new_metadata = {} for md_key, md_val in metadata.items(): new_metadata[snake_case(md_key)] = md_val upload_args["metadata"] = new_metadata if tags: upload_args["tags"] = tags if project: upload_args["project"] = getattr(project, "id", project) try: upload_info = samples_resource.init_upload(upload_args) except requests.exceptions.HTTPError as e: raise_api_error(e.response, state="init") except requests.exceptions.ConnectionError: raise_connectivity_error(file_name) return upload_info
def upload_document_fileobj(file_obj, file_name, session, documents_resource, log=None): """Uploads a single file-like object to the One Codex server directly to S3. Parameters ---------- file_obj : `FilePassthru`, or a file-like object If a file-like object is given, its mime-type will be sent as 'text/plain'. Otherwise, `FilePassthru` will send a compressed type if the file is gzip'd or bzip'd. file_name : `string` The file_name you wish to associate this file with at One Codex. fields : `dict` Additional data fields to include as JSON in the POST. session : `requests.Session` Connection to One Codex API. documents_resource : `onecodex.models.Documents` Wrapped potion-client object exposing `init_upload` and `confirm_upload` routes to mainline. Notes ----- In contrast to `upload_sample_fileobj`, this method will /only/ upload to an S3 intermediate bucket--not via our direct proxy or directly to a user's S3 bucket with a signed request. Raises ------ UploadException In the case of a fatal exception during an upload. Returns ------- `string` containing sample UUID of newly uploaded file. """ try: fields = documents_resource.init_multipart_upload() except requests.exceptions.HTTPError as e: raise_api_error(e.response, state="init") except requests.exceptions.ConnectionError: raise_connectivity_error(file_name) s3_upload = _s3_intermediate_upload( file_obj, file_name, fields, session, documents_resource._client._root_url + fields["callback_url"], # full callback url ) document_id = s3_upload.get("document_id", "<UUID not yet assigned>") logging.info("{}: finished as document {}".format(file_name, document_id)) return document_id
def upload_document_fileobj(file_obj, file_name, documents_resource): """Upload a single file-like object to One Codex directly to S3 via an intermediate bucket. Parameters ---------- file_obj : `FilePassthru`, or a file-like object If a file-like object is given, its mime-type will be sent as 'text/plain'. Otherwise, `FilePassthru` will send a compressed type if the file is gzip'd or bzip'd. file_name : `string` The file_name you wish to associate this file with at One Codex. fields : `dict` Additional data fields to include as JSON in the POST. documents_resource : `onecodex.models.Documents` Wrapped potion-client object exposing `init_upload` and `confirm_upload` routes to mainline. Notes ----- In contrast to `upload_sample_fileobj`, this method will /only/ upload to an S3 intermediate bucket--not via our direct proxy or directly to a user's S3 bucket with a signed request. Raises ------ UploadException In the case of a fatal exception during an upload. Returns ------- `string` containing sample UUID of newly uploaded file. """ try: fields = documents_resource.init_multipart_upload() except requests.exceptions.HTTPError as e: raise_api_error(e.response, state="init") except requests.exceptions.ConnectionError: raise_connectivity_error(file_name) s3_upload = _s3_intermediate_upload( file_obj, file_name, fields, documents_resource._client.session, documents_resource._client._root_url + fields["callback_url"], # full callback url ) document_id = s3_upload.get("document_id", "<UUID not yet assigned>") log.info("{}: finished as document {}".format(file_name, document_id)) return document_id
def _upload_document_fileobj(file_obj, file_name, documents_resource): """Upload a single file-like object to One Codex directly to S3 via an intermediate bucket. Parameters ---------- file_obj : `FilePassthru`, or a file-like object If a file-like object is given, its mime-type will be sent as 'text/plain'. Otherwise, `FilePassthru` will send a compressed type if the file is gzip'd or bzip'd. file_name : `string` The file_name you wish to associate this file with at One Codex. documents_resource : `onecodex.models.Documents` Wrapped potion-client object exposing `init_upload` and `confirm_upload` routes to mainline. Raises ------ UploadException In the case of a fatal exception during an upload. Returns ------- `string` containing sample UUID of newly uploaded file. """ try: fields = documents_resource.init_multipart_upload() except requests.exceptions.HTTPError as e: raise_api_error(e.response, state="init") except requests.exceptions.ConnectionError: raise_connectivity_error(file_name) s3_upload = _s3_intermediate_upload( file_obj, file_name, fields, documents_resource._client.session, documents_resource._client._root_url + fields["callback_url"], # full callback url ) msg = "{}: finished".format(file_name) document_id = s3_upload.get("document_id") if document_id is not None: msg += " as document {}".format(document_id) log.info(msg) return document_id
def _call_init_upload(file_name, file_size, metadata, tags, project, samples_resource, sample_id, external_sample_id): """Call init_upload at the One Codex API and return data used to upload the file. Parameters ---------- file_name : `string` The file_name you wish to associate this fastx file with at One Codex. file_size : `integer` Accurate size of file to be uploaded, in bytes. metadata : `dict`, optional tags : `list`, optional project : `string`, optional UUID of project to associate this sample with. samples_resource : `onecodex.models.Samples` Wrapped potion-client object exposing `init_upload` and `confirm_upload` routes to mainline. sample_id : `string`, optional If passed, will upload the file(s) to the sample with that id. Only works if the sample was pre-uploaded external_sample_id : `string`, optional If passed, will upload the file(s) to the sample with that metadata external id. Only works if the sample was pre-uploaded Returns ------- `dict` Contains, at a minimum, 'upload_url' and 'sample_id'. Should also contain various additional data used to upload the file to fastx-proxy, a user's S3 bucket, or an intermediate bucket. """ upload_args = { "filename": file_name, "size": file_size, "upload_type": "standard", # this is multipart form data "sample_id": sample_id, "external_sample_id": external_sample_id, } upload_args.update(build_upload_dict(metadata, tags, project)) try: return samples_resource.init_upload(upload_args) except requests.exceptions.HTTPError as e: raise_api_error(e.response, state="init") except requests.exceptions.ConnectionError: raise_connectivity_error(file_name)
def _s3_intermediate_upload(file_obj, file_name, fields, session, callback_url): """Upload a single file-like object to an intermediate S3 bucket. One Codex will pull the file from S3 after receiving a callback. Parameters ---------- file_obj : `FilePassthru`, or a file-like object In the case of a single file, it will simply be passed through (`FilePassthru`) to One Codex, compressed or otherwise. If a file-like object is given, its mime-type will be sent as 'text/plain'. file_name : `string` The file_name you wish to associate this fastx file with at One Codex. fields : `dict` Additional data fields to include as JSON in the POST. session : `requests.Session` Authenticated connection to One Codex API used to POST callback. callback_url : `string` API callback at One Codex which will trigger a pull from this S3 bucket. Raises ------ UploadException In the case of a fatal exception during an upload. Note we rely on boto3 to handle its own retry logic. Returns ------- `dict` : JSON results from internal confirm import callback URL """ import boto3 from boto3.s3.transfer import TransferConfig from boto3.exceptions import S3UploadFailedError boto3_session = boto3.session.Session() # actually do the upload client = boto3_session.client( "s3", aws_access_key_id=fields["upload_aws_access_key_id"], aws_secret_access_key=fields["upload_aws_secret_access_key"], ) multipart_chunksize = _choose_boto3_chunksize(file_obj) # if boto uses threads, ctrl+c won't work config = TransferConfig(use_threads=False, multipart_chunksize=multipart_chunksize) # let boto3 update our progressbar rather than our FASTX wrappers, if applicable boto_kwargs = {} if hasattr(file_obj, "progressbar"): boto_kwargs["Callback"] = file_obj.progressbar.update file_obj._progressbar = file_obj.progressbar file_obj.progressbar = None for attempt in range(1, 4): try: client.upload_fileobj(file_obj, fields["s3_bucket"], fields["file_id"], ExtraArgs={"ServerSideEncryption": "AES256"}, Config=config, **boto_kwargs) break except S3UploadFailedError as e: logging.debug( "Caught S3UploadFailedError on attempt {}/3: {}".format( attempt, str(e))) logging.error( "{}: Connectivity issue, retrying upload via intermediary ({}/3)..." .format(file_name, attempt)) # rewind the progressbar if possible, then remove so boto3 can update the bar directly if hasattr(file_obj, "_progressbar"): file_obj.progressbar = file_obj._progressbar file_obj.seek(0) file_obj.progressbar = None else: file_obj.seek(0) else: logging.debug("{}: exhausted all retries via intermediary") raise_connectivity_error(file_name) # In paired uploads, we only want to call the callback url once both files are uploaded if not callback_url: return {} # issue a callback try: # retry on 502, 503, 429, with a backoff timing of 4s, 8s, and 16s, False retries on all HTTP methods retry_strategy = Retry(total=3, backoff_factor=4, method_whitelist=False, status_forcelist=[502, 503, 429]) adapter = HTTPAdapter(max_retries=retry_strategy) session.mount(callback_url, adapter) resp = session.post( callback_url, json={ "s3_path": "s3://{}/{}".format(fields["s3_bucket"], fields["file_id"]), "filename": file_name, # "import_as_document": fields.get("import_as_document", False), }, ) except requests.exceptions.ConnectionError: raise_connectivity_error(file_name) if resp.status_code != 200: raise_connectivity_error(file_name) try: return resp.json() except ValueError: return {}
def upload_sequence_fileobj(file_obj, file_name, fields, retry_fields, session, samples_resource): """Upload a single file-like object to One Codex via either fastx-proxy or directly to S3. Parameters ---------- file_obj : `FASTXInterleave`, `FilePassthru`, or a file-like object A wrapper around a pair of fastx files (`FASTXInterleave`) or a single fastx file. In the case of paired files, they will be interleaved and uploaded uncompressed. In the case of a single file, it will simply be passed through (`FilePassthru`) to One Codex, compressed or otherwise. If a file-like object is given, its mime-type will be sent as 'text/plain'. file_name : `string` The file_name you wish to associate this fastx file with at One Codex. fields : `dict` Additional data fields to include as JSON in the POST. Must include 'sample_id' and 'upload_url' at a minimum. retry_fields : `dict` Metadata sent to `init_multipart_upload` in the case that the upload via fastx-proxy fails. session : `requests.Session` Use this session for direct uploads (via proxy or direct to a user's S3 bucket). samples_resource : `onecodex.models.Samples` Wrapped potion-client object exposing `init_upload` and `confirm_upload` routes to mainline. Raises ------ UploadException In the case of a fatal exception during an upload. Returns ------- `string` containing sample ID of newly uploaded file. """ # First attempt to upload via our validating proxy try: sample_id = fields["sample_id"] # Are we being directed to skip the proxy? If so, only do it if files ares <5GB since that's the limit for # direct uploads to S3 if ("AWSAccessKeyId" in fields["additional_fields"] and getattr(file_obj, "_fsize", 0) > 5 * 1024**3): raise RetryableUploadException # Big files are going to skip the proxy even if the backend told us the opposite # 100GB is considered big enough to defer the validation # In some cases, file_obj might be a BytesIO object instead of one of our file object so we # filter them out by checking for a `write` attribute if not hasattr(file_obj, "write") and file_obj.size() > 100 * 1024**3: raise RetryableUploadException _direct_upload(file_obj, file_name, fields, session, samples_resource) except RetryableUploadException: # upload failed -- retry direct upload to S3 intermediate; first try to cancel pending upload try: samples_resource.cancel_upload({"sample_id": sample_id}) except Exception as e: log.debug("Failed to cancel upload: {}".format(e)) log.error( "{}: Connectivity issue, trying upload via intermediary...".format( file_name)) file_obj.seek(0) # reset file_obj back to start try: retry_fields = samples_resource.init_multipart_upload(retry_fields) except requests.exceptions.HTTPError as e: raise_api_error(e.response, state="init") except requests.exceptions.ConnectionError: raise_connectivity_error(file_name) s3_upload = _s3_intermediate_upload( file_obj, file_name, retry_fields, samples_resource._client.session, samples_resource._client._root_url + retry_fields["callback_url"], # full callback url ) sample_id = s3_upload.get("sample_id", "<UUID not yet assigned>") log.info("{}: finished as sample {}".format(file_name, sample_id)) return sample_id
def _direct_upload(file_obj, file_name, fields, session, samples_resource): """Upload a single file-like object via our validating proxy. Maintains compatibility with direct upload to a user's S3 bucket in case our validating proxy is disabled for this user. Parameters ---------- file_obj : `FASTXInterleave`, `FilePassthru`, or a file-like object A wrapper around a pair of fastx files (`FASTXInterleave`) or a single fastx file. In the case of paired files, they will be interleaved and uploaded uncompressed. In the case of a single file, it will simply be passed through (`FilePassthru`) to One Codex, compressed or otherwise. If a file-like object is given, its mime-type will be sent as 'text/plain'. file_name : `string` The file_name you wish to associate this fastx file with at One Codex. fields : `dict` Additional data fields to include as JSON in the POST. Must include 'sample_id' and 'upload_url' at a minimum. session : `requests.Session` Use this session for direct uploads (via proxy or direct to a user's S3 bucket). samples_resource : `onecodex.models.Samples` Wrapped potion-client object exposing `init_upload` and `confirm_upload` routes to mainline. Raises ------ RetryableUploadException In cases where the proxy is temporarily down or we experience connectivity issues UploadException In other cases where the proxy determines the upload is invalid and should *not* be retried. """ # need an OrderedDict to preserve field order for S3, required for Python 2.7 multipart_fields = OrderedDict() for k, v in fields["additional_fields"].items(): multipart_fields[str(k)] = str(v) # this attribute is only in FASTXInterleave and FilePassthru mime_type = getattr(file_obj, "mime_type", "text/plain") multipart_fields["file"] = (file_name, file_obj, mime_type) encoder = MultipartEncoder(multipart_fields) upload_request = None try: upload_request = session.post( fields["upload_url"], data=encoder, headers={"Content-Type": encoder.content_type}, auth={}, ) except requests.exceptions.ConnectionError: pass # If we expect a status *always* try to check it, # waiting up to 4 hours for buffering to complete (~30-50GB file gzipped) if "status_url" in fields["additional_fields"]: now = time.time() while time.time() < (now + 60 * 60 * 4): try: resp = session.post( fields["additional_fields"]["status_url"], json={"sample_id": fields["sample_id"]}, ) resp.raise_for_status() except (ValueError, requests.exceptions.RequestException) as e: log.debug("Retrying due to error: {}".format(e)) raise RetryableUploadException( "Unexpected failure of direct upload proxy. Retrying...") if resp.json() and resp.json().get("complete", True) is False: log.debug( "Blocking on waiting for proxy to complete (in progress)..." ) time.sleep(30) else: break # Return is successfully processed if resp.json().get("code") in [200, 201]: file_obj.close() return elif resp.json().get("code") == 500: log.debug("Retrying due to 500 from proxy...") raise RetryableUploadException( "Unexpected issue with direct upload proxy. Retrying...") else: raise_api_error(resp, state="upload") # Direct to S3 case else: file_obj.close() if upload_request.status_code not in [200, 201]: raise UploadException( "Unknown connectivity issue with direct upload.") # Issue a callback -- this only happens in the direct-to-S3 case try: if not fields["additional_fields"].get("callback_url"): samples_resource.confirm_upload({ "sample_id": fields["sample_id"], "upload_type": "standard" }) except requests.exceptions.HTTPError as e: raise_api_error(e.response, state="callback") except requests.exceptions.ConnectionError: raise_connectivity_error(file_name)
def _s3_intermediate_upload(file_obj, file_name, fields, session, callback_url): """Uploads a single file-like object to an intermediate S3 bucket which One Codex can pull from after receiving a callback. Parameters ---------- file_obj : `FASTXInterleave`, `FilePassthru`, or a file-like object A wrapper around a pair of fastx files (`FASTXInterleave`) or a single fastx file. In the case of paired files, they will be interleaved and uploaded uncompressed. In the case of a single file, it will simply be passed through (`FilePassthru`) to One Codex, compressed or otherwise. If a file-like object is given, its mime-type will be sent as 'text/plain'. file_name : `string` The file_name you wish to associate this fastx file with at One Codex. fields : `dict` Additional data fields to include as JSON in the POST. callback_url : `string` API callback at One Codex which will trigger a pull from this S3 bucket. Raises ------ UploadException In the case of a fatal exception during an upload. Note we rely on boto3 to handle its own retry logic. Returns ------- `dict` : JSON results from internal confirm import callback URL """ import boto3 from boto3.s3.transfer import TransferConfig from boto3.exceptions import S3UploadFailedError # actually do the upload client = boto3.client( "s3", aws_access_key_id=fields["upload_aws_access_key_id"], aws_secret_access_key=fields["upload_aws_secret_access_key"], ) # if boto uses threads, ctrl+c won't work config = TransferConfig(use_threads=False) # let boto3 update our progressbar rather than our FASTX wrappers, if applicable boto_kwargs = {} if hasattr(file_obj, "progressbar"): boto_kwargs["Callback"] = file_obj.progressbar.update file_obj.progressbar = None try: client.upload_fileobj( file_obj, fields["s3_bucket"], fields["file_id"], ExtraArgs={"ServerSideEncryption": "AES256"}, Config=config, **boto_kwargs ) except S3UploadFailedError: raise_connectivity_error(file_name) # issue a callback try: resp = session.post( callback_url, json={ "s3_path": "s3://{}/{}".format(fields["s3_bucket"], fields["file_id"]), "filename": file_name, "import_as_document": fields.get("import_as_document", False), }, ) except requests.exceptions.ConnectionError: raise_connectivity_error(file_name) if resp.status_code != 200: raise_connectivity_error(file_name) try: return resp.json() except ValueError: return {}
def upload_sequence_fileobj(file_obj, file_name, fields, retry_fields, session, samples_resource): """Uploads a single file-like object to the One Codex server via either fastx-proxy or directly to S3. Parameters ---------- file_obj : `FASTXInterleave`, `FilePassthru`, or a file-like object A wrapper around a pair of fastx files (`FASTXInterleave`) or a single fastx file. In the case of paired files, they will be interleaved and uploaded uncompressed. In the case of a single file, it will simply be passed through (`FilePassthru`) to One Codex, compressed or otherwise. If a file-like object is given, its mime-type will be sent as 'text/plain'. file_name : `string` The file_name you wish to associate this fastx file with at One Codex. fields : `dict` Additional data fields to include as JSON in the POST. Must include 'sample_id' and 'upload_url' at a minimum. retry_fields : `dict` Metadata sent to `init_multipart_upload` in the case that the upload via fastx-proxy fails. session : `requests.Session` Connection to One Codex API. samples_resource : `onecodex.models.Samples` Wrapped potion-client object exposing `init_upload` and `confirm_upload` routes to mainline. Raises ------ UploadException In the case of a fatal exception during an upload. Returns ------- `string` containing sample ID of newly uploaded file. """ # First attempt to upload via our validating proxy try: _direct_upload(file_obj, file_name, fields, session, samples_resource) sample_id = fields["sample_id"] except RetryableUploadException: # upload failed -- retry direct upload to S3 intermediate; first try to cancel pending upload try: samples_resource.cancel_upload({"sample_id": sample_id}) except Exception as e: logging.debug("Failed to cancel upload: {}".format(e)) logging.error("{}: Connectivity issue, trying direct upload...".format(file_name)) file_obj.seek(0) # reset file_obj back to start try: retry_fields = samples_resource.init_multipart_upload(retry_fields) except requests.exceptions.HTTPError as e: raise_api_error(e.response, state="init") except requests.exceptions.ConnectionError: raise_connectivity_error(file_name) s3_upload = _s3_intermediate_upload( file_obj, file_name, retry_fields, session, samples_resource._client._root_url + retry_fields["callback_url"], # full callback url ) sample_id = s3_upload.get("sample_id", "<UUID not yet assigned>") logging.info("{}: finished as sample {}".format(file_name, sample_id)) return sample_id
def _direct_upload(file_obj, file_name, fields, session, samples_resource): """Uploads a single file-like object via our validating proxy. Maintains compatibility with direct upload to a user's S3 bucket as well in case we disable our validating proxy. Parameters ---------- file_obj : `FASTXInterleave`, `FilePassthru`, or a file-like object A wrapper around a pair of fastx files (`FASTXInterleave`) or a single fastx file. In the case of paired files, they will be interleaved and uploaded uncompressed. In the case of a single file, it will simply be passed through (`FilePassthru`) to One Codex, compressed or otherwise. If a file-like object is given, its mime-type will be sent as 'text/plain'. file_name : `string` The file_name you wish to associate this fastx file with at One Codex. fields : `dict` Additional data fields to include as JSON in the POST. Must include 'sample_id' and 'upload_url' at a minimum. samples_resource : `onecodex.models.Samples` Wrapped potion-client object exposing `init_upload` and `confirm_upload` routes to mainline. Raises ------ RetryableUploadException In cases where the proxy is temporarily down or we experience connectivity issues UploadException In other cases where the proxy determines the upload is invalid and should *not* be retried. """ # need an OrderedDict to preserve field order for S3, required for Python 2.7 multipart_fields = OrderedDict() for k, v in fields["additional_fields"].items(): multipart_fields[str(k)] = str(v) # this attribute is only in FASTXInterleave and FilePassthru mime_type = getattr(file_obj, "mime_type", "text/plain") multipart_fields["file"] = (file_name, file_obj, mime_type) encoder = MultipartEncoder(multipart_fields) upload_request = None try: upload_request = session.post( fields["upload_url"], data=encoder, headers={"Content-Type": encoder.content_type}, auth={}, ) except requests.exceptions.ConnectionError: pass # If we expect a status *always* try to check it, # waiting up to 4 hours for buffering to complete (~30-50GB file gzipped) if "status_url" in fields["additional_fields"]: now = time.time() while time.time() < (now + 60 * 60 * 4): try: resp = session.post( fields["additional_fields"]["status_url"], json={"sample_id": fields["sample_id"]}, ) resp.raise_for_status() except (ValueError, requests.exceptions.RequestException) as e: logging.debug("Retrying due to error: {}".format(e)) raise RetryableUploadException( "Unexpected failure of direct upload proxy. Retrying..." ) if resp.json() and resp.json().get("complete", True) is False: logging.debug("Blocking on waiting for proxy to complete (in progress)...") time.sleep(30) else: break # Return is successfully processed if resp.json().get("code") in [200, 201]: file_obj.close() return elif resp.json().get("code") == 500: logging.debug("Retrying due to 500 from proxy...") raise RetryableUploadException("Unexpected issue with direct upload proxy. Retrying...") else: raise_api_error(resp, state="upload") # Direct to S3 case else: file_obj.close() if upload_request.status_code not in [200, 201]: raise RetryableUploadException("Unknown connectivity issue with proxy upload.") # Issue a callback -- this only happens in the direct-to-S3 case try: if not fields["additional_fields"].get("callback_url"): samples_resource.confirm_upload( {"sample_id": fields["sample_id"], "upload_type": "standard"} ) except requests.exceptions.HTTPError as e: raise_api_error(e.response, state="callback") except requests.exceptions.ConnectionError: raise_connectivity_error()