def start(self): # initiate the download, check for status errors, and calculate download size try: self.response = self.session.get(self.source, stream=True, timeout=self.timeout) self.response.raise_for_status() except Exception as e: retry = retry_import(e) if not retry: raise # Catch exceptions to check if we should resume file downloading self.resume() if self.cancel_check(): self._kill_gracefully() try: self.total_size = int(self.response.headers["content-length"]) except KeyError: # When a compressed file is saved on Google Cloud Storage, # content-length is not available in the header, # but we can use X-Goog-Stored-Content-Length. gcs_content_length = self.response.headers.get( "X-Goog-Stored-Content-Length") if gcs_content_length: self.total_size = int(gcs_content_length) else: # Get size of response content when file is compressed through nginx. self.total_size = len(self.response.content) self.started = True
def resume(self): logger.info("Waiting 30s before retrying import: {}".format( self.source)) for i in range(30): if self.cancel_check(): logger.info("Canceling import: {}".format(self.source)) return sleep(1) try: byte_range_resume = None # When internet connection is lost at the beginning of start(), # self.response does not get an assigned value if hasattr(self, "response"): # Use Accept-Ranges and Content-Length header to check if range # requests are supported. For example, range requests are not # supported on compressed files byte_range_resume = self.response.headers.get( "accept-ranges", None) and self.response.headers.get( "content-length", None) resume_headers = self.response.request.headers # Only use byte-range file resuming when sources support range requests if byte_range_resume: range_headers = { "Range": "bytes={}-".format(self.transferred_size) } resume_headers.update(range_headers) self.response = self.session.get( self.source, headers=resume_headers, stream=True, timeout=self.timeout, ) else: self.response = self.session.get(self.source, stream=True, timeout=self.timeout) self.response.raise_for_status() self._content_iterator = self.response.iter_content( self.block_size) # Remove the existing content in dest_file_object when range requests are not supported if byte_range_resume is None: self.dest_file_obj.seek(0) self.dest_file_obj.truncate() except Exception as e: logger.error("Error reading download stream: {}".format(e)) retry = retry_import(e) if not retry: raise self.resume()
def next(self): if self.cancel_check(): self._kill_gracefully() try: chunk = super(FileDownload, self).next() self.transferred_size = self.transferred_size + self.block_size return chunk except Exception as e: retry = retry_import(e) if not retry: raise logger.error("Error reading download stream: {}".format(e)) self.resume() return self.next()
def _start_file_transfer(self, f, filetransfer, overall_progress_update): """ Start to transfer the file from network/disk to the destination. Return value: * True, FILE_TRANSFERRED - successfully transfer the file. * True, FILE_SKIPPED - the file does not exist so it is skipped. * True, FILE_NOT_TRANSFERRED - the transfer is cancelled. * False, FILE_NOT_TRANSFERRED - the transfer fails and needs to retry. """ try: # Save the current progress value original_value = self.progresstrackers[0].progress original_progress = self.progresstrackers[0].get_progress() with filetransfer, self.start_progress( total=filetransfer.total_size) as file_dl_progress_update: for chunk in filetransfer: if self.is_cancelled(): filetransfer.cancel() return True, FILE_NOT_TRANSFERRED length = len(chunk) overall_progress_update(length) file_dl_progress_update(length) # Ensure that if for some reason the total file size for the transfer # is less than what we have marked in the database that we make up # the difference so that the overall progress is never incorrect. # This could happen, for example for a local transfer if a file # has been replaced or corrupted (which we catch below) overall_progress_update(f.file_size - filetransfer.total_size) # If checksum of the destination file is different from the localfile # id indicated in the database, it means that the destination file # is corrupted, either from origin or during import. Skip importing # this file. checksum_correctness = compare_checksums( filetransfer.dest, f.id) if not checksum_correctness: e = "File {} is corrupted.".format(filetransfer.source) logger.error( "An error occurred during content import: {}".format( e)) os.remove(filetransfer.dest) return True, FILE_SKIPPED return True, FILE_TRANSFERRED except Exception as e: logger.error( "An error occurred during content import: {}".format(e)) retry = retry_import(e, skip_404=True) if retry: # Restore the previous progress so that the progress bar will # not reach over 100% later self.progresstrackers[0].progress = original_value self.progresstrackers[0].update_callback( original_progress.progress_fraction, original_progress) logger.info( "Waiting for 30 seconds before retrying import: {}\n". format(filetransfer.source)) sleep(30) return False, FILE_NOT_TRANSFERRED else: overall_progress_update(f.file_size) return True, FILE_SKIPPED