def work(self): # Exract chunk into temp file for upload purpose if not self.extractChunk(self.offset, self.size): logging.error('Unable to extract chunk for upload') return False dataRange = 'bytes %d-%d/*' % (self.offset, self.offset + self.size - 1) self.retry = self.retries while self.retry > 0: result = awsCommand(self.config, [ 'upload-multipart-part', '--vault-name', self.config['glacier-vault'], '--cli-input-json', '{"uploadId": "' + self.uploadId + '"}', '--body', self.tmpfile, '--range', dataRange ]) if result is not None and result[ 'json'] is not None and 'checksum' in result['json']: if self.checksum != result['json']['checksum']: logging.error('Hash does not match, expected %s got %s.', self.checksum, result['json']['checksum']) else: break else: if 'RequestTimeoutException' in result['error']: logging.warn('Timeout') else: logging.debug('Result was: ' + repr(result)) self.retry = self.retry - 1 logging.warning( '%s @ %d failed to upload, retrying in %d seconds. %d tries left', helper.formatSize(self.size), self.offset, (10 - self.retry) * 30, self.retry) time.sleep((10 - self.retry) * 30) if self.retry == 0: logging.error('Unable to upload %s at offset %d', helper.formatSize(self.size), self.offset) return -1 return self.size
def uploadFiles(config, files, bytes): logging.info("Uploading %d files (%s) to glacier, this may take a while", len(files), helper.formatSize(bytes)) i = 0 d = 0 for file in files: i += 1 file = os.path.join(config["prepdir"], file) if not uploadFile(config, "(%d of %d) " % (i, len(files)), file, d, bytes): return False d += os.path.getsize(file) return True
def uploadFiles(config, files, bytes): logging.info("Uploading %d files (%s) to glacier, this may take a while", len(files), helper.formatSize(bytes)) cmd = ["upload", config["glacier-vault"]] for f in files: cmd.append(os.path.join(config["prepdir"], f)) upload_start = round(time.time()) result = glacierCommand(config, cmd) upload_time = max(round(time.time()) - upload_start, 1) if result is None or "output" not in result or "Uploaded file" not in result[ "output"]: logging.error("Failed to upload files: %s", repr(result)) return False logging.info("Files uploaded @ %s", helper.formatSpeed(bytes / upload_time)) return True
def uploadFile(config, prefix, file, bytesDone=0, bytesTotal=0, withPath=False): if not os.path.exists(file): logging.error('File %s does not exist', file) return False name = file if not withPath: name = os.path.basename(name) size = remain = os.path.getsize(file) # Due to limit of 10000 parts in an upload, we need to make it all fit chunkSize = size / 10000 if chunkSize <= 1024**2: chunkSize = 1024**2 else: # Make sure it's a power of two factor = math.ceil(float(chunkSize) / float(1024**2)) chunkSize = int((1024**2) * factor) chunkSize -= 1 chunkSize |= chunkSize >> 1 chunkSize |= chunkSize >> 2 chunkSize |= chunkSize >> 4 chunkSize |= chunkSize >> 8 chunkSize |= chunkSize >> 16 chunkSize += 1 logging.debug( 'Using chunksize of %s based on size (%s) of the file we\'re uploading', helper.formatSize(chunkSize), helper.formatSize(size)) hashes = hashFile(file, chunkSize) if hashes is None: logging.error('Unable to hash file %s', file) return False # Initiate the upload result = awsCommand(config, [ 'initiate-multipart-upload', '--vault-name', config['glacier-vault'], '--archive-description', name, '--part-size', str(chunkSize) ]) if result is None or result['code'] != 0 or 'uploadId' not in result[ 'json']: logging.error('Unable to initiate upload: %s', repr(result)) return False uploadId = result['json']['uploadId'] # Start sending the file, one megabyte at a time until we have none left offset = 0 block = 0 work = uploadCoordinator(config['glacier-threads']) # Queue up all the work while remain > 0: chunk = remain if chunk > chunkSize: chunk = chunkSize job = uploadJob(config, file, name, offset, chunk, hashes['blocks'][block].hexdigest(), uploadId) work.add(job) block += 1 remain -= chunk offset += chunk # Wait for it... work.process() while not work.isDone(): time.sleep(1) if sys.stdout.isatty(): # Extra spaces at the end to clear remnants when numbers change if work.getSent() > 0 and work.getTime() > 0: timerem = ", " + helper.formatTime( (float(bytesTotal) - float(bytesDone + work.getSent())) / (work.getSent() / work.getTime())) + " remaining" else: timerem = "" sys.stdout.write( '%s%s @ %s, %.2f%% done (%.2f%% total%s) \r' % (prefix, name, helper.formatSpeed(work.getSent() / work.getTime()), float(work.getSent()) / float(size) * 100.0, float(bytesDone + work.getSent()) / float(bytesTotal) * 100.0, timerem)) sys.stdout.flush() if sys.stdout.isatty(): sys.stdout.write('\n') sys.stdout.flush() if not work.finish(): logging.error('Failed to upload the file, aborting') # Note! Should use JSON since plain arguments seems to not work awsCommand(config, [ 'abort-multipart-upload', '--vault-name', config['glacier-vault'], '--cli-input-json', '{"uploadId": "' + uploadId + '"}' ]) return False # Time to finalize this deal result = awsCommand(config, [ 'complete-multipart-upload', '--vault-name', config['glacier-vault'], '--cli-input-json', '{"uploadId": "' + uploadId + '"}', '--checksum', hashes['final'].hexdigest(), '--archive-size', str(size) ]) if result is None or result['code'] != 0: logging.error('Unable to complete upload of %s: %s', file, repr(result)) return False return True
def uploadFiles(config, files, bytes): logging.info("Uploading %d files (%s) to glacier, this may take a while", len(files), helper.formatSize(bytes)) cmd = ["upload", config["glacier-vault"]] for f in files: cmd.append(os.path.join(config["prepdir"], f)) upload_start = round(time.time()) result = glacierCommand(config, cmd) upload_time = max(round(time.time()) - upload_start, 1) if result is None or "output" not in result or "Uploaded file" not in result["output"]: logging.error("Failed to upload files: %s", repr(result)) return False logging.info("Files uploaded @ %s", helper.formatSpeed(bytes / upload_time)) return True