def work(self):
# Exract chunk into temp file for upload purpose
if not self.extractChunk(self.offset, self.size):
logging.error('Unable to extract chunk for upload')
return False
dataRange = 'bytes %d-%d/*' % (self.offset,
self.offset + self.size - 1)
self.retry = self.retries
while self.retry > 0:
result = awsCommand(self.config, [
'upload-multipart-part', '--vault-name',
self.config['glacier-vault'], '--cli-input-json',
'{"uploadId": "' + self.uploadId + '"}', '--body',
self.tmpfile, '--range', dataRange
])
if result is not None and result[
'json'] is not None and 'checksum' in result['json']:
if self.checksum != result['json']['checksum']:
logging.error('Hash does not match, expected %s got %s.',
self.checksum, result['json']['checksum'])
else:
break
else:
if 'RequestTimeoutException' in result['error']:
logging.warn('Timeout')
else:
logging.debug('Result was: ' + repr(result))
self.retry = self.retry - 1
logging.warning(
'%s @ %d failed to upload, retrying in %d seconds. %d tries left',
helper.formatSize(self.size), self.offset,
(10 - self.retry) * 30, self.retry)
time.sleep((10 - self.retry) * 30)
if self.retry == 0:
logging.error('Unable to upload %s at offset %d',
helper.formatSize(self.size), self.offset)
return -1
return self.size
def uploadFiles(config, files, bytes):
logging.info("Uploading %d files (%s) to glacier, this may take a while",
len(files), helper.formatSize(bytes))
i = 0
d = 0
for file in files:
i += 1
file = os.path.join(config["prepdir"], file)
if not uploadFile(config, "(%d of %d) " %
(i, len(files)), file, d, bytes):
return False
d += os.path.getsize(file)
return True
def uploadFiles(config, files, bytes):
logging.info("Uploading %d files (%s) to glacier, this may take a while",
len(files), helper.formatSize(bytes))
cmd = ["upload", config["glacier-vault"]]
for f in files:
cmd.append(os.path.join(config["prepdir"], f))
upload_start = round(time.time())
result = glacierCommand(config, cmd)
upload_time = max(round(time.time()) - upload_start, 1)
if result is None or "output" not in result or "Uploaded file" not in result[
"output"]:
logging.error("Failed to upload files: %s", repr(result))
return False
logging.info("Files uploaded @ %s",
helper.formatSpeed(bytes / upload_time))
return True
def uploadFile(config,
prefix,
file,
bytesDone=0,
bytesTotal=0,
withPath=False):
if not os.path.exists(file):
logging.error('File %s does not exist', file)
return False
name = file
if not withPath:
name = os.path.basename(name)
size = remain = os.path.getsize(file)
# Due to limit of 10000 parts in an upload, we need to make it all fit
chunkSize = size / 10000
if chunkSize <= 1024**2:
chunkSize = 1024**2
else:
# Make sure it's a power of two
factor = math.ceil(float(chunkSize) / float(1024**2))
chunkSize = int((1024**2) * factor)
chunkSize -= 1
chunkSize |= chunkSize >> 1
chunkSize |= chunkSize >> 2
chunkSize |= chunkSize >> 4
chunkSize |= chunkSize >> 8
chunkSize |= chunkSize >> 16
chunkSize += 1
logging.debug(
'Using chunksize of %s based on size (%s) of the file we\'re uploading',
helper.formatSize(chunkSize), helper.formatSize(size))
hashes = hashFile(file, chunkSize)
if hashes is None:
logging.error('Unable to hash file %s', file)
return False
# Initiate the upload
result = awsCommand(config, [
'initiate-multipart-upload', '--vault-name', config['glacier-vault'],
'--archive-description', name, '--part-size',
str(chunkSize)
])
if result is None or result['code'] != 0 or 'uploadId' not in result[
'json']:
logging.error('Unable to initiate upload: %s', repr(result))
return False
uploadId = result['json']['uploadId']
# Start sending the file, one megabyte at a time until we have none left
offset = 0
block = 0
work = uploadCoordinator(config['glacier-threads'])
# Queue up all the work
while remain > 0:
chunk = remain
if chunk > chunkSize:
chunk = chunkSize
job = uploadJob(config, file, name, offset, chunk,
hashes['blocks'][block].hexdigest(), uploadId)
work.add(job)
block += 1
remain -= chunk
offset += chunk
# Wait for it...
work.process()
while not work.isDone():
time.sleep(1)
if sys.stdout.isatty():
# Extra spaces at the end to clear remnants when numbers change
if work.getSent() > 0 and work.getTime() > 0:
timerem = ", " + helper.formatTime(
(float(bytesTotal) - float(bytesDone + work.getSent())) /
(work.getSent() / work.getTime())) + " remaining"
else:
timerem = ""
sys.stdout.write(
'%s%s @ %s, %.2f%% done (%.2f%% total%s) \r' %
(prefix, name,
helper.formatSpeed(work.getSent() / work.getTime()),
float(work.getSent()) / float(size) * 100.0,
float(bytesDone + work.getSent()) / float(bytesTotal) * 100.0,
timerem))
sys.stdout.flush()
if sys.stdout.isatty():
sys.stdout.write('\n')
sys.stdout.flush()
if not work.finish():
logging.error('Failed to upload the file, aborting')
# Note! Should use JSON since plain arguments seems to not work
awsCommand(config, [
'abort-multipart-upload', '--vault-name', config['glacier-vault'],
'--cli-input-json', '{"uploadId": "' + uploadId + '"}'
])
return False
# Time to finalize this deal
result = awsCommand(config, [
'complete-multipart-upload', '--vault-name', config['glacier-vault'],
'--cli-input-json', '{"uploadId": "' + uploadId + '"}', '--checksum',
hashes['final'].hexdigest(), '--archive-size',
str(size)
])
if result is None or result['code'] != 0:
logging.error('Unable to complete upload of %s: %s', file,
repr(result))
return False
return True
def uploadFiles(config, files, bytes):
logging.info("Uploading %d files (%s) to glacier, this may take a while", len(files), helper.formatSize(bytes))
cmd = ["upload", config["glacier-vault"]]
for f in files:
cmd.append(os.path.join(config["prepdir"], f))
upload_start = round(time.time())
result = glacierCommand(config, cmd)
upload_time = max(round(time.time()) - upload_start, 1)
if result is None or "output" not in result or "Uploaded file" not in result["output"]:
logging.error("Failed to upload files: %s", repr(result))
return False
logging.info("Files uploaded @ %s", helper.formatSpeed(bytes / upload_time))
return True