def test_digest(self):
tree_hash = TreeHash(TEST_CHUNK, algo=hashlib.md5)
tree_hash.update(TEST_DATA, TEST_INDEX)
self.assertEqual(
hashlib.md5(TEST_DATA).digest(),
tree_hash.digest()
)
def test_update(self):
tree_hash = TreeHash(TEST_CHUNK)
tree_hash.update(TEST_DATA, TEST_INDEX)
self.assertEqual(
hashlib.sha256(TEST_DATA).hexdigest(),
tree_hash.hexdigest()
)
def test_tree(self):
hashlib_result = hashlib.sha256(
hashlib.sha256(TEST_DATA).digest() +
hashlib.sha256(TEST_DATA).digest()
).hexdigest()
tree_hash = TreeHash(2, block_size=1)
tree_hash.update(TEST_DATA, 0)
tree_hash.update(TEST_DATA, 1)
self.assertEqual(hashlib_result,
tree_hash.hexdigest())
def run(self):
client = self.get_boto_client()
logging.info("Initiating job to upload")
upload_job = client.initiate_multipart_upload(
vaultName=self.vault,
archiveDescription=self.file_name,
partSize=str(UPLOAD_PART_SIZE))
upload_id = upload_job['uploadId']
treehash = TreeHash(block_size=1024**2)
cur_file = open(self.file_location, 'rb')
i = 0
for i in range(self.numof_parts - 1):
self.cur_part += 1
self.update_task()
data = cur_file.read(UPLOAD_PART_SIZE)
treehash.update(data)
cur_range = 'bytes %d-%d/*' % (i * UPLOAD_PART_SIZE,
(i + 1) * UPLOAD_PART_SIZE - 1)
client.upload_multipart_part(vaultName=self.vault,
uploadId=upload_id,
range=cur_range,
body=data)
self.cur_part += 1
self.update_task()
data = cur_file.read(UPLOAD_PART_SIZE)
treehash.update(data)
cur_range = 'bytes %d-%d/*' % (i * UPLOAD_PART_SIZE,
self.file_size - 1)
client.upload_multipart_part(vaultName=self.vault,
uploadId=upload_id,
range=cur_range,
body=data)
cur_file.close()
hash_res = treehash.hexdigest()
client.complete_multipart_upload(vaultName=self.vault,
uploadId=upload_id,
archiveSize=str(self.file_size),
checksum=hash_res)
def handle(self, *args, **options):
part_size = 8388608
print 'Contacting Amazon AWS...'
glacier = boto3.client('glacier')
multipart_upload = glacier.initiate_multipart_upload(
vaultName=settings.GLACIER_VAULT_NAME, partSize=str(part_size))
print 'Connected to Glacier Vault "' + settings.GLACIER_VAULT_NAME + '"'
upload_id = multipart_upload['uploadId']
treehash_archive = TreeHash()
db = influxdb.InfluxDBClient(settings.INFLUXDB_URI, 8086, 'root',
'root', 'seads')
archive_size = 0
for device in Device.objects.all():
start = datetime.fromtimestamp(0)
end = datetime.now() - timedelta(days=31 *
device.data_retention_policy)
start = (datetime.now() - start).total_seconds()
start = 0
end = int((datetime.now() - end).total_seconds())
end = time.time() - end
print 'Trying ' + str(device) + '...'
print 'Data Retention Policy: ' + str(
device.data_retention_policy) + ' Months'
series = 'device.' + str(device.serial)
try:
query = 'select * from ' + series + ' where time > ' + str(
start) + 's and time < ' + str(end) + 's'
points = db.query(query)
except:
print 'No data found for ' + series + '. Skipping.'
continue
print "Uploading " + series + "..."
print "Querying from " + str(datetime.fromtimestamp(
int(start))) + " to " + str(datetime.fromtimestamp(int(end)))
# store points in temporary file, break into 8MB parts
with open('/tmp/temp_archive', 'wb') as f:
f.write(json.dumps(points))
bytes_read = 0
bytes_sent = 0
with open('/tmp/temp_archive', 'rb') as f:
treehash_part = TreeHash()
part = f.read(part_size)
treehash_part.update(part)
bytes_read += len(part)
while part:
response = glacier.upload_multipart_part(
vaultName=settings.GLACIER_VAULT_NAME,
uploadId=upload_id,
range='bytes ' + str(bytes_sent) + '-' +
str(bytes_read - 1) + '/*',
body=part,
checksum=treehash_part.hexdigest())
bytes_sent += len(part)
part = f.read(part_size)
treehash_part.update(part)
bytes_read += len(part)
archive_size += 1
print "Successfully uploaded " + str(
bytes_sent) + " bytes to Glacier"
print "Deleting points from database..."
# drop from fanout series as well
series = db.query('list series')[0]['points']
rg = re.compile('device.' + str(device.serial))
for s in series:
if rg.search(s[1]):
db.query('delete from ' + s[1] + ' where time > ' +
str(start) + 's and time < ' + str(end) + 's')
print "[DONE]"
try:
with open('/tmp/temp_archive', 'rb') as f:
treehash_archive.update(f.read())
response = glacier.complete_multipart_upload(
vaultName=settings.GLACIER_VAULT_NAME,
uploadId=upload_id,
archiveSize=str(archive_size),
checksum=treehash_archive.hexdigest())
with open(settings.STATIC_PATH + 'archive_ids.log', 'a') as f:
line = {
'archiveId': response['archiveId'],
'timeEnd': str(end)
}
f.write(json.dumps(line))
f.write(';')
os.remove('/tmp/temp_archive')
print "Archival Successful"
except:
print "No data to archive. Exiting."
def test_empty(self):
self.assertEqual(
hashlib.sha256().hexdigest(),
TreeHash(0).hexdigest()
)
def test_constructor(self):
self.assertEqual(
TEST_CHUNK,
len(TreeHash(TEST_CHUNK).hashes)
)
print('file name error. should be 21341234.0123456.meta. no leading \'dot\', \'/ or full path')
exit(1)
elif int(m[2])<2017 or 2020<int(m[2]) or 12 < int(m[3]) or 31 < int(m[4]) or 24 < int(m[5]):
print('file name error. should like time stamp')
exit(1)
(fh,fs)=(open(fn,'rb'),os.path.getsize(fn))
fb=fh.read(csh)
fe=DecMethod(fn).decrypt(fb).decode('utf-8')
# fe=DecMethod(fn).decrypt(fb)
# print(int(fe[0:10]))
js=json.loads((fe[10:10+int(fe[0:10])]))
if opt['Verbose']: print(js)
(mt,dec)=(TreeHash(),DecMethod(fn))
if opt['Verbose']:
print('## because verbose option is specified, decoding results is output to %sa' % fn)
fw=open(fn+'a','wb')
for x in range(js['Size'],0,-cs):
fb=dec.decrypt(fh.read(cs))
if( cs <= x ): mt.update(fb)
else: mt.update(fb[0:x])
if not opt['Verbose']: continue
if( cs <= x ): fw.write(fb)
else: fw.write(fb[0:x])
if opt['Verbose']: fw.close()
if mt.hexdigest() == js['LongCRC']: continue
'Verbose': '--verbose' in sys.argv
}
for fn in sys.argv[1:]:
if fn == '--verbose': continue
(fs, fm) = (os.path.getsize(fn),
datetime.now().strftime('%Y%m%d.%H%M%S.meta'))
ft = ('' if 1024 * 1024 * 1024 * 32 < fs or opt['NoTmp'] else
os.environ.get('TMP') + '/') + fm + '.tmp'
# print('### tmp=',ft)
# exit(1)
print('# processing.. ', fn)
if opt['Verbose']: print('## formatting to.. ', fm)
(mt, enc) = (TreeHash(), EncMethod(fm))
(fh, fw) = (open(fn, 'rb'), open(ft, 'wb'))
fb = fh.read(css)
mt.update(fb)
if fs <= css:
fb = (fb + b'@' * 16)[0:int((len(fb) + 15) / 16) * 16]
fw.write(enc.encrypt(fb))
meta = {
'FileName': fn,
'Size': fs,
'CTime': os.path.getctime(fn),
'DTime': fm,
'ShortCRC': mt.hexdigest()
}
(fcnt, gbyte, awstmp) = (0, 1024 * 1024 * 1024,
'aws glacier {} --account-id ' + opt['Account'])
awstmp = awstmp + ' --vault-name {}'.format(opt['VaultName'])
print('## chunk size : {:6.2f} GB'.format(csize / gbyte))
for fn in sys.argv[1:]:
if re.match('^--', fn): continue
elif not os.path.exists(fn):
if opt['Verbose']: print('## file {} cannot be accessed'.format(fn))
continue
fs = os.path.getsize(fn)
if fs < opt['MinSize']:
if opt['Verbose']:
print('## file {} is shoter than lower limit {}.'.format(
fn, opt['MinSize']))
continue
(ft, mt) = (open(fn, 'rb'), TreeHash())
logevent('#{:03} target file: {}'.format(fcnt, fn))
for x in range(0, fs, csize):
mt.update(ft.read(csize))
ft.close()
arcdsc=datetime.now().strftime('v3/%Y-%m-%d@2%H@1%M@1%S/')+\
fn.replace('@','@0').replace(':','@1').replace(' ','@2').replace('/','@3')+\
'/'+mt.hexdigest()+'/0'
while (1):
(s, r) = cmd(
awstmp.format('initiate-multipart-upload') +
' --archive-description \"{}\" --part-size {}'.format(
arcdsc, csize))
if s != 0: errorexit(r)
upid = json.loads(r)['uploadId']
def test_constructor(self):
self.assertEqual(
hashlib.sha256(TEST_DATA).hexdigest(),
TreeHash(TEST_DATA).hexdigest())
def test_tree(self):
hashlib_result = hashlib.sha256(
hashlib.sha256(TEST_DATA).digest() +
hashlib.sha256(TEST_DATA).digest()).hexdigest()
self.assertEqual(hashlib_result,
TreeHash(2 * TEST_DATA, block_size=1).hexdigest())
def test_digest(self):
self.assertEqual(
hashlib.md5(TEST_DATA).digest(),
TreeHash(TEST_DATA, algo=hashlib.md5).digest())
def test_update(self):
treehash = TreeHash()
treehash.update(TEST_DATA)
self.assertEqual(
hashlib.sha256(TEST_DATA).hexdigest(), treehash.hexdigest())
