def get_hash(): SQL = """SELECT volumename,comment FROM media WHERE comment->>'state'='treehash' AND comment->>'status'='in-progress'""" volumes = glacier_chris.update_db(SQL, 'select') for vol in volumes: r = AsyncResult(vol[1]['celery_id']) if r.ready() == True: vol[1]['hash'] = r.result vol[1]['status'] = "complete" d = json.dumps(vol[1]) SQL = """UPDATE media SET comment='%(data)s' WHERE volumename='%(vol)s'""" % {'data': d, 'vol': vol[0]} glacier_chris.update_db(SQL,'update')
def submit_hash(): try: v = glacier_chris.get_vols(jobid) except: print "Failed to get volumes" raise for vol in v: fname = storage+'/'+vol[0] thash = glacier_chris.hash_file.delay(fname) data = {'state': 'treehash', 'status': 'in-progress', 'celery_id': thash.task_id, 'path': fname, 'hash': '', 'failed_parts': '', 'upoad_id': '', 'job_id': '', 'error_id': '', 'date': time} d = json.dumps(data) SQL = """UPDATE media SET comment='%(data)s' where volumename='%(vol)s'""" % {'data': d, 'vol': vol[0]} glacier_chris.update_db(SQL,'update')
def upload_volume(): SQL = """SELECT volumename,comment FROM media WHERE comment->>'state'='treehash' AND comment->>'status'='complete'""" volumes = glacier_chris.update_db(SQL, 'select') for vol in volumes: fname = vol[1]['path'] fsize = os.stat(fname).st_size if fsize > chunksize: r = 0 try: init_multi = glacier_chris.upload_multi_init(fname,vault,str(chunksize)) except: raise p = 1 with open (fname, 'rb') as f: status = {} status['file_loc'] = fname for chunk in iter(lambda: f.read(chunksize), b''): part = "part_"+str(p) if r + chunksize > fsize: incr = fsize -r else: incr = chunksize try: up = glacier_chris.upload_part.delay(vault,init_multi,"bytes " + str(r) + "-" + str(str(r + int(incr) -1)) + "/*",body=chunk) except: raise r = r + chunksize p = p + 1 if up['ResponseMetadata']['HTTPStatusCode'] != 204: status['failures'][part] = '%(start)s-%(end)s' % {'start': r, 'end': r+incr} try: fhash = vol[1]['hash'] com = glacier_chris.complete_multi(vault,init_multi,str(fsize),fhash)# complete multipart upload if com['ResponseMetadata']['HTTPStatusCode'] == 201: status['glacier_data'] = {'archiveId': com['archiveId'], 'checksum': com['checksum']} except: raise s = json.dumps(status) SQL = """UPDATE media SET comment='%(status)s' WHERE volumename='%(vol)s'""" % {'status': s, 'vol': vol[0]} glacier_chris.update_db(SQL, 'update') else: try: u = glacier_chris.upload_glacier.delay(fname,vault,fname) SQL="""UPDATE media SET comment='{"celery_id": "%(cid)s"}' WHERE volumename='%(vol)s'""" % {'cid': u.id, 'vol': vol[0]} glacier_chris.update_db(SQL, 'update') except: raise