def upload_archive(fname,vault,real_name,chunk=None,description=''):
"""
Upload a file to glacier via the web-server.
"""
if not os.path.isfile(fname):
print("%s is not a valid file! Upload failed!" % fname)
return None
if chunk is None:
chunk=WG.app.config.get("CHUNK_SIZE",1048576)
handler = get_handler()
uploader = ConcurrentUploader(handler,str(vault.name),part_size=chunk)
if WG.app.config.get("VERBOSE",False):
print("Beginning upload of file %s to Glacier. Please by patient, there is no progress bar..." % fname)
file_size = os.path.getsize(fname)
if file_size==0:
if WG.app.config.get("VERBOSE",False):
print("File size is 0. Cannot upload empty file.")
return None
csum = chunkedmd5(fname)
itime=time.time()
file_name = os.path.basename(real_name)
machine_id = str(request.remote_addr)
#Construct a meaningful description object for the file
#The limits are that the description can be no more than 1024 characters in length and must use only ascii characters between 32 and 126 (i.e., 32<=ord(char)<=126)
dscrip = description+'\\n'
dscrip = dscrip + "Uploaded at "+str(itime)+'\\n'+ "Full path "+str(real_name)+'\\n'+ "File size "+str(file_size)+'\\n' + "MD5 "+str(csum)+'\\n' + "Source machine id "+machine_id+'\\n'
archive_id = uploader.upload(fname,description)
if WG.app.config.get("VERBOSE",False):
print("Successfully uploaded %s" % fname)
archive = Archive(archive_id,description,vault,filename=file_name,fullpath=real_name,filesize=file_size,md5sum=csum)
archive.insertion_date = datetime.fromtimestamp(int(itime))
WG.db.session.add(archive)
WG.db.session.commit()
def process_callbacks():
"""
When the client has processed any commands in the queue, it will
post any information needed by the server (such as the new archive id)
back at this url.
Commands are then safely removed from the queue as they have been processed.
"""
name = request.args.get('client_name', '')
qid = str(request.remote_addr) if name == '' else name + ' (' + str(
request.remote_addr) + ')'
dat = deunicode(json.loads(request.get_data(as_text=True)))
if WG.app.config.get("VERBOSE", False):
print "Received return of %s" % str(dat)
for k, val in dat.iteritems():
#k is the hash, dat the data...
#A completed download job
if k[0] == 'd':
if WG.app.config.get("VERBOSE", False):
print "Completed download job. Returned:", val
if qid not in WG.queues or k not in WG.queues[qid]:
print "Download job not found in queue. Strange..."
else:
_ = WG.queues[qid].pop(k)
elif k[0] == 'u':
if WG.app.config.get("VERBOSE", False):
print "Completed upload job. Returned:", val
if 'error' not in val:
#Create a db object for it (provided it doesn't already exist)
vault = Vault.query.filter_by(
name=val['vault_name'], region=val['region_name']).first()
if vault is None:
print "Vault not found..."
abort(401)
archive = Archive.query.filter_by(
archive_id=val['archive_id']).first()
if archive is not None:
print "Archive already added. We shouldn't be seeing this really..."
else:
archive = Archive(val['archive_id'],
val['description'],
vault,
filename=val['file_name'],
fullpath=val['true_path'],
filesize=val['file_size'],
md5sum=val['md5sum'])
archive.insertion_date = datetime.fromtimestamp(
int(val['insert_time']))
WG.db.session.add(archive)
WG.db.session.commit()
if qid not in WG.queues or k not in WG.queues[qid]:
print "Upload job not found in queue. Strange..."
else:
_ = WG.queues[qid].pop(k)
if WG.app.config.get("VERBOSE", False):
print "After processing return, queue is %s" % str(WG.queues)
return 'Processed'
def process_callbacks():
"""
When the client has processed any commands in the queue, it will
post any information needed by the server (such as the new archive id)
back at this url.
Commands are then safely removed from the queue as they have been processed.
"""
name = request.args.get('client_name','')
qid = str(request.remote_addr) if name == '' else name+' ('+str(request.remote_addr) + ')'
dat = deunicode(json.loads(request.get_data(as_text=True)))
if WG.app.config.get("VERBOSE",False):
print "Received return of %s"%str(dat)
for k,val in dat.iteritems():
#k is the hash, dat the data...
#A completed download job
if k[0]=='d':
if WG.app.config.get("VERBOSE",False):
print "Completed download job. Returned:",val
if qid not in WG.queues or k not in WG.queues[qid]:
print "Download job not found in queue. Strange..."
else:
_ = WG.queues[qid].pop(k)
elif k[0]=='u':
if WG.app.config.get("VERBOSE",False):
print "Completed upload job. Returned:",val
if 'error' not in val:
#Create a db object for it (provided it doesn't already exist)
vault = Vault.query.filter_by(name=val['vault_name'],region=val['region_name']).first()
if vault is None:
print "Vault not found..."
abort(401)
archive = Archive.query.filter_by(archive_id=val['archive_id']).first()
if archive is not None:
print "Archive already added. We shouldn't be seeing this really..."
else:
archive = Archive(val['archive_id'],val['description'],vault,filename=val['file_name'],fullpath=val['true_path'],filesize=val['file_size'],md5sum=val['md5sum'])
archive.insertion_date = datetime.fromtimestamp(int(val['insert_time']))
WG.db.session.add(archive)
WG.db.session.commit()
if qid not in WG.queues or k not in WG.queues[qid]:
print "Upload job not found in queue. Strange..."
else:
_ = WG.queues[qid].pop(k)
if WG.app.config.get("VERBOSE",False):
print "After processing return, queue is %s"%str(WG.queues)
return 'Processed'
def process_archive(archive,vault):
"""
Like above
"""
#Check if it exists
tmp = Archive.query.filter_by(archive_id = archive["ArchiveId"]).first()
#If it doesn't, first thing is to add it
if tmp is None:
tmp = Archive(archive["ArchiveId"],archive["ArchiveDescription"],vault)
#Now make everything what it should be...
tmp.description = archive["ArchiveDescription"]
tmp.insertion_date = str_to_dt(archive["CreationDate"])
tmp.SHA256_tree_hash = archive["SHA256TreeHash"]
tmp.filesize = archive["Size"]
#Try and fill in metadata from the description if possible
tmp.populate_from_description()
WG.db.session.add(tmp)
WG.db.session.commit()
return tmp