def upload_file(vault_name):
"""
Handle file upload
"""
handler = get_handler()
region = handler.region.name
vault = Vault.query.filter_by(name=vault_name,region=region).first()
if vault is None:
abort(401)
if vault.lock:
abort(401)
file = request.files['file']
if file:
if WG.app.config.get("VERBOSE",False):
print "starting to upload file to web-server"
#Save to a temporary file on the server... Needs to be done for calculating hashes and the like.
tmp=tempfile.NamedTemporaryFile(dir=WG.app.config["TEMP_FOLDER"],delete=True)
file.save(tmp)
#Ensure that things are really really written to disc.
tmp.file.flush()
os.fsync(tmp.file.fileno())
if WG.app.config.get("VERBOSE",False):
print "Server has accepted payload"
description=request.form.get('upload_description','')
if description=="Description of file.":
description=''
upload_archive(tmp.name,vault,file.filename,description=description)
tmp.close()
return redirect(request.referrer)
def stream_output(self,chunk_size=None,file_handler=None):
"""
A generator that can be used to stream a download job from Amazon's
servers without saving it locally. Obviously the job must be live,
complete and successful. The database values are taken to be true
for each of these, so it makes sense to run this straight after
checking the status of jobs.
chunk_size has its usual meaning and will default to the config value
if not given
If file_handler is a file object, which should be open for writing,
then in addition to streaming the response, the object will be written
to file. The object will be closed after the last chunk has been processed.
"""
handler = get_handler(self.vault.region)
if self.action!='download':
raise TypeError("Can only stream download jobs")
if not self.live or not self.completed:
raise AttributeError("Job is not live and complete.")
if chunk_size is None:
chunk_size=int(WG.app.config.get('CHUNK_SIZE',1048576))
file_size = self.archive.filesize
vault_name = self.vault.name
job_id = self.job_id
num_chunks = int(math.ceil(file_size / float(chunk_size)))
for i in xrange(num_chunks):
byte_range = ((i * chunk_size), ((i + 1) * chunk_size) - 1)
response = handler.get_job_output(vault_name,job_id,byte_range)
if file_handler:
file_handler.write(response.read())
#Close after last chunk
if i==num_chunks-1:
file_handler.close()
yield response.read()
def run_jobs(vault_name):
"""
Execute a completed job. If not completed, updates its status.
"""
handler = get_handler()
#Need to get the vault as always...
region = handler.region.name
vault = Vault.query.filter_by(name=vault_name,region=region).first()
if vault is None:
abort(401)
#Get the job from our local db
job=Job.query.filter_by(job_id=(request.args['job_id'])).first()
#If we don't have the job, or our records show it's incomplete, check with amazon
if job is None or not job.completed:
if vault.lock:
abort(401)
job=process_job(handler.describe_job(vault.name,request.args['job_id']),vault)
#If it's still none, something went wrong...
if job is None or not job.completed or not job.live or not job.status_code=="Succeeded":
abort(401)
#Now we have the job, get its output
if job.action=='list':
dat=handler.get_job_output(vault.name,job.job_id)
for archive in dat["ArchiveList"]:
process_archive(archive,vault)
vault.lock=False
WG.db.session.add(vault)
WG.db.session.commit()
elif job.action=='download':
pass
#return redirect(request.referrer)
return redirect(url_for("vault_view",vault_name=vault.name))
def delete_vault():
if 'name' not in request.args:
abort(401)
handler = get_handler()
region=handler.region.name
vault = Vault.query.filter_by(name=request.args['name'],region=region).first()
if vault is None:
#Trying to delete a non-existent vault...
abort(401)
if vault.lock:
return "Vault is locked! No delete for you."
if vault.archives.count()!=0:
return "Cannot delete non-empty vault!"
try:
#First try deleting it from the amazon
handler.delete_vault(vault.name)
except UnexpectedHTTPResponseError as e:
print "Failed to delete vault. Error message was %s"%e.message
abort(401)
#If it was deleted, all jobs/archives must be ghosts...
for job in vault.jobs:
WG.db.session.delete(job)
for archive in vault.archives:
WG.db.session.delete(archive)
#Delete the vault
WG.db.session.delete(vault)
WG.db.session.commit()
return redirect(url_for('main'))
def stream_output(self, chunk_size=None, file_handler=None):
"""
A generator that can be used to stream a download job from Amazon's
servers without saving it locally. Obviously the job must be live,
complete and successful. The database values are taken to be true
for each of these, so it makes sense to run this straight after
checking the status of jobs.
chunk_size has its usual meaning and will default to the config value
if not given
If file_handler is a file object, which should be open for writing,
then in addition to streaming the response, the object will be written
to file. The object will be closed after the last chunk has been processed.
"""
handler = get_handler(self.vault.region)
if self.action != 'download':
raise TypeError("Can only stream download jobs")
if not self.live or not self.completed:
raise AttributeError("Job is not live and complete.")
if chunk_size is None:
chunk_size = int(WG.app.config.get('CHUNK_SIZE', 1048576))
file_size = self.archive.filesize
vault_name = self.vault.name
job_id = self.job_id
num_chunks = int(math.ceil(file_size / float(chunk_size)))
for i in xrange(num_chunks):
byte_range = ((i * chunk_size), ((i + 1) * chunk_size) - 1)
response = handler.get_job_output(vault_name, job_id, byte_range)
if file_handler:
file_handler.write(response.read())
#Close after last chunk
if i == num_chunks - 1:
file_handler.close()
yield response.read()
def upload_archive(fname,vault,real_name,chunk=None,description=''):
"""
Upload a file to glacier via the web-server.
"""
if not os.path.isfile(fname):
print("%s is not a valid file! Upload failed!" % fname)
return None
if chunk is None:
chunk=WG.app.config.get("CHUNK_SIZE",1048576)
handler = get_handler()
uploader = ConcurrentUploader(handler,str(vault.name),part_size=chunk)
if WG.app.config.get("VERBOSE",False):
print("Beginning upload of file %s to Glacier. Please by patient, there is no progress bar..." % fname)
file_size = os.path.getsize(fname)
if file_size==0:
if WG.app.config.get("VERBOSE",False):
print("File size is 0. Cannot upload empty file.")
return None
csum = chunkedmd5(fname)
itime=time.time()
file_name = os.path.basename(real_name)
machine_id = str(request.remote_addr)
#Construct a meaningful description object for the file
#The limits are that the description can be no more than 1024 characters in length and must use only ascii characters between 32 and 126 (i.e., 32<=ord(char)<=126)
dscrip = description+'\\n'
dscrip = dscrip + "Uploaded at "+str(itime)+'\\n'+ "Full path "+str(real_name)+'\\n'+ "File size "+str(file_size)+'\\n' + "MD5 "+str(csum)+'\\n' + "Source machine id "+machine_id+'\\n'
archive_id = uploader.upload(fname,description)
if WG.app.config.get("VERBOSE",False):
print("Successfully uploaded %s" % fname)
archive = Archive(archive_id,description,vault,filename=file_name,fullpath=real_name,filesize=file_size,md5sum=csum)
archive.insertion_date = datetime.fromtimestamp(int(itime))
WG.db.session.add(archive)
WG.db.session.commit()
def create_vault():
if 'name' not in request.args:
abort(401)
hand = get_handler()
try:
hand.create_vault(request.args['name'])
vault = process_vault(hand.describe_vault(request.args['name']))
except UnexpectedHTTPResponseError as e:
print "Failed to create vault. Error message was %s"%e.message
return redirect(url_for('main'))
def get_vaults():
"""
Queries the Amazon API and gets a list of all the jobs, which is used to
populate and/or update the local database.
"""
handler = get_handler()
try:
vaults=handler.list_vaults()
for vault in vaults["VaultList"]:
tmp=process_vault(vault)
except UnexpectedHTTPResponseError as e:
print "Failed processing/loading vaults. Error was %s"%e.message
return redirect(url_for("main"))
def download_file(vault_name):
"""
Download a file if the link is available...
"""
handler = get_handler()
region = handler.region.name
vault = Vault.query.filter_by(name=vault_name,region=region).first()
if vault is None:
abort(401)
#Need to get the archive too...
if 'archive_id' not in request.args:
abort(401)
archive = Archive.query.filter_by(archive_id=request.args['archive_id']).first()
if archive is None:
abort(401)
if archive.filename!="NOT_GIVEN":
fname=archive.filename
else:
fname=app.config["UNKNOWN_FILENAME"]
#Are we serving from cache?
#cache = archive.cached()
#if cache==1:
# print "Serving from cache."
# return send_from_directory(os.path.join(app.config["LOCAL_CACHE"],region,vault.name),archive.archive_id,attachment_filename=fname,as_attachment=True)
#Is there a finished job knocking about?
job=archive.jobs.filter_by(action='download',completed=True,live=True,status_message="Succeeded").first()
if job is None:
abort(401)
#OK, everything exists, go ahead...
if False and cache==2:
#Save to cache whilst serving
f = open(os.path.join(app.config["LOCAL_CACHE"],region,vault.name,archive.archive_id),'wb')
else:
#Don't add to cache, just serve
f = None
h=Headers()
h.add("Content-Disposition",'attachment;filename="'+fname+'"')
return Response(stream_with_context(job.stream_output(file_handler=f)),headers=h)
def request_archive(vault_name):
"""
Asks glacier to get your data. You'll have to wait for it to get back first...
"""
handler = get_handler()
#Need to get the vault as always...
region = handler.region.name
vault = Vault.query.filter_by(name=vault_name,region=region).first()
if vault is None:
abort(401)
#Need to get the archive too...
if 'archive_id' not in request.args:
abort(401)
archive = Archive.query.filter_by(archive_id=request.args['archive_id']).first()
if archive is None:
abort(401)
#OK, everything exists, go ahead...
def_opts={"Description":"Fetch archive.",
"Type":"archive-retrieval",
"ArchiveId":archive.archive_id}
job_id = handler.initiate_job(vault.name, def_opts)
job=process_job(handler.describe_job(vault.name,job_id["JobId"]),vault)
return redirect(request.referrer)
def check_job_status(vault_name):
"""
Pretty self explanatory isn't it?
"""
handler = get_handler()
region = handler.region.name
vault = Vault.query.filter_by(name=vault_name,region=region).first()
if vault is None:
abort(401)
#Get the live jobs from Amazon
live_jobs = handler.list_jobs(vault.name)
#First update/add all of them to db
for job in live_jobs['JobList']:
process_job(job,vault)
#Then kill any ones that are in our db and should be dead
jobs = vault.jobs.filter_by(live=True).all()
live_ids = [x["JobId"] for x in live_jobs['JobList']]
for job in jobs:
if job.job_id not in live_ids:
job.live=False
WG.db.session.add(job)
WG.db.session.commit()
return redirect(request.referrer)
def delete_archive(vault_name):
"""
Delete archive from glacier's servers
"""
handler = get_handler()
#Need to get the vault as always...
region = handler.region.name
vault = Vault.query.filter_by(name=vault_name,region=region).first()
if vault is None or vault.lock:
abort(401)
#Need to get the archive too...
if 'archive_id' not in request.args:
abort(401)
archive = Archive.query.filter_by(archive_id=request.args['archive_id']).first()
if archive is None:
abort(401)
#OK, everything exists, go ahead...
handler.delete_archive(vault.name,archive.archive_id)
#Delete the archive and any jobs associated with it...
for job in archive.jobs:
WG.db.session.delete(job)
WG.db.session.delete(archive)
WG.db.session.commit()
return redirect(request.referrer)
def get_inventory(vault_name):
"""
Initiates an inventory job for the specified vault.
Currently lacks any checks on if it's a good idea to submit another of these jobs
"""
handler = get_handler()
#Need to get the vault...
region = handler.region.name
vault = Vault.query.filter_by(name=vault_name,region=region).first()
if vault is None:
abort(401)
#Already asked for one, don't need another...
if vault.lock:
abort(401)
def_opts={"Description":"Auto-made inventory job.",
"Type":"inventory-retrieval","Format":"JSON"}
job_id = handler.initiate_job(vault.name, def_opts)
#Lock the vault...
vault.lock=True
WG.db.session.add(vault)
WG.db.session.commit()
#Add the job to the database
job=process_job(handler.describe_job(vault.name,job_id["JobId"]),vault)
return redirect(request.referrer)
