def _connect_glacier(self): ''' Returns Boto.Glacier.Layer1 object ''' #delete this two lines after test from boto.glacier.layer1 import Layer1 return Layer1()
def test_describe_job(): conn = Layer1(region_name="us-west-2") vault_name = "my_vault" conn.create_vault(vault_name) archive_id = conn.upload_archive( vault_name, "some stuff", "", "", "some description") job_response = conn.initiate_job(vault_name, { "ArchiveId": archive_id, "Type": "archive-retrieval", }) job_id = job_response['JobId'] job = conn.describe_job(vault_name, job_id) json.loads(job.read().decode("utf-8")).should.equal({ 'CompletionDate': '2013-03-20T17:03:43.221Z', 'VaultARN': None, 'RetrievalByteRange': None, 'SHA256TreeHash': None, 'Completed': True, 'InventorySizeInBytes': '0', 'JobId': job_id, 'Action': 'InventoryRetrieval', 'JobDescription': None, 'SNSTopic': None, 'ArchiveSizeInBytes': 0, 'ArchiveId': archive_id, 'ArchiveSHA256TreeHash': None, 'CreationDate': '2013-03-20T17:03:43.221Z', 'StatusMessage': None, 'StatusCode': 'Succeeded', })
def glacier_vault_inv(): # Submit a vault inventory job init_database() init_glconfig() timestamp = strftime("%Y-%m-%d %H:%M:%S", gmtime()) user = getpass.getuser() try: c.execute('SELECT * FROM jobs') except sqlite3.OperationalError: c.execute( 'CREATE TABLE jobs (ID integer primary key, TIMESTAMP text, USER text, VAULT text, JOBID text)' ) conn.commit() try: glacier_connect = Layer1(aws_access_key_id=key, aws_secret_access_key=secret, region_name=region) job = glacier_connect.initiate_job( vault, { 'Description': 'inventory-job', 'Type': 'inventory-retrieval', 'Format': 'JSON' }) print 'Inventory Job ID: ' + str(job['JobId']) c.execute( 'INSERT INTO jobs (TIMESTAMP,USER,VAULT,JOBID) VALUES (?,?,?,?)', (timestamp, user, vault, job['JobId'])) conn.commit() except boto.glacier.exceptions.UnexpectedHTTPResponseError as e: error_stamp('upper') print e error_stamp('lower')
def uploadToGlacier(tempTarFile=None, DEBUG_MODE=False, GLACIER_VAULT=None, SECRET_ACCESS_KEY=None, ACCESS_KEY=None, GLACIER_REALM=None): global logger if not tempTarFile: return 0 # Establish a connection to the Glacier glacier_vault_in = None my_archive = None archive_id = None try: #my_glacier = GlacierConnection(ACCESS_KEY,SECRET_ACCESS_KEY,region=GLACIER_REALM) my_glacier = Layer1(aws_access_key_id=ACCESS_KEY, aws_secret_access_key=SECRET_ACCESS_KEY, region_name=GLACIER_REALM) if DEBUG_MODE: logger.debug("Glacier Connection: %s" % my_glacier) # Create a new vault (not neccessary if you already have one!) if GLACIER_VAULT: #glacier_vault_in = my_glacier.get_vault(GLACIER_VAULT) #vaults = my_glacier.get_all_vaults() vaults = my_glacier.list_vaults() glacier_vault_in = None if GLACIER_VAULT not in vaults: glacier_vault_in = my_glacier.create_vault(GLACIER_VAULT) else: GLACIER_VAULT = id_generator(size=16) glacier_vault_in = my_glacier.create_vault(GLACIER_VAULT) if DEBUG_MODE: logger.debug("Glacier Vault: %s" % glacier_vault_in) #my_archive = GlacierArchive(tempTarFile) uploader = ConcurrentUploader(my_glacier, GLACIER_VAULT, 64 * 1024 * 1024) if DEBUG_MODE: #logger.debug("Archive created in mem: %s " % my_archive) logger.debug("Archive created in mem:%s" % uploader) #glacier_vault_in.upload(my_archive) archive_id = uploader.upload(tempTarFile, tempTarFile) if DEBUG_MODE: logger.info("upload created: %s" % glacier_vault_in) except Exception, exc: if exc.args > 0: x, y = exc.args errstr = None try: errstr = json.loads(y.read()) except: errstr = y logger.error("Error in glacier upload %s" % errstr) else: logger.error("Error in glacier upload %s" % (exc))
def get_vault_metadata(self): glacier_layer1 = Layer1(region_name=self.region_name) print("operation starting...") vault_metadata = glacier_layer1.describe_vault(self.target_vault_name) print("Success! vault metadata: %s" % vault_metadata)
def init_handlers_from_config(): """ Use the config to create handlers for Amazon Glacier """ for region in glacier.regions(): WG.handlers[region.name] = Layer1( aws_access_key_id=WG.app.config["AWS_ACCESS_KEY"], aws_secret_access_key=WG.app.config["AWS_SECRET_ACCESS_KEY"], region_name=region.name)
def get_job_output(self, job_id=None): glacier_layer1 = Layer1(region_name=self.region_name) print("operation starting...") output = glacier_layer1.get_job_output(self.target_vault_name, job_id) print output return output
def archive_delete(): init_database() init_glconfig() archive_id = raw_input('Enter archive to be deleted: ') glacier_connect = Layer1(aws_access_key_id=key, aws_secret_access_key=secret, region_name=region) glacier_connect.delete_archive(vault, archive_id)
def _connect_glacier(self): ''' Returns Boto.Glacier.Layer1 object ''' #comment for test #return __node__['ec2']['connect_glacier']() #delete this two lines after test from boto.glacier.layer1 import Layer1 return Layer1()
def glacier_vault_create(): # Create Glacier Vaults init_database() init_glconfig() print 'Creating vault: ' + vault glacier_connect = Layer1(aws_access_key_id=key, aws_secret_access_key=secret, region_name=region) glacier_connect.create_vault(vault)
def validate_glacier(access_key, secret_access_key, region): """ Validate connection to Amazon Glacier If the function executes without errors, all is well. """ tst = Layer1(aws_access_key_id=access_key, aws_secret_access_key=secret_access_key, region_name=region) a = tst.list_vaults() tst.close()
def list_jobs(self, job_id=None): glacier_layer1 = Layer1(region_name=self.region_name) print("operation starting...") if (job_id != None): print glacier_layer1.describe_job(self.target_vault_name, job_id) else: print glacier_layer1.list_jobs(self.target_vault_name, completed=False)
def upload_archive(self, file_name): glacier_layer1 = Layer1(region_name=self.region_name) uploader = ConcurrentUploader(glacier_layer1, self.target_vault_name, 32 * 1024 * 1024) print("operation starting...") archive_id = uploader.upload(file_name, file_name) print("Success! archive id: '%s'" % archive_id)
def test_init_glacier_job(): conn = Layer1(region_name="us-west-2") vault_name = "my_vault" conn.create_vault(vault_name) archive_id = conn.upload_archive(vault_name, "some stuff", "", "", "some description") job_response = conn.initiate_job(vault_name, { "ArchiveId": archive_id, "Type": "archive-retrieval", }) job_id = job_response['JobId'] job_response['Location'].should.equal("//vaults/my_vault/jobs/{0}".format(job_id))
def test_initialiate_multipart_upload(self): # Create a vault, initiate a multipart upload, # then cancel it. glacier = Layer1() glacier.create_vault('l1testvault') self.addCleanup(glacier.delete_vault, 'l1testvault') upload_id = glacier.initiate_multipart_upload( 'l1testvault', 4 * 1024 * 1024, 'double spaces here')['UploadId'] self.addCleanup(glacier.abort_multipart_upload, 'l1testvault', upload_id) response = glacier.list_multipart_uploads('l1testvault')['UploadsList'] self.assertEqual(len(response), 1) self.assertEqual(response[0]['MultipartUploadId'], upload_id)
def test_get_job_output(): conn = Layer1(region_name="us-west-2") vault_name = "my_vault" conn.create_vault(vault_name) archive_response = conn.upload_archive(vault_name, "some stuff", "", "", "some description") archive_id = archive_response['ArchiveId'] job_response = conn.initiate_job(vault_name, { "ArchiveId": archive_id, "Type": "archive-retrieval", }) job_id = job_response['JobId'] output = conn.get_job_output(vault_name, job_id) output.read().decode("utf-8").should.equal("some stuff")
def initiate_archive_retrieval_job(self, archive_id): glacier_layer1 = Layer1(region_name=self.region_name) print("operation starting...") job_id = glacier_layer1.initiate_job( self.target_vault_name, { "Description": "download-archive-job", "Type": "archive-retrieval", "ArchiveId": archive_id }) print("Success! job id: %s" % (job_id, )) return job_id
def initiate_vault_inventory_job(self): glacier_layer1 = Layer1(region_name=self.region_name) print("operation starting...") job_id = glacier_layer1.initiate_job( self.target_vault_name, { "Description": "inventory-job", "Type": "inventory-retrieval", "Format": "JSON" }) print("Success! inventory job id: %s" % (job_id, )) return job_id
def test_list_glacier_jobs(): conn = Layer1(region_name="us-west-2") vault_name = "my_vault" conn.create_vault(vault_name) archive_id1 = conn.upload_archive(vault_name, "some stuff", "", "", "some description")['ArchiveId'] archive_id2 = conn.upload_archive(vault_name, "some other stuff", "", "", "some description")['ArchiveId'] conn.initiate_job(vault_name, { "ArchiveId": archive_id1, "Type": "archive-retrieval", }) conn.initiate_job(vault_name, { "ArchiveId": archive_id2, "Type": "archive-retrieval", }) jobs = conn.list_jobs(vault_name) len(jobs['JobList']).should.equal(2)
def glacier_vault_delete(): # Delete Glacier Vaults init_database() init_glconfig() print 'Deleting vault: ' + vault glacier_connect = Layer1(aws_access_key_id=key, aws_secret_access_key=secret, region_name=region) try: glacier_connect.delete_vault(vault) c.execute('DELETE FROM config WHERE vault=(?)', (vault, )) conn.commit() except: error_stamp('upper') print 'Cannot delete vault ' + vault print 'It may contain archives that need to be deleted first' error_stamp('lower')
def test_describe_job(): conn = Layer1(region_name="us-west-2") vault_name = "my_vault" conn.create_vault(vault_name) archive_id = conn.upload_archive( vault_name, "some stuff", "", "", "some description") job_response = conn.initiate_job(vault_name, { "ArchiveId": archive_id, "Type": "archive-retrieval", }) job_id = job_response['JobId'] job = conn.describe_job(vault_name, job_id) joboutput = json.loads(job.read().decode("utf-8")) joboutput.should.have.key('Tier').which.should.equal('Standard') joboutput.should.have.key('StatusCode').which.should.equal('InProgress') joboutput.should.have.key('VaultARN').which.should.equal('arn:aws:glacier:RegionInfo:us-west-2:012345678901:vaults/my_vault')
def test_describe_job(): conn = Layer1(region_name="us-west-2") vault_name = "my_vault" conn.create_vault(vault_name) archive_id = conn.upload_archive(vault_name, "some stuff", "", "", "some description") job_response = conn.initiate_job(vault_name, { "ArchiveId": archive_id, "Type": "archive-retrieval" }) job_id = job_response["JobId"] job = conn.describe_job(vault_name, job_id) joboutput = json.loads(job.read().decode("utf-8")) joboutput.should.have.key("Tier").which.should.equal("Standard") joboutput.should.have.key("StatusCode").which.should.equal("InProgress") joboutput.should.have.key("VaultARN").which.should.equal( f"arn:aws:glacier:us-west-2:{ACCOUNT_ID}:vaults/my_vault")
def download_file(command): """ Takes a command and uses it to download the file. """ if 'action' not in command or command['action']!='DOWNLOAD': raise ValueError("Command not of type DOWNLOAD") ret={} handler = Layer1(aws_access_key_id = command['access_key'],aws_secret_access_key = command['secret_access_key'],region_name=command['region_name']) f=open(os.path.join(ddir,command['file_name']),'wb') num_chunks = int(math.ceil(command['file_size'] / float(dchunk))) print "Downloading file %s"%command['file_name'] for i in xrange(num_chunks): byte_range = ((i * dchunk), ((i + 1) * dchunk) - 1) dload = handler.get_job_output(command['vault_name'],command['job_id'],byte_range) f.write(dload.read()) print "%g %%"%(100*float(i)/float(num_chunks)) f.close() print "100 %" print "Completed." return {}
def upload_file(command): """ Uploads a file from the local machine that is specified in the given command. """ if 'action' not in command or command['action']!="UPLOAD": raise ValueError("Command not of type UPLOAD") if 'file_pattern' not in command: raise ValueError("Missing file pattern") path = command['file_pattern'] if not os.path.exists(path): raise ValueError("No valid file for upload found") returner={} handler = Layer1(aws_access_key_id = command['access_key'],aws_secret_access_key = command['secret_access_key'],region_name=command['region_name']) uploader = ConcurrentUploader(handler,command['vault_name'],part_size=uchunk) file_size = os.path.getsize(path) if file_size==0: raise ValueError("File is empty. Nothing to upload.") csum = chunkedmd5(path) itime=time.time() file_name = os.path.basename(path) machine_id = str(command['target']) if client_name == '' else client_name+' ('+str(command['target']) + ')' #Construct a meaningful description object for the file #The limits are that the description can be no more than 1024 characters in length and must use only ascii characters between 32 and 126 (i.e., 32<=ord(char)<=126) dscrip = command['description']+'\\n' dscrip = dscrip + "Uploaded at "+str(itime)+'\\n'+ "Full path "+str(path)+'\\n'+ "File size "+str(file_size)+'\\n' + "MD5 "+str(csum)+'\\n' + "Source machine id "+machine_id+'\\n' print "Uploading file %s"%file_name #Put some validation stuff here... #Do the upload archive_id = uploader.upload(path,dscrip) print "Completed successfully. Archive ID: %s"%archive_id #Done the upload, send the bastard back returner['archive_id'] = archive_id returner['description'] = dscrip returner['file_name'] = file_name returner['true_path'] = path returner['file_size'] = file_size returner['md5sum'] = csum returner['insert_time']=int(itime) returner['region_name']=command['region_name'] returner['vault_name'] = command['vault_name'] return returner
".tar.gz ~/vol/* &> /dev/null", shell=True) print "Archive done for " + volid subprocess.call("sudo umount ~/vol/", shell=True) except Exception as e: logger.error("Error in archive: " + str(e)) # Upload archive file to AWS Glacier try: vault_name = str(data['vault_name']) part_size = int(data['part_size']) num_threads = int(data['num_threads']) print "Backing archive of " + volid + " to Glacier" glacierconn = Layer1( aws_access_key_id=data['aws_access_key_id'], aws_secret_access_key=data[ 'aws_secret_access_key'], region_name=data['region']) vault = glacierconn.describe_vault(vault_name) uploader = ConcurrentUploader( glacierconn, vault_name, part_size=part_size, num_threads=num_threads) archive_id = uploader.upload( (archive_file + ".tar.gz"), "first upload") print "Download the archive using: " + archive_id except Exception as e: logger.error("Glacier error: " + str(e)) # Detach the volume from temporary instance and associate it back to
def glacier_vault_inv_out(): # Read the inventory of the vault (The job requesting the inventory must be submitted first) init_database() try: c.execute('SELECT * FROM jobs') rows = c.fetchall() if len(rows) > 0: print('\r') print 'Existing Jobs:' print '{:<22} {:<15} {:<50} {:<100}'.format( 'Time Submitted', 'User', 'Vault', 'Amazon Job ID') for row in rows: print '{:<22} {:<15} {:<50} {:<100}'.format( row[0], row[1], row[2], row[3]) print('\r') gl_id = 0 gl_id = raw_input( 'Choose a Job to get inventory? (default: %s): ' % gl_id) or gl_id gl_id = int(gl_id) if gl_id != 0: c.execute( 'SELECT jobs.jobid,jobs.vault,config.keyid,config.seckey,config.region FROM jobs INNER JOIN config ON jobs.vault = config.vault WHERE jobs.id =(?)', (gl_id, )) x = c.fetchall() job = x[0][0] vault = x[0][1] key = x[0][2] secret = x[0][3] region = x[0][4] try: glacier_connect = Layer1(aws_access_key_id=key, aws_secret_access_key=secret, region_name=region) get_job = glacier_connect.get_job_output(vault, job) list = get_job['ArchiveList'] for i in list: print 'Archive ID: ' + str(i['ArchiveId']) print '{:<25} {:<20} {:<100}'.format( 'Creation Date', 'Size', 'Archive') print '{:<25} {:<20} {:<100}'.format( i['CreationDate'], i['Size'], i['ArchiveDescription']) + '\n' except boto.glacier.exceptions.UnexpectedHTTPResponseError as e: error_stamp('upper') print e error_stamp('lower') get_job = glacier_connect.list_jobs(vault, completed=False) list = get_job['JobList'] print 'Listing currently active jobs:' print 'Request ID: ' + str(get_job['RequestId']) + '\n' for i in list: print 'Job ID: ' + str(i['JobId']) print 'Status: ' + str(i['CreationDate']) print 'Status: ' + str(i['StatusCode']) print 'Vault ARN: ' + str(i['VaultARN']) + '\n' error_stamp('lower') gl_id = 1 else: print str('-') * 121 print 'ERROR: That is not a valid choice.' print str('-') * 121 except: print 'An error has occured' pass
def __init__(self, *args, **kwargs): # Accept a passed in layer1, mainly to allow easier testing if "layer1" in kwargs: self.layer1 = kwargs["layer1"] else: self.layer1 = Layer1(*args, **kwargs)
from boto.glacier.layer1 import Layer1 from boto.glacier.concurrent import ConcurrentUploader import sys import os.path # XXX: replace these with your credentials ACCESS_KEY = "AWS_ACCESS_KEY" SECRET_KEY = "AWS_SECRET_KEY" VAULT_NAME = "VAULT_NAME" REGION_NAME = 'us-west-2' try: backup_file = sys.argv[1] except IndexError: sys.stderr.write("Usage: {} <file to backup>\n".format(sys.argv[0])) sys.exit(1) if not os.path.isfile(backup_file): sys.stderr.write("Bad upload file {}\n".format(backup_file)) sys.exit(2) glacier_layer1 = Layer1(aws_access_key_id=ACCESS_KEY, aws_secret_access_key=SECRET_KEY, region_name=REGION_NAME) uploader = ConcurrentUploader(glacier_layer1, VAULT_NAME, 32*1024*1024) sys.stdout.write("Uploading {} as {}...".format( backup_file, os.path.basename(backup_file))) archive_id = uploader.upload(backup_file, os.path.basename(backup_file)) sys.stdout.write("done\n")
def delete_archive(self, archive_id): glacier_layer1 = Layer1(region_name=self.region_name) print("operation starting...") print glacier_layer1.delete_archive(self.target_vault_name, archive_id)
import sys import os.path from boto.glacier.layer1 import Layer1 from boto.glacier.vault import Vault from boto.glacier.concurrent import ConcurrentUploader access_key_id = None secret_key = None target_vault_name = 'backups' region_name = 'us-west-2' fname = sys.argv[1] if (os.path.isfile(fname) == False): print("Can't find the file to upload!") sys.exit(-1) glacier_layer1 = Layer1(aws_access_key_id=access_key_id, aws_secret_access_key=secret_key, region_name=region_name) uploader = ConcurrentUploader(glacier_layer1, target_vault_name, 32 * 1024 * 1024) print("operation starting...") archive_id = uploader.upload(fname, fname) print("Success! archive id: '%s'" % (archive_id))