def concurrent_create_archive_from_file(self, filename, description, **kwargs): """ Create a new archive from a file and upload the given file. This is a convenience method around the :class:`boto.glacier.concurrent.ConcurrentUploader` class. This method will perform a multipart upload and upload the parts of the file concurrently. :type filename: str :param filename: A filename to upload :param kwargs: Additional kwargs to pass through to :py:class:`boto.glacier.concurrent.ConcurrentUploader`. You can pass any argument besides the ``api`` and ``vault_name`` param (these arguments are already passed to the ``ConcurrentUploader`` for you). :raises: `boto.glacier.exception.UploadArchiveError` is an error occurs during the upload process. :rtype: str :return: The archive id of the newly created archive """ uploader = ConcurrentUploader(self.layer1, self.name, **kwargs) archive_id = uploader.upload(filename, description) return archive_id
def test_calculate_required_part_size(self): self.stat_mock.return_value.st_size = 1024 * 1024 * 8 uploader = ConcurrentUploader(mock.Mock(), 'vault_name') total_parts, part_size = uploader._calculate_required_part_size( 1024 * 1024 * 8) self.assertEqual(total_parts, 2) self.assertEqual(part_size, 4 * 1024 * 1024)
def upload_archive(fname,vault,real_name,chunk=None,description=''): """ Upload a file to glacier via the web-server. """ if not os.path.isfile(fname): print("%s is not a valid file! Upload failed!" % fname) return None if chunk is None: chunk=WG.app.config.get("CHUNK_SIZE",1048576) handler = get_handler() uploader = ConcurrentUploader(handler,str(vault.name),part_size=chunk) if WG.app.config.get("VERBOSE",False): print("Beginning upload of file %s to Glacier. Please by patient, there is no progress bar..." % fname) file_size = os.path.getsize(fname) if file_size==0: if WG.app.config.get("VERBOSE",False): print("File size is 0. Cannot upload empty file.") return None csum = chunkedmd5(fname) itime=time.time() file_name = os.path.basename(real_name) machine_id = str(request.remote_addr) #Construct a meaningful description object for the file #The limits are that the description can be no more than 1024 characters in length and must use only ascii characters between 32 and 126 (i.e., 32<=ord(char)<=126) dscrip = description+'\\n' dscrip = dscrip + "Uploaded at "+str(itime)+'\\n'+ "Full path "+str(real_name)+'\\n'+ "File size "+str(file_size)+'\\n' + "MD5 "+str(csum)+'\\n' + "Source machine id "+machine_id+'\\n' archive_id = uploader.upload(fname,description) if WG.app.config.get("VERBOSE",False): print("Successfully uploaded %s" % fname) archive = Archive(archive_id,description,vault,filename=file_name,fullpath=real_name,filesize=file_size,md5sum=csum) archive.insertion_date = datetime.fromtimestamp(int(itime)) WG.db.session.add(archive) WG.db.session.commit()
def uploadToGlacier(tempTarFile=None, DEBUG_MODE=False, GLACIER_VAULT=None, SECRET_ACCESS_KEY=None, ACCESS_KEY=None, GLACIER_REALM=None): global logger if not tempTarFile: return 0 # Establish a connection to the Glacier glacier_vault_in = None my_archive = None archive_id = None try: #my_glacier = GlacierConnection(ACCESS_KEY,SECRET_ACCESS_KEY,region=GLACIER_REALM) my_glacier = Layer1(aws_access_key_id=ACCESS_KEY, aws_secret_access_key=SECRET_ACCESS_KEY, region_name=GLACIER_REALM) if DEBUG_MODE: logger.debug("Glacier Connection: %s" % my_glacier) # Create a new vault (not neccessary if you already have one!) if GLACIER_VAULT: #glacier_vault_in = my_glacier.get_vault(GLACIER_VAULT) #vaults = my_glacier.get_all_vaults() vaults = my_glacier.list_vaults() glacier_vault_in = None if GLACIER_VAULT not in vaults: glacier_vault_in = my_glacier.create_vault(GLACIER_VAULT) else: GLACIER_VAULT = id_generator(size=16) glacier_vault_in = my_glacier.create_vault(GLACIER_VAULT) if DEBUG_MODE: logger.debug("Glacier Vault: %s" % glacier_vault_in) #my_archive = GlacierArchive(tempTarFile) uploader = ConcurrentUploader(my_glacier, GLACIER_VAULT, 64 * 1024 * 1024) if DEBUG_MODE: #logger.debug("Archive created in mem: %s " % my_archive) logger.debug("Archive created in mem:%s" % uploader) #glacier_vault_in.upload(my_archive) archive_id = uploader.upload(tempTarFile, tempTarFile) if DEBUG_MODE: logger.info("upload created: %s" % glacier_vault_in) except Exception, exc: if exc.args > 0: x, y = exc.args errstr = None try: errstr = json.loads(y.read()) except: errstr = y logger.error("Error in glacier upload %s" % errstr) else: logger.error("Error in glacier upload %s" % (exc))
def test_upload(): print("create vault") glacier_layer1.create_vault(target_vault_name) print("start to upload") uploader = ConcurrentUploader(glacier_layer1, target_vault_name, 32*1024*1024) archive_id = uploader.upload(fname, fname) print("upload success! archive id: '%s'"%(archive_id)) return archive_id
def uploadToGlacier(tempTarFile=None, DEBUG_MODE=False, GLACIER_VAULT=None, SECRET_ACCESS_KEY=None, ACCESS_KEY=None, GLACIER_REALM=None): global logger if not tempTarFile: return 0 # Establish a connection to the Glacier glacier_vault_in=None my_archive=None archive_id=None try: #my_glacier = GlacierConnection(ACCESS_KEY,SECRET_ACCESS_KEY,region=GLACIER_REALM) my_glacier = Layer1(aws_access_key_id=ACCESS_KEY,aws_secret_access_key=SECRET_ACCESS_KEY,region_name=GLACIER_REALM) if DEBUG_MODE: logger.debug("Glacier Connection: %s" % my_glacier) # Create a new vault (not neccessary if you already have one!) if GLACIER_VAULT: #glacier_vault_in = my_glacier.get_vault(GLACIER_VAULT) #vaults = my_glacier.get_all_vaults() vaults = my_glacier.list_vaults() glacier_vault_in = None if GLACIER_VAULT not in vaults: glacier_vault_in = my_glacier.create_vault(GLACIER_VAULT) else: GLACIER_VAULT = id_generator(size=16) glacier_vault_in = my_glacier.create_vault(GLACIER_VAULT) if DEBUG_MODE: logger.debug("Glacier Vault: %s" % glacier_vault_in) #my_archive = GlacierArchive(tempTarFile) uploader = ConcurrentUploader(my_glacier, GLACIER_VAULT, 64*1024*1024) if DEBUG_MODE: #logger.debug("Archive created in mem: %s " % my_archive) logger.debug("Archive created in mem:%s" % uploader ) #glacier_vault_in.upload(my_archive) archive_id = uploader.upload(tempTarFile, tempTarFile) if DEBUG_MODE: logger.info("upload created: %s" % glacier_vault_in) except Exception,exc: if exc.args>0: x, y = exc.args errstr = None try: errstr = json.loads(y.read()) except: errstr = y logger.error("Error in glacier upload %s" % errstr) else: logger.error("Error in glacier upload %s" % (exc))
def test_calculate_required_part_size_too_small(self): too_small = 1 * 1024 * 1024 self.stat_mock.return_value.st_size = 1024 * 1024 * 1024 uploader = ConcurrentUploader(mock.Mock(), 'vault_name', part_size=too_small) total_parts, part_size = uploader._calculate_required_part_size( 1024 * 1024 * 1024) self.assertEqual(total_parts, 256) # Part size if 4MB not the passed in 1MB. self.assertEqual(part_size, 4 * 1024 * 1024)
def upload_archive(self, file_name): glacier_layer1 = Layer1(region_name=self.region_name) uploader = ConcurrentUploader(glacier_layer1, self.target_vault_name, 32 * 1024 * 1024) print("operation starting...") archive_id = uploader.upload(file_name, file_name) print("Success! archive id: '%s'" % archive_id)
def archive_upload(self, multipart=False, encryptor=None, part_size=DEFAULT_PART_SIZE, num_threads=DEFAULT_NUM_THREADS): # XXX: "Leading whitespace in archive descriptions is removed." # XXX: "The description must be less than or equal to 1024 bytes. The # allowable characters are 7 bit ASCII without control codes, # specifically ASCII values 32-126 decimal or 0x20-0x7E # hexadecimal." if self.args.name is not None: name = self.args.name else: try: full_name = self.args.file.name except: raise RuntimeError("Archive name not specified. Use --name.") name = os.path.basename(full_name) if not self.args.encrypt: filename = self.args.file else: filename = tempfile.NamedTemporaryFile().name logger.info("Encrypting %s to %s." % (self.args.file, filename)) encryptor.encrypt_file(self.args.file, filename) logger.info("Encryption complete: %s." % filename) vault = self.connection.get_vault(self.args.vault) if not multipart: logger.info("Uploading in a single part: %s to %s." % (filename, vault)) file_obj = file(filename) archive_id = vault.create_archive_from_file( file_obj = file_obj, description=name) else: logger.info("Uploading multi-part: %s to %s" % (filename, vault)) uploader = ConcurrentUploader(self.connection.layer1, vault.name, part_size=part_size, num_threads=num_threads) archive_id = uploader.upload(filename, description=name) logger.info("Upload complete.") logger.info("New Archive ID: %s" % archive_id) self.cache.add_archive(self.args.vault, name, archive_id) if self.args.encrypt: os.remove(filename)
class glacier_freeze: def __init__(self): self.argparser = self.argparser() self.parsecreds = self.parsecreds() self.freeze = self.freeze() #self.validatedata = self.validatedata() def argparser(self): #Setting up parsing options for inputting data parser = argparse.ArgumentParser(description="Uploads file to glacier") parser.add_argument("-a", "--archive-file", required=True, help="file to be archived") parser.add_argument("-c", "--creds", required=True, help="Path to Credential .csv file from amazon") parser.add_argument("-n", "--vault-name", required=True, help="Name of the vault to upload to") parser.add_argument("-d", "--description", required=False, help="Optional descrition for the archive") parser.add_argument("-v", "--verbose", action='store_true',required=False, default=False,help="verbose output") parser.add_argument("-b", "--block-size", required=False, default=33554432, help="block size to write. Amazon default it 33554432") parser.add_argument("-p", "--proxy", required=False, default="rcproxy.rc.fas.harvard.edu:8888", help="override proxy settings default odyssey proxy setting are => rcproxy.rc.fas.harvard.edu:8888") args = parser.parse_args() self.archive_file = args.archive_file self.creds = args.creds self.verbose = args.verbose self.block_size=args.block_size self.vault_name = args.vault_name self.proxy_address = args.proxy.split(':')[0] self.proxy_port = args.proxy.split(':')[1] self.description = args.description #if there's no description we should just make it they name as the file name for the archve. if self.description == None: self.description = self.archive_file debug = args.verbose log_level = logging.INFO if debug == True: log_level = logging.DEBUG logging.basicConfig(filename="glacier.log", level=log_level, format=LOG_FORMAT) logger.debug(" ".join(sys.argv)) def parsecreds(self): # getting credentials from CSV file assigning them to variables if not os.path.exists(self.creds): logger.critical("credential file does not exists => %s" % self.creds) else: with open(self.creds) as csvfile: reader = csv.DictReader(csvfile) for row in reader: self.glacier_users = row['User Name'] self.glacier_key_id = row['Access Key Id'] self.glacier_secret_key = row['Secret Access Key'] def freeze(self): if not os.path.exists(self.archive_file): logger.critical("archive file does not exists => %s" % self.archive_file) else: self.glacier_layer1 = Layer1(aws_access_key_id=self.glacier_key_id, aws_secret_access_key=self.glacier_secret_key) self.glacier_upload = ConcurrentUploader(self.glacier_layer1, self.vault_name, self.block_size) logger.info("pushing data to glacier... => %s" % self.glacier_upload) self.archive_id = self.glacier_upload.upload(self.archive_file,self.description) logger.info("Sucess! archive id is => %s" % self.archive_id)
def freeze(self): if not os.path.exists(self.archive_file): logger.critical("archive file does not exists => %s" % self.archive_file) else: self.glacier_layer1 = Layer1(aws_access_key_id=self.glacier_key_id, aws_secret_access_key=self.glacier_secret_key) self.glacier_upload = ConcurrentUploader(self.glacier_layer1, self.vault_name, self.block_size) logger.info("pushing data to glacier... => %s" % self.glacier_upload) self.archive_id = self.glacier_upload.upload(self.archive_file,self.description) logger.info("Sucess! archive id is => %s" % self.archive_id)
def freeze(self): if not os.path.exists(self.archive_dir): logger.critical("archive file does not exists => %s" % self.archive_dir) sys.exit() else: self.glacier_layer1 = Layer1(aws_access_key_id=self.glacier_key_id, aws_secret_access_key=self.glacier_secret_key) self.glacier_upload = ConcurrentUploader(self.glacier_layer1, self.vault_name, self.block_size) logger.info("pushing data to glacier... => %s" % self.glacier_upload) self.archive_id = self.glacier_upload.upload(self.archive_dir,self.description) logger.info("Sucess! archive id is => %s" % self.archive_id) print "Sucess! archive id is => %s \nPlease allow up to 4 hrs before checking archive" % self.archive_id
def upload_file(command): """ Uploads a file from the local machine that is specified in the given command. """ if 'action' not in command or command['action']!="UPLOAD": raise ValueError("Command not of type UPLOAD") if 'file_pattern' not in command: raise ValueError("Missing file pattern") path = command['file_pattern'] if not os.path.exists(path): raise ValueError("No valid file for upload found") returner={} handler = Layer1(aws_access_key_id = command['access_key'],aws_secret_access_key = command['secret_access_key'],region_name=command['region_name']) uploader = ConcurrentUploader(handler,command['vault_name'],part_size=uchunk) file_size = os.path.getsize(path) if file_size==0: raise ValueError("File is empty. Nothing to upload.") csum = chunkedmd5(path) itime=time.time() file_name = os.path.basename(path) machine_id = str(command['target']) if client_name == '' else client_name+' ('+str(command['target']) + ')' #Construct a meaningful description object for the file #The limits are that the description can be no more than 1024 characters in length and must use only ascii characters between 32 and 126 (i.e., 32<=ord(char)<=126) dscrip = command['description']+'\\n' dscrip = dscrip + "Uploaded at "+str(itime)+'\\n'+ "Full path "+str(path)+'\\n'+ "File size "+str(file_size)+'\\n' + "MD5 "+str(csum)+'\\n' + "Source machine id "+machine_id+'\\n' print "Uploading file %s"%file_name #Put some validation stuff here... #Do the upload archive_id = uploader.upload(path,dscrip) print "Completed successfully. Archive ID: %s"%archive_id #Done the upload, send the bastard back returner['archive_id'] = archive_id returner['description'] = dscrip returner['file_name'] = file_name returner['true_path'] = path returner['file_size'] = file_size returner['md5sum'] = csum returner['insert_time']=int(itime) returner['region_name']=command['region_name'] returner['vault_name'] = command['vault_name'] return returner
def upload_file(filename, description, vault, layer1): uploader = ConcurrentUploader(layer1, vault, 32*1024*1024) archive_id = uploader.upload(filename, description) return archive_id
def aws_create_archive(self, file, description): aws_uploader = BotoConcurrentUploader(self.boto_glacier_l1, self.vault_name, 32 * (1 << 20)) archive_id = aws_uploader.upload(file, description) return archive_id
target_vault_name = "Type your Vault name here" def upload_archive:() '''This defenition takes care of uploading the file to the AWS Glacier one file at a time''' glacier_layer1 = Layer1(aws_access_key_id='Type your AWS access ID', aws_secret_access_key='Type your AWS secret access key',region_name='Pick a region to upload') print "\nList of all files older than " + str(opts.days) + " days and greater than " + str(opts.size) + " in " + str(opts.folder) print "==========================" *len(str(opts.days)) + "===================" for root, dirs, files in walk(opts.folder): for name in files: filename = path.join(root,name) try: if stat(filename).st_mtime < now - (int(opts.days) * 86400) and stat(filename).st_size > int(size)*xsize : realsize = format_size(stat(filename).st_size) print "%s" %realsize + " %s" %filename uploader = ConcurrentUploader(glacier_layer1, target_vault_name, 32*1024*1024) archive_id = uploader.upload(filename, filename) print("upload success! archive id: '%s'"%(archive_id)) return archive_id sizelist.append(realsize) except OSError: pass def vault_inventory_jobid(): glacier_layer1 = Layer1(aws_access_key_id='Type your AWS access ID', aws_secret_access_key='Type your AWS secret access key',region_name='Pick a region to upload') # http://docs.aws.amazon.com/amazonglacier/latest/dev/api-initiate-job-post.html job_id = glacier_layer1.initiate_job(target_vault_name, {"Description":"inventory-job", "Type":"inventory-retrieval", "Format":"JSON"}) print json.dumps(job_id, indent=2) return job_id def vault_list_jobs():
def upload_file(self,vault_name, file_name): log("starting upload file %s %s" % (file_name,vault_name)) uploader = ConcurrentUploader(self.glacier_layer, vault_name, part_size = 32 * 1024 * 1024 ) archive_id = uploader.upload(file_name, "") log("upload finished! %s" % archive_id) return archive_id
# Upload archive file to AWS Glacier try: vault_name = str(data['vault_name']) part_size = int(data['part_size']) num_threads = int(data['num_threads']) print "Backing archive of " + volid + " to Glacier" glacierconn = Layer1( aws_access_key_id=data['aws_access_key_id'], aws_secret_access_key=data[ 'aws_secret_access_key'], region_name=data['region']) vault = glacierconn.describe_vault(vault_name) uploader = ConcurrentUploader( glacierconn, vault_name, part_size=part_size, num_threads=num_threads) archive_id = uploader.upload( (archive_file + ".tar.gz"), "first upload") print "Download the archive using: " + archive_id except Exception as e: logger.error("Glacier error: " + str(e)) # Detach the volume from temporary instance and associate it back to # respective instance try: conn.detach_volume(volid) time.sleep(15) vol = conn.get_all_volumes(volid) volstatus = vol[0].attachment_state()
from boto.glacier.layer1 import Layer1 from boto.glacier.concurrent import ConcurrentUploader import sys import os.path # XXX: replace these with your credentials ACCESS_KEY = "AWS_ACCESS_KEY" SECRET_KEY = "AWS_SECRET_KEY" VAULT_NAME = "VAULT_NAME" REGION_NAME = 'us-west-2' try: backup_file = sys.argv[1] except IndexError: sys.stderr.write("Usage: {} <file to backup>\n".format(sys.argv[0])) sys.exit(1) if not os.path.isfile(backup_file): sys.stderr.write("Bad upload file {}\n".format(backup_file)) sys.exit(2) glacier_layer1 = Layer1(aws_access_key_id=ACCESS_KEY, aws_secret_access_key=SECRET_KEY, region_name=REGION_NAME) uploader = ConcurrentUploader(glacier_layer1, VAULT_NAME, 32*1024*1024) sys.stdout.write("Uploading {} as {}...".format( backup_file, os.path.basename(backup_file))) archive_id = uploader.upload(backup_file, os.path.basename(backup_file)) sys.stdout.write("done\n")
# Upload archive file to AWS Glacier try: vault_name = str(data['vault_name']) part_size = int(data['part_size']) num_threads = int(data['num_threads']) print "Backing archive of " + volid + " to Glacier" glacierconn = Layer1( aws_access_key_id=data['aws_access_key_id'], aws_secret_access_key=data[ 'aws_secret_access_key'], region_name=data['region']) vault = glacierconn.describe_vault(vault_name) uploader = ConcurrentUploader( glacierconn, vault_name, part_size=part_size, num_threads=num_threads) archive_id = uploader.upload( (archive_file + ".tar.gz"), "first upload") print "Download the archive using: " + archive_id except Exception as e: logger.error("Glacier error: " + str(e)) # Detach the volume from temporary instance and associate it back to # respective instance try: conn.detach_volume(volid) time.sleep(15) vol = conn.get_all_volumes(volid)
import sys import os.path from boto.glacier.layer1 import Layer1 from boto.glacier.vault import Vault from boto.glacier.concurrent import ConcurrentUploader access_key_id = None secret_key = None target_vault_name = 'backups' region_name = 'us-west-2' fname = sys.argv[1] if (os.path.isfile(fname) == False): print("Can't find the file to upload!"); sys.exit(-1); glacier_layer1 = Layer1(aws_access_key_id=access_key_id, aws_secret_access_key=secret_key, region_name=region_name) uploader = ConcurrentUploader(glacier_layer1, target_vault_name, 32 * 1024 * 1024) print("operation starting..."); archive_id = uploader.upload(fname, fname) print("Success! archive id: '%s'" % (archive_id))
def main(): layer1 = Layer1(aws_access_key_id=config.get('glacier', 'aws_access_key_id'), aws_secret_access_key=config.get('glacier', 'aws_secret_access_key'), region_name=config.get('glacier', 'region')) threads = 1 try: threads = config.getint('glacier', 'threads') except ConfigParser.NoOptionError: pass uploader = ConcurrentUploader(layer1, config.get('glacier', 'vault'), part_size=128*1024*1024, num_threads=threads) if len(sys.argv) > 1: for filename in sys.argv[1:]: contents_file = config.get('glacier','contents') existing_contents = {} if contents_file: try: file = open(contents_file, 'r+') existing_contents = json.loads(file.read()) except IOError: file = open(contents_file, 'w+') file.close() filenames = [x['ArchiveDescription'] for x in \ existing_contents.values()] archive_description = filename.split('/')[-1] if os.path.isfile(filename) and archive_description \ not in filenames: id = uploader.upload(filename, archive_description) print('Uploaded: {0}, id: {1}'.format(filename, id)) filesize = 0 try: filesize = os.stat(filename).st_size except OSError: pass #contents_file = config.get('glacier','contents') list_of_files = {id:{'ArchiveId':id, 'ArchiveDescription':archive_description, 'CreationDate':datetime.datetime.now(). \ strftime('%Y-%m-%dT%H:%M:%SZ'), 'Size':filesize}} if contents_file: file = open(contents_file, 'w+') list_of_files = dict(list_of_files.items() + \ existing_contents.items()) file.seek(0) file.write(json.dumps(list_of_files)) file.close() else: if not os.path.isfile(filename): print('Couldn\'t find file: {0}'.format(filename)) else: print('File {filename} is already in glacier' \ .format(filename=filename)) else: print("""\ Need at least one filename on the command line, can accept globs. Example: {0} *.py""".format(sys.argv[0]))
import sys import os.path from boto.glacier.layer1 import Layer1 from boto.glacier.vault import Vault from boto.glacier.concurrent import ConcurrentUploader access_key_id = None secret_key = None target_vault_name = 'backups' region_name = 'us-west-2' fname = sys.argv[1] if (os.path.isfile(fname) == False): print("Can't find the file to upload!") sys.exit(-1) glacier_layer1 = Layer1(aws_access_key_id=access_key_id, aws_secret_access_key=secret_key, region_name=region_name) uploader = ConcurrentUploader(glacier_layer1, target_vault_name, 32 * 1024 * 1024) print("operation starting...") archive_id = uploader.upload(fname, fname) print("Success! archive id: '%s'" % (archive_id))