def concurrent_create_archive_from_file(self, filename, description,
**kwargs):
"""
Create a new archive from a file and upload the given
file.
This is a convenience method around the
:class:`boto.glacier.concurrent.ConcurrentUploader`
class. This method will perform a multipart upload
and upload the parts of the file concurrently.
:type filename: str
:param filename: A filename to upload
:param kwargs: Additional kwargs to pass through to
:py:class:`boto.glacier.concurrent.ConcurrentUploader`.
You can pass any argument besides the ``api`` and
``vault_name`` param (these arguments are already
passed to the ``ConcurrentUploader`` for you).
:raises: `boto.glacier.exception.UploadArchiveError` is an error
occurs during the upload process.
:rtype: str
:return: The archive id of the newly created archive
"""
uploader = ConcurrentUploader(self.layer1, self.name, **kwargs)
archive_id = uploader.upload(filename, description)
return archive_id
def test_calculate_required_part_size(self):
self.stat_mock.return_value.st_size = 1024 * 1024 * 8
uploader = ConcurrentUploader(mock.Mock(), 'vault_name')
total_parts, part_size = uploader._calculate_required_part_size(
1024 * 1024 * 8)
self.assertEqual(total_parts, 2)
self.assertEqual(part_size, 4 * 1024 * 1024)
def upload_archive(fname,vault,real_name,chunk=None,description=''):
"""
Upload a file to glacier via the web-server.
"""
if not os.path.isfile(fname):
print("%s is not a valid file! Upload failed!" % fname)
return None
if chunk is None:
chunk=WG.app.config.get("CHUNK_SIZE",1048576)
handler = get_handler()
uploader = ConcurrentUploader(handler,str(vault.name),part_size=chunk)
if WG.app.config.get("VERBOSE",False):
print("Beginning upload of file %s to Glacier. Please by patient, there is no progress bar..." % fname)
file_size = os.path.getsize(fname)
if file_size==0:
if WG.app.config.get("VERBOSE",False):
print("File size is 0. Cannot upload empty file.")
return None
csum = chunkedmd5(fname)
itime=time.time()
file_name = os.path.basename(real_name)
machine_id = str(request.remote_addr)
#Construct a meaningful description object for the file
#The limits are that the description can be no more than 1024 characters in length and must use only ascii characters between 32 and 126 (i.e., 32<=ord(char)<=126)
dscrip = description+'\\n'
dscrip = dscrip + "Uploaded at "+str(itime)+'\\n'+ "Full path "+str(real_name)+'\\n'+ "File size "+str(file_size)+'\\n' + "MD5 "+str(csum)+'\\n' + "Source machine id "+machine_id+'\\n'
archive_id = uploader.upload(fname,description)
if WG.app.config.get("VERBOSE",False):
print("Successfully uploaded %s" % fname)
archive = Archive(archive_id,description,vault,filename=file_name,fullpath=real_name,filesize=file_size,md5sum=csum)
archive.insertion_date = datetime.fromtimestamp(int(itime))
WG.db.session.add(archive)
WG.db.session.commit()
def uploadToGlacier(tempTarFile=None,
DEBUG_MODE=False,
GLACIER_VAULT=None,
SECRET_ACCESS_KEY=None,
ACCESS_KEY=None,
GLACIER_REALM=None):
global logger
if not tempTarFile:
return 0
# Establish a connection to the Glacier
glacier_vault_in = None
my_archive = None
archive_id = None
try:
#my_glacier = GlacierConnection(ACCESS_KEY,SECRET_ACCESS_KEY,region=GLACIER_REALM)
my_glacier = Layer1(aws_access_key_id=ACCESS_KEY,
aws_secret_access_key=SECRET_ACCESS_KEY,
region_name=GLACIER_REALM)
if DEBUG_MODE:
logger.debug("Glacier Connection: %s" % my_glacier)
# Create a new vault (not neccessary if you already have one!)
if GLACIER_VAULT:
#glacier_vault_in = my_glacier.get_vault(GLACIER_VAULT)
#vaults = my_glacier.get_all_vaults()
vaults = my_glacier.list_vaults()
glacier_vault_in = None
if GLACIER_VAULT not in vaults:
glacier_vault_in = my_glacier.create_vault(GLACIER_VAULT)
else:
GLACIER_VAULT = id_generator(size=16)
glacier_vault_in = my_glacier.create_vault(GLACIER_VAULT)
if DEBUG_MODE:
logger.debug("Glacier Vault: %s" % glacier_vault_in)
#my_archive = GlacierArchive(tempTarFile)
uploader = ConcurrentUploader(my_glacier, GLACIER_VAULT,
64 * 1024 * 1024)
if DEBUG_MODE:
#logger.debug("Archive created in mem: %s " % my_archive)
logger.debug("Archive created in mem:%s" % uploader)
#glacier_vault_in.upload(my_archive)
archive_id = uploader.upload(tempTarFile, tempTarFile)
if DEBUG_MODE:
logger.info("upload created: %s" % glacier_vault_in)
except Exception, exc:
if exc.args > 0:
x, y = exc.args
errstr = None
try:
errstr = json.loads(y.read())
except:
errstr = y
logger.error("Error in glacier upload %s" % errstr)
else:
logger.error("Error in glacier upload %s" % (exc))
def test_calculate_required_part_size_too_small(self):
too_small = 1 * 1024 * 1024
self.stat_mock.return_value.st_size = 1024 * 1024 * 1024
uploader = ConcurrentUploader(mock.Mock(), 'vault_name',
part_size=too_small)
total_parts, part_size = uploader._calculate_required_part_size(
1024 * 1024 * 1024)
self.assertEqual(total_parts, 256)
# Part size if 4MB not the passed in 1MB.
self.assertEqual(part_size, 4 * 1024 * 1024)
def upload_archive(self, file_name):
glacier_layer1 = Layer1(region_name=self.region_name)
uploader = ConcurrentUploader(glacier_layer1, self.target_vault_name,
32 * 1024 * 1024)
print("operation starting...")
archive_id = uploader.upload(file_name, file_name)
print("Success! archive id: '%s'" % archive_id)
def upload_file(command):
"""
Uploads a file from the local machine that is specified in the given command.
"""
if 'action' not in command or command['action']!="UPLOAD":
raise ValueError("Command not of type UPLOAD")
if 'file_pattern' not in command:
raise ValueError("Missing file pattern")
path = command['file_pattern']
if not os.path.exists(path):
raise ValueError("No valid file for upload found")
returner={}
handler = Layer1(aws_access_key_id = command['access_key'],aws_secret_access_key = command['secret_access_key'],region_name=command['region_name'])
uploader = ConcurrentUploader(handler,command['vault_name'],part_size=uchunk)
file_size = os.path.getsize(path)
if file_size==0:
raise ValueError("File is empty. Nothing to upload.")
csum = chunkedmd5(path)
itime=time.time()
file_name = os.path.basename(path)
machine_id = str(command['target']) if client_name == '' else client_name+' ('+str(command['target']) + ')'
#Construct a meaningful description object for the file
#The limits are that the description can be no more than 1024 characters in length and must use only ascii characters between 32 and 126 (i.e., 32<=ord(char)<=126)
dscrip = command['description']+'\\n'
dscrip = dscrip + "Uploaded at "+str(itime)+'\\n'+ "Full path "+str(path)+'\\n'+ "File size "+str(file_size)+'\\n' + "MD5 "+str(csum)+'\\n' + "Source machine id "+machine_id+'\\n'
print "Uploading file %s"%file_name
#Put some validation stuff here...
#Do the upload
archive_id = uploader.upload(path,dscrip)
print "Completed successfully. Archive ID: %s"%archive_id
#Done the upload, send the bastard back
returner['archive_id'] = archive_id
returner['description'] = dscrip
returner['file_name'] = file_name
returner['true_path'] = path
returner['file_size'] = file_size
returner['md5sum'] = csum
returner['insert_time']=int(itime)
returner['region_name']=command['region_name']
returner['vault_name'] = command['vault_name']
return returner
# Upload archive file to AWS Glacier
try:
vault_name = str(data['vault_name'])
part_size = int(data['part_size'])
num_threads = int(data['num_threads'])
print "Backing archive of " + volid + " to Glacier"
glacierconn = Layer1(
aws_access_key_id=data['aws_access_key_id'],
aws_secret_access_key=data[
'aws_secret_access_key'],
region_name=data['region'])
vault = glacierconn.describe_vault(vault_name)
uploader = ConcurrentUploader(
glacierconn,
vault_name,
part_size=part_size,
num_threads=num_threads)
archive_id = uploader.upload(
(archive_file + ".tar.gz"), "first upload")
print "Download the archive using: " + archive_id
except Exception as e:
logger.error("Glacier error: " + str(e))
# Detach the volume from temporary instance and associate it back to
# respective instance
try:
conn.detach_volume(volid)
time.sleep(15)
vol = conn.get_all_volumes(volid)
volstatus = vol[0].attachment_state()
import sys
import os.path
from boto.glacier.layer1 import Layer1
from boto.glacier.vault import Vault
from boto.glacier.concurrent import ConcurrentUploader
access_key_id = None
secret_key = None
target_vault_name = 'backups'
region_name = 'us-west-2'
fname = sys.argv[1]
if (os.path.isfile(fname) == False):
print("Can't find the file to upload!")
sys.exit(-1)
glacier_layer1 = Layer1(aws_access_key_id=access_key_id,
aws_secret_access_key=secret_key,
region_name=region_name)
uploader = ConcurrentUploader(glacier_layer1, target_vault_name,
32 * 1024 * 1024)
print("operation starting...")
archive_id = uploader.upload(fname, fname)
print("Success! archive id: '%s'" % (archive_id))
from boto.glacier.layer1 import Layer1
from boto.glacier.concurrent import ConcurrentUploader
import sys
import os.path
# XXX: replace these with your credentials
ACCESS_KEY = "AWS_ACCESS_KEY"
SECRET_KEY = "AWS_SECRET_KEY"
VAULT_NAME = "VAULT_NAME"
REGION_NAME = 'us-west-2'
try:
backup_file = sys.argv[1]
except IndexError:
sys.stderr.write("Usage: {} <file to backup>\n".format(sys.argv[0]))
sys.exit(1)
if not os.path.isfile(backup_file):
sys.stderr.write("Bad upload file {}\n".format(backup_file))
sys.exit(2)
glacier_layer1 = Layer1(aws_access_key_id=ACCESS_KEY,
aws_secret_access_key=SECRET_KEY,
region_name=REGION_NAME)
uploader = ConcurrentUploader(glacier_layer1, VAULT_NAME, 32*1024*1024)
sys.stdout.write("Uploading {} as {}...".format(
backup_file, os.path.basename(backup_file)))
archive_id = uploader.upload(backup_file, os.path.basename(backup_file))
sys.stdout.write("done\n")
