def connect(region, debug_level=0, access_key=None, secret_key=None):
""" Connect to a specific region """
layer2 = connect_to_region(region,
aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
debug=debug_level)
if layer2 is None:
raise Exception('Invalid region %s, or bad creds' % region)
return layer2
def connect(region, debug_level=0, access_key=None, secret_key=None):
""" Connect to a specific region """
layer2 = connect_to_region(region,
aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
debug=debug_level)
if layer2 is None:
raise Exception('Invalid region %s, or bad creds' % region)
return layer2
def connect(region, debug_level=0, access_key=None, secret_key=None):
""" Connect to a specific region """
layer2 = connect_to_region(region,
aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
debug=debug_level)
if layer2 is None:
print('Invalid region (%s)' % region)
sys.exit(1)
return layer2
def connect_to_glacier_get_vault(region=REGION, vault=VAULT_NAME):
cn = glacier.connect_to_region(region)
print 'available regions:', glacier.regions()
vaults = cn.list_vaults()
print 'available vaults in region:', vaults
print 'using', vault, 'vault in region', region
if not vault in [v.name for v in vaults]:
print 'vault', vault, 'does not exist, creating it...'
return cn.create_vault(vault)
else:
return cn.get_vault(vault)
def main():
config = get_config()
LOG.info('initiating Glacier backup')
LOG.info('AWS region: %s' % config['login']['aws_region'])
LOG.info('currently in local directory: %s' % APP_ROOT)
# get a connection to Glacier
LOG.info('creating connection')
try:
conn = glacier.connect_to_region(
config['login']['aws_region'],
aws_access_key_id=config['login']['aws_access_key_id'],
aws_secret_access_key=config['login']['aws_secret_access_key'],
)
except Exception, e:
LOG.exception('connection failed, exception was:')
raise SystemExit, 1
if not vault_name:
print "You need to give a vault name and region in the first line of the config file, e.g. `MyVault|us-west-1`."
terminate(1)
if not len(dirs):
print r"You need to give the full path to a folder to sync in the second line of the config file, e.g. `C:\backups`. You can list multiple folders, e.g. `C:\backups|D:\backups`"
terminate(1)
for dir in dirs:
if not os.path.exists(dir):
print "Sync directory not found: " + dir
terminate(1)
# Cool! Let's set up everything.
connect_to_region(vault_info[1],
aws_access_key_id=access_key_id,
aws_secret_access_key=secret_key)
glacier = Layer2(aws_access_key_id=access_key_id,
aws_secret_access_key=secret_key,
region_name=region)
vault = glacier.get_vault(vault_name)
print "Beginning job on " + vault.arn
# Ah, we don't have a vault listing yet.
if not ls_present:
# No job yet? Initiate a job.
if not inventory_job:
inventory_job = vault.retrieve_inventory()
write()
print "Requested an inventory. This usually takes about four hours."
if not vault_name:
print "You need to give a vault name and region in the first line of the config file, e.g. `MyVault|us-west-1`."
sys.exit(1)
if not len(dirs):
print r"You need to give the full path to a folder to sync in the second line of the config file, e.g. `C:\backups`. You can list multiple folders, e.g. `C:\backups|D:\backups`"
sys.exit(1)
for dir in dirs:
if not os.path.isdir(dir):
print "Sync directory not found: " + dir
sys.exit(1)
# Cool! Let's set up everything.
connect_to_region(vault_info[1], aws_access_key_id=access_key_id, aws_secret_access_key=secret_key)
glacier = Layer2(aws_access_key_id=access_key_id, aws_secret_access_key=secret_key, region_name=region)
if arg.list_vaults:
print_vault_list(glacier.list_vaults())
sys.exit()
elif arg.create:
print "Creating vault " + vault_name
vault = glacier.create_vault(vault_name)
else:
vault = glacier.get_vault(vault_name)
# If we need to list all jobs
if arg.list_jobs:
list_jobs = vault.list_jobs()
for job in list_jobs:
def main():
config = read_config()
# Cool! Let's set up everything.
connect_to_region(config.region, aws_access_key_id=access_key_id, aws_secret_access_key=secret_key)
glacier = Layer2(aws_access_key_id=access_key_id, aws_secret_access_key=secret_key, region_name=config.region)
vault = glacier.get_vault(config.vault_name)
# workaround for UnicodeDecodeError
# https://github.com/boto/boto/issues/3318
vault.name = str(vault.name)
print "Beginning job on " + vault.arn
# Ah, we don't have a vault listing yet.
if not config.ls_present:
# No job yet? Initiate a job.
if not config.inventory_job:
config.inventory_job = vault.retrieve_inventory()
config.write()
print "Requested an inventory. This usually takes about four hours."
terminate(0)
# We have a job, but it's not finished.
job = vault.get_job(config.inventory_job)
if not job.completed:
print "Waiting for an inventory. This usually takes about four hours."
terminate(0)
# Finished!
try:
data = json.loads(job.get_output().read())
except ValueError:
print "Something went wrong interpreting the data Amazon sent!"
terminate(1)
config.ls = {}
for archive in data['ArchiveList']:
config.ls[archive['ArchiveDescription']] = {
'id': archive['ArchiveId'],
'last_modified': int(float(time.mktime(parse_ts(archive['CreationDate']).timetuple()))),
'size': int(archive['Size']),
'hash': archive['SHA256TreeHash']
}
config.ls_present = '-'
config.inventory_job = ''
config.write()
print "Imported a new inventory from Amazon."
database = Database(
host=db_host,
port=db_port,
username=db_username,
password=db_password,
name=db_name
)
print "Connected to database."
# Let's upload!
os.stat_float_times(False)
try:
i = 0
transferred = 0
time_begin = time.time()
for dir in config.dirs:
print "Syncing " + dir
for file in database.files():
path = dir + os.sep + file
if not os.path.exists(path):
#print >> sys.stderr, "'%s' does not exist" % path
print "\n" + "'%s' does not exist" % path
continue
# If it's a directory, then ignore it
if not os.path.isfile(path):
continue
last_modified = int(os.path.getmtime(path))
size = os.path.getsize(path)
updating = False
if file in config.ls:
# Has it not been modified since?
if config.ls[file]['last_modified'] >= last_modified and config.ls[file]['size'] == size:
continue
# It's been changed... we should delete the old one
else:
vault.delete_archive(config.ls[file]['id'])
del config.ls[file]
updating = True
config.write()
try:
print file + ": uploading... ",
id = vault.concurrent_create_archive_from_file(path, file)
config.ls[file] = {
'id': id,
'size': size,
'last_modified': last_modified
}
config.write()
i += 1
transferred += size
if updating:
print "updated."
else:
print "done."
database.update(file, id, vault)
except UploadArchiveError:
print "FAILED TO UPLOAD."
finally:
database.close()
elapsed = time.time() - time_begin
print "\n" + str(i) + " files successfully uploaded."
print "Transferred " + format_bytes(transferred) + " in " + format_time(elapsed) + " at rate of " + format_bytes(transferred / elapsed) + "/s."
terminate(0)
def setUp(self):
self.conn = connect_to_region("us-east-1")
self.vault_name = 'boto-test-vault-%s' % (uuid.uuid1(), )
self.conn.create_vault(self.vault_name)
self.vault = self.conn.get_vault(self.vault_name)
