def __init__(self,
bucket,
vault=None,
c=None,
gc=None,
status=None,
retrieve_bph=1491308088):
if vault is None:
vault = bucket
self.retries = 40
self.status = status
if c is None:
c = boto.connect_s3()
if gc is None:
gc = boto.connect_glacier()
self.bucket = bucket
self.vault = vault
self.c = c
try:
self.b = c.get_bucket(bucket)
except boto.exception.S3ResponseError:
self.b = c.create_bucket(bucket)
try:
self.v = gc.get_vault(vault)
except glacierexceptions.UnexpectedHTTPResponseError:
self.v = gc.create_vault(vault)
self.gl1 = self.v.layer1
self.gc = gc
self.get_jobs()
self.chunk_queue = []
self.aid_cache = {}
self.last_job_creation = 0
self.bph = retrieve_bph
def __init__(self, bucket, vault=None, c=None, gc=None,
status=None, retrieve_bph=1491308088):
if vault is None:
vault = bucket
self.retries = 40
self.status = status
if c is None:
c = boto.connect_s3()
if gc is None:
gc = boto.connect_glacier()
self.bucket = bucket
self.vault = vault
self.c = c
try:
self.b = c.get_bucket(bucket)
except boto.exception.S3ResponseError:
self.b = c.create_bucket(bucket)
try:
self.v = gc.get_vault(vault)
except glacierexceptions.UnexpectedHTTPResponseError:
self.v = gc.create_vault(vault)
self.gl1 = self.v.layer1
self.gc = gc
self.get_jobs()
self.chunk_queue = []
self.aid_cache = {}
self.last_job_creation = 0
self.bph = retrieve_bph
def __init__(self, conf):
if conf is None:
try:
access_key = config.get("aws", "access_key")
secret_key = config.get("aws", "secret_key")
vault_name = config.get("aws", "glacier_vault")
try:
region_name = config.get("aws", "region_name")
except ConfigParser.NoOptionError:
region_name = DEFAULT_LOCATION
except ConfigParser.NoOptionError:
log.error("Configuration file not available.")
log.info("Use 'bakthat configure' to create one.")
return
else:
access_key = conf.get("access_key")
secret_key = conf.get("secret_key")
vault_name = conf.get("vault")
region_name = conf.get("region_name", DEFAULT_LOCATION)
con = boto.connect_glacier(aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
region_name=region_name)
self.conf = conf
self.vault = con.create_vault(vault_name)
self.backup_key = "bakthat_glacier_inventory"
self.container = "Glacier vault: {0}".format(vault_name)
def connect(db):
glacier_conn = boto.connect_glacier(
aws_access_key_id = ACCESS_KEY_ID,
aws_secret_access_key = SECRET_ACCESS_KEY,
)
db_conn = sqlite3.connect(db)
return db_conn, glacier_conn
def execute(self, args, config):
glacier_connection = boto.connect_glacier(aws_access_key_id=config.get('configuration', 'aws_key'), aws_secret_access_key=config.get('configuration', 'aws_secret'))
try:
vault = glacier_connection.get_vault(args[2])
print "Vault info:\n\tname={}\n\tarn={}\n\tcreation_date={}\n\tlast_inventory_date={}\n\tsize={}\n\tnumber_of_archives={}".format(vault.name, vault.arn, vault.creation_date, vault.last_inventory_date, vault.size, vault.number_of_archives)
except:
print "Vault named '{}' does not exist.".format(args[2])
def uploadToGlacier(filename): inventory = shelve.open(settings.ROOT_DIR+'backup/'+'SECURE_sensibledtu1k_glacier_inventory') glacier_connection = boto.connect_glacier(aws_access_key_id=SECURE_settings.GLACIER['ACCESS_KEY_ID'], aws_secret_access_key=SECURE_settings.GLACIER['SECRET_ACCESS_KEY'], region_name='eu-west-1') vault = glacier_connection.get_vault(SECURE_settings.GLACIER['VAULT']) archive_id = glacier.upload(inventory, vault, filename) inventory.close() print archive_id return archive_id
def __init__(self, vault_name):
"""
Initialize the vault
"""
layer2 = boto.connect_glacier(aws_access_key_id=ACCESS_KEY_ID,
aws_secret_access_key=SECRET_ACCESS_KEY)
self.vault = layer2.get_vault(vault_name)
def __init__(self, vault_name):
"""
Initialize the vault
"""
layer2 = boto.connect_glacier(aws_access_key_id = AWS_ACCESS_KEY_ID,
aws_secret_access_key = AWS_SECRET_ACCESS_KEY)
self.vault = layer2.get_vault(vault_name)
def __init__(self, conf={}, profile="default"):
BakthatBackend.__init__(self, conf, profile)
con = boto.connect_glacier(aws_access_key_id=self.conf["access_key"], aws_secret_access_key=self.conf["secret_key"], region_name=self.conf["region_name"])
self.vault = con.create_vault(self.conf["glacier_vault"])
self.backup_key = "bakthat_glacier_inventory"
self.container = self.conf["glacier_vault"]
self.container_key = "glacier_vault"
def __init__(self, *args, **kwargs):
super(AWSGlacier, self).__init__()
self.__dict__.update(kwargs)
self.path = kwargs.get('path', None)
self.name_backup = kwargs.get('name_backup', None)
self.glacier_connection = \
boto.connect_glacier(aws_access_key_id=AWS_ACCESS_KEY,
aws_secret_access_key=AWS_SECRET_KEY)
self.vault = self.glacier_connection.get_vault(AWS_VAULT_NAME)
def __init__(self, *args, **kwargs):
super(AWSGlacier, self).__init__()
self.__dict__.update(kwargs)
self.path = kwargs.get('path', None)
self.name_backup = kwargs.get('name_backup', None)
self.glacier_connection = \
boto.connect_glacier(aws_access_key_id=AWS_ACCESS_KEY,
aws_secret_access_key=AWS_SECRET_KEY)
self.vault = self.glacier_connection.get_vault(AWS_VAULT_NAME)
def __init__(self, conf=None):
BakthatBackend.__init__(self, conf, extra_conf=["glacier_vault", "s3_bucket"])
con = boto.connect_glacier(aws_access_key_id=self.conf["access_key"],
aws_secret_access_key=self.conf["secret_key"],
region_name=self.conf["region_name"])
self.vault = con.create_vault(self.conf["glacier_vault"])
self.backup_key = "bakthat_glacier_inventory"
self.container = "Glacier vault: {0}".format(self.conf["glacier_vault"])
def __init__(self, vault_name):
"""
Initialize the vault
"""
layer2 = boto.connect_glacier()
self.vault = layer2.get_vault(vault_name)
## Bodge to fix Boto unicode bug
## see https://github.com/boto/boto/issues/2603
self.vault.name = str(self.vault.name)
def __init__(self, config_file=CONFIG_FILE, key_id=None, key=None, region_name=REGION_NAME, record_file=RECORD_FILE, job_log=JOB_LOG):
if key_id is None and key is None:
logging.info('not enough credentials passed in, sourcing from config file')
with open(config_file, 'r') as fh:
cfg = parse_config_file(fh)
key_id = cfg['AWSAccessKeyId']
key = cfg['AWSSecretKey']
region_name = region_name
self.record_file = record_file
self.job_log = job_log
self.glacier = boto.connect_glacier(aws_access_key_id=key_id, aws_secret_access_key=key, region_name=region_name)
def __init__(self, vault_name):
"""
Initialize the vault
"""
layer2 = boto.connect_glacier()
self.vault = layer2.get_vault(vault_name)
## Bodge to fix Boto unicode bug
## see https://github.com/boto/boto/issues/2603
self.vault.name = str(self.vault.name)
def init(access_key, secret_key):
global glacier
glacier = connect_glacier(
aws_access_key_id = access_key,
aws_secret_access_key = secret_key
)
# Create the jobs table if it doesn't already exist
cursor.execute("CREATE TABLE IF NOT EXISTS jobs (id text, action text, status_code text, vault_name text)")
db.commit()
def uploadToGlacier(filename):
inventory = shelve.open(settings.ROOT_DIR + 'backup/' +
'SECURE_sensibledtu1k_glacier_inventory')
glacier_connection = boto.connect_glacier(
aws_access_key_id=SECURE_settings.GLACIER['ACCESS_KEY_ID'],
aws_secret_access_key=SECURE_settings.GLACIER['SECRET_ACCESS_KEY'],
region_name='eu-west-1')
vault = glacier_connection.get_vault(SECURE_settings.GLACIER['VAULT'])
archive_id = glacier.upload(inventory, vault, filename)
inventory.close()
print archive_id
return archive_id
def _connect_glacier(self):
for key in ['aws_access_key','aws_secret_key','aws_region_name']:
if not self._config.has_key(key):
raise GlacierSyncConfigException('The following configuration key is required: "%s"'%key)
if not self._profile.has_key('vault_name'):
raise GlacierSyncConfigException('The following profile configuration key is required: %s'%msg)
self._glacier_connection=boto.connect_glacier(
aws_access_key_id=self._config['aws_access_key'],
aws_secret_access_key=self._config['aws_secret_key'],
region_name=self._config['aws_region_name'],
)
self._glacier_vault=self._glacier_connection.create_vault(self._profile['vault_name'])
def __get_glacier_connection(self):
"""
Returns a connection to AWS Glacier.
:returns: glacier connection
:rtype: boto.glacier.layer2.Layer2
"""
return boto.connect_glacier(
aws_access_key_id=settings.AWS_ACCESS_KEY_ID,
aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY,
region_name=settings.AWS_REGION_NAME,
)
def __get_glacier_connection(self):
"""
Returns a connection to AWS Glacier.
:returns: glacier connection
:rtype: boto.glacier.layer2.Layer2
"""
return boto.connect_glacier(
aws_access_key_id=settings.AWS_ACCESS_KEY_ID,
aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY,
region_name=settings.AWS_REGION_NAME,
)
def upload(awsKeyId,awsSecret,awsVault,file,inventory,description=None):
connection=None
vault=None
try:
with open(inventory,'a') as f:
print("Verified inventory file is writable.")
except Exception as err:
raise Exception(
"Could not verify write access to inventory file. " + \
"ERROR: {}".format(err)
)
try:
with open(file,'rb') as f:
print("source file readability verified.")
except Exception as err:
raise Exception(
"Could not verify readability of source file." + \
"ERROR: {}".format(err)
)
try:
connection = boto.connect_glacier(
aws_access_key_id=awsKeyId,
aws_secret_access_key=awsSecret
)
except Exception as err:
showException("ERROR_CONNECTING",err)
try:
vault = connection.get_vault(awsVault)
except Exception as err:
try:
print("ERROR: Failed to get vault. ERROR:{}".format(err))
print("Attempting to create vaulte [{}]".format(awsVault))
vault = connection.create_vault(awsVault)
except Exception as err:
showException("ERROR_CREATING_VAULT:",err)
start=time.time()
try:
print("\n\nStarting upload [{}] at {}\n\n".format(file,start))
id = vault.upload_archive(file)
except Exception as err:
showException("ERROR_UPLOADING",err)
stop=time.time()
print("Done id:{} time:{}".format(id,stop))
with open(inventory,'a') as f:
f.write("'{}','{}','{}','{}'\n".format(file,start,stop,id))
return id
def delete_backup(path,vault,dbCreds,verbosity):
conn = mdb.connect(dbCreds['server'] , dbCreds['user'], dbCreds['password'], dbCreds['database']);
c = conn.cursor()
c.execute('''select x_amz_archive_id from {} where path = '%s' '''.format(vault) % ( path ) )
archiveID = c.fetchone()
try:
glacier_connection = boto.connect_glacier()
vaultObj = glacier_connection.get_vault(vault)
response = vaultObj.delete_archive(archiveID)
except Exception, e:
print("error on " + path + " " + str(e))
response = str(e)
def execute(self, args, config):
glacier_connection = boto.connect_glacier(aws_access_key_id=config.get('configuration', 'aws_key'), aws_secret_access_key=config.get('configuration', 'aws_secret'))
try:
vault = glacier_connection.get_vault(args[6])
except:
vault = None
if vault is None:
print "Vault named '{}' does not exist.".format(args[6])
else:
try:
job = vault.get_job(args[3])
print "Job info:\n\tAction: {}\n\tArchive Id: {}\n\tArchive Size:{}\n\tCompleted: {}\n\tCompletion Date: {}\n\tCreation Date: {} \n\tInventory Size: {}\n\tDescription: {}\n\tJobID: {}\n\tSNSTopic: {}\n\tStatus Code: {}\n\tStatus Message: {}\n".format(job.action, job.archive_id, job.archive_size, job.completed, job.completion_date, job.creation_date, job.inventory_size, job.description, job.id, job.sns_topic, job.status_code, job.status_message)
except UnexpectedHTTPResponseError as error:
print "Job '{}' can not be found:\n\t {}".format(args[3], error)
def _connect_glacier(self):
for key in ['aws_access_key', 'aws_secret_key', 'aws_region_name']:
if not self._config.has_key(key):
raise GlacierSyncConfigException(
'The following configuration key is required: "%s"' % key)
if not self._profile.has_key('vault_name'):
raise GlacierSyncConfigException(
'The following profile configuration key is required: %s' %
msg)
self._glacier_connection = boto.connect_glacier(
aws_access_key_id=self._config['aws_access_key'],
aws_secret_access_key=self._config['aws_secret_key'],
region_name=self._config['aws_region_name'],
)
self._glacier_vault = self._glacier_connection.create_vault(
self._profile['vault_name'])
def execute(self, args, config):
glacier_connection = boto.connect_glacier(aws_access_key_id=config.get('configuration', 'aws_key'), aws_secret_access_key=config.get('configuration', 'aws_secret'))
try:
vault = glacier_connection.get_vault(args[2])
except:
vault = None
if vault is None:
print "Vault named '{}' does not exist.".format(args[2])
else:
try:
vault = glacier_connection.delete_vault(args[2])
print "Vault deleted: '{}'".format(args[2])
except UnexpectedHTTPResponseError as error:
print "Vault can not be deleted:\n\t {}".format(error)
def execute(self, args, config):
glacier_connection = boto.connect_glacier(aws_access_key_id=config.get('configuration', 'aws_key'), aws_secret_access_key=config.get('configuration', 'aws_secret'))
try:
vault = glacier_connection.get_vault(args[4])
except:
vault = None
if vault is None:
print "Vault named '{}' does not exist.".format(args[4])
else:
jobs = vault.list_jobs()
if not jobs:
print "No jobs found for {}".format(args[4])
else:
count = 1
for job in jobs:
print "{}. Job info:\n\tAction: {}\n\tArchive Id: {}\n\tArchive Size:{}\n\tCompleted: {}\n\tCompletion Date: {}\n\tCreation Date: {} \n\tInventory Size: {}\n\tDescription: {}\n\tJobID: {}\n\tSNSTopic: {}\n\tStatus Code: {}\n\tStatus Message: {}\n".format(count, job.action, job.archive_id, job.archive_size, job.completed, job.completion_date, job.creation_date, job.inventory_size, job.description, job.id, job.sns_topic, job.status_code, job.status_message)
count += 1
def execute(self, args, config):
glacier_connection = boto.connect_glacier(aws_access_key_id=config.get('configuration', 'aws_key'), aws_secret_access_key=config.get('configuration', 'aws_secret'))
try:
vault = glacier_connection.get_vault(args[5])
except:
vault = None
if vault is None:
print "Vault named '{}' does not exist.".format(args[5])
else:
dynamo_connection = DynamoDBConnection(aws_access_key_id=config.get('configuration', 'aws_key'), aws_secret_access_key=config.get('configuration', 'aws_secret'))
archive_id_table = Table(config.get('configuration', 'dynamodb_table'), connection=dynamo_connection, schema=[HashKey('Account ID')])
count = 1
for archive in archive_id_table.scan():
time_str = datetime.datetime.fromtimestamp(archive['upload_timestamp']).strftime('%b %d %Y %X')
print "{}.\tFilename: {}\n\tTimestamp: {}\n\tArchive ID: {}".format(count, archive['filename'], time_str, archive['archive_id'])
count += 1
def upload_glacier(path,vault,dbCreds):
description = get_description(path,dbCreds,vault)
try:
glacier_connection = boto.connect_glacier()
vaultObj = glacier_connection.get_vault(vault)
start = time.time()
archiveID = vaultObj.upload_archive(path,description)
elapsed = time.time() - start
transferred = os.path.getsize(path)
Mbps = ((transferred/elapsed)*8)/(1024*1024)
except Exception, e:
if 'ThrottlingException' in str(e):
response = "ThrottlingException"
return (response)
else:
print("error on " + path + " " + str(e))
archiveID = 'FAILED'
Mbps = 0
def open_store(accesskey, sharedKey, host):
logger.debug('Connecting with creds %s : %s to Glacier' % (accesskey, sharedKey))
try:
conn = boto.connect_glacier(aws_access_key_id=accesskey,
aws_secret_access_key=sharedKey)
except UnexpectedHTTPResponseError as e:
logger.error('unable to connect to Glacier : HTTP %d - %s' %s (e.status, e.message))
sys.exit(2)
myvaultname = ('backup')
try:
vault = conn.get_vault(myvaultname)
except UnexpectedHTTPResponseError:
try:
logger.info('open_glacier: creating new vault %s' % myvaultname)
vault = conn.create_vault(myvaultname)
except UnexpectedHTTPResponseError as e:
logger.error('unable to create vault %s : HTTP %d - %s' % (myvaultname, e.status, e.message))
sys.exit(3)
return vault
def execute(self, args, config):
glacier_connection = boto.connect_glacier(aws_access_key_id=config.get('configuration', 'aws_key'), aws_secret_access_key=config.get('configuration', 'aws_secret'))
try:
vault = glacier_connection.get_vault(args[4])
except:
vault = None
if vault is None:
print "Vault named '{}' does not exist.".format(args[4])
else:
try:
vault.delete_archive(args[2])
dynamo_connection=DynamoDBConnection(aws_access_key_id=config.get('configuration', 'aws_key'), aws_secret_access_key=config.get('configuration', 'aws_secret'))
archive_table = Table(config.get('configuration', 'dynamodb_table'), connection=dynamo_connection)
archive_table.delete_item(archive_id=args[2])
print "Archive deleted: '{}'".format(args[2])
except UnexpectedHTTPResponseError as error:
print "Archive can not be deleted:\n\t {}".format(error)
def upload(self):
if not self.is_valid():
raise ValidationException(self.error_message)
# create the connection to AWS
connection = boto.connect_glacier(
aws_access_key_id=self.arguments.aws_access_key_id,
aws_secret_access_key=self.arguments.aws_secret_access_key,
region_name=self.arguments.region,
)
# get the target vault
vault = connection.get_vault(self.arguments.vault)
# set uploaded object id
self.upload_id = vault.create_archive_from_file(
file_obj=self.arguments.file,
description=self.arguments.description if self.arguments.description else None
)
# return the upload id
return self.upload_id
def __init__(self, conf, logger):
if conf is None:
try:
access_key = config.get("aws", "access_key")
secret_key = config.get("aws", "secret_key")
vault_name = config.get("aws", "glacier_vault")
except ConfigParser.NoOptionError:
app_logger.error("Configuration file not available.")
app_logger.info("Use 'bakthat configure' to create one.")
return
else:
access_key = conf.get("access_key")
secret_key = conf.get("secret_key")
vault_name = conf.get("vault")
con = boto.connect_glacier(aws_access_key_id=access_key,
aws_secret_access_key=secret_key)
self.conf = conf
self.vault = con.create_vault(vault_name)
self.logger = logger
self.backup_key = "bakthat_glacier_inventory"
self.container = "Glacier vault: {}".format(vault_name)
source.generate(archives)
return archives
def discover(meta, archives):
for archive in archives.archives:
if not meta.contains(archive.meta_hash()):
log.info("Adding archive meta (%s)" % archive)
meta.add(archive.paths)
if __name__ == "__main__":
top = boto.config.get('Ice', 'top')
password = boto.config.get('Ice', 'password')
bucket_name = boto.config.get('Ice', 'bucket')
vault_name = boto.config.get('Ice', 'vault')
s3 = meta.S3(password, bucket_name)
meta = meta.Meta(s3)
meta.read()
archives = get_archive_set(top)
discover(meta, archives)
glacier = boto.connect_glacier()
vault = glacier.create_vault(vault_name)
for archive in archives.archives:
log.info("Archive: %s" % archive)
meta_hash = archive.meta_hash()
sha1 = meta.get_sha1(meta_hash)
archiver.upload_archive(vault, sha1, archive.paths, upload=True, dryRun=True)
meta.write()
# EOF
def execute(self, args, config):
glacier_connection = boto.connect_glacier(aws_access_key_id=config.get('configuration', 'aws_key'), aws_secret_access_key=config.get('configuration', 'aws_secret'))
for index, vault in enumerate(glacier_connection.list_vaults()):
print "{}.\tname={}\n\tarn={}\n\tcreation_date={}\n\tlast_inventory_date={}\n\tsize={}\n\tnumber_of_archives={}".format(index+1, vault.name, vault.arn, vault.creation_date, vault.last_inventory_date, vault.size, vault.number_of_archives)
def __init__(self, vault_name):
con = boto.connect_glacier()
self.vault = con.create_vault(vault_name)
# [SECTION] # configitem = value # # It expects one section with the values: # [DEFAULT] # aws_access_key_id = <value> # aws_secret_access_key = <value> # aws_sns_topic_arn = <value> # aws_glacier_vault_id = <value> config = configparser.ConfigParser() ACCESS_KEY_ID = config['DEFAULT']['aws_access_key_id'] SECRET_ACCESS_KEY = config['DEFAULT']['aws_secret_access_key'] SNS_TOPIC = config['DEFAULT']['aws_sns_topic_arn'] VAULT_ID = config['DEFAULT']['aws_glacier_vault_id'] # Connection to Glacier glacier_connection = boto.connect_glacier(aws_access_key_id=ACCESS_KEY_ID, aws_secret_access_key=SECRET_ACCESS_KEY) # Connect to specific vault vault = glacier_connection.get_vault(VAULT_ID) # initiate a job to retrieve the vault inventory inventory_job = vault.retrieve_inventory(sns_topic=SNS_TOPIC) print(inventory_job)
print
print 'filename -> archive_id'
print '---------------------------'
for k, v in inventory.items():
print '%s -> %s' % (k, v)
def list_jobs(vault):
for j in vault.list_jobs():
print j.__dict__
if __name__ == "__main__":
inventory = shelve.open('inventory')
glacier_connection = boto.connect_glacier(aws_access_key_id=GLACIER['ACCESS_KEY_ID'],
aws_secret_access_key=GLACIER['SECRET_ACCESS_KEY'],
region_name='eu-west-1')
vault = glacier_connection.get_vault(GLACIER['VAULT'])
if len(sys.argv) < 2:
print 'usage: ul | dl | inv | jobs'
elif sys.argv[1] == 'jobs':
list_jobs(vault)
elif sys.argv[1] == 'ul':
upload(inventory, vault, sys.argv[2])
elif sys.argv[1] == 'dl':
download(inventory, vault, sys.argv[2])
elif sys.argv[1] == 'inv':
local_inventory(inventory)
else:
print 'usage: ul | dl | inv'
def list_vaults():
connection = boto.connect_glacier()
print('\n'.join([vault.name for vault in connection.list_vaults()]))
