def __init__(self, conf=None, extra_conf=[], section="aws"): self.custom_conf = None self.conf = {} if not conf: try: if section == "aws": self.conf["access_key"] = config.get(section, "access_key") self.conf["secret_key"] = config.get(section, "secret_key") self.conf["region_name"] = config.get(section, "region_name") for key in extra_conf: try: self.conf[key] = config.get(section, key) except Exception, exc: log.exception(exc) log.error("Missing configuration variable.") self.conf[key] = None except ConfigParser.NoOptionError: if section == "aws": log.error("Configuration file not available.") log.info("Use 'bakthat configure' to create one.") else: log.error("No {0} section available in configuration file.".format(section)) else: if section == "aws": self.conf["access_key"] = conf.get("access_key") self.conf["secret_key"] = conf.get("secret_key") self.conf["region_name"] = conf.get("region_name", DEFAULT_LOCATION) for key in extra_conf: self.conf[key] = conf.get(key)
def __init__(self, conf=None): conf = {} if conf is None else conf sync_conf = dict(url=config.get("sync", {}).get("url"), username=config.get("sync", {}).get("username"), password=config.get("sync", {}).get("password")) sync_conf.update(conf) self.sync_auth = (sync_conf["username"], sync_conf["password"]) self.api_url = sync_conf["url"] self.request_kwargs = dict(auth=self.sync_auth) self.request_kwargs["headers"] = {'content-type': 'application/json', 'bakthat-client': socket.gethostname()} self.get_resource = lambda x: self.api_url + "/{0}".format(x)
def configure(profile="default"): new_conf = config.copy() new_conf[profile] = config.get(profile, {}) new_conf_file = open(CONFIG_FILE, "w") new_conf[profile]["access_key"] = raw_input("AWS Access Key: ") new_conf[profile]["secret_key"] = raw_input("AWS Secret Key: ") new_conf[profile]["s3_bucket"] = raw_input("S3 Bucket Name: ") new_conf[profile]["glacier_vault"] = raw_input("Glacier Vault Name: ") while 1: default_destination = raw_input("Default destination ({0}): ".format(DEFAULT_DESTINATION)) if default_destination: default_destination = default_destination.lower() if default_destination in ("s3", "glacier"): break else: log.error("Invalid default_destination, should be s3 or glacier, try again.") else: default_destination = DEFAULT_DESTINATION break new_conf[profile]["default_destination"] = default_destination region_name = raw_input("Region Name ({0}): ".format(DEFAULT_LOCATION)) if not region_name: region_name = DEFAULT_LOCATION new_conf[profile]["region_name"] = region_name yaml.dump(new_conf, new_conf_file, default_flow_style=False) log.info("Config written in %s" % CONFIG_FILE) log.info("Run bakthat configure_backups_rotation if needed.")
def _get_database(): """Determine database in use and credentials if required.""" database = None conf = config.get('default') if conf.get("database_type"): database_type = conf.get("database_type") if database_type == "mysql": if conf.get("database_host") and conf.get("database_name") and conf.get("database_user") and conf.get("database_pass") and conf.get("database_port"): database_host = conf.get("database_host") database_name = conf.get("database_name") database_user = conf.get("database_user") database_pass = conf.get("database_pass") database_port = conf.get("database_port") database = peewee.MySQLDatabase(database_name, host=database_host, port=database_port, user=database_user, passwd=database_pass) else: log.error("You must specify all config options if using mysql database.") if database_type == 'sqlite': database = peewee.SqliteDatabase(DATABASE) else: log.info("Defaulting to using SQLITE.") database = peewee.SqliteDatabase(DATABASE) return database
def __init__(self, conf={}, profile="default"): self.conf = conf if not conf: self.conf = config.get(profile) if not self.conf: log.error("No {0} profile defined in {1}.".format(profile, CONFIG_FILE)) if not "access_key" in self.conf or not "secret_key" in self.conf: log.error("Missing access_key/secret_key in {0} profile ({1}).".format(profile, CONFIG_FILE))
def show_glacier_inventory(**kwargs): if config.get("aws", "s3_bucket"): conf = kwargs.get("conf", None) glacier_backend = GlacierBackend(conf) loaded_archives = glacier_backend.load_archives_from_s3() log.info(json.dumps(loaded_archives, sort_keys=True, indent=4, separators=(",", ": "))) else: log.error("No S3 bucket defined.") return loaded_archives
def __init__(self, conf=None): conf = {} if conf is None else conf sync_conf = dict(url=config.get("sync", {}).get("url"), username=config.get("sync", {}).get("username"), password=config.get("sync", {}).get("password")) sync_conf.update(conf) self.sync_auth = (sync_conf["username"], sync_conf["password"]) self.api_url = sync_conf["url"] self.request_kwargs = dict(auth=self.sync_auth) self.request_kwargs["headers"] = { 'content-type': 'application/json', 'bakthat-client': socket.gethostname() } self.get_resource = lambda x: self.api_url + "/{0}".format(x)
def configure(profile="default"): try: new_conf = config.copy() new_conf[profile] = config.get(profile, {}) while 1: database_type = raw_input("Database type: ") if database_type: database_type = database_type.lower() if database_type in ("mysql", "sqlite"): break else: log.error("Invalid database_type, should be mysql or sqlite, try again.") new_conf[profile]["database_type"] = database_type if database_type == "mysql": new_conf[profile]["database_host"] = raw_input("Database host: ") new_conf[profile]["database_name"] = raw_input("Database name: ") new_conf[profile]["database_user"] = raw_input("Database user: "******"database_pass"] = raw_input("Database pass: "******"database_port"] = raw_input("Database port: ") new_conf[profile]["access_key"] = raw_input("AWS Access Key: ") new_conf[profile]["secret_key"] = raw_input("AWS Secret Key: ") new_conf[profile]["s3_bucket"] = raw_input("S3 Bucket Name: ") new_conf[profile]["glacier_vault"] = raw_input("Glacier Vault Name: ") while 1: default_destination = raw_input("Default destination ({0}): ".format(DEFAULT_DESTINATION)) if default_destination: default_destination = default_destination.lower() if default_destination in ("s3", "glacier", "swift"): break else: log.error("Invalid default_destination, should be s3 or glacier, swift, try again.") else: default_destination = DEFAULT_DESTINATION break new_conf[profile]["default_destination"] = default_destination region_name = raw_input("Region Name ({0}): ".format(DEFAULT_LOCATION)) if not region_name: region_name = DEFAULT_LOCATION new_conf[profile]["region_name"] = region_name if default_destination in ("swift"): new_conf[profile]["auth_version"] = raw_input("Swift Auth Version: ") new_conf[profile]["auth_url"] = raw_input("Swift Auth URL: ") yaml.dump(new_conf, open(CONFIG_FILE, "w"), default_flow_style=False) log.info("Config written in %s" % CONFIG_FILE) log.info("Run bakthat configure_backups_rotation if needed.") except KeyboardInterrupt: log.error("Cancelled by user")
def backup_inventory(self): """Backup the local inventory from shelve as a json string to S3.""" if config.get("aws", "s3_bucket"): archives = self.load_archives() s3_bucket = S3Backend(self.conf).bucket k = Key(s3_bucket) k.key = self.backup_key k.set_contents_from_string(json.dumps(archives)) k.set_acl("private")
def restore_inventory(self): """Restore inventory from S3 to local shelve.""" if config.get("aws", "s3_bucket"): loaded_archives = self.load_archives_from_s3() with glacier_shelve() as d: archives = {} for a in loaded_archives: print a archives[a["filename"]] = a["archive_id"] d["archives"] = archives else: raise Exception("You must set s3_bucket in order to backup/restore inventory to/from S3.")
def restore_inventory(self): """Restore inventory from S3 to local shelve.""" if config.get("aws", "s3_bucket"): loaded_archives = self.load_archives_from_s3() with glacier_shelve() as d: if not d.has_key("archives"): d["archives"] = dict() archives = loaded_archives d["archives"] = archives else: raise Exception("You must set s3_bucket in order to backup/restore inventory to/from S3.")
def _get_profile_query(profile): if profile: profile_conf = config.get(profile) if profile_conf: s3_hash_key = hashlib.sha512(profile_conf.get("access_key") + \ profile_conf.get("s3_bucket")).hexdigest() glacier_hash_key = s3_hash_key = hashlib.sha512(profile_conf.get("access_key") + \ profile_conf.get("glacier_vault")).hexdigest() return " ((backend == 's3' AND backend_hash == '{0}') OR \ (backend == 'glacier' AND backend_hash == '{1}'))".format(s3_hash_key, glacier_hash_key) else: raise Exception("Profile {0} not found.".format(profile)) return ""
def match_filename(cls, filename, destination, **kwargs): profile = config.get(kwargs.get("profile", "default")) s3_key = hashlib.sha512(profile.get("access_key") + profile.get("s3_bucket")).hexdigest() glacier_key = hashlib.sha512(profile.get("access_key") + profile.get("glacier_vault")).hexdigest() try: fquery = "{0}*".format(filename) query = Backups.select().where(Backups.filename % fquery | Backups.stored_filename % fquery, Backups.backend == destination, Backups.backend_hash << [s3_key, glacier_key]) query = query.order_by(Backups.backup_date.desc()) return query.get() except Backups.DoesNotExist: return
def search(cls, query="", destination="", **kwargs): if not destination: destination = ["s3", "glacier"] if isinstance(destination, (str, unicode)): destination = [destination] profile = config.get(kwargs.get("profile", "default")) s3_key = hashlib.sha512(profile.get("access_key") + profile.get("s3_bucket")).hexdigest() glacier_key = hashlib.sha512(profile.get("access_key") + profile.get("glacier_vault")).hexdigest() query = "*{0}*".format(query) wheres = [] wheres.append(Backups.filename % query | Backups.stored_filename % query) wheres.append(Backups.backend << destination) wheres.append(Backups.backend_hash << [s3_key, glacier_key]) wheres.append(Backups.is_deleted == False) older_than = kwargs.get("older_than") if older_than: wheres.append(Backups.backup_date < older_than) backup_date = kwargs.get("backup_date") if backup_date: wheres.append(Backups.backup_date == backup_date) last_updated_gt = kwargs.get("last_updated_gt") if last_updated_gt: wheres.append(Backups.last_updated >= last_updated_gt) tags = kwargs.get("tags", []) if tags: if isinstance(tags, (str, unicode)): tags = tags.split() tags_query = ["Backups.tags % '*{0}*'".format(tag) for tag in tags] tags_query = eval("({0})".format(" and ".join(tags_query))) wheres.append(tags_query) return Backups.select().where(*wheres).order_by(Backups.last_updated.desc())
def configure(profile="default"): try: new_conf = config.copy() new_conf[profile] = config.get(profile, {}) new_conf[profile]["access_key"] = raw_input("AWS Access Key: ") new_conf[profile]["secret_key"] = raw_input("AWS Secret Key: ") new_conf[profile]["s3_bucket"] = raw_input("S3 Bucket Name: ") new_conf[profile]["glacier_vault"] = raw_input("Glacier Vault Name: ") while 1: default_destination = raw_input( "Default destination ({0}): ".format(DEFAULT_DESTINATION)) if default_destination: default_destination = default_destination.lower() if default_destination in ("s3", "glacier", "swift"): break else: log.error( "Invalid default_destination, should be s3 or glacier, swift, try again." ) else: default_destination = DEFAULT_DESTINATION break new_conf[profile]["default_destination"] = default_destination region_name = raw_input("Region Name ({0}): ".format(DEFAULT_LOCATION)) if not region_name: region_name = DEFAULT_LOCATION new_conf[profile]["region_name"] = region_name if default_destination in ("swift"): new_conf[profile]["auth_version"] = raw_input( "Swift Auth Version: ") new_conf[profile]["auth_url"] = raw_input("Swift Auth URL: ") yaml.dump(new_conf, open(CONFIG_FILE, "w"), default_flow_style=False) log.info("Config written in %s" % CONFIG_FILE) log.info("Run bakthat configure_backups_rotation if needed.") except KeyboardInterrupt: log.error("Cancelled by user")
def backup(filename=os.getcwd(), destination=None, prompt="yes", tags=[], profile="default", config=CONFIG_FILE, key=None, **kwargs): """Perform backup. :type filename: str :param filename: File/directory to backup. :type destination: str :param destination: s3|glacier|swift :type prompt: str :param prompt: Disable password promp, disable encryption, only useful when using bakthat in command line mode. :type tags: str or list :param tags: Tags either in a str space separated, either directly a list of str (if calling from Python). :type password: str :keyword password: Password, empty string to disable encryption. :type conf: dict :keyword conf: Override/set AWS configuration. :type custom_filename: str :keyword custom_filename: Override the original filename (only in metadata) :rtype: dict :return: A dict containing the following keys: stored_filename, size, metadata, backend and filename. """ storage_backend, destination, conf = _get_store_backend( config, destination, profile) backup_file_fmt = "{0}.{1}.tgz" # Check if compression is disabled on the configuration. if conf: compress = conf.get("compress", True) else: compress = config.get(profile).get("compress", True) if not compress: backup_file_fmt = "{0}.{1}" log.info("Backing up " + filename) arcname = filename.strip('/').split('/')[-1] now = datetime.utcnow() date_component = now.strftime("%Y%m%d%H%M%S") stored_filename = backup_file_fmt.format(arcname, date_component) backup_date = int(now.strftime("%s")) backup_data = dict(filename=kwargs.get("custom_filename", arcname), backup_date=backup_date, last_updated=backup_date, backend=destination, is_deleted=False) # Useful only when using bakmanager.io hook backup_key = key password = kwargs.get("password", os.environ.get("BAKTHAT_PASSWORD")) if password is None and prompt.lower() != "no": password = getpass("Password (blank to disable encryption): ") if password: password2 = getpass("Password confirmation: ") if password != password2: log.error("Password confirmation doesn't match") return if not compress: log.info("Compression disabled") outname = filename with open(outname) as outfile: backup_data["size"] = os.fstat(outfile.fileno()).st_size bakthat_compression = False # Check if the file is not already compressed elif mimetypes.guess_type(arcname) == ('application/x-tar', 'gzip'): log.info("File already compressed") outname = filename # removing extension to reformat filename new_arcname = re.sub(r'(\.t(ar\.)?gz)', '', arcname) stored_filename = backup_file_fmt.format(new_arcname, date_component) with open(outname) as outfile: backup_data["size"] = os.fstat(outfile.fileno()).st_size bakthat_compression = False else: # If not we compress it log.info("Compressing...") with tempfile.NamedTemporaryFile(delete=False) as out: with closing(tarfile.open(fileobj=out, mode="w:gz")) as tar: tar.add(filename, arcname=arcname) outname = out.name out.seek(0) backup_data["size"] = os.fstat(out.fileno()).st_size bakthat_compression = True bakthat_encryption = False if password: bakthat_encryption = True log.info("Encrypting...") encrypted_out = tempfile.NamedTemporaryFile(delete=False) encrypt_file(outname, encrypted_out.name, password) stored_filename += ".enc" # We only remove the file if the archive is created by bakthat if bakthat_compression: os.remove(outname) # remove non-encrypted tmp file outname = encrypted_out.name encrypted_out.seek(0) backup_data["size"] = os.fstat(encrypted_out.fileno()).st_size # Handling tags metadata if isinstance(tags, list): tags = " ".join(tags) backup_data["tags"] = tags backup_data["metadata"] = dict(is_enc=bakthat_encryption, client=socket.gethostname()) backup_data["stored_filename"] = stored_filename access_key = storage_backend.conf.get("access_key") container_key = storage_backend.conf.get(storage_backend.container_key) backup_data["backend_hash"] = hashlib.sha512(access_key + container_key).hexdigest() log.info("Uploading...") storage_backend.upload(stored_filename, outname) # We only remove the file if the archive is created by bakthat if bakthat_compression or bakthat_encryption: os.remove(outname) log.debug(backup_data) # Insert backup metadata in SQLite Backups.create(**backup_data) BakSyncer(conf).sync_auto() # bakmanager.io hook, enable with -k/--key paramter if backup_key: bakmanager_hook(conf, backup_data, backup_key) return backup_data
def backup(filename=os.getcwd(), destination=None, profile="default", config=CONFIG_FILE, prompt="yes", tags=[], key=None, exclude_file=None, s3_reduced_redundancy=False, **kwargs): """Perform backup. :type filename: str :param filename: File/directory to backup. :type destination: str :param destination: s3|glacier|swift :type prompt: str :param prompt: Disable password promp, disable encryption, only useful when using bakthat in command line mode. :type tags: str or list :param tags: Tags either in a str space separated, either directly a list of str (if calling from Python). :type password: str :keyword password: Password, empty string to disable encryption. :type conf: dict :keyword conf: Override/set AWS configuration. :type custom_filename: str :keyword custom_filename: Override the original filename (only in metadata) :rtype: dict :return: A dict containing the following keys: stored_filename, size, metadata, backend and filename. """ storage_backend, destination, conf = _get_store_backend(config, destination, profile) backup_file_fmt = "{0}.{1}.tgz" session_id = str(uuid.uuid4()) events.before_backup(session_id) # Check if compression is disabled on the configuration. if conf: compress = conf.get("compress", True) else: compress = config.get(profile).get("compress", True) if not compress: backup_file_fmt = "{0}.{1}" log.info("Backing up " + filename) if exclude_file and os.path.isfile(exclude_file): EXCLUDE_FILES.insert(0, exclude_file) _exclude = lambda filename: False if os.path.isdir(filename): join = functools.partial(os.path.join, filename) for efile in EXCLUDE_FILES: efile = join(efile) if os.path.isfile(efile): _exclude = _get_exclude(efile) log.info("Using {0} to exclude files.".format(efile)) arcname = filename.strip('/').split('/')[-1] now = datetime.utcnow() date_component = now.strftime("%Y%m%d%H%M%S") stored_filename = backup_file_fmt.format(arcname, date_component) backup_date = int(now.strftime("%s")) backup_data = dict(filename=kwargs.get("custom_filename", arcname), backup_date=backup_date, last_updated=backup_date, backend=destination, is_deleted=False) # Useful only when using bakmanager.io hook backup_key = key password = kwargs.get("password", os.environ.get("BAKTHAT_PASSWORD")) if password is None and prompt.lower() != "no": password = getpass("Password (blank to disable encryption): ") if password: password2 = getpass("Password confirmation: ") if password != password2: log.error("Password confirmation doesn't match") return if not compress: log.info("Compression disabled") outname = filename with open(outname) as outfile: backup_data["size"] = os.fstat(outfile.fileno()).st_size bakthat_compression = False # Check if the file is not already compressed elif mimetypes.guess_type(arcname) == ('application/x-tar', 'gzip'): log.info("File already compressed") outname = filename # removing extension to reformat filename new_arcname = re.sub(r'(\.t(ar\.)?gz)', '', arcname) stored_filename = backup_file_fmt.format(new_arcname, date_component) with open(outname) as outfile: backup_data["size"] = os.fstat(outfile.fileno()).st_size bakthat_compression = False else: # If not we compress it log.info("Compressing...") with tempfile.NamedTemporaryFile(delete=False) as out: with closing(tarfile.open(fileobj=out, mode="w:gz")) as tar: tar.add(filename, arcname=arcname, exclude=_exclude) outname = out.name out.seek(0) backup_data["size"] = os.fstat(out.fileno()).st_size bakthat_compression = True bakthat_encryption = False if password: bakthat_encryption = True log.info("Encrypting...") encrypted_out = tempfile.NamedTemporaryFile(delete=False) encrypt_file(outname, encrypted_out.name, password) stored_filename += ".enc" # We only remove the file if the archive is created by bakthat if bakthat_compression: os.remove(outname) # remove non-encrypted tmp file outname = encrypted_out.name encrypted_out.seek(0) backup_data["size"] = os.fstat(encrypted_out.fileno()).st_size # Handling tags metadata if isinstance(tags, list): tags = " ".join(tags) backup_data["tags"] = tags backup_data["metadata"] = dict(is_enc=bakthat_encryption, client=socket.gethostname()) stored_filename = os.path.join(os.path.dirname(kwargs.get("custom_filename", "")), stored_filename) backup_data["stored_filename"] = stored_filename access_key = storage_backend.conf.get("access_key") container_key = storage_backend.conf.get(storage_backend.container_key) backup_data["backend_hash"] = hashlib.sha512(access_key + container_key).hexdigest() log.info("Uploading...") storage_backend.upload(stored_filename, outname, s3_reduced_redundancy=s3_reduced_redundancy) # We only remove the file if the archive is created by bakthat if bakthat_compression or bakthat_encryption: os.remove(outname) log.debug(backup_data) # Insert backup metadata in SQLite backup = Backups.create(**backup_data) BakSyncer(conf).sync_auto() # bakmanager.io hook, enable with -k/--key paramter if backup_key: bakmanager_hook(conf, backup_data, backup_key) events.on_backup(session_id, backup) return backup
def sync_auto(self): """Trigger sync if autosync is enabled.""" if config.get("sync", {}).get("auto", False): self.sync()
def _get_store_backend(conf, destination=DEFAULT_DESTINATION, profile="default"): if not destination: destination = config.get("aws", "default_destination") return STORAGE_BACKEND[destination](conf, profile)