def delete_job(self, filename):
"""Delete the job entry for the filename.
:type filename: str
:param filename: Stored filename.
"""
dump_truck.execute("DELETE FROM jobs WHERE filename == '{0}'".format(filename))
def dump_truck_delete_backup(stored_filename):
"""Change status of is_deleted to 1 for the given stored_filename.
:type stored_filename: str
:param stored_filename: Filename
"""
dump_truck.execute("UPDATE backups SET is_deleted = 1, last_updated= {0} \
WHERE stored_filename == '{1}'".format(int(datetime.utcnow().strftime("%s")),
stored_filename))
def dump_truck_backups_generator(per=50):
"""Generator for iterating user stored backups.
:type per: int
:param per: Number of backups in each DumpTruck (SQLite) request
"""
cnt = dump_truck.execute("SELECT COUNT(*) FROM backups")[0]["COUNT(*)"]
for i in range(int(math.ceil(cnt / float(per)))):
limit = (i * per, (i + 1) * per)
#for result in dump_truck.execute("SELECT * FROM backups LIMIT {0:d}, {1:d}".format(*limit)):
# yield result
yield dump_truck.execute("SELECT * FROM backups LIMIT {0:d}, {1:d}".format(*limit))
def delete(self, keyname):
archive_id = self.get_archive_id(keyname)
if archive_id:
self.vault.delete_archive(archive_id)
with glacier_shelve() as d:
archives = d["archives"]
if keyname in archives:
del archives[keyname]
d["archives"] = archives
dump_truck.execute("DELETE FROM inventory WHERE filename == '{0}'".format(keyname))
self.backup_inventory()
def dump_truck_get_backup(filename, destination="", profile="default"):
query = _get_query(destination=destination, stored_filename=filename)
profile_query = _get_profile_query(profile)
query = "SELECT stored_filename FROM backups \
WHERE {0} AND {1} ORDER BY backup_date DESC LIMIT 0, 1".format(query, profile_query)
print query
backups = dump_truck.execute(query)
if backups:
return backups[0]
def get_job_id(self, filename):
"""Get the job_id corresponding to the filename.
:type filename: str
:param filename: Stored filename.
"""
res = dump_truck.execute('SELECT job_id FROM jobs WHERE filename LIKE "{0}%"'.format(filename))
if res:
return res[0].get("job_id")
def sync(self):
"""Draft for implementing bakthat clients (hosts) backups data synchronization.
Synchronize Bakthat sqlite database via a HTTP POST request.
Backups are never really deleted from sqlite database, we just update the is_deleted key.
It sends the last server sync timestamp along with data updated since last sync.
Then the server return backups that have been updated on the server since last sync.
Both side (bakthat and the sync server) make upserts of the latest data avaible:
- if it doesn't exist yet, it will be created.
- if it has been modified (e.g deleted, since it's the only action we can take) we update it.
"""
log.debug("Start syncing")
self.register()
last_sync_ts = dump_truck.get_var("sync_ts")
to_insert_in_mongo = dump_truck.execute("SELECT * FROM backups WHERE last_updated > {0:d}".format(last_sync_ts))
data = dict(sync_ts=last_sync_ts, to_insert_in_mongo=to_insert_in_mongo)
r_kwargs = self.request_kwargs.copy()
log.debug("Initial payload: {0}".format(data))
r_kwargs.update({"data": json.dumps(data)})
r = requests.post(self.get_resource("backups/sync/status"), **r_kwargs)
if r.status_code != 200:
log.error("An error occured during sync: {0}".format(r.text))
return
log.debug("Sync result: {0}".format(r.json()))
to_insert_in_bakthat = r.json().get("to_insert_in_bakthat")
sync_ts = r.json().get("sync_ts")
for newbackup in to_insert_in_bakthat:
log.debug("Upsert {0}".format(newbackup))
dump_truck_upsert_backup(newbackup)
dump_truck.save_var("sync_ts", sync_ts)
log.debug("Sync succcesful")
def delete_older_than(filename, interval, destination=DEFAULT_DESTINATION, profile="default", **kwargs):
"""Delete backups matching the given filename older than the given interval string.
:type filename: str
:param filename: File/directory name.
:type interval: str
:param interval: Interval string like 1M, 1W, 1M3W4h2s...
(s => seconds, m => minutes, h => hours, D => days, W => weeks, M => months, Y => Years).
:type destination: str
:param destination: glacier|s3
:type conf: dict
:keyword conf: Override/set AWS configuration.
:rtype: list
:return: A list containing the deleted keys (S3) or archives (Glacier).
"""
conf = kwargs.get("conf")
storage_backend = _get_store_backend(conf, destination, profile)
interval_seconds = _interval_string_to_seconds(interval)
deleted = []
backup_date_filter = int(datetime.utcnow().strftime("%s")) - interval_seconds
query = "SELECT stored_filename FROM backups WHERE backend == '{0}' \
AND backup_date < {1:d} AND filename LIKE '{2}%' AND is_deleted == 0"
for backup in dump_truck.execute(query.format(destination, backup_date_filter, filename)):
real_key = backup.get("stored_filename")
log.info("Deleting {0}".format(real_key))
storage_backend.delete(real_key)
dump_truck_delete_backup(real_key)
deleted.append(real_key)
return deleted
def show(query="", destination=DEFAULT_DESTINATION, tags=[], profile="", help="Profile, blank to show all"):
query = _get_query(tags=tags, destination=destination, query=query, profile=profile)
if query:
query = "WHERE " + query
backups = dump_truck.execute("SELECT * FROM backups {0} ORDER BY last_updated DESC".format(query))
_display_backups(backups)
def rotate_backups(filename, destination=DEFAULT_DESTINATION, profile="default", **kwargs):
"""Rotate backup using grandfather-father-son rotation scheme.
:type filename: str
:param filename: File/directory name.
:type destination: str
:param destination: s3|glacier
:type conf: dict
:keyword conf: Override/set AWS configuration.
:type days: int
:keyword days: Number of days to keep.
:type weeks: int
:keyword weeks: Number of weeks to keep.
:type months: int
:keyword months: Number of months to keep.
:type first_week_day: str
:keyword first_week_day: First week day (to calculate wich weekly backup keep, saturday by default).
:rtype: list
:return: A list containing the deleted keys (S3) or archives (Glacier).
"""
conf = kwargs.get("conf", None)
storage_backend = _get_store_backend(conf, destination, profile)
rotate = RotationConfig(conf, profile)
if not rotate:
raise Exception("You must run bakthat configure_backups_rotation or provide rotation configuration.")
deleted = []
query = "SELECT backup_date FROM backups WHERE backend == '{0}' \
AND filename LIKE '{1}%' AND is_deleted == 0".format(
destination, filename
)
backups = dump_truck.execute(query)
backups_date = [datetime.fromtimestamp(float(backup["backup_date"])) for backup in backups]
to_delete = grandfatherson.to_delete(
backups_date,
days=int(rotate.conf["days"]),
weeks=int(rotate.conf["weeks"]),
months=int(rotate.conf["months"]),
firstweekday=int(rotate.conf["first_week_day"]),
now=datetime.utcnow(),
)
for delete_date in to_delete:
backup_date = int(delete_date.strftime("%s"))
query = "SELECT stored_filename FROM backups WHERE backend == '{0}' \
AND filename LIKE '{1}%' AND backup_date == {2:d} \
AND is_deleted == 0".format(
destination, filename, backup_date
)
backups = dump_truck.execute(query)
if backups:
real_key = backups[0].get("stored_filename")
log.info("Deleting {0}".format(real_key))
storage_backend.delete(real_key)
dump_truck_delete_backup(real_key)
deleted.append(real_key)
return deleted
def get_archive_id(self, filename):
"""Get the archive_id corresponding to the filename."""
res = dump_truck.execute('SELECT archive_id FROM inventory WHERE filename == "{0}"'.format(filename))
if res:
return res[0].get("archive_id")
