def backup_meta(config_path):
'''
Makes tar out of database directory + config, encrypts it and copies
to dropbox folder as 'meta' file
'''
logger = logging.getLogger('mylog')
config = SafeConfigParser()
config.read(config_path)
db_path = config.get('DB', 'db_path')
password = config.get('Credentials', 'password')
logger.info('Backing up database ' + db_path)
temp_dir = tempfile.gettempdir()
archive = temp_dir + '/' + META_DB_NAME
with tarfile.open(archive, "w:") as tar:
tar.add(db_path, arcname=path_leaf(db_path))
tar.add(config_path, arcname=path_leaf(config_path))
tar.close()
enc_file_name = encrypt_file(password, archive)
#Remove unencrypted file
os.remove(archive)
#Move encrypted meta file to dropboxdir
dropboxdir = config.get('DropBox', 'drop_box_dir')
if os.path.isfile(os.path.join(dropboxdir, '') +
path_leaf(enc_file_name)):
os.remove(os.path.join(dropboxdir, '') + path_leaf(enc_file_name))
shutil.move(enc_file_name, dropboxdir)
logger.info('Database was backed up.')
def test4():
'''
Initally dropbox folder is empty and database is empty.
Check if deleted source folder (which contains files) causes deletion
of this files from dropbox folder.
'''
print '-' * 50
print "Test 4"
clear_all()
create_test_config()
config = SafeConfigParser()
config.read(TEST_PATH + TEST_CONFIG_NAME)
directory_to_backup = config.get('Backup', 'to_backup') + '/some_dir'
os.makedirs(directory_to_backup)
file_to_backup = config.get('Backup', 'to_backup') + '/some_dir/some_file1'
with open(file_to_backup, 'w') as some_file:
some_file.write('Very first line')
file_to_backup = config.get('Backup', 'to_backup') + '/some_dir/some_file2'
with open(file_to_backup, 'w') as some_file:
some_file.write('The other very first line')
#Backup
os.system(BACKUP_TOOL + "backup -c" + TEST_PATH + "/" + TEST_CONFIG_NAME)
#Delete
shutil.rmtree(directory_to_backup)
#Backup again
os.system(BACKUP_TOOL + "backup -c" + TEST_PATH + "/" + TEST_CONFIG_NAME)
#Check if file doesn't exist
for subdir, dirs, files in os.walk(config.get('DropBox', 'drop_box_dir')):
for f in files:
#Meta entry is not in database
if path_leaf(f) == backup.META_DB_NAME + ".xyz":
continue
else:
print "Test 4 failed - deleted file exists in dropbox folder"
sys.exit(1)
#Database shall not contain any entry
entries = 0
db = leveldb.LevelDB(config.get('DB', 'db_path'))
dump_file = os.path.join(tempfile.gettempdir(), '') + 'foo'
entries = dump_database(db, dump_file)
if entries != 0:
print "Test 4 failed - deleted file has entry in database"
sys.exit(1)
print "Test 4 ok"
def test4():
'''
Initally dropbox folder is empty and database is empty.
Check if deleted source folder (which contains files) causes deletion
of this files from dropbox folder.
'''
print '-'*50
print "Test 4"
clear_all()
create_test_config()
config = SafeConfigParser()
config.read(TEST_PATH+TEST_CONFIG_NAME)
directory_to_backup = config.get('Backup', 'to_backup')+'/some_dir'
os.makedirs(directory_to_backup)
file_to_backup = config.get('Backup', 'to_backup')+'/some_dir/some_file1'
with open(file_to_backup, 'w') as some_file:
some_file.write('Very first line')
file_to_backup = config.get('Backup', 'to_backup')+'/some_dir/some_file2'
with open(file_to_backup, 'w') as some_file:
some_file.write('The other very first line')
#Backup
os.system(BACKUP_TOOL + "backup -c" + TEST_PATH + "/" + TEST_CONFIG_NAME)
#Delete
shutil.rmtree(directory_to_backup)
#Backup again
os.system(BACKUP_TOOL + "backup -c" + TEST_PATH + "/" + TEST_CONFIG_NAME)
#Check if file doesn't exist
for subdir, dirs, files in os.walk(config.get('DropBox','drop_box_dir')):
for f in files:
#Meta entry is not in database
if path_leaf(f) == backup.META_DB_NAME + ".xyz":
continue
else:
print "Test 4 failed - deleted file exists in dropbox folder"
sys.exit(1)
#Database shall not contain any entry
entries = 0
db = leveldb.LevelDB(config.get('DB', 'db_path'))
dump_file = os.path.join(tempfile.gettempdir(), '') + 'foo'
entries = dump_database(db, dump_file)
if entries != 0:
print "Test 4 failed - deleted file has entry in database"
sys.exit(1)
print "Test 4 ok"
def consistency_check(config, database, dry_run=False):
"""
DB->dropbox
Consistency is when the db values (encrypted hash and encrypted file name )
matches the dropbox file hash and name.
dropbox->DB
If there is no entry in db for fropbox file then it's incosistency and must
be reported. It cannot be fixes automatically.
I don't remove any file from dropbox. In some cases I just report problem
which has to be manually fixed.
"""
logger = logging.getLogger("mylog")
dropbox_folder = config.get('DropBox', 'drop_box_dir')
batch = leveldb.WriteBatch()
#DB->dropbox
#Little bit more verbose to report cases.
dropbox_names = set()
result = True
for key, value in database.RangeIter():
(_, encrypted_hash, dropbox_file) = \
value.split(FileProcessor.HASH_SEPARATOR)
dropbox_names.add(dropbox_file)
dropbox_file = os.path.join(dropbox_folder, '') + dropbox_file
if not os.path.isfile(key):
if not os.path.isfile(dropbox_file):
result = False
logger.warning('File ' + dropbox_file + ' orginally ' + key +
'does not exist in dropbox dir.')
if not dry_run:
logger.warning('Removing entry from db.')
batch.Delete(key)
else:
#Only for informational purposes
hsh = compute_hash(dropbox_file)
if hsh != encrypted_hash:
result = False
logger.warning('File ' + dropbox_file + ' orginally ' +
key + 'has hash that does not match db entry.')
else:
if not os.path.isfile(dropbox_file):
result = False
logger.warning('File ' + dropbox_file + ' orginally ' + key +
' does not exist in dropbox dir.')
if not dry_run:
logger.warning('Removing entry from db.')
batch.Delete(key)
else:
#Only for informational purposes
hsh = compute_hash(dropbox_file)
if hsh != encrypted_hash:
result = False
logger.warning('File ' + dropbox_file + ' orginally ' +
key + 'has hash that does not match db entry.')
if not dry_run:
logger.warning('Removing entry from db and file ' +
'from dropbox')
batch.Delete(key)
os.remove(dropbox_file)
database.Write(batch, sync=True)
#dropbox->DB
for _, _, files in os.walk(dropbox_folder):
for single_file in files:
#Meta entry is not in database
if path_leaf(single_file) == backup.META_DB_NAME + ".xyz":
continue
if single_file not in dropbox_names:
result = False
logger.warning('File ' + single_file +
' from dropbox does not have db entry !')
return result
def consistency_check(config, database, dry_run=False):
"""
DB->dropbox
Consistency is when the db values (encrypted hash and encrypted file name )
matches the dropbox file hash and name.
dropbox->DB
If there is no entry in db for fropbox file then it's incosistency and must
be reported. It cannot be fixes automatically.
I don't remove any file from dropbox. In some cases I just report problem
which has to be manually fixed.
"""
logger = logging.getLogger("mylog")
dropbox_folder = config.get('DropBox', 'drop_box_dir')
batch = leveldb.WriteBatch()
#DB->dropbox
#Little bit more verbose to report cases.
dropbox_names = set()
result = True
for key, value in database.RangeIter():
(_, encrypted_hash, dropbox_file) = \
value.split(FileProcessor.HASH_SEPARATOR)
dropbox_names.add(dropbox_file)
dropbox_file = os.path.join(dropbox_folder, '') + dropbox_file
if not os.path.isfile(key):
if not os.path.isfile(dropbox_file):
result = False
logger.warning('File ' + dropbox_file + ' orginally ' + key +
'does not exist in dropbox dir.')
if not dry_run:
logger.warning('Removing entry from db.')
batch.Delete(key)
else:
#Only for informational purposes
hsh = compute_hash(dropbox_file)
if hsh != encrypted_hash:
result = False
logger.warning('File ' + dropbox_file + ' orginally ' +
key +
'has hash that does not match db entry.')
else:
if not os.path.isfile(dropbox_file):
result = False
logger.warning('File ' + dropbox_file + ' orginally ' + key +
' does not exist in dropbox dir.')
if not dry_run:
logger.warning('Removing entry from db.')
batch.Delete(key)
else:
#Only for informational purposes
hsh = compute_hash(dropbox_file)
if hsh != encrypted_hash:
result = False
logger.warning('File ' + dropbox_file + ' orginally ' +
key +
'has hash that does not match db entry.')
if not dry_run:
logger.warning('Removing entry from db and file ' +
'from dropbox')
batch.Delete(key)
os.remove(dropbox_file)
database.Write(batch, sync=True)
#dropbox->DB
for _, _, files in os.walk(dropbox_folder):
for single_file in files:
#Meta entry is not in database
if path_leaf(single_file) == backup.META_DB_NAME + ".xyz":
continue
if single_file not in dropbox_names:
result = False
logger.warning('File ' + single_file +
' from dropbox does not have db entry !')
return result
def restore(dropbox_path, destiantion, override):
'''
Restoring module
dropbox_path: direcotry of dropbox_path
destiantion: directory where database shall be put
'''
logger = logging.getLogger('mylog')
db_in_dropbox = os.path.join(dropbox_path, '') + backup.META_DB_NAME + \
'.xyz'
db_destination = os.path.join(destiantion, '') + backup.META_DB_NAME + \
'.xyz'
logger.info('DB restoring from ' + db_in_dropbox + ' to ' + db_destination)
logger.debug('Copying ' + db_in_dropbox + ' to ' + db_destination)
shutil.copy2(db_in_dropbox, db_destination)
password = getpass.getpass()
decrypt_file(password, db_destination)
tar = tarfile.open(os.path.join(destiantion, '') + backup.META_DB_NAME)
tar.extractall(path=os.path.join(destiantion, ''))
tar.close()
#From extracted config file get the name of database directory
config = SafeConfigParser()
config.read(os.path.join(destiantion, '') + "config")
db_name = path_leaf(config.get('DB', 'db_path'))
logger.debug('DB name extracted from config: ' + db_name)
#Now we can read database
database = leveldb.LevelDB(os.path.join(destiantion, '') + db_name)
for key, value in database.RangeIter():
(_, _, dropbox_file) = value.split(FileProcessor.HASH_SEPARATOR)
#If dropbox file does not exist skip it
dropbox_file_with_path = os.path.join(dropbox_path, '') + dropbox_file
if not os.path.isfile(dropbox_file_with_path):
logger.warning('File ' + dropbox_file_with_path +
' does not exist. Skipping.')
continue
#If destination file exists skip it
if os.path.isfile(key):
logger.info(key + ' already exists. Skiping.')
continue
logger.info('Extracting ' + key)
dest_dir = ntpath.dirname(key)
#Append topmost directory
if override:
override = os.path.join(override, '')
dest_dir = '/' + override + dest_dir
logging.debug('Overridden path: ' + dest_dir)
dest_file_name = path_leaf(key)
logger.debug('Dir: ' + dest_dir + ', file name: ' + dest_file_name)
#Create destination directory if not present
if not os.path.exists(dest_dir):
logger.info(dest_dir + ' does not exitst. Creating.')
os.makedirs(dest_dir)
#Decrypt file
decrypt_file(password, dropbox_file_with_path,
os.path.join(dest_dir, '') + dest_file_name)