def setUp(self):
self.fs = MagicMock(spec=Filesystem)
self.vfs = MagicMock(spec=VirtualFS)
self.fs.exists.return_value = False
self.vfs.exists.return_value = False
self.path_mocked = Path(self.fs, self.vfs)
self.dummyfs = DummyFS({'c1', 'c2', 'c3'})
self.dummyvfs = DummyVirtualFS(example_paths)
self.path = Path(self.dummyfs, self.dummyvfs)
def test_argument_path(self):
with self.subTest('single path argument'):
path = Path('/hello/<name>')
match = path.match('/hello/bob')
self.assertIsNotNone(match)
args = match.groupdict()
self.assertTrue('name' in args)
self.assertEqual('bob', args['name'])
match = path.match('/hello/bob/anderson')
self.assertIsNotNone(match)
args = match.groupdict()
self.assertTrue('name' in args)
self.assertEqual('bob', args['name'])
match = path.match('/bob')
self.assertIsNone(match)
with self.subTest('multiple path arguments'):
path = Path('/<name>/<foo>/<bar>')
match = path.match('/bob/foo/bar')
self.assertIsNotNone(match)
args = match.groupdict()
self.assertTrue('name' in args)
self.assertTrue('foo' in args)
self.assertTrue('bar' in args)
self.assertEqual(args['name'], 'bob')
self.assertEqual(args['foo'], 'foo')
self.assertEqual(args['bar'], 'bar')
match = path.match('/bob')
self.assertIsNone(match)
def __init__(self, site_info, backup_or_files):
self.backup = False
self.files = False
if backup_or_files == 'backup':
self.backup = True
if site_info.is_postgres:
file_type = 'postgres'
elif site_info.is_mysql:
file_type = 'mysql'
else:
abort(
"Sorry, this site is not using a 'postgres' database. "
"I don't know how to restore it (yet)."
)
elif backup_or_files == 'files':
self.files = True
file_type = 'files'
else:
abort(
"Only 'backup' and 'files' are valid operations for duplicity "
"commands (not '{}')".format(backup_or_files)
)
self.backup_or_files = backup_or_files
self.path = Path(site_info.domain, file_type)
self.site_info = site_info
def test_nested_tree(self):
def func(req, res):
pass
tree = RouteTree()
path = Path('path/')
tree[path] = func
sub = RouteTree()
routerpath = Path('sub/')
tree[routerpath] = sub
path = Path('path/')
sub[path] = func
tree.build_path(Path('/'))
print(tree)
def test_relative(self):
path = '/image/rainbow'
rule = Path('/image')
relative = rule.mk_rel(path)
self.assertEqual(relative, '/rainbow/')
path = '/image'
rule = Path('/image')
relative = rule.mk_rel(path)
self.assertEqual(relative, '/')
path = '/'
rule = Path('/')
relative = rule.mk_rel(path)
self.assertEqual(relative, '/')
def files(self):
"""
the files you wanted. sequence.
:return: list
"""
if self.ext == ".files":
_files = Path(self.location).children()
else:
_files = glob.glob("%s/*%s" % (self.location, self.ext))
_files = [f.replace("\\", "/") for f in _files]
return _files
def backup_php_site(server_name, site_name):
"""
legalsecretaryjournal_com
TODO Only really need to backup everything in the 'images' and 'sites'
folder.
"""
site_info = SiteInfo(server_name, site_name)
backup_path = site_info.backup().get('path')
print(green("Backup files on '{}'").format(env.host_string))
path = Path(site_name, 'files')
print(yellow(path.remote_folder()))
run('mkdir -p {0}'.format(path.remote_folder()))
# remove '.gz' from end of tar file
# tar_file = os.path.splitext(path.remote_file())[0]
# with cd('/home/legalsec/legalsecretaryjournal.com/'):
# first = True
# for folder in path.php_folders():
# if first:
# first = False
# run('tar cvf {} {}'.format(tar_file, folder))
# else:
# run('tar rvf {} {}'.format(tar_file, folder))
# run('gzip {}'.format(tar_file))
# # list the contents of the archive
# run('tar ztvf {}'.format(path.remote_file()))
with cd(backup_path):
run('tar -cvzf {} .'.format(path.remote_file()))
get(path.remote_file(), path.local_file())
def backup_ftp():
if not env.site_info.is_ftp():
abort("'{}' is not set-up for 'ftp'".format(env.site_info.site_name))
print(green("Backup FTP files on '{}'").format(env.host_string))
path = Path(env.site_info.site_name, 'ftp')
run('mkdir -p {0}'.format(path.remote_folder()))
with cd(path.ftp_folder(env.site_info.site_name)):
run('tar -cvzf {} .'.format(path.remote_file()))
get(path.remote_file(), path.local_file())
def backup_files():
"""
To backup the files 'rs.connexionsw' server:
fab -H rs.web.connexionsw backup_files
"""
print(green("Backup files on '{}'").format(env.host_string))
name = env.host_string.replace('.', '_')
name = name.replace('-', '_')
path = Path(name, 'files')
run('mkdir -p {0}'.format(path.remote_folder()))
with cd(path.files_folder()), hide('stdout'):
run('tar -czf {} .'.format(path.remote_file()))
get(path.remote_file(), path.local_file())
def test_absolute(self):
rootpath = Path('/root/')
subpath = Path('/sub/')
subpath.abs_to(rootpath)
self.assertIsNotNone(subpath.match('/root/sub'))
self.assertIsNotNone(subpath.match('/root/sub/abc'))
self.assertIsNone(subpath.match('/root'))
rootpath = Path('/')
subpath = Path('%')
subpath.abs_to(rootpath)
self.assertIsNotNone(subpath.match('/'))
self.assertIsNotNone(subpath.match('/abc'))
self.assertIsNotNone(subpath.match('/abc/def'))
def _create_endpoint(self, handler, rule, methods):
path = Path(rule, endpoint=True)
self.tree[path] = Middleware(handler, methods)
class Duplicity(object):
def __init__(self, site_info, backup_or_files):
self.backup = False
self.files = False
if backup_or_files == 'backup':
self.backup = True
if site_info.is_postgres:
file_type = 'postgres'
elif site_info.is_mysql:
file_type = 'mysql'
else:
abort(
"Sorry, this site is not using a 'postgres' database. "
"I don't know how to restore it (yet)."
)
elif backup_or_files == 'files':
self.files = True
file_type = 'files'
else:
abort(
"Only 'backup' and 'files' are valid operations for duplicity "
"commands (not '{}')".format(backup_or_files)
)
self.backup_or_files = backup_or_files
self.path = Path(site_info.domain, file_type)
self.site_info = site_info
def _display_backup_not_restored(self, restore_to, sql_file):
print
count = 0
files = file_paths(filtered_walk(restore_to))
for item in files:
if sql_file == item:
pass
else:
count = count + 1
print('{}. {}'.format(count, item))
if count:
print(yellow(
"The {} files listed above were not restored "
"(just so you know).".format(count)
))
print
def _display_files_not_restored(self, restore_to):
print
count = 0
files = file_paths(filtered_walk(restore_to))
for item in files:
count = count + 1
print('{}. {}'.format(count, item))
if count:
print(yellow(
"The {} files listed above were not restored "
"(just so you know).".format(count)
))
print
def _find_sql(self, restore_to):
result = None
found = None
match = glob.glob('{}/*.sql'.format(restore_to))
for item in match:
print('found: {}'.format(os.path.basename(item)))
file_name, extension = os.path.splitext(os.path.basename(item))
file_date_time = datetime.strptime(file_name, '%Y%m%d_%H%M')
if not found or file_date_time > found:
found = file_date_time
result = item
if result:
print('restoring: {}'.format(os.path.basename(result)))
else:
abort("Cannot find any SQL files to restore.")
return result
def _heading(self, command):
print(yellow("{}: {} for {}").format(
command,
self.backup_or_files,
self.site_info.domain,
))
def _repo(self):
return '{}{}/{}'.format(
self.site_info.rsync_ssh,
self.site_info.domain,
self.backup_or_files,
)
def _restore(self, restore_to):
env = {
'PASSPHRASE': self.site_info.rsync_gpg_password,
}
with shell_env(**env):
local('duplicity restore {} {}'.format(
self._repo(),
restore_to,
))
def _restore_database(self, restore_to):
sql_file = self._find_sql(restore_to)
print(green("restore to test database: {}".format(sql_file)))
if self.site_info.is_mysql:
self._restore_database_mysql(sql_file)
elif self.site_info.is_postgres:
self._restore_database_postgres(restore_to, sql_file)
else:
abort("Nothing to do... (this is a problem)")
return sql_file
def _restore_database_mysql(self, sql_file):
local_project_folder = self.path.local_project_folder(
self.site_info.domain
)
to_sql_file = os.path.join(local_project_folder, 'mysqldump.sql')
from_to = []
from_to.append((sql_file, to_sql_file))
self._remove_files_folders(from_to)
# move the files/folders to the project folder
for from_file, to_file in from_to:
shutil.move(from_file, to_file)
def _restore_database_postgres(self, restore_to, sql_file):
database_name = self.path.test_database_name()
if local_database_exists(database_name):
drop_local_database(database_name)
local_database_create(database_name)
if not local_user_exists(self.site_info):
local_user_create(self.site_info)
local_load_file(database_name, sql_file)
local_reassign_owner(
database_name,
self.site_info.db_name,
self.path.user_name()
)
print(green("psql {}").format(database_name))
def _remove_file_or_folder(self, file_name):
if os.path.exists(file_name):
if os.path.isdir(file_name):
shutil.rmtree(file_name)
elif os.path.isfile(file_name):
os.remove(file_name)
else:
abort("Is not a file or folder: {}".format(file_name))
def _remove_files_folders(self, from_to):
count = 0
print
for ignore, to_file in from_to:
if os.path.exists(to_file):
count = count + 1
print('{}. {}'.format(count, to_file))
if count:
confirm = ''
while confirm not in ('Y', 'N'):
confirm = prompt(
"Are you happy to remove these {} files/folders?".format(
count
))
confirm = confirm.strip().upper()
if confirm == 'Y':
for ignore, to_file in from_to:
self._remove_file_or_folder(to_file)
else:
abort("Cannot restore unless existing files are removed.")
else:
print("No files or folders to remove from the project folder.")
def _get_from_to(self, temp_folder, project_folder):
result = []
match = glob.glob('{}/*'.format(temp_folder))
for item in match:
folder = os.path.join(
project_folder,
os.path.basename(item),
)
result.append((item, folder))
result.sort() # required for the test
return result
def _restore_files(self, restore_to):
if self.site_info.is_php:
self._restore_files_php_site(restore_to)
else:
self._restore_files_django_site(restore_to)
def _restore_files_django_site(self, restore_to):
from_to = self._get_from_to(
os.path.join(restore_to, 'public'),
self.path.local_project_folder_media(self.site_info.package),
)
from_to = from_to + self._get_from_to(
os.path.join(restore_to, 'private'),
self.path.local_project_folder_media_private(self.site_info.package),
)
self._remove_files_folders(from_to)
# move the files/folders to the project folder
for from_file, to_file in from_to:
shutil.move(from_file, to_file)
def _restore_files_php_site(self, restore_to):
from_to = self._get_from_to(
restore_to,
self.path.local_project_folder(self.site_info.domain),
)
self._remove_files_folders(from_to)
# move the files/folders to the project folder
for from_file, to_file in from_to:
shutil.move(from_file, to_file)
def list_current(self):
self._heading('list_current')
local('duplicity collection-status {}'.format(self._repo()))
def restore(self):
self._heading('restore')
try:
restore_to = tempfile.mkdtemp()
self._restore(restore_to)
if self.backup:
sql_file = self._restore_database(restore_to)
self._display_backup_not_restored(restore_to, sql_file)
elif self.files:
self._restore_files(restore_to)
self._display_files_not_restored(restore_to)
else:
abort("Nothing to do... (this is a problem)")
finally:
if os.path.exists(restore_to):
#shutil.rmtree(restore_to)
pass
self._heading('Complete')
def test_index_middleware_path(self):
path = Path('/', endpoint=False)
self.assertIsNotNone(path.match('/'))
self.assertIsNotNone(path.match('/abc'))
self.assertIsNotNone(path.match('/abc/def'))
def test_remote_folder_files(self):
path = Path('csw_web', 'files')
remote_folder = path.remote_folder()
self.assertIn('/repo/backup/files', remote_folder)
class TestPath(unittest.TestCase):
def setUp(self):
self.path = Path('csw_web', 'postgres')
self.user = getpass.getuser()
self.year = datetime.now().strftime('%Y')
def test_backup_file_name(self):
backup_file_name = self.path.backup_file_name()
self.assertIn('csw_web_', backup_file_name)
self.assertIn(self.user, backup_file_name)
self.assertIn(self.year, backup_file_name)
self.assertIn('sql', backup_file_name)
def test_backup_file_name_postgres(self):
backup_file_name = self.path.backup_file_name()
self.assertIn('sql', backup_file_name)
def test_backup_file_name_files(self):
path = Path('csw_web', 'files')
backup_file_name = path.backup_file_name()
self.assertIn('tar.gz', backup_file_name)
def test_backup_file_name_user(self):
path = Path('raymond@csw_web', 'postgres')
self.assertNotIn('raymond', path.backup_file_name())
def test_database_name(self):
database_name = self.path.database_name()
self.assertNotIn('sql', database_name)
def test_files_folder(self):
self.assertEquals('/home/web/repo/files', self.path.files_folder())
def test_invalid_name(self):
with self.assertRaises(TaskError) as cm:
Path('csw_web_*', 'postgres')
self.assertIn('invalid characters', cm.exception.value)
def test_invalid_file_type(self):
with self.assertRaises(TaskError) as cm:
Path('csw_web_', 'smartie')
self.assertIn('invalid file type', cm.exception.value)
def test_local_file_files(self):
path = Path('csw_web', 'files')
local_file = path.local_file()
self.assertIn('/repo/backup/files/csw_web_', local_file)
self.assertIn('.tar.gz', local_file)
def test_local_file_postgres(self):
local_file = self.path.local_file()
self.assertIn('home', local_file)
self.assertIn(self.user, local_file)
self.assertIn('/repo/backup/postgres/csw_web_', local_file)
self.assertIn('.sql', local_file)
def test_remote_file_files(self):
path = Path('csw_web', 'files')
remote_file = path.remote_file()
self.assertIn('/repo/backup/files/csw_web_', remote_file)
self.assertIn('.tar.gz', remote_file)
def test_remote_file_name_postgres(self):
remote_file = self.path.remote_file()
self.assertIn('/repo/backup/postgres/csw_web_', remote_file)
self.assertIn('.sql', remote_file)
def test_remote_folder_files(self):
path = Path('csw_web', 'files')
remote_folder = path.remote_folder()
self.assertIn('/repo/backup/files', remote_folder)
def test_remote_folder_postgres(self):
remote_folder = self.path.remote_folder()
self.assertIn('/repo/backup/postgres', remote_folder)
def test_test_database_name(self):
self.assertEquals(
self.path.test_database_name(),
'test_csw_web_{}'.format(getpass.getuser())
)
def test_user_name(self):
self.assertEquals(self.path.user_name(), self.user)
def test_local_file_files(self):
path = Path('csw_web', 'files')
local_file = path.local_file()
self.assertIn('/repo/backup/files/csw_web_', local_file)
self.assertIn('.tar.gz', local_file)
def test_remote_file_files(self):
path = Path('csw_web', 'files')
remote_file = path.remote_file()
self.assertIn('/repo/backup/files/csw_web_', remote_file)
self.assertIn('.tar.gz', remote_file)
def __init__(self, root='/'):
self.tree = RouteTree()
self.components = []
self.root = Path(root)
def remove_location(self):
"""
before execute, you'd better remove the old
:return:
"""
Path(self.location).remove()
def remove_thumbnail(self):
"""
after execute, you'd better remove the old thumbnail
:return:
"""
Path(self.thumbnail).remove()
def test_endpoint_path(self):
path = Path('/', endpoint=True)
self.assertIsNotNone(path.match('/'))
path = Path('/abc', endpoint=True)
self.assertIsNotNone(path.match('/abc'))
self.assertIsNone(path.match('/a'))
self.assertIsNone(path.match('/abcd'))
path = Path('/abc/def', endpoint=True)
self.assertIsNotNone(path.match('/abc/def'))
self.assertIsNone(path.match('/abc/d'))
class PathTest(unittest.TestCase):
# noinspection PyTypeChecker
def setUp(self):
self.fs = MagicMock(spec=Filesystem)
self.vfs = MagicMock(spec=VirtualFS)
self.fs.exists.return_value = False
self.vfs.exists.return_value = False
self.path_mocked = Path(self.fs, self.vfs)
self.dummyfs = DummyFS({'c1', 'c2', 'c3'})
self.dummyvfs = DummyVirtualFS(example_paths)
self.path = Path(self.dummyfs, self.dummyvfs)
def testIsObject_NonExistent(self):
self.assertFalse(self.path_mocked.exists("asdf"))
def testIsObject_ExistsInFS(self):
self.fs.exists.return_value = True
self.assertTrue(self.path_mocked.exists("asdf"))
def testIsObject_ExistsInVFS(self):
self.vfs.exists.return_value = True
self.assertTrue(self.path_mocked.exists("asdf"))
def testFetchObject_NonExistent(self):
self.assertRaises(FileNotFoundError, self.path_mocked.fetch_object, "asdf")
def testFetchObject_FromFS(self):
self.fs.exists.return_value = True
self.fs.get_object.return_value = DummyFSObject("", "", b"")
self.assertIsInstance(self.path_mocked.fetch_object("asdf"), FSObject)
def testFetchObject_FromVFS(self):
self.vfs.exists.return_value = True
fs_obj_mock = MagicMock(spec=FSObject)
self.vfs.get_object.return_value = VFSObject(None, fs_obj_mock)
self.fs.get_object.return_value = fs_obj_mock
test_obj_name = "asdf"
self.assertIsInstance(self.path_mocked.fetch_object(test_obj_name), FSObject)
self.vfs.get_object.assert_called_once_with(test_obj_name)
self.fs.get_object.assert_called_once_with(fs_obj_mock)
def testWalk(self):
walk_result = list(self.path.walk('/'))
expected_result = [
('/', ['asdf', 'a'], []),
('/a', ['b'], []),
('/a/b', ['subdir1', 'subdir2', 'subdir3'], [f1, f2]),
('/a/b/subdir3', ['sub1', 'sub3', 'sub2'], []),
('/a/b/subdir3/sub2', [], []),
('/a/b/subdir3/sub3', [], []),
('/a/b/subdir3/sub1', [], []),
('/a/b/subdir2', [], []),
('/a/b/subdir1', [], []),
('/asdf', [], [])
]
self.assertCountEqual(((p1, sorted(p2), sorted(p.object_name for p in p3)) for p1, p2, p3 in expected_result),
((p1, sorted(p2), sorted(p.object_name for p in p3)) for p1, p2, p3 in walk_result))
def create_folder(self):
"""
create toolset directory
:return: None
"""
Path(self.directory).create()
def name(self):
"""
current toolset name
:return: str
"""
return Path(self.directory).basename()
def test_wildcards_path(self):
with self.subTest('test wildcard char'):
path = Path('/h+llo')
self.assertIsNotNone(path.match('/hello'))
self.assertIsNotNone(path.match('/hallo'))
self.assertIsNone(path.match('/haello'))
with self.subTest('test wildcard string'):
path = Path('/*/world')
self.assertIsNotNone(path.match('/hello/world'))
self.assertIsNotNone(path.match('/hell/world'))
self.assertIsNone(path.match('/world'))
self.assertIsNone(path.match('/hello/nice/world'))
path = Path('/*o/world')
self.assertIsNotNone(path.match('/hello/world'))
self.assertIsNotNone(path.match('/foo/world'))
self.assertIsNone(path.match('/hell/world'))
with self.subTest('test wildcard segments'):
path = Path('/%/world')
self.assertIsNotNone(path.match('/hello/world'))
self.assertIsNotNone(path.match('/hell/world'))
self.assertIsNotNone(path.match('/hello/nice/world'))
self.assertIsNone(path.match('/world'))
path = Path('%')
self.assertIsNotNone(path.match('/'))
self.assertIsNotNone(path.match('/a1/b1'))
self.assertIsNotNone(path.match('/a/b/c'))
self.assertIsNotNone(path.match('/a/'))
self.assertIsNotNone(path.match('/a1'))
def setUp(self):
self.path = Path('csw_web', 'postgres')
self.user = getpass.getuser()
self.year = datetime.now().strftime('%Y')
def _create_middleware(self, handler, rule, methods):
path = Path(rule)
self.tree[path] = Middleware(handler, methods)
def test_backup_file_name_files(self):
path = Path('csw_web', 'files')
backup_file_name = path.backup_file_name()
self.assertIn('tar.gz', backup_file_name)
def mount(self, rule, router):
path = Path(rule, endpoint=False)
if path in self.tree:
raise IndexError(f'{path} cannot be used twice')
self.tree[path] = router.tree
def test_backup_file_name_user(self):
path = Path('raymond@csw_web', 'postgres')
self.assertNotIn('raymond', path.backup_file_name())
