def main(args=None):
"""
Main entry point
"""
# parse args
if args is None:
args = parse_args(sys.argv[1:])
# set logging level
if args.verbose > 1:
set_log_debug(logger)
elif args.verbose == 1:
set_log_info(logger)
s = FileSyncer(
args.BUCKET_NAME,
prefix=args.prefix,
dry_run=args.dry_run,
ssec_key=read_keyfile(args.key_file)
)
if args.FILELIST_PATH is not None:
files = read_filelist(args.FILELIST_PATH)
else:
files = args.PATH
s.restore(args.LOCAL_PREFIX, files)
def main(args=None):
"""
Main entry point
"""
# parse args
if args is None:
args = parse_args(sys.argv[1:])
# set logging level
if args.verbose > 1:
set_log_debug(logger)
elif args.verbose == 1:
set_log_info(logger)
s = FileSyncer(args.BUCKET_NAME,
prefix=args.prefix,
dry_run=args.dry_run,
ssec_key=read_keyfile(args.key_file))
files = read_filelist(args.FILELIST_PATH)
exclude = []
if args.exclude_file is not None:
exclude = read_filelist(args.exclude_file)
stats = s.run(files, exclude_paths=exclude)
if args.summary:
print(stats.summary)
class TestMakeRestoreFileList(object):
def setup(self):
with patch('%s.S3Wrapper' % pbm, autospec=True) as mock_s3:
self.cls = FileSyncer('bname')
self.mock_s3 = mock_s3
def test_make_restore_file_list(self):
s3files = {
'/foo': {},
'/bar/baz/blarg1': {},
'/bar/baz/blarg2': {},
'/bar/baz/blarg/quux': {},
'/baz': {},
'/blam': {}
}
restore_paths = [
'/foo',
'/bar/baz',
'/quux'
]
expected = [
'/foo',
'/bar/baz/blarg1',
'/bar/baz/blarg2',
'/bar/baz/blarg/quux'
]
res = self.cls._make_restore_file_list(restore_paths, s3files)
assert sorted(res) == sorted(expected)
class TestListdir(object):
def setup(self):
with patch('%s.S3Wrapper' % pbm, autospec=True) as mock_s3:
self.cls = FileSyncer('bname')
self.mock_s3 = mock_s3
def test_simple(self):
def se_isfile(p):
if p == '/foo/foo2':
return False
return True
result = [
('/foo', ['bar'], ['foo1', 'foo2']),
('/foo/bar', ['baz', 'blam'], ['foobar1']),
('/foo/bar/baz', [], ['foobarbaz1', 'foobarbaz2'])
]
expected = [
'/foo/foo1',
'/foo/bar/foobar1',
'/foo/bar/baz/foobarbaz1',
'/foo/bar/baz/foobarbaz2'
]
with patch('%s.os.walk' % pbm, autospec=True) as mock_walk:
with patch('%s.os.path.isfile' % pbm, autospec=True) as mock_if:
mock_walk.return_value = iter(result)
mock_if.side_effect = se_isfile
res = self.cls._listdir('/foo')
assert sorted(res) == sorted(expected)
assert mock_walk.mock_calls == [call('/foo')]
class TestFilterFilelist(object):
def setup(self):
with patch('%s.S3Wrapper' % pbm, autospec=True) as mock_s3:
self.cls = FileSyncer('bname')
self.mock_s3 = mock_s3
def test_simple(self):
local_files = [
'/foo/one',
'/foo/two',
'/foo/three',
'/foo/bar/one',
'/foo/bar/three',
'/foo/barzzz',
]
exclude_paths = ['/foo/bar/']
expected = [
'/foo/one',
'/foo/two',
'/foo/three',
'/foo/barzzz'
]
res = self.cls._filter_filelist(local_files, exclude_paths)
assert res == expected
class TestFileMeta(object):
def setup(self):
with patch('%s.S3Wrapper' % pbm, autospec=True) as mock_s3:
self.cls = FileSyncer('bname')
self.mock_s3 = mock_s3
def test_simple(self):
with patch('%s.md5_file' % pbm, autospec=True) as mock_md5:
mock_md5.side_effect = [
'abcd1234a',
'abcd1234b'
]
with patch('%s.os.path.getsize' % pbm, autospec=True) as mock_sz:
with patch('%s.os.path.getmtime' % pbm,
autospec=True) as mock_mtime:
mock_sz.side_effect = [6789, 1234]
mock_mtime.side_effect = [
123456789.0123,
987654321.5432
]
res = self.cls._file_meta(['a', 'b'])
assert res == {
'a': (6789, 123456789.0123, 'abcd1234a'),
'b': (1234, 987654321.5432, 'abcd1234b')
}
assert mock_md5.mock_calls == [call('a'), call('b')]
assert mock_sz.mock_calls == [call('a'), call('b')]
assert mock_mtime.mock_calls == [call('a'), call('b')]
def test_init_prefix_no_slash(self):
m_s3 = Mock()
with patch('%s.S3Wrapper' % pbm, autospec=True) as mock_s3:
mock_s3.return_value = m_s3
cls = FileSyncer('bname', prefix='foo', ssec_key='foo')
assert cls.s3 == m_s3
assert mock_s3.mock_calls == [
call('bname', prefix='foo', dry_run=False, ssec_key='foo')
]
def test_init_args(self):
m_s3 = Mock()
with patch('%s.S3Wrapper' % pbm, autospec=True) as mock_s3:
mock_s3.return_value = m_s3
cls = FileSyncer('bname', prefix='/foo', dry_run=True)
assert cls.s3 == m_s3
assert mock_s3.mock_calls == [
call('bname', prefix='/foo', dry_run=True, ssec_key=None)
]
assert cls._dry_run is True
class TestUploadFiles(object):
def setup(self):
with patch('%s.S3Wrapper' % pbm, autospec=True) as mock_s3:
self.cls = FileSyncer('bname')
self.mock_s3 = mock_s3
def test_upload_files(self):
files = {
'/foo/one': (111, 12345.67, 'aaaa'),
'/foo/two': (222, 23456.78, 'bbbb'),
'/foo/three': (333, 34567.89, 'cccc'),
}
def se_put(f, sz, mt, md):
if f == '/foo/two':
raise RuntimeError()
return None
self.mock_s3.return_value.put_file.side_effect = se_put
res = self.cls._upload_files(files)
assert res == (['/foo/two'], 444)
class TestFilesToUpload(object):
def setup(self):
with patch('%s.S3Wrapper' % pbm, autospec=True) as mock_s3:
self.cls = FileSyncer('bname')
self.mock_s3 = mock_s3
def test_simple(self):
local_files = {
'/foo/one': (111, 12345.67, 'aaaa'),
'/foo/two': (222, 23456.78, 'bbbb'),
'/foo/three': (333, 34567.89, 'cccc'),
}
s3_files = {
'/foo/one': (444, 45678.9, 'dddd'),
'/foo/three': (555, 456789.01, 'cccc')
}
res = self.cls._files_to_upload(local_files, s3_files)
assert res == {
'/foo/one': (111, 12345.67, 'aaaa'),
'/foo/two': (222, 23456.78, 'bbbb')
}
class TestS3Files(object):
def setup(self):
with patch('%s.S3Wrapper' % pbm, autospec=True) as mock_s3:
self.cls = FileSyncer('bname')
self.mock_s3 = mock_s3
def test_simple(self):
s3files = {
'/foo': {},
'/bar': {
'size_b': '1234',
'mtime': '1234.5678',
'md5sum': 'foobar',
'upload_by': 's3sfe v1.0.0'
}
}
self.mock_s3.return_value.get_filelist.return_value = s3files
res = self.cls._s3_files()
assert res == {
'/foo': (0, 0.0, None),
'/bar': (1234, 1234.5678, 'foobar')
}
def setup(self):
with patch('%s.S3Wrapper' % pbm, autospec=True) as mock_s3:
self.cls = FileSyncer('bname')
self.mock_s3 = mock_s3
class TestListAllFiles(object):
def setup(self):
with patch('%s.S3Wrapper' % pbm, autospec=True) as mock_s3:
self.cls = FileSyncer('bname')
self.mock_s3 = mock_s3
def test_simple(self):
paths = [
'/bar',
'/foo/bar',
'/foo/baz.txt',
'/foo/notfile',
'/foo/bar/one'
]
def se_listdir(_, p):
return ['/foo/bar/one', '/foo/bar/two', '/foo/bar/three/four']
def se_exists(p):
if p in [
'/foo/bar', '/foo/baz.txt', '/foo/notfile', '/foo/bar/one'
]:
return True
return False
def se_isfile(p):
if p == '/foo/baz.txt' or p == '/foo/bar/one':
return True
return False
def se_isdir(p):
if p == '/foo/bar':
return True
return False
with patch('%s.logger' % pbm) as mock_logger:
with patch('%s._listdir' % pb, autospec=True) as mock_listdir:
mock_listdir.side_effect = se_listdir
with patch.multiple(
'%s.os.path' % pbm,
autospec=True,
exists=DEFAULT,
isfile=DEFAULT,
isdir=DEFAULT
) as mock_os:
mock_os['exists'].side_effect = se_exists
mock_os['isfile'].side_effect = se_isfile
mock_os['isdir'].side_effect = se_isdir
res = self.cls._list_all_files(paths)
assert sorted(res) == sorted([
'/foo/bar/one',
'/foo/bar/two',
'/foo/bar/three/four',
'/foo/baz.txt'
])
assert mock_os['exists'].mock_calls == [
call(x) for x in paths
]
assert mock_os['isfile'].mock_calls == [
call('/foo/bar'),
call('/foo/baz.txt'),
call('/foo/notfile'),
call('/foo/bar/one')
]
assert mock_os['isdir'].mock_calls == [
call('/foo/bar'),
call('/foo/notfile')
]
assert mock_listdir.mock_calls == [
call(self.cls, '/foo/bar')
]
assert mock_logger.mock_calls == [
call.info('Listing files under %d paths', 5),
call.warning('Skipping non-existent path: %s', '/bar'),
call.debug('Found %d files under %s', 3, '/foo/bar'),
call.warning('Skipping unknown path type: %s', '/foo/notfile'),
call.debug('Done finding candidate files.')
]
class TestRestore(object):
def setup(self):
with patch('%s.S3Wrapper' % pbm, autospec=True) as mock_s3:
self.cls = FileSyncer('bname')
self.mock_s3 = mock_s3
def test_simple(self):
s3files = {
'/foo': {},
'/bar/baz/blarg1': {},
'/bar/baz/blarg2': {},
'/bar/baz/blarg/quux': {},
'/baz': {},
'/blam': {}
}
restore_paths = [
'/foo',
'/bar/baz',
'/quux'
]
restore_files = [
'/foo',
'/bar/baz/blarg1',
'/bar/baz/blarg2',
'/bar/baz/blarg/quux'
]
ms3 = self.mock_s3.return_value
ms3.get_filelist.return_value = s3files
with patch('%s._make_restore_file_list' % pb,
autospec=True) as mock_mrfl:
mock_mrfl.return_value = restore_files
self.cls.restore('/l/p', restore_paths)
assert mock_mrfl.mock_calls == [
call(self.cls, restore_paths, s3files)
]
assert ms3.mock_calls == [
call.get_filelist(),
call.get_file('/foo', '/l/p'),
call.get_file('/bar/baz/blarg1', '/l/p'),
call.get_file('/bar/baz/blarg2', '/l/p'),
call.get_file('/bar/baz/blarg/quux', '/l/p'),
]
def test_error(self):
def se_get(fpath, local_prefix):
if fpath == '/bar/baz/blarg1':
raise RuntimeError('foo')
return None
s3files = {
'/foo': {},
'/bar/baz/blarg1': {},
'/bar/baz/blarg2': {},
'/bar/baz/blarg/quux': {},
'/baz': {},
'/blam': {}
}
restore_paths = [
'/foo',
'/bar/baz',
'/quux'
]
restore_files = [
'/foo',
'/bar/baz/blarg1',
'/bar/baz/blarg2',
'/bar/baz/blarg/quux'
]
ms3 = self.mock_s3.return_value
ms3.get_filelist.return_value = s3files
ms3.get_file.side_effect = se_get
with patch('%s._make_restore_file_list' % pb,
autospec=True) as mock_mrfl:
mock_mrfl.return_value = restore_files
self.cls.restore('/l/p', restore_paths)
assert mock_mrfl.mock_calls == [
call(self.cls, restore_paths, s3files)
]
assert ms3.mock_calls == [
call.get_filelist(),
call.get_file('/foo', '/l/p'),
call.get_file('/bar/baz/blarg1', '/l/p'),
call.get_file('/bar/baz/blarg2', '/l/p'),
call.get_file('/bar/baz/blarg/quux', '/l/p'),
]
class TestRun(object):
def setup(self):
with patch('%s.S3Wrapper' % pbm, autospec=True) as mock_s3:
self.cls = FileSyncer('bname')
self.mock_s3 = mock_s3
def test_simple(self):
paths = ['a', 'b']
s3_files = {
'one': (1, 2, 'three'),
'two': (4, 5, 'six')
}
local_files = {
'one': (1, 2, 'three'),
'two': (4, 5, 'NOTsix'),
'three': (6, 7, 'eight')
}
to_upload = {
'two': (4, 5, 'NOTsix'),
'three': (6, 7, 'eight')
}
dates = [
'dt_start',
'dt_meta',
'dt_query',
'dt_calc',
'dt_upload',
'dt_end'
]
with patch('%s.dtnow' % pbm, autospec=True) as mock_dtnow:
mock_dtnow.side_effect = dates
with patch('%s.RunStats' % pbm, autospec=True) as mock_stats:
with patch.multiple(
pb,
autospec=True,
_list_all_files=DEFAULT,
_file_meta=DEFAULT,
_files_to_upload=DEFAULT,
_upload_files=DEFAULT,
_filter_filelist=DEFAULT,
_s3_files=DEFAULT
) as mocks:
mocks['_list_all_files'].return_value = [
'one', 'two', 'three']
mocks['_file_meta'].return_value = local_files
mocks['_files_to_upload'].return_value = to_upload
mocks['_upload_files'].return_value = (['one'], 123)
mocks['_s3_files'].return_value = s3_files
res = self.cls.run(paths)
assert mocks['_list_all_files'].mock_calls == [
call(self.cls, paths)
]
assert mocks['_file_meta'].mock_calls == [
call(self.cls, ['one', 'two', 'three'])
]
assert mocks['_s3_files'].mock_calls == [call(self.cls)]
assert mocks['_files_to_upload'].mock_calls == [
call(self.cls, local_files, s3_files)
]
assert mocks['_upload_files'].mock_calls == [
call(self.cls, to_upload)
]
assert mock_stats.mock_calls == [
call(
'dt_start', 'dt_meta', 'dt_query', 'dt_calc', 'dt_upload',
'dt_end', 3, 2, ['one'], 11, 123, dry_run=False
)
]
assert mocks['_filter_filelist'].mock_calls == []
assert res == mock_stats.return_value
def test_exclude(self):
paths = ['a', 'b']
s3_files = {
'one': (1, 2, 'three'),
'two': (4, 5, 'six')
}
local_files = {
'one': (1, 2, 'three'),
'two': (4, 5, 'NOTsix'),
'three': (6, 7, 'eight')
}
to_upload = {
'two': (4, 5, 'NOTsix'),
'three': (6, 7, 'eight')
}
dates = [
'dt_start',
'dt_meta',
'dt_query',
'dt_calc',
'dt_upload',
'dt_end'
]
with patch('%s.dtnow' % pbm, autospec=True) as mock_dtnow:
mock_dtnow.side_effect = dates
with patch('%s.RunStats' % pbm, autospec=True) as mock_stats:
with patch.multiple(
pb,
autospec=True,
_list_all_files=DEFAULT,
_file_meta=DEFAULT,
_files_to_upload=DEFAULT,
_upload_files=DEFAULT,
_filter_filelist=DEFAULT,
_s3_files=DEFAULT
) as mocks:
mocks['_list_all_files'].return_value = [
'one', 'two', 'three']
mocks['_file_meta'].return_value = local_files
mocks['_files_to_upload'].return_value = to_upload
mocks['_upload_files'].return_value = (['one'], 123)
mocks['_filter_filelist'].return_value = ['one', 'two']
mocks['_s3_files'].return_value = s3_files
res = self.cls.run(paths, exclude_paths=['a', 'b'])
assert mocks['_list_all_files'].mock_calls == [
call(self.cls, paths)
]
assert mocks['_file_meta'].mock_calls == [
call(self.cls, ['one', 'two'])
]
assert mocks['_s3_files'].mock_calls == [call(self.cls)]
assert mocks['_files_to_upload'].mock_calls == [
call(self.cls, local_files, s3_files)
]
assert mocks['_upload_files'].mock_calls == [
call(self.cls, to_upload)
]
assert mocks['_filter_filelist'].mock_calls == [
call(self.cls, ['one', 'two', 'three'], ['a', 'b'])
]
assert mock_stats.mock_calls == [
call(
'dt_start', 'dt_meta', 'dt_query', 'dt_calc', 'dt_upload',
'dt_end', 2, 2, ['one'], 11, 123, dry_run=False
)
]
assert res == mock_stats.return_value