def main(args=None):
"""
Main entry point
"""
# parse args
if args is None:
args = parse_args(sys.argv[1:])
# set logging level
if args.verbose > 1:
set_log_debug(logger)
elif args.verbose == 1:
set_log_info(logger)
s = FileSyncer(args.BUCKET_NAME,
prefix=args.prefix,
dry_run=args.dry_run,
ssec_key=read_keyfile(args.key_file))
files = read_filelist(args.FILELIST_PATH)
exclude = []
if args.exclude_file is not None:
exclude = read_filelist(args.exclude_file)
stats = s.run(files, exclude_paths=exclude)
if args.summary:
print(stats.summary)
class TestRun(object):
def setup(self):
with patch('%s.S3Wrapper' % pbm, autospec=True) as mock_s3:
self.cls = FileSyncer('bname')
self.mock_s3 = mock_s3
def test_simple(self):
paths = ['a', 'b']
s3_files = {
'one': (1, 2, 'three'),
'two': (4, 5, 'six')
}
local_files = {
'one': (1, 2, 'three'),
'two': (4, 5, 'NOTsix'),
'three': (6, 7, 'eight')
}
to_upload = {
'two': (4, 5, 'NOTsix'),
'three': (6, 7, 'eight')
}
dates = [
'dt_start',
'dt_meta',
'dt_query',
'dt_calc',
'dt_upload',
'dt_end'
]
with patch('%s.dtnow' % pbm, autospec=True) as mock_dtnow:
mock_dtnow.side_effect = dates
with patch('%s.RunStats' % pbm, autospec=True) as mock_stats:
with patch.multiple(
pb,
autospec=True,
_list_all_files=DEFAULT,
_file_meta=DEFAULT,
_files_to_upload=DEFAULT,
_upload_files=DEFAULT,
_filter_filelist=DEFAULT,
_s3_files=DEFAULT
) as mocks:
mocks['_list_all_files'].return_value = [
'one', 'two', 'three']
mocks['_file_meta'].return_value = local_files
mocks['_files_to_upload'].return_value = to_upload
mocks['_upload_files'].return_value = (['one'], 123)
mocks['_s3_files'].return_value = s3_files
res = self.cls.run(paths)
assert mocks['_list_all_files'].mock_calls == [
call(self.cls, paths)
]
assert mocks['_file_meta'].mock_calls == [
call(self.cls, ['one', 'two', 'three'])
]
assert mocks['_s3_files'].mock_calls == [call(self.cls)]
assert mocks['_files_to_upload'].mock_calls == [
call(self.cls, local_files, s3_files)
]
assert mocks['_upload_files'].mock_calls == [
call(self.cls, to_upload)
]
assert mock_stats.mock_calls == [
call(
'dt_start', 'dt_meta', 'dt_query', 'dt_calc', 'dt_upload',
'dt_end', 3, 2, ['one'], 11, 123, dry_run=False
)
]
assert mocks['_filter_filelist'].mock_calls == []
assert res == mock_stats.return_value
def test_exclude(self):
paths = ['a', 'b']
s3_files = {
'one': (1, 2, 'three'),
'two': (4, 5, 'six')
}
local_files = {
'one': (1, 2, 'three'),
'two': (4, 5, 'NOTsix'),
'three': (6, 7, 'eight')
}
to_upload = {
'two': (4, 5, 'NOTsix'),
'three': (6, 7, 'eight')
}
dates = [
'dt_start',
'dt_meta',
'dt_query',
'dt_calc',
'dt_upload',
'dt_end'
]
with patch('%s.dtnow' % pbm, autospec=True) as mock_dtnow:
mock_dtnow.side_effect = dates
with patch('%s.RunStats' % pbm, autospec=True) as mock_stats:
with patch.multiple(
pb,
autospec=True,
_list_all_files=DEFAULT,
_file_meta=DEFAULT,
_files_to_upload=DEFAULT,
_upload_files=DEFAULT,
_filter_filelist=DEFAULT,
_s3_files=DEFAULT
) as mocks:
mocks['_list_all_files'].return_value = [
'one', 'two', 'three']
mocks['_file_meta'].return_value = local_files
mocks['_files_to_upload'].return_value = to_upload
mocks['_upload_files'].return_value = (['one'], 123)
mocks['_filter_filelist'].return_value = ['one', 'two']
mocks['_s3_files'].return_value = s3_files
res = self.cls.run(paths, exclude_paths=['a', 'b'])
assert mocks['_list_all_files'].mock_calls == [
call(self.cls, paths)
]
assert mocks['_file_meta'].mock_calls == [
call(self.cls, ['one', 'two'])
]
assert mocks['_s3_files'].mock_calls == [call(self.cls)]
assert mocks['_files_to_upload'].mock_calls == [
call(self.cls, local_files, s3_files)
]
assert mocks['_upload_files'].mock_calls == [
call(self.cls, to_upload)
]
assert mocks['_filter_filelist'].mock_calls == [
call(self.cls, ['one', 'two', 'three'], ['a', 'b'])
]
assert mock_stats.mock_calls == [
call(
'dt_start', 'dt_meta', 'dt_query', 'dt_calc', 'dt_upload',
'dt_end', 2, 2, ['one'], 11, 123, dry_run=False
)
]
assert res == mock_stats.return_value