def test_verify_happy_path(self, boto3):
dataset = make_configargs({'content_length': 180})
bucket = Mock()
bucket.Object.return_value = dataset
s3 = Mock()
s3.Bucket.return_value = bucket
boto3.resource.return_value = s3
self.optional.insert(0, '--dump-config')
argv = build_argv(self.optional, self.positional)
with captured_output(argv) as (out, err):
sut.main()
# assert calls
boto3.resource.assert_called_once_with('s3')
s3.Bucket.assert_called_once_with('foo')
# assert file size update
with open(self.json_size_file, 'r') as f:
prev_json_size = int(f.read())
self.assertTrue(prev_json_size == 180)
# assert cache size update
with open(self.cache_size_file, 'r') as f:
prev_cache_size = int(f.read())
self.assertTrue(prev_cache_size != 0)
def test_verify_json_file_invalid(self, boto3):
invalid_count_file = \
toAbsolute('__fixtures__/prev_json_size_invalid.txt')
test_positional = [
'json_data.json', invalid_count_file, self.cache_size_file
]
dataset = make_configargs({'content_length': 1})
bucket = Mock()
bucket.Object.return_value = dataset
s3 = Mock()
s3.Bucket.return_value = bucket
boto3.resource.return_value = s3
argv = build_argv(self.optional, test_positional)
with captured_output(argv) as (out, err):
sut.main()
# assert json size update
with open(invalid_count_file, 'r') as f:
prev_json_size = int(f.read())
self.assertTrue(prev_json_size != 0)
# Clean up
with open(invalid_count_file, 'w+') as f:
f.write(str('Invalid'))
def test_main_happy_path_check_latest_no_new_data(self, boto3):
dataset = make_configargs({
'last_modified': datetime.fromtimestamp(self.timestamp, pytz.utc)
})
bucket = Mock()
bucket.Object.return_value = dataset
s3 = Mock()
s3.Bucket.return_value = bucket
boto3.resource.return_value = s3
self.optional.insert(0, '--check-latest')
# Make sure the file exists and reflects the expected timestamp
open(self.actual_file, 'a').close()
os.utime(self.actual_file, (self.timestamp, self.timestamp))
argv = build_argv(self.optional)
with captured_output(argv) as (out, err):
sut.main()
boto3.resource.assert_called_once_with('s3')
s3.Bucket.assert_called_once_with('foo')
bucket.Object.assert_called_once_with('bar')
self.assertTrue(True, '\nNo new data set since 08:00 ' +
'PM Sunday, September 08, 2019\n' in
out.getvalue())
def test_callback(self):
options = make_configargs({
'infile': 'foo.bar'
})
instance = sut.ProgressPercentage(options)
with captured_output([]) as (out, err):
instance(100)
self.assertEqual(out.getvalue(), '\rfoo.bar 100 bytes')
def test_get_es_connection(self, mock_es):
options = make_configargs({
'es_host': 'www.example.org',
'es_port': '9222',
'es_username': '******',
'es_password': '******',
})
mock_es.return_value = 'passed'
actual = sut.get_es_connection(options)
self.assertEqual(actual, 'passed')
mock_es.assert_called_once_with('http://www.example.org:9222',
http_auth=('king', 'kong'),
timeout=1000,
user_ssl=True)
def test_verify_file_verify_failure(self, boto3):
dataset = make_configargs({'content_length': -1})
bucket = Mock()
bucket.Object.return_value = dataset
s3 = Mock()
s3.Bucket.return_value = bucket
boto3.resource.return_value = s3
with self.assertRaises(SystemExit) as ex:
argv = build_argv(self.optional, self.positional)
with captured_output(argv) as (out, err):
sut.main()
# assert calls
boto3.resource.assert_called_once_with('s3')
s3.Bucket.assert_called_once_with('foo')
# assert exit code
self.assertEqual(ex.exception.code, 2)
def test_main_happy_path_check_latest(self, boto3):
dataset = make_configargs({
'last_modified': datetime.fromtimestamp(self.timestamp, pytz.utc)
})
bucket = Mock()
bucket.Object.return_value = dataset
s3 = Mock()
s3.Bucket.return_value = bucket
boto3.resource.return_value = s3
self.optional.insert(0, '--check-latest')
argv = build_argv(self.optional)
with captured_output(argv) as (out, err):
sut.main()
boto3.resource.assert_called_once_with('s3')
s3.Bucket.assert_called_once_with('foo')
bucket.Object.assert_called_once_with('bar')
stat = os.stat(self.actual_file)
self.assertEqual(self.timestamp, stat.st_mtime)
def setUp(self):
self.options = make_configargs({'interval': '3y'})
