def test_global_stats_aggregator():
with moto.mock_s3_deprecated():
s3_conn = boto.connect_s3()
aggregator = GlobalStatsAggregator(s3_conn=s3_conn)
add_s3_content(
s3_conn, {
'test-bucket/stats/dataset_summaries/CB.json':
json.dumps(sample_dataset_stats()),
'test-bucket/stats/dataset_summaries/VA.json':
json.dumps(sample_dataset_stats()),
})
with freeze_time('2017-01-10'):
aggregator.run('test-bucket/stats')
expected_stats = {
'total': 8,
'output_counts': {
'title': 8,
'description': 4
},
'output_percentages': {
'title': 1.0,
'description': 0.5
},
'last_updated': '2017-01-10T00:00:00',
}
key = s3_conn.get_bucket('test-bucket').get_key('stats/summary.json')
assert json.loads(
key.get_contents_as_string().decode('utf-8')) == expected_stats
def test_dataset_stats_counter():
counter = DatasetStatsCounter(quarter='2014Q1', dataset_id='VA')
counter.track(input_document={
'jobtitle': 'test',
'jobdesc': 'test'
},
output_document={
'title': 'test',
'description': 'test'
})
counter.track(input_document={
'jobtitle': 'test',
'jobdesc': '',
'extra': 'test'
},
output_document={
'title': 'test',
'description': ''
})
assert counter.stats['total'] == 2
assert counter.stats['output_counts']['title'] == 2
assert counter.stats['output_counts']['description'] == 1
assert counter.stats['input_counts']['jobtitle'] == 2
assert counter.stats['input_counts']['jobdesc'] == 1
assert counter.stats['input_counts']['extra'] == 1
with moto.mock_s3_deprecated():
with freeze_time('2017-01-10'):
s3_conn = boto.connect_s3()
s3_conn.create_bucket('test-bucket')
counter.save(s3_conn, 'test-bucket/stats')
key = s3_conn.get_bucket('test-bucket')\
.get_key('stats/quarterly/VA_2014Q1')
expected_stats = {
'total': 2,
'output_counts': {
'title': 2,
'description': 1
},
'input_counts': {
'jobtitle': 2,
'jobdesc': 1,
'extra': 1
},
'output_percentages': {
'title': 1.0,
'description': 0.5
},
'input_percentages': {
'jobtitle': 1.0,
'jobdesc': 0.5,
'extra': 0.5
},
'last_updated': '2017-01-10T00:00:00',
'quarter': '2014Q1',
}
assert json.loads(
key.get_contents_as_string().decode('utf-8')) == expected_stats
def setUp(self):
mock_s3 = moto.mock_s3_deprecated()
mock_s3.start()
self.addCleanup(mock_s3.stop)
self.s3 = boto.connect_s3()
self.s3.create_bucket('test_s3_bucket')
def test_s3_save(self):
with mock_s3_deprecated():
s3_conn = boto.connect_s3()
bucket_name = 'fake-matrix-bucket'
bucket = s3_conn.create_bucket(bucket_name)
matrix_store_list = self.matrix_store()
for matrix_store in matrix_store_list:
matrix_store.save(project_path='s3://fake-matrix-bucket',
name='test')
# HDF
hdf = HDFMatrixStore(
matrix_path='s3://fake-matrix-bucket/test.h5',
metadata_path='s3://fake-matrix-bucket/test.yaml')
# CSV
csv = CSVMatrixStore(
matrix_path='s3://fake-matrix-bucket/test.csv',
metadata_path='s3://fake-matrix-bucket/test.yaml')
assert csv.metadata == matrix_store_list[0].metadata
assert csv.matrix.to_dict() == matrix_store_list[0].matrix.to_dict(
)
assert hdf.metadata == matrix_store_list[0].metadata
assert hdf.matrix.to_dict() == matrix_store_list[0].matrix.to_dict(
)
def test_partner_list():
with moto.mock_s3_deprecated():
s3_conn = boto.connect_s3()
s3_conn.create_bucket('stats-bucket')
bucket = s3_conn.get_bucket('stats-bucket')
upload_partner_rollup(bucket, 'XX', 45)
upload_partner_rollup(bucket, 'YY', 55)
upload_partner_rollup(bucket, 'ZZ', 0)
assert DatasetStatsAggregator.partners(
s3_conn, config['partner_stats']['s3_path']) == ['XX', 'YY']
def setUp(self):
from moto import mock_s3_deprecated
self.mock_s3 = mock_s3_deprecated()
self.mock_s3.start()
# Make our fake bucket
self.s3 = boto.connect_s3(self.aws_creds.access_id,
self.aws_creds.secret_key,
calling_format=OrdinaryCallingFormat())
self.s3.create_bucket(self.old_style_bucket_id)
self.s3.create_bucket(DOWNLOAD_SITE_BUCKET)
def s3_conn(request):
mock = mock_s3_deprecated()
mock.start()
conn = boto.connect_s3()
def tear_down():
mock.stop()
request.addfinalizer(tear_down)
return conn
def test_template_image_image_url_s3(self):
"""Meta image links should work if using S3 storage."""
mock_s3 = moto.mock_s3_deprecated()
mock_s3.start()
s3 = boto.connect_s3()
s3.create_bucket('test_s3_bucket')
try:
# There should be no root required as the image rendition URL
# should generate a fully qualified S3 path.
self.check_template_meta_image_url(expected_root='')
finally:
mock_s3.stop()
def setUp(self):
video_proto = VideoProto()
video_proto.veda_id = 'XXXXXXXX2014-V00TEST'
self.upload_filepath = os.path.join(
os.path.dirname(os.path.abspath(__file__)), 'test_files',
'OVTESTFILE_01.mp4')
with patch.object(Hotstore, '_READ_AUTH',
PropertyMock(return_value=lambda: CONFIG_DATA)):
self.hotstore = Hotstore(video_object=video_proto,
upload_filepath=self.upload_filepath,
video_proto=video_proto)
# do s3 mocking
mock = mock_s3_deprecated()
mock.start()
conn = S3Connection()
conn.create_bucket(CONFIG_DATA['veda_s3_hotstore_bucket'])
self.addCleanup(mock.stop)
def test_quarterly_posting_stats():
with moto.mock_s3_deprecated():
s3_conn = boto.connect_s3()
s3_conn.create_bucket('stats-bucket')
bucket = s3_conn.get_bucket('stats-bucket')
upload_quarterly_dataset_counts(bucket, 'XX', '2014Q1', 5)
upload_quarterly_dataset_counts(bucket, 'XX', '2014Q2', 6)
upload_quarterly_dataset_counts(bucket, 'XX', '2014Q3', 7)
upload_quarterly_dataset_counts(bucket, 'XX', '2014Q4', 8)
upload_quarterly_dataset_counts(bucket, 'ZZ', '2014Q1', 10)
upload_quarterly_dataset_counts(bucket, 'ZZ', '2014Q2', 9)
upload_quarterly_dataset_counts(bucket, 'ZZ', '2014Q3', 8)
upload_quarterly_dataset_counts(bucket, 'ZZ', '2014Q4', 10)
assert DatasetStatsCounter.quarterly_posting_stats(
s3_conn, config['partner_stats']['s3_path']) == {
'2014Q1': 15,
'2014Q2': 15,
'2014Q3': 15,
'2014Q4': 18
}
def test_dataset_stats_counter_empty():
counter = DatasetStatsCounter(quarter='2013Q1', dataset_id='VA')
with moto.mock_s3_deprecated():
with freeze_time('2017-01-10'):
s3_conn = boto.connect_s3()
s3_conn.create_bucket('test-bucket')
counter.save(s3_conn, 'test-bucket/stats')
key = s3_conn.get_bucket('test-bucket')\
.get_key('stats/quarterly/VA_2013Q1')
expected_stats = {
'total': 0,
'output_counts': {},
'input_counts': {},
'output_percentages': {},
'input_percentages': {},
'last_updated': '2017-01-10T00:00:00',
'quarter': '2013Q1',
}
assert json.loads(
key.get_contents_as_string().decode('utf-8')) == expected_stats
def test_total_job_postings():
with moto.mock_s3_deprecated():
s3_conn = boto.connect_s3()
s3_conn.create_bucket('stats-bucket')
bucket = s3_conn.get_bucket('stats-bucket')
key = boto.s3.key.Key(bucket=bucket, name='partner-etl/summary.json')
key.set_contents_from_string(
json.dumps({
'total': 8,
'output_counts': {
'title': 8,
'description': 4
},
'output_percentages': {
'title': 1.0,
'description': 0.5
},
'last_updated': '2017-01-10T00:00:00',
}))
assert GlobalStatsAggregator(s3_conn)\
.saved_total(config['partner_stats']['s3_path']) == 8
def test_dataset_stats_aggregator():
with moto.mock_s3_deprecated():
s3_conn = boto.connect_s3()
aggregator = DatasetStatsAggregator(dataset_id='CB', s3_conn=s3_conn)
add_s3_content(
s3_conn, {
'test-bucket/stats/quarterly/CB_2014Q1':
json.dumps(sample_quarter_stats('2014Q1')),
'test-bucket/stats/quarterly/CB_2014Q2':
json.dumps(sample_quarter_stats('2014Q2')),
'test-bucket/stats/quarterly/VA_2014Q1':
json.dumps(sample_quarter_stats('2014Q1')),
})
with freeze_time('2017-01-10'):
aggregator.run('test-bucket/stats')
expected_stats = sample_dataset_stats()
key = s3_conn.get_bucket('test-bucket')\
.get_key('stats/dataset_summaries/CB.json')
assert json.loads(
key.get_contents_as_string().decode('utf-8')) == expected_stats
def test_field_value_counter():
counter = FieldValueCounter(quarter='2014Q1',
field_values=['jobtitle', 'jobdesc'])
counter.track(input_document={'jobtitle': 'test', 'jobdesc': 'test'}, )
counter.track(input_document={
'jobtitle': 'test',
'jobdesc': '',
'extra': 'test'
}, )
assert counter.accumulator['jobtitle']['test'] == 2
assert counter.accumulator['jobdesc']['test'] == 1
assert counter.accumulator['jobdesc'][''] == 1
with moto.mock_s3_deprecated():
s3_conn = boto.connect_s3()
s3_conn.create_bucket('test-bucket')
counter.save(s3_conn, 'test-bucket/stats')
key = s3_conn.get_bucket('test-bucket')\
.get_key('stats/field_values/2014Q1/jobtitle.csv')
expected_count = 'test,2'
assert key.get_contents_as_string().decode(
'utf-8').rstrip() == expected_count
def setUp(self):
super(MockS3Mixin, self).setUp()
self._mock_s3 = moto.mock_s3_deprecated()
self._mock_s3.start()