def _prepare_preconditions(self):
ckan.verify_if_evaluser_exists()
ckan.verify_if_organization_exists('tu-wien')
ckan.ensure_package_does_not_exist('ucbtrace')
if self._db_type == 'mongodb':
mongodb.purge_indexes('CKAN_Datastore')
package = ckan.client.action.package_create(
name='ucbtrace',
title='UC Berkeley Home IP Web Traces',
private=False,
owner_org='tu-wien',
author='Steve Gribble',
maintainer='',
license='other-open',
extras=[{
'key': 'year',
'value': '2017'
}])
self._resource_id = ckan.client.action.resource_create(
package_id=package['id'],
name='UC Berkeley Home IP Web Trace')['id']
with open('data/datasets/trace_fields.json', 'r') as trace_fields:
ckan.client.action.datastore_create(resource_id=self._resource_id,
force='True',
fields=json.load(trace_fields),
indexes="client_port",
primary_key='id')
self._queries = self._random_query_generator.generate_random_queries(
20, ['client_port'])
def _check_precondition(self):
ckan.verify_if_evaluser_exists()
ckan.verify_if_organization_exists('tu-wien')
ckan.reset_package_to_initial_state(PACKAGE, RESOURCE_FILE_LOCATION)
ckan.verify_package_does_exist('rr-dataset')
self._resource_id = ckan.verify_package_contains_resource('rr-dataset',
{'name': 'countries_dataset.csv',
'datastore_active': True})
def _prepare_preconditions(self):
ckan.verify_if_evaluser_exists()
ckan.verify_if_organization_exists('tu-wien')
mongodb.purge_indexes('CKAN_Datastore')
ckan.ensure_package_does_not_exist('ucbtrace')
package = ckan.client.action.package_create(
name='ucbtrace',
title='UC Berkeley Home IP Web Traces',
private=False,
owner_org='tu-wien',
author='Steve Gribble',
maintainer='',
license='other-open',
extras=[{
'key': 'year',
'value': '2017'
}])
self._resource_id = ckan.client.action.resource_create(
package_id=package['id'],
name='UC Berkeley Home IP Web Trace')['id']
with open('data/datasets/trace_fields.json', 'r') as trace_fields:
ckan.client.action.datastore_create(resource_id=self._resource_id,
force='True',
fields=json.load(trace_fields),
primary_key='id')
with (open('data/datasets/preprocessed_trace_1000000')) as trace_file:
line_count = 1
records = []
while line := trace_file.readline():
records.append(line_to_trace_record(line, line_count))
if line_count % self._chunksize == 0:
self.logger.info(
f'{str(line_count).rjust(8, " ")} records uploaded')
ckan.client.action.datastore_upsert(
resource_id=self._resource_id,
records=records,
force=True,
method='insert')
records = []
line_count += 1
def _prepare_preconditions(self):
ckan.verify_if_evaluser_exists()
ckan.verify_if_organization_exists('tu-wien')
ckan.ensure_package_does_not_exist('ucbtrace')
package = ckan.client.action.package_create(
name='ucbtrace',
title='UC Berkeley Home IP Web Traces',
private=False,
owner_org='tu-wien',
author='Steve Gribble',
maintainer='',
license='other-open',
extras=[{
'key': 'year',
'value': '2017'
}])
self._resource_id = ckan.client.action.resource_create(
package_id=package['id'],
name='UC Berkeley Home IP Web Trace')['id']
with open('data/datasets/trace_fields.json', 'r') as trace_fields:
ckan.client.action.datastore_create(resource_id=self._resource_id,
force='True',
fields=json.load(trace_fields),
primary_key='id')
self.filter_queries = self._random_query_generator.generate_random_queries(
size=20)
self.fulltext_queries = ['GET', 'gif', 'html']
with open(
os.path.join(self.results_dir, 'csv',
f'{self.tag}_{self._testcase_name}_result.csv'),
'a') as result_file:
result_file.writelines(RESULT_FILE_HEADER)
def _check_precondition(self):
ckan.verify_if_evaluser_exists()
ckan.verify_if_organization_exists('tu-wien')
ckan.ensure_package_does_not_exist('rr-dataset')