def _prepare_preconditions(self):
        ckan.verify_if_evaluser_exists()
        ckan.verify_if_organization_exists('tu-wien')

        ckan.ensure_package_does_not_exist('ucbtrace')
        if self._db_type == 'mongodb':
            mongodb.purge_indexes('CKAN_Datastore')

        package = ckan.client.action.package_create(
            name='ucbtrace',
            title='UC Berkeley Home IP Web Traces',
            private=False,
            owner_org='tu-wien',
            author='Steve Gribble',
            maintainer='',
            license='other-open',
            extras=[{
                'key': 'year',
                'value': '2017'
            }])

        self._resource_id = ckan.client.action.resource_create(
            package_id=package['id'],
            name='UC Berkeley Home IP Web Trace')['id']

        with open('data/datasets/trace_fields.json', 'r') as trace_fields:
            ckan.client.action.datastore_create(resource_id=self._resource_id,
                                                force='True',
                                                fields=json.load(trace_fields),
                                                indexes="client_port",
                                                primary_key='id')

        self._queries = self._random_query_generator.generate_random_queries(
            20, ['client_port'])
Ejemplo n.º 2
0
    def _check_precondition(self):
        ckan.verify_if_evaluser_exists()
        ckan.verify_if_organization_exists('tu-wien')
        ckan.reset_package_to_initial_state(PACKAGE, RESOURCE_FILE_LOCATION)

        ckan.verify_package_does_exist('rr-dataset')
        self._resource_id = ckan.verify_package_contains_resource('rr-dataset',
                                                                  {'name': 'countries_dataset.csv',
                                                                   'datastore_active': True})
    def _prepare_preconditions(self):
        ckan.verify_if_evaluser_exists()
        ckan.verify_if_organization_exists('tu-wien')

        mongodb.purge_indexes('CKAN_Datastore')

        ckan.ensure_package_does_not_exist('ucbtrace')
        package = ckan.client.action.package_create(
            name='ucbtrace',
            title='UC Berkeley Home IP Web Traces',
            private=False,
            owner_org='tu-wien',
            author='Steve Gribble',
            maintainer='',
            license='other-open',
            extras=[{
                'key': 'year',
                'value': '2017'
            }])

        self._resource_id = ckan.client.action.resource_create(
            package_id=package['id'],
            name='UC Berkeley Home IP Web Trace')['id']

        with open('data/datasets/trace_fields.json', 'r') as trace_fields:
            ckan.client.action.datastore_create(resource_id=self._resource_id,
                                                force='True',
                                                fields=json.load(trace_fields),
                                                primary_key='id')

        with (open('data/datasets/preprocessed_trace_1000000')) as trace_file:
            line_count = 1
            records = []
            while line := trace_file.readline():
                records.append(line_to_trace_record(line, line_count))

                if line_count % self._chunksize == 0:
                    self.logger.info(
                        f'{str(line_count).rjust(8, " ")} records uploaded')
                    ckan.client.action.datastore_upsert(
                        resource_id=self._resource_id,
                        records=records,
                        force=True,
                        method='insert')
                    records = []

                line_count += 1
Ejemplo n.º 4
0
    def _prepare_preconditions(self):
        ckan.verify_if_evaluser_exists()
        ckan.verify_if_organization_exists('tu-wien')

        ckan.ensure_package_does_not_exist('ucbtrace')

        package = ckan.client.action.package_create(
            name='ucbtrace',
            title='UC Berkeley Home IP Web Traces',
            private=False,
            owner_org='tu-wien',
            author='Steve Gribble',
            maintainer='',
            license='other-open',
            extras=[{
                'key': 'year',
                'value': '2017'
            }])

        self._resource_id = ckan.client.action.resource_create(
            package_id=package['id'],
            name='UC Berkeley Home IP Web Trace')['id']

        with open('data/datasets/trace_fields.json', 'r') as trace_fields:
            ckan.client.action.datastore_create(resource_id=self._resource_id,
                                                force='True',
                                                fields=json.load(trace_fields),
                                                primary_key='id')

        self.filter_queries = self._random_query_generator.generate_random_queries(
            size=20)
        self.fulltext_queries = ['GET', 'gif', 'html']

        with open(
                os.path.join(self.results_dir, 'csv',
                             f'{self.tag}_{self._testcase_name}_result.csv'),
                'a') as result_file:
            result_file.writelines(RESULT_FILE_HEADER)
Ejemplo n.º 5
0
 def _check_precondition(self):
     ckan.verify_if_evaluser_exists()
     ckan.verify_if_organization_exists('tu-wien')
     ckan.ensure_package_does_not_exist('rr-dataset')