class TestClient(TestCase): def setUp(self): self.mock_requests = requests_mock.Mocker() self.mock_requests.start() self.addCleanup(self.mock_requests.stop) self.url = "http://example.test" self.username = "******" self.pcode = "pcode" cookies = {"csrftoken": "token"} self.mock_requests.get("{0}/api/login/".format(self.url), status_code=200) self.mock_requests.post("{0}/api/login/".format(self.url), status_code=200, cookies=cookies) self.mock_requests.get(self.url, status_code=200, cookies=cookies) self.mock_requests.get("{0}/create".format(self.url), status_code=200, cookies=cookies) self.mock_requests.get("{0}/api/providers".format(self.url), status_code=200) self.mock_requests.get("{0}/api/runs".format(self.url), status_code=200, cookies=cookies) with self.settings(SESSION_COOKIE_DOMAIN=self.url): self.client = EventKitClient(self.url, self.username, self.pcode) def test_get_providers(self): expected_response = {"provider": "provider_name"} self.mock_requests.get(self.client.providers_url, text=json.dumps(expected_response), status_code=200) providers = self.client.get_providers() self.assertEqual(expected_response, providers) with self.assertRaises(Exception): self.mock_requests.get(self.client.providers_url, text=json.dumps(expected_response), status_code=400) self.client.get_providers() def test_get_runs(self): request_response = {"runs": "runs"} expected_response = ["runs"] self.mock_requests.register_uri( "GET", "{0}/filter".format(self.client.runs_url), [{ "text": json.dumps(request_response), "status_code": 200 }, { "text": "", "status_code": 404 }], ) runs = self.client.search_runs() self.assertEqual(expected_response, runs) with self.assertRaises(Exception): self.mock_requests.get("{0}/filter".format(self.client.runs_url), text=json.dumps(expected_response), status_code=400) self.client.search_runs() def test_run_job(self): expected_response = {"runs": "runs"} self.mock_requests.post(self.client.jobs_url, text=json.dumps(expected_response), status_code=202) job_response = self.client.create_job(name="Name", description="Description", project="Project", selection=None, provider_tasks=None) self.assertEqual(expected_response, job_response) with self.assertRaises(Exception): self.mock_requests.post(self.client.jobs_url, text=json.dumps(expected_response), status_code=400) self.client.create_job(name="Name", description="Description", project="Project", selection=None, provider_tasks=None) def test_delete_run(self): example_run_uid = uuid.uuid4() expected_status = 204 url = "{}/{}".format(self.client.runs_url.rstrip("/"), example_run_uid) self.mock_requests.delete(url, status_code=expected_status) self.client.delete_run(example_run_uid) with self.assertRaises(Exception): wrong_status = 500 self.mock_requests.delete(url, status_code=wrong_status) self.client.delete_run(example_run_uid) def test_delete_job(self): example_job_uid = uuid.uuid4() expected_status = 204 url = "{}/{}".format(self.client.jobs_url.rstrip("/"), example_job_uid) self.mock_requests.delete(url, status_code=expected_status) self.client.delete_job(example_job_uid) with self.assertRaises(Exception): wrong_status = 500 self.mock_requests.delete(url, status_code=wrong_status) self.client.delete_job(example_job_uid) def test_cancel_export_provider_task(self): example_job_uid = uuid.uuid4() expected_status = 200 url = "{}/{}".format(self.client.provider_tasks_url.rstrip("/"), example_job_uid) self.mock_requests.patch(url, status_code=expected_status) self.client.cancel_provider(example_job_uid) with self.assertRaises(Exception): wrong_status = 500 self.mock_requests.patch(url, status_code=wrong_status) self.client.cancel_provider(example_job_uid) def test_wait_for_run(self): example_run_uid = uuid.uuid4() in_progress_response = [{ "status": "PENDING", "provider_tasks": [{ "tasks": [{ "status": "IN_PROGRESS", "errors": "" }] }] }] finished_response = [{ "status": "COMPLETED", "provider_tasks": [{ "tasks": [{ "status": "SUCCESS", "errors": "" }] }] }] expected_errors = ["EXAMPLE", "ERROR"] failure_reponse = [{ "provider_tasks": [{ "tasks": [{ "status": "FAILED", "errors": expected_errors }] }], "status": "INCOMPLETE" }] url = "{}/{}".format(self.client.runs_url.rstrip("/"), example_run_uid) self.mock_requests.register_uri( "GET", url, [{ "text": json.dumps(finished_response), "status_code": 200 }]) response = self.client.wait_for_run(run_uid=example_run_uid, run_timeout=2) self.assertEqual(response.get("status"), finished_response[0].get("status")) # Test failed to get status with self.assertRaises(Exception): self.mock_requests.register_uri( "GET", url, [{ "text": json.dumps(failure_reponse), "status_code": 404 }]) self.client.wait_for_run(run_uid=example_run_uid, run_timeout=2) # Test failed run with self.assertRaises(Exception): self.mock_requests.register_uri( "GET", url, [{ "text": json.dumps(failure_reponse), "status_code": 200 }]) self.client.wait_for_run(run_uid=example_run_uid, run_timeout=2) # Test timeout. with self.assertRaises(Exception): self.mock_requests.register_uri( "GET", url, [{ "text": json.dumps(in_progress_response), "status_code": 200 }]) self.client.wait_for_run(run_uid=example_run_uid, run_timeout=1) def test_check_provider(self): example_slug = "test" url = "{}/{}/status".format(self.client.providers_url, example_slug) success_response = '{"status": "SUCCESS"}' fail_response = '{"status": "ERROR"}' self.mock_requests.get(url, text=success_response) response = self.client.check_provider(example_slug) self.assertTrue(response) self.mock_requests.get(url, text=fail_response) response = self.client.check_provider(example_slug) self.assertFalse(response) def test_parse_duration(self): def with_timedelta(td): self.assertEqual(td.seconds, parse_duration(str(td))) # All possible inputs to timedelta - "9 days, 3:04:05.006007" with_timedelta( datetime.timedelta(weeks=1, days=2, hours=3, minutes=4, seconds=5, milliseconds=6, microseconds=7)) with_timedelta(datetime.timedelta( days=1, hours=2, minutes=3)) # No plural - "1 day, 2:03:00" with_timedelta(datetime.timedelta(hours=2, minutes=3, seconds=4)) # Just hms "2:03:04" def test_parse_size_unit(self): self.assertEqual(parse_size_unit("B"), 1) self.assertEqual(parse_size_unit("KB"), 1e3) self.assertEqual(parse_size_unit("MB"), 1e6) self.assertEqual(parse_size_unit("GB"), 1e9) self.assertEqual(parse_size_unit("TB"), 1e12) def test_parse_byte_size(self): self.assertAlmostEqual(256000, parse_byte_size("256 MB", "KB")) self.assertAlmostEqual(256, parse_byte_size("256 MB", "MB")) self.assertAlmostEqual(0.256, parse_byte_size("256000 KB", "GB")) self.assertAlmostEqual(0.000256, parse_byte_size("256000 KB", "TB"), places=10)
def main(): parser = argparse.ArgumentParser() parser.add_argument( 'url', help='The EventKit instance base url (i.e. http://cloud.eventkit.test).' ) parser.add_argument('file', help='A geojson file to be used for location data.') parser.add_argument( '--name', default='name', help= 'The field to be used for the name of the location which will be the datapack name.' ) parser.add_argument( '--description', default='description', help= 'The field to be used for the description of the location which will be the datapack description.' ) parser.add_argument( '--project', default='project', help= 'The project name, will be the same for all datapacks (not based on file).' ) parser.add_argument( '-s', '--sources', nargs='+', default='', help= 'The slugs of sources to check, if not included all visible sources are checked.' ) parser.add_argument( '--levels', nargs='+', default='1 10', help='The levels to see (i.e. 1 10) would seed from levels 1-10.') parser.add_argument('--limit', type=int, default=0, help='The max number of jobs to create.') parser.add_argument( '--start', type=int, default=0, help= 'The index (0-based) of the first geojson feature to use to create a datapack' ) parser.add_argument( '--verify', default='true', help= 'True to enable ssl verification, false to disable ssl verification') parser.add_argument( '--certificate', default='', help='The path to a certificate to use for authentication') args = parser.parse_args() user = password = None certificate = args.certificate if not certificate: user = os.getenv('EVENTKIT_USER') if not user: user = input("EventKit Username: "******"EventKit Password: "******"provider": provider.get('slug'), "formats": ["gpkg"], "min_zoom": args.levels[0], 'max_zoom': args.levels[1] }] else: provider_tasks = [{ "provider": provider.get('slug'), "formats": ["gpkg"], "min_zoom": 1, 'max_zoom': 1 } for provider in client.get_providers()] with open(args.file, 'rb') as geojson_file: geojson_data = json.load(geojson_file) count = args.limit or len( geojson_data['features']) # number of jobs to submit index = args.start # current feature # Stop when count gets to zero (user gets desired number of jobs) # or we run out of features to create jobs for. while count or (index > len(geojson_data['features'])): feature = geojson_data['features'][index] # Iterate index independently of the count because we might skip some jobs which would iterate the # features but not change the number of jobs submitted. index += 1 name = feature['properties'].get(args.name) description = feature['properties'].get( args.description) or "Created using the seed_data script." project = feature['properties'].get(args.project) or "seed" if name in [ run['job']['name'] for run in client.search_runs(search_term=name) ]: print(( "Skipping {0} because data already exists in a DataPack with the same name." .format(name))) continue if not name: print(("Skipping: \n {0} \n" "because a valid name wasn't provided or found.".format( feature))) continue response = client.create_job(name=name, description=description, project=project, provider_tasks=provider_tasks, selection=feature) if response: print(('Submitted job for {0}'.format(name))) count -= 1 else: print(('Failed to submit job for {0}'.format(name))) print(response)
class BenchmarkEventkit(object): def __init__(self, url, username, passwd, file_name, name_field, sources): self.url = url self.file = file_name self.sources = sources self.name = name_field self.username = username self.client = EventKitClient(self.url, username, passwd) def run_tests(self, batch_sizes): times = [] start_time = time.time() times += [start_time] print("Starting tests:") for batch_size in batch_sizes.split(','): batch_size = int(batch_size) print("Running test for batch size: {0}".format(batch_size)) batch_start_time = time.time() run_uid = self.run_iteration(batch_size) self.wait_for_run(run_uid) self.delete_runs() batch_finish_time = time.time() print("Test for batch size: {0} finished in {1} seconds".format( batch_size, batch_finish_time - batch_start_time)) times += [batch_finish_time] print("Tests finished in {0} seconds.".format(batch_finish_time - start_time)) return times def run_iteration(self, batch_size): if self.sources: provider_tasks = [] for provider in self.client.get_providers(): if provider.get('slug') in self.sources: provider_tasks += [{ "provider": provider.get('name'), "formats": ["gpkg"] }] else: provider_tasks = [{ "provider": provider.get('name'), "formats": ["gpkg"] } for provider in self.client.get_providers()] with open(self.file, 'r') as geojson_file: geojson_data = json.load(geojson_file) count = batch_size index = 0 while count or (index > len(geojson_data['features'])): feature = geojson_data['features'][index] index += 1 name = feature['properties'].get(self.name) description = "Created using the benchmark script." project = "benchmark" runs = [run for run in self.client.search_runs(search_term=name)] if name in [run['job']['name'] for run in runs ] and self.username in [run['user'] for run in runs]: print( "Skipping {0} because data already exists in a DataPack with the same name." .format(name)) continue if not name: print("Skipping: \n {0} \n" "because a valid name wasn't provided or found.".format( feature)) continue response = self.client.create_job(name=name, description=description, project=project, provider_tasks=provider_tasks, selection=feature) if response: print('Submitted job for {0}'.format(name)) count -= 1 if not count: return response['uid'] else: print('Failed to submit job for {0}'.format(name)) print(response) def delete_runs(self): response = self.client.search_runs() while response: for run in response: if run.get('user') == self.username: job_url = run['job']['url'] if job_url: self.client.client.delete( run['job']['url'], headers={'X-CSRFToken': self.client.csrftoken}) response = self.client.search_runs() def wait_for_run(self, job_uid, run_timeout=0): finished = False response = None first_check = time.time() while not finished: time.sleep(1) response = self.client.client.get(self.client.runs_url, params={ "job_uid": job_uid }, headers={ 'X-CSRFToken': self.client.csrftoken }).json() status = response[0].get('status') if status == "COMPLETED": finished = True last_check = time.time() if run_timeout and last_check - first_check > run_timeout: raise Exception( 'Run timeout ({}s) exceeded'.format(run_timeout)) return response[0]