class TestClient(TestCase):
def setUp(self):
self.mock_requests = requests_mock.Mocker()
self.mock_requests.start()
self.addCleanup(self.mock_requests.stop)
self.url = 'http://example.test'
self.username = "******"
self.pcode = "pcode"
cookies = {'csrftoken': 'token'}
self.mock_requests.get("{0}/api/login/".format(self.url), status_code=200)
self.mock_requests.post("{0}/api/login/".format(self.url), status_code=200, cookies=cookies)
self.mock_requests.get(self.url, status_code=200, cookies=cookies)
self.mock_requests.get("{0}/create".format(self.url), status_code=200, cookies=cookies)
self.mock_requests.get("{0}/api/runs".format(self.url), status_code=200, cookies=cookies)
with self.settings(SESSION_COOKIE_DOMAIN=self.url):
self.client = EventKitClient(self.url, self.username, self.pcode)
def test_get_providers(self):
expected_response = {"provider": "provider_name"}
self.mock_requests.get(self.client.providers_url, text=json.dumps(expected_response), status_code=200)
providers = self.client.get_providers()
self.assertEqual(expected_response, providers)
with self.assertRaises(Exception):
self.mock_requests.get(self.client.providers_url, text=json.dumps(expected_response), status_code=400)
self.client.get_providers()
def test_get_runs(self):
request_response = {"runs": "runs"}
expected_response = ["runs"]
self.mock_requests.register_uri('GET', "{0}/filter".format(self.client.runs_url),
[{'text': json.dumps(request_response), 'status_code': 200},
{'text': '', 'status_code': 404}])
runs = self.client.get_runs()
self.assertEqual(expected_response, runs)
with self.assertRaises(Exception):
self.mock_requests.get("{0}/filter".format(self.client.runs_url), text=json.dumps(expected_response), status_code=400)
self.client.get_runs()
def test_run_job(self):
expected_response = {"runs": "runs"}
self.mock_requests.post(self.client.jobs_url, text=json.dumps(expected_response), status_code=202)
job_response = self.client.run_job(name='Name', description='Description', project='Project')
self.assertEqual(expected_response, job_response)
with self.assertRaises(Exception):
self.mock_requests.post(self.client.jobs_url, text=json.dumps(expected_response), status_code=400)
self.client.run_job(name='Name', description='Description', project='Project')
with self.assertRaises(Exception):
self.mock_requests.post(self.client.jobs_url, text=json.dumps(expected_response), status_code=202)
self.client.run_job(name=None)
def test_parse_duration(self):
def with_timedelta(td):
self.assertEqual(td.seconds, parse_duration(str(td)))
# All possible inputs to timedelta - "9 days, 3:04:05.006007"
with_timedelta(datetime.timedelta(weeks=1, days=2, hours=3, minutes=4, seconds=5, milliseconds=6,
microseconds=7))
with_timedelta(datetime.timedelta(days=1, hours=2, minutes=3)) # No plural - "1 day, 2:03:00"
with_timedelta(datetime.timedelta(hours=2, minutes=3, seconds=4)) # Just hms "2:03:04"
def test_parse_size_unit(self):
self.assertEqual(parse_size_unit('B'), 1)
self.assertEqual(parse_size_unit('KB'), 1e3)
self.assertEqual(parse_size_unit('MB'), 1e6)
self.assertEqual(parse_size_unit('GB'), 1e9)
self.assertEqual(parse_size_unit('TB'), 1e12)
def test_parse_byte_size(self):
self.assertAlmostEqual(256000, parse_byte_size('256 MB', 'KB'))
self.assertAlmostEqual(256, parse_byte_size('256 MB', 'MB'))
self.assertAlmostEqual(.256, parse_byte_size('256000 KB', 'GB'))
self.assertAlmostEqual(.000256, parse_byte_size('256000 KB', 'TB'), places=10)
class BenchmarkEventkit(object):
def __init__(self, url, username, passwd, file_name, name_field, sources):
self.url = url
self.file = file_name
self.sources = sources
self.name = name_field
self.username = username
self.client = EventKitClient(self.url, username, passwd)
def run_tests(self, batch_sizes):
times = []
start_time = time.time()
times += [start_time]
print("Starting tests:")
for batch_size in batch_sizes.split(','):
batch_size = int(batch_size)
print("Running test for batch size: {0}".format(batch_size))
batch_start_time = time.time()
run_uid = self.run_iteration(batch_size)
self.wait_for_run(run_uid)
self.delete_runs()
batch_finish_time = time.time()
print("Test for batch size: {0} finished in {1} seconds".format(batch_size, batch_finish_time - batch_start_time))
times += [batch_finish_time]
print("Tests finished in {0} seconds.".format(batch_finish_time - start_time))
return times
def run_iteration(self, batch_size):
if self.sources:
provider_tasks = []
for provider in self.client.get_providers():
if provider.get('slug') in self.sources:
provider_tasks += [{"provider": provider.get('name'), "formats": ["gpkg"]}]
else:
provider_tasks = [{"provider": provider.get('name'), "formats": ["gpkg"]} for provider in
self.client.get_providers()]
with open(self.file, 'r') as geojson_file:
geojson_data = json.load(geojson_file)
count = batch_size
index = 0
while count or (index > len(geojson_data['features'])):
feature = geojson_data['features'][index]
index += 1
name = feature['properties'].get(self.name)
description = "Created using the benchmark script."
project = "benchmark"
runs = [run for run in self.client.get_runs(search_term=name)]
if name in [run['job']['name'] for run in runs] and self.username in [run['user'] for run in runs]:
print("Skipping {0} because data already exists in a DataPack with the same name.".format(name))
continue
if not name:
print("Skipping: \n {0} \n"
"because a valid name wasn't provided or found.".format(feature))
continue
response = self.client.run_job(name=name, description=description, project=project,
provider_tasks=provider_tasks, selection=feature)
if response:
print('Submitted job for {0}'.format(name))
count -= 1
if not count:
return response['uid']
else:
print('Failed to submit job for {0}'.format(name))
print(response)
def delete_runs(self):
response = self.client.get_runs()
while response:
for run in response:
if run.get('user') == self.username:
job_url = run['job']['url']
if job_url:
self.client.client.delete(run['job']['url'],
headers={'X-CSRFToken': self.client.csrftoken})
response = self.client.get_runs()
def wait_for_run(self, job_uid, run_timeout=0):
finished = False
response = None
first_check = time.time()
while not finished:
time.sleep(1)
response = self.client.client.get(
self.client.runs_url,
params={"job_uid": job_uid},
headers={'X-CSRFToken': self.client.csrftoken
}).json()
status = response[0].get('status')
if status == "COMPLETED":
finished = True
last_check = time.time()
if run_timeout and last_check - first_check > run_timeout:
raise Exception('Run timeout ({}s) exceeded'.format(run_timeout))
return response[0]
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
'url',
help='The EventKit instance base url (i.e. http://cloud.eventkit.test).'
)
parser.add_argument('file',
help='A geojson file to be used for location data.')
parser.add_argument(
'--name',
default='name',
help=
'The field to be used for the name of the location which will be the datapack name.'
)
parser.add_argument(
'--description',
default='description',
help=
'The field to be used for the description of the location which will be the datapack description.'
)
parser.add_argument(
'--project',
default='project',
help=
'The project name, will be the same for all datapacks (not based on file).'
)
parser.add_argument(
'--sources',
nargs='*',
default='',
help=
'The project name, will be the same for all datapacks (not based on file).'
)
parser.add_argument(
'--limit',
type=int,
default=0,
help=
'The project name, will be the same for all datapacks (not based on file).'
)
parser.add_argument(
'--verify',
default='true',
help=
'True to enable ssl verification, false to disable ssl verification')
args = parser.parse_args()
user = os.getenv('EVENTKIT_USER')
if not user:
user = input("EventKit Username: "******"EventKit Password: "******"provider": provider.get('name'),
"formats": ["gpkg"]
}]
else:
provider_tasks = [{
"provider": provider.get('name'),
"formats": ["gpkg"]
} for provider in client.get_providers()]
with open(args.file, 'rb') as geojson_file:
geojson_data = json.load(geojson_file)
count = args.limit or len(
geojson_data['features']) # number of jobs to submit
index = 0 # current feature
# Stop when count gets to zero (user gets desired number of jobs)
# or we run our of features to create jobs for.
while count or (index > len(geojson_data['features'])):
feature = geojson_data['features'][index]
# Iterate index independently of the count because we might skip some jobs which would iterate the
# features but not change the number of jobs submitted.
index += 1
name = feature['properties'].get(args.name)
description = feature['properties'].get(
args.description) or "Created using the seed_data script."
project = feature['properties'].get(args.project) or "seed"
if name in [
run['job']['name'] for run in client.get_runs(search_term=name)
]:
print((
"Skipping {0} because data already exists in a DataPack with the same name."
.format(name)))
continue
if not name:
print(("Skipping: \n {0} \n"
"because a valid name wasn't provided or found.".format(
feature)))
continue
response = client.run_job(name=name,
description=description,
project=project,
provider_tasks=provider_tasks,
selection=feature)
if response:
print(('Submitted job for {0}'.format(name)))
count -= 1
else:
print(('Failed to submit job for {0}'.format(name)))
print(response)
class BenchmarkEventkit(object):
def __init__(self, url, username, passwd, file_name, name_field, sources):
self.url = url
self.file = file_name
self.sources = sources
self.name = name_field
self.username = username
self.client = EventKitClient(self.url, username, passwd)
def run_tests(self, batch_sizes):
times = []
start_time = time.time()
times += [start_time]
print("Starting tests:")
for batch_size in batch_sizes.split(','):
batch_size = int(batch_size)
print("Running test for batch size: {0}".format(batch_size))
batch_start_time = time.time()
run_uid = self.run_iteration(batch_size)
self.wait_for_run(run_uid)
self.delete_runs()
batch_finish_time = time.time()
print("Test for batch size: {0} finished in {1} seconds".format(
batch_size, batch_finish_time - batch_start_time))
times += [batch_finish_time]
print("Tests finished in {0} seconds.".format(batch_finish_time -
start_time))
return times
def run_iteration(self, batch_size):
if self.sources:
provider_tasks = []
for provider in self.client.get_providers():
if provider.get('slug') in self.sources:
provider_tasks += [{
"provider": provider.get('name'),
"formats": ["gpkg"]
}]
else:
provider_tasks = [{
"provider": provider.get('name'),
"formats": ["gpkg"]
} for provider in self.client.get_providers()]
with open(self.file, 'r') as geojson_file:
geojson_data = json.load(geojson_file)
count = batch_size
index = 0
while count or (index > len(geojson_data['features'])):
feature = geojson_data['features'][index]
index += 1
name = feature['properties'].get(self.name)
description = "Created using the benchmark script."
project = "benchmark"
runs = [run for run in self.client.get_runs(search_term=name)]
if name in [run['job']['name'] for run in runs
] and self.username in [run['user'] for run in runs]:
print(
"Skipping {0} because data already exists in a DataPack with the same name."
.format(name))
continue
if not name:
print("Skipping: \n {0} \n"
"because a valid name wasn't provided or found.".format(
feature))
continue
response = self.client.run_job(name=name,
description=description,
project=project,
provider_tasks=provider_tasks,
selection=feature)
if response:
print('Submitted job for {0}'.format(name))
count -= 1
if not count:
return response['uid']
else:
print('Failed to submit job for {0}'.format(name))
print(response)
def delete_runs(self):
response = self.client.get_runs()
while response:
for run in response:
if run.get('user') == self.username:
job_url = run['job']['url']
if job_url:
self.client.client.delete(
run['job']['url'],
headers={'X-CSRFToken': self.client.csrftoken})
response = self.client.get_runs()
def wait_for_run(self, job_uid, run_timeout=0):
finished = False
response = None
first_check = time.time()
while not finished:
time.sleep(1)
response = self.client.client.get(self.client.runs_url,
params={
"job_uid": job_uid
},
headers={
'X-CSRFToken':
self.client.csrftoken
}).json()
status = response[0].get('status')
if status == "COMPLETED":
finished = True
last_check = time.time()
if run_timeout and last_check - first_check > run_timeout:
raise Exception(
'Run timeout ({}s) exceeded'.format(run_timeout))
return response[0]