def do_test_post_table(verb1,
verb2,
src,
cnttype,
tablename,
kvp1,
kvp2,
content1,
content2,
has_comments=False):
print("cwd =", os.getcwd())
ctx = reload.context()
pServer = Process(target=server.server, args=[ctx], kwargs={})
pServer.start()
time.sleep(WAIT_PERIOD)
pWorker = Process(target=reload.startWorker)
pWorker.start()
time.sleep(WAIT_PERIOD)
try:
print("get " + tablename)
resp = requests.get("http://localhost:5000/table/" + tablename)
assert (resp.json() == [])
print("post " + tablename)
resp = do_request_table(verb1,
tablename,
kvp1,
src,
cnttype,
has_comments=has_comments)
print(resp.text)
assert resp.status_code == 200
taskid = resp.json()
assert isinstance(taskid, str)
wait_for_task_to_finish(taskid)
print("get " + tablename)
resp = requests.get("http://localhost:5000/table/" + tablename)
respjson = resp.json()
assert (bag_contains(respjson, content1))
print("post " + tablename)
resp = do_request_table(verb2,
tablename,
kvp2,
src,
cnttype,
has_comments=has_comments)
assert resp.status_code == 200
taskid = resp.json()
assert isinstance(taskid, str)
wait_for_task_to_finish(taskid)
print("get " + tablename)
resp = requests.get("http://localhost:5000/table/" + tablename)
respjson = resp.json()
assert (bag_contains(respjson, content2))
finally:
pWorker.terminate()
pServer.terminate()
reload.clearTasks()
reload.clearDatabase(ctx)
reload.createTables(ctx)
def test_get_all_tasks():
ctx = reload.context()
pServer = Process(target = server.server, args=[ctx], kwargs={})
print("starting server ctx = " + str(ctx))
pServer.start()
print("server started, waiting for " + str(WAIT_PERIOD))
time.sleep(WAIT_PERIOD)
print("clearing tasks")
reload.clearTasks()
print("clearing database")
reload.clearDatabase(ctx)
print("creating tables")
reload.createTables(ctx)
print("starting worker")
pWorker = Process(target = reload.startWorker)
pWorker.start()
print("worker started, waiting for " + str(WAIT_PERIOD))
time.sleep(WAIT_PERIOD)
print("set up")
try:
resp0 = requests.get("http://localhost:5000/task")
assert len(resp0.json()["queued"]) == 0
resp1 = requests.post("http://localhost:5000/sync")
task_id = resp1.json()
wait_for_task_to_start(task_id)
resp2 = requests.get("http://localhost:5000/task")
assert resp2.json() == {
"queued": [],
"started": {
"job_ids": [task_id],
"expired_job_ids": []
},
"finished": {
"job_ids": [],
"expired_job_ids": []
},
"failed": {
"job_ids": [],
"expired_job_ids": []
},
"deferred": {
"job_ids": [],
"expired_job_ids": []
}
}
finally:
pWorker.terminate()
pServer.terminate()
reload.clearTasks()
reload.clearDatabase(ctx)
reload.createTables(ctx)
def test_back_up_endpoint():
ctx = reload.context()
p = Process(target = server.server, args=[ctx], kwargs={})
p.start()
time.sleep(WAIT_PERIOD)
try:
resp = requests.get("http://localhost:5000/backup")
assert resp.status_code == 200
print(resp.json())
assert isinstance(resp.json(), list)
finally:
p.terminate()
reload.clearTasks()
def do_test_post_error(verb1, src, cnttype, tablename, kvp1, status_code,
resp_text):
ctx = reload.context()
pServer = Process(target=server.server, args=[ctx], kwargs={})
pServer.start()
time.sleep(WAIT_PERIOD)
pWorker = Process(target=reload.startWorker)
pWorker.start()
time.sleep(WAIT_PERIOD)
try:
resp = do_request_table(verb1, tablename, kvp1, src, cnttype)
assert resp.status_code == status_code
taskid = resp.text
assert re.match(resp_text, taskid)
finally:
pWorker.terminate()
pServer.terminate()
reload.clearTasks()
reload.clearDatabase(ctx)
reload.createTables(ctx)
def test_get_task():
ctx = reload.context()
p = Process(target = server.server, args=[ctx], kwargs={})
p.start()
time.sleep(WAIT_PERIOD)
try:
resp = requests.post("http://localhost:5000/backup")
resp2 = requests.get("http://localhost:5000/task/" + resp.json())
assert "name" in resp2.json()
assert "created_at" in resp2.json()
assert "ended_at" in resp2.json()
assert "started_at" in resp2.json()
assert "enqueued_at" in resp2.json()
assert "description" in resp2.json()
assert "status" in resp2.json()
assert "result" in resp2.json()
finally:
p.terminate()
reload.clearTasks()
def test_task():
ctx = reload.context()
p = Process(target = server.server, args=[ctx], kwargs={})
p.start()
time.sleep(WAIT_PERIOD)
try:
resp0 = requests.get("http://localhost:5000/task")
assert len(resp0.json()["queued"]) == 0
resp = requests.post("http://localhost:5000/backup")
resp2 = requests.get("http://localhost:5000/task")
assert "queued" in resp2.json()
assert len(resp2.json()["queued"]) == 1
for status in ["started", "finished", "failed", "deferred"]:
assert status in resp2.json()
for category in ["job_ids", "expired_job_ids"]:
assert category in resp2.json()[status]
assert len(resp2.json()[status][category]) == 0
finally:
p.terminate()
reload.clearTasks()
def test_delete_task():
ctx = reload.context()
p = Process(target = server.server, args=[ctx], kwargs={})
p.start()
time.sleep(WAIT_PERIOD)
try:
resp0 = requests.get("http://localhost:5000/task")
assert len(resp0.json()["queued"]) == 0
resp = requests.post("http://localhost:5000/sync")
resp1 = requests.post("http://localhost:5000/sync")
resp2 = requests.get("http://localhost:5000/task")
assert len(resp2.json()["queued"]) == 2
assert resp.json() in resp2.json()["queued"]
assert resp1.json() in resp2.json()["queued"]
requests.delete("http://localhost:5000/task/" + resp1.json())
resp3 = requests.get("http://localhost:5000/task")
assert len(resp3.json()["queued"]) == 1
assert resp.json() in resp3.json()["queued"]
assert resp1.json() not in resp3.json()["queued"]
finally:
p.terminate()
reload.clearTasks()
def test_post_table_column():
ctx = reload.context()
fn = "/tmp/ssd1.csv"
fn2 = "/tmp/ssd2.csv"
csv1 = [[i, i] for i in range(10)]
csv2 = [[i, i + 1] for i in range(1, 11)]
n = len(csv1)
n2 = len(csv2)
write_csv(fn, ["ProposalID", "siteNumber"], csv1)
write_csv(fn2, ["ProposalID", "siteNumber"], csv2)
tablename = "SiteInformation"
column = "ProposalID"
kvp1 = kvp2 = {}
cnttype = "text/csv"
verb1 = verb2 = requests.post
content1 = [{
"siteNumber": str(row[1]),
"ProposalID": str(row[0])
} for row in csv1]
content2 = [{
"siteNumber": str(row[1]),
"ProposalID": str(row[0])
} for row in csv1 if row[0] not in list(map(lambda x: x[0], csv2))
] + [{
"siteNumber": str(row[1]),
"ProposalID": str(row[0])
} for row in csv2]
pServer = Process(target=server.server, args=[ctx], kwargs={})
pServer.start()
time.sleep(WAIT_PERIOD)
pWorker = Process(target=reload.startWorker)
pWorker.start()
time.sleep(WAIT_PERIOD)
try:
resp = do_request_table_column(verb1, tablename, column, kvp1, fn,
cnttype)
assert resp.status_code == 200
taskid = resp.json()
assert isinstance(taskid, str)
wait_for_task_to_finish(taskid)
print("get " + tablename)
resp = requests.get("http://localhost:5000/table/" + tablename)
respjson = resp.json()
assert (bag_contains(respjson, content1))
print("post " + tablename)
resp = do_request_table_column(verb2, tablename, column, kvp2, fn2,
cnttype)
assert resp.status_code == 200
taskid = resp.json()
assert isinstance(taskid, str)
wait_for_task_to_finish(taskid)
print("get " + tablename)
resp = requests.get("http://localhost:5000/table/" + tablename)
respjson = resp.json()
assert (bag_contains(respjson, content2))
finally:
pWorker.terminate()
pServer.terminate()
reload.clearTasks()
reload.clearDatabase(ctx)
reload.createTables(ctx)