def get_paginated_docs(endpoint):
try:
r = requests.get(endpoint)
return r.json()
except requests.ConnectionError as error:
eprint("Error with request {0}".format(str(error)))
return None
def print_test(test_data):
doc_to_dump = {"type": "test", "info": {}}
for field in ["username", "start_time", "end_time", "experiment_id", "test_id", "test_name"]:
if field not in test_data:
eprint("Field {0} missing from test {1}".format(field, test_data["test_id"]))
else:
doc_to_dump["info"][field] = test_data[field]
print(json.dumps(doc_to_dump))
def send_test_docs(self, documents):
for test in documents:
if not self.experiment_exists(test["experiment_id"],
test["username"]):
# Experiment doesn't exist
eprint("Experiment {0} doesn't exist".format(
test["experiment_id"]))
continue
info = self.get_test(test["experiment_id"], test["test_name"],
test["username"])
self.post_doc(test, info, self.tests_full_endpoint)
def get_doc(self, endpoint, doc_name):
tries = 0
while tries <= MAX_CONNECTION_TRIES:
status_code = None
try:
r = requests.get(endpoint)
if r.status_code == 200:
experiments = r.json()["_items"]
if experiments:
return r.json()["_items"][0]
else:
return {}
elif r.status_code == 404:
return {}
else:
status_code = r.status_code
except requests.ConnectionError as error:
eprint("Error with request {0}".format(str(error)))
eprint(
"Couldn't get document {0}, trying again for the {1} time out of {2}"
.format(doc_name, tries, MAX_CONNECTION_TRIES))
if status_code:
eprint("Status code was {0}".format(str(status_code)))
tries += 1
if tries > MAX_CONNECTION_TRIES:
error_string = "Information retrieval for document {0} failed too many times, aborting".format(
doc_name)
eprint(error_string)
raise requests.ConnectionError(error_string)
def post_doc(self, doc, info, endpoint):
tries = 0
while tries <= MAX_CONNECTION_TRIES:
if info == {}:
# Document doesn't exist, create
r = requests.post(endpoint,
headers=headers,
data=json.dumps(doc))
if r.status_code != 201:
eprint(
"Couldn't properly put document to address {0}".format(
endpoint))
eprint(r.text)
tries += 1
else:
iprint("Document created at: {0}".format(
time.strftime("%D %H:%M:%S", time.localtime()) +
" timestamp is " + str(time.time())))
break
else:
# Test exists, update
etag = info["_etag"]
doc_id = info["_id"]
doc = self.merge_data_from_existing_doc(info, doc)
these_headers = headers
these_headers["If-Match"] = etag
r = requests.put(endpoint + "/" + str(doc_id),
headers=these_headers,
data=json.dumps(doc))
if r.status_code != 200:
eprint(
"Couldn't properly put document to address {0}".format(
endpoint))
eprint(r.text)
tries += 1
else:
iprint("Document updated at: " +
time.strftime("%D %H:%M:%S", time.localtime()))
break
if tries > MAX_CONNECTION_TRIES:
error_string = "Information posting for document {0} failed too many times, aborting".format(
str(doc))
eprint(error_string)
raise ConnectionError(error_string)
def __init__(self, os_env=None):
if not os_env:
self.tests_post_endpoint = os.getenv(
TESTS_POST_ENDPOINT_OS_VARNAME, default_tests_database_name)
self.experiments_post_endpoint = os.getenv(
EXPERIMENTS_POST_ENDPOINT_OS_VARNAME,
default_experiments_database_name)
self.mongodb_ip = os.getenv(MONGODB_IP_OS_VARNAME,
default_mongodb_ip)
try:
self.mongodb_port = str(
int(
os.getenv(MONGODB_PORT_OS_VARNAME,
default_mongodb_port)))
except ValueError:
eprint("Invalid port configuration, using default '" +
str(default_mongodb_port) + "'")
self.mongodb_port = str(default_mongodb_port)
else:
try:
self.tests_post_endpoint = os_env[
TESTS_POST_ENDPOINT_OS_VARNAME]
except KeyError:
self.tests_post_endpoint = default_tests_database_name
try:
self.experiments_post_endpoint = os_env[
EXPERIMENTS_POST_ENDPOINT_OS_VARNAME]
except KeyError:
self.experiments_post_endpoint = default_experiments_database_name
try:
self.mongodb_ip = os_env[MONGODB_IP_OS_VARNAME]
except KeyError:
self.mongodb_ip = default_mongodb_ip
try:
self.mongodb_port = str(int(os_env[MONGODB_PORT_OS_VARNAME]))
except (ValueError, KeyError):
self.mongodb_port = str(default_mongodb_port)
self.tests_full_endpoint = "http://{0}:{1}/{2}".format(
self.mongodb_ip, self.mongodb_port, self.tests_post_endpoint)
self.experiments_full_endpoint = "http://{0}:{1}/{2}".format(
self.mongodb_ip, self.mongodb_port, self.experiments_post_endpoint)
def delete_test(self, experiment_id, test_name, username):
if self.experiment_exists(experiment_id, username):
test = self.get_test(experiment_id, test_name, username)
if not test:
iprint("Document doesn't {0} exist".format(test_name))
return
headers["If-Match"] = test["_etag"]
tests_full_endpoint = "{0}/{1}".format(self.tests_full_endpoint,
test["_id"])
r = requests.delete(tests_full_endpoint, headers=headers)
if r.status_code != 204:
eprint(
"Couldn't properly delete document to address {0}".format(
tests_full_endpoint))
eprint(r.text)
else:
iprint("Document deleted at: " +
time.strftime("%D %H:%M:%S", time.localtime()))
else:
return False
def get_test_info(experiment_name, test_name, username):
if not mongodb_agent.experiment_exists(experiment_name, username):
eprint("Couldn't find experiment with id {0}".format(experiment_name))
exit(0)
if not test_name or test_name == "ALL":
tests = mongodb_agent.get_experiment_tests(experiment_name, username)
else:
test = mongodb_agent.get_test(experiment_name, test_name, username)
if not test:
tests = []
else:
tests = [test]
if not tests:
eprint("Couldn't find tests for experiment {0}".format(experiment_name))
exit(0)
for test in tests:
print_test(test)
def get_experiment_info(experiment_name, username):
if experiment_name == "ALL":
data = mongodb_agent.get_all_experiments(username)
if not data:
eprint("Couldn't find experiment data, maybe there is none?")
exit(0)
else:
data = mongodb_agent.get_experiment(experiment_name, username)
if not data:
eprint(
"Couldn't find experiment with id {0}".format(experiment_name))
exit(0)
if type(data) == type(dict()):
print_experiment(data)
elif type(data) == type(list()):
for experiment in data:
print_experiment(experiment)
else:
print_experiment(data)
def delete_experiment(self, experiment_id, username):
if self.experiment_exists(experiment_id, username):
experiment = self.get_experiment(experiment_id, username)
headers["If-Match"] = experiment["_etag"]
experiment_full_endoint = "{0}/{1}".format(
self.experiments_full_endpoint, experiment["_id"])
r = requests.delete(experiment_full_endoint, headers=headers)
if r.status_code != 204:
eprint("Couldn't properly delete document in {0}".format(
experiment_full_endoint))
eprint(r.text)
else:
iprint("Document deleted at: " +
time.strftime("%D %H:%M:%S", time.localtime()))
else:
eprint("Document doesn't {0} exist".format(experiment_id))
return False
return r.json()
except requests.ConnectionError as error:
eprint("Error with request {0}".format(str(error)))
return None
if __name__ == '__main__':
abort = False
docs = dict()
docs["experiment"] = []
docs["test"] = []
agent = MongoDBTimestampAgent()
eprint("Mongodb agent started, waiting for input to send.")
# eprint("Endpoints to use will be {0} to experiments and {1} to tests".format(
# agent.get_experiments_endpoint(), agent.get_tests_endpoint()
# ))
try:
# UNAFFECTED BY BUFFERING
while True:
line = sys.stdin.readline()
if line == "":
# Reached EOF
break
try:
new_doc = ast.literal_eval(line)
if __name__ == '__main__':
mongodb_agent = MongoDBTimestampAgent()
parser = argparse.ArgumentParser(description='Signal for the start, end times or for the deletion of a test.')
parser.add_argument('option', metavar='option', type=str,
help='an operation option "info", "start", "end" or "delete"')
parser.add_argument('experiment_name', metavar='experiment_name', type=str,
help='The name of the experiment that encompasses this test')
parser.add_argument('test_name', metavar='test_name', type=str,
help='The name of the test')
parser.add_argument('--time', type=str, default=None,
help="A time string in the form 'yyyy/mm/dd-HH:MM:SS' (e.g., '2018/06/01-12:34:36')")
parser.add_argument('--username', type=str, default=None,
help="The username")
args = parser.parse_args()
username = get_username(args)
if args.option == "start" or args.option == "end":
signal_test(args.experiment_name, args.test_name, username, args.option, get_timestamp(args))
elif args.option == "delete":
mongodb_agent.delete_test(args.experiment_name, args.test_name, username)
elif args.option == "info":
get_test_info(args.experiment_name, args.test_name, username)
else:
eprint("Bad option")
exit(1)