def test_result_api(self, client, make_archive, send_file):
temp_session = Session("abcd", "", "", "", "5abcd")
temp_session.status = 2
mock_db = mongomock.MongoClient().db
client().__getitem__.return_value = mock_db
temp_session.save(mock_db)
make_archive.return_value = "Test mock"
with app.test_request_context():
send_file.return_value = jsonify({"response": "File sent"})
response = self.app.get("/result/5abcd")
self.assertEqual(response.status_code, 200)
self.assertTrue(make_archive.called)
self.assertTrue(send_file.called)
def test_start_analysis(self, write_dataset, cross_val, client):
mock_db = mongomock.MongoClient().db
client().__getitem__.return_value = mock_db
session = {
"id": self.session_id,
"moses_options": moses_options,
"crossval_options": crossval_options,
"dataset": self.dataset,
"mnemonic": "abcdr4e",
"target_feature": "case",
"swd": None,
"filter_opts": {
"score": "precision",
"value": 0.4
}
}
cross_val.return_value.run_folds.return_value = "Run folds"
write_dataset.return_value = ("", "")
start_analysis.delay(**session)
tmp_session = Session.get_session(mock_db, session_id=self.session_id)
self.assertEqual(tmp_session.status, 2)
self.assertEqual(tmp_session.progress, 100)
def filter_models(mnemonic):
filter_type = request.args.get("filter")
filter_value = request.args.get("value")
overfitness = int(request.args.get("overfitness"))
if filter_type is None or filter_value is None:
return jsonify({"message":"Filter type and value required"}), 400
try:
value = float(filter_value)
assert 0 < value < 1
except Exception:
return jsonify({"message": "Filter value must be a float between 0 and 1"}), 400
db = pymongo.MongoClient(MONGODB_URI)[DB_NAME]
session = Session.get_session(db, mnemonic=mnemonic)
if session is None:
return jsonify({"message": "Session not found"}), 404
client = redis.Redis(host="redis")
if client.exists(mnemonic) > 0:
client.delete(mnemonic)
post_process_filter = PostProcess(filter_type, value, mnemonic, overfitness)
try:
models = post_process_filter.filter_models()
except AssertionError:
return jsonify({"message": "Session File not found"}), 404
json_dump = jsonpickle.encode(models, unpicklable=False)
client.set(mnemonic, json_dump, ex=600)
return jsonify({"models": json_dump}), 200
def get_session_info(mnemonic):
db = pymongo.MongoClient(MONGODB_URI)[DB_NAME]
session = Session.get_session(db, mnemonic=mnemonic)
if session:
return jsonify({
"moses_opts": session.moses_options,
"cross_val_opts": session.crossval_options,
"target_feature": session.target_feature
})
else:
return jsonify({
"message": "Session not found"
}), 404
def test_result_api_expired(self, client):
temp_session = Session("abcd", "", "", "", "5abcd")
temp_session.expired = True
temp_session.status = 2
mock_db = mongomock.MongoClient().db
client().__getitem__.return_value = mock_db
temp_session.save(mock_db)
response = self.app.get("/result/5abcd")
self.assertEqual(response.status_code, 400)
def check_status(mnemonic):
db = pymongo.MongoClient(MONGODB_URI)[DB_NAME]
session = Session.get_session(db, mnemonic=mnemonic)
if session:
if session.status == 1:
return jsonify({"status": session.status, "progress": session.progress, "start_time": session.start_time, "end_time": session.end_time, "message": session.message}), 200
elif session.status == 2 or session.status == -1:
td = timedelta(days=EXPIRY_SPAN)
time_to_expire = td.total_seconds() - (time.time() - session.end_time)
return jsonify({"status": session.status, "progress": session.progress, "start_time": session.start_time, "message": session.message, "end_time": session.end_time, "expire_time": time_to_expire}), 200
else:
return jsonify({"response": "Session not found"}), 404
def get_expired_sessions(db, time_span):
"""
Given a database object reference, it will return all the sessions where the difference between
their end_time and the current time is greater than or equal to time_span
:param db: Mongo object
:param time_span: the time difference in seconds
:return:
"""
sessions = Session.get_finished_sessions(db)
now = time.time()
if len(sessions) > 0:
expired_sessions = filter(lambda k: (now - k.end_time) > time_span,
sessions)
return expired_sessions
return None
def send_result(mnemonic):
db = pymongo.MongoClient(MONGODB_URI)[DB_NAME]
session = Session.get_session(db, mnemonic=mnemonic)
if session:
if session.status == 2 and not session.expired:
swd = os.path.join(DATASET_DIR, f"session_{mnemonic}")
zip_fp = zipfile.ZipFile(f"{swd}.zip", "w", zipfile.ZIP_DEFLATED)
zip_dir(swd, zip_fp)
zip_fp.close()
return send_file(f"{swd}.zip", as_attachment=True), 200
elif session.expired:
return jsonify({"response": "Session has expired."}), 400
elif session.status != 2:
return jsonify({"response", "Session not finished"}), 401
else:
return jsonify({"response": "Session not found"}), 404
def test_start_analysis_error_path(self, cross_val, client):
mock_db = mongomock.MongoClient().db
client().__getitem__.return_value = mock_db
session = {
"id": self.session_id,
"moses_options": moses_options,
"crossval_options": crossval_options,
"dataset": self.dataset,
"mnemonic": "abcdr4e",
"target_feature": "case"
}
mock_db.sessions.insert_one(session)
cross_val.side_effect = Exception("Mock exception")
self.assertRaises(Exception, start_analysis.delay(**session))
tmp_session = Session.get_session(mock_db, session_id=self.session_id)
self.assertEqual(tmp_session.status, -1)
def test_status(self, client):
temp_session = Session("abcd", "", "", "", "5abcd")
temp_session.status = 1
temp_session.progress = 40
mock_db = mongomock.MongoClient().db
client().__getitem__.return_value = mock_db
temp_session.save(mock_db)
response = self.app.get("/status/5abcd")
self.assertEqual(response.status_code, 200)
self.assertEqual(response.json["progress"], 40)
self.assertEqual(response.json["status"], 1)
def test_start_analysis(self, cross_val, client):
mock_db = mongomock.MongoClient().db
client().__getitem__.return_value = mock_db
session = {
"id": self.session_id,
"moses_options": moses_options,
"crossval_options": crossval_options,
"dataset": self.dataset,
"mnemonic": "abcdr4e",
"target_feature": "case",
"swd": None
}
mock_db.sessions.insert_one(session)
cross_val.return_value.run_folds.return_value = "Run folds"
start_analysis.delay(**session)
tmp_session = Session.get_session(mock_db, session_id=self.session_id)
self.assertEqual(tmp_session.status, 2)
self.assertEqual(tmp_session.progress, 100)
def start_analysis(**kwargs):
"""
A celery task that runs the MOSES analysis
:param session: The session object
:param cwd: Current working directory to store the results of the analysis
:param db: A reference to the mongo database
:return:
"""
db = pymongo.MongoClient(MONGODB_URI)[DB_NAME]
swd, file_path = write_dataset(kwargs["dataset"], kwargs["mnemonic"])
session = Session(kwargs["id"], kwargs["moses_options"],
kwargs["crossval_options"], file_path,
kwargs["mnemonic"], kwargs["target_feature"])
logger = get_logger(session.mnemonic)
session.save(db)
session.status = 1
session.progress = 1
session.start_time = time.time()
session.update_session(db)
try:
filter_type, value = kwargs["filter_opts"]["score"], kwargs[
"filter_opts"]["value"]
filter_cls = loader.get_score_filters(filter_type)
moses_cross_val = CrossValidation(session, db, filter_cls, value, swd)
logger.info("Started cross-validation run")
moses_cross_val.run_folds()
logger.info("Cross-validation done successfully")
session.status = 2
session.message = "Success"
session.progress = 100
except Exception as e:
session.status = -1
session.message = e.__str__()
logger.error(
f"Task failed with {traceback.format_exception(etype=type(e), value=e, tb=e.__traceback__)}"
)
finally:
session.end_time = time.time()
session.update_session(db)
def setUp(self):
dataset = os.path.join(TEST_DATA_DIR, "bin_truncated.csv")
session_id = str(uuid.uuid4())
self.session = Session(session_id, moses_options, crossval_options,
dataset, "abcd")
self.train_file, self.test_file = os.path.join(
TEST_DATA_DIR, "train_file"), os.path.join(TEST_DATA_DIR,
"test_file")
self.test_matrix = [[
1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0,
1, 0, 1, 1, 1, 1, 0, 1
],
[
1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0,
0, 1, 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 0, 1
],
[
0, 1, 0, 0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0,
0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 1
],
[
1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0,
0, 1, 0, 1, 1, 0, 1, 0, 1, 1, 1, 1, 0, 1
],
[
1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0,
0, 1, 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 0, 1
],
[
0, 1, 0, 0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0,
0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 1
],
[
0, 1, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0,
0, 1, 0, 1, 1, 0, 1, 0, 1, 1, 1, 0, 0, 1
],
[
0, 1, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0,
0, 1, 0, 1, 1, 0, 1, 1, 1, 1, 1, 0, 0, 1
],
[
0, 1, 0, 0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0,
0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 1
],
[
1, 1, 1, 0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0,
0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 1, 0, 1
],
[
1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0,
0, 1, 0, 1, 1, 0, 1, 0, 1, 1, 1, 1, 0, 1
],
[
1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, 0, 0, 1, 0,
0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 1, 1, 0, 1
],
[
0, 1, 0, 0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0,
0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 1
],
[
1, 1, 1, 1, 1, 0, 1, 0, 0, 0, 0, 1, 0, 0, 1, 0,
0, 1, 0, 1, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1
],
[
1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 1, 1,
0, 1, 0, 1, 1, 0, 1, 0, 1, 1, 1, 1, 0, 1
]]
def start_annotation(**kwargs):
logger = logging.getLogger("annotation-service")
session = Session(id=kwargs["session_id"], mnemonic=kwargs["mnemonic"],
annotations=kwargs["payload"]["annotations"], genes=kwargs["payload"]["genes"])
db = pymongo.MongoClient(MONGODB_URI)[DB_NAME]
try:
session.save(db)
session.status = 1
session.start_time = time.time()
session.update_session(db)
path = os.path.join(RESULT_DIR, session.mnemonic)
if not os.path.exists(path):
os.makedirs(path)
logger.info("when executing atoms:" + scheme_eval(atomspace, "(count-all)").decode("utf-8"))
annotate(atomspace, kwargs["payload"]["annotations"], kwargs["payload"]["genes"],
session.mnemonic)
scm_dir = "/root/result/{session}".format(session=session.mnemonic)
json_file = "/root/result/{session}/{session}.json".format(session=session.mnemonic)
logger.info("Result dir: " + scm_dir)
session.status = 2
session.result = json_file
logger.info("Applying Multi-level Layout")
out_dict = multi_level_layout(json_file)
with open(json_file, "w") as fp:
json.dump({"elements": out_dict}, fp)
session.results_file = scm_dir
csv_file = to_csv(session.mnemonic)
logger.info(csv_file)
session.csv_file = csv_file
session.update_session(db)
return True
except Exception as ex:
msg = "Error: " + ex.__str__()
session.status = -1
session.update_session(db)
session.message = msg
logger.error(msg)
traceback.print_exc()
return False
finally:
session.end_time = time.time()
session.update_session(db)
