def corpusMenuInitializer(menu, prevMenu):
menu.text = """
The corpora are used for training new models and testing existing
models. The corpora installable here are from the two BioNLP Shared
Tasks (BioNLP'09 and BioNLP'11) on Event Extraction (organized by
University of Tokyo), and the First Challenge Task: Drug-Drug Interaction
Extraction (DDI'11, organized by Universidad Carlos III de Madrid).
The corpora are downloaded as interaction XML files, generated from the
original Shared Task files. If you need to convert the corpora from
the original files, you can use the convertBioNLP.py and convertDDI.py programs
located at Utils/Convert.
It is also recommended to download the official BioNLP Shared Task evaluator
programs, which will be used by TEES when training or testing on those corpora.
"""
# Mark "skip" as default option, this will be re-marked as install if a corpus is missing
menu.setDefault("s")
# If CORPUS_DIR setting is not set set it now
if not hasattr(Settings, "CORPUS_DIR") or getattr(Settings, "CORPUS_DIR") == None:
Settings.setLocal("CORPUS_DIR", os.path.join(menu.system.defaultInstallDir, "corpora"))
print >>sys.stderr
# Initialize handlers
handlers = []
handlerArgs = []
corpusInstallPath = os.path.join(menu.system.defaultInstallDir, "corpora")
corpusDownloadPath = os.path.join(menu.system.defaultInstallDir, "corpora/download")
# Check which corpora need to be installed
redownload = menu.optDict["1"].toggle
for corpus in ["GE", "EPI", "ID", "BB", "BI", "CO", "REL", "REN"]:
if menu.optDict["2"].toggle or (menu != prevMenu and not checkCorpusInstall(corpus)):
menu.setDefault("i")
menu.optDict["2"].toggle = True
handlers.append(convertBioNLP.installPreconverted)
handlerArgs.append(["BIONLP_11_CORPORA", corpusInstallPath, corpusDownloadPath, redownload, True])
break
if menu.optDict["3"].toggle or (menu != prevMenu and not checkCorpusInstall("GE09")):
menu.setDefault("i")
menu.optDict["3"].toggle = True
handlers.append(convertBioNLP.installPreconverted)
handlerArgs.append(["BIONLP_09_CORPUS", corpusInstallPath, corpusDownloadPath, redownload, True])
if menu.optDict["4"].toggle or (menu != prevMenu and not checkCorpusInstall("DDI", ("-train.xml", "-devel.xml"))):
menu.setDefault("i")
menu.optDict["4"].toggle = True
handlers.append(convertBioNLP.installPreconverted)
handlerArgs.append(["DDI_11_CORPUS", corpusInstallPath, corpusDownloadPath, redownload, True])
# A handler for installing BioNLP'11 evaluators
evaluatorInstallPath = os.path.join(menu.system.defaultInstallDir, "tools/evaluators")
evaluatorDownloadPath = os.path.join(menu.system.defaultInstallDir, "tools/download")
if menu.optDict["5"].toggle or (
menu != prevMenu
and (not hasattr(Settings, "BIONLP_EVALUATOR_DIR") or getattr(Settings, "BIONLP_EVALUATOR_DIR") == None)
):
menu.setDefault("i")
menu.optDict["5"].toggle = True
handlers.append(convertBioNLP.installEvaluators)
handlerArgs.append([evaluatorInstallPath, evaluatorDownloadPath, redownload, True])
# Add the handlers to install option
menu.optDict["i"].handler = handlers
menu.optDict["i"].handlerArgs = handlerArgs
def create_data_set(self,
name,
owner,
data,
cleaned,
access_user_list,
access_business_unit_list,
description,
storage_type,
size=0):
datasetID = "dataset_" + st.create_id()
hash_of_dataset = st.hash_data(data)
size = data.size
dataset = ds.DataSet(
datasetID=datasetID,
name=name,
owner=owner,
size=size,
hash_of_dataset=hash_of_dataset,
cleaned=cleaned,
access_user_list=access_user_list,
access_business_unit_list=access_business_unit_list,
description=description,
storage_type=storage_type,
data=data)
local = st.enum_storage_type_bool(storage_type=storage_type)
self.insert_dataset_db(DataManager, dataset=dataset, local=local)
self.insert_dataset_data_db(DataManager, dataset=dataset, local=local)
self.insert_user_dataset_access_relation_db(DataManager,
dataset=dataset)
self.insert_department_dataset_access_relation_db(DataManager,
dataset=dataset)
return dataset
def update_dataset():
result = {}
body = fl.request.get_json(force=True)
dataset_id = body['dataset_id']
name = body['name']
cleaned = body['cleaned']
# access_user_list = body['access_user_list']
# access_business_unit_list = body['access_business_unit_list']
description = body['description']
storage_type = body['storage_type']
dataset = dm.DataManager.get_dataset_by_id(
dm.DataManager,
dataset_id=dataset_id,
local=st.enum_storage_type_bool(storage_type=storage_type))
# Set the new values/changes in the dataset obkect
dataset.set_name(name=name)
dataset.set_cleaned(cleaned=cleaned)
dataset.set_description(description=description)
# if access_user_list == ""
dm.DataManager.update_dataset(
dm.DataManager,
dataset=dataset,
local=st.enum_storage_type_bool(storage_type=storage_type))
result = DataManagerEndpoints.data_set_to_dict(DataManagerEndpoints,
dataset=dataset)
return fl.jsonify(result), 200
def installPreconverted(url="BIONLP_CORPORA", destPath=None, downloadPath=None, redownload=False, updateLocalSettings=False):
print >> sys.stderr, "---------------", "Downloading preconverted corpora", "---------------"
if destPath == None:
destPath = os.path.join(Settings.DATAPATH, "corpora")
if downloadPath == None:
downloadPath = os.path.join(Settings.DATAPATH, "corpora/download")
Utils.Download.downloadAndExtract(Settings.URL[url], destPath, downloadPath, redownload=redownload)
Settings.setLocal("CORPUS_DIR", destPath, updateLocalSettings)
def install(destPath=None, redownload=False, updateLocalSettings=True):
if hasattr(Settings, "SE10T8_CORPUS"): # Already installed
return
print >> sys.stderr, "---------------", "Downloading the SemEval 2010 Task 8 corpus", "---------------"
if destPath == None:
destPath = os.path.join(Settings.DATAPATH, "resources/SemEval2010_task8_all_data.zip")
Utils.Download.download(Settings.URL["SE10T8_CORPUS"], destPath, addName=False, clear=redownload)
Settings.setLocal("SE10T8_CORPUS", destPath, updateLocalSettings)
def installPreconverted(url="BIONLP_CORPORA", destPath=None, downloadPath=None, redownload=False, updateLocalSettings=False):
print >> sys.stderr, "---------------", "Downloading preconverted corpora", "---------------"
if destPath == None:
destPath = os.path.join(Settings.DATAPATH, "corpora")
if downloadPath == None:
downloadPath = os.path.join(Settings.DATAPATH, "corpora/download")
Utils.Download.downloadAndExtract(Settings.URL[url], destPath, downloadPath, redownload=redownload)
Settings.setLocal("CORPUS_DIR", destPath, updateLocalSettings)
def installEvaluators(destPath=None, downloadPath=None, redownload=False, updateLocalSettings=False):
print >> sys.stderr, "---------------", "Downloading BioNLP Shared Task evaluators", "---------------"
if destPath == None:
destPath = os.path.join(Settings.DATAPATH, "tools/evaluators")
if downloadPath == None:
downloadPath = os.path.join(Settings.DATAPATH, "tools/download")
Utils.Download.downloadAndExtract(Settings.URL["BIONLP11_EVALUATORS"], destPath, downloadPath, redownload=redownload)
Settings.setLocal("BIONLP_EVALUATOR_DIR", destPath, updateLocalSettings)
Settings.setLocal("BIONLP_EVALUATOR_GOLD_DIR", os.path.join(destPath, "gold"), updateLocalSettings)
def installBBData(destPath=None, downloadPath=None, redownload=False, updateLocalSettings=False):
print >>sys.stderr, "---------------", "Downloading TEES data files for BB", "---------------"
print >>sys.stderr, "Bacteria tokens derived from LPSN (http://www.bacterio.cict.fr/)"
if destPath == None:
destPath = os.path.join(Settings.DATAPATH, "resources")
if downloadPath == None:
downloadPath = os.path.join(Settings.DATAPATH, "resources/download")
Utils.Download.downloadAndExtract(Settings.URL["TEES_RESOURCES"], destPath, downloadPath, redownload=redownload)
Settings.setLocal("TEES_RESOURCES", destPath, updateLocalSettings)
def installEvaluators(destPath=None, downloadPath=None, redownload=False, updateLocalSettings=False):
print >> sys.stderr, "---------------", "Downloading BioNLP Shared Task evaluators", "---------------"
if destPath == None:
destPath = os.path.join(Settings.DATAPATH, "tools/evaluators")
if downloadPath == None:
downloadPath = os.path.join(Settings.DATAPATH, "tools/download")
Utils.Download.downloadAndExtract(Settings.URL["BIONLP11_EVALUATORS"], destPath, downloadPath, redownload=redownload)
Settings.setLocal("BIONLP_EVALUATOR_DIR", destPath, updateLocalSettings)
Settings.setLocal("BIONLP_EVALUATOR_GOLD_DIR", os.path.join(destPath, "gold"), updateLocalSettings)
def finalizeInstall(programs, testCommand={}, programDir=None, settings={}, updateLocalSettings=False):
if checkPrograms(programs, testCommand, programDir):
setVariable = updateLocalSettings
else:
print >> sys.stderr, "All programs may not have been installed correctly"
print >> sys.stderr, "Do not use the following settings if not sure:"
setVariable = False
for key in sorted(settings.keys()):
Settings.setLocal(key, settings[key], setVariable)
def installRENData(destPath=None, downloadPath=None, redownload=False, updateLocalSettings=False):
print >> sys.stderr, "---------------", "Downloading TEES data files for REN", "---------------"
print >> sys.stderr, "These files are derived from UniProt bacsu and SubtiWiki"
if destPath == None:
destPath = os.path.join(Settings.DATAPATH, "resources")
if downloadPath == None:
downloadPath = os.path.join(Settings.DATAPATH, "resources/download")
Utils.Download.downloadAndExtract(Settings.URL["TEES_RESOURCES"], destPath, downloadPath, redownload=redownload)
Settings.setLocal("TEES_RESOURCES", destPath, updateLocalSettings)
def installDrugBank(destPath=None, downloadPath=None, redownload=False, updateLocalSettings=False):
print >> sys.stderr, "---------------", "Downloading Drug Bank XML", "---------------"
print >> sys.stderr, "See http://www.drugbank.ca/downloads for conditions of use"
if destPath == None:
destPath = os.path.join(Settings.DATAPATH, "resources")
if downloadPath == None:
downloadPath = os.path.join(Settings.DATAPATH, "resources/download")
filenames = Utils.Download.downloadAndExtract(Settings.URL["DRUG_BANK_XML"], destPath, downloadPath, redownload=redownload)
assert len(filenames) == 1
Settings.setLocal("DRUG_BANK_XML", os.path.join(destPath, filenames[0]), updateLocalSettings)
def installDrugBank(destPath=None, downloadPath=None, redownload=False, updateLocalSettings=False):
print >> sys.stderr, "---------------", "Downloading Drug Bank XML", "---------------"
print >> sys.stderr, "See http://www.drugbank.ca/downloads for conditions of use"
if destPath == None:
destPath = os.path.join(Settings.DATAPATH, "resources")
if downloadPath == None:
downloadPath = os.path.join(Settings.DATAPATH, "resources/download")
filenames = Utils.Download.downloadAndExtract(Settings.URL["DRUG_BANK_XML"], destPath, downloadPath, redownload=redownload)
assert len(filenames) == 1
Settings.setLocal("DRUG_BANK_XML", os.path.join(destPath, filenames[0]), updateLocalSettings)
def __init__(self, type, playdataxml, settingsxml=None):
# Load settings
Settings.SETTINGS = Settings(settingsxml)
# Load game data
print "====> Loading XML:", playdataxml
self._gameSettings = Settings(playdataxml)
self.typeg = type
# Internal data initialization
self.__cityEvolution = CityEvolution()
def finalizeInstall(programs,
testCommand={},
programDir=None,
settings={},
updateLocalSettings=False):
if checkPrograms(programs, testCommand, programDir):
setVariable = updateLocalSettings
else:
print >> sys.stderr, "All programs may not have been installed correctly"
print >> sys.stderr, "Do not use the following settings if not sure:"
setVariable = False
for key in sorted(settings.keys()):
Settings.setLocal(key, settings[key], setVariable)
def create_user(self, first_name: str, last_name: str, email: str,
password: str, business_unit: str, access_rights_pillars, admin, role_manager, operation_issuer):
if self.check_admin(UserManager, operation_issuer):
user_id = "user_" + st.create_id()
password = st.create_hash_password_sha512(
password=password, complementary_input=user_id)
access_rights_pillars_dict = self.__parse_access_rigths_pillars(
UserManager, access_rights_pillars)
user = us.User(userID=user_id, first_name=first_name, last_name=last_name, email=email,
password=password, business_unit=business_unit,
access_rights_pillars=access_rights_pillars_dict, admin=admin,
role_manager=role_manager)
self.insert_user_db(UserManager, user=user)
return user
def installRENData(destPath=None,
downloadPath=None,
redownload=False,
updateLocalSettings=False):
print >> sys.stderr, "---------------", "Downloading TEES data files for REN", "---------------"
print >> sys.stderr, "These files are derived from UniProt bacsu and SubtiWiki"
if destPath == None:
destPath = os.path.join(Settings.DATAPATH, "resources")
if downloadPath == None:
downloadPath = os.path.join(Settings.DATAPATH, "resources/download")
Utils.Download.downloadAndExtract(Settings.URL["TEES_RESOURCES"],
destPath,
downloadPath,
redownload=redownload)
Settings.setLocal("TEES_RESOURCES", destPath, updateLocalSettings)
def installBBData(destPath=None,
downloadPath=None,
redownload=False,
updateLocalSettings=False):
print >> sys.stderr, "---------------", "Downloading TEES data files for BB", "---------------"
print >> sys.stderr, "Bacteria tokens derived from LPSN (http://www.bacterio.cict.fr/)"
if destPath == None:
destPath = os.path.join(Settings.DATAPATH, "resources")
if downloadPath == None:
downloadPath = os.path.join(Settings.DATAPATH, "resources/download")
Utils.Download.downloadAndExtract(Settings.URL["TEES_RESOURCES"],
destPath,
downloadPath,
redownload=redownload)
Settings.setLocal("TEES_RESOURCES", destPath, updateLocalSettings)
def parse_dataset_obj(self, db_row):
if db_row:
dataset_id = db_row[0]
name = db_row[1]
owner = db_row[2]
hash_of_dataset = db_row[3]
cleaned = ds.DataSet.enum_cleaned(ds.DataSet, cleaned=db_row[4])
size = db_row[5]
access_user_list = db_row[6]
access_business_unit_list = db_row[7]
storage_type = db_row[8]
description = db_row[9]
local = st.enum_storage_type_bool(storage_type=storage_type)
data = self.get_table_as_df(DataManager,
table=dataset_id,
local=local)
dataset = ds.DataSet(
datasetID=dataset_id,
name=name,
owner=owner,
size=size,
hash_of_dataset=hash_of_dataset,
cleaned=cleaned,
access_user_list=access_user_list,
access_business_unit_list=access_business_unit_list,
description=description,
storage_type=storage_type,
data=data)
return dataset
pass
def create_architecture_view(self, name, description, components):
architecture_viewID = "architecture_view_" + st.create_id()
architecture_view = av.ArchitectureView(
architecture_viewID, name, description, components)
check_unique = self.insert_architecture_view_db(
ArchitectureViewManager, architecture_view=architecture_view)
return architecture_view, check_unique
def finalizeInstall(programs, testCommand={}, programDir=None, settings={}, updateLocalSettings=False):
installOK = checkPrograms(programs, testCommand, programDir)
if installOK:
setVariable = updateLocalSettings
else:
print >> sys.stderr, "All programs may not have been installed correctly"
print >> sys.stderr, "Do not use the following settings if not sure:"
setVariable = False
for key in sorted(settings.keys()):
if settings[key] != None:
#raise Exception("Local setting " + str(key) + " is undefined")
Settings.setLocal(key, settings[key], setVariable)
else:
print >> sys.stderr, "Warning, local setting " + str(key) + " is undefined"
if not installOK:
raise Exception("Error installing programs: " + ", ".join(programs))
def update_password(self, new_password, user_id):
local = False
new_password = st.create_hash_password_sha512(
password=new_password, complementary_input=user_id)
db_utils.DataBaseUtils.execute_sql(
db_utils.DataBaseUtils, sql_statement=sql_stmt.DataBaseSQL.update_value(
sql_stmt.DataBaseSQL, table=st.TABLE_USER, column=st.TB_USER_COL_PASSWORD, value=new_password,
condition=st.TB_USER_COL_USER_ID, condition_value=user_id), local=local)
def update_header_list(self, dataset_id, header_list):
if isinstance(header_list, list):
header_list = st.make_list_to_str(header_list)
if isinstance(header_list, str):
header_list = header_list.split(',')
header_list_tmp = dm.DataManager.get_dataset_table_headers(
dm.DataManager, dataset_id=dataset_id, local=False)
if not st.check_list_identical(header_list, header_list_tmp):
if header_list[0] == '' and len(header_list) == 1:
header_list = header_list_tmp
elif not all(x in header_list for x in header_list_tmp):
[
header_list.append(val) for val in header_list_tmp
if not val in header_list
]
header_list = st.make_list_to_str(header_list)
return header_list
def enum_role_manger(self, role_manager):
enum_role_manger = {
'role_manager': 1,
'not_role_manager': 0,
True: 1,
False: 0,
1: 1,
0: 0,
str(1): 1,
str(0): 0
}
try:
res = enum_role_manger[role_manager]
except:
res = 0
print('Not Able to identify if User is role manager\n' +
'Line: ' + str(st.get_linenumber_of_occured_error())
+ 'File: ' + str(st.get_filename_of_occured_error()))
return res
def enum_admin(self, admin):
enum_admin = {
'admin': 1,
'not_admin': 0,
True: 1,
False: 0,
1: 1,
0: 0,
str(1): 1,
str(0): 0
}
try:
res = enum_admin[admin]
except:
res = 0
print('Not Able to identify if User is admin\n' +
'Line: ' + str(st.get_linenumber_of_occured_error())
+ 'File: ' + str(st.get_filename_of_occured_error()))
return res
def enum_admin_bool(self, admin):
enum_admin = {
'admin': True,
'not_admin': False,
'True': True,
'False': False,
1: True,
0: False,
'1': True,
'0': False
}
try:
res = enum_admin[admin]
except:
res = 0
print('Not Able to identify if User is admin\n' +
'Line: ' + str(st.get_linenumber_of_occured_error())
+ 'File: ' + str(st.get_filename_of_occured_error()))
return res
def get_dataset(dataset_id, storage_type):
result = {}
local = st.enum_storage_type_bool(storage_type=storage_type)
dataset = dm.DataManager.get_dataset_by_id(dm.DataManager,
dataset_id=dataset_id,
local=local)
if dataset:
result = DataManagerEndpoints.data_set_to_dict(DataManagerEndpoints,
dataset=dataset)
else:
DataManagerEndpoints(404, "DATASET_NOT_FOUND")
return fl.jsonify(result), 200
def finalizeInstall(programs,
testCommand={},
programDir=None,
settings={},
updateLocalSettings=False):
installOK = checkPrograms(programs, testCommand, programDir)
if installOK:
setVariable = updateLocalSettings
else:
print >> sys.stderr, "All programs may not have been installed correctly"
print >> sys.stderr, "Do not use the following settings if not sure:"
setVariable = False
for key in sorted(settings.keys()):
if settings[key] != None:
#raise Exception("Local setting " + str(key) + " is undefined")
Settings.setLocal(key, settings[key], setVariable)
else:
print >> sys.stderr, "Warning, local setting " + str(
key) + " is undefined"
if not installOK:
raise Exception("Error installing programs: " + ", ".join(programs))
def enum_cleaned(self, cleaned):
enum_cleaned = {
'cleaned': 1,
'not_cleaned': 0,
True: 1,
False: 0,
'true': 1,
'false': 0,
'True': 1,
'False': 0,
1: 1,
0: 0,
'1': 1,
'0': 0
}
try:
res = enum_cleaned[cleaned]
except:
res = 0
print('Not Able to identify if DataSet cleaned\n' + 'Line: ' +
str(st.get_linenumber_of_occured_error()) + 'File: ' +
str(st.get_filename_of_occured_error()))
return res
def check_password(self, password_user, user_id):
local = False
result = db_utils.DataBaseUtils.execute_sql(
db_utils.DataBaseUtils, sql_statement=sql_stmt.DataBaseSQL.
select_object_by_condition(sql_stmt.DataBaseSQL, table=st.TABLE_USER, condition=st.TB_USER_COL_USER_ID,
condition_value=user_id), fetchone=True, local=local)
user = self.__parse_user_obj(UserManager, result)
password_db = user.get_password()
password_user = st.create_hash_password_sha512(
password=password_user, complementary_input=user_id)
if password_db == password_user:
return True
else:
return False
def insert_department_db(self, department: str, user_id):
if self.check_admin(UserManager, user_id):
local = False
dep_id = "dep_" + st.create_id()
# Creates Table
db_utils.DataBaseUtils.execute_sql(
db_utils.DataBaseUtils, sql_statement=sql_stmt.DataBaseSQL.create_department_table_sql(sql_stmt.DataBaseSQL), local=False)
# Inserts data
db_utils.DataBaseUtils.execute_sql(db_utils.DataBaseUtils,
sql_statement=sql_stmt.DataBaseSQL.
insert_department_values(
sql_stmt.DataBaseSQL, departmentID=dep_id, department_name=department), local=local)
return True
else:
return False
def setKey(key, value):
if value == None:
print >> sys.stderr, "No value defined for key '" + str(key) + "'"
return
if key not in Settings.KEY_TYPE:
print >> sys.stderr, "Unknown key '" + str(key) + "'"
return
if Settings.KEY_TYPE[key].get("type") == "file":
fullPath = os.path.abspath(value)
if not os.path.exists(os.path.abspath(value)):
print >> sys.stderr, "No file at '" + fullPath + "'"
return
if not os.path.isfile(fullPath):
print >> sys.stderr, "'" + fullPath + "' is not a file"
return
expectedMD5 = Settings.KEY_TYPE[key].get("md5")
if Settings.KEY_TYPE[key].get("md5") != None:
print >> sys.stderr, "Determining MD5 for '" + fullPath + "'"
md5 = FileUtils.getFileMd5(fullPath)
if md5 != expectedMD5:
print >> sys.stderr, "MD5 '" + md5 + "' does not match expected value '" + expectedMD5 + "'"
return
#print >> sys.stderr, "Defining value '" + fullPath + "' for key '" + key + "'"
Settings.setLocal(key, value)
def setKey(key, value):
if value == None:
print >> sys.stderr, "No value defined for key '" + str(key) + "'"
return
if key not in Settings.KEY_TYPE:
print >> sys.stderr, "Unknown key '" + str(key) + "'"
return
if Settings.KEY_TYPE[key].get("type") == "file":
fullPath = os.path.abspath(value)
if not os.path.exists(os.path.abspath(value)):
print >> sys.stderr, "No file at '" + fullPath + "'"
return
if not os.path.isfile(fullPath):
print >> sys.stderr, "'" + fullPath + "' is not a file"
return
expectedMD5 = Settings.KEY_TYPE[key].get("md5")
if Settings.KEY_TYPE[key].get("md5") != None:
print >> sys.stderr, "Determining MD5 for '" + fullPath + "'"
md5 = FileUtils.getFileMd5(fullPath)
if md5 != expectedMD5:
print >> sys.stderr, "MD5 '" + md5 + "' does not match expected value '" + expectedMD5 + "'"
return
#print >> sys.stderr, "Defining value '" + fullPath + "' for key '" + key + "'"
Settings.setLocal(key, value)
def get_all_suitable_cleansers_by_header(self, user_id):
suitable_cleaners = []
cleansers = self.get_all_cleansers(DataCleanser)
datasets = dm.DataManager.get_all_datasets_only_id(
dm.DataManager, user_id)
for cleanser in cleansers:
for dataset_id in datasets:
dataset_header = dm.DataManager.get_dataset_table_headers(
dm.DataManager, dataset_id=dataset_id, local=False)
if cleanser.get_header_list() == st.make_list_to_str(
dataset_header):
suitable_cleaners.append(cleanser)
elif all(x in cleanser.get_header_list()
for x in dataset_header):
suitable_cleaners.append(cleanser)
return list(set(suitable_cleaners))
def auth_user():
email = fl.request.args.get('email', None)
passwd = fl.request.args.get('passwd', None)
result = {}
if email:
user = um.UserManager.get_user_by_email(um.UserManager, email)
else:
UserManagerEndpoints.endpoints_exception(400, "EMAIL_PARAM_NOT_FOUND")
if passwd and st.create_hash_password_sha512(
passwd, user.get_userID()) == user.get_password():
token = us.User.generate_token(us.User, user)
result = {"token": token.decode('utf-8'), "duration": 6600}
else:
UserManagerEndpoints.endpoints_exception(UserManagerEndpoints, 400,
"BAD_OR_MISSING_PASSWORD")
return fl.jsonify(result), 200
def create_cleanser(self,
name,
description,
datasets,
cleanser_operation_types,
local=False):
cleanser_id = 'cleanser_' + st.create_id()
header_list = []
for dataset_id in datasets.split(','):
header_list = self.update_header_list(DataCleanser,
dataset_id=dataset_id,
header_list=header_list)
cleanser = cl.Cleanser(cleanser_id, name, description, header_list,
datasets, cleanser_operation_types)
check = self.insert_cleanser_db(DataCleanser, cleanser)
for dataset_id in datasets.split(','):
self.insert_cleanser_dataset_compatibility_relation_db(
DataCleanser, cleanser, dataset_id)
return cleanser, check
def delete_dataset(dataset_id, storage_type):
result = {}
local = st.enum_storage_type_bool(storage_type=storage_type)
dataset = dm.DataManager.get_dataset_by_id(dm.DataManager,
dataset_id=dataset_id,
local=local)
try:
dm.DataManager.delete_dataset(dm.DataManager,
dataset_id=dataset_id,
local=local)
except:
print('Deleting Dataset from Dataset table unsuccessful')
try:
dm.DataManager.drop_dataset_table(dm.DataManager,
dataset_id=dataset_id,
local=local)
except:
print('Droping Dataset data table unsuccessful')
result = DataManagerEndpoints.data_set_to_dict(DataManagerEndpoints,
dataset=dataset)
return fl.jsonify(result), 200
def get_table_df(dataset_id, storage_type):
result = dm.DataManager.get_table_as_df(
dm.DataManager,
table=dataset_id,
local=st.enum_storage_type_bool(storage_type=storage_type)).to_json()
return fl.jsonify(result), 200
def initLocalSettings(filename):
assert Menu.system.defaultInstallDir != None
Settings.initLocalSettings(filename, Menu.system.defaultInstallDir)
# Reset local settings
os.environ["TEES_SETTINGS"] = filename
reload(Settings)
def create_settings(self):
# The inputs are linked to a tkinter variable. Those values will have to be retrieved from each variable
# and passed on to the settings objects
return Settings(self.pdf_dir.get(), self.json_dir.get(),
self.out_dir.get())
def corpusMenuInitializer(menu, prevMenu):
menu.text = """
The corpora are used for training new models and testing existing
models. The corpora installable here are from the three BioNLP Shared
Tasks (2009, 2011 and 2013) on Event Extraction (organized by
University of Tokyo), and the two Drug-Drug Interaction Extraction
tasks (DDI'11 and 13, organized by Universidad Carlos III de Madrid).
The 2009 and 2011 corpora are downloaded as interaction XML files,
generated from the original Shared Task files. If you need to convert
the corpora from the original files, you can use the convertBioNLP.py,
convertDDI.py and convertDDI13.py programs located at Utils/Convert.
The 2013 corpora will be converted to interaction XML from the official
corpus files, downloaded automatically from the task websites. Installing
the BioNLP'13 corpora will take about 10 minutes.
It is also recommended to download the official BioNLP Shared Task evaluator
programs, which will be used by TEES when training or testing on those corpora.
"""
# Mark "skip" as default option, this will be re-marked as install if a corpus is missing
menu.setDefault("s")
# If CORPUS_DIR setting is not set set it now
if not hasattr(Settings, "CORPUS_DIR") or getattr(Settings, "CORPUS_DIR") == None:
Settings.setLocal("CORPUS_DIR", os.path.join(menu.system.defaultInstallDir, "corpora"))
print >> sys.stderr
# Initialize handlers
handlers = []
handlerArgs = []
corpusInstallPath = os.path.join(menu.system.defaultInstallDir, "corpora")
corpusDownloadPath = os.path.join(menu.system.defaultInstallDir, "corpora/download")
# Check which corpora need to be installed
redownload = menu.optDict["1"].toggle
# 2009-2011 corpora
for corpus in ["GE11", "EPI11", "ID11", "BB11", "BI11", "CO11", "REL11", "REN11"]:
if menu.optDict["2"].toggle or (menu != prevMenu and not checkCorpusInstall(corpus)):
menu.setDefault("i")
menu.optDict["2"].toggle = True
handlers.append(convertBioNLP.installPreconverted)
handlerArgs.append(["BIONLP_11_CORPORA", corpusInstallPath, corpusDownloadPath, redownload, True])
break
if menu.optDict["3"].toggle or (menu != prevMenu and not checkCorpusInstall("GE09")):
menu.setDefault("i")
menu.optDict["3"].toggle = True
handlers.append(convertBioNLP.installPreconverted)
handlerArgs.append(["BIONLP_09_CORPUS", corpusInstallPath, corpusDownloadPath, redownload, True])
if menu.optDict["4"].toggle or (menu != prevMenu and not checkCorpusInstall("DDI11", ("-train.xml", "-devel.xml"))):
menu.setDefault("i")
menu.optDict["4"].toggle = True
handlers.append(convertBioNLP.installPreconverted)
handlerArgs.append(["DDI_11_CORPUS", corpusInstallPath, corpusDownloadPath, redownload, True])
# 2013 corpora
bioNLP13Corpora = ["GE13", "CG13", "PC13", "GRO13", "GRN13", "BB13T2", "BB13T3"]
for corpus in bioNLP13Corpora:
if menu.optDict["5"].toggle or (menu != prevMenu and not checkCorpusInstall(corpus)):
menu.setDefault("i")
menu.optDict["5"].toggle = True
#handlers.append(convertBioNLP.convert)
#handlerArgs.append([bioNLP13Corpora, corpusInstallPath, corpusDownloadPath, redownload, False])
handlers.append(convertBioNLP.installPreconverted)
handlerArgs.append(["BIONLP_13_CORPORA", corpusInstallPath, corpusDownloadPath, redownload, True])
break
if menu.optDict["6"].toggle or (menu != prevMenu and not checkCorpusInstall("DDI13", ("-train.xml",))):
menu.setDefault("i")
menu.optDict["6"].toggle = False #True
handlers.append(convertDDI13.convertDDI13)
handlerArgs.append([corpusInstallPath, corpusDownloadPath, ["DDI13_TRAIN", "DDI13_TEST_TASK_9.1", "DDI13_TEST_TASK_9.2"], redownload])
# A handler for installing BioNLP'11 evaluators
evaluatorInstallPath = os.path.join(menu.system.defaultInstallDir, "tools/evaluators")
evaluatorDownloadPath = os.path.join(menu.system.defaultInstallDir, "tools/download")
if menu.optDict["7"].toggle or (menu != prevMenu and (not hasattr(Settings, "BIONLP_EVALUATOR_DIR") or getattr(Settings, "BIONLP_EVALUATOR_DIR") == None)):
menu.setDefault("i")
menu.optDict["7"].toggle = True
handlers.append(convertBioNLP.installEvaluators)
handlerArgs.append([evaluatorInstallPath, evaluatorDownloadPath, redownload, True])
# Add the handlers to install option
menu.optDict["i"].handler = handlers
menu.optDict["i"].handlerArgs = handlerArgs