def install(destDir=None, downloadDir=None, redownload=False, updateLocalSettings=False): url = Settings.URL["BLLIP_SOURCE"] if downloadDir == None: downloadDir = os.path.join(Settings.DATAPATH) + "/tools/download" if destDir == None: destDir = Settings.DATAPATH + "/tools/BLLIP" items = Download.downloadAndExtract(url, destDir, downloadDir + "/bllip.zip", None, False) print >> sys.stderr, "Installing BLLIP parser" Tool.testPrograms("BLLIP parser", ["make", "flex"], {"flex":"flex --version"}) parserPath = Download.getTopDir(destDir, items) cwd = os.getcwd() os.chdir(parserPath) print >> sys.stderr, "Compiling first-stage parser" subprocess.call("make", shell=True) print >> sys.stderr, "Compiling second-stage parser" subprocess.call("make reranker", shell=True) os.chdir(cwd) print >> sys.stderr, "Installing the McClosky biomedical parsing model" url = "http://bllip.cs.brown.edu/download/bioparsingmodel-rel1.tar.gz" Download.downloadAndExtract(url, destDir, downloadDir, None) bioModelDir = os.path.abspath(destDir + "/biomodel") # Check that everything works Tool.finalizeInstall(["first-stage/PARSE/parseIt", "second-stage/programs/features/best-parses"], {"first-stage/PARSE/parseIt":"first-stage/PARSE/parseIt " + bioModelDir + "/parser/ < /dev/null", "second-stage/programs/features/best-parses":"second-stage/programs/features/best-parses -l " + bioModelDir + "/reranker/features.gz " + bioModelDir + "/reranker/weights.gz < /dev/null"}, parserPath, {"BLLIP_PARSER_DIR":os.path.abspath(parserPath), "MCCLOSKY_BIOPARSINGMODEL_DIR":bioModelDir}, updateLocalSettings)
def prepareGoldForEvaluation(corpus, sourceDir, tempdir, goldDir=None, goldPackage=None): if os.path.exists(os.path.join(tempdir, "gold")): shutil.rmtree(os.path.join(tempdir, "gold")) if goldDir == None: if not hasattr(Settings, "BIONLP_EVALUATOR_GOLD_DIR"): print >> sys.stderr, corpus, "BIONLP_EVALUATOR_GOLD_DIR setting not defined" return None if goldPackage == None: goldPackage = Settings.EVALUATOR[corpus + "_DEVEL-gold"] goldDir = os.path.join(Settings.BIONLP_EVALUATOR_GOLD_DIR, goldPackage) if not os.path.exists(goldDir): print >> sys.stderr, corpus, "Evaluator gold data directory", goldDir, "does not exist" goldDir = None if goldDir != None and goldDir.endswith(".tar.gz") or goldDir.endswith( ".tgz"): goldDir = Download.getTopDir( os.path.join(tempdir, "gold"), Download.extractPackage(goldDir, os.path.join(tempdir, "gold"))) print >> sys.stderr, "Uncompressed evaluation gold to", goldDir if goldDir != None and not hasGoldDocuments(sourceDir, goldDir): print >> sys.stderr, "Evaluation input has no gold documents" goldDir = None return goldDir
def install(destDir=None, downloadDir=None, redownload=False, compile=True, updateLocalSettings=False): print >> sys.stderr, "Installing SVM-Multiclass" if compile: url = Settings.URL["SVM_MULTICLASS_SOURCE"] else: if 8 * struct.calcsize("P") == 32: # check for 32 bit or 64 bit environment (see http://stackoverflow.com/questions/1405913/how-do-i-determine-if-my-python-shell-is-executing-in-32bit-or-64bit-mode-on-os) url = Settings.URL["SVM_MULTICLASS_LINUX_32"] else: url = Settings.URL["SVM_MULTICLASS_LINUX_64"] if downloadDir == None: downloadDir = os.path.join(Settings.DATAPATH, "tools/download/") if destDir == None: destDir = Settings.DATAPATH destDir += "/tools/SVMMultiClass" Download.downloadAndExtract(url, destDir, downloadDir, redownload=redownload) if compile: print >> sys.stderr, "Compiling SVM-Multiclass" Tools.Tool.testPrograms("SVM-Multiclass", ["make"]) subprocess.call("cd " + destDir + "; make", shell=True) Tools.Tool.finalizeInstall(["svm_multiclass_learn", "svm_multiclass_classify"], {"svm_multiclass_learn":"echo | ./svm_multiclass_learn -? > /dev/null", "svm_multiclass_classify":"echo | ./svm_multiclass_classify -? > /dev/null"}, destDir, {"SVM_MULTICLASS_DIR":destDir}, updateLocalSettings)
def install(destDir=None, downloadDir=None, redownload=False, compile=True, updateLocalSettings=False): print >> sys.stderr, "Installing SVM-Multiclass" if compile: url = Settings.URL["SVM_MULTICLASS_SOURCE"] else: url = Settings.URL["SVM_MULTICLASS_LINUX"] if downloadDir == None: downloadDir = os.path.join(Settings.DATAPATH, "tools/download/") if destDir == None: destDir = Settings.DATAPATH destDir += "/tools/SVMMultiClass" Download.downloadAndExtract(url, destDir, downloadDir, redownload=redownload) if compile: print >> sys.stderr, "Compiling SVM-Multiclass" Tools.Tool.testPrograms("SVM-Multiclass", ["make"]) subprocess.call("cd " + destDir + "; make", shell=True) Tools.Tool.finalizeInstall( ["svm_multiclass_learn", "svm_multiclass_classify"], { "svm_multiclass_learn": "echo | ./svm_multiclass_learn -? > /dev/null", "svm_multiclass_classify": "echo | ./svm_multiclass_classify -? > /dev/null" }, destDir, {"SVM_MULTICLASS_DIR": destDir}, updateLocalSettings)
def checkEvaluator(corpus, sourceDir, goldDir = None): # Check evaluator if not hasattr(Settings, "BIONLP_EVALUATOR_DIR"): print >> sys.stderr, corpus, "BIONLP_EVALUATOR_DIR setting not defined" evaluatorDir = None else: evaluatorDir = os.path.join(Settings.BIONLP_EVALUATOR_DIR, Settings.EVALUATOR[corpus]) # Check source data tempdir = tempfile.mkdtemp() if sourceDir.endswith(".tar.gz"): Download.extractPackage(sourceDir, os.path.join(tempdir, "source")) sourceDir = os.path.join(tempdir, "source") else: #if corpus in ("GE09", "BB11", "BI11"): # GE09 a2 files have to be renamed and relation identifier "R" has to be replaced with "E" for the BB11 and BI11 relations. # X-lines have to be removed from all tasks shutil.copytree(sourceDir, os.path.join(tempdir, "source")) sourceDir = os.path.join(tempdir, "source") # Filter extra data removeXLines(sourceDir) # Check gold data print >> sys.stderr, "Using devel set gold for evaluation" goldDir = prepareGoldForEvaluation(corpus, sourceDir, tempdir, goldDir) if goldDir == None and Settings.EVALUATOR.has_key(corpus + "_TEST-gold"): print >> sys.stderr, "Using test set gold for evaluation" goldDir = prepareGoldForEvaluation(corpus, sourceDir, tempdir, goldDir, Settings.EVALUATOR[corpus + "_TEST-gold"]) # Use absolute paths sourceDir = os.path.abspath(sourceDir) if evaluatorDir != None: evaluatorDir = os.path.abspath(evaluatorDir) if goldDir != None: goldDir = os.path.abspath(goldDir) if tempdir != None: tempdir = os.path.abspath(tempdir) return evaluatorDir, sourceDir, goldDir, tempdir
def install(self, destDir=None, downloadDir=None, redownload=False, updateLocalSettings=False): url = Settings.URL["BLLIP_SOURCE"] if downloadDir == None: downloadDir = os.path.join(Settings.DATAPATH) + "/tools/download" if destDir == None: destDir = Settings.DATAPATH + "/tools/BLLIP" items = Download.downloadAndExtract(url, destDir, downloadDir + "/bllip.zip", None, False) print >> sys.stderr, "Installing BLLIP parser" Tool.testPrograms("BLLIP parser", ["make", "flex"], {"flex":"flex --version"}) parserPath = Download.getTopDir(destDir, items) cwd = os.getcwd() os.chdir(parserPath) print >> sys.stderr, "Compiling first-stage parser" subprocess.call("make", shell=True) print >> sys.stderr, "Compiling second-stage parser" subprocess.call("make reranker", shell=True) os.chdir(cwd) print >> sys.stderr, "Installing the McClosky biomedical parsing model" url = "http://bllip.cs.brown.edu/download/bioparsingmodel-rel1.tar.gz" Download.downloadAndExtract(url, destDir, downloadDir, None) bioModelDir = os.path.abspath(destDir + "/biomodel") # Check that everything works Tool.finalizeInstall(["first-stage/PARSE/parseIt", "second-stage/programs/features/best-parses"], {"first-stage/PARSE/parseIt":"first-stage/PARSE/parseIt " + bioModelDir + "/parser/ < /dev/null", "second-stage/programs/features/best-parses":"second-stage/programs/features/best-parses -l " + bioModelDir + "/reranker/features.gz " + bioModelDir + "/reranker/weights.gz < /dev/null"}, parserPath, {"BLLIP_PARSER_DIR":os.path.abspath(parserPath), "MCCLOSKY_BIOPARSINGMODEL_DIR":bioModelDir}, updateLocalSettings)
def install(destDir=None, downloadDir=None, redownload=False): url = Settings.URL["GENIA_SENTENCE_SPLITTER"] packageName = "geniass" if downloadDir == None: downloadDir = os.path.join(Settings.DATAPATH, "tools/download/") if destDir == None: destDir = os.path.join(Settings.DATAPATH, "tools/") Download.downloadAndExtract(url, packageName, destDir, downloadDir)
def test(progDir): cwd = os.getcwd() os.chdir(progDir) print >> sys.stderr, "Testing svm_multiclass_learn...", trainOK = Download.checkReturnCode(os.system("echo | ./svm_multiclass_learn -? > /dev/null")) print >> sys.stderr, "Testing svm_multiclass_classify...", classifyOK = Download.checkReturnCode(os.system("echo | ./svm_multiclass_classify -? > /dev/null")) os.chdir(cwd) return trainOK and classifyOK
def test(progDir): cwd = os.getcwd() os.chdir(progDir) print >> sys.stderr, "Testing svm_multiclass_learn...", trainOK = Download.checkReturnCode( os.system("echo | ./svm_multiclass_learn -? > /dev/null")) print >> sys.stderr, "Testing svm_multiclass_classify...", classifyOK = Download.checkReturnCode( os.system("echo | ./svm_multiclass_classify -? > /dev/null")) os.chdir(cwd) return trainOK and classifyOK
def install(self, destDir=None, downloadDir=None, redownload=False, updateLocalSettings=False): print >> sys.stderr, "Installing Stanford Parser" if downloadDir == None: downloadDir = os.path.join(Settings.DATAPATH, "tools/download/") if destDir == None: destDir = os.path.join(Settings.DATAPATH, "tools/") items = Download.downloadAndExtract(Settings.URL["STANFORD_PARSER"], destDir, downloadDir) stanfordPath = Download.getTopDir(destDir, items) Tool.finalizeInstall(["stanford-parser.jar"], {"stanford-parser.jar":"java -cp stanford-parser.jar edu.stanford.nlp.trees.EnglishGrammaticalStructure"}, stanfordPath, {"STANFORD_PARSER_DIR":stanfordPath}, updateLocalSettings)
def install(destDir=None, downloadDir=None, redownload=False, updateLocalSettings=False): print >> sys.stderr, "Installing Stanford Parser" if downloadDir == None: downloadDir = os.path.join(Settings.DATAPATH, "tools/download/") if destDir == None: destDir = os.path.join(Settings.DATAPATH, "tools/") items = Download.downloadAndExtract(Settings.URL["STANFORD_PARSER"], destDir, downloadDir) stanfordPath = Download.getTopDir(destDir, items) Tool.finalizeInstall(["stanford-parser.jar"], {"stanford-parser.jar":"java -cp stanford-parser.jar:slf4j-api.jar edu.stanford.nlp.trees.EnglishGrammaticalStructure"}, stanfordPath, {"STANFORD_PARSER_DIR":stanfordPath}, updateLocalSettings)
def install(destDir=None, downloadDir=None, redownload=False): print >> sys.stderr, "Installing BANNER" url = Settings.URL["BANNER_SOURCE"] if downloadDir == None: downloadDir = os.path.join(Settings.DATAPATH, "tools/download") if destDir == None: destDir = Settings.DATAPATH Download.downloadAndExtract(url, destDir + "/tools/BANNER", downloadDir + "/banner.tar.gz", "trunk", False, redownload=redownload) print >> sys.stderr, "Compiling BANNER with ANT" subprocess.call("cd " + destDir + "/tools/BANNER; export JAVA_HOME=/usr/lib/jvm/java-6-openjdk; ant -f build_ext.xml", shell=True)
def install(destDir=None, downloadDir=None, redownload=False): print >> sys.stderr, "Installing BioNLP'11 evaluators" settings = {} if downloadDir == None: downloadDir = Settings.DATAPATH if destDir == None: destDir = Settings.DATAPATH for corpus in ["GE", "BB", "BI", "CO"]: print >> sys.stderr, "Installing BioNLP'11", corpus, "evaluator" settings[corpus + "_EVALUATOR"] = Download.getTopDir(destDir + "/tools/evaluators/", Download.downloadAndExtract(Settings.URL[corpus + "_EVALUATOR"], destDir + "/tools/evaluators/", downloadDir + "/tools/download/")) print >> sys.stderr, "Installing BioNLP'11", corpus, "evaluator gold data" Download.downloadAndExtract(Settings.URL[corpus + "_DEVEL"], destDir + "/tools/evaluators/gold/" + corpus + "-devel", downloadDir + "/corpora/BioNLP11-original/corpus/", os.path.basename(Settings.URL[corpus + "_DEVEL"])[:-len(".tar.gz")]) return settings
def checkEvaluator(corpus, sourceDir, goldDir=None): # Check evaluator if not hasattr(Settings, "BIONLP_EVALUATOR_DIR"): print >> sys.stderr, corpus, "BIONLP_EVALUATOR_DIR setting not defined" evaluatorDir = None else: evaluatorDir = os.path.join(Settings.BIONLP_EVALUATOR_DIR, Settings.EVALUATOR[corpus]) # Check source data tempdir = None if sourceDir.endswith(".tar.gz"): tempdir = tempfile.mkdtemp() Download.extractPackage(sourceDir, os.path.join(tempdir, "source")) sourceDir = os.path.join(tempdir, "source") else: #if corpus in ("GE09", "BB11", "BI11"): # GE09 a2 files have to be renamed and relation identifier "R" has to be replaced with "E" for the BB11 and BI11 relations. # X-lines have to be removed from all tasks tempdir = tempfile.mkdtemp() shutil.copytree(sourceDir, os.path.join(tempdir, "source")) sourceDir = os.path.join(tempdir, "source") # Filter extra data removeXLines(sourceDir) # Check gold data if goldDir == None: if not hasattr(Settings, "BIONLP_EVALUATOR_GOLD_DIR"): print >> sys.stderr, corpus, "BIONLP_EVALUATOR_GOLD_DIR setting not defined" return evaluatorDir, None goldDir = os.path.join(Settings.BIONLP_EVALUATOR_GOLD_DIR, Settings.EVALUATOR[corpus + "-gold"]) if not os.path.exists(goldDir): print >> sys.stderr, corpus, "Evaluator gold data directory", goldDir, "does not exist" goldDir = None if goldDir != None and goldDir.endswith(".tar.gz"): if tempdir == None: tempdir = tempfile.mkdtemp() goldDir = Download.getTopDir( os.path.join(tempdir, "gold"), Download.extractPackage(goldDir, os.path.join(tempdir, "gold"))) print >> sys.stderr, "Uncompressed evaluation gold to", goldDir if goldDir != None and not hasGoldDocuments(sourceDir, goldDir): print >> sys.stderr, "Evaluation input has no gold documents" goldDir = None # Use absolute paths sourceDir = os.path.abspath(sourceDir) if evaluatorDir != None: evaluatorDir = os.path.abspath(evaluatorDir) if goldDir != None: goldDir = os.path.abspath(goldDir) if tempdir != None: tempdir = os.path.abspath(tempdir) return evaluatorDir, sourceDir, goldDir, tempdir
def checkEvaluator(corpus, sourceDir, goldDir = None): # Check evaluator if not hasattr(Settings, "BIONLP_EVALUATOR_DIR"): print >> sys.stderr, corpus, "BIONLP_EVALUATOR_DIR setting not defined" evaluatorDir = None else: evaluatorDir = os.path.join(Settings.BIONLP_EVALUATOR_DIR, Settings.EVALUATOR[corpus]) # Check source data tempdir = None if sourceDir.endswith(".tar.gz"): tempdir = tempfile.mkdtemp() Download.extractPackage(sourceDir, os.path.join(tempdir, "source")) sourceDir = os.path.join(tempdir, "source") else: #if corpus in ("GE09", "BB11", "BI11"): # GE09 a2 files have to be renamed and relation identifier "R" has to be replaced with "E" for the BB11 and BI11 relations. # X-lines have to be removed from all tasks tempdir = tempfile.mkdtemp() shutil.copytree(sourceDir, os.path.join(tempdir, "source")) sourceDir = os.path.join(tempdir, "source") # Filter extra data removeXLines(sourceDir) # Check gold data if goldDir == None: if not hasattr(Settings, "BIONLP_EVALUATOR_GOLD_DIR"): print >> sys.stderr, corpus, "BIONLP_EVALUATOR_GOLD_DIR setting not defined" return evaluatorDir, None goldDir = os.path.join(Settings.BIONLP_EVALUATOR_GOLD_DIR, Settings.EVALUATOR[corpus + "-gold"]) print >> sys.stderr, "Found gold data directory", goldDir if not os.path.exists(goldDir): print >> sys.stderr, corpus, "Evaluator gold data directory", goldDir, "does not exist" goldDir = None if goldDir != None and goldDir.endswith(".tar.gz"): if tempdir == None: tempdir = tempfile.mkdtemp() goldDir = Download.getTopDir(os.path.join(tempdir, "gold"), Download.extractPackage(goldDir, os.path.join(tempdir, "gold"))) print >> sys.stderr, "Uncompressed evaluation gold to", goldDir if goldDir != None and not hasGoldDocuments(sourceDir, goldDir): print >> sys.stderr, "Evaluation input has no gold documents" goldDir = None # Use absolute paths sourceDir = os.path.abspath(sourceDir) if evaluatorDir != None: evaluatorDir = os.path.abspath(evaluatorDir) if goldDir != None: goldDir = os.path.abspath(goldDir) if tempdir != None: tempdir = os.path.abspath(tempdir) return evaluatorDir, sourceDir, goldDir, tempdir
def install(destDir=None, downloadDir=None, redownload=False, updateLocalSettings=False): print >> sys.stderr, "Installing GENIA Sentence Splitter" if downloadDir == None: downloadDir = os.path.join(Settings.DATAPATH, "tools/download/") if destDir == None: destDir = os.path.join(Settings.DATAPATH, "tools/geniass") Download.downloadAndExtract(Settings.URL["GENIA_SENTENCE_SPLITTER"], destDir, downloadDir, "geniass") print >> sys.stderr, "Compiling GENIA Sentence Splitter" Tool.testPrograms("Genia Sentence Splitter", ["make", "ruby"]) cwd = os.getcwd() os.chdir(destDir) print >> sys.stderr, "Compiling Genia Sentence Splitter" subprocess.call("make", shell=True) os.chdir(cwd) Tool.finalizeInstall(["./run_geniass.sh"], {"./run_geniass.sh":"./run_geniass.sh README /dev/null " + Settings.RUBY_PATH}, destDir, {"GENIA_SENTENCE_SPLITTER_DIR":destDir}, updateLocalSettings)
def checkEvaluator(corpus, sourceDir, goldDir=None): # Check evaluator if not hasattr(Settings, "BIONLP_EVALUATOR_DIR"): print >> sys.stderr, corpus, "BIONLP_EVALUATOR_DIR setting not defined" evaluatorDir = None else: evaluatorDir = os.path.join(Settings.BIONLP_EVALUATOR_DIR, Settings.EVALUATOR[corpus]) # Check source data tempdir = None if sourceDir.endswith(".tar.gz"): tempdir = tempfile.mkdtemp() Download.extractPackage(sourceDir, os.path.join(tempdir, "source")) sourceDir = os.path.join(tempdir, "source") elif corpus == "GE09": # a2 files have to be renamed tempdir = tempfile.mkdtemp() shutil.copytree(sourceDir, os.path.join(tempdir, "source")) sourceDir = os.path.join(tempdir, "source") # Check gold data if goldDir == None: if not hasattr(Settings, "BIONLP_EVALUATOR_GOLD_DIR"): print >> sys.stderr, corpus, "BIONLP_EVALUATOR_GOLD_DIR setting not defined" return evaluatorDir, None goldDir = os.path.join(Settings.BIONLP_EVALUATOR_GOLD_DIR, Settings.EVALUATOR[corpus + "-gold"]) if not os.path.exists(goldDir): print >> sys.stderr, corpus, "Evaluator gold data directory", goldDir, "does not exist" goldDir = None if goldDir != None and goldDir.endswith(".tar.gz"): if tempdir == None: tempdir = tempfile.mkdtemp() goldDir = Download.getTopDir( os.path.join(tempdir, "gold"), Download.extractPackage(goldDir, os.path.join(tempdir, "gold")) ) print >> sys.stderr, "Uncompressed evaluation gold to", goldDir if goldDir != None and not hasGoldDocuments(sourceDir, goldDir): print >> sys.stderr, "Evaluation input has no gold documents" goldDir = None # Use absolute paths sourceDir = os.path.abspath(sourceDir) if evaluatorDir != None: evaluatorDir = os.path.abspath(evaluatorDir) if goldDir != None: goldDir = os.path.abspath(goldDir) if tempdir != None: tempdir = os.path.abspath(tempdir) return evaluatorDir, sourceDir, goldDir, tempdir
def install(destDir=None, downloadDir=None, redownload=False, compile=True): print >> sys.stderr, "Installing SVM-Multiclass" if compile: url = Settings.URL["SVM_MULTICLASS_SOURCE"] else: url = Settings.URL["SVM_MULTICLASS_LINUX"] if downloadDir == None: downloadDir = os.path.join(Settings.DATAPATH, "tools/download/") if destDir == None: destDir = Settings.DATAPATH destDir += "/tools/SVMMultiClass" Download.downloadAndExtract(url, destDir, downloadDir, redownload=redownload) if compile: print >> sys.stderr, "Compiling SVM-Multiclass" subprocess.call("cd " + destDir + "; make", shell=True) test(destDir)
def install(destDir=None, downloadDir=None, redownload=False, compile=False, javaHome=None, updateLocalSettings=False): print >> sys.stderr, "Installing BANNER" if downloadDir == None: downloadDir = os.path.join(Settings.DATAPATH, "tools/download") if destDir == None: destDir = Settings.DATAPATH if compile: Download.downloadAndExtract(Settings.URL["BANNER_SOURCE"], destDir + "/tools/BANNER", downloadDir + "/banner.tar.gz", "trunk", False, redownload=redownload) print >> sys.stderr, "Compiling BANNER with ANT" Tool.testPrograms("BANNER", ["ant"], {"ant":"ant -version"}) #/usr/lib/jvm/java-6-openjdk if javaHome == None or javaHome.strip() == "": subprocess.call("cd " + destDir + "/tools/BANNER; ant -f build_ext.xml", shell=True) else: subprocess.call("cd " + destDir + "/tools/BANNER; export JAVA_HOME=" + javaHome + "; ant -f build_ext.xml", shell=True) else: print >> sys.stderr, "Downloading precompiled BANNER" Download.downloadAndExtract(Settings.URL["BANNER_COMPILED"], destDir + "/tools", downloadDir, redownload=redownload) Tool.finalizeInstall([], None, destDir + "/tools/BANNER", {"BANNER_DIR":destDir + "/tools/BANNER"}, updateLocalSettings)
def prepareGoldForEvaluation(corpus, sourceDir, tempdir, goldDir=None, goldPackage=None): if os.path.exists(os.path.join(tempdir, "gold")): shutil.rmtree(os.path.join(tempdir, "gold")) if goldDir == None: if not hasattr(Settings, "BIONLP_EVALUATOR_GOLD_DIR"): print >> sys.stderr, corpus, "BIONLP_EVALUATOR_GOLD_DIR setting not defined" return None if goldPackage == None: goldPackage = Settings.EVALUATOR[corpus + "_DEVEL-gold"] goldDir = os.path.join(Settings.BIONLP_EVALUATOR_GOLD_DIR, goldPackage) if not os.path.exists(goldDir): print >> sys.stderr, corpus, "Evaluator gold data directory", goldDir, "does not exist" goldDir = None if goldDir != None and goldDir.endswith(".tar.gz") or goldDir.endswith(".tgz"): goldDir = Download.getTopDir(os.path.join(tempdir, "gold"), Download.extractPackage(goldDir, os.path.join(tempdir, "gold"))) print >> sys.stderr, "Uncompressed evaluation gold to", goldDir if goldDir != None and not hasGoldDocuments(sourceDir, goldDir): print >> sys.stderr, "Evaluation input has no gold documents" goldDir = None return goldDir
def install(destDir=None, downloadDir=None, redownload=False, compile=False, javaHome=None, updateLocalSettings=False): print >> sys.stderr, "Installing BANNER" if downloadDir == None: downloadDir = os.path.join(Settings.DATAPATH, "tools/download") if destDir == None: destDir = Settings.DATAPATH if compile: Download.downloadAndExtract(Settings.URL["BANNER_SOURCE"], destDir + "/tools/BANNER", downloadDir + "/banner.tar.gz", "trunk", False, redownload=redownload) print >> sys.stderr, "Compiling BANNER with ANT" Tool.testPrograms("BANNER", ["ant"], {"ant": "ant -version"}) #/usr/lib/jvm/java-6-openjdk if javaHome == None or javaHome.strip() == "": subprocess.call("cd " + destDir + "/tools/BANNER; ant -f build_ext.xml", shell=True) else: subprocess.call("cd " + destDir + "/tools/BANNER; export JAVA_HOME=" + javaHome + "; ant -f build_ext.xml", shell=True) else: print >> sys.stderr, "Downloading precompiled BANNER" Download.downloadAndExtract(Settings.URL["BANNER_COMPILED"], destDir + "/tools", downloadDir, redownload=redownload) Tool.finalizeInstall([], None, destDir + "/tools/BANNER", {"BANNER_DIR": destDir + "/tools/BANNER"}, updateLocalSettings)
def install(destDir=None, downloadDir=None, redownload=False, compile=True, updateLocalSettings=False): print >> sys.stderr, "Installing SVM-Multiclass" if compile: url = Settings.URL["SVM_MULTICLASS_SOURCE"] else: url = Settings.URL["SVM_MULTICLASS_LINUX"] if downloadDir == None: downloadDir = os.path.join(Settings.DATAPATH, "tools/download/") if destDir == None: destDir = Settings.DATAPATH destDir += "/tools/SVMMultiClass" Download.downloadAndExtract(url, destDir, downloadDir, redownload=redownload) if compile: print >> sys.stderr, "Compiling SVM-Multiclass" Tools.Tool.testPrograms("SVM-Multiclass", ["make"]) subprocess.call("cd " + destDir + "; make", shell=True) Tools.Tool.finalizeInstall(["svm_multiclass_learn", "svm_multiclass_classify"], {"svm_multiclass_learn":"echo | ./svm_multiclass_learn -? > /dev/null", "svm_multiclass_classify":"echo | ./svm_multiclass_classify -? > /dev/null"}, destDir, {"SVM_MULTICLASS_DIR":destDir}, updateLocalSettings)
def install(destDir=None, downloadDir=None, redownload=False): print >> sys.stderr, "Installing BioNLP'11 evaluators" settings = {} if downloadDir == None: downloadDir = Settings.DATAPATH if destDir == None: destDir = Settings.DATAPATH for corpus in ["GE11", "BB11", "BI11", "CO11"]: print >> sys.stderr, "Installing BioNLP'11", corpus, "evaluator" settings[corpus + "_EVALUATOR"] = Download.getTopDir( destDir + "/tools/evaluators/", Download.downloadAndExtract(Settings.URL[corpus + "_EVALUATOR"], destDir + "/tools/evaluators/", downloadDir + "/tools/download/")) print >> sys.stderr, "Installing BioNLP'11", corpus, "evaluator gold data" Download.downloadAndExtract( Settings.URL[corpus + "_DEVEL"], destDir + "/tools/evaluators/gold/" + corpus + "-devel", downloadDir + "/corpora/BioNLP11-original/corpus/", os.path.basename(Settings.URL[corpus + "_DEVEL"])[:-len(".tar.gz")]) return settings
def install(destDir=None, downloadDir=None, redownload=False): url = Settings.URL["BLLIP_SOURCE"] if downloadDir == None: downloadDir = os.path.join(Settings.DATAPATH) if destDir == None: destDir = Settings.DATAPATH items = Download.downloadAndExtract( url, destDir + "/tools/BLLIP", downloadDir + "/tools/download/bllip.zip", None, False) # Install the parser # topDirs = [] # for item in items: # if item.endswith("/") and item.count("/") == 1: # topDirs.append(item) # assert len(topDirs) == 1 # parserPath = os.path.join(destDir + "/tools/BLLIP", topDirs[0]) # cwd = os.getcwd() # os.chdir(parserPath) # subprocess.call("make") # os.chdir(cwd) url = "http://bllip.cs.brown.edu/download/bioparsingmodel-rel1.tar.gz" Download.downloadAndExtract(url, destDir + "/tools/BLLIP", downloadDir + "/tools/download/", None)
def install(destDir=None, downloadDir=None, redownload=False, compile=True, updateLocalSettings=False): print >> sys.stderr, "Installing SVM-Multiclass" if compile: url = Settings.URL["SVM_MULTICLASS_SOURCE"] else: if 8 * struct.calcsize( "P" ) == 32: # check for 32 bit or 64 bit environment (see http://stackoverflow.com/questions/1405913/how-do-i-determine-if-my-python-shell-is-executing-in-32bit-or-64bit-mode-on-os) url = Settings.URL["SVM_MULTICLASS_LINUX_32"] else: url = Settings.URL["SVM_MULTICLASS_LINUX_64"] if downloadDir == None: downloadDir = os.path.join(Settings.DATAPATH, "tools/download/") if destDir == None: destDir = Settings.DATAPATH destDir += "/tools/SVMMultiClass" Download.downloadAndExtract(url, destDir, downloadDir, redownload=redownload) if compile: print >> sys.stderr, "Compiling SVM-Multiclass" Tools.Tool.testPrograms("SVM-Multiclass", ["make"]) subprocess.call("cd " + destDir + "; make", shell=True) Tools.Tool.finalizeInstall( ["svm_multiclass_learn", "svm_multiclass_classify"], { "svm_multiclass_learn": "echo | ./svm_multiclass_learn -? > /dev/null", "svm_multiclass_classify": "echo | ./svm_multiclass_classify -? > /dev/null" }, destDir, {"SVM_MULTICLASS_DIR": destDir}, updateLocalSettings)
def checkEvaluator(corpus, sourceDir, goldDir=None): # Check evaluator if not hasattr(Settings, "BIONLP_EVALUATOR_DIR"): print >> sys.stderr, corpus, "BIONLP_EVALUATOR_DIR setting not defined" evaluatorDir = None else: evaluatorDir = os.path.join(Settings.BIONLP_EVALUATOR_DIR, Settings.EVALUATOR[corpus]) # Check source data tempdir = tempfile.mkdtemp() if sourceDir.endswith(".tar.gz"): Download.extractPackage(sourceDir, os.path.join(tempdir, "source")) sourceDir = os.path.join(tempdir, "source") else: #if corpus in ("GE09", "BB11", "BI11"): # GE09 a2 files have to be renamed and relation identifier "R" has to be replaced with "E" for the BB11 and BI11 relations. # X-lines have to be removed from all tasks shutil.copytree(sourceDir, os.path.join(tempdir, "source")) sourceDir = os.path.join(tempdir, "source") # Filter extra data removeXLines(sourceDir) # Check gold data print >> sys.stderr, "Using devel set gold for evaluation" goldDir = prepareGoldForEvaluation(corpus, sourceDir, tempdir, goldDir) if goldDir == None and Settings.EVALUATOR.has_key(corpus + "_TEST-gold"): print >> sys.stderr, "Using test set gold for evaluation" goldDir = prepareGoldForEvaluation( corpus, sourceDir, tempdir, goldDir, Settings.EVALUATOR[corpus + "_TEST-gold"]) # Use absolute paths sourceDir = os.path.abspath(sourceDir) if evaluatorDir != None: evaluatorDir = os.path.abspath(evaluatorDir) if goldDir != None: goldDir = os.path.abspath(goldDir) if tempdir != None: tempdir = os.path.abspath(tempdir) return evaluatorDir, sourceDir, goldDir, tempdir
def install(destDir=None, downloadDir=None, redownload=False): url = URL["STANFORD_PARSER"] packageName = url.split("/")[-1].split(".")[0] # Download if downloadDir == None: downloadDir = os.path.join(Settings.DATAPATH, "tools/download/") downloadFile = Download.download(url, downloadDir, clear=redownload) # Prepare destination if destDir == None: destDir = os.path.join(Settings.DATAPATH, "tools/") installDir = os.path.join(destDir, packageName) if os.path.exists(installDir): print >> sys.stderr, "Removing existing installation at", installDir shutil.rmtree(installDir) # Unpack print >> sys.stderr, "Extracting", downloadFile, "to", destDir f = tarfile.open(downloadFile, 'r:gz') f.extractall(destDir) f.close()