def _init_jvm_options():
import imglyb_config
import jnius_config
PAINTERA_ENDPOINT = 'org.janelia.saalfeldlab:paintera:{}'.format(
__paintera_version__)
RELEVANT_MAVEN_REPOS = {
'imagej.public': 'https://maven.imagej.net/content/groups/public',
'saalfeldlab': 'https://saalfeldlab.github.io/maven'
}
imglyb_config.add_endpoints(
PAINTERA_ENDPOINT, '{}'.format(
os.getenv('PAINTERA_SLF4J_BINDING',
'org.slf4j:slf4j-simple:1.7.25')))
imglyb_config.add_repositories(RELEVANT_MAVEN_REPOS)
JVM_OPTIONS_STR = 'JVM_OPTIONS'
if JVM_OPTIONS_STR in os.environ:
jnius_config.add_options(*os.environ[JVM_OPTIONS_STR].split(' '))
jnius_config.add_classpath(__jfx_rt_jar)
return jnius_config
def _cli(self):
##### jnius
import jnius_config
LIQUIBASE_CLASSPATH: list = [
self.liquibase_dir + "/liquibase.jar",
self.liquibase_lib_dir + "/*",
self.liquibase_internal_dir + "/*",
self.liquibase_internal_lib_dir + "/*",
self.jdbc_drivers_dir + "/*"]
if self.additional_classpath:
LIQUIBASE_CLASSPATH.append(self.additional_classpath)
if not jnius_config.vm_running:
jnius_config.add_classpath(*LIQUIBASE_CLASSPATH)
else:
log.warning("VM is already running, can't set classpath/options")
log.debug("classpath: %s" % jnius_config.get_classpath())
from jnius import JavaClass, MetaJavaClass, JavaMethod
#####
class LiquibaseCommandLine(JavaClass, metaclass=MetaJavaClass):
__javaclass__ = "liquibase/integration/commandline/LiquibaseCommandLine"
# methods
execute = JavaMethod("([Ljava/lang/String;)I")
return LiquibaseCommandLine()
def __init__(self, skip_errors=False, log_file=None, keep_unbalanced_ions=False, id_tag='Reaction_ID',
action_on_isotopes=0, keep_reagents=False, logger=None, ignore_mapping=False, jvm_path=None,
rdkit_dearomatization=False, remove_unchanged_parts=True, skip_tautomerize=False,
jchem_path=None, add_reagents_to_reactants=False) -> None:
if logger is None:
self.logger = self._config_log(log_file, logger_name='logger')
else:
self.logger = logger
self._skip_errors = skip_errors
self._keep_unbalanced_ions = keep_unbalanced_ions
self._id_tag = id_tag
self._action_on_isotopes = action_on_isotopes
self._keep_reagents = keep_reagents
self._ignore_mapping = ignore_mapping
self._remove_unchanged_parts_flag = remove_unchanged_parts
self._skip_tautomerize = skip_tautomerize
self._dearomatize_by_rdkit = rdkit_dearomatization
self._reagents_to_reactants = add_reagents_to_reactants
if not skip_tautomerize:
if jvm_path:
os.environ['JDK_HOME'] = jvm_path
os.environ['JAVA_HOME'] = jvm_path
os.environ['PATH'] += f';{PurePosixPath(jvm_path).joinpath("bin").joinpath("server")};' \
f'{PurePosixPath(jvm_path).joinpath("bin").joinpath("server")};'
if jchem_path:
import jnius_config
jnius_config.add_classpath(jchem_path)
from jnius import autoclass
Standardizer = autoclass('chemaxon.standardizer.Standardizer')
self._Molecule = autoclass('chemaxon.struc.Molecule')
self._MolHandler = autoclass('chemaxon.util.MolHandler')
self._standardizer = Standardizer('tautomerize')
def test_add_classpath(self):
assert jnius_config.vm_running is False
assert jnius_config.classpath is None
jnius_config.add_classpath(".")
assert jnius_config.classpath == ["."]
jnius_config.add_classpath("/usr/local/fem/plugins/*")
assert jnius_config.classpath == [".", "/usr/local/fem/plugins/*"]
def __init__(self, model_name=os.path.join(os.path.dirname(__file__), 'purepos/szeged.model'), morphology=None,
source_fields=None, target_fields=None):
if not jnius_config.vm_running:
jnius_config.add_classpath(PurePOS.class_path)
self._autoclass = import_pyjnius()
self._params = {}
self._model_name = model_name
# We have to use it later...
self._java_string_class = self._autoclass('java.lang.String')
self._java_list_class = self._autoclass('java.util.ArrayList')
self._java_pair_class = self._autoclass('org.apache.commons.lang3.tuple.Pair')
self._java_anal_item_class = self._autoclass('hu.ppke.itk.nlpg.purepos.common.TAnalysisItem')
self._model_jfile = self._java_string_class(self._model_name.encode('UTF-8'))
self.morphology = morphology
self._model = None
self._tagger = None
# Field names for e-magyar TSV
if source_fields is None:
source_fields = set()
if target_fields is None:
target_fields = []
self.source_fields = source_fields
self.target_fields = target_fields
def add_classpath(self, *claspath):
if jnius_config.vm_running:
raise ValueError(
"VM is already running, can't set classpath/options; VM started at %s"
% jnius_config.vm_started_at)
jnius_config.add_classpath(*claspath)
log.info("VM Classpath: %s" % jnius_config.get_classpath())
def setup(classpath, java_home=None):
if java_home is not None:
os.environ['JAVA_HOME'] = java_home
# os.environ['CLASSPATH'] = classpath
# os.environ['PATH'] = "C:\Program Files\Java\jre1.8.0_74\bin\server;" + os.environ['PATH']
import jnius_config
for i in classpath:
jnius_config.add_classpath(i)
def JVM_setup():
# jnius_config.add_options('-Xmx512m')
OceanDir = "C:\Program Files\Ocean Optics\OmniDriver\OOI_HOME"
lsj = [
i for i in listdir(OceanDir)
if isfile(join(OceanDir, i)) and i.endswith(".jar")
]
for j in lsj:
jnius_config.add_classpath(join(OceanDir, j))
def set_lshvec_jar_path(jar_path):
"""
Set jlshvec jar library path
"""
if jnius_config.vm_running:
logger.warn("jvm is running, cannot set java path")
else:
LSHVec.lshvec_jar_path = jar_path
jnius_config.add_classpath(jar_path)
def __new__(cls, *args, **kwargs):
if cls.__standardizer is None: # load only once
from jnius_config import add_classpath
add_classpath(*jars)
from jnius import autoclass
cls.__standardizer = autoclass(
'chemaxon.standardizer.Standardizer')
cls.__importer = autoclass('chemaxon.formats.MolImporter')
cls.__exporter = autoclass('chemaxon.formats.MolExporter')
return super().__new__(cls)
def add_classpath(path):
"""
Appends items to the classpath for the JVM to use.
Replaces any existing classpath, overriding the CLASSPATH environment variable.
"""
import jnius_config
# JDK 1.6以后好像可以不配置CLASSPATH了，现在为了这个配置上了。
if ':' + path in os.environ['CLASSPATH']:
return
# os.environ['CLASSPATH'] += ':' + path
jnius_config.add_classpath(path)
def _init_jvm():
import scyjava_config
import jnius_config
import jgo
if jnius_config.vm_running:
_logger.warning(
'JVM is already running, will not add endpoints to classpath -- required classes might not be on classpath..'
)
import jnius
return jnius
PYJNIUS_JAR_STR = 'PYJNIUS_JAR'
if PYJNIUS_JAR_STR not in globals():
try:
PYJNIUS_JAR = os.environ[PYJNIUS_JAR_STR]
jnius_config.add_classpath(PYJNIUS_JAR)
except KeyError as e:
if e.args[0] == PYJNIUS_JAR_STR:
_logger.error(
'Unable to import scyjava: %s environment variable not defined.',
PYJNIUS_JAR_STR)
else:
raise e
return None
endpoints = scyjava_config.get_endpoints()
repositories = scyjava_config.get_repositories()
_logger.debug('Adding jars from endpoints %s', endpoints)
if len(endpoints) > 0:
endpoints = endpoints[:1] + sorted(endpoints[1:])
_logger.debug('Using endpoints %s', endpoints)
_, workspace = jgo.resolve_dependencies(
'+'.join(endpoints),
m2_repo=scyjava_config.get_m2_repo(),
cache_dir=scyjava_config.get_cache_dir(),
repositories=repositories,
verbose=scyjava_config.get_verbose())
jnius_config.add_classpath(os.path.join(workspace, '*'))
try:
import jnius
return jnius
except KeyError as e:
if e.args[0] == 'JAVA_HOME':
_logger.error(
'Unable to import scyjava: JAVA_HOME environment variable not defined, cannot import jnius.'
)
else:
raise e
return None
def addClasspath(self, *paths):
"""
Appends items to the classpath for the JVM to use.
Replaces any existing classpath, overriding the CLASSPATH environment variable.
:type paths: list
"""
super().addClasspath(*paths)
try:
for path in paths:
jnius_config.add_classpath(path)
except ValueError as e:
raise JVMError(e)
def configure_classpath(anserini_root="."):
"""
Parameters
----------
anserini_root : str
(Optional) path to root anserini directory.
"""
paths = glob.glob(os.path.join(anserini_root, 'anserini-*-fatjar.jar'))
if not paths:
raise Exception('No matching jar file found in {}'.format(os.path.abspath(anserini_root)))
latest = max(paths, key=os.path.getctime)
jnius_config.add_classpath(latest)
def __init__(self, classpath, java_home=None):
if java_home is not None:
os.environ['JAVA_HOME'] = java_home
# os.environ['CLASSPATH'] = classpath
import jnius_config
for path in classpath:
jnius_config.add_classpath(path)
from jnius import autoclass # import jnius here so that CLASSPATH and JAVA_HOME can be set beforehand
self.ByteArrayInputStream = autoclass('java.io.ByteArrayInputStream')
self.ImageIO = autoclass('javax.imageio.ImageIO')
self.ByteBuffer = autoclass('java.nio.ByteBuffer')
self.BufferedImageLuminanceSource = autoclass('com.google.zxing.client.j2se.BufferedImageLuminanceSource')
self.HybridBinarizer = autoclass('com.google.zxing.common.HybridBinarizer')
self.BinaryBitmap = autoclass('com.google.zxing.BinaryBitmap')
self.MultiFormatReader = autoclass('com.google.zxing.MultiFormatReader')
def gen_client(
account, pwd,
clientcert_path=None, clientcert_passphase=None,
truststore_path=None, truststore_passphase=None,
java_home=None, class_path=None
):
try:
if java_home:
logging.debug('Setting JAVA_HOME to {}'.format(java_home))
os.environ['JAVA_HOME'] = java_home
except:
logging.warning('Exception when setting JAVA_HOME.')
logging.warning(traceback.format_exc())
try:
if class_path:
logging.debug(
'Adding CLASSPATH to {}'.format(
','.join(class_path)
)
)
jnius_config.add_classpath(*class_path)
except:
logging.warning('Exception when setting CLASSPATH.')
logging.warning(traceback.format_exc())
try:
from jnius import autoclass
GisacClient = autoclass('gisac.api.GisacClient')
except:
logging.error('Exception when setting autoclass GisacClient.')
logging.error(traceback.format_exc())
raise
try:
client = GisacClient(
clientcert_path, clientcert_passphase,
truststore_path, truststore_passphase,
account, pwd
)
except:
logging.error('Exception when creating GisacClient object.')
logging.error(traceback.format_exc())
raise
return client
def __init__(self):
"""
Constructor to set up local pipeline
@param: file_name, the file name of the custom config file
"""
super(LocalPipeline, self).__init__()
self.PipelineFactory = None
self.pipeline = None
self.ProtobufSerializer = None
pipeline_config.log_current_config(self.config, False)
# set up JVM and load classes needed
try:
import jnius_config
jnius_config.add_options('-Xmx16G')
jnius_config.add_classpath(get_model_path() + '/*')
except Exception as e:
logger.warning(
"Couldn't set JVM config; this might be because you're setting up the multiple instances of the local pipeline."
)
logger.warning(str(e))
try:
from jnius import autoclass
self.PipelineFactory = autoclass(
'edu.illinois.cs.cogcomp.pipeline.main.PipelineFactory')
self.SerializationHelper = autoclass(
'edu.illinois.cs.cogcomp.core.utilities.SerializationHelper')
# self.ProtobufSerializer = autoclass('edu.illinois.cs.cogcomp.core.utilities.protobuf.ProtobufSerializer')
self.Boolean = autoclass('java.lang.Boolean')
self.JString = autoclass('java.lang.String')
except Exception as e:
logger.error(
'Fail to load models, please check if your Java version is up to date.'
)
logger.error(str(e))
return None
self.pipeline = self.PipelineFactory.buildPipelineWithAllViews(
self.Boolean(True))
logger.info("pipeline has been set up")
def gen_client(account,
pwd,
clientcert_path=None,
clientcert_passphase=None,
truststore_path=None,
truststore_passphase=None,
java_home=None,
class_path=None):
try:
if java_home:
logging.debug('Setting JAVA_HOME to {}'.format(java_home))
os.environ['JAVA_HOME'] = java_home
except:
logging.warning('Exception when setting JAVA_HOME.')
logging.warning(traceback.format_exc())
try:
if class_path:
logging.debug('Adding CLASSPATH to {}'.format(
','.join(class_path)))
jnius_config.add_classpath(*class_path)
except:
logging.warning('Exception when setting CLASSPATH.')
logging.warning(traceback.format_exc())
try:
from jnius import autoclass
GisacClient = autoclass('gisac.api.GisacClient')
except:
logging.error('Exception when setting autoclass GisacClient.')
logging.error(traceback.format_exc())
raise
try:
client = GisacClient(clientcert_path, clientcert_passphase,
truststore_path, truststore_passphase, account,
pwd)
except:
logging.error('Exception when creating GisacClient object.')
logging.error(traceback.format_exc())
raise
return client
def uw_sectioner(documents):
#assert(documents)
cwd = os.path.dirname(__file__)
jnius_config.add_classpath(os.path.join("./Java/*"))
jnius_config.add_classpath(os.path.join(cwd,os.path.join("Java","*")))
out_file="/nethome/Clinical_Data/NER_Training_Corpus/sectioned",
Chunker = autoclass('edu.uw.bhi.bionlp.sectionchunker.decode.App')
chunker = Chunker()
model_path_and_prefix = os.path.join(cwd, "Model","uw-bhi-bionlp-clinical-record-section-chunker")
log_file = os.path.join(cwd, "Log", "log.sectionchunker.out")
config_file = os.path.join(cwd, "Config", "sample_decode_config.xml")
args = ["--model-path-and-prefix", model_path_and_prefix,
"--input-format", "TEXT",
"--input-directory", "/home/wlane/nethome/Clinical_Data/NER_Training_Corpus/txt",
"--output-directory", "/nethome/Clinical_Data/NER_Training_Corpus/sectioned",
"--log-file", log_file,
"--logging",
"--config-file", config_file
]
chunker.main(args)
return out_file
def find_most_violated_constraint(x, y, sm, sparm):
"""Return ybar associated with x's most violated constraint.
Returns the label ybar for pattern x corresponding to the most
violated constraint according to SVM^struct cost function. To
find which cost function you should use, check sparm.loss_type for
whether this is slack or margin rescaling (1 or 2 respectively),
and check sparm.slack_norm for whether the slack vector is in an
L1-norm or L2-norm in the QP (1 or 2 respectively).
If this function is not implemented, this function is equivalent
to 'classify(x, sm, sparm)'. The optimality guarantees of
Tsochantaridis et al. no longer hold, since this doesn't take the
loss into account at all, but it isn't always a terrible
approximation. One still technically maintains the empirical
risk bound condition, but without any regularization.
score = classify_example(x,sm,sparm)
discy, discny = y*score, -y*score + 1
if discy > discny: return y
return -y"""
import os
import os.path as path
import jnius_config
from jnius import autoclass
global x_to_file_dictionary
jnius_config.add_classpath(os.path.dirname(__file__) + "/*")
print jnius_config.expand_classpath()
System = autoclass('java.lang.System')
print System.getProperty('java.class.path')
IHT = autoclass('edu.albany.cs.ssvmCSD.IHT_Bridge')
file_name = x_to_file_dictionary[str(x)]
print 'constrain func:', file_name
iht_ins = IHT(
path.abspath(path.join(__file__, "../../../../../..")) + "/data/" +
file_name)
ybar = iht_ins.getX()
print 'ybar:-', ybar
return ybar
def __init__(self, classpath=None):
''' This is a kluge to allow autoclass & find_javaclass to be 'private' functions
of this class. In reality, they are global to the geomesa_jnius_setup namespace'''
global _autoclass, _cast, _detach, _find_javaclass, _JavaClass, _JavaMethod, _JavaObject, _MetaJavaClass
if classpath is not None:
java_classpath = os.path.expanduser(classpath)
os.environ['CLASSPATH'] = java_classpath
jar_list = test_jnius.GetFileList(java_classpath, ["jar"])
for jar in jar_list:
jnius_config.add_classpath(jar)
try:
from jnius import autoclass as _autoclass
from jnius import cast as _cast
from jnius import detach as _detach
from jnius import find_javaclass as _find_javaclass
from jnius import JavaClass as _JavaClass
from jnius import JavaMethod as _JavaMethod
from jnius import JavaObject as _JavaObject
from jnius import MetaJavaClass as _MetaJavaClass
except:
print(
"Error: the jnius package has not been installed. Please install it with:"
)
print("\t\tsudo -H pip install jnius\n\n")
_detach()
sys.exit(-1)
if self.testClasspath():
self.classpath_status = True
print("Your classpath is valid.")
else:
print(
"Error:there are errors in your classpath; refer to the logged tests above."
)
_detach()
sys.exit(-1)
def _init_jvm_options():
import jnius_config
import os
PYJNIUS_JAR_STR = 'PYJNIUS_JAR'
IMGLYB_JAR_STR = 'IMGLYB_JAR'
if PYJNIUS_JAR_STR not in globals():
try:
PYJNIUS_JAR = os.environ[PYJNIUS_JAR_STR]
except KeyError as e:
print(
"Path to pyjnius.jar not defined! Use environment variable {} to define it."
.format(PYJNIUS_JAR_STR))
raise e
if IMGLYB_JAR_STR not in globals():
try:
IMGLYB_JAR = os.environ[IMGLYB_JAR_STR]
except KeyError as e:
print(
"Path to imglib2-imglyb jar not defined! Use environment variable {} to define it."
.format(IMGLYB_JAR_STR))
raise e
if 'classpath' in globals():
jnius_config.add_classpath(classpath)
CLASSPATH_STR = 'CLASSPATH'
if CLASSPATH_STR in os.environ:
for path in os.environ[CLASSPATH_STR].split(jnius_config.split_char):
jnius_config.add_classpath(path)
jnius_config.add_classpath(PYJNIUS_JAR)
jnius_config.add_classpath(IMGLYB_JAR)
JVM_OPTIONS_STR = 'JVM_OPTIONS'
if JVM_OPTIONS_STR in os.environ:
jnius_config.add_options(*os.environ[JVM_OPTIONS_STR].split(' '))
return jnius_config
def set_jnius_config():
try:
import jnius_config
path = get_dir()
if path[-1] == '*':
jnius_config.add_classpath(path)
elif os.path.isfile(path):
jnius_config.add_classpath(path)
else:
path = os.path.join(path, '*')
jnius_config.add_classpath(path)
# Further options can be set by individual projects
except ImportError:
warnings.warn('Pyjnius not installed.')
#-*- coding:utf-8 -*-
import jnius_config
jnius_config.add_classpath('.', 'alipaySdk-20160120.jar', 'paydemoactivity.jar', 'libammsdk.jar')
from jnius import autoclass, cast
from toast import toast
from tools import *
PythonActivity = autoclass('org.renpy.android.PythonActivity')
context = cast('android.app.Activity', PythonActivity.mActivity)
'''
PythonActivity = autoclass('org.renpy.android.PythonActivity')
Intent = autoclass('android.content.Intent')
Uri = autoclass('android.net.Uri')
Class = autoclass('java.lang.Class')
Bundle = autoclass('android.os.Bundle')
String = autoclass('java.lang.String')
current = cast('android.app.Activity', PythonActivity.mActivity)
#AndroidManifest.xml
#<uses-permission android:name='android.permission.INTERNET' />
#<activity android:name='com.alipay.sdk.pay.demo.H5PayDemoActivity' />
#modify: .buildozer/android/platform/python-for-android/dist/myapp/templates/AndroidManifest.tmpl.xml
#delete: setContentView(R.layout.pay_main);
def h5paydemo():
H5PayDemoActivity = autoclass('com.alipay.sdk.pay.demo.H5PayDemoActivity')
s = ''
try:
intent = Intent(current, Class.forName('com.alipay.sdk.pay.demo.H5PayDemoActivity'))
'''
python test_pyltvae_fixed_var.py --lr 0.0001 --stepwise_em_lr 0.01 --epochs 20 --everyepochs 10 --model model/sdae.pt --name pyltvae
acc: 0.86318, nmi: 0.81245
'''
import sys
sys.path.append("..")
sys.path.append("../../pyLTM")
import os
# os.environ["JAVA_HOME"] = "/usr/local/packages/j2se-8"
import jnius_config
jnius_config.add_options('-Xms1024m', '-Xmx2048m')
jnius_config.add_classpath('../pltm.jar','../../JAR/colt/lib/colt.jar',
'../../JAR/colt/lib/concurrent.jar','../../JAR/commons-cli-1.2/commons-cli-1.2.jar',
'../../JAR/commons-math3-3.6.1/commons-math3-3.6.1.jar')
import torch
import torch.utils.data
from torchvision import datasets, transforms
import numpy as np
# from lib.ltvae_pyltm import LTVAE
from lib.ltvae_pyltm_fixed_var import LTVAE
from lib.plt_pyltm import learn_latentTree, learn_pouchlatentTree
from lib.datasets import MNIST
from lib.utils import saveToArff
import argparse
from lib.utils import init_logging
import logging
import time
import warnings
warnings.filterwarnings('error')
import click
import yaml
from bigquery_etl.view.generate_stable_views import get_stable_table_schemas
stable_views = None
try:
import jnius_config # noqa: E402
if not jnius_config.vm_running:
# this has to run before jnius is imported the first time
root = Path(__file__).parent.parent / "target" / "dependency"
for path in root.glob("*.jar"):
jnius_config.add_classpath(path.resolve().as_posix())
except ImportError:
# ignore so this module can be imported safely without java installed
pass
def extract_table_references(sql: str) -> List[str]:
"""Return a list of tables referenced in the given SQL."""
# import jnius here so this module can be imported safely without java installed
import jnius # noqa: E402
try:
Analyzer = jnius.autoclass("com.google.zetasql.Analyzer")
AnalyzerOptions = jnius.autoclass("com.google.zetasql.AnalyzerOptions")
except jnius.JavaException:
# replace jnius.JavaException because it's not available outside this function
def add_classpath(*path):
jnius_config.add_classpath(*path)
def add_classpath(path):
try:
import jnius_config
jnius_config.add_classpath(path)
except ImportError:
warnings.warn('Pyjnius not installed.')
def init(version=None,
mem=None,
packages=[],
jvm_opts=[],
redirect_io=True,
logging='WARN',
home_dir=None,
boot_packages=[]):
"""
Function necessary to be called before Terrier classes and methods can be used.
Loads the Terrier.jar file and imports classes. Also finds the correct version of Terrier to download if no version is specified.
Args:
version(str): Which version of Terrier to download. Default=None.
If None, find the newest Terrier version in maven and download it.
mem(str): Maximum memory allocated for java heap in MB. Default is 1/4 of physical memory.
packages(list(str)): Extra maven package coordinates files to load. Default=[]. More information at https://github.com/terrier-org/terrier-core/blob/5.x/doc/terrier_develop.md
jvm_opts(list(str)): Extra options to pass to the JVM. Default=[].
redirect_io(boolean): If True, the Java System.out and System.err will be redirected to Pythons sys.out and sys.err. Default=True.
logging(str): the logging level to use.
Can be one of 'INFO', 'DEBUG', 'TRACE', 'WARN', 'ERROR'. The latter is the quietest.
Default='WARN'.
home_dir(str): the home directory to use. Default to PYTERRIER_HOME environment variable.
"""
global ApplicationSetup
global properties
global firstInit
global file_path
global HOME_DIR
# we keep a local directory
if home_dir is not None:
HOME_DIR = home_dir
if "PYTERRIER_HOME" in os.environ:
HOME_DIR = os.environ["PYTERRIER_HOME"]
else:
from os.path import expanduser
userhome = expanduser("~")
HOME_DIR = os.path.join(userhome, ".pyterrier")
if not os.path.exists(HOME_DIR):
os.mkdir(HOME_DIR)
# get the initial classpath for the JVM
classpathTrJars = setup_terrier(HOME_DIR,
version,
boot_packages=boot_packages)
# Import pyjnius and other classes
import jnius_config
for jar in classpathTrJars:
jnius_config.add_classpath(jar)
if jvm_opts is not None:
for opt in jvm_opts:
jnius_config.add_options(opt)
if mem is not None:
jnius_config.add_options('-Xmx' + str(mem) + 'm')
from jnius import autoclass, cast
properties = autoclass('java.util.Properties')()
ApplicationSetup = autoclass('org.terrier.utility.ApplicationSetup')
from .batchretrieve import BatchRetrieve, FeaturesBatchRetrieve
from .utils import Utils
from .index import Indexer, FilesIndexer, TRECCollectionIndexer, DFIndexer, DFIndexUtils, IndexingType
from .pipelines import LTR_pipeline, XGBoostLTR_pipeline
# Make imports global
globals()["autoclass"] = autoclass
globals()["cast"] = cast
globals()["ApplicationSetup"] = ApplicationSetup
global rewrite
global anserini
global pipelines
global index
global transformer
rewrite = importlib.import_module('.rewrite', package='pyterrier')
anserini = importlib.import_module('.anserini', package='pyterrier')
pipelines = importlib.import_module('.pipelines', package='pyterrier')
index = importlib.import_module('.index', package='pyterrier')
transformer = importlib.import_module('.transformer', package='pyterrier')
# append the python helpers
if packages is None:
packages = []
# Import other java packages
if packages != []:
pkgs_string = ",".join(packages)
properties.put("terrier.mvn.coords", pkgs_string)
ApplicationSetup.bootstrapInitialisation(properties)
if redirect_io:
# this ensures that the python stdout/stderr and the Java are matched
redirect_stdouterr()
setup_logging(logging)
setup_jnius()
globals()["BatchRetrieve"] = BatchRetrieve
globals()["Indexer"] = Indexer
globals()["FeaturesBatchRetrieve"] = FeaturesBatchRetrieve
globals()["TRECCollectionIndexer"] = TRECCollectionIndexer
globals()["FilesIndexer"] = FilesIndexer
globals()["DFIndexer"] = DFIndexer
globals()["DFIndexUtils"] = DFIndexUtils
globals()["Utils"] = Utils
globals()["LTR_pipeline"] = LTR_pipeline
globals()["XGBoostLTR_pipeline"] = XGBoostLTR_pipeline
globals()["IndexFactory"] = autoclass(
"org.terrier.structures.IndexFactory")
globals()["IndexRef"] = autoclass("org.terrier.querying.IndexRef")
globals()["IndexingType"] = IndexingType
firstInit = True
import inspect, os import jnius_config thisfile = inspect.getfile(inspect.currentframe()) testdir = os.path.dirname(os.path.abspath(thisfile)) jnius_config.add_classpath(testdir)
# repository: https://github.com/asad/ReactionDecoder
#############################################################################
from multiprocessing import Pool
from scripts.standardizer import Standardizer
def __run(i):
standardizer = Standardizer(skip_errors=True,
keep_unbalanced_ions=False,
id_tag='Reaction_ID',
log_file='./Log_' + str(i) + '.txt',
keep_reagents=False,
ignore_mapping=True,
action_on_isotopes=2)
data = standardizer.standardize_file(input_file='./chunk_' + str(i) +
'.smi', )
standardizer.write(output_file='./clean_' + str(i) + '.rdf', data=data)
standardizer.logger.handlers.pop().close()
if __name__ == '__main__':
import jnius_config
jnius_config.add_classpath("/home/opt/chemaxon/jchemsuite/lib/jchem.jar")
n_threads = 45
p = Pool(
processes=n_threads) # Number of threads that can be used in parallel
n_chunks = 14843
p.map(__run, [i for i in range(0, n_chunks)]) # Enumerate the chunks
def init(version=None,
mem=None,
packages=[],
jvm_opts=[],
redirect_io=True,
logging='WARN',
home_dir=None,
boot_packages=[],
tqdm=None):
"""
Function necessary to be called before Terrier classes and methods can be used.
Loads the Terrier .jar file and imports classes. Also finds the correct version of Terrier to download if no version is specified.
Args:
version(str): Which version of Terrier to download. Default is `None`.
* If None, find the newest Terrier released version in MavenCentral and download it.
* If `"snapshot"`, will download the latest build from `Jitpack <https://jitpack.io/>`_.
mem(str): Maximum memory allocated for the Java virtual machine heap in MB. Corresponds to java `-Xmx` commandline argument. Default is 1/4 of physical memory.
boot_packages(list(str)): Extra maven package coordinates files to load before starting Java. Default=`[]`. There is more information about loading packages in the `Terrier documentation <https://github.com/terrier-org/terrier-core/blob/5.x/doc/terrier_develop.md>`_
packages(list(str)): Extra maven package coordinates files to load, using the Terrier classloader. Default=`[]`. See also `boot_packages` above.
jvm_opts(list(str)): Extra options to pass to the JVM. Default=`[]`. For instance, you may enable Java assertions by setting `jvm_opts=['-ea']`
redirect_io(boolean): If True, the Java `System.out` and `System.err` will be redirected to Pythons sys.out and sys.err. Default=True.
logging(str): the logging level to use:
* Can be one of `'INFO'`, `'DEBUG'`, `'TRACE'`, `'WARN'`, `'ERROR'`. The latter is the quietest.
* Default is `'WARN'`.
home_dir(str): the home directory to use. Default to PYTERRIER_HOME environment variable.
tqdm: The `tqdm <https://tqdm.github.io/>`_ instance to use for progress bars within PyTerrier. Defaults to tqdm.tqdm. Available options are `'tqdm'`, `'auto'` or `'notebook'`.
**Locating the Terrier .jar file:** PyTerrier is not tied to a specific version of Terrier and will automatically locate and download a recent Terrier .jar file. However, inevitably, some functionalities will require more recent Terrier versions.
* If set, PyTerrier uses the `version` init kwarg to determine the .jar file to look for.
* If the `version` init kwarg is not set, Terrier will query MavenCentral to determine the latest Terrier release.
* If `version` is set to `"snapshot"`, the latest .jar file build derived from the `Terrier Github repository <https://github.com/terrier-org/terrier-core/>`_ will be downloaded from `Jitpack <https://jitpack.io/>`_.
* Otherwise the local (`~/.mvn`) and MavenCentral repositories are searched for the jar file at the given version.
In this way, the default setting is to download the latest release of Terrier from MavenCentral. The user is also able to use a locally installed copy in their private Maven repository, or track the latest build of Terrier from Jitpack.
"""
set_tqdm(tqdm)
global ApplicationSetup
global properties
global firstInit
global file_path
global HOME_DIR
# we keep a local directory
if home_dir is not None:
HOME_DIR = home_dir
elif "PYTERRIER_HOME" in os.environ:
HOME_DIR = os.environ["PYTERRIER_HOME"]
else:
from os.path import expanduser
userhome = expanduser("~")
HOME_DIR = os.path.join(userhome, ".pyterrier")
if not os.path.exists(HOME_DIR):
os.mkdir(HOME_DIR)
# get the initial classpath for the JVM
classpathTrJars = setup_terrier(HOME_DIR,
version,
boot_packages=boot_packages)
# Import pyjnius and other classes
import jnius_config
for jar in classpathTrJars:
jnius_config.add_classpath(jar)
if jvm_opts is not None:
for opt in jvm_opts:
jnius_config.add_options(opt)
if mem is not None:
jnius_config.add_options('-Xmx' + str(mem) + 'm')
from jnius import autoclass, cast
# we only accept Java version 11 and newer; so anything starting 1. or 9. is too old
java_version = autoclass("java.lang.System").getProperty("java.version")
if java_version.startswith("1.") or java_version.startswith("9."):
raise RuntimeError(
"Pyterrier requires Java 11 or newer, we only found Java version %s;"
+
" install a more recent Java, or change os.environ['JAVA_HOME'] to point to the proper Java installation",
java_version)
tr_version = autoclass('org.terrier.Version')
version_string = tr_version.VERSION
if "BUILD_DATE" in dir(tr_version):
version_string += " (built by %s on %s)" % (tr_version.BUILD_USER,
tr_version.BUILD_DATE)
print("PyTerrier %s has loaded Terrier %s" % (__version__, version_string))
properties = autoclass('java.util.Properties')()
ApplicationSetup = autoclass('org.terrier.utility.ApplicationSetup')
from .batchretrieve import BatchRetrieve, FeaturesBatchRetrieve
from .utils import Utils
from .datasets import get_dataset, list_datasets
from .index import Indexer, FilesIndexer, TRECCollectionIndexer, DFIndexer, DFIndexUtils, IterDictIndexer, FlatJSONDocumentIterator, IndexingType
from .pipelines import LTR_pipeline, XGBoostLTR_pipeline, Experiment
# Make imports global
globals()["autoclass"] = autoclass
globals()["cast"] = cast
globals()["ApplicationSetup"] = ApplicationSetup
for sub_module_name in [
'anserini', 'apply', 'cache', 'index', 'io', 'model', 'new', 'ltr',
'pipelines', 'rewrite', 'text', 'transformer'
]:
globals()[sub_module_name] = importlib.import_module(
'.' + sub_module_name, package='pyterrier')
# append the python helpers
if packages is None:
packages = []
# Import other java packages
if packages != []:
pkgs_string = ",".join(packages)
properties.put("terrier.mvn.coords", pkgs_string)
ApplicationSetup.bootstrapInitialisation(properties)
if redirect_io:
# this ensures that the python stdout/stderr and the Java are matched
redirect_stdouterr()
_logging(logging)
setup_jnius()
globals()["get_dataset"] = get_dataset
globals()["list_datasets"] = list_datasets
globals()["Experiment"] = Experiment
globals()["BatchRetrieve"] = BatchRetrieve
globals()["Indexer"] = Indexer
globals()["FeaturesBatchRetrieve"] = FeaturesBatchRetrieve
globals()["TRECCollectionIndexer"] = TRECCollectionIndexer
globals()["FilesIndexer"] = FilesIndexer
globals()["DFIndexer"] = DFIndexer
globals()["DFIndexUtils"] = DFIndexUtils
globals()["IterDictIndexer"] = IterDictIndexer
globals()["FlatJSONDocumentIterator"] = FlatJSONDocumentIterator
globals()["Utils"] = Utils
globals()["LTR_pipeline"] = LTR_pipeline
globals()["XGBoostLTR_pipeline"] = XGBoostLTR_pipeline
globals()["IndexFactory"] = autoclass(
"org.terrier.structures.IndexFactory")
globals()["IndexRef"] = autoclass("org.terrier.querying.IndexRef")
globals()["IndexingType"] = IndexingType
firstInit = True
import jnius_config
jnius_config.add_classpath(
'jedai-core/jedai-core-3.0-jar-with-dependencies.jar')
from jnius import autoclass
from timeit import default_timer as timer
# Pipeline
# F1-Extended Sorted Neighborhood
# F2-Block Filtering
# F3-Comparison Propagation
# F4-Group Linkage
# F5-Ricochet SR Clustering
# AmazonProducts attributes GoogleProducts attributes PerfectMapping attributes
# "id", "id", "idAmazon",
# "title", "name", "idGoogleBase"
# "description", "description",
# "manufacturer", "manufacturer",
# "price" "price"
# 1362 rows 3226 rows 1300 rows
def loadFile(filepath):
"""
Load CSV file (separator ','; attribute names in first row)
Return CSV Entity
"""
dataReader = autoclass(
'org.scify.jedai.datareader.entityreader.EntityCSVReader')
csvReader = dataReader(filepath)
csvReader.setAttributeNamesInFirstRow(True)
def JVM_setup():
# jnius_config.add_options('-Xmx512m')
OceanDir = "C:\Program Files (x86)\Ocean Optics\OmniDriver\OOI_HOME"
lsj = [i for i in listdir(OceanDir) if isfile(join(OceanDir,i)) and i.endswith(".jar")]
for j in lsj:
jnius_config.add_classpath(join(OceanDir,j))
import matplotlib.patches as mpatches
try:
import modest_image
except ImportError:
modest_image = None
# Patch np.fft to use pyfftw so skimage utilities can benefit.
np.fft = pyfftw.interfaces.numpy_fft
if not jnius_config.vm_running:
pkg_root = pathlib.Path(__file__).parent.resolve()
bf_jar_path = pkg_root / 'jars' / 'loci_tools.jar'
if not bf_jar_path.exists():
raise RuntimeError("loci_tools.jar missing from distribution"
" (expected it at %s)" % bf_jar_path)
jnius_config.add_classpath(str(bf_jar_path))
import jnius
JString = jnius.autoclass('java.lang.String')
DebugTools = jnius.autoclass('loci.common.DebugTools')
IFormatReader = jnius.autoclass('loci.formats.IFormatReader')
MetadataStore = jnius.autoclass('loci.formats.meta.MetadataStore')
ServiceFactory = jnius.autoclass('loci.common.services.ServiceFactory')
OMEXMLService = jnius.autoclass('loci.formats.services.OMEXMLService')
ChannelSeparator = jnius.autoclass('loci.formats.ChannelSeparator')
UNITS = jnius.autoclass('ome.units.UNITS')
# Another workaround for pyjnius #300 (see below).
DebugTools.enableLogging(JString("ERROR"))
