def process_data(pos, model_path, sentences):
os.environ["CLASSPATH"] = model_path
import jnius_config
jnius_config.add_options("-Xmx4056M")
from jnius import autoclass
enum = autoclass(
"org.lambda3.text.simplification.discourse.processing.ProcessingType"
)
model = autoclass(
"org.lambda3.text.simplification.discourse.processing.DiscourseSimplifier"
)()
results = {}
for id, sentence in tqdm(sentences.items(),
position=pos,
mininterval=1):
try:
with suppress_stdout():
result = model.doDiscourseSimplification(
str(sentence), enum.SEPARATE).serializeToJSON()
o_id = ray.put(json.decode(result))
results[id] = o_id
except:
logger.error(f"Unable to parse {sentence}")
results[id] = None
return results
def initialize(self):
# we need to delay the initialization of Java in order for this class to
# properly work with multiprocessing
if not OpenNLPSentenceTokenizer._java_initialized:
import jnius_config
jnius_config.add_options("-Xrs")
jnius_config.set_classpath(
pkg_resources.resource_filename(
__name__, "/resources/opennlp-tools-1.5.3.jar"))
OpenNLPSentenceTokenizer._java_initialized = True
from jnius import autoclass
File = autoclass("java.io.File")
SentenceModel = autoclass("opennlp.tools.sentdetect.SentenceModel")
SentenceDetectorME = autoclass(
"opennlp.tools.sentdetect.SentenceDetectorME")
sentence_model_file = pkg_resources.resource_filename(
__name__, "resources/en-sent.bin")
sentence_model = SentenceModel(File(sentence_model_file))
self._tokenizer = SentenceDetectorME(sentence_model)
self._initialized = True
def test_add_options(self):
assert jnius_config.vm_running is False
assert jnius_config.options == []
jnius_config.add_options("option1", "option2")
assert jnius_config.options == ["option1", "option2"]
jnius_config.add_options("option3")
assert jnius_config.options == ["option1", "option2", "option3"]
def main():
parser = argparse.ArgumentParser(description='Tensorflow hello world')
parser.add_argument('-c', '--carbon-sdk-path', type=str, default=DEFAULT_CARBONSDK_PATH,
help='carbon sdk path')
args = parser.parse_args()
jnius_config.set_classpath(args.carbon_sdk_path)
jnius_config.add_options('-Xrs', '-Xmx6096m')
# jnius_config.add_options('-Xrs', '-Xmx6096m', '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5555')
print("Start")
start = time.time()
key = "OF0FTHGASIHDTRYHBCWU"
secret = "fWWjJwh89NFaMDPrFdhu68Umus4vftlIzcNuXvwV"
endpoint = "http://obs.cn-north-5.myhuaweicloud.com"
just_read_batch_obs(key, secret, endpoint,
'modelarts-carbon', 'imageNet_resize/imageNet_whole_resize_small1', '/tmp/download/')
shutil.rmtree('/tmp/download/')
end = time.time()
print("all time: " + str(end - start))
print("Finish")
def main():
parser = argparse.ArgumentParser(description='carbon read from obs manifest')
parser.add_argument('-c', '--carbon-sdk-path', type=str, default=DEFAULT_CARBONSDK_PATH,
help='carbon sdk path')
parser.add_argument('-ak', '--access_key', type=str, required=True,
help='access_key of obs')
parser.add_argument('-sk', '--secret_key', type=str, required=True,
help='secret_key of obs')
parser.add_argument('-endpoint', '--end_point', type=str, required=True,
help='end_point of obs')
args = parser.parse_args()
jnius_config.set_classpath(args.carbon_sdk_path)
jnius_config.add_options('-Xrs', '-Xmx6096m')
print("Start")
start = time.time()
path = "s3a://manifest/carbon/manifestcarbon/obsbinary1557717977531.manifest"
just_read_batch_obs(path, args.access_key, args.secret_key, args.end_point)
just_unified_read_batch_obs(path, args.access_key, args.secret_key, args.end_point)
end = time.time()
print("all time: " + str(end - start))
print("Finish")
def main():
parser = argparse.ArgumentParser(description='Tensorflow hello world')
parser.add_argument('-c', '--carbon-sdk-path', type=str, default=DEFAULT_CARBONSDK_PATH,
help='carbon sdk path')
parser.add_argument('-ak', '--access_key', type=str, required=True,
help='access_key of obs')
parser.add_argument('-sk', '--secret_key', type=str, required=True,
help='secret_key of obs')
parser.add_argument('-endpoint', '--end_point', type=str, required=True,
help='end_point of obs')
args = parser.parse_args()
jnius_config.set_classpath(args.carbon_sdk_path)
jnius_config.add_options('-Xrs', '-Xmx6096m')
# jnius_config.add_options('-Xrs', '-Xmx6096m', '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5555')
print("Start")
start = time.time()
just_read_batch_obs(key=args.access_key, secret=args.secret_key, endpoint=args.end_point)
just_unified_read_batch_obs(key=args.access_key, secret=args.secret_key, endpoint=args.end_point)
shutil.rmtree('/tmp/download/')
end = time.time()
print("all time: " + str(end - start))
print("Finish")
def _init_jvm_options():
import imglyb_config
import jnius_config
PAINTERA_ENDPOINT = 'org.janelia.saalfeldlab:paintera:{}'.format(
__paintera_version__)
RELEVANT_MAVEN_REPOS = {
'imagej.public': 'https://maven.imagej.net/content/groups/public',
'saalfeldlab': 'https://saalfeldlab.github.io/maven'
}
imglyb_config.add_endpoints(
PAINTERA_ENDPOINT, '{}'.format(
os.getenv('PAINTERA_SLF4J_BINDING',
'org.slf4j:slf4j-simple:1.7.25')))
imglyb_config.add_repositories(RELEVANT_MAVEN_REPOS)
JVM_OPTIONS_STR = 'JVM_OPTIONS'
if JVM_OPTIONS_STR in os.environ:
jnius_config.add_options(*os.environ[JVM_OPTIONS_STR].split(' '))
jnius_config.add_classpath(__jfx_rt_jar)
return jnius_config
def loadLibrary():
'''
Loads Siddi Library
:return:
'''
siddhi_home = os.getenv("SIDDHISDK_HOME")
# Test whether Java Library is already loaded
if "siddhi_api_configured" in globals():
if globals()["siddhi_api_configured"] != 4:
raise Exception(
"Unable to use multiple versions of Siddhi Library")
# Resume global variables if already loaded
_resumeLibrary()
return
# Add Java library to class path of jvm
import jnius_config
# NOTE: The following code-line is required on Linux Kernel 4.4.0-81-generic and above to avoid segmentation fault
# at initialization of pyjnius
jnius_config.add_options('-Xss1280k')
jnius_config.add_options('-Djava.library.path=' + PySiddhi.root_path +
"/__PySiddhiProxy")
# Determine library class path
class_paths = ['.', os.path.join(siddhi_home, 'lib', '*')]
# Add Extensions
if not "extensions" in globals():
global extensions
extensions = []
for extension in extensions:
class_paths.append(extension)
jnius_config.set_classpath(*(tuple(class_paths)))
# Mark API configured
global siddhi_api_configured
siddhi_api_configured = 4
logging.info("Classpath Configured")
# Load Pyjnius (Starts JVM)
from jnius import autoclass, java_method, PythonJavaClass, JavaClass
# Generate references and store in backup
global t_siddhi_api_core, t_siddhi_api_core_inst, t_java_method, t_PythonJavaClass, t_JavaClass
t_siddhi_api_core = autoclass(
"io.siddhi.pythonapi.proxy.core.SiddhiAPICoreProxy")
t_siddhi_api_core_inst = t_siddhi_api_core(sys.version_info[0],
sys.version_info[1])
t_java_method = java_method
t_PythonJavaClass = PythonJavaClass
t_JavaClass = JavaClass
# Resume references stored in backup
_resumeLibrary()
def add_java_options(*args):
"""
Add jnius java options (refer to jnius_config.add_options)
"""
if jnius_config.vm_running:
logger.warn("jvm is running, cannot add options")
else:
jnius_config.add_options(*args)
def _get_jnius():
import jnius_config
jnius_config.add_options(
'-ea',
'-Xrs',
'-Xmx256M',
'-Dlogback.configurationFile=logback-tools.xml',
)
jnius_config.set_classpath(*get_cassandra_classpath())
import jnius
return jnius
def addOptions(self, *opts):
"""
Append options to the list of JVM start-up options
:type opts: list
"""
super().addOptions(*opts)
try:
jnius_config.add_options(*opts)
except ValueError as e:
raise JVMError(e)
def load_class(*options, cls: str = CLASS_PATH):
import jnius_config
for option in options:
jnius_config.add_options(option)
jnius_config.set_classpath(cls)
from jnius import autoclass
Vocabulary = autoclass('corenlp.rdrsegmenter.Vocabulary')
WordSegmenter = autoclass('corenlp.rdrsegmenter.WordSegmenter')
return Vocabulary, WordSegmenter
def run(self, *args: str):
try:
jnius_config.add_options(*args)
log.info("Configured jvm options:%s" % jnius_config.get_options())
from jnius import autoclass
DebeziumServer = autoclass('io.debezium.server.Main')
_dbz = DebeziumServer()
return _dbz.main()
finally:
from jnius import detach
detach()
def __init__(self, **kwargs):
self.root = kwargs['root']
self.version = kwargs['version']
import jnius_config
jnius_config.add_options('-Xmx2g')
jnius_config.set_classpath('{0}/lib/btrbl-je-{1}.jar'.format(self.root, self.version),
'{0}/lib/slf4j-api-1.7.5.jar'.format(self.root),
'{0}/lib/slf4j-simple-1.7.5.jar'.format(self.root))
from jnius import autoclass
self.BaseLinguisticsFactory = autoclass('com.basistech.rosette.bl.BaseLinguisticsFactory')
self.Annotator = autoclass('com.basistech.rosette.dm.Annotator')
self.BaseLinguisticsOption = autoclass('com.basistech.rosette.bl.BaseLinguisticsOption')
self.EnumMap = autoclass('java.util.EnumMap')
self.String = autoclass('java.lang.String')
def init_pyjnius():
if not jnius_config.vm_running:
if "JAVA_HOME" not in os.environ:
logger.info("'JAVA_HOME' environment variable missing, "
"trying to get from settings.py")
os.environ["JAVA_HOME"] = settings.JAVA_HOME
logger.info("JAVA_HOME set to '{}'".format(os.environ["JAVA_HOME"]))
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
jnius_config.add_options('-Djava.awt.headless=true')
jnius_config.add_options('-Xmx1536m')
jnius_config.set_classpath(
os.path.join(BASE_DIR, "simccs", "lib",
"simccs-app-1.0-jar-with-dependencies.jar"))
logger.info("Initialized jnius with classpath={}".format(
jnius_config.get_classpath()))
import jnius # noqa
def __init__(self, **kwargs):
self.root = kwargs['root']
self.version = kwargs['version']
import jnius_config
jnius_config.add_options('-Xmx2g')
jnius_config.set_classpath(
'{0}/lib/btrbl-je-{1}.jar'.format(self.root, self.version),
'{0}/lib/slf4j-api-1.7.5.jar'.format(self.root),
'{0}/lib/slf4j-simple-1.7.5.jar'.format(self.root))
from jnius import autoclass
self.BaseLinguisticsFactory = autoclass(
'com.basistech.rosette.bl.BaseLinguisticsFactory')
self.Annotator = autoclass('com.basistech.rosette.dm.Annotator')
self.BaseLinguisticsOption = autoclass(
'com.basistech.rosette.bl.BaseLinguisticsOption')
self.EnumMap = autoclass('java.util.EnumMap')
self.String = autoclass('java.lang.String')
def initialize(self):
if self._autoclass is not None:
logger.debug('jnius already initialized')
return
with logger.duration('initializing jnius'):
log_fifo, l4j12_file, l4j24_file = self._init_java_logger_interface(
)
import jnius_config
jnius_config.set_classpath(*self._jars,
os.path.dirname(l4j12_file))
jnius_config.add_options(f'-Dlog4j.configuration={l4j12_file}')
jnius_config.add_options(f'-Dlog4j.configurationFile={l4j24_file}')
from jnius import autoclass
self._autoclass = autoclass
# self.PropertyConfigurator.configure(l4j12_file)
for key, path in self._defs.items():
self._cache[key] = self._autoclass(path)
def install(*options, jar_path: str = CLASS_PATH):
if not os.path.exists(MODEL_DIR):
import urllib.request
from io import BytesIO
from zipfile import ZipFile
url = 'https://github.com/NgHoangDat/python_vncorenlp/raw/master/python_vncorenlp/data/models.zip'
response = urllib.request.urlopen(url)
data = BytesIO(response.read())
with ZipFile(data) as f:
f.extractall(DATA_DIR)
import jnius_config
for option in options:
jnius_config.add_options(option)
jnius_config.set_classpath(jar_path)
def _init_jvm_options():
import jnius_config
import os
PYJNIUS_JAR_STR = 'PYJNIUS_JAR'
IMGLYB_JAR_STR = 'IMGLYB_JAR'
if PYJNIUS_JAR_STR not in globals():
try:
PYJNIUS_JAR = os.environ[PYJNIUS_JAR_STR]
except KeyError as e:
print(
"Path to pyjnius.jar not defined! Use environment variable {} to define it."
.format(PYJNIUS_JAR_STR))
raise e
if IMGLYB_JAR_STR not in globals():
try:
IMGLYB_JAR = os.environ[IMGLYB_JAR_STR]
except KeyError as e:
print(
"Path to imglib2-imglyb jar not defined! Use environment variable {} to define it."
.format(IMGLYB_JAR_STR))
raise e
if 'classpath' in globals():
jnius_config.add_classpath(classpath)
CLASSPATH_STR = 'CLASSPATH'
if CLASSPATH_STR in os.environ:
for path in os.environ[CLASSPATH_STR].split(jnius_config.split_char):
jnius_config.add_classpath(path)
jnius_config.add_classpath(PYJNIUS_JAR)
jnius_config.add_classpath(IMGLYB_JAR)
JVM_OPTIONS_STR = 'JVM_OPTIONS'
if JVM_OPTIONS_STR in os.environ:
jnius_config.add_options(*os.environ[JVM_OPTIONS_STR].split(' '))
return jnius_config
def __init__(self):
"""
Constructor to set up local pipeline
@param: file_name, the file name of the custom config file
"""
super(LocalPipeline, self).__init__()
self.PipelineFactory = None
self.pipeline = None
self.ProtobufSerializer = None
pipeline_config.log_current_config(self.config, False)
# set up JVM and load classes needed
try:
import jnius_config
jnius_config.add_options('-Xmx16G')
jnius_config.add_classpath(get_model_path() + '/*')
except Exception as e:
logger.warning(
"Couldn't set JVM config; this might be because you're setting up the multiple instances of the local pipeline."
)
logger.warning(str(e))
try:
from jnius import autoclass
self.PipelineFactory = autoclass(
'edu.illinois.cs.cogcomp.pipeline.main.PipelineFactory')
self.SerializationHelper = autoclass(
'edu.illinois.cs.cogcomp.core.utilities.SerializationHelper')
# self.ProtobufSerializer = autoclass('edu.illinois.cs.cogcomp.core.utilities.protobuf.ProtobufSerializer')
self.Boolean = autoclass('java.lang.Boolean')
self.JString = autoclass('java.lang.String')
except Exception as e:
logger.error(
'Fail to load models, please check if your Java version is up to date.'
)
logger.error(str(e))
return None
self.pipeline = self.PipelineFactory.buildPipelineWithAllViews(
self.Boolean(True))
logger.info("pipeline has been set up")
def get_tagger():
global _tagger
if _tagger is None:
import jnius_config
import os
os.environ['JAVA_HOME'] = configuration.java_home
jnius_config.add_options('-Xrs', '-Xmx128M')
jnius_config.set_classpath(os.path.abspath(os.path.join(os.path.dirname(__file__), '../lib/ark-tweet-nlp-0.3.2/ark-tweet-nlp-0.3.2.jar')))
from jnius import autoclass
_Tagger = autoclass('cmu.arktweetnlp.Tagger')
_tagger=_Tagger()
_tagger.loadModel('/cmu/arktweetnlp/model.20120919')
return _tagger
def __init__(self, name, index, k, num_threads, Xms=None, Xmx=None):
super().__init__(name)
if Xms and Xmx:
# to solve Insufficient memory for the Java Runtime Environment
jnius_config.add_options("-Xms{}".format(Xms),
"-Xmx{}".format(Xmx),
"-XX:-UseGCOverheadLimit")
print("Configured options:", jnius_config.get_options())
self.num_threads = min(num_threads, int(multiprocessing.cpu_count()))
# initialize a ranker per thread
self.arguments = []
for id in tqdm(range(self.num_threads)):
self.arguments.append({
"id": id,
"index": index,
"k": k,
})
def init(ij_dir):
"""
quietly set up the whole environment
:param ij_dir: System path for Fiji.app
:return: an instance of the net.imagej.ImageJ gateway
"""
jnius_config.add_options('-Djava.awt.headless=true')
imglyb_path = configure_path()
# ImageJ
if imglyb_path is not None:
num_jars = set_ij_env(ij_dir, imglyb_path)
else:
return
print("Added " + str(num_jars + 1) + " JARs to the Java classpath.")
import imglyb
from jnius import autoclass
ImageJ = autoclass('net.imagej.ImageJ')
return ImageJ()
def main():
parser = argparse.ArgumentParser(description='Tensorflow hello world')
parser.add_argument('-c', '--carbon-sdk-path', type=str, default=DEFAULT_CARBONSDK_PATH,
help='carbon sdk path')
args = parser.parse_args()
jnius_config.set_classpath(args.carbon_sdk_path)
jnius_config.add_options('-Xrs', '-Xmx6096m')
# jnius_config.add_options('-Xrs', '-Xmx6096m', '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5555')
print("Start")
start = time.time()
just_read()
# just_read_batch()
end = time.time()
print("all time: " + str(end - start))
print("Finish")
def init(config):
# Validate the JARs folder
# Path is relative to the current working directory
if not os.path.isdir('./java_libs/'):
raise Exception("Java library path could not be located.")
# Construct classpath for JVM
import jnius_config
jnius_config.add_options('-Xrs',
'-Xmx4m') # JVM options in comma separated
jnius_config.set_classpath(
'./java_libs/*') # Path for the Java library
import jnius
# Build Payment Method Token Recipient
Builder = jnius.autoclass(
'com.google.crypto.tink.apps.paymentmethodtoken.PaymentMethodTokenRecipient$Builder'
)
builder = Builder()
builder.addSenderVerifyingKey(config['GoogleSigningKey'])
builder.recipientId(config['RecipientId'])
builder.protocolVersion(config['ProtocolVersion'])
builder.addRecipientPrivateKey(config['RecipientPrivateKey'])
Decryptor.paymentMethodTokenRecipient = builder.build()
def main():
parser = argparse.ArgumentParser(
description='Support read carbon by manifest path in python code')
parser.add_argument('-c',
'--carbon-sdk-path',
type=str,
default=DEFAULT_CARBONSDK_PATH,
help='carbon sdk path')
args = parser.parse_args()
jnius_config.set_classpath(args.carbon_sdk_path)
jnius_config.add_options('-Xrs', '-Xmx6096m')
print("Start")
start = time.time()
just_read_batch()
just_unified_read_batch()
end = time.time()
print("all time: " + str(end - start))
print("Finish")
def _has_xmx(options):
for option in options:
if option.startswith('-Xmx'):
return True
return False
default_mem_limit = get_config("INDRA_DEFAULT_JAVA_MEM_LIMIT")
if default_mem_limit is None:
# Set to 8g if not specified in the configuration
default_mem_limit = '8g'
if not _has_xmx(jnius_config.get_options()):
if not jnius_config.vm_running:
jnius_config.add_options('-Xmx%s' % default_mem_limit)
else:
logger.warning("Couldn't set memory limit for Java VM because the VM "
"is already running.")
path_here = os.path.dirname(os.path.realpath(__file__))
cp = os.path.join(path_here, 'sources/biopax/jars/paxtools.jar')
cp_existing = os.environ.get('CLASSPATH')
if cp_existing is not None:
os.environ['CLASSPATH'] = cp + ':' + cp_existing
else:
os.environ['CLASSPATH'] = cp
logger.debug('Classpath when starting Java VM: %s' % os.environ['CLASSPATH'])
import os, frappe
import jasper_erpnext_report as jr
_logger = frappe.logger("jasper_erpnext_report")
try:
import jnius_config as jc
jr.pyjnius = True
if not jc.vm_running:
jc.add_options('-Djava.awt.headless=true')
else:
_logger.info("vm_running: {}".format(jc.vm_running))
except:
print "jnius_config not found"
jr.pyjnius = False
norm_path = os.path.normpath
join_path = os.path.join
dirname = os.path.dirname
parent_path = dirname(dirname(__file__))
rel_path = os.path.relpath(os.path.join(parent_path, "java"),dirname(__file__))
rel_path_curr = os.path.relpath(parent_path, os.getcwd())
try:
os.environ['CLASSPATH'] = norm_path(join_path(parent_path,"java/lib/*")) + ":.:" + os.environ.get('CLASSPATH',"")
print "CLASSPATH {}".format(os.environ['CLASSPATH'])
except:
print "Error in setting java classpath."
try:
from jnius import autoclass
# __author__ = 'WeiFu'
from __future__ import print_function, division
import jnius_config
jnius_config.add_options('-Xrs', '-Xmx4096')
jnius_config.set_classpath('.', '/Users/WeiFu/Github/HDP_Jython/jar/weka.jar','/Users/WeiFu/Github/HDP_Jython/jar/commons-math3-3.5/commons-math3-3.5.jar')
import pdb
import random
from os import listdir
from os.path import isfile, join
from jnius import autoclass
#
#
#
#
#
#
# import weka.core.Instances
# import java.io.BufferedReader
# import java.io.FileReader
# import weka.attributeSelection.Ranker as Ranker
# import weka.attributeSelection.ReliefFAttributeEval as ReliefFAttributeEval
# import weka.attributeSelection.AttributeSelection as attributeSelection
# import weka.classifiers.functions.Logistic as Logistic
# import weka.classifiers.Evaluation as Evaluation
# import weka.core.converters.ArffSaver as Saver
# import weka.filters.unsupervised.attribute.Remove as Remove
# import weka.filters.unsupervised.instance.Randomize as Randomize
# import weka.filters.unsupervised.instance.RemoveFolds as RemoveFolds
# import weka.core.jvm as jvm
"""Handles all imports from jnius to prevent conflicts resulting from attempts
to set JVM options while the VM is already running."""
import jnius_config
if '-Xmx4g' not in jnius_config.get_options():
if not jnius_config.vm_running:
jnius_config.add_options('-Xmx4g')
else:
warnings.warn("Couldn't set memory limit for Java VM because the VM "
"is already running.")
from jnius import autoclass, JavaException, cast
"""Handles all imports from jnius to prevent conflicts resulting from attempts
to set JVM options while the VM is already running."""
import os
import warnings
import jnius_config
if "-Xmx4g" not in jnius_config.get_options():
if not jnius_config.vm_running:
jnius_config.add_options("-Xmx4g")
else:
warnings.warn("Couldn't set memory limit for Java VM because the VM " "is already running.")
path_here = os.path.dirname(os.path.realpath(__file__))
cp = path_here + "/biopax/jars/paxtools.jar"
os.environ["CLASSPATH"] = cp
from jnius import autoclass, JavaException, cast
os.environ['JAVA_HOME'] = expandPath('@resources/java/')
if WIN32:
path = os.getenv('Path', '')
path = path.split(os.pathsep)
path.insert(0, os.path.join(os.environ['JAVA_HOME'], 'bin', 'client'))
os.environ['Path'] = os.pathsep.join(path)
classpath = []
resources = expandPath('@resources/lib/')
for name in os.listdir(resources):
path = os.path.join(resources, name)
if os.path.isfile(path):
classpath.append(path)
jnius_config.add_options('-ea', '-Xmx512m')
jnius_config.set_classpath(*classpath)
#############################################
# END JAVA VIRTUAL MACHINE SETUP
#################################
from argparse import ArgumentParser
from unittest import TestCase
from unittest.util import safe_repr
from PyQt5 import QtCore
from PyQt5 import QtTest
from eddy import APPNAME, ORGANIZATION, WORKSPACE
from eddy.core.application import Eddy
def init(version=None,
mem=None,
packages=[],
jvm_opts=[],
redirect_io=True,
logging='WARN',
home_dir=None,
boot_packages=[],
tqdm=None):
"""
Function necessary to be called before Terrier classes and methods can be used.
Loads the Terrier .jar file and imports classes. Also finds the correct version of Terrier to download if no version is specified.
Args:
version(str): Which version of Terrier to download. Default is `None`.
* If None, find the newest Terrier released version in MavenCentral and download it.
* If `"snapshot"`, will download the latest build from `Jitpack <https://jitpack.io/>`_.
mem(str): Maximum memory allocated for the Java virtual machine heap in MB. Corresponds to java `-Xmx` commandline argument. Default is 1/4 of physical memory.
boot_packages(list(str)): Extra maven package coordinates files to load before starting Java. Default=`[]`. There is more information about loading packages in the `Terrier documentation <https://github.com/terrier-org/terrier-core/blob/5.x/doc/terrier_develop.md>`_
packages(list(str)): Extra maven package coordinates files to load, using the Terrier classloader. Default=`[]`. See also `boot_packages` above.
jvm_opts(list(str)): Extra options to pass to the JVM. Default=`[]`. For instance, you may enable Java assertions by setting `jvm_opts=['-ea']`
redirect_io(boolean): If True, the Java `System.out` and `System.err` will be redirected to Pythons sys.out and sys.err. Default=True.
logging(str): the logging level to use:
* Can be one of `'INFO'`, `'DEBUG'`, `'TRACE'`, `'WARN'`, `'ERROR'`. The latter is the quietest.
* Default is `'WARN'`.
home_dir(str): the home directory to use. Default to PYTERRIER_HOME environment variable.
tqdm: The `tqdm <https://tqdm.github.io/>`_ instance to use for progress bars within PyTerrier. Defaults to tqdm.tqdm. Available options are `'tqdm'`, `'auto'` or `'notebook'`.
**Locating the Terrier .jar file:** PyTerrier is not tied to a specific version of Terrier and will automatically locate and download a recent Terrier .jar file. However, inevitably, some functionalities will require more recent Terrier versions.
* If set, PyTerrier uses the `version` init kwarg to determine the .jar file to look for.
* If the `version` init kwarg is not set, Terrier will query MavenCentral to determine the latest Terrier release.
* If `version` is set to `"snapshot"`, the latest .jar file build derived from the `Terrier Github repository <https://github.com/terrier-org/terrier-core/>`_ will be downloaded from `Jitpack <https://jitpack.io/>`_.
* Otherwise the local (`~/.mvn`) and MavenCentral repositories are searched for the jar file at the given version.
In this way, the default setting is to download the latest release of Terrier from MavenCentral. The user is also able to use a locally installed copy in their private Maven repository, or track the latest build of Terrier from Jitpack.
"""
set_tqdm(tqdm)
global ApplicationSetup
global properties
global firstInit
global file_path
global HOME_DIR
# we keep a local directory
if home_dir is not None:
HOME_DIR = home_dir
elif "PYTERRIER_HOME" in os.environ:
HOME_DIR = os.environ["PYTERRIER_HOME"]
else:
from os.path import expanduser
userhome = expanduser("~")
HOME_DIR = os.path.join(userhome, ".pyterrier")
if not os.path.exists(HOME_DIR):
os.mkdir(HOME_DIR)
# get the initial classpath for the JVM
classpathTrJars = setup_terrier(HOME_DIR,
version,
boot_packages=boot_packages)
# Import pyjnius and other classes
import jnius_config
for jar in classpathTrJars:
jnius_config.add_classpath(jar)
if jvm_opts is not None:
for opt in jvm_opts:
jnius_config.add_options(opt)
if mem is not None:
jnius_config.add_options('-Xmx' + str(mem) + 'm')
from jnius import autoclass, cast
# we only accept Java version 11 and newer; so anything starting 1. or 9. is too old
java_version = autoclass("java.lang.System").getProperty("java.version")
if java_version.startswith("1.") or java_version.startswith("9."):
raise RuntimeError(
"Pyterrier requires Java 11 or newer, we only found Java version %s;"
+
" install a more recent Java, or change os.environ['JAVA_HOME'] to point to the proper Java installation",
java_version)
tr_version = autoclass('org.terrier.Version')
version_string = tr_version.VERSION
if "BUILD_DATE" in dir(tr_version):
version_string += " (built by %s on %s)" % (tr_version.BUILD_USER,
tr_version.BUILD_DATE)
print("PyTerrier %s has loaded Terrier %s" % (__version__, version_string))
properties = autoclass('java.util.Properties')()
ApplicationSetup = autoclass('org.terrier.utility.ApplicationSetup')
from .batchretrieve import BatchRetrieve, FeaturesBatchRetrieve
from .utils import Utils
from .datasets import get_dataset, list_datasets
from .index import Indexer, FilesIndexer, TRECCollectionIndexer, DFIndexer, DFIndexUtils, IterDictIndexer, FlatJSONDocumentIterator, IndexingType
from .pipelines import LTR_pipeline, XGBoostLTR_pipeline, Experiment
# Make imports global
globals()["autoclass"] = autoclass
globals()["cast"] = cast
globals()["ApplicationSetup"] = ApplicationSetup
for sub_module_name in [
'anserini', 'apply', 'cache', 'index', 'io', 'model', 'new', 'ltr',
'pipelines', 'rewrite', 'text', 'transformer'
]:
globals()[sub_module_name] = importlib.import_module(
'.' + sub_module_name, package='pyterrier')
# append the python helpers
if packages is None:
packages = []
# Import other java packages
if packages != []:
pkgs_string = ",".join(packages)
properties.put("terrier.mvn.coords", pkgs_string)
ApplicationSetup.bootstrapInitialisation(properties)
if redirect_io:
# this ensures that the python stdout/stderr and the Java are matched
redirect_stdouterr()
_logging(logging)
setup_jnius()
globals()["get_dataset"] = get_dataset
globals()["list_datasets"] = list_datasets
globals()["Experiment"] = Experiment
globals()["BatchRetrieve"] = BatchRetrieve
globals()["Indexer"] = Indexer
globals()["FeaturesBatchRetrieve"] = FeaturesBatchRetrieve
globals()["TRECCollectionIndexer"] = TRECCollectionIndexer
globals()["FilesIndexer"] = FilesIndexer
globals()["DFIndexer"] = DFIndexer
globals()["DFIndexUtils"] = DFIndexUtils
globals()["IterDictIndexer"] = IterDictIndexer
globals()["FlatJSONDocumentIterator"] = FlatJSONDocumentIterator
globals()["Utils"] = Utils
globals()["LTR_pipeline"] = LTR_pipeline
globals()["XGBoostLTR_pipeline"] = XGBoostLTR_pipeline
globals()["IndexFactory"] = autoclass(
"org.terrier.structures.IndexFactory")
globals()["IndexRef"] = autoclass("org.terrier.querying.IndexRef")
globals()["IndexingType"] = IndexingType
firstInit = True
import sys, traceback
import jpype
import numpy as np
import os
import logging
logger = logging.getLogger(__name__)
CLASSPATH = os.environ.get("CLASSPATH", "OpenRocket-15.03.jar")
import jnius_config
jnius_config.add_options("-ea")
jnius_config.set_classpath(".", CLASSPATH)
import jnius
from jnius import autoclass
class OpenRocketInstance(object):
""" When instantiated, this class starts up a new openrocket instance.
This class is designed to be called using the 'with' construct. This
will ensure that no matter what happens within that context, the
JVM will always be shutdown.
"""
def __init__(self, jar_path=CLASSPATH, log_level="ERROR"):
""" jar_path is the full path of the OpenRocket .jar file to use
log_level can be either ERROR, WARN, USER, INFO, DEBUG or VBOSE
"""
import jnius_config
import os
jnius_config.add_options('-Dorg.bytedeco.javacpp.nopointergc=true')
jnius_class_path = os.environ.get('JUMPY_CLASS_PATH')
if not jnius_class_path:
raise Exception('Environment variable JUMPY_CLASS_PATH not set.')
elif not os.path.exists(jnius_class_path):
raise Exception('File not found : ' + jnius_class_path)
jnius_config.set_classpath(jnius_class_path)
from jnius import autoclass
Nd4j = autoclass('org.nd4j.linalg.factory.Nd4j')
INDArray = autoclass('org.nd4j.linalg.api.ndarray.INDArray')
Transforms = autoclass('org.nd4j.linalg.ops.transforms.Transforms')
NDArrayIndex = autoclass('org.nd4j.linalg.indexing.NDArrayIndex')
DataBuffer = autoclass('org.nd4j.linalg.api.buffer.DataBuffer')
System = autoclass('java.lang.System')
Integer = autoclass('java.lang.Integer')
Float = autoclass('java.lang.Float')
Double = autoclass('java.lang.Double')
Shape = autoclass('org.nd4j.linalg.api.shape.Shape')
BinarySerde = autoclass('org.nd4j.serde.binary.BinarySerde')
NativeOpsHolder = autoclass('org.nd4j.nativeblas.NativeOpsHolder')
DoublePointer = autoclass('org.bytedeco.javacpp.DoublePointer')
FloatPointer = autoclass('org.bytedeco.javacpp.FloatPointer')
IntPointer = autoclass('org.bytedeco.javacpp.IntPointer')
DataTypeUtil = autoclass('org.nd4j.linalg.api.buffer.util.DataTypeUtil')
MemoryManager = autoclass('org.nd4j.linalg.memory.MemoryManager')
SameDiff = autoclass('org.nd4j.autodiff.samediff.SameDiff')
import os, frappe
import jasper_erpnext_report as jr
_logger = frappe.get_logger("jasper_erpnext_report")
try:
import jnius_config as jc
jr.pyjnius = True
if not jc.vm_running:
jc.add_options('-Djava.awt.headless=true')
else:
_logger.info("vm_running: {}".format(jc.vm_running))
except:
print "jnius_config not found"
jr.pyjnius = False
norm_path = os.path.normpath
join_path = os.path.join
dirname = os.path.dirname
parent_path = dirname(dirname(__file__))
rel_path = os.path.relpath(os.path.join(parent_path, "java"),
dirname(__file__))
rel_path_curr = os.path.relpath(parent_path, os.getcwd())
try:
os.environ['CLASSPATH'] = norm_path(join_path(
parent_path, "java/lib/*")) + ":.:" + os.environ.get('CLASSPATH', "")
except:
print "Error in setting java classpath."
try:
'''
python3调用自定义的java类和方法建立索引
'''
import jnius_config
jnius_config.add_options('-Xrs', '-Xmx1024m')
jnius_config.set_classpath('.', 'C:\Program Files\java\jdk1.8.0_111\lib\*')
from jnius import autoclass
myclass = autoclass("jingyi.IndexMovies")
indexclass = myclass()
try:
indexclass.Create_index()
except Exception as err:
print(err)
finally:
print("success")