Exemple #1
0
    def process_data(pos, model_path, sentences):
        os.environ["CLASSPATH"] = model_path
        import jnius_config

        jnius_config.add_options("-Xmx4056M")
        from jnius import autoclass

        enum = autoclass(
            "org.lambda3.text.simplification.discourse.processing.ProcessingType"
        )
        model = autoclass(
            "org.lambda3.text.simplification.discourse.processing.DiscourseSimplifier"
        )()

        results = {}
        for id, sentence in tqdm(sentences.items(),
                                 position=pos,
                                 mininterval=1):
            try:
                with suppress_stdout():
                    result = model.doDiscourseSimplification(
                        str(sentence), enum.SEPARATE).serializeToJSON()
                    o_id = ray.put(json.decode(result))
                    results[id] = o_id
            except:

                logger.error(f"Unable to parse {sentence}")
                results[id] = None
        return results
    def initialize(self):
        # we need to delay the initialization of Java in order for this class to
        # properly work with multiprocessing
        if not OpenNLPSentenceTokenizer._java_initialized:
            import jnius_config

            jnius_config.add_options("-Xrs")
            jnius_config.set_classpath(
                pkg_resources.resource_filename(
                    __name__, "/resources/opennlp-tools-1.5.3.jar"))
            OpenNLPSentenceTokenizer._java_initialized = True

        from jnius import autoclass

        File = autoclass("java.io.File")
        SentenceModel = autoclass("opennlp.tools.sentdetect.SentenceModel")
        SentenceDetectorME = autoclass(
            "opennlp.tools.sentdetect.SentenceDetectorME")

        sentence_model_file = pkg_resources.resource_filename(
            __name__, "resources/en-sent.bin")
        sentence_model = SentenceModel(File(sentence_model_file))
        self._tokenizer = SentenceDetectorME(sentence_model)

        self._initialized = True
 def test_add_options(self):
     assert jnius_config.vm_running is False
     assert jnius_config.options == []
     jnius_config.add_options("option1", "option2")
     assert jnius_config.options == ["option1", "option2"]
     jnius_config.add_options("option3")
     assert jnius_config.options == ["option1", "option2", "option3"]
Exemple #4
0
def main():
  parser = argparse.ArgumentParser(description='Tensorflow hello world')
  parser.add_argument('-c', '--carbon-sdk-path', type=str, default=DEFAULT_CARBONSDK_PATH,
                      help='carbon sdk path')

  args = parser.parse_args()

  jnius_config.set_classpath(args.carbon_sdk_path)

  jnius_config.add_options('-Xrs', '-Xmx6096m')
  # jnius_config.add_options('-Xrs', '-Xmx6096m', '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5555')

  print("Start")
  start = time.time()

  key = "OF0FTHGASIHDTRYHBCWU"
  secret = "fWWjJwh89NFaMDPrFdhu68Umus4vftlIzcNuXvwV"
  endpoint = "http://obs.cn-north-5.myhuaweicloud.com"

  just_read_batch_obs(key, secret, endpoint,
                      'modelarts-carbon', 'imageNet_resize/imageNet_whole_resize_small1', '/tmp/download/')
  shutil.rmtree('/tmp/download/')

  end = time.time()
  print("all time: " + str(end - start))
  print("Finish")
Exemple #5
0
def main():
  parser = argparse.ArgumentParser(description='carbon read from obs manifest')
  parser.add_argument('-c', '--carbon-sdk-path', type=str, default=DEFAULT_CARBONSDK_PATH,
                      help='carbon sdk path')
  parser.add_argument('-ak', '--access_key', type=str, required=True,
                      help='access_key of obs')
  parser.add_argument('-sk', '--secret_key', type=str, required=True,
                      help='secret_key of obs')
  parser.add_argument('-endpoint', '--end_point', type=str, required=True,
                      help='end_point of obs')

  args = parser.parse_args()

  jnius_config.set_classpath(args.carbon_sdk_path)

  jnius_config.add_options('-Xrs', '-Xmx6096m')
  print("Start")
  start = time.time()

  path = "s3a://manifest/carbon/manifestcarbon/obsbinary1557717977531.manifest"

  just_read_batch_obs(path, args.access_key, args.secret_key, args.end_point)

  just_unified_read_batch_obs(path, args.access_key, args.secret_key, args.end_point)

  end = time.time()
  print("all time: " + str(end - start))
  print("Finish")
Exemple #6
0
def main():
  parser = argparse.ArgumentParser(description='Tensorflow hello world')
  parser.add_argument('-c', '--carbon-sdk-path', type=str, default=DEFAULT_CARBONSDK_PATH,
                      help='carbon sdk path')
  parser.add_argument('-ak', '--access_key', type=str, required=True,
                      help='access_key of obs')
  parser.add_argument('-sk', '--secret_key', type=str, required=True,
                      help='secret_key of obs')
  parser.add_argument('-endpoint', '--end_point', type=str, required=True,
                      help='end_point of obs')

  args = parser.parse_args()

  jnius_config.set_classpath(args.carbon_sdk_path)

  jnius_config.add_options('-Xrs', '-Xmx6096m')
  # jnius_config.add_options('-Xrs', '-Xmx6096m', '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5555')

  print("Start")
  start = time.time()

  just_read_batch_obs(key=args.access_key, secret=args.secret_key, endpoint=args.end_point)

  just_unified_read_batch_obs(key=args.access_key, secret=args.secret_key, endpoint=args.end_point)

  shutil.rmtree('/tmp/download/')

  end = time.time()
  print("all time: " + str(end - start))
  print("Finish")
Exemple #7
0
def _init_jvm_options():

    import imglyb_config

    import jnius_config

    PAINTERA_ENDPOINT = 'org.janelia.saalfeldlab:paintera:{}'.format(
        __paintera_version__)
    RELEVANT_MAVEN_REPOS = {
        'imagej.public': 'https://maven.imagej.net/content/groups/public',
        'saalfeldlab': 'https://saalfeldlab.github.io/maven'
    }

    imglyb_config.add_endpoints(
        PAINTERA_ENDPOINT, '{}'.format(
            os.getenv('PAINTERA_SLF4J_BINDING',
                      'org.slf4j:slf4j-simple:1.7.25')))
    imglyb_config.add_repositories(RELEVANT_MAVEN_REPOS)

    JVM_OPTIONS_STR = 'JVM_OPTIONS'

    if JVM_OPTIONS_STR in os.environ:
        jnius_config.add_options(*os.environ[JVM_OPTIONS_STR].split(' '))

    jnius_config.add_classpath(__jfx_rt_jar)

    return jnius_config
Exemple #8
0
def loadLibrary():
    '''
    Loads Siddi Library
    :return: 
    '''
    siddhi_home = os.getenv("SIDDHISDK_HOME")
    # Test whether Java Library is already loaded
    if "siddhi_api_configured" in globals():
        if globals()["siddhi_api_configured"] != 4:
            raise Exception(
                "Unable to use multiple versions of Siddhi Library")
        # Resume global variables if already loaded
        _resumeLibrary()
        return

    # Add Java library to class path of jvm
    import jnius_config

    # NOTE: The following code-line is required on Linux Kernel 4.4.0-81-generic and above to avoid segmentation fault
    # at initialization of pyjnius
    jnius_config.add_options('-Xss1280k')

    jnius_config.add_options('-Djava.library.path=' + PySiddhi.root_path +
                             "/__PySiddhiProxy")

    # Determine library class path
    class_paths = ['.', os.path.join(siddhi_home, 'lib', '*')]

    # Add Extensions
    if not "extensions" in globals():
        global extensions
        extensions = []

    for extension in extensions:
        class_paths.append(extension)

    jnius_config.set_classpath(*(tuple(class_paths)))

    # Mark API configured
    global siddhi_api_configured
    siddhi_api_configured = 4

    logging.info("Classpath Configured")

    # Load Pyjnius (Starts JVM)
    from jnius import autoclass, java_method, PythonJavaClass, JavaClass

    # Generate references and store in backup
    global t_siddhi_api_core, t_siddhi_api_core_inst, t_java_method, t_PythonJavaClass, t_JavaClass
    t_siddhi_api_core = autoclass(
        "io.siddhi.pythonapi.proxy.core.SiddhiAPICoreProxy")
    t_siddhi_api_core_inst = t_siddhi_api_core(sys.version_info[0],
                                               sys.version_info[1])
    t_java_method = java_method
    t_PythonJavaClass = PythonJavaClass
    t_JavaClass = JavaClass

    # Resume references stored in backup
    _resumeLibrary()
Exemple #9
0
    def add_java_options(*args):
        """
            Add jnius java options (refer to jnius_config.add_options)
        """

        if jnius_config.vm_running:
            logger.warn("jvm is running, cannot add options")
        else:
            jnius_config.add_options(*args)
Exemple #10
0
def _get_jnius():
    import jnius_config
    jnius_config.add_options(
        '-ea',
        '-Xrs',
        '-Xmx256M',
        '-Dlogback.configurationFile=logback-tools.xml',
    )
    jnius_config.set_classpath(*get_cassandra_classpath())
    import jnius
    return jnius
Exemple #11
0
        def addOptions(self, *opts):
            """
            Append options to the list of JVM start-up options

            :type opts: list
            """
            super().addOptions(*opts)
            try:
                jnius_config.add_options(*opts)
            except ValueError as e:
                raise JVMError(e)
Exemple #12
0
def load_class(*options, cls: str = CLASS_PATH):
    import jnius_config

    for option in options:
        jnius_config.add_options(option)

    jnius_config.set_classpath(cls)
    from jnius import autoclass
    Vocabulary = autoclass('corenlp.rdrsegmenter.Vocabulary')
    WordSegmenter = autoclass('corenlp.rdrsegmenter.WordSegmenter')

    return Vocabulary, WordSegmenter
Exemple #13
0
    def run(self, *args: str):

        try:
            jnius_config.add_options(*args)
            log.info("Configured jvm options:%s" % jnius_config.get_options())

            from jnius import autoclass
            DebeziumServer = autoclass('io.debezium.server.Main')
            _dbz = DebeziumServer()
            return _dbz.main()
        finally:
            from jnius import detach
            detach()
Exemple #14
0
 def __init__(self, **kwargs):
     self.root = kwargs['root']
     self.version = kwargs['version']
     import jnius_config
     jnius_config.add_options('-Xmx2g')
     jnius_config.set_classpath('{0}/lib/btrbl-je-{1}.jar'.format(self.root, self.version),
                                '{0}/lib/slf4j-api-1.7.5.jar'.format(self.root),
                                '{0}/lib/slf4j-simple-1.7.5.jar'.format(self.root))
     from jnius import autoclass
     self.BaseLinguisticsFactory = autoclass('com.basistech.rosette.bl.BaseLinguisticsFactory')
     self.Annotator = autoclass('com.basistech.rosette.dm.Annotator')
     self.BaseLinguisticsOption = autoclass('com.basistech.rosette.bl.BaseLinguisticsOption')
     self.EnumMap = autoclass('java.util.EnumMap')
     self.String = autoclass('java.lang.String')
Exemple #15
0
def init_pyjnius():
    if not jnius_config.vm_running:
        if "JAVA_HOME" not in os.environ:
            logger.info("'JAVA_HOME' environment variable missing, "
                        "trying to get from settings.py")
            os.environ["JAVA_HOME"] = settings.JAVA_HOME
        logger.info("JAVA_HOME set to '{}'".format(os.environ["JAVA_HOME"]))
        BASE_DIR = os.path.dirname(os.path.abspath(__file__))
        jnius_config.add_options('-Djava.awt.headless=true')
        jnius_config.add_options('-Xmx1536m')
        jnius_config.set_classpath(
            os.path.join(BASE_DIR, "simccs", "lib",
                         "simccs-app-1.0-jar-with-dependencies.jar"))
        logger.info("Initialized jnius with classpath={}".format(
            jnius_config.get_classpath()))
        import jnius  # noqa
Exemple #16
0
 def __init__(self, **kwargs):
     self.root = kwargs['root']
     self.version = kwargs['version']
     import jnius_config
     jnius_config.add_options('-Xmx2g')
     jnius_config.set_classpath(
         '{0}/lib/btrbl-je-{1}.jar'.format(self.root, self.version),
         '{0}/lib/slf4j-api-1.7.5.jar'.format(self.root),
         '{0}/lib/slf4j-simple-1.7.5.jar'.format(self.root))
     from jnius import autoclass
     self.BaseLinguisticsFactory = autoclass(
         'com.basistech.rosette.bl.BaseLinguisticsFactory')
     self.Annotator = autoclass('com.basistech.rosette.dm.Annotator')
     self.BaseLinguisticsOption = autoclass(
         'com.basistech.rosette.bl.BaseLinguisticsOption')
     self.EnumMap = autoclass('java.util.EnumMap')
     self.String = autoclass('java.lang.String')
Exemple #17
0
 def initialize(self):
     if self._autoclass is not None:
         logger.debug('jnius already initialized')
         return
     with logger.duration('initializing jnius'):
         log_fifo, l4j12_file, l4j24_file = self._init_java_logger_interface(
         )
         import jnius_config
         jnius_config.set_classpath(*self._jars,
                                    os.path.dirname(l4j12_file))
         jnius_config.add_options(f'-Dlog4j.configuration={l4j12_file}')
         jnius_config.add_options(f'-Dlog4j.configurationFile={l4j24_file}')
         from jnius import autoclass
         self._autoclass = autoclass
         # self.PropertyConfigurator.configure(l4j12_file)
         for key, path in self._defs.items():
             self._cache[key] = self._autoclass(path)
def install(*options, jar_path: str = CLASS_PATH):
    if not os.path.exists(MODEL_DIR):
        import urllib.request
        from io import BytesIO
        from zipfile import ZipFile

        url = 'https://github.com/NgHoangDat/python_vncorenlp/raw/master/python_vncorenlp/data/models.zip'
        response = urllib.request.urlopen(url)
        data = BytesIO(response.read())
        with ZipFile(data) as f:
            f.extractall(DATA_DIR)

    import jnius_config

    for option in options:
        jnius_config.add_options(option)

    jnius_config.set_classpath(jar_path)
def _init_jvm_options():

    import jnius_config

    import os

    PYJNIUS_JAR_STR = 'PYJNIUS_JAR'
    IMGLYB_JAR_STR = 'IMGLYB_JAR'

    if PYJNIUS_JAR_STR not in globals():
        try:
            PYJNIUS_JAR = os.environ[PYJNIUS_JAR_STR]
        except KeyError as e:
            print(
                "Path to pyjnius.jar not defined! Use environment variable {} to define it."
                .format(PYJNIUS_JAR_STR))
            raise e

    if IMGLYB_JAR_STR not in globals():
        try:
            IMGLYB_JAR = os.environ[IMGLYB_JAR_STR]
        except KeyError as e:
            print(
                "Path to imglib2-imglyb jar not defined! Use environment variable {} to define it."
                .format(IMGLYB_JAR_STR))
            raise e

    if 'classpath' in globals():
        jnius_config.add_classpath(classpath)

    CLASSPATH_STR = 'CLASSPATH'
    if CLASSPATH_STR in os.environ:
        for path in os.environ[CLASSPATH_STR].split(jnius_config.split_char):
            jnius_config.add_classpath(path)

    jnius_config.add_classpath(PYJNIUS_JAR)
    jnius_config.add_classpath(IMGLYB_JAR)

    JVM_OPTIONS_STR = 'JVM_OPTIONS'

    if JVM_OPTIONS_STR in os.environ:
        jnius_config.add_options(*os.environ[JVM_OPTIONS_STR].split(' '))

    return jnius_config
    def __init__(self):
        """
        Constructor to set up local pipeline

        @param: file_name, the file name of the custom config file
        """
        super(LocalPipeline, self).__init__()

        self.PipelineFactory = None
        self.pipeline = None
        self.ProtobufSerializer = None

        pipeline_config.log_current_config(self.config, False)

        # set up JVM and load classes needed
        try:
            import jnius_config
            jnius_config.add_options('-Xmx16G')
            jnius_config.add_classpath(get_model_path() + '/*')
        except Exception as e:
            logger.warning(
                "Couldn't set JVM config; this might be because you're setting up the multiple instances of the local pipeline."
            )
            logger.warning(str(e))
        try:
            from jnius import autoclass
            self.PipelineFactory = autoclass(
                'edu.illinois.cs.cogcomp.pipeline.main.PipelineFactory')
            self.SerializationHelper = autoclass(
                'edu.illinois.cs.cogcomp.core.utilities.SerializationHelper')
            # self.ProtobufSerializer = autoclass('edu.illinois.cs.cogcomp.core.utilities.protobuf.ProtobufSerializer')
            self.Boolean = autoclass('java.lang.Boolean')
            self.JString = autoclass('java.lang.String')
        except Exception as e:
            logger.error(
                'Fail to load models, please check if your Java version is up to date.'
            )
            logger.error(str(e))
            return None
        self.pipeline = self.PipelineFactory.buildPipelineWithAllViews(
            self.Boolean(True))

        logger.info("pipeline has been set up")
Exemple #21
0
def get_tagger():
    global _tagger
    if _tagger is None:

        import jnius_config
        import os

        os.environ['JAVA_HOME'] = configuration.java_home
        jnius_config.add_options('-Xrs', '-Xmx128M')

        jnius_config.set_classpath(os.path.abspath(os.path.join(os.path.dirname(__file__), '../lib/ark-tweet-nlp-0.3.2/ark-tweet-nlp-0.3.2.jar')))

        from jnius import autoclass


        _Tagger = autoclass('cmu.arktweetnlp.Tagger')
        _tagger=_Tagger()
        _tagger.loadModel('/cmu/arktweetnlp/model.20120919')

    return _tagger
Exemple #22
0
    def __init__(self, name, index, k, num_threads, Xms=None, Xmx=None):
        super().__init__(name)

        if Xms and Xmx:
            # to solve Insufficient memory for the Java Runtime Environment
            jnius_config.add_options("-Xms{}".format(Xms),
                                     "-Xmx{}".format(Xmx),
                                     "-XX:-UseGCOverheadLimit")
            print("Configured options:", jnius_config.get_options())

        self.num_threads = min(num_threads, int(multiprocessing.cpu_count()))

        # initialize a ranker per thread
        self.arguments = []
        for id in tqdm(range(self.num_threads)):
            self.arguments.append({
                "id": id,
                "index": index,
                "k": k,
            })
Exemple #23
0
def init(ij_dir):
    """
    quietly set up the whole environment

    :param ij_dir: System path for Fiji.app
    :return: an instance of the net.imagej.ImageJ gateway
    """

    jnius_config.add_options('-Djava.awt.headless=true')
    imglyb_path = configure_path()
    # ImageJ
    if imglyb_path is not None:
        num_jars = set_ij_env(ij_dir, imglyb_path)
    else:
        return
    print("Added " + str(num_jars + 1) + " JARs to the Java classpath.")
    import imglyb
    from jnius import autoclass
    ImageJ = autoclass('net.imagej.ImageJ')
    return ImageJ()
def main():
  parser = argparse.ArgumentParser(description='Tensorflow hello world')
  parser.add_argument('-c', '--carbon-sdk-path', type=str, default=DEFAULT_CARBONSDK_PATH,
                      help='carbon sdk path')

  args = parser.parse_args()

  jnius_config.set_classpath(args.carbon_sdk_path)

  jnius_config.add_options('-Xrs', '-Xmx6096m')
  # jnius_config.add_options('-Xrs', '-Xmx6096m', '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5555')
  print("Start")
  start = time.time()

  just_read()

  # just_read_batch()

  end = time.time()
  print("all time: " + str(end - start))
  print("Finish")
Exemple #25
0
    def init(config):
        # Validate the JARs folder
        # Path is relative to the current working directory
        if not os.path.isdir('./java_libs/'):
            raise Exception("Java library path could not be located.")

        # Construct classpath for JVM
        import jnius_config
        jnius_config.add_options('-Xrs',
                                 '-Xmx4m')  # JVM options in comma separated
        jnius_config.set_classpath(
            './java_libs/*')  # Path for the Java library
        import jnius

        # Build Payment Method Token Recipient
        Builder = jnius.autoclass(
            'com.google.crypto.tink.apps.paymentmethodtoken.PaymentMethodTokenRecipient$Builder'
        )
        builder = Builder()
        builder.addSenderVerifyingKey(config['GoogleSigningKey'])
        builder.recipientId(config['RecipientId'])
        builder.protocolVersion(config['ProtocolVersion'])
        builder.addRecipientPrivateKey(config['RecipientPrivateKey'])
        Decryptor.paymentMethodTokenRecipient = builder.build()
Exemple #26
0
def main():
    parser = argparse.ArgumentParser(
        description='Support read carbon by manifest path in python code')
    parser.add_argument('-c',
                        '--carbon-sdk-path',
                        type=str,
                        default=DEFAULT_CARBONSDK_PATH,
                        help='carbon sdk path')

    args = parser.parse_args()

    jnius_config.set_classpath(args.carbon_sdk_path)

    jnius_config.add_options('-Xrs', '-Xmx6096m')
    print("Start")
    start = time.time()

    just_read_batch()

    just_unified_read_batch()

    end = time.time()
    print("all time: " + str(end - start))
    print("Finish")
Exemple #27
0
def _has_xmx(options):
    for option in options:
        if option.startswith('-Xmx'):
            return True
    return False


default_mem_limit = get_config("INDRA_DEFAULT_JAVA_MEM_LIMIT")
if default_mem_limit is None:
    # Set to 8g if not specified in the configuration
    default_mem_limit = '8g'

if not _has_xmx(jnius_config.get_options()):
    if not jnius_config.vm_running:
        jnius_config.add_options('-Xmx%s' % default_mem_limit)
    else:
        logger.warning("Couldn't set memory limit for Java VM because the VM "
                       "is already running.")

path_here = os.path.dirname(os.path.realpath(__file__))
cp = os.path.join(path_here, 'sources/biopax/jars/paxtools.jar')
cp_existing = os.environ.get('CLASSPATH')

if cp_existing is not None:
    os.environ['CLASSPATH'] = cp + ':' + cp_existing
else:
    os.environ['CLASSPATH'] = cp

logger.debug('Classpath when starting Java VM: %s' % os.environ['CLASSPATH'])
import os, frappe
import jasper_erpnext_report as jr

_logger = frappe.logger("jasper_erpnext_report")

try:
	import jnius_config as jc
	jr.pyjnius = True

	if not jc.vm_running:
		jc.add_options('-Djava.awt.headless=true')
	else:
		_logger.info("vm_running: {}".format(jc.vm_running))
except:
	print "jnius_config not found"
	jr.pyjnius = False

norm_path = os.path.normpath
join_path = os.path.join
dirname = os.path.dirname
parent_path = dirname(dirname(__file__))
rel_path = os.path.relpath(os.path.join(parent_path, "java"),dirname(__file__))
rel_path_curr = os.path.relpath(parent_path, os.getcwd())
try:
	os.environ['CLASSPATH'] = norm_path(join_path(parent_path,"java/lib/*")) + ":.:" + os.environ.get('CLASSPATH',"")
	print "CLASSPATH {}".format(os.environ['CLASSPATH'])
except:
	print "Error in setting java classpath."

try:
	from jnius import autoclass
Exemple #29
0
# __author__ = 'WeiFu'
from __future__ import print_function, division
import jnius_config
jnius_config.add_options('-Xrs', '-Xmx4096')
jnius_config.set_classpath('.', '/Users/WeiFu/Github/HDP_Jython/jar/weka.jar','/Users/WeiFu/Github/HDP_Jython/jar/commons-math3-3.5/commons-math3-3.5.jar')
import pdb
import random
from os import listdir
from os.path import isfile, join
from jnius import autoclass

#
#
#
#
#
#
# import weka.core.Instances
# import java.io.BufferedReader
# import java.io.FileReader
# import weka.attributeSelection.Ranker as Ranker
# import weka.attributeSelection.ReliefFAttributeEval as ReliefFAttributeEval
# import weka.attributeSelection.AttributeSelection as attributeSelection
# import weka.classifiers.functions.Logistic as Logistic
# import weka.classifiers.Evaluation as Evaluation
# import weka.core.converters.ArffSaver as Saver
# import weka.filters.unsupervised.attribute.Remove as Remove
# import weka.filters.unsupervised.instance.Randomize as Randomize
# import weka.filters.unsupervised.instance.RemoveFolds as RemoveFolds

# import weka.core.jvm as jvm
Exemple #30
0
"""Handles all imports from jnius to prevent conflicts resulting from attempts
to set JVM options while the VM is already running."""

import jnius_config

if '-Xmx4g' not in jnius_config.get_options():
    if not jnius_config.vm_running:
        jnius_config.add_options('-Xmx4g')
    else:
        warnings.warn("Couldn't set memory limit for Java VM because the VM "
                      "is already running.")

from jnius import autoclass, JavaException, cast

Exemple #31
0
"""Handles all imports from jnius to prevent conflicts resulting from attempts
to set JVM options while the VM is already running."""

import os
import warnings
import jnius_config

if "-Xmx4g" not in jnius_config.get_options():
    if not jnius_config.vm_running:
        jnius_config.add_options("-Xmx4g")
    else:
        warnings.warn("Couldn't set memory limit for Java VM because the VM " "is already running.")

path_here = os.path.dirname(os.path.realpath(__file__))
cp = path_here + "/biopax/jars/paxtools.jar"
os.environ["CLASSPATH"] = cp

from jnius import autoclass, JavaException, cast
Exemple #32
0
    os.environ['JAVA_HOME'] = expandPath('@resources/java/')

if WIN32:
    path = os.getenv('Path', '')
    path = path.split(os.pathsep)
    path.insert(0, os.path.join(os.environ['JAVA_HOME'], 'bin', 'client'))
    os.environ['Path'] = os.pathsep.join(path)

classpath = []
resources = expandPath('@resources/lib/')
for name in os.listdir(resources):
    path = os.path.join(resources, name)
    if os.path.isfile(path):
        classpath.append(path)

jnius_config.add_options('-ea', '-Xmx512m')
jnius_config.set_classpath(*classpath)

#############################################
# END JAVA VIRTUAL MACHINE SETUP
#################################

from argparse import ArgumentParser
from unittest import TestCase
from unittest.util import safe_repr

from PyQt5 import QtCore
from PyQt5 import QtTest

from eddy import APPNAME, ORGANIZATION, WORKSPACE
from eddy.core.application import Eddy
Exemple #33
0
def init(version=None,
         mem=None,
         packages=[],
         jvm_opts=[],
         redirect_io=True,
         logging='WARN',
         home_dir=None,
         boot_packages=[],
         tqdm=None):
    """
    Function necessary to be called before Terrier classes and methods can be used.
    Loads the Terrier .jar file and imports classes. Also finds the correct version of Terrier to download if no version is specified.

    Args:
        version(str): Which version of Terrier to download. Default is `None`.

         * If None, find the newest Terrier released version in MavenCentral and download it.
         * If `"snapshot"`, will download the latest build from `Jitpack <https://jitpack.io/>`_.

        mem(str): Maximum memory allocated for the Java virtual machine heap in MB. Corresponds to java `-Xmx` commandline argument. Default is 1/4 of physical memory.
        boot_packages(list(str)): Extra maven package coordinates files to load before starting Java. Default=`[]`. There is more information about loading packages in the `Terrier documentation <https://github.com/terrier-org/terrier-core/blob/5.x/doc/terrier_develop.md>`_
        packages(list(str)): Extra maven package coordinates files to load, using the Terrier classloader. Default=`[]`. See also `boot_packages` above.
        jvm_opts(list(str)): Extra options to pass to the JVM. Default=`[]`. For instance, you may enable Java assertions by setting `jvm_opts=['-ea']`
        redirect_io(boolean): If True, the Java `System.out` and `System.err` will be redirected to Pythons sys.out and sys.err. Default=True.
        logging(str): the logging level to use:

         * Can be one of `'INFO'`, `'DEBUG'`, `'TRACE'`, `'WARN'`, `'ERROR'`. The latter is the quietest.
         * Default is `'WARN'`.

        home_dir(str): the home directory to use. Default to PYTERRIER_HOME environment variable.
        tqdm: The `tqdm <https://tqdm.github.io/>`_ instance to use for progress bars within PyTerrier. Defaults to tqdm.tqdm. Available options are `'tqdm'`, `'auto'` or `'notebook'`.

   
    **Locating the Terrier .jar file:** PyTerrier is not tied to a specific version of Terrier and will automatically locate and download a recent Terrier .jar file. However, inevitably, some functionalities will require more recent Terrier versions. 
    
     * If set, PyTerrier uses the `version` init kwarg to determine the .jar file to look for.
     * If the `version` init kwarg is not set, Terrier will query MavenCentral to determine the latest Terrier release.
     * If `version` is set to `"snapshot"`, the latest .jar file build derived from the `Terrier Github repository <https://github.com/terrier-org/terrier-core/>`_ will be downloaded from `Jitpack <https://jitpack.io/>`_.
     * Otherwise the local (`~/.mvn`) and MavenCentral repositories are searched for the jar file at the given version.
     
    In this way, the default setting is to download the latest release of Terrier from MavenCentral. The user is also able to use a locally installed copy in their private Maven repository, or track the latest build of Terrier from Jitpack.
    
    """
    set_tqdm(tqdm)

    global ApplicationSetup
    global properties
    global firstInit
    global file_path
    global HOME_DIR

    # we keep a local directory
    if home_dir is not None:
        HOME_DIR = home_dir
    elif "PYTERRIER_HOME" in os.environ:
        HOME_DIR = os.environ["PYTERRIER_HOME"]
    else:
        from os.path import expanduser
        userhome = expanduser("~")
        HOME_DIR = os.path.join(userhome, ".pyterrier")
        if not os.path.exists(HOME_DIR):
            os.mkdir(HOME_DIR)

    # get the initial classpath for the JVM
    classpathTrJars = setup_terrier(HOME_DIR,
                                    version,
                                    boot_packages=boot_packages)

    # Import pyjnius and other classes
    import jnius_config
    for jar in classpathTrJars:
        jnius_config.add_classpath(jar)
    if jvm_opts is not None:
        for opt in jvm_opts:
            jnius_config.add_options(opt)
    if mem is not None:
        jnius_config.add_options('-Xmx' + str(mem) + 'm')
    from jnius import autoclass, cast

    # we only accept Java version 11 and newer; so anything starting 1. or 9. is too old
    java_version = autoclass("java.lang.System").getProperty("java.version")
    if java_version.startswith("1.") or java_version.startswith("9."):
        raise RuntimeError(
            "Pyterrier requires Java 11 or newer, we only found Java version %s;"
            +
            " install a more recent Java, or change os.environ['JAVA_HOME'] to point to the proper Java installation",
            java_version)

    tr_version = autoclass('org.terrier.Version')
    version_string = tr_version.VERSION
    if "BUILD_DATE" in dir(tr_version):
        version_string += " (built by %s on %s)" % (tr_version.BUILD_USER,
                                                    tr_version.BUILD_DATE)
    print("PyTerrier %s has loaded Terrier %s" % (__version__, version_string))
    properties = autoclass('java.util.Properties')()
    ApplicationSetup = autoclass('org.terrier.utility.ApplicationSetup')

    from .batchretrieve import BatchRetrieve, FeaturesBatchRetrieve
    from .utils import Utils
    from .datasets import get_dataset, list_datasets
    from .index import Indexer, FilesIndexer, TRECCollectionIndexer, DFIndexer, DFIndexUtils, IterDictIndexer, FlatJSONDocumentIterator, IndexingType
    from .pipelines import LTR_pipeline, XGBoostLTR_pipeline, Experiment

    # Make imports global
    globals()["autoclass"] = autoclass
    globals()["cast"] = cast
    globals()["ApplicationSetup"] = ApplicationSetup

    for sub_module_name in [
            'anserini', 'apply', 'cache', 'index', 'io', 'model', 'new', 'ltr',
            'pipelines', 'rewrite', 'text', 'transformer'
    ]:
        globals()[sub_module_name] = importlib.import_module(
            '.' + sub_module_name, package='pyterrier')

    # append the python helpers
    if packages is None:
        packages = []

    # Import other java packages
    if packages != []:
        pkgs_string = ",".join(packages)
        properties.put("terrier.mvn.coords", pkgs_string)
    ApplicationSetup.bootstrapInitialisation(properties)

    if redirect_io:
        # this ensures that the python stdout/stderr and the Java are matched
        redirect_stdouterr()
    _logging(logging)
    setup_jnius()

    globals()["get_dataset"] = get_dataset
    globals()["list_datasets"] = list_datasets
    globals()["Experiment"] = Experiment
    globals()["BatchRetrieve"] = BatchRetrieve
    globals()["Indexer"] = Indexer
    globals()["FeaturesBatchRetrieve"] = FeaturesBatchRetrieve
    globals()["TRECCollectionIndexer"] = TRECCollectionIndexer
    globals()["FilesIndexer"] = FilesIndexer
    globals()["DFIndexer"] = DFIndexer
    globals()["DFIndexUtils"] = DFIndexUtils
    globals()["IterDictIndexer"] = IterDictIndexer
    globals()["FlatJSONDocumentIterator"] = FlatJSONDocumentIterator
    globals()["Utils"] = Utils
    globals()["LTR_pipeline"] = LTR_pipeline
    globals()["XGBoostLTR_pipeline"] = XGBoostLTR_pipeline
    globals()["IndexFactory"] = autoclass(
        "org.terrier.structures.IndexFactory")
    globals()["IndexRef"] = autoclass("org.terrier.querying.IndexRef")
    globals()["IndexingType"] = IndexingType

    firstInit = True
Exemple #34
0
import sys, traceback
import jpype
import numpy as np

import os
import logging

logger = logging.getLogger(__name__)

CLASSPATH = os.environ.get("CLASSPATH", "OpenRocket-15.03.jar")

import jnius_config

jnius_config.add_options("-ea")
jnius_config.set_classpath(".", CLASSPATH)

import jnius
from jnius import autoclass


class OpenRocketInstance(object):
    """ When instantiated, this class starts up a new openrocket instance.
        This class is designed to be called using the 'with' construct. This
        will ensure that no matter what happens within that context, the 
        JVM will always be shutdown.
    """
    def __init__(self, jar_path=CLASSPATH, log_level="ERROR"):
        """ jar_path is the full path of the OpenRocket .jar file to use
            log_level can be either ERROR, WARN, USER, INFO, DEBUG or VBOSE
        """
Exemple #35
0
import jnius_config
import os

jnius_config.add_options('-Dorg.bytedeco.javacpp.nopointergc=true')
jnius_class_path = os.environ.get('JUMPY_CLASS_PATH')
if not jnius_class_path:
    raise Exception('Environment variable JUMPY_CLASS_PATH not set.')
elif not os.path.exists(jnius_class_path):
    raise Exception('File not found : ' + jnius_class_path)
jnius_config.set_classpath(jnius_class_path)

from jnius import autoclass

Nd4j = autoclass('org.nd4j.linalg.factory.Nd4j')
INDArray = autoclass('org.nd4j.linalg.api.ndarray.INDArray')
Transforms = autoclass('org.nd4j.linalg.ops.transforms.Transforms')
NDArrayIndex = autoclass('org.nd4j.linalg.indexing.NDArrayIndex')
DataBuffer = autoclass('org.nd4j.linalg.api.buffer.DataBuffer')
System = autoclass('java.lang.System')
Integer = autoclass('java.lang.Integer')
Float = autoclass('java.lang.Float')
Double = autoclass('java.lang.Double')
Shape = autoclass('org.nd4j.linalg.api.shape.Shape')
BinarySerde = autoclass('org.nd4j.serde.binary.BinarySerde')
NativeOpsHolder = autoclass('org.nd4j.nativeblas.NativeOpsHolder')
DoublePointer = autoclass('org.bytedeco.javacpp.DoublePointer')
FloatPointer = autoclass('org.bytedeco.javacpp.FloatPointer')
IntPointer = autoclass('org.bytedeco.javacpp.IntPointer')
DataTypeUtil = autoclass('org.nd4j.linalg.api.buffer.util.DataTypeUtil')
MemoryManager = autoclass('org.nd4j.linalg.memory.MemoryManager')
SameDiff = autoclass('org.nd4j.autodiff.samediff.SameDiff')
Exemple #36
0
import os, frappe
import jasper_erpnext_report as jr

_logger = frappe.get_logger("jasper_erpnext_report")

try:
    import jnius_config as jc
    jr.pyjnius = True

    if not jc.vm_running:
        jc.add_options('-Djava.awt.headless=true')
    else:
        _logger.info("vm_running: {}".format(jc.vm_running))
except:
    print "jnius_config not found"
    jr.pyjnius = False

norm_path = os.path.normpath
join_path = os.path.join
dirname = os.path.dirname
parent_path = dirname(dirname(__file__))
rel_path = os.path.relpath(os.path.join(parent_path, "java"),
                           dirname(__file__))
rel_path_curr = os.path.relpath(parent_path, os.getcwd())
try:
    os.environ['CLASSPATH'] = norm_path(join_path(
        parent_path, "java/lib/*")) + ":.:" + os.environ.get('CLASSPATH', "")
except:
    print "Error in setting java classpath."

try:
'''
python3调用自定义的java类和方法建立索引
'''

import jnius_config

jnius_config.add_options('-Xrs', '-Xmx1024m')
jnius_config.set_classpath('.', 'C:\Program Files\java\jdk1.8.0_111\lib\*')
from jnius import autoclass

myclass = autoclass("jingyi.IndexMovies")
indexclass = myclass()
try:
    indexclass.Create_index()
except Exception as err:
    print(err)
finally:
    print("success")