def weka_local_libsvm(input_dict):
if not jp.isThreadAttachedToJVM():
jp.attachThreadToJVM()
model = jp.JClass('weka.classifiers.functions.LibSVM')()
model.setOptions(common.parse_options(input_dict['params']))
sclassifier = common.serialize_weka_object(model)
return {'LibSVM_learner': sclassifier}
def weka_local_generic_learner(input_dict):
if not jp.isThreadAttachedToJVM():
jp.attachThreadToJVM()
model = jp.JClass(input_dict['weka_class'])()
model.setOptions(common.parse_options(input_dict['params']))
sclassifier = common.serialize_weka_object(model)
return {'Generic_Weka_learner': sclassifier}
def handle(config):
try:
flagcheck=True
logger.info("calling jmx for metrics")
#check for jmx hosts file
#TODO add code for handling metrics from multiple JMX hosts
#
#JAVA(libjvm='./lib/jmx/libjvm.so')
#JAVA()
jpype.attachThreadToJVM()
jmx=JMX(host='96.119.153.107',port=9999)
DS=DataStore()
for condition in config.get('if').get('jmx'):
baseline=DS.getbaseline(condition)
current=jmx.get_attr(condition.get('object'),condition.get('type'),condition.get('attribute'))
logger.debug(current)
logger.debug(str(current) + condition.get('operator') + repr(baseline))
out=eval(str(current) + condition.get('operator') + repr(baseline))
if not bool(out):
flagcheck=False
break
DS.setbaseline(current.floatValue(),baseline,condition)
del jmx
return flagcheck
except Exception,e:
print "in exception"
print e
logger.error(e)
return False
def __init__(self,arff_string,weights,nb_seeds,nb_steps):
if not jp.isThreadAttachedToJVM():
jp.attachThreadToJVM()
Monostrategy.__init__(self,arff_string,weights)
self.nb_seeds = int(nb_seeds)
self.nb_steps = int(nb_steps)
def __init__(self, *args, **kwargs):
if len(args) == 0:
extractor = 'ArticleExtractor'
elif len(args) == 1:
extractor = args[0]
else:
raise Exception('Invalid extractor param')
self.extractor_name = extractor
if kwargs.get('url'):
self.setUrl(kwargs['url'])
elif kwargs.get('html'):
self.setHtml(kwargs['html'])
try:
# make it thread-safe
if threading.activeCount() > 1:
if jpype.isThreadAttachedToJVM() == False:
jpype.attachThreadToJVM()
lock.acquire()
self.extractor = jpype.JClass(
"de.l3s.boilerpipe.extractors." + self.extractor_name).INSTANCE
finally:
lock.release()
def weka_local_arff_to_weka_instances(input_dict):
'''
Reads a dataset into a format suitable for WEKA methods
'''
if not jp.isThreadAttachedToJVM():
jp.attachThreadToJVM()
tmp = common.TemporaryFile(suffix='.arff')
tmp.writeString(input_dict['arff'])
try:
class_index = int(input_dict['class_index'])
except:
class_index = None
source = jp.JClass('weka.core.converters.ConverterUtils$DataSource')(tmp.name)
instances = source.getDataSet()
if class_index is None:
print 'Warning: class is set to the last attribute!'
class_index = instances.numAttributes() - 1
elif class_index == -1:
class_index = instances.numAttributes() - 1
instances.setClassIndex(class_index)
return {'instances': common.serialize_weka_object(instances)}
def parsing():
text = request.args.get('text')
app.logger.info('GET parsing: text(%s)' % (text))
jpype.attachThreadToJVM()
return jsonify(morphs=do_parsing_without_threading(text))
def correlation_basedfeat_sel(bunch):
"""Correlation-based Feature Subset Selection, as implemented by the CfsSubsetEval class of Weka
:param bunch: dataset
:return: new dataset
"""
if not jp.isThreadAttachedToJVM():
jp.attachThreadToJVM()
# Instances data!
data = utilities.convertBunchToWekaInstances(bunch)
Filter = jp.JClass('weka.filters.Filter')
AttributeSelection = jp.JClass('weka.filters.supervised.attribute.AttributeSelection')
attsel_filter = AttributeSelection()
CfsSubsetEval = jp.JClass('weka.attributeSelection.CfsSubsetEval')
attsel_eval = CfsSubsetEval()
GreedyStepwise = jp.JClass('weka.attributeSelection.BestFirst')
attsel_search = GreedyStepwise()
# attsel_search.setSearchBackwards(True) # True, true
attsel_filter.setEvaluator(attsel_eval)
attsel_filter.setSearch(attsel_search)
attsel_filter.setInputFormat(data)
new_instances = Filter.useFilter(data, attsel_filter)
return utilities.convertWekaInstancesToBunch(new_instances)
def run(self):
if not jp.isThreadAttachedToJVM():
jp.attachThreadToJVM()
HybridClassification = jp.JClass('jcl.learning.methods.multistrategy.samarah.HybridClassification')
self.hybrid_classification = HybridClassification()
for agent_python in self.agents_python:
self.hybrid_classification.addAgent(agent_python.get_parameters(), agent_python.get_data())
self.hybrid_classification.classify()
clusRes = self.hybrid_classification.getClusteringResult()
temp_file_url = tempfile.mkdtemp()
timenow = int(round(time.time() * 1000))
output_file_url=os.path.join(temp_file_url,"result%s.arff"%str(timenow))
ARFFWriter = jp.JClass('jcl.io.arff.ARFFWriter')
writer = ARFFWriter(output_file_url,clusRes)
writer.write()
with open(output_file_url, 'r') as f:
output_file = f.read()
return output_file
def do_concurrent_tagging(start, end, lines, result):
jpype.attachThreadToJVM()
l = [k.sentences(lines[i]) for i in range(start, end)]
result.append(l)
l = [k.nouns(lines[i]) for i in range(start, end)]
result.append(l)
return
def __init__(self, path):
jpype.attachThreadToJVM()
jpype.java.lang.System.setProperty("neo4j.ext.udc.source", "altneo4j")
self._db = EmbeddedGraphDatabase(path)
operations = GlobalGraphOperations.at(self._db)
self.nodes = Nodes(self._db, operations)
self.relationships = Relationships(self._db, operations)
def _jdbc_connect_jpype(jclassname, *args):
import jpype
# Ensure JVN is running
if not jpype.isJVMStarted():
jpype.startJVM(jpype.getDefaultJVMPath())
# Connect Jpype to JVM Thread
if not jpype.isThreadAttachedToJVM():
jpype.attachThreadToJVM()
# Initialize Type Conversion Mapping
if _converters is None:
types = jpype.java.sql.Types
types_map = {}
for i in types.__javaclass__.getClassFields():
types_map[i.getName()] = i.getStaticAttribute()
_init_converters(types_map)
global _java_array_byte
if _java_array_byte is None:
def _java_array_byte(data):
return jpype.JArray(jpype.JByte, 1)(data)
# register driver for DriverManager
jpype.JClass(jclassname)
return jpype.java.sql.DriverManager.getConnection(*args)
def __init__(self, extractor='DefaultExtractor', **kwargs):
if kwargs.get('url'):
response = requests.request('GET', kwargs['url'], headers=self.headers)
self.data = response.text
elif kwargs.get('html'):
self.data = kwargs['html']
if not isinstance(self.data, unicode):
self.data = unicode(self.data, charade.detect(self.data)['encoding'])
else:
raise Exception('No text or url provided')
try:
# make it thread-safe
if threading.activeCount() > 1:
if jpype.isThreadAttachedToJVM() == False:
jpype.attachThreadToJVM()
lock.acquire()
self.extractor = jpype.JClass(
"de.l3s.boilerpipe.extractors."+extractor).INSTANCE
finally:
lock.release()
reader = StringReader(self.data)
self.source = BoilerpipeSAXInput(InputSource(reader)).getTextDocument()
self.extractor.process(self.source)
def _jdbc_connect_jpype(jclassname, jars, libs, *driver_args):
import jpype
if not jpype.isJVMStarted():
args = []
class_path = []
if jars:
class_path.extend(jars)
class_path.extend(_get_classpath())
if class_path:
args.append('-Djava.class.path=%s' %
os.path.pathsep.join(class_path))
if libs:
# path to shared libraries
libs_path = os.path.pathsep.join(libs)
args.append('-Djava.library.path=%s' % libs_path)
# jvm_path = ('/usr/lib/jvm/java-6-openjdk'
# '/jre/lib/i386/client/libjvm.so')
jvm_path = jpype.getDefaultJVMPath()
jpype.startJVM(jvm_path, *args)
if not jpype.isThreadAttachedToJVM():
jpype.attachThreadToJVM()
if _jdbc_name_to_const is None:
types = jpype.java.sql.Types
types_map = {}
for i in types.__javaclass__.getClassFields():
types_map[i.getName()] = i.getStaticAttribute()
_init_types(types_map)
global _java_array_byte
if _java_array_byte is None:
def _java_array_byte(data):
return jpype.JArray(jpype.JByte, 1)(data)
# register driver for DriverManager
jpype.JClass(jclassname)
return jpype.java.sql.DriverManager.getConnection(*driver_args)
def joystickTimerEvent( self ) :
if self.__directConnection :
if not jpype.isThreadAttachedToJVM() :
jpype.attachThreadToJVM()
if self.joystickActive :
self.joystick.update()
if not self.joystick.getButton(0) :
if self.joystickPrevButtonDown : # if button was just recently let up
self.joystickPrevButtonDown = False
self.updateMovementCanvas( 0.0 , 0.0 )
else :
self.joystickPrevButtonDown = True
if self.joystick.getButton(1) :
self.updateMovementCanvas( self.joystick.getAxis(0) , -self.joystick.getAxis(1) )
elif self.joystick.getButton(3) :
self.updateMovementCanvas( self.joystick.getAxis(0) , -self.joystick.getAxis(1) / 2.0 )
elif self.joystick.getButton(2) :
self.updateMovementCanvas( self.joystick.getAxis(0) , -self.joystick.getAxis(1) / 4.0 )
else :
self.updateMovementCanvas( self.joystick.getAxis(0) , -self.joystick.getAxis(1) / 6.0 )
self.joystickTimer = Timer( self.joystickTimerPeriod , self.joystickTimerEvent ).start()
else : #joystick is not active
if self.joystickTimer :
self.joystickTimer.stop()
self.joystickTimer = None
self.joystickPrevButtonDown = False
self.updateMovementCanvas( 0.0 , 0.0 )
def run(self):
"""Execute a java plugin instance and collect results"""
self._log_info("Acquiring JVM lock...")
self._lock.acquire()
self._log_info("JVM lock acquired")
self._log_info("Attaching thread to JVM...")
jpype.attachThreadToJVM()
self._log_info("Thread attached to JVM")
for action in self._initial_actions:
self._log_info(
"Initial action passed to plugin: {}".format(action)
)
self._log_info("Initialising OpenStackAPI interfaces")
# interfaces
try:
metrics = Metrics(self.plugin_name)
j_metrics = jpype.JProxy(
"intel.adaptationengine.Metrics",
inst=metrics
)
except Exception, err:
LOGGER.error("Error initialising OpenStackAPI interface (Metrics)")
LOGGER.exception(err)
def __init__(self, extractor='DefaultExtractor', **kwargs):
if kwargs.get('url'):
request = urllib2.urlopen(kwargs['url'])
self.data = request.read()
encoding = request.headers['content-type'].lower().split('charset=')[-1]
if encoding.lower() == 'text/html':
encoding = chardet.detect(self.data)['encoding']
self.data = unicode(self.data, encoding)
elif kwargs.get('html'):
self.data = kwargs['html']
if not isinstance(self.data, unicode):
self.data = unicode(self.data, chardet.detect(self.data)['encoding'])
else:
raise Exception('No text or url provided')
try:
# make it thread-safe
if threading.activeCount() > 1:
if jpype.isThreadAttachedToJVM() == False:
jpype.attachThreadToJVM()
lock.acquire()
self.extractor = jpype.JClass(
"de.l3s.boilerpipe.extractors."+extractor).INSTANCE
finally:
lock.release()
reader = StringReader(self.data)
self.source = BoilerpipeSAXInput(InputSource(reader)).getTextDocument()
self.extractor.process(self.source)
def __init__(self, extractor='DefaultExtractor', **kwargs):
if kwargs.get('url'):
request = urllib2.Request(kwargs['url'], headers=self.headers)
connection = urllib2.urlopen(request)
self.data = connection.read()
encoding = connection.headers['content-type'].lower().split('charset=')[-1]
if encoding.lower() == 'text/html':
encoding = charade.detect(self.data)['encoding']
# self.data = unicode(self.data, 'gbk')
#self.data = self.data.decode(encoding, 'ignore')
try:
self.data = unicode(self.data, charade.detect(self.data)['encoding'])
except UnicodeError:
encoding = charade.detect(self.data)['encoding']
self.data = self.data.decode(encoding, 'ignore')
elif kwargs.get('html'):
self.data = kwargs['html']
if not isinstance(self.data, unicode):
try:
self.data = unicode(self.data,'gbk')
#self.data = unicode(self.data, charade.detect(self.data)['encoding'])
#try:
# self.data = unicode(self.data, charade.detect(self.data)['encoding'])
except UnicodeError:
encoding = charade.detect(self.data)['encoding']
print "charset is :",encoding
self.data = self.data.decode(encoding, 'ignore')
## Extractor(extractor='ArticleExtractor',file='/tmp/a.html')
elif kwargs.get('file'):
Path = kwargs['file']
f = open(Path, 'r')
self.data = f.read()
f.close()
if not isinstance(self.data, unicode):
try:
self.data = unicode(self.data, charade.detect(self.data)['encoding'])
except UnicodeError:
encoding = charade.detect(self.data)['encoding']
self.data = self.data.decode(encoding, 'ignore')
else:
raise Exception('No text or url provided')
try:
# make it thread-safe
if threading.activeCount() > 1:
if jpype.isThreadAttachedToJVM() == False:
jpype.attachThreadToJVM()
lock.acquire()
self.extractor = jpype.JClass(
"de.l3s.boilerpipe.extractors."+extractor).INSTANCE
finally:
lock.release()
reader = StringReader(self.data)
self.source = BoilerpipeSAXInput(InputSource(reader)).getTextDocument()
self.extractor.process(self.source)
def serialize_weka_object(obj):
if not jp.isThreadAttachedToJVM():
jp.attachThreadToJVM()
s = jp.JClass('weka.core.SerializationHelper')
tfile = TemporaryFile(flags='wb+')
s.write(tfile.name, obj)
return b64encode(tfile.fp.read())
def deserialize_weka_object(objString):
if not jp.isThreadAttachedToJVM():
jp.attachThreadToJVM()
d = jp.JClass('weka.core.SerializationHelper')
tfile = TemporaryFile(flags='wb+')
tfile.writeString(b64decode(objString))
return d.read(tfile.name)
def print_classifier(self):
if not jp.isThreadAttachedToJVM():
jp.attachThreadToJVM()
try:
classifier = common.deserialize_weka_object(self.sclassifier)
return classifier.toString()
except:
raise Exception("Only WEKA classifiers/models supported. Please provide a valid WEKA learner.")
def __init__(self, extractor='DefaultExtractor', **kwargs):
if kwargs.get('url'):
request = urllib2.Request(kwargs['url'], headers=self.headers)
# Version without headers
# request = urllib2.Request(kwargs['url'])
connection = urllib2.urlopen(request)
self.data = connection.read()
encoding = connection.headers['content-type'].lower().split('charset=')[-1]
# Try requests
# request = requests.get(kwargs['url'], headers=self.headers, verify=False)
# self.data = request.text
# encoding = request.headers['content-type'].lower().split('charset=')[-1]
if encoding.lower() == 'text/html':
encoding = charade.detect(self.data)['encoding']
try:
self.data = unicode(self.data, encoding, errors='replace')
except LookupError as e:
print e
import ipdb; ipdb.set_trace() # XXX BREAKPOINT
elif kwargs.get('html'):
self.data = kwargs['html']
if not isinstance(self.data, unicode):
self.data = unicode(self.data, charade.detect(self.data)['encoding'], errors='replace')
import ipdb; ipdb.set_trace() # XXX BREAKPOINT
else:
raise Exception('No text or url provided')
try:
# make it thread-safe
if threading.activeCount() > 1:
if jpype.isThreadAttachedToJVM() == False:
jpype.attachThreadToJVM()
lock.acquire()
self.extractor = jpype.JClass(
"de.l3s.boilerpipe.extractors."+extractor).INSTANCE
finally:
lock.release()
reader = StringReader(self.data)
self.source = BoilerpipeSAXInput(InputSource(reader)).getTextDocument()
self.extractor.process(self.source)
def _cypher_engine(self):
if not hasattr(self, '__cached_cypher_engine'):
try:
import jpype
jpype.attachThreadToJVM()
except Exception:
pass
self.__cached_cypher_engine = CypherEngine(self)
return self.__cached_cypher_engine
def weka_local_rep_tree(input_dict):
'''A REP Tree, which is a fast decision tree learner. Builds a decision/regression tree using information gain/variance and prunes it using reduced-error pruning
'''
if not jp.isThreadAttachedToJVM():
jp.attachThreadToJVM()
model = jp.JClass('weka.classifiers.trees.REPTree')()
model.setOptions(common.parse_options(input_dict['params']))
sclassifier = common.serialize_weka_object(model)
return {'REPTree_learner': sclassifier}
def weka_local_multilayer_perceptron(input_dict):
'''Feedforward artificial neural network, using backpropagation to classify instances
'''
if not jp.isThreadAttachedToJVM():
jp.attachThreadToJVM()
model = jp.JClass('weka.classifiers.functions.MultilayerPerceptron')()
model.setOptions(common.parse_options(input_dict['params']))
sclassifier = common.serialize_weka_object(model)
return {'Multilayer_Perceptron_learner': sclassifier}
def weka_local_smo(input_dict):
'''A support vector classifier, trained using the Sequential Minimal Optimization (SMO) algorithm
'''
if not jp.isThreadAttachedToJVM():
jp.attachThreadToJVM()
model = jp.JClass('weka.classifiers.functions.SMO')()
model.setOptions(common.parse_options(input_dict['params']))
sclassifier = common.serialize_weka_object(model)
return {'SMO_learner': sclassifier}
def do_concurrent_tagging(isNoun, start, end, lines, result):
jpype.attachThreadToJVM()
if (isNoun == True):
# check nouns
l = [k.nouns(lines[i]) for i in range(start, end)]
else:
# check sentences
l = [k.sentences(lines[i]) for i in range(start, end)]
result.append(l)
return 0
def weka_local_random_forest(input_dict):
'''Random Forest learner by Weka
'''
if not jp.isThreadAttachedToJVM():
jp.attachThreadToJVM()
model = jp.JClass('weka.classifiers.trees.RandomForest')()
model.setOptions(common.parse_options(input_dict['params']))
sclassifier = common.serialize_weka_object(model)
return {'RandomForest_learner': sclassifier}
def weka_local_naive_bayes(input_dict):
'''Naive Bayes classifier provided by Weka. Naive Bayes is a simple probabilistic classifier based on applying the Bayes' theorem.
'''
if not jp.isThreadAttachedToJVM():
jp.attachThreadToJVM()
model = jp.JClass('weka.classifiers.bayes.NaiveBayes')()
model.setOptions(common.parse_options(input_dict['params']))
sclassifier = common.serialize_weka_object(model)
return {'Naive_Bayes_learner': sclassifier}
def weka_local_k_star(input_dict):
'''Instance-Based learner K* by Weka
'''
if not jp.isThreadAttachedToJVM():
jp.attachThreadToJVM()
model = jp.JClass('weka.classifiers.lazy.KStar')()
model.setOptions(common.parse_options(input_dict['params']))
sclassifier = common.serialize_weka_object(model)
return {'KStar_learner': sclassifier}
def _wrapper(*args, **kwargs):
import jpype
if not jpype.isThreadAttachedToJVM():
jpype.attachThreadToJVM()
return wrapped(*args, **kwargs)
def __init__(self,
filename,
meta=True,
java_memory='512m',
read_mode='auto',
series=0):
global loci
super(BioformatsReader, self).__init__()
if read_mode not in ['auto', 'jpype', 'stringbuffer', 'javacasting']:
raise ValueError('Invalid read_mode value.')
# Make sure that file exists before starting java
if not os.path.isfile(filename):
raise IOError('The file "{}" does not exist.'.format(filename))
# Start java VM and initialize logger (globally)
if not jpype.isJVMStarted():
loci_path = _find_jar()
# If we can turn off string auto-conversion, do so,
# since this is the recommended practice.
if LooseVersion(jpype.__version__) >= LooseVersion('0.7.0'):
startJVM_kwargs = {'convertStrings': False}
else:
startJVM_kwargs = {
} # convertStrings kwarg not supported for earlier jpype versions
jpype.startJVM(jpype.getDefaultJVMPath(), '-ea',
'-Djava.class.path=' + loci_path,
'-Xmx' + java_memory, **startJVM_kwargs)
log4j = jpype.JPackage('org.apache.log4j')
log4j.BasicConfigurator.configure()
log4j_logger = log4j.Logger.getRootLogger()
log4j_logger.setLevel(log4j.Level.ERROR)
if not jpype.isThreadAttachedToJVM():
jpype.attachThreadToJVM()
loci = jpype.JPackage('loci')
# Initialize reader and metadata
self.filename = str(filename)
self.rdr = loci.formats.ChannelSeparator(loci.formats.ChannelFiller())
# patch for issue with ND2 files and the Chunkmap implemented in 5.4.0
# See https://github.com/openmicroscopy/bioformats/issues/2955
# circumventing the reserved keyword 'in'
mo = getattr(loci.formats, 'in').DynamicMetadataOptions()
mo.set('nativend2.chunkmap', 'False') # Format Bool as String
self.rdr.setMetadataOptions(mo)
if meta:
self._metadata = loci.formats.MetadataTools.createOMEXMLMetadata()
self.rdr.setMetadataStore(self._metadata)
self.rdr.setId(self.filename)
if meta:
self.metadata = MetadataRetrieve(self._metadata)
# Checkout reader dtype and define read mode
isLittleEndian = self.rdr.isLittleEndian()
LE_prefix = ['>', '<'][isLittleEndian]
FormatTools = loci.formats.FormatTools
self._dtype_dict = {
FormatTools.INT8: 'i1',
FormatTools.UINT8: 'u1',
FormatTools.INT16: LE_prefix + 'i2',
FormatTools.UINT16: LE_prefix + 'u2',
FormatTools.INT32: LE_prefix + 'i4',
FormatTools.UINT32: LE_prefix + 'u4',
FormatTools.FLOAT: LE_prefix + 'f4',
FormatTools.DOUBLE: LE_prefix + 'f8'
}
self._dtype_dict_java = {}
for loci_format in self._dtype_dict.keys():
self._dtype_dict_java[loci_format] = \
(FormatTools.getBytesPerPixel(loci_format),
FormatTools.isFloatingPoint(loci_format),
isLittleEndian)
# Set the correct series and initialize the sizes
self.size_series = self.rdr.getSeriesCount()
if series >= self.size_series or series < 0:
self.rdr.close()
raise IndexError('Series index out of bounds.')
self._series = series
self._change_series()
# Set read mode. When auto, tryout fast and check the image size.
if read_mode == 'auto':
Jarr = self.rdr.openBytes(0)
if isinstance(Jarr[:], np.ndarray):
read_mode = 'jpype'
else:
warn('Due to an issue with JPype 0.6.0, reading is slower. '
'Please consider upgrading JPype to 0.6.1 or later.')
try:
im = self._jbytearr_stringbuffer(Jarr)
im.reshape(self._sizeRGB, self._sizeX, self._sizeY)
except (AttributeError, ValueError):
read_mode = 'javacasting'
else:
read_mode = 'stringbuffer'
self.read_mode = read_mode
# Define the names of the standard per frame metadata.
self.frame_metadata = {}
if meta:
if hasattr(self.metadata, 'PlaneDeltaT'):
self.frame_metadata['t_s'] = 'PlaneDeltaT'
if hasattr(self.metadata, 'PlanePositionX'):
self.frame_metadata['x_um'] = 'PlanePositionX'
if hasattr(self.metadata, 'PlanePositionY'):
self.frame_metadata['y_um'] = 'PlanePositionY'
if hasattr(self.metadata, 'PlanePositionZ'):
self.frame_metadata['z_um'] = 'PlanePositionZ'
def read_stderr():
jpype.attachThreadToJVM()
for line in xenon.conversions.read_lines(J.streams.getStderr()):
print(job_config.name + ": " + line, file=sys.stderr, flush=True)
def validate(request):
""" Handle Validate api request """
if request.method == 'GET':
""" Return all validate api request """
query = ValidateFileUpload.objects.all()
serializer = ValidateSerializer(query, many=True)
return Response(serializer.data)
elif request.method == 'POST':
""" Return validate tool result on the post file"""
serializer = ValidateSerializer(data=request.data)
if serializer.is_valid():
if (jpype.isJVMStarted() == 0):
""" If JVM not already started, start it, attach a Thread and start processing the request """
classpath = settings.JAR_ABSOLUTE_PATH
jpype.startJVM(jpype.getDefaultJVMPath(), "-ea",
"-Djava.class.path=%s" % classpath)
"""Attach a Thread and start processing the request """
jpype.attachThreadToJVM()
package = jpype.JPackage("org.spdx.tools")
verifyclass = package.Verify
query = ValidateFileUpload.objects.create(
owner=request.user, file=request.data.get('file'))
uploaded_file = str(query.file)
uploaded_file_path = str(query.file.path)
try:
if request.FILES["file"]:
""" Call the java function with parameter"""
retval = verifyclass.verify(uploaded_file_path)
if (len(retval) > 0):
result = "The following error(s)/warning(s) were raised: " + str(
retval)
returnstatus = status.HTTP_400_BAD_REQUEST
httpstatus = 400
jpype.detachThreadFromJVM()
else:
result = "This SPDX Document is valid."
returnstatus = status.HTTP_201_CREATED
httpstatus = 201
jpype.detachThreadFromJVM()
else:
result = "File Not Uploaded"
returnstatus = status.HTTP_400_BAD_REQUEST
httpstatus = 400
jpype.detachThreadFromJVM()
except jpype.JavaException as ex:
""" Error raised by verifyclass.verify without exiting the application"""
result = jpype.JavaException.message(
ex
) #+ "This SPDX Document is not a valid RDF/XML or tag/value format"
returnstatus = status.HTTP_400_BAD_REQUEST
httpstatus = 400
jpype.detachThreadFromJVM()
except:
""" Other errors raised"""
result = format_exc()
returnstatus = status.HTTP_400_BAD_REQUEST
httpstatus = 400
jpype.detachThreadFromJVM()
query.result = result
query.status = httpstatus
ValidateFileUpload.objects.filter(file=uploaded_file).update(
result=result, status=httpstatus)
serial = ValidateSerializerReturn(instance=query)
return Response(serial.data, status=returnstatus)
else:
return Response(serializer.errors,
status=status.HTTP_400_BAD_REQUEST)
def compare(request):
""" Handle Compare api request """
if request.method == 'GET':
""" Return all compare api request """
query = CompareFileUpload.objects.all()
serializer = CompareSerializerReturn(query, many=True)
return Response(serializer.data)
elif request.method == 'POST':
""" Return compare tool result on the post file"""
serializer = CompareSerializer(data=request.data)
if serializer.is_valid():
if (jpype.isJVMStarted() == 0):
""" If JVM not already started, start it, attach a Thread and start processing the request """
classpath = settings.JAR_ABSOLUTE_PATH
jpype.startJVM(jpype.getDefaultJVMPath(), "-ea",
"-Djava.class.path=%s" % classpath)
""" Attach a Thread and start processing the request """
jpype.attachThreadToJVM()
package = jpype.JPackage("org.spdx.tools")
verifyclass = package.Verify
compareclass = package.CompareMultpleSpdxDocs
result = ""
message = "Success"
erroroccurred = False
rfilename = request.POST["rfilename"]
query = CompareFileUpload.objects.create(
owner=request.user,
file1=request.data.get('file1'),
file2=request.data.get('file2'),
rfilename=rfilename,
)
uploaded_file1 = str(query.file1)
uploaded_file2 = str(query.file2)
uploaded_file1_path = str(query.file1.path)
uploaded_file2_path = str(query.file2.path)
try:
if (request.FILES["file1"] and request.FILES["file2"]):
""" Saving file to the media directory """
if (extensionGiven(rfilename) == False):
rfilename = rfilename + ".xlsx"
file1 = request.FILES["file1"]
file2 = request.FILES["file2"]
folder = "/".join(uploaded_file1_path.split('/')[:-1])
callfunc = [folder + "/" + rfilename]
callfunc.append(uploaded_file1_path)
callfunc.append(uploaded_file2_path)
""" Call the java function with parameters as list"""
retval1 = verifyclass.verifyRDFFile(uploaded_file1_path)
if (len(retval1) > 0):
erroroccurred = True
message = "The following error(s)/warning(s) were raised by " + str(
uploaded_file1) + ": " + str(retval1)
retval2 = verifyclass.verifyRDFFile(uploaded_file2_path)
if (len(retval2) > 0):
erroroccurred = True
message += "The following error(s)/warning(s) were raised by " + str(
uploaded_file2) + ": " + str(retval2)
try:
compareclass.onlineFunction(callfunc)
"""Return only the path starting with MEDIA_URL"""
index = folder.split("/").index('media')
result = "/" + ("/".join(
folder.split("/")[index:])) + '/' + rfilename
returnstatus = status.HTTP_201_CREATED
httpstatus = 201
except:
message += "While running compare tool " + format_exc()
returnstatus = status.HTTP_400_BAD_REQUEST
httpstatus = 400
if (erroroccurred == False):
returnstatus = status.HTTP_201_CREATED
httpstatus = 201
else:
returnstatus = status.HTTP_406_BAD_REQUEST
httpstatus = 406
jpype.detachThreadFromJVM()
else:
message = "File Not Uploaded"
returnstatus = status.HTTP_400_BAD_REQUEST
httpstatus = 400
jpype.detachThreadFromJVM()
except jpype.JavaException as ex:
""" Error raised by verifyclass.verify without exiting the application"""
message = jpype.JavaException.message(
ex
) #+ "This SPDX Document is not a valid RDF/XML or tag/value format"
returnstatus = status.HTTP_400_BAD_REQUEST
httpstatus = 400
jpype.detachThreadFromJVM()
except:
message = format_exc()
returnstatus = status.HTTP_400_BAD_REQUEST
httpstatus = 400
jpype.detachThreadFromJVM()
query.message = message
query.result = result
query.status = httpstatus
CompareFileUpload.objects.filter(file1=uploaded_file1).filter(
file2=uploaded_file2).update(message=message,
result=result,
status=httpstatus)
serial = CompareSerializerReturn(instance=query)
return Response(serial.data, status=returnstatus)
else:
return Response(serializer.errors,
status=status.HTTP_400_BAD_REQUEST)
def do_GET(self):
url = urlparse(self.path)
if url.path == '/tile':
query_components = parse_qs(urlparse(self.path).query)
dataset = query_components["dataset"][0]
level = query_components["level"][0]
x = query_components["x"][0]
y = query_components["y"][0]
time_from = transToStamp(query_components["time_from"][0])
time_to = transToStamp(query_components["time_to"][0])
# start = time.time()
tile_quadkey = mercantile.quadkey(int(x), int(y), int(level))
tile_id = quadkey_to_num(tile_quadkey)
print tile_id, str(time_from), str(time_to)
jpype.attachThreadToJVM()
hbase_response = cp.spatialSum("nycTaxi", tile_id, str(time_from),
str(time_to))
print hbase_response
# end = time.time()
# print "query_time" + ": " + str(end - start)
#
#
# q_time1.append(end - start)
self.send_response(200)
self.send_header('Content-type', 'application/json')
self.send_header('Access-Control-Allow-Origin', '*')
self.end_headers()
self.wfile.write(hbase_response)
# if len(q_time1) == 20:
# print "hundred"
# plt.plot(range(len(q_time1)), q_time1)
# plt.ylabel('tile query time')
# plt.axis([1, 20, 0, 2])
# print float(sum(q_time1)) / len(q_time1)
# plt.show()
return
if url.path == '/time_series':
query_components = parse_qs(urlparse(self.path).query)
dataset = query_components["dataset"][0]
level = query_components["level"][0]
bounds = query_components["bounds"][0]
bounds = bounds.split(",")
time_from = transToStamp(query_components["time_from"][0])
time_to = transToStamp(query_components["time_to"][0])
start = time.time()
new_bounds = check_bounds(bounds)
tiles = mercantile.tiles(new_bounds[0],new_bounds[1],new_bounds[2], \
new_bounds[3], [int(level), ])
tile_numbers = []
for tile in list(tiles):
t_quadkey = mercantile.quadkey(tile)
t_num = quadkey_to_num(t_quadkey)
tile_numbers.append(t_num)
tile_numbers = ' '.join(str(x) for x in tile_numbers)
jpype.attachThreadToJVM()
res = cp.timeSeriesCount(dataset, tile_numbers, str(time_from),
str(time_to))
end = time.time()
print "query_time" + ": " + str(end - start)
# q_time2.append(end - start)
self.send_response(200)
self.send_header('Content-type', 'application/json')
self.send_header('Access-Control-Allow-Origin', '*')
self.end_headers()
self.wfile.write(res)
# if len(q_time2) == 20:
# print "hundred"
# plt.plot(range(len(q_time2)), q_time2)
# plt.ylabel('timeseries query time')
# plt.axis([1, 20, 0, 1])
# print float(sum(q_time2)) / len(q_time2)
# plt.show()
return
def search_data(request):
logger = logging.getLogger(__name__)
searchtext = request.POST.get('searchtext', '')
# initilize genesisChunk...
genesisChunks = []
candidate = []
syntax = []
result = []
# receving data..
if jpype.isJVMStarted():
jpype.attachThreadToJVM()
# model setting (아담 / Noun)
sentences_vocab = SentenceReader(
'/usr/local/web/encyclopedia/dictionary/genesis1p.txt')
sentences_train = SentenceReader(
'/usr/local/web/encyclopedia/dictionary/genesis1p.txt')
model = Word2Vec()
model.build_vocab(sentences_vocab)
model.train(sentences_train,
total_examples=model.corpus_count,
epochs=model.epochs)
model.save('model')
model = Word2Vec.load('model')
# model setting
sentences_vocab = SentenceReader(
'/usr/local/web/encyclopedia/dictionary/genesis2p.txt')
sentences_train = SentenceReader(
'/usr/local/web/encyclopedia/dictionary/genesis2p.txt')
modelOne = Word2Vec()
modelOne.build_vocab(sentences_vocab)
modelOne.train(sentences_train,
total_examples=modelOne.corpus_count,
epochs=modelOne.epochs)
modelOne.save('model')
modelOne = Word2Vec.load('model')
# search text
posMaker = Twitter()
searchMetatext = posMaker.pos(searchtext)
# (아담/Noun)
searchtextZero = str("/".join(searchMetatext[0]))
# (Noun)
searchtextOne = searchMetatext[0][1]
# Previously Items are attached form base /
# Later Items are attached from result /
for item in model.most_similar(positive=[searchtextZero], topn=100):
candidate.append(item[0])
for item in modelOne.most_similar(positive=[searchtextOne], topn=100):
syntax.append(item[0])
result.append(searchtext)
index = 0
for item in candidate:
if ((item.split("/")[1] == syntax[index])
or (item.split("/")[1] == "Noun")):
result.append(item.split("/")[0])
logger.error(result)
logger.error(syntax)
context = json.dumps(result, ensure_ascii=False)
return JsonResponse(context, safe=False)
def __init__(self,
filename,
meta=True,
java_memory='512m',
read_mode='auto',
series=0):
global loci
super(BioformatsReader, self).__init__()
if read_mode not in ['auto', 'jpype', 'stringbuffer', 'javacasting']:
raise ValueError('Invalid read_mode value.')
# Make sure that file exists before starting java
if not os.path.isfile(filename):
raise IOError('The file "{}" does not exist.'.format(filename))
# Start java VM and initialize logger (globally)
if not jpype.isJVMStarted():
loci_path = _find_jar()
jpype.startJVM(jpype.getDefaultJVMPath(), '-ea',
'-Djava.class.path=' + loci_path,
'-Xmx' + java_memory)
log4j = jpype.JPackage('org.apache.log4j')
log4j.BasicConfigurator.configure()
log4j_logger = log4j.Logger.getRootLogger()
log4j_logger.setLevel(log4j.Level.ERROR)
if not jpype.isThreadAttachedToJVM():
jpype.attachThreadToJVM()
loci = jpype.JPackage('loci')
# Initialize reader and metadata
self.filename = str(filename)
self.rdr = loci.formats.ChannelSeparator(loci.formats.ChannelFiller())
if meta:
self._metadata = loci.formats.MetadataTools.createOMEXMLMetadata()
self.rdr.setMetadataStore(self._metadata)
self.rdr.setId(self.filename)
if meta:
self.metadata = MetadataRetrieve(self._metadata)
# Checkout reader dtype and define read mode
isLittleEndian = self.rdr.isLittleEndian()
LE_prefix = ['>', '<'][isLittleEndian]
FormatTools = loci.formats.FormatTools
self._dtype_dict = {
FormatTools.INT8: 'i1',
FormatTools.UINT8: 'u1',
FormatTools.INT16: LE_prefix + 'i2',
FormatTools.UINT16: LE_prefix + 'u2',
FormatTools.INT32: LE_prefix + 'i4',
FormatTools.UINT32: LE_prefix + 'u4',
FormatTools.FLOAT: LE_prefix + 'f4',
FormatTools.DOUBLE: LE_prefix + 'f8'
}
self._dtype_dict_java = {}
for loci_format in self._dtype_dict.keys():
self._dtype_dict_java[loci_format] = \
(FormatTools.getBytesPerPixel(loci_format),
FormatTools.isFloatingPoint(loci_format),
isLittleEndian)
# Set the correct series and initialize the sizes
self.size_series = self.rdr.getSeriesCount()
if series >= self.size_series or series < 0:
self.rdr.close()
raise IndexError('Series index out of bounds.')
self._series = series
self._change_series()
# Set read mode. When auto, tryout fast and check the image size.
if read_mode == 'auto':
Jarr = self.rdr.openBytes(0)
if isinstance(Jarr[:], np.ndarray):
read_mode = 'jpype'
else:
warn('Due to an issue with JPype 0.6.0, reading is slower. '
'Please consider upgrading JPype to 0.6.1 or later.')
try:
im = self._jbytearr_stringbuffer(Jarr)
im.reshape(self._sizeRGB, self._sizeX, self._sizeY)
except (AttributeError, ValueError):
read_mode = 'javacasting'
else:
read_mode = 'stringbuffer'
self.read_mode = read_mode
# Define the names of the standard per frame metadata.
self.frame_metadata = {}
if meta:
if hasattr(self.metadata, 'PlaneDeltaT'):
self.frame_metadata['t_s'] = 'PlaneDeltaT'
if hasattr(self.metadata, 'PlanePositionX'):
self.frame_metadata['x_um'] = 'PlanePositionX'
if hasattr(self.metadata, 'PlanePositionY'):
self.frame_metadata['y_um'] = 'PlanePositionY'
if hasattr(self.metadata, 'PlanePositionZ'):
self.frame_metadata['z_um'] = 'PlanePositionZ'
def tokenize_kkma_noun(doc):
jpype.attachThreadToJVM()
token_doc = [
'/'.join(word) for word in kkma.pos(doc) if word[1] in filter_kkma
]
return token_doc
def fetchMessage(request):
"""
input: GET
(str) text: message
(int) type: 0 - initialize, 1 - question, 2 - normal message, 3 - end session
(int) index: -1 - initialize, 0 - normal message, 1 - 어휘, 2 - 문법, 3 - 발음, 4 - 기타
(str) userid: id
return: JSON
(str) text: list of messages
(int) type: 0 - bot, 1 - question, 2 - user, 3 - end, 4 - init
(int) success: 0 - fail, 1 - success
(str) userid: id
"""
_text = str(request.GET['text'])
_type = int(request.GET['type'])
_userid = str(request.GET['userid'])
# Initialize Bot
if _type == 0:
topic = request.GET.get('topic')
hasTense = False
txtfile = 'scenario/' + Filename.objects.get(topic=topic +
".xlsx").filename
lines = parser(txtfile).split('\n')
bot = Bot(line=lines)
bot.hasTense = hasTense
msg = [bot.lines[bot.index]]
_userid = bot.id
users[_userid] = bot
js = {
"text": msg,
"type": 4,
"success": 1,
"userid": _userid,
"nextline": "",
"original": _text,
"hasTense": bot.hasTense,
}
else:
bot = users[_userid]
# _index = int(request.GET['index'])
if _type == 2:
pass
# _questionType = ["어휘","문법","발음","기타"]
# line = bot.current_line()
# words = line.split()
# word = words[int(_text)]
# _questiontype = _index-1
# q = QuestionType(questionType=_questiontype,questionID=int(_text),dialogueIndex=bot.index)
# q.save()
# msg = bot.next_line()
# if msg[-1] == False:
# js = {
# "text": [""],
# "type": 3,
# "success": 1,
# "userid": _userid,
# }
# else:
# js = {
# "text": msg,
# "type": 0 if (bot.index-1)%2==0 else 2,
# "success": 1,
# "userid": _userid,
# }
elif _type == 1:
corrected_result = spell_checker.check(_text).as_dict()
num_errors = corrected_result['errors']
correct_sent = []
for x in corrected_result['words']:
correct_sent.append((x, corrected_result['words'][x]))
# if corrected_result['words'][x] == 0:
# correct_sent += "<span>" + x + " </span>"
# elif corrected_result['words'][x] == 1:
# correct_sent += "<span className='redFont'>" + x + " </span>"
# elif corrected_result['words'][x] == 2:
# correct_sent += "<span className='greenFont'>" + x + " </span>"
# elif corrected_result['words'][x] == 3:
# correct_sent += "<span className='purpleFont'>" + x + " </span>"
# elif corrected_result['words'][x] == 4:
# correct_sent += "<span className='blueFont'>" + x + " </span>"
userline = bot.lines[bot.index]
ul_s = userline.split('(')
n_list = []
for x in ul_s:
if ')' in x:
n_list.append(x[:x.index(')')])
if jpype.isJVMStarted():
jpype.attachThreadToJVM()
# bot.replace_pairs.extend(match_nouns(n_list, _text))
msg, next_line = bot.next_line()
msg = process_msg(msg, bot.tense)
if msg[-1] == False:
js = {
"text": msg[:-1],
"type": 1,
"success": 1,
"userid": _userid,
"nextline": next_line,
"original": _text,
"errorcount": num_errors,
"corrected": correct_sent,
}
else:
js = {
"text": msg,
"type": 0,
"success": 1,
"userid": _userid,
"nextline": next_line,
"original": _text,
"errorcount": num_errors,
"corrected": correct_sent,
}
return HttpResponse(json.dumps(js), content_type="application/json")
def decorator(*args, **kwargs):
if not jpype.isThreadAttachedToJVM():
jpype.attachThreadToJVM()
return fn(*args, **kwargs)
def pos(self, sentence):
jpype.attachThreadToJVM()
return self.komoran.pos(sentence)
def Normalize(self, request, context):
jpype.attachThreadToJVM(
) # XXX: Performance Incresed. (Still don't know yet)
return global_pb2.StringArrayResponse(
results=[self.engine.normalize(request.payload)],
options=request.options) # FIXME: proto can't handle this.
def _jdbc_connect_jpype(jclassname, url, driver_args, jars, libs):
import jpype
if not jpype.isJVMStarted():
args = []
class_path = []
if jars:
class_path.extend(jars)
class_path.extend(_get_classpath())
if class_path:
args.append('-Djava.class.path=%s' %
os.path.pathsep.join(class_path))
if libs:
# path to shared libraries
libs_path = os.path.pathsep.join(libs)
args.append('-Djava.library.path=%s' % libs_path)
# jvm_path = ('/usr/lib/jvm/java-6-openjdk'
# '/jre/lib/i386/client/libjvm.so')
jvm_path = jpype.getDefaultJVMPath()
global old_jpype
if hasattr(jpype, '__version__'):
try:
ver_match = re.match('\d+\.\d+', jpype.__version__)
if ver_match:
jpype_ver = float(ver_match.group(0))
if jpype_ver < 0.7:
old_jpype = True
except ValueError:
pass
if old_jpype:
jpype.startJVM(jvm_path, *args)
else:
jpype.startJVM(jvm_path, *args, ignoreUnrecognized=True,
convertStrings=True)
if not jpype.isThreadAttachedToJVM():
jpype.attachThreadToJVM()
jpype.java.lang.Thread.currentThread().setContextClassLoader(jpype.java.lang.ClassLoader.getSystemClassLoader())
if _jdbc_name_to_const is None:
types = jpype.java.sql.Types
types_map = {}
if old_jpype:
for i in types.__javaclass__.getClassFields():
const = i.getStaticAttribute()
types_map[i.getName()] = const
else:
for i in types.class_.getFields():
if jpype.java.lang.reflect.Modifier.isStatic(i.getModifiers()):
const = i.get(None)
types_map[i.getName()] = const
_init_types(types_map)
global _java_array_byte
if _java_array_byte is None:
def _java_array_byte(data):
return jpype.JArray(jpype.JByte, 1)(data)
# register driver for DriverManager
jpype.JClass(jclassname)
if isinstance(driver_args, dict):
Properties = jpype.java.util.Properties
info = Properties()
for k, v in driver_args.items():
info.setProperty(k, v)
dargs = [ info ]
else:
dargs = driver_args
return jpype.java.sql.DriverManager.getConnection(url, *dargs)
def attachThread():
jpype.attachThreadToJVM()
def prod_execute_sql(conn_type='prod', sqltype='insert', sql=''):
"""
:param conn_type: 连接类型
prod 生产集群环境
test 测试集群环境
:param sqltype:
:param sql:
:return:
"""
jars_path = '/you_filed_algos/jars/'
dirver = "org.apache.hive.jdbc.HiveDriver"
is_prod_env = True
PROD = 'prod' # 生产环境
TEST = 'test' # 测试环境
#print('**** prod_execute_sql ****')
#print('* conn_type=', conn_type)
if conn_type == PROD:
# 生产集群使用KUDU
url = "jdbc:hive2://hadoop-pro-017:7180/default;ssl=true;sslTrustStore=/you_filed_algos/prod-cm-auto-global_truststore.jks;principal=impala/[email protected]"
elif conn_type == TEST:
# 开发集群使用KUDU
# jdbc:hive2://bigdata-dev-014:7180/;ssl=true;sslTrustStore=/home/user/java/keytab/cm-auto-global_truststore.jks;principal=impala/[email protected]
url = "jdbc:hive2://bigdata-dev-014:7180/;ssl=true;sslTrustStore=/you_filed_algos/cm-auto-global_truststore_kaifa.jks;principal=impala/[email protected]"
jars_file_ls = []
jars_file_str = ''
for jar in os.listdir(jars_path):
jars_file_ls.append(jars_path + jar)
jars_file_str = ':'.join(jars_file_ls)
# jvm_options = ["-Djava.class.path=" + jars_file_str, '-Xmx2G','-Xms512M']
jvm_options = "-Djava.class.path=" + jars_file_str
# jvm = jpype.getDefaultJVMPath()
jvm = '/you_filed_algos/jdk8/jre/lib/amd64/server/libjvm.so'
# if not jpype.isJVMStarted():
# try:
# # print('--------startjvm---------')
# jpype.startJVM(jvm, jvm_options)
# # print("JVM path:"+ jpype.getDefaultJVMPath())
# # print('----- running jvm -------------')
#
# except Exception as e:
# print('====== throw error ======')
# traceback.print_exc()
# jpype.shutdownJVM()
try:
if not jpype.isJVMStarted():
print('--------startjvm---------')
jpype.startJVM(jvm, jvm_options)
# jpype.startJVM(jvm, "-ea", jvm_options, '-Xmx5g', '-Xms5g', '-Xmn2g', '-XX:+UseParNewGC',
# '-XX:ParallelGCThreads=8', '-XX:SurvivorRatio=6', '-XX:+UseConcMarkSweepGC')
if jpype.isJVMStarted() and not jpype.isThreadAttachedToJVM():
print('-----attaching jvm-----')
jpype.attachThreadToJVM()
jpype.java.lang.Thread.currentThread().setContextClassLoader(
jpype.java.lang.ClassLoader.getSystemClassLoader())
# print('--------startjvm---------')
# jpype.startJVM(jvm, jvm_options)
# print("JVM path:"+ jpype.getDefaultJVMPath())
# print('----- running jvm -------------')
except Exception as e:
print('====== throw error ======')
traceback.print_exc()
raise RuntimeError(e)
try:
# print('----- running jvm ，' , jpype.isJVMStarted())
System = jpype.java.lang.System
if conn_type == PROD:
# 生产集群使用KUDU
System.setProperty("java.security.krb5.conf",
"/you_filed_algos/prod-krb5.conf")
elif conn_type == TEST:
# 测试集群使用KUDU
System.setProperty("java.security.krb5.conf",
"/you_filed_algos/krb5_kaifa.conf")
Configuration = jpype.JPackage('org.apache.hadoop.conf').Configuration
conf = Configuration()
conf.set("hadoop.security.authentication", "kerberos")
UserGroupInformation = jpype.JClass(
'org.apache.hadoop.security.UserGroupInformation')
UserGroupInformation.setConfiguration(conf)
if conn_type == PROD:
# 生产集群使用KUDU
UserGroupInformation.loginUserFromKeytab(
"sjfw_wangsh12348", "/you_filed_algos/sjfw_wangsh12348.keytab")
elif conn_type == TEST:
# 测试集群使用KUDU
UserGroupInformation.loginUserFromKeytab(
"sjfw_wangsh12348",
"/you_filed_algos/sjfw_wangsh12348_kaifa.keytab")
conn = jaydebeapi.connect(dirver, url)
cur = conn.cursor()
if sqltype == 'insert':
cur.execute(sql)
else:
cur.execute(sql)
result = cur.fetchall()
# 关闭游标
cur.close()
# 关闭连接
conn.close()
if sqltype != 'insert':
return result
except Exception as ex:
print(ex)
traceback.print_exc()
raise RuntimeError(ex)
# q_time2.append(end - start)
self.send_response(200)
self.send_header('Content-type', 'application/json')
self.send_header('Access-Control-Allow-Origin', '*')
self.end_headers()
self.wfile.write(res)
# if len(q_time2) == 20:
# print "hundred"
# plt.plot(range(len(q_time2)), q_time2)
# plt.ylabel('timeseries query time')
# plt.axis([1, 20, 0, 1])
# print float(sum(q_time2)) / len(q_time2)
# plt.show()
return
if __name__ == '__main__':
jpype.startJVM(jpype.getDefaultJVMPath(), "-ea ",
"-Djava.class.path=%s" % ('/tmp/GroupByInterface.jar'))
jpype.attachThreadToJVM()
coprocessor = jpype.JClass("com.hbase.client.MainEntrance")
cp = coprocessor()
httpd = ThreadedHTTPServer((HOST_IP, PORT), myHandler)
print "serving at port", PORT
httpd.serve_forever()
def send_sms(phoneNumber, smsData):
#获取jar包存放路径
jarpath = "./release/"
#jvm路径
#jvmpath=jpype.getDefaultJVMPath()
jvmpath = "/usr/lib/jvm//jre/lib/amd64/server/libjvm.so"
#待使用jar包文件
jarFile = 'SMSGatewayMidwarezxc.jar'
#jarBase='SMSGatewayMidwareBase.jar'
try:
#检测并启动jvm,get init args first
if not jpype.isJVMStarted(): #test whether the JVM is started
logging.error('jvm is not start!')
jpype.startJVM(jvmpath, "-ea",
"-Djava.class.path=%s" % (jarpath + jarFile))
while True:
if jpype.isJVMStarted():
break
logging.error("jvm has started!")
if jpype.isJVMStarted():
# +++20150915 必须加入这个方法才能在jpype环境下使用多线程
if not jpype.isThreadAttachedToJVM():
jpype.attachThreadToJVM()
logging.error('jvm is start!%s', time.time())
#get ipv4 and type
IPV4Util = jpype.JClass("com.cisi.TWBase.IPV4Util")
KSystemType = jpype.JClass("com.cisi.TWBase.KSystemType")
#get args
systemtype = KSystemType.EUserManageSystem
smsgateway = "122.13.138.180" #gate ip
smsgate = IPV4Util.ipToInt(smsgateway)
smsport = int(10320)
# logging.error("three init arg is : %s , %s and %s , and ip is : %s",systemtype,smsgate,smsport,smsgateway)
#create the Java class
SMSManager = jpype.JClass("com.cisi.client.midware.SMSManager")
smsmanager = SMSManager()
if not smsmanager.Init(systemtype, smsgate, smsport):
logging.error('init result is: %s,time is %s',
smsmanager.Init(systemtype, smsgate, smsport),
time.time())
raise
logging.error('init succeed!')
#发送短信
if not check_pnumber(phoneNumber):
print "wrong number"
#String to Bytes
# pnumber=phoneNumber
# smsdata1=smsData.encode("UnicodeBigUnmarked")
# smsdata=bytes(smsdata1)
#+++ 20150721 增加特殊短信头
# smsdata=str(bytearray([0xa,0xa])+bytearray(smsdata))
# sendresult=smsmanager.SendSMS(0,0,pnumber,smsdata)
print "开始发送短信的时间" + str(time.time())
sendresult = smsmanager.SendOtherSMS(phoneNumber, smsData)
logging.error(
'sms is sended!phoneNumber is %s,the result is %s,time is %s',
phoneNumber, sendresult, time.time())
#+++ 20150612
if sendresult == -6030:
return False
return True
except Exception, ex:
logging.error('send_sms. ex:%s', ex)
return False
def convert(request):
""" Handle Convert api request """
if request.method == 'GET':
""" Return all convert api request """
query = ConvertFileUpload.objects.all()
serializer = ConvertSerializer(query, many=True)
return Response(serializer.data)
elif request.method == 'POST':
""" Return convert tool result on the post file"""
serializer = ConvertSerializer(data=request.data)
if serializer.is_valid():
if (jpype.isJVMStarted() == 0):
""" If JVM not already started, start it, attach a Thread and start processing the request """
classpath = settings.JAR_ABSOLUTE_PATH
jpype.startJVM(jpype.getDefaultJVMPath(), "-ea",
"-Djava.class.path=%s" % classpath)
""" Attach a Thread and start processing the request """
jpype.attachThreadToJVM()
package = jpype.JPackage("org.spdx.tools")
result = ""
tagToRdfFormat = None
message = "Success"
query = ConvertFileUpload.objects.create(
owner=request.user,
file=request.data.get('file'),
from_format=request.POST["from_format"],
to_format=request.POST["to_format"],
cfilename=request.POST["cfilename"],
)
uploaded_file = str(query.file)
uploaded_file_path = str(query.file.path)
try:
if request.FILES["file"]:
folder = "/".join(uploaded_file_path.split('/')[:-1])
option1 = request.POST["from_format"]
option2 = request.POST["to_format"]
convertfile = request.POST["cfilename"]
warningoccurred = False
if (extensionGiven(convertfile) == False):
extension = getFileFormat(option2)
convertfile = convertfile + extension
""" Call the java function with parameters as list"""
if (option1 == "Tag"):
print("Verifing for Tag/Value Document")
if (option2 == "RDF"):
try:
tagToRdfFormat = request.POST["tagToRdfFormat"]
except:
tagToRdfFormat = 'RDF/XML-ABBREV'
option3 = tagToRdfFormat
if option3 not in [
'RDF/XML-ABBREV', 'RDF/XML', 'N-TRIPLET',
'TURTLE'
]:
message, returnstatus, httpstatus = convertError(
'400')
tagtordfclass = package.TagToRDF
retval = tagtordfclass.onlineFunction([
uploaded_file_path, folder + "/" + convertfile,
option3
])
if (len(retval) > 0):
warningoccurred = True
elif (option2 == "Spreadsheet"):
tagtosprdclass = package.TagToSpreadsheet
retval = tagtosprdclass.onlineFunction([
uploaded_file_path, folder + "/" + convertfile
])
if (len(retval) > 0):
warningoccurred = True
else:
message, returnstatus, httpstatus = convertError(
'400')
elif (option1 == "RDF"):
print("Verifing for RDF Document")
if (option2 == "Tag"):
rdftotagclass = package.RdfToTag
retval = rdftotagclass.onlineFunction([
uploaded_file_path, folder + "/" + convertfile
])
if (len(retval) > 0):
warningoccurred = True
elif (option2 == "Spreadsheet"):
rdftosprdclass = package.RdfToSpreadsheet
retval = rdftosprdclass.onlineFunction([
uploaded_file_path, folder + "/" + convertfile
])
if (len(retval) > 0):
warningoccurred = True
elif (option2 == "HTML"):
rdftohtmlclass = package.RdfToHtml
retval = rdftohtmlclass.onlineFunction([
uploaded_file_path, folder + "/" + convertfile
])
if (len(retval) > 0):
warningoccurred = True
else:
message, returnstatus, httpstatus = convertError(
'400')
elif (option1 == "Spreadsheet"):
print("Verifing for Spreadsheet Document")
if (option2 == "Tag"):
sprdtotagclass = package.SpreadsheetToTag
retval = sprdtotagclass.onlineFunction([
uploaded_file_path, folder + "/" + convertfile
])
if (len(retval) > 0):
warningoccurred = True
elif (option2 == "RDF"):
sprdtordfclass = package.SpreadsheetToRDF
retval = sprdtordfclass.onlineFunction([
uploaded_file_path, folder + "/" + convertfile
])
if (len(retval) > 0):
warningoccurred = True
else:
message, returnstatus, httpstatus = convertError(
'400')
if (warningoccurred == True):
message = "The following error(s)/warning(s) were raised: " + str(
retval)
index = folder.split("/").index('media')
result = "/" + "/".join(
folder.split("/")[index:]) + '/' + convertfile
returnstatus = status.HTTP_406_NOT_ACCEPTABLE
httpstatus = 406
jpype.detachThreadFromJVM()
else:
"""return only the path starting with MEDIA_URL"""
index = folder.split("/").index('media')
result = "/" + ("/".join(
folder.split("/")[index:])) + '/' + convertfile
returnstatus = status.HTTP_201_CREATED
httpstatus = 201
jpype.detachThreadFromJVM()
else:
message, returnstatus, httpstatus = convertError('404')
except jpype.JavaException as ex:
message = jpype.JavaException.message(ex)
returnstatus = status.HTTP_400_BAD_REQUEST
httpstatus = 400
jpype.detachThreadFromJVM()
except:
message = format_exc()
returnstatus = status.HTTP_400_BAD_REQUEST
httpstatus = 400
jpype.detachThreadFromJVM()
query.tagToRdfFormat = tagToRdfFormat
query.message = message
query.status = httpstatus
query.result = result
ConvertFileUpload.objects.filter(file=uploaded_file).update(
tagToRdfFormat=tagToRdfFormat,
message=message,
status=httpstatus,
result=result)
serial = ConvertSerializerReturn(instance=query)
return Response(serial.data, status=returnstatus)
else:
return Response(serializer.errors,
status=status.HTTP_400_BAD_REQUEST)
def _connection(self):
if self._cached_connection is None:
try:
# Check if the instantclient is available
cx_Oracle.clientversion()
except cx_Oracle.DatabaseError as e:
# Fallback to JDBC
use_oracle_client = False
self.log.debug(
'Oracle instant client unavailable, falling back to JDBC: %s',
e)
connect_string = self.JDBC_CONNECT_STRING.format(
self._server, self._service)
else:
use_oracle_client = True
self.log.debug('Running cx_Oracle version %s',
cx_Oracle.version)
connect_string = self.CX_CONNECT_STRING.format(
self._user, self._password, self._server, self._service)
if use_oracle_client:
connection = cx_Oracle.connect(connect_string)
self.log.debug(
"Connected to Oracle DB using Oracle Instant Client")
elif JDBC_IMPORT_ERROR:
self.log.error(
"Oracle client is unavailable and the integration is unable to import JDBC libraries. You may not "
"have the Microsoft Visual C++ Runtime 2015 installed on your system. Please double check your "
"installation and refer to the Datadog documentation for more information."
)
raise JDBC_IMPORT_ERROR
else:
try:
if jpype.isJVMStarted(
) and not jpype.isThreadAttachedToJVM():
jpype.attachThreadToJVM()
jpype.java.lang.Thread.currentThread(
).setContextClassLoader(
jpype.java.lang.ClassLoader.getSystemClassLoader())
connection = jdb.connect(self.ORACLE_DRIVER_CLASS,
connect_string,
[self._user, self._password],
self._jdbc_driver)
self.log.debug(
"Connected to Oracle DB using JDBC connector")
except Exception as e:
if "Class {} not found".format(
self.ORACLE_DRIVER_CLASS) in str(e):
msg = """Cannot run the Oracle check until either the Oracle instant client or the JDBC Driver
is available.
For the Oracle instant client, see:
http://www.oracle.com/technetwork/database/features/instant-client/index.html
You will also need to ensure the `LD_LIBRARY_PATH` is also updated so the libs are reachable.
For the JDBC Driver, see:
http://www.oracle.com/technetwork/database/application-development/jdbc/downloads/index.html
You will also need to ensure the jar is either listed in your $CLASSPATH or in the yaml
configuration file of the check.
"""
self.log.error(msg)
raise
self._cached_connection = connection
return self._cached_connection
def check_license(request):
""" Handle Check License api request """
if request.method == 'GET':
""" Return all check license api request """
query = CheckLicenseFileUpload.objects.all()
serializer = CheckLicenseSerializer(query, many=True)
return Response(serializer.data)
elif request.method == 'POST':
""" Return check license tool result on the post file"""
serializer = CheckLicenseSerializer(data=request.data)
if serializer.is_valid():
if (jpype.isJVMStarted() == 0):
""" If JVM not already started, start it, attach a Thread and start processing the request """
classpath = settings.JAR_ABSOLUTE_PATH
jpype.startJVM(jpype.getDefaultJVMPath(), "-ea",
"-Djava.class.path=%s" % classpath)
""" Attach a Thread and start processing the request """
jpype.attachThreadToJVM()
package = jpype.JPackage("org.spdx.compare")
compareclass = package.LicenseCompareHelper
query = CheckLicenseFileUpload.objects.create(
owner=request.user, file=request.data.get('file'))
uploaded_file = str(query.file)
uploaded_file_path = str(query.file.path)
""" Reading the license text file into a string variable """
licensetext = query.file.read()
try:
if request.FILES["file"]:
"""Call the java function with parameter"""
matching_licenses = compareclass.matchingStandardLicenseIds(
licensetext)
if (matching_licenses and len(matching_licenses) > 0):
matching_str = "The following license ID(s) match: "
matching_str += matching_licenses[0]
for i in range(1, len(matching_licenses)):
matching_str += ", "
matching_str += matching_licenses[i]
result = matching_str
returnstatus = status.HTTP_201_CREATED
httpstatus = 201
jpype.detachThreadFromJVM()
else:
result = "There are no matching SPDX listed licenses"
returnstatus = status.HTTP_400_BAD_REQUEST
httpstatus = 400
jpype.detachThreadFromJVM()
else:
result = "File Not Uploaded"
returnstatus = status.HTTP_400_BAD_REQUEST
httpstatus = 400
jpype.detachThreadFromJVM()
except jpype.JavaException as ex:
""" Java exception raised without exiting the application """
result = jpype.JavaException.message(ex)
returnstatus = status.HTTP_400_BAD_REQUEST
httpstatus = 400
jpype.detachThreadFromJVM()
except:
""" Other errors raised"""
result = format_exc()
returnstatus = status.HTTP_400_BAD_REQUEST
httpstatus = 400
jpype.detachThreadFromJVM()
query.result = result
query.status = httpstatus
CheckLicenseFileUpload.objects.filter(file=uploaded_file).update(
result=result, status=httpstatus)
serial = CheckLicenseSerializerReturn(instance=query)
return Response(serial.data, status=returnstatus)
else:
return Response(serializer.errors,
status=status.HTTP_400_BAD_REQUEST)
def _attach_jvm_to_thread():
"""
use attachThreadToJVM to fix multi-thread issues: https://github.com/hankcs/pyhanlp/issues/7
"""
if not isThreadAttachedToJVM():
attachThreadToJVM()
def sample(self, sess, chars, vocab, num=200, prime=' ', sampling_type=1):
"""
prime: starting character sequence.
sampling_type: 0 to use max at each timestep, 1 to sample at each timestep, 2 to sample on spaces
"""
state = sess.run(self.cell.zero_state(1, tf.float32))
print('Generate sample start with:', prime)
# make rnn state by feeding in prime sequence.
if jpype.isJVMStarted():
jpype.attachThreadToJVM()
twitter = Twitter()
tuple_example = twitter.pos(prime, stem=True, norm=False)[0]
char = tuple_example[0] + "/" + tuple_example[1]
print('put this to rnn to make state:', char)
x = np.zeros((1, 1)) # 1x1 matrix
x[0, 0] = vocab[char]
feed = {self.input_data: x, self.initial_state: state}
[state] = sess.run([self.final_state], feed)
def weighted_pick(weights):
t = np.cumsum(weights)
s = np.sum(weights)
return (int(np.searchsorted(t, np.random.rand(1) * s)))
li = []
li.append(char)
ret = prime.split("/")[0] + ' ' #result text
for n in range(num):
x = np.zeros((1, 1))
x[0, 0] = vocab[char]
feed = {self.input_data: x, self.initial_state: state}
[probs, state] = sess.run([self.probs, self.final_state], feed)
p = probs[0]
if sampling_type == 0:
sample = np.argmax(p)
elif sampling_type == 2:
if char == ' ':
sample = weighted_pick(p)
else:
sample = np.argmax(p)
else: # sampling_type == 1 default:
sample = weighted_pick(p)
pred = chars[sample] # 품사 정보 포함
if pred != "<Pad>/Pad":
#candidate.append(pred)
pred_word_tag = pred.split("/")
pred_word = pred_word_tag[0] # 품사 정보 불포함
pred_tag = pred_word_tag[1]
no_space_tag = [
'Eomi', 'PreEomi', 'Punctuation', 'Josa', 'Suffix'
]
if pred_tag in no_space_tag:
ret = ret[:-1] + pred_word + " "
else:
ret += pred_word + ' '
li.append(pred)
char = pred
else:
break
#print(li)
#score = sentence_bleu(reference, candidate, smoothing_function = sf.method1)
#return ret, score
return ret
def add_device(request):
if request.method == 'POST':
req = json.loads(request.body)
keyuser = req['keyusername']
keypwd = req['keypassword']
a = login_require(keyuser, keypwd)
if a == 0:
reason = "未登录"
elif a == 1:
reason = "普通用户无此权限"
elif a == 2:
try:
req = json.loads(request.body)
devicename = req['devicename']
type = req['type']
except:
reason = "没有数据耶"
j = jsonedit(reason)
return HttpResponse(j)
username = req['username']
if username: # 单纯添加设备，不与用户绑定
try:
userid = User.objects.get(username=username).id
except:
reason = "无此用户"
j = jsonedit(reason)
return HttpResponse(j)
flag = 1
else:
flag = 0
device = DeviceInfo.objects.filter(devicename=devicename)
if device.exists():
print(device)
reason = "重名啦"
j = jsonedit(reason)
return HttpResponse(j)
if not jpype.isJVMStarted():
jpype.startJVM(jvmPath, jvmArg)
jpype.attachThreadToJVM()
Main = jpype.JClass("yuer.yueriot")
jd = Main()
secret = jd.yuerregist(devicename)
try:
id = DeviceInfo.objects.create(devicename=devicename,
devicesecret=secret,
type=type).id
except:
reason = "add_device出错"
j = jsonedit(reason)
return HttpResponse(j)
if flag == 1:
UandD.objects.create(username=username,
deviceid_id=id,
devicename=devicename,
userid_id=userid,
device_type=type)
user_info_all()
device_status_all()
reason = None
j = jsonedit(reason)
return HttpResponse(j)
def cb_sensors(self, msg):
"""lpz sensors callback: receive sensor values, sos algorithm attached"""
if not self.init: return
# self.msg.data = []
self.x_t = np.roll(self.x_t, -1, axis=1) # push back past
# self.z = np.roll(self.y, 1, axis=1) # push back past
# update input with new sensor data
u = np.reshape(np.asarray(msg.data), (self.idim, 1))
self.x_t[:,-1] = u.reshape((self.idim,))
# compute network output
self.res.execute(u)
# print self.res.z
# learning
dw = 0
# if self.cnt > 0: # self.piwin:
if self.cnt > self.piwin:
for sysdim in range(self.idim):
attachThreadToJVM()
# x_tmp = self.x_t[sysdim,:]
# x_tmp = (((self.x_t[sysdim,:] / (2*np.pi)) + 0.5) * 999).astype(int)
# discrete ENT / PI / AIS / ...
# FIXME: use digitize and determine bin boundaries from min/max
x_tmp = (((self.x_t[sysdim,:] / 3.) + 0.5) * 999).astype(int)
# print "x_tmp", sysdim, x_tmp
# x_tmp = x_tmp - np.min(x_tmp)
# x_tmp = ((x_tmp / np.max(x_tmp)) * (LPZRosEH.base-1)).astype(int)
# print "x_tmp", x_tmp
# # print "jvm", isThreadAttachedToJVM()
# pis = LPZRosEH.piCalc10.computeLocal(x_tmp)
if self.mode == LPZRosEH.modes["eh_ent_d"]: # EH learning, discrete PI
# plain entropy
pis = LPZRosEH.entCalc.computeLocal(x_tmp)
self.perf[0,sysdim] = list(pis)[-1] * -1
elif self.mode == LPZRosEH.modes["eh_pi_d"]: # EH learning, discrete PI
# predictive information
# pis = LPZRosEH.piCalc10.computeLocal(x_tmp)
# self.perf[0,sysdim] = list(pis)[-1]
pis = LPZRosEH.piCalc10.computeAverageLocal(x_tmp)
self.perf[0,sysdim] = pis
elif self.mode == LPZRosEH.modes["eh_ais_d"]: # EH learning, discrete PI
# pis = LPZRosEH.aisCalc.computeLocal(x_tmp)
# self.perf[0,sysdim] = list(pis)[-1]
pis = LPZRosEH.aisCalc.computeAverageLocal(x_tmp)
self.perf[0,sysdim] = pis
elif self.mode == LPZRosEH.modes["eh_pi_c_l"]: # EH learning, discrete PI
# local continuous predictive information
LPZRosEH.piCalc.initialise(1, 1, 0.5); # Use history length 1 (Schreiber k=1),
x_src = np.atleast_2d(self.x_t[sysdim,0:-1]).T
x_dst = np.atleast_2d(self.x_t[sysdim,1:]).T
LPZRosEH.piCalc.setObservations(x_src, x_dst)
pis = LPZRosEH.piCalc.computeLocalOfPreviousObservations()
self.perf[0,sysdim] = list(pis)[-1]
elif self.mode == LPZRosEH.modes["eh_pi_c_avg"]: # EH learning, discrete PI
# average continuous predictive information
LPZRosEH.piCalc.initialise(1, 1, 0.5); # Use history length 1 (Schreiber k=1),
x_src = np.atleast_2d(self.x_t[sysdim,0:-1]).T
x_dst = np.atleast_2d(self.x_t[sysdim,1:]).T
LPZRosEH.piCalc.setObservations(x_src, x_dst)
self.perf[0,sysdim] = LPZRosEH.piCalc.computeAverageLocalOfObservations()
elif self.mode == LPZRosEH.modes["eh_var"]: # EH learning, discrete PI
# variance
self.perf[0,sysdim] = np.var(self.x_t[sysdim,:])
else:
self.perf[0,sysdim] = 0.
# print "pis", pis
print "perf", self.perf
# recent performance
self.perf_lp = self.perf_lp * (1 - self.coeff_a) + self.perf * self.coeff_a
############################################################
# learning
# FIXME: put that into res / model member function
for ro_idx in range(self.odim):
# if gaussian / acc based
# if perf[0, ro_idx] > (perf_lp[0, ro_idx] + 0.1):
# for information based
# FIXME: consider single perf / modulator for all readouts
if self.perf[0, ro_idx] > self.perf_lp[0, ro_idx]:
self.mdltr[0, ro_idx] = 1.
else:
self.mdltr[0, ro_idx] = 0.
eta = self.eta_init / (1 + (self.cnt/self.T))
# eta = eta_init
# dw = eta * (zn_t[0, ti - 20] - zn_lp) * mdltr * r
# dw = eta * (zn.T - zn_lp) * mdltr * r
# print "2D dbg" zn.shape, zn_lp.shape
if True: # self.cnt < self.test_rate * self.simtime_len:
# dw = eta * (zn.T - zn_lp) * mdltr * r
# wo += dw
dw = eta * (self.res.zn.T - self.res.zn_lp.T) * self.mdltr * self.res.r
# print dw
self.res.wo += dw
# FIXME: apply soft bounding on weights or weight decay
self.soft_bound()
else:
dw = np.zeros(self.res.r.shape)
self.mdltr[0,:] = 0.
# if np.abs(ip2d.x[ti,0]) > 10. or np.abs(ip2d.x[ti,1]) > 10.:
# sys.exit()
# # check this
# bins = np.arange(-1, 1.1, 0.1)
# x_tmp = np.digitize(x[:,0], bins)
# base = np.max(x_tmp)+1 # 1000
# basehalf = base/2
# piCalc10 = piCalcClassD(base,3)
# aisCalc10 = aisCalcClassD(base,5)
# pi = list(piCalc10.computeLocal(x_tmp))
# ais = list(aisCalc10.computeLocal(x_tmp))
self.msg.data = self.res.zn.flatten().tolist()
# print self.msg.data
# print("sending msg", msg)
self.pub_motors.publish(self.msg)
self.msg_res_r.data = self.res.r.flatten().tolist()
self.pub_res_r.publish(self.msg_res_r)
self.msg_res_w.data = np.linalg.norm(self.res.wo, 2, axis=0)
self.pub_res_w.publish(self.msg_res_w)
self.msg_res_perf.data = self.perf.flatten().tolist()
self.pub_res_perf.publish(self.msg_res_perf)
self.msg_res_perf_lp.data = self.perf_lp.flatten().tolist()
self.pub_res_perf_lp.publish(self.msg_res_perf_lp)
self.msg_res_mdltr.data = self.mdltr.flatten().tolist()
self.pub_res_mdltr.publish(self.msg_res_mdltr)
# time.sleep(0.1)
# if self.cnt > 20:
# rospy.signal_shutdown("stop")
# sys.exit(0)
self.cnt += 1
def Phrases(self, request, context):
jpype.attachThreadToJVM(
) # XXX: Performance Incresed. (Still don't know yet)
return global_pb2.StringArrayResponse(results=self.engine.phrases(
request.payload),
options=request.options)
def attach_thread(logger=None):
if not jp.isThreadAttachedToJVM():
if logger is not None:
logger.debug("Attaching thread to JVM")
jp.attachThreadToJVM()
def __dir__(self):
if jpype.isThreadAttachedToJVM() == 0:
jpype.attachThreadToJVM()
v = sorted(
[self._cleanName(i) for i in self._get_vars() if len(i) > 0])
return sorted(self._dict.keys()) + v
def GetWordCloud(
request
): # post처리가 일어나는곳. -> Redirct처리(CSRF보안에러 일어날수도 있음), matplotlib's Tkinter always use main thread
check = request.POST['lang']
image = request.POST['image']
URL = request.POST['address']
user_agent = 'forget'
overloadText = ""
response = rq.get(URL, headers={'User-Agent': user_agent})
soup = BeautifulSoup(response.text,
'html.parser') # 전체 HTML에서 특정 부분만 찾기 위한 html parsing
if image != "":
mask = np.array(Image.open(image)) # select mask
image_colors = ImageColorGenerator(mask)
else:
mask = np.array(
Image.open(
"WCProgram/static/WCProgram/images/oval.jpg")) # default mask
alldata = soup.find_all()
taglist = set()
# get all tags(どんなサイトでもデータを習得)
for tag in alldata:
taglist.add(tag.name)
if (check == 'kr'):
# 사용자를 대신하여 일을 수행하는 소프트웨어 에이전트, get방식으로 값을 가져왔는데 내용이 안보이면 header에 User-Agent 아무값이나 넣어보자
Article = soup.findAll(taglist) # multiple tag
for index in Article:
overloadText = overloadText + index.text + " "
# For [No core dump will be written] Exception Clear
if jpype.isJVMStarted():
jpype.attachThreadToJVM()
tokens_ko = t.nouns(overloadText) # konlpy Package의 t를 이용하여 단어 나눔
stop_words = [
'거', '왜', '좀', '레', '뭐', '임', '코', '페', '타', '함', '요', '이', '어',
'온', '내'
] # Excepted Wrods
tokens_ko = [
each_word for each_word in tokens_ko if each_word not in stop_words
] # for, if, not in 이용해서 제외단어 이외의 단어만 남김
ko = nltk.Text(tokens_ko, name='갤DB')
data = ko.vocab().most_common(
500) # 정렬된 list-tuple형식으로 변경(Count해줌. 최대 500)
tmp_data = dict(data) # dict형식으로 데이터 변경
# in Mac
wc = WordCloud(
font_path="/Library/Fonts/AppleGothic.ttf",
mask=mask,
stopwords=stop_words,
min_font_size=5,
max_words=2000,
background_color="white").generate_from_frequencies(tmp_data)
# in Ubuntu - 한글
# wc = WordCloud(font_path="/usr/share/fonts/truetype/nanum/NanumBarunGothic.ttf",
# mask=mask, min_font_size=5, max_words=2000, background_color="white").generate_from_frequencies(tmp_data)
elif (check == 'jp'):
Article = soup.findAll(taglist) # 닉네임 전부 가져옴
for index in Article:
overloadText = overloadText + index.text
tokens_jp = mecab_analysis(
overloadText
) # konlpy Package의 t를 이용하여 단어 나눔, "形容詞", "動詞","名詞", "副詞"만 필요
jp = nltk.Text(tokens_jp, name='杏') # 중복 제거를 위해 token형식으로 만듬
data = jp.vocab().most_common(
500) # 정렬된 list-tuple형식으로 변경(Count해줌. 최대 500)(
tmp_data = dict(data) # dict형식으로 데이터 변경
stop_words = [
'てる', 'いる', 'なる', 'れる', 'する', 'ある', 'こと', 'これ', 'さん', 'して', 'くれる',
'やる', 'くださる', 'そう', 'せる', 'した', '思う', 'それ', 'ここ', 'ちゃん', 'くん', '',
'て', 'に', 'を', 'は', 'の', 'が', 'と', 'た', 'し', 'で', 'ない', 'も', 'な',
'い', 'か', 'ので', 'よう', '[', ']', '/'
]
# Python 3.0부턴 유니코드 표현을 위해 앞에 u를 붙일 필요가 없음(이미 유니코드로 표현되므로)
# in Max
wc = WordCloud(
font_path="/Library/Fonts/Hannari.otf",
mask=mask,
stopwords=stop_words,
max_words=2000,
background_color='white').generate_from_frequencies(tmp_data)
# in Ubuntu - 日本語
# wc = WordCloud(font_path="/usr/share/fonts/truetype/fonts-japanese-gothic.ttf",
# mask=mask, min_font_size=5, max_words=2000, background_color="white").generate_from_frequencies(tmp_data)
elif (check == 'en'):
Article = soup.findAll(taglist)
for index in Article:
overloadText = overloadText + index.text
text = overloadText
stopwords = set(STOPWORDS) # Use STOPWORDS Class
stopwords.add("int")
stopwords.add("ext")
# English Don't need font check
wc = WordCloud(mask=mask,
stopwords=stopwords,
max_words=2000,
background_color='white').generate(text)
# killed error：ラムが足りないときによく発動
plt.figure(figsize=(16, 16))
if image != "":
plt.imshow(wc.recolor(color_func=image_colors),
interpolation='bilinear')
else:
plt.imshow(wc, interpolation="bilinear")
plt.axis("off")
#plt.savefig('GalleryDataWC.png') # word cloud 세이브
buf = io.BytesIO()
plt.savefig(buf, format='png')
del mask
plt.clf()
plt.close() # If you don't close, you can see a lot of error
response.close()
response = HttpResponse(
buf.getvalue(), content_type='image/png') # これままでは、イメージがWeb画面に見せるだけです。
response[
'Content-Disposition'] = "attachment; filename=picture.png" # Downloadまでするコード
#response = HttpResponseRedirect(reverse('WCProgram:index'))
buf.close()
Article.clear()
gc.collect() # garbage collect empty(for server memory)
return response
def tokenize_kkma(doc):
jpype.attachThreadToJVM()
token_doc = ['/'.join(word) for word in kkma.pos(doc)]
return token_doc
