pca_vector = pca.transform(scaled_vector.reshape(1, -1))
prediction = clf.predict(pca_vector.reshape(1, -1))
if prediction == 1:
prediction = "BIOFILM"
else:
prediction = "OTHER"
return prediction
if __name__ == '__main__':
# connect to the JVM
gateway = JavaGateway(gateway_parameters=GatewayParameters(port=25335))
# get the patient path.
patient = gateway.entry_point
load_path = patient.getPathCampi()
# get the results path
biofilm_path = patient.getPathBiofilmsi()
other_path = patient.getPathBiofilmno()
# load the scaler, the PCA and the classifier
scaler = joblib.load('scaler.pkl')
clf = joblib.load('clf.pkl')
pca = joblib.load('pca.pkl')
for file in os.listdir(load_path):
def _launch_gateway(self, class_path, popen_kwargs=None):
"""
launch jvm gateway
:param popen_kwargs: Dictionary of kwargs to pass to Popen when spawning
the py4j JVM. This is a developer feature intended for use in
customizing how pyspark interacts with the py4j JVM (e.g., capturing
stdout/stderr).
"""
command = ["java"]
command.append("-cp")
command.append(class_path)
command.append("org.apache.spark.deploy.raydp.AppMasterEntryPoint")
# Create a temporary directory where the gateway server should write the connection
# information.
conn_info_dir = tempfile.mkdtemp()
try:
fd, conn_info_file = tempfile.mkstemp(dir=conn_info_dir)
os.close(fd)
os.unlink(conn_info_file)
env = dict(os.environ)
env["_RAYDP_APPMASTER_CONN_INFO_PATH"] = conn_info_file
# Launch the Java gateway.
popen_kwargs = {} if popen_kwargs is None else popen_kwargs
# We open a pipe to stdin so that the Java gateway can die when the pipe is broken
popen_kwargs['stdin'] = PIPE
# We always set the necessary environment variables.
popen_kwargs['env'] = env
# Don't send ctrl-c / SIGINT to the Java gateway:
def preexec_func():
signal.signal(signal.SIGINT, signal.SIG_IGN)
popen_kwargs['preexec_fn'] = preexec_func
proc = Popen(command, **popen_kwargs)
# Wait for the file to appear, or for the process to exit, whichever happens first.
while not proc.poll() and not os.path.isfile(conn_info_file):
time.sleep(0.1)
if not os.path.isfile(conn_info_file):
raise Exception(
"Java gateway process exited before sending its port number"
)
with open(conn_info_file, "rb") as info:
length = info.read(4)
if not length:
raise EOFError
gateway_port = struct.unpack("!i", length)[0]
finally:
shutil.rmtree(conn_info_dir)
gateway = JavaGateway(gateway_parameters=GatewayParameters(
port=gateway_port, auto_convert=True))
# Store a reference to the Popen object for use by the caller (e.g., in reading stdout/stderr)
gateway.proc = proc
return gateway
def launch_gateway(conf=None):
"""
launch jvm gateway
:param conf: spark configuration passed to spark-submit
:return:
"""
if "PYSPARK_GATEWAY_PORT" in os.environ:
gateway_port = int(os.environ["PYSPARK_GATEWAY_PORT"])
gateway_secret = os.environ["PYSPARK_GATEWAY_SECRET"]
else:
SPARK_HOME = _find_spark_home()
# Launch the Py4j gateway using Spark's run command so that we pick up the
# proper classpath and settings from spark-env.sh
on_windows = platform.system() == "Windows"
script = "./bin/spark-submit.cmd" if on_windows else "./bin/spark-submit"
command = [os.path.join(SPARK_HOME, script)]
if conf:
for k, v in conf.getAll():
command += ['--conf', '%s=%s' % (k, v)]
submit_args = os.environ.get("PYSPARK_SUBMIT_ARGS", "pyspark-shell")
if os.environ.get("SPARK_TESTING"):
submit_args = ' '.join(
["--conf spark.ui.enabled=false", submit_args])
command = command + shlex.split(submit_args)
# Create a temporary directory where the gateway server should write the connection
# information.
conn_info_dir = tempfile.mkdtemp()
try:
fd, conn_info_file = tempfile.mkstemp(dir=conn_info_dir)
os.close(fd)
os.unlink(conn_info_file)
env = dict(os.environ)
env["_PYSPARK_DRIVER_CONN_INFO_PATH"] = conn_info_file
# Launch the Java gateway.
# We open a pipe to stdin so that the Java gateway can die when the pipe is broken
if not on_windows:
# Don't send ctrl-c / SIGINT to the Java gateway:
def preexec_func():
signal.signal(signal.SIGINT, signal.SIG_IGN)
proc = Popen(command,
stdin=PIPE,
preexec_fn=preexec_func,
env=env)
else:
# preexec_fn not supported on Windows
proc = Popen(command, stdin=PIPE, env=env)
# Wait for the file to appear, or for the process to exit, whichever happens first.
while not proc.poll() and not os.path.isfile(conn_info_file):
time.sleep(0.1)
if not os.path.isfile(conn_info_file):
raise Exception(
"Java gateway process exited before sending its port number"
)
with open(conn_info_file, "rb") as info:
gateway_port = read_int(info)
gateway_secret = UTF8Deserializer().loads(info)
finally:
shutil.rmtree(conn_info_dir)
# In Windows, ensure the Java child processes do not linger after Python has exited.
# In UNIX-based systems, the child process can kill itself on broken pipe (i.e. when
# the parent process' stdin sends an EOF). In Windows, however, this is not possible
# because java.lang.Process reads directly from the parent process' stdin, contending
# with any opportunity to read an EOF from the parent. Note that this is only best
# effort and will not take effect if the python process is violently terminated.
if on_windows:
# In Windows, the child process here is "spark-submit.cmd", not the JVM itself
# (because the UNIX "exec" command is not available). This means we cannot simply
# call proc.kill(), which kills only the "spark-submit.cmd" process but not the
# JVMs. Instead, we use "taskkill" with the tree-kill option "/t" to terminate all
# child processes in the tree (http://technet.microsoft.com/en-us/library/bb491009.aspx)
def killChild():
Popen([
"cmd", "/c", "taskkill", "/f", "/t", "/pid",
str(proc.pid)
])
atexit.register(killChild)
# Connect to the gateway
gateway = JavaGateway(gateway_parameters=GatewayParameters(
port=gateway_port, auth_token=gateway_secret, auto_convert=True))
# Import the classes used by PySpark
java_import(gateway.jvm, "org.apache.spark.SparkConf")
java_import(gateway.jvm, "org.apache.spark.api.java.*")
java_import(gateway.jvm, "org.apache.spark.api.python.*")
java_import(gateway.jvm, "org.apache.spark.ml.python.*")
java_import(gateway.jvm, "org.apache.spark.mllib.api.python.*")
# TODO(davies): move into sql
java_import(gateway.jvm, "org.apache.spark.sql.*")
java_import(gateway.jvm, "org.apache.spark.sql.api.python.*")
java_import(gateway.jvm, "org.apache.spark.sql.hive.*")
java_import(gateway.jvm, "scala.Tuple2")
return gateway
def __init__(self, port=25333):
self.gateway = JavaGateway(gateway_parameters=GatewayParameters(
port=port))
def launch_gateway():
SPARK_HOME = os.environ["SPARK_HOME"]
gateway_port = -1
if "PYSPARK_GATEWAY_PORT" in os.environ:
gateway_port = int(os.environ["PYSPARK_GATEWAY_PORT"])
else:
# Launch the Py4j gateway using Spark's run command so that we pick up the
# proper classpath and settings from spark-env.sh
on_windows = platform.system() == "Windows"
script = "./bin/spark-submit.cmd" if on_windows else "./bin/spark-submit"
submit_args = os.environ.get("PYSPARK_SUBMIT_ARGS")
submit_args = submit_args if submit_args is not None else ""
submit_args = shlex.split(submit_args)
command = [os.path.join(SPARK_HOME, script), "pyspark-shell"
] + submit_args
if not on_windows:
# Don't send ctrl-c / SIGINT to the Java gateway:
def preexec_func():
signal.signal(signal.SIGINT, signal.SIG_IGN)
proc = Popen(command,
stdout=PIPE,
stdin=PIPE,
preexec_fn=preexec_func)
else:
# preexec_fn not supported on Windows
proc = Popen(command, stdout=PIPE, stdin=PIPE)
try:
# Determine which ephemeral port the server started on:
gateway_port = proc.stdout.readline()
gateway_port = int(gateway_port)
except ValueError:
(stdout, _) = proc.communicate()
exit_code = proc.poll()
error_msg = "Launching GatewayServer failed"
error_msg += " with exit code %d! " % exit_code if exit_code else "! "
error_msg += "(Warning: unexpected output detected.)\n\n"
error_msg += gateway_port + stdout
raise Exception(error_msg)
# Create a thread to echo output from the GatewayServer, which is required
# for Java log output to show up:
class EchoOutputThread(Thread):
def __init__(self, stream):
Thread.__init__(self)
self.daemon = True
self.stream = stream
def run(self):
while True:
line = self.stream.readline()
sys.stderr.write(line)
EchoOutputThread(proc.stdout).start()
# Connect to the gateway
gateway = JavaGateway(GatewayClient(port=gateway_port), auto_convert=False)
# Import the classes used by PySpark
java_import(gateway.jvm, "org.apache.spark.SparkConf")
java_import(gateway.jvm, "org.apache.spark.api.java.*")
java_import(gateway.jvm, "org.apache.spark.api.python.*")
java_import(gateway.jvm, "org.apache.spark.mllib.api.python.*")
java_import(gateway.jvm, "org.apache.spark.sql.SQLContext")
java_import(gateway.jvm, "org.apache.spark.sql.hive.HiveContext")
java_import(gateway.jvm, "org.apache.spark.sql.hive.LocalHiveContext")
java_import(gateway.jvm, "org.apache.spark.sql.hive.TestHiveContext")
java_import(gateway.jvm, "scala.Tuple2")
return gateway
def __init__(self,case_files, dyn_sim_config_file,rl_config_file , server_port_num = 25333, cnts=[2,2,2]):
# change from global to class-level variable to support parallel process
#global gateway
self.a_gateway = JavaGateway(gateway_parameters=GatewayParameters(port = server_port_num,auto_convert=True))
#global ipss_app
self.ipss_app = self.a_gateway.entry_point
from gym import spaces
_case_files = transfer2JavaStringAry(self.a_gateway,case_files)
#initialize the power system simulation service
# {observation_history_length,observation_space_dim, action_location_num, action_level_num};
dim_ary= self.ipss_app.initStudyCase(_case_files,dyn_sim_config_file,rl_config_file)
print ('observation_history_length,observation_space_dim, action_location_num, action_level_num = ')
print (dim_ary[0], dim_ary[1],dim_ary[2], dim_ary[3])
observation_history_length = dim_ary[0]
observation_space_dim = dim_ary[1]
action_location_num = dim_ary[2]
action_level_num = dim_ary[3]
num = action_level_num ** action_location_num
self.action_space = spaces.Discrete(num)
self.cnts = cnts
#define action and observation spaces
"""
if(action_location_num == 1):
self.action_space = spaces.Discrete(action_level_num) # Discrete, 1-D dimension
else:
#print('N-D dimension Discrete Action space is not supported it yet...TODO')
# the following is based on the latest gym dev version
# action_def_vector = np.ones(action_location_num, dtype=np.int32)*action_level_num
# for gym version 0.10.4, it is parametrized by passing an array of arrays containing [min, max] for each discrete action space
# for exmaple, MultiDiscrete([ [0,4], [0,1], [0,1] ])
action_def_vector = np.ones((action_location_num,2),dtype=np.int32)
action_def_vector[:,1] = action_level_num -1
aa = np.asarray(action_def_vector, dtype=np.int32)
self.action_space = spaces.MultiDiscrete(action_def_vector) # Discrete, N-D dimension
"""
self.observation_space = spaces.Box(-999,999,shape=(observation_history_length * observation_space_dim,)) # Continuous
self.seed()
#TOOD get the initial states
self.state = None
self.steps_beyond_done = None
self.restart_simulation = True
else:
result = json.dumps(
list(
filter(lambda x: not re.match("^__.*", x),
list(completionList))))
self.interpreter.setStatementsFinished(result, False)
host = sys.argv[1]
port = int(sys.argv[2])
if "PY4J_GATEWAY_SECRET" in os.environ:
from py4j.java_gateway import GatewayParameters
gateway_secret = os.environ["PY4J_GATEWAY_SECRET"]
gateway = JavaGateway(gateway_parameters=GatewayParameters(
address=host, port=port, auth_token=gateway_secret, auto_convert=True))
else:
gateway = JavaGateway(GatewayClient(address=host, port=port),
auto_convert=True)
intp = gateway.entry_point
_zcUserQueryNameSpace = {}
completion = PythonCompletion(intp, _zcUserQueryNameSpace)
_zcUserQueryNameSpace["__zeppelin_completion__"] = completion
_zcUserQueryNameSpace["gateway"] = gateway
from zeppelin_context import PyZeppelinContext
if intp.getZeppelinContext():
z = __zeppelin__ = PyZeppelinContext(intp.getZeppelinContext(), gateway)
# Test script. Tests brightness and contrast commands.
# Created by Toni Sagrista
from py4j.java_gateway import JavaGateway, GatewayParameters
gateway = JavaGateway(gateway_parameters=GatewayParameters(auto_convert=True))
gs = gateway.entry_point
gs.disableInput()
gs.cameraStop()
gs.print("Testing brightness")
gs.setBrightnessLevel(-1.0)
gs.sleep(1)
gs.setBrightnessLevel(-0.5)
gs.sleep(1)
gs.setBrightnessLevel(-0.0)
gs.sleep(1)
gs.setBrightnessLevel(0.5)
gs.sleep(1)
gs.setBrightnessLevel(1.0)
gs.sleep(1)
gs.setBrightnessLevel(0.0)
gs.sleep(1)
gs.print("Testing contrast")
gs.setContrastLevel(0.0)
gs.sleep(1)
gs.setContrastLevel(0.5)
def run(self, parent_data_objs):
# Run the java py4j entry point
comp_dir = self._dag_node.comp_root_path()
self._logger.info("comp_dir: {}".format(comp_dir))
jar_files = glob.glob(os.path.join(comp_dir, "*.jar"))
self._logger.info("Java classpath files: {}".format(jar_files))
component_class = self._dag_node.comp_class()
java_jars = [self._mlcomp_jar] + jar_files
class_path = ":".join(java_jars)
java_gateway = None
all_ok = False
monitor_proc = None
try:
total_phys_mem_size_mb = ByteConv.from_bytes(
psutil.virtual_memory().total).mbytes
jvm_heap_size_option = "-Xmx{}m".format(
int(math.ceil(total_phys_mem_size_mb)))
java_opts = [jvm_heap_size_option]
self._logger.info("JVM options: {}".format(java_opts))
# Note: the jarpath is set to be the path to the mlcomp jar since the launch_gateway code is checking
# for the existence of the jar. The py4j jar is packed inside the mlcomp jar.
java_port = launch_gateway(port=0,
javaopts=java_opts,
die_on_exit=True,
jarpath=self._mlcomp_jar,
classpath=class_path,
redirect_stdout=sys.stdout,
redirect_stderr=sys.stderr)
java_gateway = JavaGateway(
gateway_parameters=GatewayParameters(port=java_port),
callback_server_parameters=CallbackServerParameters(port=0),
python_server_entry_point=MLOpsPY4JWrapper())
python_port = java_gateway.get_callback_server(
).get_listening_port()
self._logger.debug("Python port: {}".format(python_port))
java_gateway.java_gateway_server.resetCallbackClient(
java_gateway.java_gateway_server.getCallbackClient().
getAddress(), python_port)
mlops_wrapper = MLOpsPY4JWrapper()
entry_point = java_gateway.jvm.com.parallelm.mlcomp.ComponentEntryPoint(
component_class)
component_via_py4j = entry_point.getComponent()
component_via_py4j.setMLOps(mlops_wrapper)
# Configure
m = java_gateway.jvm.java.util.HashMap()
for key in self._params.keys():
m[key] = self._params[key]
component_via_py4j.configure(m)
# Materialized
l = java_gateway.jvm.java.util.ArrayList()
for obj in parent_data_objs:
l.append(obj)
self._logger.info("Parent obj: {} type {}".format(
obj, type(obj)))
self._logger.info("Parent objs: {}".format(l))
if mlops_loaded:
monitor_proc = ProcessMonitor(mlops, self._ml_engine)
monitor_proc.start()
py4j_out_objs = component_via_py4j.materialize(l)
self._logger.debug(type(py4j_out_objs))
self._logger.debug(len(py4j_out_objs))
python_out_objs = []
for obj in py4j_out_objs:
self._logger.debug("Obj:")
self._logger.debug(obj)
python_out_objs.append(obj)
self._logger.info(
"Done running of materialize and getting output objects")
all_ok = True
except Py4JJavaError as e:
self._logger.error("Error in java code: {}".format(e))
raise MLCompException(str(e))
except Exception as e:
self._logger.error("General error: {}".format(e))
raise MLCompException(str(e))
finally:
self._logger.info("In finally block: all_ok {}".format(all_ok))
if java_gateway:
java_gateway.close_callback_server()
java_gateway.shutdown()
if mlops_loaded and monitor_proc:
monitor_proc.stop_gracefully()
return python_out_objs
doors.write("\n")
doors.close()
if __name__ == "__main__":
try:
numero_pruebas = int(sys.argv[1])
plan_elegido = PLANARRAY[int(sys.argv[2])]
print u"Total de evaluaciones a realizar por plan: ", sys.argv[1]
print u'El plan elegido es: ', plan_elegido
except:
numero_pruebas = 10
plan_elegido = PLANARRAY[0]
print u"El número de pruebas a realizar será el por defecto (10 rondas)"
print u'El plan elegido es: ', plan_elegido
gw = JavaGateway() # New gateway connection
bridge = gw.entry_point # The actual NetLogoBridge object
bridge.openModel(NETLOGOMODEL)
createPlanFile(plan_elegido)
posiciones = obtenerPosiciones(PLANFILE)
# en genes se guardan todas las posibles puertas
genes = obtenerGenes(posiciones)
# generarArchivo(genes)
individuos = generarPoblacion(genes, 100)
# aca especificamos el numero de generaciones
for x in range(0, numero_pruebas):
print "generacion: {0}".format(x)
for individuo in individuos:
def index():
movie = request.form['movie']
print movie
imdbTitle = movie.replace(" ","%20")
link = 'http://www.omdbapi.com/?i=&t=' + imdbTitle +'&tomatoes=true'
linkRequest = Request(link)
try:
linkResponse = urlopen(linkRequest)
j = json.load(linkResponse)
print j['Title']
except :
pass
return render_template('index.html',errMsg="Pls Enter Correct Movie")
try:
rd = j['Released']
date_time = datetime.datetime.strptime(rd,"%d %b %Y")
t1= datetime.datetime.strftime(date_time, "%Y-%m-%d")
Dates = t1.split()
Year= Dates[0][0:4]
Month = Dates[0][5:7]
t = Year+Month
Dates = t1.split()
Day= Dates[0][8:10]
Month = Dates[0][5:7]
if Month == "09":
Month = 9
elif Month == "08":
Month = 8
elif Day == "09":
Day = 9
elif Day == "08":
Day = 8
Month = int(Month)
Day = int(Day)
d = (Month,Day)
print d
d1 = (01,9)
d2 = (01,19)
d3 = (02,06)
d4 = (02,16)
d5 = (05,15)
d6 = (05,25)
d7 = (06,24)
d8 = (07,04)
d9 = (8,28)
d10 = (9,07)
d11 = (10,02)
d12 =(10,12)
d13 = (10,21)
d14= (10,31)
d15 = (11,01)
d16 = (11,11)
d17 = (11,16)
d18 = (11,26)
d19 = (12,15)
d20 = (12,31)
d21 = (01,01)
if (d1 <= d <= d2) or ( d3 <= d <= d4) or (d5<= d <= d6) or (d7 <= d <= d8) or (d9 <= d <= d10) or (d11<= d<= d12) or (d13<= d<=d14) or (d15<= d<= d16) or (d17 <= d <= d18) or (d19<= d<= d20) :
status = "1"
else:
if (d == d21) :
status = "1"
else:
status = "0";
try:
wiki_url = 'http://stats.grok.se/json/en/'+t+'/'+movie
print wiki_url
wiki_read = json.loads(urlopen(wiki_url).read())
total_views = sum([count for count in wiki_read['daily_views'].values()])
total_str =str(total_views)
print total_str
except :
pass
imdb_genre = j['Genre'].replace(",","-")
j['Title'] = j['Title'].replace(",","-")
imdb_Actors = j['Actors'].replace(",","-")
imdb_Director = j['Director'].replace(",","-")
imdb_Awards = j['Awards'].replace(",","-")
imdb_Production = j['Production'].replace(",","-")
imdb_votes = j['imdbVotes'].replace(",","")
tomatoUserReviews = j['tomatoUserReviews'].replace(",","")
tomatoUserRating = j['tomatoUserRating'].replace(",","")
tomatoUserMeter = j['tomatoUserMeter'].replace(",","")
imdb_rating = j['imdbRating'].replace(",","")
Rated = j['Rated'].replace(",","")
Metascore = j['Metascore'].replace(",","")
print "*********"
if 'USA' in j['Country']:
print "comin"
resultant = j['Title'] +','+ imdb_genre +','+ imdb_Actors +','+imdb_Director +','+imdb_Awards+',' +imdb_Production+','+imdb_rating +',' + imdb_votes + ','+j['Released']+','+ tomatoUserReviews +','+tomatoUserRating+','+tomatoUserMeter
print resultant
else:
return render_template('index.html',errMsg="Pls Enter USA Movie")
except:
return render_template('index.html',errMsg="Pls Enter Correct Movie")
try:
genre=imdb_genre.strip()
query = {"genre":genre}
colums = {"_id":1}
doc = genreColl.find_one(query,colums)
if doc is None:
genreId=0
else:
genreId = doc['_id']
print "genre Id " + `genreId`
topActorCount=0
actors=imdb_Actors
actorList = actors.split("-")
for actors in actorList:
actor = actors.strip()
query = {"actor":actor}
colums = {"_id":1}
doc = actorsColl.find_one(query,colums)
if doc is None:
print actor
else:
topActorCount = topActorCount+1
print "topActorCount "+`topActorCount`
topDirectorCount=0
imdb_Director = j['Director'].replace(",","-")
dires=imdb_Director
direList = dires.split("-")
for directors in direList:
director = directors.strip()
query = {"director":director}
colums = {"_id":1}
doc = direColl.find_one(query,colums)
if doc is None:
print director
else:
topDirectorCount = topDirectorCount+1
print("topDirectorCount "+`topDirectorCount`)
prod=imdb_Production
prodList = prod.split("/")
cnt = 0
for prodHouse in prodList:
cnt = cnt+ 1
if (cnt == 1):
productioHouse = prodHouse.strip()
#print productioHouse
if "Screen Media" in productioHouse:
productioHouse="SMedia"
elif "Screen Gems" in productioHouse:
productioHouse="SGems"
elif "Open Road Films" in productioHouse:
productioHouse = "OpRd"
elif "Metro-Goldwyn-Mayer" in productioHouse:
productioHouse ="MGM"
elif "FilmBuff" in productioHouse:
productioHouse = "Buff"
elif "Tribeca" in productioHouse:
productioHouse = "Tribecca"
elif "FilmDistrict" in productioHouse:
productioHouse="District"
elif "New Films" in productioHouse:
productioHouse = "NeFm"
elif "LD Entertainment" in productioHouse:
productioHouse = "LDEnte"
elif "Vitagraph" in productioHouse:
productioHouse ="ViGrp"
elif "Area 23a" in productioHouse:
productioHouse = "Area23A"
elif "Arc Entertainment" in productioHouse:
productioHouse = "ArcEn"
elif "FilmDisctrict" in productioHouse:
productioHouse = "District"
elif "High Top" in productioHouse:
productioHouse="Hightop"
elif "Millennium" in productioHouse:
productioHouse="Milennium"
elif "Radius-TWC" in productioHouse:
productioHouse="Radius"
elif "World Wide"in productioHouse:
productioHouse="WrlWid"
elif "First Independent Pictures" in productioHouse:
productioHouse = "FirInde"
elif "New Line Cinema" in productioHouse:
productioHouse = "Warner"
elif "TriStar" in productioHouse:
productioHouse = "Sony"
elif "Stage 6" in productioHouse:
productioHouse = "Sony"
elif "Touchstone Pictures" in productioHouse:
productioHouse = "Walt"
elif "Disneynature"in productioHouse:
productioHouse = "Walt"
elif "Focus Features"in productioHouse:
productioHouse = "Universal"
elif "Nickelodeon" in productioHouse:
productioHouse = "Paramount"
elif "Roadside" in productioHouse:
productioHouse = "Lionsgate"
elif "Dimension" in productioHouse:
productioHouse = "Weinstein"
doc = db.command("text", "topProdHouse",search=productioHouse, project={"_id": 1})
rs = doc['results']
#print rs
if rs == []:
prodHouseId=0
else:
print productioHouse
prodHouseId=1
print "prodHouseId "+`prodHouseId`
except:
return render_template('index.html',errMsg="Error while transforming the data")
gateway = JavaGateway()
predictionModel = gateway.entry_point
#Genre Production Released Actor Director imdbRating Metascore Wikistats BoxOffice */
output=predictionModel.boxOfficePre(str(genreId),str(prodHouseId),str(status),str(topActorCount),str(topDirectorCount),imdb_rating,Metascore,total_str)
print "Hi "+`output`
boxOfficeValue=""
if output==0.0:
boxOfficeValue="181--200000"
elif output==1.0:
boxOfficeValue="200000--5000000"
elif output==2.0:
boxOfficeValue="5000000--20000000"
elif output==3.0:
boxOfficeValue="20000000--40000000"
elif output==4.0:
boxOfficeValue="40000000--80000000"
elif output==5.0:
boxOfficeValue="80000000--760500000"
parts = boxOfficeValue.split("--")
value1 = parts[0]
value2 = parts[1]
mid = (int(value1) + int(value2))/2
if mid > 100000000:
mid = 100000000
return render_template('predict.html',boxOffice=boxOfficeValue,high_chart = mid, title = movie,poster = j['Poster'],plot = j['Plot'])
def __init__(self):
"""Starts a new instance of SystemDSContext, in which the connection to a JVM systemds instance is handled
Any new instance of this SystemDS Context, would start a separate new JVM.
Standard out and standard error form the JVM is also handled in this class, filling up Queues,
that can be read from to get the printed statements from the JVM.
"""
root = os.environ.get("SYSTEMDS_ROOT")
if root == None:
# If there is no systemds install default to use the PIP packaged java files.
root = os.path.join(get_module_dir(), "systemds-java")
# nt means its Windows
cp_separator = ";" if os.name == "nt" else ":"
lib_cp = os.path.join(root, "target","lib", "*")
systemds_cp = os.path.join(root,"target","SystemDS.jar")
classpath = cp_separator.join([lib_cp , systemds_cp])
command = ["java", "-cp", classpath]
if os.environ.get("SYSTEMDS_ROOT") != None:
files = glob(os.path.join(root, "conf", "log4j*.properties"))
if len(files) > 1:
print("WARNING: Multiple logging files found selecting: " + files[0])
if len(files) == 0:
print("WARNING: No log4j file found at: "
+ os.path.join(root, "conf")
+ " therefore using default settings")
else:
command.append("-Dlog4j.configuration=file:" + files[0])
command.append("org.apache.sysds.api.PythonDMLScript")
# TODO add an argument parser here
# Find a random port, and hope that no other process
# steals it while we wait for the JVM to startup
port = self.__get_open_port()
command.append(str(port))
process = Popen(command, stdout=PIPE, stdin=PIPE, stderr=PIPE)
first_stdout = process.stdout.readline()
if(b"GatewayServer Started" in first_stdout):
print("Startup success")
else:
stderr = process.stderr.readline().decode("utf-8")
if(len(stderr) > 1):
raise Exception("Exception in startup of GatewayServer: " + stderr)
outputs = []
outputs.append(first_stdout.decode("utf-8"))
max_tries = 10
for i in range(max_tries):
next_line = process.stdout.readline()
if(b"GatewayServer Started" in next_line):
print("WARNING: Stdout corrupted by prints: " + str(outputs))
print("Startup success")
break
else:
outputs.append(next_line)
if (i == max_tries-1):
raise Exception("Error in startup of systemDS gateway process: \n gateway StdOut: " + str(outputs) + " \n gateway StdErr" + process.stderr.readline().decode("utf-8") )
# Handle Std out from the subprocess.
self.__stdout = Queue()
self.__stderr = Queue()
Thread(target=self.__enqueue_output, args=(
process.stdout, self.__stdout), daemon=True).start()
Thread(target=self.__enqueue_output, args=(
process.stderr, self.__stderr), daemon=True).start()
# Py4j connect to the started process.
gateway_parameters = GatewayParameters(
port=port, eager_load=True, read_timeout=5)
self.java_gateway = JavaGateway(
gateway_parameters=gateway_parameters, java_process=process)
import numpy
from collections import Counter as mset
"""""" """""" """""" """
similarity score parameters a,b,c,d
""" """""" """""" """"""
a = float(1 / 11) # commen Noun
b = float(5 / 11) # properNoun
c = float(1 / 11) # verb
d = float(4 / 11) # hashtag threshold
threshold = 0.3
threshold2 = 0.4
threshold3 = .4
"""""" """""" """
Initializing NER model and files input
""" ""
gateway = JavaGateway() # coect to the JVM
inputFile = "../Data/mytweet2"
Alltweets = pd.read_csv(inputFile, ",")
# print(len(Alltweets))
Outfilename = "trainingOutputs/clustersIds03_04_04.csv"
output = open(Outfilename, mode='wt', encoding='utf-8')
fieldnames = ['clusterno', 'tweetd']
writer = csv.DictWriter(output,
fieldnames=fieldnames,
quoting=csv.QUOTE_MINIMAL)
writer.writeheader()
# File path which consists of Abbreviations.
fileName = "../utils/slang.txt" #
# File Access mode [Read Mode]
accessMode = "r"
abbrRemov = {}
# -*- coding: utf-8 -*- # @Time : 2020/3/7 13:30 # @Author : Deng Wenxing # @Email : [email protected] # @File : qq_msg.py # @Software: PyCharm # @content : qq相关信息 import argparse import subprocess from py4j.java_gateway import JavaGateway if __name__ == '__main__': getaway = JavaGateway() random = getaway.jvm.java.util.Random() print random.nextInt(10)
def __init__(self, bigdl_type, port=25333):
self.value = JavaGateway(GatewayClient(port=port), auto_convert=True)
def testPythonToJavaToPythonClose(self):
def play_with_ping(gateway):
ping = InstrumentedPythonPing()
pingpong = gateway.jvm.py4j.examples.PingPong()
total = pingpong.start(ping)
return total
def internal_work(assert_memory):
gateway2 = InstrJavaGateway(
gateway_parameters=GatewayParameters(port=DEFAULT_PORT + 5),
callback_server_parameters=CallbackServerParameters(
port=DEFAULT_PYTHON_PROXY_PORT + 5))
sleep()
play_with_ping(gateway2)
python_gc()
sleep()
gateway2.close(close_callback_server_connections=True,
keep_callback_server=True)
sleep()
assert_memory()
gateway2.shutdown()
sleep()
with gateway_server_example_app_process():
gateway = JavaGateway()
gateway.entry_point.startServer2()
def perform_memory_tests():
python_gc()
gateway.jvm.py4j.instrumented.MetricRegistry.\
forceFinalization()
sleep()
createdSet = gateway.jvm.py4j.instrumented.MetricRegistry.\
getCreatedObjectsKeySet()
finalizedSet = gateway.jvm.py4j.instrumented.MetricRegistry.\
getFinalizedObjectsKeySet()
# 10 objects: GatewayServer, 4 GatewayConnection,
# CallbackClient, 4 CallbackConnection
self.assertEqual(10, len(createdSet))
# 13 objects: JavaGateway, CallbackSerer, GatewayClient,
# GatewayProperty, PythonPing, 4 GatewayConnection,
# 4 CallbackConnection. Notice the symmetry between callback
# and gateway connections.
self.assertEqual(13, len(CREATED))
# 4 gateway connections, 3 callback connections.
# There is still one callback connection staying around
# following Java finalization that called back Python.
self.assertEqual(7, len(finalizedSet))
# Same amount of connections for the Python side
self.assertEqual(7, len(FINALIZED))
internal_work(perform_memory_tests)
python_gc()
gateway.jvm.py4j.instrumented.MetricRegistry.forceFinalization()
sleep()
gateway.shutdown()
# 14 objects: JavaGateway, CallbackSerer, GatewayClient,
# GatewayProperty, PythonPing, 5 GatewayConnection,
# 4 CallbackConnection. Notice the symmetry
# One more gateway connection created because we called shutdown
# after close (which requires a connection to send a shutdown
# command).
assert_python_memory(self, 14)
# limitations under the License.
#
from py4j.java_gateway import java_import, JavaGateway, GatewayClient
from pyspark.conf import SparkConf
from pyspark.context import SparkContext
# for back compatibility
from pyspark.sql import SQLContext
# start JVM gateway
if "PY4J_GATEWAY_SECRET" in os.environ:
from py4j.java_gateway import GatewayParameters
gateway_secret = os.environ["PY4J_GATEWAY_SECRET"]
gateway = JavaGateway(gateway_parameters=GatewayParameters(address="${JVM_GATEWAY_ADDRESS}",
port=${JVM_GATEWAY_PORT}, auth_token=gateway_secret, auto_convert=True))
else:
gateway = JavaGateway(GatewayClient(address="${JVM_GATEWAY_ADDRESS}", port=${JVM_GATEWAY_PORT}), auto_convert=True)
java_import(gateway.jvm, "org.apache.spark.SparkEnv")
java_import(gateway.jvm, "org.apache.spark.SparkConf")
java_import(gateway.jvm, "org.apache.spark.api.java.*")
java_import(gateway.jvm, "org.apache.spark.api.python.*")
java_import(gateway.jvm, "org.apache.spark.mllib.api.python.*")
intp = gateway.entry_point
jsc = intp.getJavaSparkContext()
java_import(gateway.jvm, "org.apache.spark.sql.*")
java_import(gateway.jvm, "org.apache.spark.sql.hive.*")
java_import(gateway.jvm, "scala.Tuple2")
def setUp(self):
self.p = start_example_app_process()
self.gateway = JavaGateway()
obs, rew, done, _ = env.step(action)
episode_rew += rew
cnt += 1
print("Episode reward", episode_rew)
print('total cnt', cnt)
return np.array(actions), np.array(observations)
folder_dir = r'C:\Users\huan289\git\RLGC'
#os.chdir(folder_dir)
print(os.getcwd())
java_port = 25333
gateway = JavaGateway(
gateway_parameters=GatewayParameters(port=java_port, auto_convert=True))
ipss_app = gateway.entry_point
# case_files = ['testData\\Kundur-2area\\kunder_2area_ver30.raw','testData\\Kundur-2area\\kunder_2area.dyr']
# Need to use the following way to define a String array in Python for Py4J
case_files_array = gateway.new_array(gateway.jvm.String, 2)
case_files_array[
0] = folder_dir + '\\' + 'testData\\IEEE39\\IEEE39bus_multiloads_xfmr4_smallX_v30.raw'
case_files_array[1] = folder_dir + '\\' + 'testData\\IEEE39\\IEEE39bus_3AC.dyr'
dyn_config_file = folder_dir + '\\' + 'testData\\IEEE39\\json\\IEEE39_dyn_config_v0.65.json'
rl_config_file = folder_dir + '\\' + 'testData\\IEEE39\\json\\IEEE39_RL_loadShedding_3motor_3levels_v0.65.json'
sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
def Cluster(dir, coef=0.2):
gateway = JavaGateway()
Entry = gateway.entry_point
SimilarityMatrix = GenerateMatrix(dir, coef)
print(QTCluster(SimilarityMatrix, Entry))
# Формируем цель обучения сети
q_values[0][action] = target
# Обучение сети
self.model.fit(state, q_values, verbose=0)
if self.freeroll_rate > self.freeroll_min:
self.freeroll_rate *= self.freeroll_decay
if __name__ == "__main__":
def get_state(java_state):
done = java_state.isDone()
reward = java_state.getReward()
state = np.array([[java_state.getX(), java_state.getXDot(), java_state.getTheta(), java_state.getThetaDot()]], dtype = np.float64)
return state, done, reward
# DQN - глубокая Q-нейронная сеть
java_gateway = JavaGateway()
env = java_gateway.jvm.ru.dutov.cartpole.env.CartPoleEnv()
observ_space = env.getObservationSpace()
action_space = env.getActionSpace()
agent = Agent(observ_space, action_space) # Создаем агента
episodes = 500 # Число игровых эпизодов
scores = deque(maxlen = 100)
# scores - хранит длительность последних 100 игр
for e in range(episodes + 1):
# Получаем начальное состояние объекта перед началом каждого эпизода
state, done, reward = get_state(env.reset())
# Python side of Java-Python bridge # see info.kwarc.mmt.python.Py4JGateway (an MMT extension that must be added to MMT for the bridge to work) for the counterpart from py4j.java_gateway import JavaGateway, JavaObject, GatewayParameters, CallbackServerParameters import py4j # create the gateway that communicates with the JVM gwp = GatewayParameters(auto_field=True, auto_convert=True) cbp = CallbackServerParameters() gateway = JavaGateway(gateway_parameters=gwp, callback_server_parameters=cbp) # MMT sets the entry point to be the MMT controller controller = gateway.entry_point # jvm yields access to Java namespace mmt = gateway.jvm.info.kwarc.mmt api = mmt.api # everything below here are optional improvements to smoothen the Scala-Python integration # they also provide examples how to use the bridge # General remarks for handling from Python Scala features that are not present in the JVM anymore # companion object: fields are static methods, call as usual # nullary functions: () is mandatory # sequence arguments: needs a Seq, use Seq(pythonlist) conversion # default arguments: all arguments must be provided # implicit conversions: apply explicitly # magic functions: some correspondence established below # symbolic method names as operators: use .M below, some infix operators can be mapped to Python magic functions (see below)
def launch_gateway(conf=None):
"""
launch jvm gateway
:param conf: spark configuration passed to spark-submit
:return:
"""
if "PYSPARK_GATEWAY_PORT" in os.environ:
gateway_port = int(os.environ["PYSPARK_GATEWAY_PORT"])
else:
SPARK_HOME = _find_spark_home()
# Launch the Py4j gateway using Spark's run command so that we pick up the
# proper classpath and settings from spark-env.sh
on_windows = platform.system() == "Windows"
script = "./bin/spark-submit.cmd" if on_windows else "./bin/spark-submit"
command = [os.path.join(SPARK_HOME, script)]
if conf:
for k, v in conf.getAll():
command += ['--conf', '%s=%s' % (k, v)]
submit_args = os.environ.get("PYSPARK_SUBMIT_ARGS", "pyspark-shell")
if os.environ.get("SPARK_TESTING"):
submit_args = ' '.join(
["--conf spark.ui.enabled=false", submit_args])
command = command + shlex.split(submit_args)
# Start a socket that will be used by PythonGatewayServer to communicate its port to us
callback_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
callback_socket.bind(('127.0.0.1', 0))
callback_socket.listen(1)
callback_host, callback_port = callback_socket.getsockname()
env = dict(os.environ)
env['_PYSPARK_DRIVER_CALLBACK_HOST'] = callback_host
env['_PYSPARK_DRIVER_CALLBACK_PORT'] = str(callback_port)
# Launch the Java gateway.
# We open a pipe to stdin so that the Java gateway can die when the pipe is broken
if not on_windows:
# Don't send ctrl-c / SIGINT to the Java gateway:
def preexec_func():
signal.signal(signal.SIGINT, signal.SIG_IGN)
proc = Popen(command, stdin=PIPE, preexec_fn=preexec_func, env=env)
else:
# preexec_fn not supported on Windows
proc = Popen(command, stdin=PIPE, env=env)
gateway_port = None
# We use select() here in order to avoid blocking indefinitely if the subprocess dies
# before connecting
while gateway_port is None and proc.poll() is None:
timeout = 1 # (seconds)
readable, _, _ = select.select([callback_socket], [], [], timeout)
if callback_socket in readable:
gateway_connection = callback_socket.accept()[0]
# Determine which ephemeral port the server started on:
gateway_port = read_int(gateway_connection.makefile(mode="rb"))
gateway_connection.close()
callback_socket.close()
if gateway_port is None:
raise Exception(
"Java gateway process exited before sending the driver its port number"
)
# In Windows, ensure the Java child processes do not linger after Python has exited.
# In UNIX-based systems, the child process can kill itself on broken pipe (i.e. when
# the parent process' stdin sends an EOF). In Windows, however, this is not possible
# because java.lang.Process reads directly from the parent process' stdin, contending
# with any opportunity to read an EOF from the parent. Note that this is only best
# effort and will not take effect if the python process is violently terminated.
if on_windows:
# In Windows, the child process here is "spark-submit.cmd", not the JVM itself
# (because the UNIX "exec" command is not available). This means we cannot simply
# call proc.kill(), which kills only the "spark-submit.cmd" process but not the
# JVMs. Instead, we use "taskkill" with the tree-kill option "/t" to terminate all
# child processes in the tree (http://technet.microsoft.com/en-us/library/bb491009.aspx)
def killChild():
Popen([
"cmd", "/c", "taskkill", "/f", "/t", "/pid",
str(proc.pid)
])
atexit.register(killChild)
# Connect to the gateway
gateway = JavaGateway(GatewayClient(port=gateway_port), auto_convert=True)
# Import the classes used by PySpark
java_import(gateway.jvm, "org.apache.spark.SparkConf")
java_import(gateway.jvm, "org.apache.spark.api.java.*")
java_import(gateway.jvm, "org.apache.spark.api.python.*")
java_import(gateway.jvm, "org.apache.spark.ml.python.*")
java_import(gateway.jvm, "org.apache.spark.mllib.api.python.*")
# TODO(davies): move into sql
java_import(gateway.jvm, "org.apache.spark.sql.*")
java_import(gateway.jvm, "org.apache.spark.sql.hive.*")
java_import(gateway.jvm, "scala.Tuple2")
return gateway
from py4j.java_gateway import JavaGateway, GatewayParameters
from py4j.protocol import Py4JNetworkError
from django.conf import settings
import re
from decimal import Decimal
gateway = JavaGateway(gateway_parameters=GatewayParameters(
address=settings.BPAY_GATEWAY, port=25333, read_timeout=10))
def test_connection():
try:
gateway.jvm.System.currentTimeMillis()
except Py4JNetworkError:
raise Exception('Cannot connect to gateway server')
except Exception as e:
raise e
finally:
gateway.close()
def getCRN(number):
if settings.BPAY_GATEWAY:
test_connection()
crn = gateway.entry_point.getCRN()
return crn.generateBPAYCrnWithMod10V05(str(number))
return generate_crn(number)
def getICRN(number, amount, option='ICRNAMT'):
if settings.BPAY_GATEWAY:
else:
raise ValueError("fmt must be 'png' or 'svg'")
html = "<div style='width:{width};height:{height}'>{img}<div>"
#print(html.format(width=width, height=height, img=img_str))
intp.showHTML(html.format(width=width, height=height, img=img_str))
img.close()
signal.signal(signal.SIGINT, handler_stop_signals)
_pUserQueryNameSpace = {}
client = GatewayClient(port=int(sys.argv[1]))
#gateway = JavaGateway(client, auto_convert = True)
gateway = JavaGateway(client)
output = Logger()
sys.stdout = output
sys.stderr = output
intp = gateway.entry_point
show = __show__ = PythonContext(intp)
__show__._setup_matplotlib()
intp.onPythonScriptInitialized(os.getpid())
while True:
req = intp.getStatements()
if req == None:
break
def __init__(self):
self.gateway = JavaGateway()
from pyspark.accumulators import Accumulator, AccumulatorParam
from pyspark.broadcast import Broadcast
from pyspark.serializers import MarshalSerializer, PickleSerializer
from time import sleep
# for back compatibility
from pyspark.sql import SparkSession, DataFrame, Row
client = GatewayClient(port=int(sys.argv[1]))
sparkVersion = sys.argv[2]
print("PYTHON:: Starting gateway")
if re.match("^1\.[456]\..*$", sparkVersion) or re.match(
"^2\..*$", sparkVersion):
gateway = JavaGateway(client, auto_convert=True)
else:
gateway = JavaGateway(client)
print("PYTHON:: Gateway started")
java_import(gateway.jvm, "org.apache.spark.SparkEnv")
java_import(gateway.jvm, "org.apache.spark.SparkConf")
java_import(gateway.jvm, "org.apache.spark.api.java.*")
java_import(gateway.jvm, "org.apache.spark.api.python.*")
java_import(gateway.jvm, "org.apache.spark.mllib.api.python.*")
bridge = gateway.entry_point
state = bridge.state()
state.markReady()
if sparkVersion.startswith("1.2"):
from py4j.java_gateway import JavaGateway, CallbackServerParameters
class PythonListener(object):
def __init__(self, gateway):
self.gateway = gateway
def notify(self, obj):
print("Notified by Java")
print(obj)
gateway.jvm.System.out.println("Hello from python!")
return "A Return Value"
class Java:
implements = ["py4j.examples._3_3_callback.ExampleListener"]
if __name__ == "__main__":
gateway = JavaGateway(
callback_server_parameters=CallbackServerParameters())
listener = PythonListener(gateway)
gateway.entry_point.registerListener(listener)
gateway.entry_point.notifyAllListeners()
gateway.shutdown()
#in the cmd window ,the python output is as follows:
# Notified by Java
# <ListenerApplication> instance
from py4j.java_gateway import JavaGateway
from py4jfml.knowledgebasevariable import AggregatedFuzzyVariableType as afvt
from py4jfml.knowledgebasevariable import FuzzyVariableType as fvt
from py4jfml.term import TskTermType as tsktt
gateway = JavaGateway()
class TskVariableType:
'''
Python class for tskVariableType complex type
'''
def __init__(self, name=None):
'''
:param name:
'''
if name == None:
self.java_kbv = gateway.entry_point.getJFMLKnowledgebaseVariable_Factory(
).createTskVariableType()
else:
assert type(name) == str
self.java_kbv = gateway.entry_point.getJFMLKnowledgebaseVariable_Factory(
).createTskVariableType(name)
# Method of class KnowledgeBaseVariable
def isInput(self):
'''
Tests if the variable is input type
:return: true if the variable is input type; false otherwise
'''
return self.java_kbv.isInput()
def get_jvm() -> JVMView:
import geopyspark
pysc = geopyspark.get_spark_context()
gateway = JavaGateway(eager_load=True, gateway_parameters=pysc._gateway.gateway_parameters)
jvm = gateway.jvm
return jvm
