def getVector_semantic(self, tweets, lang):
now = datetime.datetime.now()
dia = now.day
mes = now.month
anyo = now.year
#configuracion del sistema
conf = Conf()
path = conf.getAbsPath()
model_loc = '%s/LuigiTasks/TrainText/Doc2VecLang_semantic/%s/%s/%s_%s.model'%(path, anyo, mes, dia, lang)
days_minus = 1
while os.path.isfile(model_loc) == False and days_minus < 20:
now = datetime.datetime.now() - datetime.timedelta(days=days_minus)
dia = now.day
mes = now.month
anyo = now.year
model_loc = '%s/LuigiTasks/TrainText/Doc2VecLang_semantic/%s/%s/%s_%s.model'%(path, anyo, mes, dia, lang)
days_minus += 1
d2v = None
if model_loc in self.models_opened:
d2v = self.models_opened[model_loc]
else:
d2v = Doc2Vec()
self.models_opened[model_loc] = d2v
vectorText = self.generateVectorText_semantic(tweets, lang).split(" ")
vector = np.array(d2v.simulateVectorsFromVectorText(vectorText, model_loc))
return vector / np.linalg.norm(vector)
def output(self):
conf = Conf()
path = conf.getAbsPath()
now = datetime.datetime.now()
dia = now.day
mes = now.month
anyo = now.year
hour = now.hour
return luigi.LocalTarget('%s/LuigiTasks/contenido/%s/%s/%s/%s/%s' %
(path, anyo, mes, dia, hour, self.busqueda))
def __init__(self):
infoSQL = Conf().getSQLInfo()
self.conn = psycopg2.connect(database=infoSQL.database,
user=infoSQL.user,
password=infoSQL.password,
host=infoSQL.host)
infoSQL = Conf().getSQLPoliceInfo()
self.policeConn = psycopg2.connect(database=infoSQL.database,
user=infoSQL.user,
password=infoSQL.password,
host=infoSQL.host)
def run(self):
with self.output().open("w") as out:
d2v = Doc2Vec_sent()
savePath = self.output().path.replace("check", "model")
conf = Conf()
d2v.train(self.input().path,
savePath,
dimension=conf.getDimVectors(),
epochs=20,
method="DM",
isString=True)
out.write("OK")
def run(self):
#configuracion del sistema
conf = Conf()
path = conf.getAbsPath()
comand = "luigi --module LuigiTasks.GenerateSentiment "
if self.mentions == True:
comand += "GenerateSentimentMetions"
else:
comand += "GenerateSentimentUser"
comand += " --lang " + self.lang + " --user " + self.username
comand += " > /dev/null 2>&1"
os.popen(comand)
def output(self):
conf = Conf()
path = conf.getAbsPath()
now = datetime.datetime.now()
dia = now.day
mes = now.month
anyo = now.year
try:
usuario = self.usuario.replace("@", "")
self.usuario = usuario
except:
pass
return luigi.LocalTarget('%s/LuigiTasks/circulo/%s/%s/%s' %
(path, anyo, mes, self.usuario))
def __init__(self):
self.conf = (
SparkConf().setMaster("local[8]").setAppName("My app").set(
"spark.executor.memory",
"6g").set("spark.driver.memory",
"4g").setSparkHome(Conf().getSparkHome()))
self.sc = SparkContext(conf=self.conf)
def output(self):
conf = Conf()
path = conf.getAbsPath()
"""
"""
now = datetime.datetime.now()
dia = now.day
mes = now.month
anyo = now.year
try:
usuario = self.usuario.replace("@", "")
self.usuario = usuario
except:
pass
#return luigi.LocalTarget('%s/LuigiTasks/graphs/gephi/%s/%s/%s_%s_%s'%(path, anyo, mes, self.usuario))
return luigi.LocalTarget('%s/graphs/pagerank/%s/%s/%s.gexf' %
(path, anyo, mes, self.usuario))
def getCursor_police(self):
try:
return self.policeConn.cursor()
except:
infoSQL = Conf().getSQLPoliceInfo()
self.policeConn = psycopg2.connect(database=infoSQL.database,
user=infoSQL.user,
password=infoSQL.password,
host=infoSQL.host)
return self.policeConn.cursor()
def __init__(self):
config = Conf()
host = config.getRedis('host')
port = config.getRedis('port')
db = config.getRedis('db')
password = config.getRedis('password')
self.__time = config.getRedis('activeTime')
self.__redis = redis.StrictRedis(host, port, db, password)
def __init__(self):
cluster_cass = Cluster()
self.session = cluster_cass.connect(Conf().getCassandraKeyspace())
self.session.default_timeout = 100
self.session_instagram = cluster_cass.connect(
Conf().getCassandraKeyspaceInstagram())
# -*- coding: utf-8 -*-
import os
import sys
from Config.Conf import Conf
lib_path = os.path.abspath(Conf().getSparkHome() + '/python')
sys.path.append(lib_path)
from pyspark import SparkConf, SparkContext
class SparkContexto():
"""docstring for SparkContexto"""
class __impl:
"""docstring for __impl"""
def __init__(self):
self.conf = (
SparkConf().setMaster("local[8]").setAppName("My app").set(
"spark.executor.memory",
"6g").set("spark.driver.memory",
"4g").setSparkHome(Conf().getSparkHome()))
self.sc = SparkContext(conf=self.conf)
def getContexto(self):
return self.sc
#return None
# storage for the instance reference
__instance = None
def __init__(self):
if SparkContexto.__instance is None:
SparkContexto.__instance = SparkContexto.__impl()
def output(self):
conf = Conf()
path = conf.getAbsPath()
return luigi.LocalTarget('%s/Data/results/%s.%s.json'%(path, self.lang, self.user))
def output(self):
conf = Conf()
path = conf.getAbsPath()
return luigi.LocalTarget('%s/Data/%s.mod_def' % (path, self.lang))
def __init__(self):
authenticate("localhost:7474", "neo4j", Conf().getNeo4jPassword())
self.graph = Graph()
def output(self):
conf = Conf()
path = conf.getAbsPath()
return luigi.LocalTarget('%s/Data/tree_%s.clasi' % (path, self.lang))
def __str__(self):
conf = Conf()
str = "访问%s失败，请检查网络连接" % conf.getDefault('TestIP')
return str
def output(self):
conf = Conf()
path = conf.getAbsPath()
return luigi.LocalTarget(
'%s/Data/%s.train' % (path, self.lang),
format=luigi.format.TextFormat(encoding='utf8'))
def __init__(self):
conf = Conf()
self.DB = conf.getDefault('DBData')
def __str__(self):
conf = Conf()
str = "使用DB_CONNECT_STRING:%s--连接数据库失败" % conf.getDefault('DBData')
return str