def index():
input = request.json
for key in input:
print key, input[key]
className = input['class']
parent = input['parent']
mutexs = input['mutex']
terms = input['term']
seeds = input['seed']
"""
kb.class_relation
"""
#sql = 'select count(*) from class_relation where class_src=%s relation=%d class_dest=%s' %(parent, 1, className)
#rows = Mysql('kb', host=MYSQL_HOST).execute(sql)
#count = int(rows[0][0])
#if count == 0:
mysql = Mysql('kb', host=MYSQL_HOST)
mysql.connect()
sql = 'insert into class_relation(class_src,relation,class_dest) values (\'%s\', %d, \'%s\')' % (parent, 1, className)
try:
rows = Mysql('kb', host=MYSQL_HOST).execute(sql)
except MySQLdb.Error, e:
#key exists
if e.args[0] == 1062:
pass
else:
print traceback.format_exc()
def lista_tipo():
retorno = []
bd = Mysql()
cursor = bd.cursor
data = inicio_fim_semana()
query = (
"SELECT 'origem' as tipo,origem.descricao,tipo_veiculo.descricao,count(*) as qtd FROM checkin\
inner join motorista ON (motorista.codigo = checkin.codigo_motorista)\
inner join tipo_veiculo ON (tipo_veiculo.codigo = motorista.tipo_veiculo)\
INNER JOIN local origem ON (origem.codigo = checkin.codigo_origem)\
GROUP BY origem.descricao,tipo_veiculo.descricao\
UNION\
SELECT 'destino' as tipo ,destino.descricao,tipo_veiculo.descricao,count(*) as qtd FROM checkin\
inner join motorista ON (motorista.codigo = checkin.codigo_motorista)\
inner join tipo_veiculo ON (tipo_veiculo.codigo = motorista.tipo_veiculo)\
INNER JOIN local destino ON (destino.codigo = checkin.codigo_destino)\
GROUP BY destino.descricao,tipo_veiculo.descricao\
")
bd.query(query)
resultado = bd.fetchall()
for dado in resultado:
retorno.append({
'tipo': dado[0],
'descricao_local': dado[1],
'descricao_tipo_veiculo': dado[2],
'qtd_veiculos': dado[3],
})
return retornar(retorno, 200)
def veiculo_proprio():
bd = Mysql()
cursor = bd.cursor
query = (
"SELECT nome, idade, sexo, tipo_cnh, veiculo_proprio, tipo_veiculo.descricao AS tipo_veiculo_descricao, \
(SELECT carregado FROM checkin WHERE checkin.codigo_motorista = motorista.codigo order by data DESC LIMIT 1) AS carregado\
FROM motorista\
INNER JOIN tipo_veiculo ON (tipo_veiculo.codigo = motorista.tipo_veiculo)\
WHERE (SELECT carregado FROM checkin WHERE checkin.codigo_motorista = motorista.codigo order by data DESC LIMIT 1) = 0\
")
bd.query(query)
retorno = []
resultado = bd.fetchall()
for dado in resultado:
retorno.append({
'name': dado[0],
'idade': dado[1],
'sexo': dado[2],
'tipo_cnh': dado[3],
'veiculo_proprio': dado[4],
'tipo_veiculo_descricao': dado[5],
'carregado': dado[6],
})
return retornar(retorno, 200)
def InitDb(host, uid, pwd, db, table, inputfile):
createDbAndTable(host, uid, pwd, db)
#连接数据库
mydb = Mysql(database = db)
#初始化搜索列表
urlfile = open(inputfile, "r")
mydb.putUrl(table, urlfile)
def retorna_motorista(nome):
bd = Mysql()
cursor = bd.cursor
query = ("SELECT codigo FROM motorista Where nome = '" + str(nome) + "'")
bd.query(query)
resultado = bd.fetchall()
for dado in resultado:
return dado[0]
return False
class Database(object):
def __init__(self, host, username, pwd, database, table=''):
self._table = table
self._conn = Mysql(host, username, pwd, database)
def insert(self, table, *args, **kwargs):
return self._conn.insert(table, *args, **kwargs)
def delete(self, table, where=None, *args):
return self._conn.delete(table, where, *args)
def select(self, table, where=None, *args, **kwargs):
return self._conn.select(table, where, *args, **kwargs)
def update_ports(self, sw_name, act, stand):
logging.info("Updating ActivePort=%s and StandbyPort=%s for switch=%s", act, stand, sw_name)
self._conn.update(self._table, 'switch_name=%s', sw_name, active_port=act, standby_port=stand)
def update_power_status(self, sw_name, power):
logging.info("Updating is_powered_on as %s for switch=%s", power, sw_name)
self._conn.update(self._table, 'switch_name=%s', sw_name, is_powered_on=power)
def update_images(self, sw_name, kick, sys):
logging.info("Updating kickstart=%s and system=%s images for switch=%s", kick, sys, sw_name)
self._conn.update(self._table, 'switch_name=%s', sw_name, kickstart=kick, system=sys)
def retorna_codigo_local(descricao):
bd = Mysql()
cursor = bd.cursor
query = ("SELECT codigo FROM local Where descricao = '" + str(descricao) +
"'")
bd.query(query)
resultado = bd.fetchall()
for dado in resultado:
return dado[0]
return False
def index():
indicators = []
mysql = Mysql(db = DB,
host = MYSQL_HOST,
user = USER,
passwd = PASSWD)
sql = 'select name from indicator'
rows = mysql.execute(sql)
for row in rows:
indicators.append(row[0])
return {'msg':'ok', 'value':indicators}
def __init__(self, log):
self.extract = Extract()
self.sql = Mysql("localhost", "root", "hello", "tweets", 3306)
# self.colors = Colors()
self.stream_url = self.extract.api['TwitterStream']
self.stream_rules_url = self.extract.api['TwitterStreamRules']
self.rules = self.extract.api['TwitterStreamRulesSettings']
self.auth = lambda x: {
"Authorization": "Bearer {}".format(x),
"Content-type": "application/json"
}
self.conn = 0
self.log = lambda log: (log == 1)
def checkRepeat(url=''):
try:
mysql = Mysql()
target_data_list = mysql.getAll(
"SELECT id from tb_article where href='" + url + "'")
mysql.dispose()
if len(target_data_list) > 0:
return False
else:
return True
except Exception as e:
log("[ERROR]checkRepeat is error for " + url)
log(str(e))
return False
def index():
input = request.json
className = input['class']
seed = input['seed']
sentence = input['sentence_src']
sentence_dest = input['sentence_dest']
label = input['label']
sql = 'update sentence set flag=%d,sentence_dest=\'%s\' where class=\'%s\' and sentence_src=\'%s\'' %(label, sentence_dest,className, sentence)
mysql = Mysql('healthai', host=MYSQL_HOST, user=USER, passwd=PASSWD)
mysql.connect()
try:
rows = mysql.executeNoConn(sql)
except MySQLdb.Error, e:
print traceback.format_exc()
def veiculo_proprio():
retorno = []
bd = Mysql()
cursor = bd.cursor
query = (
"SELECT count(*) AS qtd FROM motorista WHERE veiculo_proprio = 1")
bd.query(query)
resultado = bd.fetchall()
for dado in resultado:
retorno.append({'qtd': dado[0]})
return retornar(retorno, 200)
def main():
#在mysql中创建数据库
host = 'localhost'
uid = 'root'
pwd = '123456'
db = 'WebSpider'
table = 'urlnew'
inputfile = "input.txt"
InitDb(host, uid, pwd, db, table, inputfile)
delNum = 0 #数据库中被删除的url个数
while True:
mydb = Mysql(database = db)
urlRequest = mydb.getUrl(table, 20)
length = len(urlRequest)
if 0 == len(urlRequest):
print "URL is over"
break
delNum = delNum + length
mydb.delUrl(table, delNum)
usingThreadpool(10, urlRequest)
mylock.acquire()
while True:
if 0 == urlResult.qsize():
#print "urlResult is Empty!"
break
url = urlResult.get()
value = ("urlnew", url)
mydb.insertOneUrl(value)
mydb.commit()
mylock.release()
class Database(object):
def __init__(self, host, username, pwd, database):
self._conn = Mysql(host, username, pwd, database)
def insert(self, table, *args, **kwargs):
return self._conn.insert(table, *args, **kwargs)
def delete(self, table, where=None, *args):
return self._conn.delete(table, where, *args)
def select(self, table, where=None, *args, **kwargs):
return self._conn.select(table, where, *args, **kwargs)
def update(self, table, where=None, *args, **kwargs):
return self._conn.update(table, where, *args, **kwargs)
class Database(object):
def __init__(self, host, username, pwd, database):
self._conn = Mysql(host, username, pwd, database)
def insert(self, table, *args, **kwargs):
return self._conn.insert(table, *args, **kwargs)
def delete(self, table, where=None, *args):
return self._conn.delete(table, where, *args)
def select(self, table, where=None, *args, **kwargs):
return self._conn.select(table, where, *args, **kwargs)
def update(self, table, where=None, *args, **kwargs):
return self._conn.update(table, where, *args, **kwargs)
def reemplazar(cls, id_jugador, nombre_nuevo):
""" reemplaza el nombre del jugador parseado por el que esta almacenado en la bbdd"""
sql = Mysql.conectar()
sql.execute("UPDATE jugador SET nombre=%s WHERE id=%s", (nombre_nuevo, id_jugador))
cls.__jugadores.clear()
Jugadores.load_jugadores()
def emotionType(words):
mysql = Mysql()
article_config = mysql.getOne(
"SELECT * from tb_article_config where ac_id=1")
mysql.dispose()
words_positive = article_config.get("ac_words_positive",
'').split(",") #正面
words_negative = article_config.get("ac_words_negative",
'').split(",") #负面
type = 0
if len(words) > 0:
negaCheck = list(set(words).intersection(set(words_negative)))
if len(negaCheck) > 0:
type = 2
elif len(list(set(words).intersection(set(words_positive)))):
type = 1
return type
def getDBTargetList():
mysql = Mysql()
target_data_list = mysql.getAll(
"SELECT * from tb_crawler_config where status=1 order by id asc")
mysql.dispose()
target_data_list = list(target_data_list)
for i in xrange(len(target_data_list)):
target_data_list[i]['headers'] = json.loads(
target_data_list[i]['headers'])
target_data_list[i]['login_info'] = json.loads(
target_data_list[i]['login_info'])
target_data_list[i]['data_rule'] = json.loads(
target_data_list[i]['data_rule'])
target_data_list[i]['url_params'] = json.loads(
target_data_list[i]['url_params'])
target_data_list[i]['comment_rule'] = json.loads(
target_data_list[i]['comment_rule'])
return target_data_list
def atualizar():
bd = Mysql()
cursor = bd.cursor
post = json.loads(request.body.getvalue().decode('utf-8'))
#motorista
codigo = post["codigo"]
nome = post["nome"]
idade = post["idade"]
veiculo_proprio = post["veiculo_proprio"]
tipo_cnh = post["tipo_cnh"]
sexo = post["sexo"]
tipo_veiculo = post["tipo_veiculo"]
codigo = retorna_motorista_por_codigo(codigo)
if (codigo != False):
sql_motorista = """ UPDATE `motorista`
SET `nome` = %s ,`idade` = %s ,`sexo` = %s ,`veiculo_proprio` = %s ,`tipo_veiculo` = %s ,`tipo_cnh` = %s WHERE codigo = %s"""
bd.query(sql_motorista, (nome, idade, sexo, veiculo_proprio,
tipo_veiculo, tipo_cnh, codigo))
bd.commit()
if (cursor.rowcount > 0):
return retornar('Alterado com sucesso!', 200)
else:
return retornar('Nenhum registro foi alterado!', 200)
else:
retorno = 'Motorista Informado não existe'
return retornar(retorno, 400)
def sqlOperate(funName, sql):
mysql = Mysql(host, port, db, username, pwd)
print sql
if funName.strip() == "loadFile":
mysql.loadFile(sql)
elif funName.strip() == "insert":
mysql.insert(sql)
else:
print "no mysql method : " + str(funName)
def veiculo_por_terminal():
retorno = []
bd = Mysql()
cursor = bd.cursor
data = inicio_fim_semana()
query = (
"SELECT (SELECT COUNT(*) FROM checkin WHERE Day(data) = Day(NOW())) as dia,\
(SELECT COUNT(*) FROM checkin WHERE data between '" + data[0] +
"' and '" + data[1] + "') as semana,\
(SELECT COUNT(*) FROM checkin WHERE Month(data) = Month(NOW())) as mes\
FROM checkin LIMIT 1")
bd.query(query)
resultado = bd.fetchall()
for dado in resultado:
retorno.append({
'dia': dado[0],
'semana': dado[1],
'mes': dado[2],
})
return retornar(retorno, 200)
class Database(object):
def __init__(self, host, username, pwd, database, table=''):
self._table = table
self._conn = Mysql(host, username, pwd, database)
def insert(self, table, *args, **kwargs):
return self._conn.insert(table, *args, **kwargs)
def delete(self, table, where=None, *args):
return self._conn.delete(table, where, *args)
def select(self, table, where=None, *args, **kwargs):
return self._conn.select(table, where, *args, **kwargs)
def update_ports(self, sw_name, act, stand):
logging.info("Updating ActivePort=%s and StandbyPort=%s for switch=%s",
act, stand, sw_name)
self._conn.update(self._table,
'switch_name=%s',
sw_name,
active_port=act,
standby_port=stand)
def update_power_status(self, sw_name, power):
logging.info("Updating is_powered_on as %s for switch=%s", power,
sw_name)
self._conn.update(self._table,
'switch_name=%s',
sw_name,
is_powered_on=power)
def update_images(self, sw_name, kick, sys):
logging.info(
"Updating kickstart=%s and system=%s images for switch=%s", kick,
sys, sw_name)
self._conn.update(self._table,
'switch_name=%s',
sw_name,
kickstart=kick,
system=sys)
def load_jugadores(cls):
sql = Mysql.conectar()
sql.execute("""select e.nombre as equipo, j.id, j.nombre
from jugador j
inner join equipo e ON e.id=j.id_equipo
order by e.nombre, j.nombre""")
for fila in sql.fetchall():
equipo = str(fila['equipo'])
if equipo not in cls.__jugadores.keys():
"""
si no existe el equipo como key del diccionario es crea una una key con una lista vacia para poder
incorporar los jugadores despues
"""
cls.__jugadores[equipo] = []
cls.__jugadores[equipo].append({"nombre": str(fila['nombre']), "id": int(fila['id'])})
def nuevo(cls, jugador):
""" inserta un nuevo jugador que no existe en la bbdd """
sql = Mysql.conectar()
sql.execute("SELECT id FROM equipo WHERE nombre='{}'".format(jugador['equipo']))
id_equipo = sql.fetchone()
sql.execute("""INSERT INTO jugador (nombre, id_equipo, posicion, nacionalidad, pais_nacimiento,
fecha_nacimiento, dorsal, imagen) VALUES ('{}', {}, '{}', '{}', '{}', '{}', {}, '')""".format(
jugador['nombre'], str(id_equipo['id']), jugador['posicion'], jugador['nacionalidad'],
jugador['pais_nacimiento'], jugador['fecha_nacimiento'], jugador['dorsal']
))
sql.execute("SELECT id FROM jugador ORDER BY id DESC LIMIT 1")
id_nuevo = sql.fetchone()
id_jugador = id_nuevo['id']
cls.__jugadores.clear()
cls.load_jugadores()
return id_jugador
element = browser.find_element_by_css_selector("form")
schedule_parser(element.text, court_number, mysql)
# browser.save_screenshot("images/" + dtstr + ".png")
if __name__ == "__main__":
try:
# 接続する
conn = mysql.connector.connect(host="db",
port=3306,
user="******",
password="******")
mysql = Mysql()
# browser = webdriver.Firefox() # 普通のFilefoxを制御する場合
# browser = webdriver.Chrome() # 普通のChromeを制御する場合
# HEADLESSブラウザに接続
browser = webdriver.Remote(
command_executor="http://selenium-hub:4444/wd/hub",
desired_capabilities=DesiredCapabilities.CHROME,
)
# Googleで検索実行
bunkyo_scraper(browser, mysql)
finally:
# 終了
def text_reply(msg):
sql = Mysql()
now_time = time.strftime("%Y,%m,%d")
year = now_time.split(",")[0]
month = now_time.split(",")[1]
date = now_time.split(",")[2]
global flag
global money
if flag == 0:
if (msg.text == "1" or msg.text == "2" or msg.text == "3"):
flag = 1
else:
message = "请输入要进行的操作\n1.记账\n2.显示本月消费\n3.图形化显示每月消费"
if flag == 1:
if (msg.text == "1"):
message = "请输入消费金额"
flag = 2
elif (msg.text == "2"):
message = sql.month_consume(month)
elif (msg.text == "3"):
message = "模块建设中..."
elif flag == 2:
money = msg.text
message = "请选择消费类型\n1.吃饭\n2.水果零食\n3.学习\n4.运动\n0.回到首页"
flag = 3
else:
if (msg.text == "1"):
message = "吃饭记账成功"
sql.weixin_account(money, "吃饭", year, month, date)
print(money)
flag = 0
elif (msg.text == "2"):
message = "水果零食记账成功"
sql.weixin_account(money, "水果零食", year, month, date)
print(money)
flag = 0
elif (msg.text == "3"):
message = "学习用品记账成功"
sql.weixin_account(money, "学习用品", year, month, date)
print(money)
flag = 0
elif (msg.text == "4"):
message = "运动消费记账成功"
sql.weixin_account(money, "运动消费", year, month, date)
print(money)
flag = 0
else:
message = "请输入要进行的操作\n1.记账\n2.显示本月消费\n3.图形化显示每月消费"
flag = 0
msg.user.send(message)
def __init__(self, host, username, pwd, database):
self._conn = Mysql(host, username, pwd, database)
def index():
projectid = request.json['projectid']
indicators = request.json['indicator']
try:
mysql = Mysql(db = DB,
host = MYSQL_HOST,
user = USER,
passwd = PASSWD)
mysql.connect()
for idk in indicators:
# check if exists
idk = idk.strip().decode('utf8')
sql = "select indicator_id from indicator where name = '%s'" %(idk,)
print sql
res = mysql.executeNoConn(sql)
if len(res) > 0:
idk_id = res[0][0]
else:
# insert and get indicator_id
sql = "insert into indicator(name, type) values('%s', %s)" %(idk, 1)
print sql
mysql.executeNoConn(sql)
sql = "select indicator_id from indicator where name = '%s'" %(idk,)
print sql
res = mysql.executeNoConn(sql)
idk_id = res[0][0]
# sav projectid & indicatorid
sql = "insert into project_indicator values('%s', '%s')" %(projectid, idk_id)
mysql.executeNoConn(sql)
mysql.close()
except MySQLdb.Error, e:
print traceback.format_exc()
mysql.close()
def index():
#保存修改数据,写入数据库
projectid = request.json['projectid']
emrid = request.json['emrid']
indicators = request.json['indicator']
try:
mysql = Mysql(db = DB,
host = MYSQL_HOST,
user = USER,
passwd = PASSWD)
mysql.connect()
sql = "select emr_id from emr where detail = '%s'" %(emrid,)
res = mysql.executeNoConn(sql)
emr_id = res[0][0]
sql = "insert into project_emr values(%s, %s)" %(projectid, emr_id)
print sql
try:
mysql.executeNoConn(sql)
except:
pass
for idk in indicators:
value = indicators[idk].strip()
sql = "select indicator_id from indicator where name = '%s'" %(idk,)
res = mysql.executeNoConn(sql)
idk_id = res[0][0]
# sav emr_id & ink_id
sql = "insert into emr_indicator values(%s, %s, '%s')" %(emr_id, idk_id, value)
print sql
try:
mysql.executeNoConn(sql)
except:
pass
mysql.close()
except MySQLdb.Error, e:
print traceback.format_exc()
mysql.close()
def index():
print request
projectid = request.query.get('projectid')
emrid = request.query.get('emrid')
#参数要带projectId
#1.根据EMR_id, 查询EMR_full表，获取EMR_text, 展现在左侧
#2.根据EMR_id, 查询EMR_indicator表，获取indicator, value，展现在右侧
#3.保存按钮,提交修改的数据到EMR_indicator
indicator = {}
try:
mysql = Mysql(db = DB,
host = MYSQL_HOST,
user = USER,
passwd = PASSWD)
mysql.connect()
try:
sql = "select emr_id from emr where detail = '%s'" %(emrid,)
res = mysql.executeNoConn(sql)
print sql, res
emr_id = res[0][0]
except:
pass
sql = 'select indicator_id from project_indicator where project_id = %s' %(projectid,)
idk_ids = []
rows = mysql.executeNoConn(sql)
for row in rows:
idk_ids.append(row[0])
for idk_id in idk_ids:
sql = "select name from indicator where indicator_id = %s" %(idk_id,)
rows = mysql.executeNoConn(sql)
name = rows[0][0]
sql = "select value from emr_indicator where emr_id = %s and indicator_id = %s" %(emr_id, idk_id)
rows = mysql.executeNoConn(sql)
value = ""
if len(rows) > 0:
value = rows[0][0]
indicator[name] = value
mysql.close()
except MySQLdb.Error, e:
print traceback.format_exc()
mysql.close()
def __init__(self, host, username, pwd, database, table=''):
self._table = table
self._conn = Mysql(host, username, pwd, database)
def cadastrar():
bd = Mysql()
cursor = bd.cursor
post = json.loads(request.body.getvalue().decode('utf-8'))
#motorista
nome = post["nome"]
idade = post["idade"]
veiculo_proprio = post["veiculo_proprio"]
tipo_cnh = post["tipo_cnh"]
sexo = post["sexo"]
tipo_veiculo = post["tipo_veiculo"]
#checkin
carregado = post["carregado"]
#local
origem_descricao = post["origem_descricao"]
origem_latitude = post["origem_latitude"]
origem_longitude = post["origem_longitude"]
destino_descricao = post["destino_descricao"]
destino_latitude = post["destino_latitude"]
destino_longitude = post["destino_longitude"]
sql_local = """ INSERT INTO `local`
(`descricao`, `latitude`, `longitude`) VALUES (%s,%s,%s)"""
sql_motorista = """ INSERT INTO `motorista`
(`nome`, `idade`, `sexo`, `veiculo_proprio`,`tipo_veiculo`,`tipo_cnh`) VALUES (%s,%s,%s,%s,%s,%s)"""
sql_checkin = """ INSERT INTO `checkin`
(`data`, `codigo_origem`, `codigo_destino`, `carregado`,`codigo_motorista`) VALUES (%s,%s,%s,%s,%s)"""
id_origem = retorna_codigo_local(origem_descricao)
if (id_origem == False):
bd.query(sql_local,
(origem_descricao, origem_latitude, origem_longitude))
id_origem = cursor.lastrowid
id_destino = retorna_codigo_local(destino_descricao)
if (id_destino == False):
bd.query(sql_local,
(destino_descricao, destino_latitude, destino_longitude))
id_destino = cursor.lastrowid
id_motorista = retorna_motorista(nome)
if (id_motorista == False):
bd.query(sql_motorista,
(nome, idade, sexo, veiculo_proprio, tipo_veiculo, tipo_cnh))
id_motorista = cursor.lastrowid
bd.query(sql_checkin, (datetime.datetime.now(), id_origem, id_destino,
carregado, id_motorista))
id_checkin = cursor.lastrowid
bd.commit()
if (id_checkin):
return retornar('sucesso', 200)
else:
return retornar('erro', 400)
class StreamTwitter:
def __init__(self, log):
self.extract = Extract()
self.sql = Mysql("localhost", "root", "hello", "tweets", 3306)
# self.colors = Colors()
self.stream_url = self.extract.api['TwitterStream']
self.stream_rules_url = self.extract.api['TwitterStreamRules']
self.rules = self.extract.api['TwitterStreamRulesSettings']
self.auth = lambda x: {
"Authorization": "Bearer {}".format(x),
"Content-type": "application/json"
}
self.conn = 0
self.log = lambda log: (log == 1)
def connect_stream(self):
try:
access_token = os.environ.get('ACCESS_TOKEN_TWT')
if (self.conn == 0):
response = requests.get(self.stream_url,
headers=self.auth(access_token),
stream=True,
params={"format": "detailed"})
self.conn = 1
for line in response.iter_lines():
if (line):
json_dict = json.loads(line)
if (self.log):
pprint(json_dict)
self.load_twt_to_db(json_dict)
except Exception as err:
print(err)
response.close()
sys.exit()
else:
print("success !")
response.close()
def load_twt_to_db(self, tweet_dict):
try:
author_id = tweet_dict['data']['author_id']
created_at = tweet_dict['data']['created_at']
lang = tweet_dict['data']['lang']
possibly_sensitive = (
tweet_dict['data']['possibly_sensitive'] == 'True')
like_count = tweet_dict['data']['stats']['like_count']
retweet_count = tweet_dict['data']['stats']['retweet_count']
reply_count = tweet_dict['data']['stats']['reply_count']
quote_count = tweet_dict['data']['stats']['quote_count']
twt_body = tweet_dict['data']['text']
self.sql.db_add(author_id, created_at, lang, possibly_sensitive,
like_count, retweet_count, reply_count,
quote_count, twt_body)
except Exception as err:
print(err)
sys.exit()
def set_rules(self, rules):
payload = []
try:
access_token = os.environ.get('ACCESS_TOKEN_TWT')
'''
Evaluat Rule given if it's in our Rules list !
Examined file : 'data_sources.json'
Check if the given rule has some content*
* (content) : the keyword, hashtags, user's that we will match againts twitter and filter tweets upon !
'''
for rule in rules:
if rule not in self.rules:
raise ModuleNotFoundError('Rule type is uncorrect')
elif self.rules[rule] == None:
raise KeyError('Error : empty rules')
'''
After checking the rules given, we transform them to the correct format* for the api call.
* (format) : json format example : [{"value" : "cat has:media*"}, {"value" : "dog"}]
* (has:media) : optional parametrs check twitter operators !
'''
for rule in rules:
for key in self.rules[rule]:
if key:
payload.append({'value': key})
'''
manage all errors raised above.
'''
rules = dict({"add": payload})
response = requests.post(self.stream_rules_url,
headers=self.auth(access_token),
json=rules)
if response.status_code is not 201:
raise AssertionError(response.text)
except AssertionError as set_rule_error:
print(f"Rules were not setted")
print(f"Error : %s" % set_rule_error)
except KeyError as err:
print(err)
sys.exit()
except ValueError as v_err:
print(f"Error : %s." % v_err)
print(f"Update : '%s' file." % self.extract.data_source_file_name)
print(f"Section : %s." % self.stream_rules_url)
sys.exit()
except ModuleNotFoundError as err:
print(f"Error : %s" % err)
print("Try on of the followings keywords")
for rules in self.rules:
print("* " + rules + " *")
print("\nOR\n")
print(
f"Add your own rules in %s using the same format as in the file"
% self.extract.data_source_file_name)
sys.exit()
finally:
if (self.log):
for rule in rules:
print(rule, end='\n')
print('')
print("rules were added succefully")
def get_rules(self, log=0):
try:
access_token = os.environ.get('ACCESS_TOKEN_TWT')
response = requests.get(self.stream_rules_url,
headers=self.auth(access_token))
if (response.status_code is not 200):
raise Exception(response.text)
if (log == 1):
pprint(response.json())
except Exception as err:
print(err)
sys.exit()
finally:
if (log == 1):
pprint(response.json())
return (response.json())
def reset_rules(self):
rules = self.get_rules(0)
rules_ids = list()
if ('data' not in rules):
print('Empty set of rules.')
return
for rule in rules['data']:
rules_ids.append(rule['id'])
rules = dict({'delete': {"ids": rules_ids}})
try:
acces_token = os.environ['ACCESS_TOKEN_TWT']
response = requests.post(self.stream_rules_url,
headers=self.auth(acces_token),
json=rules)
print(response.text)
except Exception as err:
print(err)
sys.exit()
finally:
return (response.json())
def index():
projectid = request.query.get('projectid')
#参数要带projectId
#1.根据projectId，查询project_EMR, EMR_indicator, project_indicator表，生产web报表，并且异步写成xls
#2.导出按钮提供下载
#emrid, indicator1, indicator2, ...
try:
mysql = Mysql(db = DB,
host = MYSQL_HOST,
user = USER,
passwd = PASSWD)
mysql.connect()
sql = 'select emr_id from project_emr where project_id = %s' %(projectid,)
emrids = []
rows = mysql.executeNoConn(sql)
for row in rows:
emrids.append(row[0])
sql = 'select indicator_id from project_indicator where project_id = %s' %(projectid,)
idk_ids = []
rows = mysql.executeNoConn(sql)
for row in rows:
idk_ids.append(row[0])
fields = ['emrid']
for idk_id in idk_ids:
sql = "select name from indicator where indicator_id = %s" %(idk_id,)
rows = mysql.executeNoConn(sql)
name = rows[0][0]
fields.append(name)
values = []
for emrid in emrids:
value = [emrid]
for idk_id in idk_ids:
sql = "select value from emr_indicator where emr_id = %s and indicator_id = %s" %(emrid, idk_id)
rows = mysql.executeNoConn(sql)
value.append(rows[0][0])
values.append(value)
mysql.close()
except MySQLdb.Error, e:
print traceback.format_exc()
mysql.close()
def __init__(self, host, username, pwd, database, table=''):
self._table = table
self._conn = Mysql(host, username, pwd, database)
def index():
query = request.query
# name
proj_name = query.get('projectname')
if not proj_name:
print "error: proj_name has not been set"
proj_name = proj_name.strip()
if proj_name == "":
print "error: proj_name can not be empty string"
# detail
proj_detail = query.get('detail').strip()
if not proj_detail:
print "error: proj_detail has not been set"
proj_detail = proj_detail.strip()
if proj_detail == "":
print "error: proj_detail can not be empty string"
# owner
proj_owner = query.get('owner').strip()
if not proj_owner:
print "error: proj_owner has not been set"
proj_owner = proj_owner.strip()
if proj_owner == "":
print "error: proj_owner can not be empty string"
proj_id = -1
# save
try:
mysql = Mysql(db = DB,
host = MYSQL_HOST,
user = USER,
passwd = PASSWD)
mysql.connect()
# check if duplicated
sql = "select project_id from project where name = \'%s\' and detail = \'%s\' and owner = \'%s\'" %(proj_name, proj_detail, proj_owner)
print sql
res = mysql.executeNoConn(sql)
if len(res) > 0:
print "error: project exists, please modify!"
# insert and return project_id
sql = "insert into project(name, detail, owner) values('%s', '%s', '%s')" %(proj_name, proj_detail, proj_owner)
mysql.executeNoConn(sql)
sql = "select project_id from project where name = '%s' and detail = '%s' and owner = '%s'" %(proj_name, proj_detail, proj_owner)
res = mysql.executeNoConn(sql)
proj_id = res[0][0]
mysql.close()
except MySQLdb.Error, e:
print traceback.format_exc()
mysql.close()
def offline_client(sock):
print "-------[IP:%s Port:%s] disconnected-------" % addr
sock.close()
conn_list.remove(sock)
if __name__ == "__main__":
conn_list = []
server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
server_socket.bind(("0.0.0.0", PORT))
server_socket.listen(MAX_CLIENT)
conn_list.append(server_socket)
print SERVER_START_INFO
print "TCP server starts at Port " + str(PORT)
mysql_op = Mysql()
mysql_op.conn()
exchange = Exchange()
acc = Account()
while True:
read_sockets, write_sockets, error_sockets = select.select(
conn_list, [], [])
for sock in read_sockets:
#New connection
if sock == server_socket:
sockfd, addr = server_socket.accept()
conn_list.append(sockfd)
print "-------[IP:%s Port:%s] connected-------" % addr
sockfd.send("Connected\n")
def saveData(target, info):
comments = info.pop('comments')
mysql = Mysql()
if (target.get('check_repeat_type') == 'url&title'):
checkRepeat_title = mysql.getAll(
"SELECT id from tb_article where title='" + info.get('title') +
"'")
if len(checkRepeat_title) > 0:
mysql.dispose()
return False
log("SegmentedWords start:")
words_title = segmentedWords(info.get('title'))
words_content = segmentedWords(info.get('content_text'))
words = list(set(words_title + words_content))
log("SegmentedWords end;")
insert_set_str = "set emotion_type = " + str(emotionType(words)) + ","
info['content'] = unicode(info.get('content').encode("utf-8"))
for k, v in info.items():
insert_set_str = insert_set_str + " `" + str(
k) + "`= '%s'," % (MySQLdb.escape_string(str(v).encode('utf-8')), )
log("start:" + str(time.time()))
# 1==1:
try:
print insert_set_str
article_id = mysql.insertOne("insert into tb_article " +
str(insert_set_str[:-1]))
article_id = str(article_id)
# 评论插入
if len(comments) > 0:
for c_i in xrange(len(comments)):
insert_comment_str = "set article_id = " + article_id + ","
for c_k, c_v in comments[c_i].items():
insert_comment_str = insert_comment_str + " `" + c_k + "`='%s'," % (
MySQLdb.escape_string(str(c_v).encode('utf-8')), )
mysql.insertOne("insert into tb_article_comment " +
insert_comment_str[:-1])
# 分词插入
for i in xrange(len(words)):
if words[i] != '':
findwords = mysql.getOne(
"SELECT * from tb_words where w_name='%s'" %
(MySQLdb.escape_string(words[i]), ))
if len(findwords) > 0:
w_a_ids_str = findwords.get("w_a_ids")
w_a_ids = w_a_ids_str.split(',')
w_a_ids.append(article_id)
w_a_ids_str = ','.join(w_a_ids)
mysql.update("update tb_words set `w_a_ids`='" +
MySQLdb.escape_string(w_a_ids_str) +
"',`w_num`=" + str(len(w_a_ids)) +
" where `w_id`=" + str(findwords.get('w_id')))
else:
w_a_ids = []
w_a_ids.append(article_id)
w_a_ids_str = ','.join(w_a_ids)
w_id = mysql.insertOne(
"insert into tb_words set `w_name`='" +
MySQLdb.escape_string(words[i]) + "',`w_a_ids`=" +
w_a_ids_str + ",`w_num`=" + str(len(w_a_ids)))
mysql.end('commit')
except Exception as e:
log("[ERROR]saveData is error:")
log(e)
mysql.end()
log("end:" + str(time.time()))
mysql.dispose()
# 运行事件循环， loop以参数的形式传递进来运行
asyncio.set_event_loop(loop)
loop.run_forever()
def save_account(username, pwd, cookies):
insert_sql = """
insert into user_info(USERNAME, PASSWORD, COOKIES, STATUS, USED, LOCKED, TYPE)
VALUES (%s, %s, %s, %s, %s, %s, %s)
"""
id = mysql.insertOne(insert_sql, (username, pwd, cookies, 1, 0, 0, 2))
print(id)
loop = ''
mysql = Mysql()
# mysql连接池
pool = ''
if __name__ == '__main__':
# if platform.system() == "Linux":
# display = Display(visible=0, size=(800, 600))
# display.start()
# thread_loop = asyncio.new_event_loop() # 获取一个事件循环
# run_loop_thread = threading.Thread(target=start_loop, args=(thread_loop,)) # 将次事件循环运行在一个线程中，防止阻塞当前主线程
# run_loop_thread.start() # 运行线程，同时协程事件循环也会运行
loop = asyncio.get_event_loop()
for i in range(1):
loop.run_until_complete(login('15143554689', '89f6010c1337'))
class Database(object):
def __init__(self, host, username, pwd, database, table=''):
self._table = table
self._conn = Mysql(host, username, pwd, database)
def insert(self, table, *args, **kwargs):
return self._conn.insert(table, *args, **kwargs)
def delete(self, table, where=None, *args):
return self._conn.delete(table, where, *args)
def select(self, table, where=None, *args, **kwargs):
return self._conn.select(table, where, *args, **kwargs)
def update_ports(self, sw_name, act, stand):
logging.info("Updating ActivePort=%s and StandbyPort=%s for switch=%s",
act, stand, sw_name)
self._conn.update(self._table,
'switch_name=%s',
sw_name,
active_port=act,
standby_port=stand)
def update_power_status(self, sw_name, power):
logging.info("Updating is_powered_on as %s for switch=%s", power,
sw_name)
self._conn.update(self._table,
'switch_name=%s',
sw_name,
is_powered_on=power)
def update_images(self, sw_name, kick, sys):
logging.info(
"Updating kickstart=%s and system=%s images for switch=%s", kick,
sys, sw_name)
self._conn.update(self._table,
'switch_name=%s',
sw_name,
kickstart=kick,
system=sys)
def update_start_end_time(self, sw_name, start, end):
self._conn.update(self._table,
'switch_name=%s',
sw_name,
start_time=start,
end_time=end)
def increment_end_time(self, switch_dict):
logging.info(
"Incrementing end time based on weekday/weekend for switch=%s",
str(switch_dict['switch_name']))
dt = util.get_next_end_time(switch_dict)
sw_name = str(switch_dict['switch_name'])
self._conn.update(self._table, 'switch_name=%s', sw_name, end_time=dt)
def increment_start_time(self, switch_dict):
logging.info(
"Incrementing start timei based on weekday/weekend for switch=%s",
str(switch_dict['switch_name']))
dt = util.get_next_start_time(switch_dict)
sw_name = str(switch_dict['switch_name'])
self._conn.update(self._table,
'switch_name=%s',
sw_name,
start_time=dt)
def update_is_sanity_activated(self, sw_name, status):
logging.info("Updating is_sanity_activated as %s for switch=%s",
status, sw_name)
self._conn.update(self._table,
'switch_name=%s',
sw_name,
is_sanity_activated=status)
def __init__(self, host, username, pwd, database):
self._conn = Mysql(host, username, pwd, database)
level=logging.NOTSET,
format=
'%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s',
datefmt='%a, %d %b %Y %H:%M:%S',
filename='records.log'
# filemode='w'
)
console = logging.StreamHandler()
console.setLevel(logging.INFO)
formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s')
console.setFormatter(formatter)
logging.getLogger('').addHandler(console)
app = Flask(__name__)
db = Mysql()
@app.route('/r', methods=['POST'])
def record():
try:
ip = request.headers['X-Forwarded-For']
url = 'http://ip.taobao.com/service/getIpInfo.php?ip=%s' % (ip)
urlobject = urllib2.urlopen(url)
urlcontent = urlobject.read()
res = json.loads(urlcontent)
city = res['data']['city']
choices = request.form['choices']
user_type = request.form['user_type']
user_interest = request.form['user_interest']
user_name = request.form['user_name']
if opcion == "1": #Calcular Tempratura y humedad
#CALCULANDO TEMPERATURA Y HUMEDAD
Sensor = Sensores()
data = {
'sensor': "TEMPERATURA_HUMEDA",
'tipo': "DHT11",
'pin': 4
}
print("Calculando Temperatura y Humedad...")
response = Sensor.sensor(data)
print(data['message'])
#CALCULANDO TEMPERATURA Y HUMEDAD
print("Guardando datos en...")
#GUARDANDO DATOS EN MYSQL
mysql = Mysql('localhost', 'administrador', 'admin',
'sensores_python')
sql = 'INSERT INTO valores (sensor,temperatura, humedad, fecha) VALUES (' + 1 + ',' + data[
'message'].temperatura + ', ' + data[
'message'].humedad + ',' + datetime.now() + ');'
mysql.commit(sql)
mysql.close()
#GUARDANDO DATOS EN MYSQL
print("Mysql...")
#GUARDANDO DATOS EN MONGODB
#CARLOS
mongo = Database2('localhost', 'Sensores',
'Temperatura_Humedad', 27017)
db = mongo.Temperatura_Humedad
collection = db.Sensores
collection.insert_one({"Temperatura", "Humedad"})