def getMysqlConnect():
'''
获取mysql 的连接信息
:return: 与mysql 的连接
'''
try:
configure = getConfig()
host = configure['mysql']['host']
username = configure['mysql']['username']
password = configure['mysql']['password']
database = configure['mysql']['database']
port = configure.getint('mysql', 'port')
charset = configure['mysql']['charset']
# connect_timeout = configure['mysql'].getint('connect_timeout') or 100
conn = pymysql.connect(host=host,
user=username,
passwd=password,
db=database,
port=port,
charset=charset,
autocommit=False,
cursorclass=pymysql.cursors.Cursor)
except:
logger.error('connect mysql failure', exc_info=1)
return conn
def getSpark():
global_config = getConfig()
spark_submit_str = ('--driver-memory 40g --executor-memory 3g --packages org.apache.spark:spark-sql_2.11:2.4.0,org.apache.bahir:spark-sql-cloudant_2.11:2.3.2,com.datastax.spark:spark-cassandra-connector_2.11:2.4.0'
' --driver-class-path ./jars/mysql-connector-java-8.0.16.jar'
' --jars ./jars/mysql-connector-java-8.0.16.jar'
' pyspark-shell')
os.environ['PYSPARK_SUBMIT_ARGS'] = spark_submit_str
spark = SparkSession\
.builder\
.config('spark.sql.repl.eagerEval.enabled', True)\
.config('cloudant.host', global_config['couch']['host'])\
.config('cloudant.username', global_config['couch']['username'])\
.config('cloudant.password', global_config['couch']['password'])\
.config('cloudant.protocol', global_config['couch']['protocol'])\
.config("jsonstore.rdd.partitions", 15000)\
.config('spark.driver.maxResultSize', "15000M")\
.config('spark.sql.crossJoin.enabled', True)\
.config('spark.sql.autoBroadcastJoinThreshold', 0)\
.config("spark.cassandra.connection.host", global_config['cassandra']['host'])\
.config("spark.cassandra.auth.username", global_config['cassandra']['username'])\
.config("spark.cassandra.auth.password", global_config['cassandra']['password'])\
.config("spark.cassandra.output.consistency.level", "ANY")\
.config("spark.sql.shuffle.partitions", 1000)\
.getOrCreate()
spark.sparkContext.setLogLevel("INFO")
return spark
def getOpt(model):
def opt():
pass
if not model in mode_switch:
return
opt.model = mode_switch[model][0]
opt.modelDef = mode_switch[model][1]
opt.ramCoef = mode_switch[model][2][config.getRunType()]
opt.cropsize = config.getConfig()[1 if model[:4] == 'lite' else 2]
opt.modelCached = initModel(opt, opt.model, 'DN' + model)
return opt
def getMongoConn():
'''
获取mongodb client 的连接
:return: client
'''
configure = getConfig()
host = configure['mongodb']['host']
username = configure['mongodb']['username']
password = configure['mongodb']['password']
database = configure['mongodb']['database']
port = configure.getint('mongodb', 'port')
client = pymongo.MongoClient(host= host ,port = port,tz_aware = True,maxPoolSize= 10,maxIdleTimeMS = 20000,connectTimeoutMS=20000)
return client[database]
def getOpt(model):
def opt():
pass
if not model in mode_switch:
return
opt.model = mode_switch[model][0]
opt.modelDef = mode_switch[model][1]
opt.ramCoef = mode_switch[model][2][config.getRunType()]
opt.cropsize = config.getConfig()[1 if model[:4] == 'lite' else 2]
opt.modelCached = initModel(getModel(opt),
load(opt.model, map_location='cpu'))
return opt
def getOpt(optDN):
model = optDN['model']
if not model in mode_switch:
return
opt = Option(mode_switch[model][0])
opt.modelDef = mode_switch[model][1]
opt.ramCoef = mode_switch[model][2][config.getRunType()]
opt.cropsize = config.getConfig()[1 if model[:4] == 'lite' else 2]
opt.modelCached = initModel(opt, opt.model, 'DN' + model)
opt.squeeze = lambda x: x.squeeze(1)
opt.unsqueeze = lambda x: x.unsqueeze(1)
opt.padding = 5
return opt
def getOpt(model):
def opt():
pass
if not model in model_dict:
return
opt.model = model_dict[model]
modelType = 0 if model[:4] == 'lite' else 1
opt.ramCoef = ramCoef[config.getRunType() * 2 + modelType]
opt.cropsize = config.getConfig()[modelType + 1]
if opt.cropsize:
print('当前denoise切块大小：', opt.cropsize)
opt.modelCached = getModel(opt, False)
return opt
def read_from_mysql(self, query, db_name, config):
global_config = getConfig()
return self.get_spark().read.format("jdbc").\
option("url", "jdbc:mysql://"+global_config['mysql']['host']+":" + global_config['mysql']['port']+ "/" + db_name + "?zeroDateTimeBehavior=convertToNull").\
option("useUnicode", "true").\
option("continueBatchOnError","true").\
option("useSSL", "false").\
option("user", global_config['mysql']['username']).\
option("password", global_config['mysql']['password']).\
option("dbtable",query).\
option("partitionColumn", config['partitionColumn']).\
option("fetchSize", config['fetchsize']).\
option("lowerBound", config['lowerBound']).\
option("upperBound", config['upperBound']).\
option("numPartitions", config['numPartitions']).\
load()
def start(self):
dataConfig = getConfig().get("DataHandler", "DataClass")
data = DataSourceFactory.getDataHandler(dataConfig)
currRow, prevRow = data.getData()
currRow = currRow['50dayEWM']
prevRow = prevRow['50dayEWM']
if (currRow > prevRow):
return "Buy"
elif (currRow < prevRow):
return "Sell"
else:
return "Nothing"
def get_pool(self):
if self.pool is None:
print("create db pool == dw_new")
dev = "dev." if str(config.getConfig('public',
'dev')) == "true" else ""
pool = PooledDB(
creator=pymysql,
host=config.getConfig(dev + 'database.dw_new', 'host'),
port=int(config.getConfig(dev + 'database.dw_new', 'port')),
user=config.getConfig(dev + 'database.dw_new', 'user'),
passwd=config.getConfig(dev + 'database.dw_new', 'password'),
charset=config.getConfig(dev + 'database.dw_new', 'charset'),
db=config.getConfig(dev + 'database.dw_new', 'db'))
self.pool = pool
return self.pool
def getOpt(scale, mode, ensemble):
def opt():pass
nmode = mode+str(scale)
if not nmode in mode_switch:
return
opt.C2B = mode[:3] != 'gan'
opt.mode = mode
opt.model = mode_switch[nmode][0]
opt.modelDef = mode_switch[nmode][1]
opt.scale = scale
opt.ensemble = ensemble
opt.ramCoef = mode_switch[nmode][2][config.getRunType()]
opt.cropsize = config.getConfig()[0]
if opt.cropsize:
print('当前SR切块大小：', opt.cropsize)
opt.modelCached = initModel(getModel(opt), load(opt.model, map_location='cpu'))
return opt
def getOpt(scale, mode, ensemble):
def opt():
pass
nmode = mode + str(scale)
if not nmode in mode_switch:
return
opt.C2B = mode[:3] != 'gan'
opt.mode = mode
opt.model = mode_switch[nmode][0]
opt.modelDef = mode_switch[nmode][1]
opt.scale = scale
opt.ensemble = ensemble
opt.ramCoef = mode_switch[nmode][2][config.getRunType()]
opt.cropsize = config.getConfig()[0]
opt.modelCached = initModel(opt, opt.model, 'SR' + nmode)
return opt
def getOpt(optSR):
opt = Option()
opt.mode = optSR['model']
opt.scale = optSR['scale']
nmode = opt.mode + str(opt.scale)
if not nmode in mode_switch:
return
if opt.mode[:3] != 'gan':
opt.squeeze = lambda x: x.squeeze(1)
opt.unsqueeze = lambda x: x.unsqueeze(1)
opt.padding = 9 if opt.scale == 3 else 5
opt.model = mode_switch[nmode][0]
opt.modelDef = mode_switch[nmode][1]
opt.ensemble = optSR['ensemble'] if 'ensemble' in optSR and (
0 <= optSR['ensemble'] <= 7) else config.ensembleSR
opt.ramCoef = mode_switch[nmode][2][config.getRunType()]
opt.cropsize = config.getConfig()[0]
opt.modelCached = initModel(opt, opt.model, 'SR' + nmode)
return opt
def getOpt(scale, mode, ensemble):
def opt():
pass
nmode = mode + str(scale)
if not nmode in mode_switch:
return
opt.C2B = mode[:3] != 'gan'
opt.mode = mode
opt.model = mode_switch[nmode]
opt.scale = scale
opt.ensemble = ensemble
modelType = (scale - 2) * 3
if mode[:3] == 'gan':
modelType += 9
opt.ramCoef = ramCoef[config.getRunType() + modelType]
opt.cropsize = config.getConfig()[0]
if opt.cropsize:
print('当前SR切块大小：', opt.cropsize)
opt.modelCached = getModel(opt, False)
return opt
def start_job(self, spark, ssc):
# start kafka stream
config = getConfig()['kafka']
kafka_stream = KafkaStream.create_source(ssc, config['config'],
config['topics'])
# filter streams
new_encounters = get_encounter_stream(kafka_stream)
new_obs = get_new_obs_stream(kafka_stream)
updated_obs = get_updated_obs_stream(kafka_stream)
# save encounters to parquet
save_stream_to_parquet(new_encounters,
path="spark-warehouse/encounters")
#transform new obs
# to do:
# 1) add code to deal with updated obs (voided)
# 2) add code to deal with new obs but existing encounters
# 3) add code to deal with obs group
self.transform_data(spark, new_obs)
# -*- coding:utf8 -*-
#__author__ = "neo"
from config.config import getConfig
import pymysql
host = getConfig('mysql', 'host')
port = getConfig('mysql', 'port')
user = getConfig('mysql', 'username')
passwd = getConfig('mysql', 'password')
db = getConfig('mysql', 'db')
def query(sql, data=None):
conn = pymysql.connect(host=host,
port=int(port),
user=user,
passwd=passwd,
db=db,
charset='utf8')
cursor = conn.cursor()
cursor.execute(sql, data)
result = cursor.fetchone()
conn.commit()
cursor.close()
conn.close()
return result
wordcloud = WordCloud(font_path='../resources/simsun.ttf').generate(
" ".join(text))
import matplotlib.pyplot as plt
plt.imshow(wordcloud, interpolation='bilinear')
plt.imshow(wordcloud)
plt.axis("off")
plt.show()
if __name__ == '__main__':
env = 'dev'
if len(sys.argv) == 2:
env = sys.argv[1]
configure = updateConfig('env', 'env_active', env)
configure = getConfig()
# 加载词库到默认路径
# loadDicDataToFile()
## extract_tag:
# extract_tag()
## tf-idf
# tfidf_keywords()
#cloud
# wordCould()
##get data
getData()
#-*- coding:utf8 -*-
import unittest
from login import get_session
import requests
from config.config import getConfig
from command.optionexcel import OperationExcel
headers = eval(getConfig('header', 'headers'))
json_header = eval(getConfig('header', 'json_header'))
data_no_permission = {
'username': '******',
'password': '******',
'rememberMe': False
} #无权限用户
login_admin = {
'username': '******',
'password': '******',
'rememberMe': False
} #超级管理员
dict_add_api = getConfig('swagger', 'baseurl') + ':' + getConfig(
'swagger', 'port') + OperationExcel(u'字典管理').get_value(4, 1) #添加字典
session_api = getConfig('swagger', 'baseurl') + ':' + getConfig(
'swagger', 'port') + OperationExcel(u'会话管理').get_value(12, 1) #查询在线用户列表
delete_api = getConfig('swagger', 'baseurl') + ':' + getConfig(
'swagger', 'port') + OperationExcel(u'会话管理').get_value(4, 1) #踢出用户
dict_delete_api = getConfig('swagger', 'baseurl') + ':' + getConfig(
'swagger', 'port') + OperationExcel(u'字典管理').get_value(13, 1) #删除字典
captcha_api = getConfig('swagger', 'baseurl') + ':' + getConfig(
'swagger', 'port') + OperationExcel(u'其他组件').get_value(4, 1) #获取验证码
dicts_api = getConfig('swagger', 'baseurl') + ':' + getConfig(
def getBroker():
type = getConfig().get("Broker", "BrokerClass")
if (type == "LiveBroker"):
return IBapi()
'''
处理与mysql相关的信息
'''
#coding:utf-8
import sys
sys.path.append("..")
from config.config import getConfig
import redis
config = getConfig()
pool = redis.ConnectionPool(
host=config['redis']['host'],
port=config.getint('redis', 'port'),
password=config['redis']['password'],
db=config.getint('redis', 'database'),
decode_responses=True) # host是redis主机，需要redis服务端和客户端都起着 redis默认端口是6379
def getRedisConn(key):
r = redis.Redis(connection_pool=pool)
return r.get(key)
def getKey(key):
r = redis.Redis(connection_pool=pool)
result = r.get(key)
return result
def apiInvoke():
'''
api 调用该接口，执行评论提取流程
:return: 成功：true 失败：false
'''
# stopWords = set(readFile(stop_file))
response = {}
start = datetime.now()
try:
if request.method == 'GET':
strDate = request.args.get('date', None)
else:
strDate = request.get_json()['date']
env = 'dev'
if len(sys.argv) == 2:
env = sys.argv[1]
updateConfig('env', 'env_active', env)
configure = getConfig()
clearDbData()
poolSize = configure.getint('pool', 'poolSize') ## 线程池大小
collections = ["Mall_Comment_Info",
"Shop_Comment_Info"] ## mongodb 待分析的表
## 每次处理的数量
batchSize = configure.getint('pool', 'batchSize')
batchSize = 100 if batchSize is None else batchSize
shopIds = findAllShopId()
mallIds = findAllMallId()
for collection in collections:
ids = shopIds if collection == 'Shop_Comment_Info' else mallIds
## 每次创建 POOL_SIZE 个线程 ，一共创建size/POOL_SIZE 个
t1 = datetime.now()
pool = multiprocessing.Pool(processes=poolSize) # 创建POOL_SIZE个进程
i = 0
while i < len(ids):
logger.info("i = {0} ".format(i))
if (i + batchSize) >= len(ids):
logger.info("最后一批数据，索引范围：{0}-{1}".format(i, len(ids)))
subset = ids[i:]
else:
subset = ids[i:i + batchSize]
# processMongoData(collection,subset, strDate)
pool.apply_async(processMongoData,
args=(
collection,
subset,
strDate,
))
i += batchSize
#
pool.close() ## 关闭进程
pool.join() ## 等待所有进程执行完成
t2 = datetime.now()
logger.info("处理{0}数据耗时{1}s".format(collection, (t2 - t1).seconds))
end = datetime.now()
logger.info("spend time:{0}s".format((end - start)))
response["code"] = 200
response["msg"] = "执行成功。耗时{0}s".format((end - start).seconds)
except:
end = datetime.now()
logger.error("exception now", exc_info=1)
response["code"] = 501
response["msg"] = "程序异常。耗时{0}s".format((end - start).seconds)
# stopWords = None ## 清空，防止内存不释放
return json.dumps(response)
def start(self):
config = getConfig()['spark']
SparkApp.start_spark(app_name=config['app_name'],
master=config['master'],
spark_config=config['configs'],
callback=self.start_job)
class LogFormatter(tornado.log.LogFormatter):
def __init__(self):
super(LogFormatter, self).__init__(
fmt=
'%(color)s[%(asctime)s %(filename)s:%(funcName)s:%(lineno)d %(levelname)s]%(end_color)s %(message)s',
datefmt='%Y-%m-%d %H:%M:%S')
db1 = DB_dw_new()
db1.get_pool()
#数据预加载
sysInitData = SysInitData()
sysInitData.init_load()
if __name__ == "__main__":
dev = "dev." if str(config.getConfig('public', 'dev')) == "true" else ""
application = tornado.web.Application(
handlers=[
# (r"/deploy/init2", InitHandler2),
(r"/deploy/publish2", OuterHandler2),
(r"/deploy/publishall2", OuterAllHandler2),
(r"/deploy/find2", FindVersionHandler2),
(r"/deploy/findPath2", FindPathHandler2)
],
debug=True
if str(config.getConfig(dev + 'web', 'debug')) == "true" else False)
# 日志设置
tornado.options.parse_command_line()
[i.setFormatter(LogFormatter()) for i in logging.getLogger().handlers]
application.listen(int(config.getConfig(dev + 'web', 'port')))
def main():
'''
This is the main Function which will be responsible for running all the functions for the version 1 , console application
:return:
'''
path = str(config.getConfig()['MongoDb']['URL'])
print("Connecting to Database")
con = connection.Mongo(path)
n = Notes.Notes("", "")
data = con.FindAll(n.tablename)
if data:
for i in data:
allNotes.append(
Notes.Notes(i['name'], i['data'], i['_id'], i['created_at']))
print(allNotes[-1])
while True:
os.system("clear")
try:
print("Welcome To Notes Application , version 1.0.0")
print(
"1.Add a Note\n2.Find a note\n3.Delete a Note\n4.See all Notes\n5.exit"
)
try:
n = int(input("Enter Your Choice >>"))
except ValueError:
continue
if n == 5:
raise KeyboardInterrupt
elif n == 1:
node = add()
id = con.Insert(node.tablename, node.retData())
node.setId(id)
allNotes.append(node)
print("Added")
time.sleep(2)
elif n == 2:
result = find()
if result['status'] == 'OK':
for i in result['data']:
print(i)
else:
name = result['name']
node = Notes.Notes(name, "")
data = con.Find(node.tablename, node.name)
if data:
for i in data:
allNotes.append(
Notes.Notes(i['name'], i['data'], i['_id'],
i['created_at']))
print(allNotes[-1])
input()
elif n == 3:
arr = delete()
if not arr:
continue
if len(arr) <= 0:
print("Enter The Correct Indexes again please !!")
time.sleep(1.2)
continue
n = Notes.Notes("", "")
count = con.Delete(n.tablename, arr)
print("Delete Count: {}".format(count))
input()
elif n == 4:
if not findAll():
n = Notes.Notes("", "")
data = con.FindAll(n.tablename)
if data:
for i in data:
allNotes.append(
Notes.Notes(i['name'], i['data'], i['_id'],
i['created_at']))
print(allNotes[-1])
pass
input()
else:
continue
except Exception as E:
print("Exception has occured {} \n".format(E))
raise KeyboardInterrupt
except KeyboardInterrupt:
print("quitting...")
exit(1)
##
# Win!
# Logging
logLocation = config.getConfig('logLocation')
logger = logging.getLogger('docker-aws-python')
hdlr = logging.FileHandler(logLocation)
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
hdlr.setFormatter(formatter)
logger.addHandler(hdlr)
logger.setLevel(logging.DEBUG)
# #
# Records
# a = records.getAAlias()
# c = records.getCAlias()
# ip = records.getPublicIp()
# #
#-*- coding:utf8 -*-
import requests
from config.config import getConfig
from command.optionexcel import *
headers = getConfig('header', 'headers')
data = {'username': '******', 'password': '******', 'rememberMe': False}
login_api = getConfig('swagger', 'baseurl') + ':' + getConfig(
'swagger', 'port') + OperationExcel(u'用户个人管理').get_value(4, 1)
def get_session(data=data):
login_session = requests.Session()
login_session.post(login_api, headers=eval(headers), data=data)
return login_session
