def getMysqlConnect(): ''' 获取mysql 的连接信息 :return: 与mysql 的连接 ''' try: configure = getConfig() host = configure['mysql']['host'] username = configure['mysql']['username'] password = configure['mysql']['password'] database = configure['mysql']['database'] port = configure.getint('mysql', 'port') charset = configure['mysql']['charset'] # connect_timeout = configure['mysql'].getint('connect_timeout') or 100 conn = pymysql.connect(host=host, user=username, passwd=password, db=database, port=port, charset=charset, autocommit=False, cursorclass=pymysql.cursors.Cursor) except: logger.error('connect mysql failure', exc_info=1) return conn
def getSpark(): global_config = getConfig() spark_submit_str = ('--driver-memory 40g --executor-memory 3g --packages org.apache.spark:spark-sql_2.11:2.4.0,org.apache.bahir:spark-sql-cloudant_2.11:2.3.2,com.datastax.spark:spark-cassandra-connector_2.11:2.4.0' ' --driver-class-path ./jars/mysql-connector-java-8.0.16.jar' ' --jars ./jars/mysql-connector-java-8.0.16.jar' ' pyspark-shell') os.environ['PYSPARK_SUBMIT_ARGS'] = spark_submit_str spark = SparkSession\ .builder\ .config('spark.sql.repl.eagerEval.enabled', True)\ .config('cloudant.host', global_config['couch']['host'])\ .config('cloudant.username', global_config['couch']['username'])\ .config('cloudant.password', global_config['couch']['password'])\ .config('cloudant.protocol', global_config['couch']['protocol'])\ .config("jsonstore.rdd.partitions", 15000)\ .config('spark.driver.maxResultSize', "15000M")\ .config('spark.sql.crossJoin.enabled', True)\ .config('spark.sql.autoBroadcastJoinThreshold', 0)\ .config("spark.cassandra.connection.host", global_config['cassandra']['host'])\ .config("spark.cassandra.auth.username", global_config['cassandra']['username'])\ .config("spark.cassandra.auth.password", global_config['cassandra']['password'])\ .config("spark.cassandra.output.consistency.level", "ANY")\ .config("spark.sql.shuffle.partitions", 1000)\ .getOrCreate() spark.sparkContext.setLogLevel("INFO") return spark
def getOpt(model): def opt(): pass if not model in mode_switch: return opt.model = mode_switch[model][0] opt.modelDef = mode_switch[model][1] opt.ramCoef = mode_switch[model][2][config.getRunType()] opt.cropsize = config.getConfig()[1 if model[:4] == 'lite' else 2] opt.modelCached = initModel(opt, opt.model, 'DN' + model) return opt
def getMongoConn(): ''' 获取mongodb client 的连接 :return: client ''' configure = getConfig() host = configure['mongodb']['host'] username = configure['mongodb']['username'] password = configure['mongodb']['password'] database = configure['mongodb']['database'] port = configure.getint('mongodb', 'port') client = pymongo.MongoClient(host= host ,port = port,tz_aware = True,maxPoolSize= 10,maxIdleTimeMS = 20000,connectTimeoutMS=20000) return client[database]
def getOpt(model): def opt(): pass if not model in mode_switch: return opt.model = mode_switch[model][0] opt.modelDef = mode_switch[model][1] opt.ramCoef = mode_switch[model][2][config.getRunType()] opt.cropsize = config.getConfig()[1 if model[:4] == 'lite' else 2] opt.modelCached = initModel(getModel(opt), load(opt.model, map_location='cpu')) return opt
def getOpt(optDN): model = optDN['model'] if not model in mode_switch: return opt = Option(mode_switch[model][0]) opt.modelDef = mode_switch[model][1] opt.ramCoef = mode_switch[model][2][config.getRunType()] opt.cropsize = config.getConfig()[1 if model[:4] == 'lite' else 2] opt.modelCached = initModel(opt, opt.model, 'DN' + model) opt.squeeze = lambda x: x.squeeze(1) opt.unsqueeze = lambda x: x.unsqueeze(1) opt.padding = 5 return opt
def getOpt(model): def opt(): pass if not model in model_dict: return opt.model = model_dict[model] modelType = 0 if model[:4] == 'lite' else 1 opt.ramCoef = ramCoef[config.getRunType() * 2 + modelType] opt.cropsize = config.getConfig()[modelType + 1] if opt.cropsize: print('当前denoise切块大小:', opt.cropsize) opt.modelCached = getModel(opt, False) return opt
def read_from_mysql(self, query, db_name, config): global_config = getConfig() return self.get_spark().read.format("jdbc").\ option("url", "jdbc:mysql://"+global_config['mysql']['host']+":" + global_config['mysql']['port']+ "/" + db_name + "?zeroDateTimeBehavior=convertToNull").\ option("useUnicode", "true").\ option("continueBatchOnError","true").\ option("useSSL", "false").\ option("user", global_config['mysql']['username']).\ option("password", global_config['mysql']['password']).\ option("dbtable",query).\ option("partitionColumn", config['partitionColumn']).\ option("fetchSize", config['fetchsize']).\ option("lowerBound", config['lowerBound']).\ option("upperBound", config['upperBound']).\ option("numPartitions", config['numPartitions']).\ load()
def start(self): dataConfig = getConfig().get("DataHandler", "DataClass") data = DataSourceFactory.getDataHandler(dataConfig) currRow, prevRow = data.getData() currRow = currRow['50dayEWM'] prevRow = prevRow['50dayEWM'] if (currRow > prevRow): return "Buy" elif (currRow < prevRow): return "Sell" else: return "Nothing"
def get_pool(self): if self.pool is None: print("create db pool == dw_new") dev = "dev." if str(config.getConfig('public', 'dev')) == "true" else "" pool = PooledDB( creator=pymysql, host=config.getConfig(dev + 'database.dw_new', 'host'), port=int(config.getConfig(dev + 'database.dw_new', 'port')), user=config.getConfig(dev + 'database.dw_new', 'user'), passwd=config.getConfig(dev + 'database.dw_new', 'password'), charset=config.getConfig(dev + 'database.dw_new', 'charset'), db=config.getConfig(dev + 'database.dw_new', 'db')) self.pool = pool return self.pool
def getOpt(scale, mode, ensemble): def opt():pass nmode = mode+str(scale) if not nmode in mode_switch: return opt.C2B = mode[:3] != 'gan' opt.mode = mode opt.model = mode_switch[nmode][0] opt.modelDef = mode_switch[nmode][1] opt.scale = scale opt.ensemble = ensemble opt.ramCoef = mode_switch[nmode][2][config.getRunType()] opt.cropsize = config.getConfig()[0] if opt.cropsize: print('当前SR切块大小:', opt.cropsize) opt.modelCached = initModel(getModel(opt), load(opt.model, map_location='cpu')) return opt
def getOpt(scale, mode, ensemble): def opt(): pass nmode = mode + str(scale) if not nmode in mode_switch: return opt.C2B = mode[:3] != 'gan' opt.mode = mode opt.model = mode_switch[nmode][0] opt.modelDef = mode_switch[nmode][1] opt.scale = scale opt.ensemble = ensemble opt.ramCoef = mode_switch[nmode][2][config.getRunType()] opt.cropsize = config.getConfig()[0] opt.modelCached = initModel(opt, opt.model, 'SR' + nmode) return opt
def getOpt(optSR): opt = Option() opt.mode = optSR['model'] opt.scale = optSR['scale'] nmode = opt.mode + str(opt.scale) if not nmode in mode_switch: return if opt.mode[:3] != 'gan': opt.squeeze = lambda x: x.squeeze(1) opt.unsqueeze = lambda x: x.unsqueeze(1) opt.padding = 9 if opt.scale == 3 else 5 opt.model = mode_switch[nmode][0] opt.modelDef = mode_switch[nmode][1] opt.ensemble = optSR['ensemble'] if 'ensemble' in optSR and ( 0 <= optSR['ensemble'] <= 7) else config.ensembleSR opt.ramCoef = mode_switch[nmode][2][config.getRunType()] opt.cropsize = config.getConfig()[0] opt.modelCached = initModel(opt, opt.model, 'SR' + nmode) return opt
def getOpt(scale, mode, ensemble): def opt(): pass nmode = mode + str(scale) if not nmode in mode_switch: return opt.C2B = mode[:3] != 'gan' opt.mode = mode opt.model = mode_switch[nmode] opt.scale = scale opt.ensemble = ensemble modelType = (scale - 2) * 3 if mode[:3] == 'gan': modelType += 9 opt.ramCoef = ramCoef[config.getRunType() + modelType] opt.cropsize = config.getConfig()[0] if opt.cropsize: print('当前SR切块大小:', opt.cropsize) opt.modelCached = getModel(opt, False) return opt
def start_job(self, spark, ssc): # start kafka stream config = getConfig()['kafka'] kafka_stream = KafkaStream.create_source(ssc, config['config'], config['topics']) # filter streams new_encounters = get_encounter_stream(kafka_stream) new_obs = get_new_obs_stream(kafka_stream) updated_obs = get_updated_obs_stream(kafka_stream) # save encounters to parquet save_stream_to_parquet(new_encounters, path="spark-warehouse/encounters") #transform new obs # to do: # 1) add code to deal with updated obs (voided) # 2) add code to deal with new obs but existing encounters # 3) add code to deal with obs group self.transform_data(spark, new_obs)
# -*- coding:utf8 -*- #__author__ = "neo" from config.config import getConfig import pymysql host = getConfig('mysql', 'host') port = getConfig('mysql', 'port') user = getConfig('mysql', 'username') passwd = getConfig('mysql', 'password') db = getConfig('mysql', 'db') def query(sql, data=None): conn = pymysql.connect(host=host, port=int(port), user=user, passwd=passwd, db=db, charset='utf8') cursor = conn.cursor() cursor.execute(sql, data) result = cursor.fetchone() conn.commit() cursor.close() conn.close() return result
wordcloud = WordCloud(font_path='../resources/simsun.ttf').generate( " ".join(text)) import matplotlib.pyplot as plt plt.imshow(wordcloud, interpolation='bilinear') plt.imshow(wordcloud) plt.axis("off") plt.show() if __name__ == '__main__': env = 'dev' if len(sys.argv) == 2: env = sys.argv[1] configure = updateConfig('env', 'env_active', env) configure = getConfig() # 加载词库到默认路径 # loadDicDataToFile() ## extract_tag: # extract_tag() ## tf-idf # tfidf_keywords() #cloud # wordCould() ##get data getData()
#-*- coding:utf8 -*- import unittest from login import get_session import requests from config.config import getConfig from command.optionexcel import OperationExcel headers = eval(getConfig('header', 'headers')) json_header = eval(getConfig('header', 'json_header')) data_no_permission = { 'username': '******', 'password': '******', 'rememberMe': False } #无权限用户 login_admin = { 'username': '******', 'password': '******', 'rememberMe': False } #超级管理员 dict_add_api = getConfig('swagger', 'baseurl') + ':' + getConfig( 'swagger', 'port') + OperationExcel(u'字典管理').get_value(4, 1) #添加字典 session_api = getConfig('swagger', 'baseurl') + ':' + getConfig( 'swagger', 'port') + OperationExcel(u'会话管理').get_value(12, 1) #查询在线用户列表 delete_api = getConfig('swagger', 'baseurl') + ':' + getConfig( 'swagger', 'port') + OperationExcel(u'会话管理').get_value(4, 1) #踢出用户 dict_delete_api = getConfig('swagger', 'baseurl') + ':' + getConfig( 'swagger', 'port') + OperationExcel(u'字典管理').get_value(13, 1) #删除字典 captcha_api = getConfig('swagger', 'baseurl') + ':' + getConfig( 'swagger', 'port') + OperationExcel(u'其他组件').get_value(4, 1) #获取验证码 dicts_api = getConfig('swagger', 'baseurl') + ':' + getConfig(
def getBroker(): type = getConfig().get("Broker", "BrokerClass") if (type == "LiveBroker"): return IBapi()
''' 处理与mysql相关的信息 ''' #coding:utf-8 import sys sys.path.append("..") from config.config import getConfig import redis config = getConfig() pool = redis.ConnectionPool( host=config['redis']['host'], port=config.getint('redis', 'port'), password=config['redis']['password'], db=config.getint('redis', 'database'), decode_responses=True) # host是redis主机,需要redis服务端和客户端都起着 redis默认端口是6379 def getRedisConn(key): r = redis.Redis(connection_pool=pool) return r.get(key) def getKey(key): r = redis.Redis(connection_pool=pool) result = r.get(key) return result
def apiInvoke(): ''' api 调用该接口,执行评论提取流程 :return: 成功:true 失败:false ''' # stopWords = set(readFile(stop_file)) response = {} start = datetime.now() try: if request.method == 'GET': strDate = request.args.get('date', None) else: strDate = request.get_json()['date'] env = 'dev' if len(sys.argv) == 2: env = sys.argv[1] updateConfig('env', 'env_active', env) configure = getConfig() clearDbData() poolSize = configure.getint('pool', 'poolSize') ## 线程池大小 collections = ["Mall_Comment_Info", "Shop_Comment_Info"] ## mongodb 待分析的表 ## 每次处理的数量 batchSize = configure.getint('pool', 'batchSize') batchSize = 100 if batchSize is None else batchSize shopIds = findAllShopId() mallIds = findAllMallId() for collection in collections: ids = shopIds if collection == 'Shop_Comment_Info' else mallIds ## 每次创建 POOL_SIZE 个线程 ,一共创建size/POOL_SIZE 个 t1 = datetime.now() pool = multiprocessing.Pool(processes=poolSize) # 创建POOL_SIZE个进程 i = 0 while i < len(ids): logger.info("i = {0} ".format(i)) if (i + batchSize) >= len(ids): logger.info("最后一批数据,索引范围:{0}-{1}".format(i, len(ids))) subset = ids[i:] else: subset = ids[i:i + batchSize] # processMongoData(collection,subset, strDate) pool.apply_async(processMongoData, args=( collection, subset, strDate, )) i += batchSize # pool.close() ## 关闭进程 pool.join() ## 等待所有进程执行完成 t2 = datetime.now() logger.info("处理{0}数据耗时{1}s".format(collection, (t2 - t1).seconds)) end = datetime.now() logger.info("spend time:{0}s".format((end - start))) response["code"] = 200 response["msg"] = "执行成功。耗时{0}s".format((end - start).seconds) except: end = datetime.now() logger.error("exception now", exc_info=1) response["code"] = 501 response["msg"] = "程序异常。耗时{0}s".format((end - start).seconds) # stopWords = None ## 清空,防止内存不释放 return json.dumps(response)
def start(self): config = getConfig()['spark'] SparkApp.start_spark(app_name=config['app_name'], master=config['master'], spark_config=config['configs'], callback=self.start_job)
class LogFormatter(tornado.log.LogFormatter): def __init__(self): super(LogFormatter, self).__init__( fmt= '%(color)s[%(asctime)s %(filename)s:%(funcName)s:%(lineno)d %(levelname)s]%(end_color)s %(message)s', datefmt='%Y-%m-%d %H:%M:%S') db1 = DB_dw_new() db1.get_pool() #数据预加载 sysInitData = SysInitData() sysInitData.init_load() if __name__ == "__main__": dev = "dev." if str(config.getConfig('public', 'dev')) == "true" else "" application = tornado.web.Application( handlers=[ # (r"/deploy/init2", InitHandler2), (r"/deploy/publish2", OuterHandler2), (r"/deploy/publishall2", OuterAllHandler2), (r"/deploy/find2", FindVersionHandler2), (r"/deploy/findPath2", FindPathHandler2) ], debug=True if str(config.getConfig(dev + 'web', 'debug')) == "true" else False) # 日志设置 tornado.options.parse_command_line() [i.setFormatter(LogFormatter()) for i in logging.getLogger().handlers] application.listen(int(config.getConfig(dev + 'web', 'port')))
def main(): ''' This is the main Function which will be responsible for running all the functions for the version 1 , console application :return: ''' path = str(config.getConfig()['MongoDb']['URL']) print("Connecting to Database") con = connection.Mongo(path) n = Notes.Notes("", "") data = con.FindAll(n.tablename) if data: for i in data: allNotes.append( Notes.Notes(i['name'], i['data'], i['_id'], i['created_at'])) print(allNotes[-1]) while True: os.system("clear") try: print("Welcome To Notes Application , version 1.0.0") print( "1.Add a Note\n2.Find a note\n3.Delete a Note\n4.See all Notes\n5.exit" ) try: n = int(input("Enter Your Choice >>")) except ValueError: continue if n == 5: raise KeyboardInterrupt elif n == 1: node = add() id = con.Insert(node.tablename, node.retData()) node.setId(id) allNotes.append(node) print("Added") time.sleep(2) elif n == 2: result = find() if result['status'] == 'OK': for i in result['data']: print(i) else: name = result['name'] node = Notes.Notes(name, "") data = con.Find(node.tablename, node.name) if data: for i in data: allNotes.append( Notes.Notes(i['name'], i['data'], i['_id'], i['created_at'])) print(allNotes[-1]) input() elif n == 3: arr = delete() if not arr: continue if len(arr) <= 0: print("Enter The Correct Indexes again please !!") time.sleep(1.2) continue n = Notes.Notes("", "") count = con.Delete(n.tablename, arr) print("Delete Count: {}".format(count)) input() elif n == 4: if not findAll(): n = Notes.Notes("", "") data = con.FindAll(n.tablename) if data: for i in data: allNotes.append( Notes.Notes(i['name'], i['data'], i['_id'], i['created_at'])) print(allNotes[-1]) pass input() else: continue except Exception as E: print("Exception has occured {} \n".format(E)) raise KeyboardInterrupt except KeyboardInterrupt: print("quitting...") exit(1)
## # Win! # Logging logLocation = config.getConfig('logLocation') logger = logging.getLogger('docker-aws-python') hdlr = logging.FileHandler(logLocation) formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s') hdlr.setFormatter(formatter) logger.addHandler(hdlr) logger.setLevel(logging.DEBUG) # # # Records # a = records.getAAlias() # c = records.getCAlias() # ip = records.getPublicIp() # #
#-*- coding:utf8 -*- import requests from config.config import getConfig from command.optionexcel import * headers = getConfig('header', 'headers') data = {'username': '******', 'password': '******', 'rememberMe': False} login_api = getConfig('swagger', 'baseurl') + ':' + getConfig( 'swagger', 'port') + OperationExcel(u'用户个人管理').get_value(4, 1) def get_session(data=data): login_session = requests.Session() login_session.post(login_api, headers=eval(headers), data=data) return login_session