def UpdateLotto638(): client = Mongo() table = client['admin']['Lottery.Draw.Lotto638'] cnt = table.count_documents({}) if not cnt: # insert first time # fill old data table.create_index([('期別', 1), ('開獎日', 1)], unique=True) for y in range(103, datetime.today().year): for m in range(1, 13): print(y, m) i = 0 while i < 3: final_data = Lotto638(y, m) if final_data: print(final_data) table.insert_many(final_data) final_data = [] break time.sleep(5) i += 1 time.sleep(15) # fill current year data td = datetime.today() y = td.year for m in range(1, td.month + 1): i = 0 while i < 3: final_data = Lotto638(y, m) if final_data: table.insert_many(final_data) final_data = [] break time.sleep(5) else: # Daily Update td = datetime.today() y = td.year m = td.month i = 0 while i < 3: final_data = Lotto638(y, m) if final_data: break i += 1 time.sleep(5) # time.sleep(5) info = getInfoData(LotteryType.Lotto638) if not info: if final_data: table.insert_many(final_data) else: final_data = [ x for x in final_data if x['開獎日'] > info[0]['UpdateDate'] ] if final_data: table.insert_many(final_data) # Update Information updateInfo(LotteryType.Lotto638)
def UpdateOldData(): client = Mongo() table = client['admin']['Stock.Info'] old_df = table.find({}) for data in old_df: try: data['已發行普通股數或TDR原股發行股數'] = float(data['已發行普通股數或TDR原股發行股數'].split('\n')[0][:-1].replace(',', '')) except: pass if '_id' in data: table.update_one({'_id':data['_id']}, {'$set':data}, upsert=True) else: table.update_one(data, {'$set':data}, upsert=True)
def updateInfo(datas): client = Mongo() table = client['admin']['Stock.Info'] cnt = table.count_documents({}) if not cnt: table.create_index([('股票代號', 1)], unique=True) table.insert_many(datas) else: old_df = table.find({}) _id_mapping = dict((x['股票代號'], x['_id']) for x in old_df) for data in datas: try: data.update({'_id':_id_mapping[data['股票代號']]}) except: pass if '_id' in data: table.update_one({'_id':data['_id']}, {'$set':data}, upsert=True) else: table.update_one(data, {'$set':data}, upsert=True)
def updateInfo(lotteryType: LotteryType): client = Mongo() table = client['admin']['Lottery.Info'] data = list(table.find({'Name':{'$eq':lotteryType.value}})) if not data: temp_data = {'UpdateDate':datetime.today().strftime('%Y-%m-%d')} if lotteryType == LotteryType.Lotto649: # 大樂透 temp_data.update({'Nums':[str(x).zfill(2) for x in range(1, 50)], 'Name':lotteryType.Lotto649.value, 'UpdateDate':datetime.today().strftime('%Y-%m-%d')}) elif lotteryType == LotteryType.Lotto638: # 威力透 temp_data.update({'Nums':{'Block1':[str(x).zfill(2) for x in range(1, 39)], 'Block2':[str(x).zfill(2) for x in range(1, 9)]}, 'Name':lotteryType.Lotto638.value, 'UpdateDate':datetime.today().strftime('%Y-%m-%d')}) else: temp_data = data[0] temp_data.update({'UpdateDate':datetime.today().strftime('%Y-%m-%d')}) table.update_one({'_id':temp_data['_id']} if '_id' in temp_data else temp_data, {'$set':temp_data}, upsert=True)
from flask import Flask, request, abort, send_file from flask_cors import CORS from modules import Mongo, Validator, Jwt, Img import pprint, json, re, subprocess, os os.environ['DB_USER'] = ( os.getenv('DB_USER'), '' ) [ os.getenv('DB_USER') == None ] os.environ['DB_PASS'] = ( os.getenv('DB_PASS'), '' ) [ os.getenv('DB_PASS') == None ] os.environ['DB_HOST'] = ( os.getenv('DB_HOST'), 'localhost' ) [ os.getenv('DB_HOST') == None ] os.environ['DB_PORT'] = ( os.getenv('DB_PORT'), '27017' ) [ os.getenv('DB_PORT') == None ] os.environ['DB_NAME'] = ( os.getenv('DB_NAME'), 'skk' ) [ os.getenv('DB_NAME') == None ] os.environ['DB_POOL'] = ( os.getenv('DB_POOL'), '20' ) [ os.getenv('DB_POOL') == None ] os.environ['API_SECRET'] = ( os.getenv('API_SECRET'), "awVYjjzt7ssPDRxwmrIY" ) [ os.getenv('API_SECRET') == None ] mng = Mongo('mongodb://' + ( os.getenv('DB_USER') + ':' + os.getenv('DB_PASS') + '@', '') [ os.getenv('DB_USER') == '' and os.getenv('DB_PASS') == '' ] + os.getenv('DB_HOST') + ':' + os.getenv('DB_PORT'), os.getenv('DB_NAME'), int(os.getenv('DB_POOL'))) Img.prepare(mng) app = Flask(__name__) CORS(app) secret = os.getenv('API_SECRET') data_model = { "user" : { "pseudo" : "string", "mail" : "mail", "password" : "string" }, "room" : { "users" : "array:string", "banned" : "array:string", "song" : { "id" : "string",
if not os.path.isdir(path): path = os.getcwd() def update_data_dict(d): del d['Index'] d['Adj Close'] = d['Close'] return d def update_one(table, d): table.update_one(d, {'$set': d}, upsert=True) time.sleep(0.001) if __name__ == '__main__': client = Mongo() table = client['admin']['TWSE']['historicalPrice'] table.create_index([('Date', 1), ('Ticker', 1)], unique=True) df = read_csv(os.path.join(path, 'TWSE_HistoricalPrice.txt'), sep='\t') data = [dict(x._asdict().items()) for x in df.itertuples()] start_time = time.time() with ThreadPoolExecutor(50) as executor: exes = [executor.submit(update_data_dict, d) for d in data] data = [exe.result() for exe in exes] table.insert_many(data) duration = round((time.time() - start_time) / 60, 4) Tele().sendMessage(f'insert {len(data)} data use {duration} mins', group='UpdateMessage')
def GetStockTickers(): client = Mongo() table = client['admin']['TWSE']['StockList'] updateDate = list(table.distinct('UpdateDate'))[-1] datas = table.find({'UpdateDate':{'$eq':updateDate}, 'AssetType':{'$eq':'股票'}}) return [x['Ticker'] for x in datas]
parser.add_argument( '--autoxbit', action='store_true', help='add auto generated XBIT SET name to parsed rules') parser.add_argument('--userdef', nargs=2, help='add user defined (single) tag to parsed rules') Arguments = parser.parse_args() ########################[Arguments section - End]################################ global Configs # Read MongoDB config from file Configs = dict(ConfRead.Read().items('db')) # Make a connection to MongoDB Connection = Mongo.DBConnection(Host=Configs['mongo_ip'], Port=Configs['mongo_port'], User=Configs['mongo_user'], Password=Configs['mongo_password'], DbName=Configs['mongo_dbname']) # Connect [or make] DB and collection in MongoDB global RulesCollection dbnames = Connection.list_database_names() if 'siem' not in dbnames: MainDB = Connection.siem MainDB = Connection['siem'] RulesCollection = MainDB['rules'] # Read & Pars! listOfFiles = list() for (dirpath, dirnames, filenames) in os.walk(Arguments.path): listOfFiles += [os.path.join(dirpath, file) for file in filenames] for Files in listOfFiles: ReadRuleFile(Files)
def getInfoData(lotteryType: LotteryType): client = Mongo() table = client['admin']['Lottery.Info'] data = list(table.find({'Name':{'$eq':lotteryType.value}})) return data