def deleteDuplicatedPrediction(mode): duplicatedPredictions = list(stockMongo.findDuplicatedPrediction(mode)); dateSymbol_df = pd.DataFrame([item['_id'] for item in duplicatedPredictions]) ''' Date Symbol 0 2017-08-10 SFBC 1 2017-08-10 SVBI 2 2017-08-10 SENEB ''' # k: v for k, v in df.groupby('Region') dateSymbols = {date:list(group_df['Symbol']) for date, group_df in dateSymbol_df.groupby('Date')} # for date, symbol_df in dateSymbol_df.groupby('Date'): # dateSymbols[date] = list(symbol_df['Symbol']) ''' dateSymbols { '2017-06-12': ['JIVE'], '2017-06-15': ['BNCN'], '2017-06-16': ['CNCO', 'SPAN', 'GNVC'], ... } ''' parallel.runToAllDone(stockMongo.deletePredictionBySymbolAndDate, [(mode, symbols, date) for date, symbols in dateSymbols.items()], NUMBER_OF_PROCESSES=8)
def deleteDuplicatedQuote(startDate, endDate): # check data month by month # symbols = stockMongo.findAllActiveSymbols() # startDate = '2017-04-01' # endDate = '2017-07-01' print(startDate, endDate) duplicatedQuotes = stockMongo.findDuplicatedQuotes(startDate, endDate) runToAllDone(deleteQuoteHasNoNextClose, [(quoteCount,) for quoteCount in duplicatedQuotes])
def isPreviousNextCloseUpdated(date): symbols = stockMongo.findAllActiveSymbols(); start = dt.datetime.now() func = functools.partial(checkAndUpdate, date=date) parallel.runToAllDone(func, [(symbol,) for symbol in symbols], NUMBER_OF_PROCESSES=8) end = dt.datetime.now() print('time consumed:', (end - start).seconds)
def checkSymbolVolume(): startDate = getPassedDate(120).strftime("%Y-%m-%d") symbols = pd.DataFrame( list(stockMongo.findSymbolsVolumesLessThan(startDate, 100000)))['_id'].values print(len(symbols)) print(symbols) runToAllDone(stockMongo.addSymbolStatus, [(symbol, "lowVolume") for symbol in symbols])
def learn(machineLearningMode, sepDate): symbols = pd.DataFrame(list(stockMongo.findAllActiveSymbols()))['Symbol'].values # symbols = ['KO'] learningFunction = functools.partial(initialMachineLearning, machineLearningMode, None, sepDate) runToAllDone(learningFunction, [(symbol,) for symbol in symbols]) saveAccuracyFunc = functools.partial(saveAccuracy, machineLearningMode, None, sepDate) runToAllDone(saveAccuracyFunc, [(symbol,) for symbol in symbols]) # verify the prediction of the rest data startDate = sepDate endDate = None runToAllDone(functools.partial(verifyQuote, machineLearningMode, startDate, endDate), [(symbol,) for symbol in symbols])
def runDailyJob(): print(now(), "daily job is started...") # start = datetime.datetime.now().strftime("%Y-%m-%d") # end = datetime.datetime.now().strftime("%Y-%m-%d") start = '2017-08-28' end = '2017-08-28' # check if previous predict exists, if not, do it symbols = list(findAllActiveSymbols()) fetchAndStore = functools.partial(fetchAndStoreQuotes, start=start, end=end) # fetchAndStore = lambda symbol: fetchAndStoreQuotes(symbol, start, end) runToAllDone(fetchAndStore, [(symbol,) for symbol in symbols], NUMBER_OF_PROCESSES=12) print(now(), "quote csv files were all downloaded...") loadAllQuoteFiles() time.sleep(60) print(now(), "save next tx data...") quotes = stockMongo.findQuotesByPeriod(start, end) runToAllDone(saveNextTxDayData, [(quote,) for quote in quotes]) print(now(), 'starting prediction for next Tx day') runToAllDone(predictAndSave, [(symbol, start) for symbol in symbols]) # , NUMBER_OF_PROCESSES=1 print(now(), 'generating predict report') predictReport() # verify for date in pd.date_range(datetime.datetime.strptime(start, "%Y-%m-%d"), datetime.datetime.strptime(end, "%Y-%m-%d")): print('check', date) dataHealth.dailyCheck(date.strftime("%Y-%m-%d")) print(now(), "daily job is done...")
def findAndDeleteDuplicatedQuote(): dataRanges = getDateRanges('2016-08-20', '2017-04-01', 100) runToAllDone(deleteDuplicatedQuote, dataRanges)
def loadAllQuoteFiles(): csvFiles = list(filter(isFile, os.listdir(fileUtil.QUOTES_DIR))) # csvFiles = ['INF_2017-06-21_2017-06-21.csv'] # with multiprocessing.Pool(multiprocessing.cpu_count() - 1) as p: runToAllDone(loadCsv, [(csvFile, ) for csvFile in csvFiles])