def run_tests(): global test_result TEST_ENV = { 'stopped': False, 'test': True, 'send': False, } TEST_ENV['sender'] = TestSender() TEST_ENV['enumerator'] = Enumerator(TEST_ENV) TEST_ENV['parser'] = Parser(TEST_ENV) TEST_ENV['screener'] = Screener(TEST_ENV) TEST_ENV['logger'] = open('/dev/null', 'w') TEST_ENV['processor'] = processor = Processor(TEST_ENV) num_failures = 0 for (rcpttos, expected) in tests: test_result = {} processor.process_message({ 'peer': None, 'mailfrom': 'mailfrom', 'rcpttos': rcpttos, 'data': 'message', }) print 'testing sending to', rcpttos, '...' for key in expected: assert key in test_result if type(test_result[key]) == type([]): assert set(test_result[key]) == set(expected[key]) else: assert test_result[key] == expected[key] for key in test_result: assert key in expected
def screener(): if request.method == 'GET': default_dictionary = {} return render_template( 'screener.html', **default_dictionary) elif request.method == 'POST': query = request.form.get('query') try: results = Screener().run_screen(query) except Exception as e: print(e) return '''Something was wrong with your query. Try again.''' output = { 'query': query, 'results': results } return render_template('screener.html', **output)
def getInstruments(self): print 'getInstruments()' self.screener = Screener() screener_model = self.screener.fetch() self.instruments = screener_model.get('DataList')
class Collector: def __init__(self, dt=datetime.now(), predict=False): print 'Sequencer initialized...' self.predict = predict self.observations = Observations() print 'Observations instance initialized with length:', self.observations.length if self.predict: self.today = dt + timedelta(days=1) else: self.today = dt self.yesterday_market_times = utils.previous_market_times(self.today) print 'Getting yesterdays chart for:', datetime.fromtimestamp( self.yesterday_market_times[0]) self.today_market_times = utils.current_market_times(self.today) self.charts = Tickcharts() def start(self): self.sequence() def sequence(self): self.getInstruments() self.instruments.each(self.getChart) self.initialize_observations() self.setNews() self.setNumNewHighs() self.setNumNewLows() self.setTallestCandles() if not self.predict: self.observations.create() else: self.printObservations() def printObservations(self): printer = [] prediction_file = open( './trading-data/' + config.get('predict.filename'), 'w+') for observation in self.observations.toJSON(): item = observation item['market_open_datetime'] = str( observation['market_open_datetime']) item['market_close_datetime'] = str( observation['market_close_datetime']) printer.append(item) prediction_file.write(json.dumps(printer)) def printObservation(self, observation, index): print observation.toJSON() def appendObservation(self, model, index): self.observations.append(model) def initialize_observations(self): print 'initialize_observations()' self.charts.each(self.appendObservation) def getInstruments(self): print 'getInstruments()' self.screener = Screener() screener_model = self.screener.fetch() self.instruments = screener_model.get('DataList') def getChart(self, instrument, index): print 'getChart()' symbol = instrument.get('Eqsm') regex = re.compile(r'(\S+[\.\/]\S+$)', re.I) search_symbol = regex.search(symbol) if search_symbol: return {} chart_instance = ChartAnalysis({ 'startdate': self.yesterday_market_times[0], 'enddate': self.yesterday_market_times[1], 'ticker': symbol }) chart = chart_instance.fetch() try: results = chart.get('chart').get('result').at(0) except: return None if not results.has('timestamp'): return None timestamps = results.get('timestamp') opens = results.get('indicators').get('quote').at(0).get('open') closes = results.get('indicators').get('quote').at(0).get('close') highs = results.get('indicators').get('quote').at(0).get('high') lows = results.get('indicators').get('quote').at(0).get('low') volumes = results.get('indicators').get('quote').at(0).get('volume') return_chart = Tickchart({ 'symbol': symbol, 'timestamps': timestamps, 'opens': opens, 'closes': closes, 'highs': highs, 'lows': lows, 'volumes': volumes }) if not self.predict: last_close = return_chart.get('closes')[len(closes) - 1] tomorrow_open_margin = self.getTodaysOpenMargin(symbol, last_close) return_chart.set('next_market_open_margin', tomorrow_open_margin) if (return_chart.get('next_market_open_margin') != None and not self.predict) or self.predict: self.charts.append(return_chart) return return_chart def getTodaysOpenMargin(self, symbol, last_close): print 'getTodaysOpenMargin()' chart_instance = ChartAnalysis({ 'startdate': self.today_market_times[0], 'enddate': self.today_market_times[1], 'ticker': symbol }) chart = chart_instance.fetch() try: chart.get('chart') results = chart.get('chart').get('result').at(0) except: print 'Could not parse results from chart: %s' % symbol return None if not results.has('timestamp'): return None today_timestamps = results.get('timestamp') today_opens = results.get('indicators').get('quote').at(0).get('open') today_closes = results.get('indicators').get('quote').at(0).get( 'close') today_highs = results.get('indicators').get('quote').at(0).get('high') today_lows = results.get('indicators').get('quote').at(0).get('low') today_volumes = results.get('indicators').get('quote').at(0).get( 'volume') today_chart = Tickchart({ 'symbol': symbol, 'timestamps': today_timestamps, 'opens': today_opens, 'closes': today_closes, 'highs': today_highs, 'lows': today_lows, 'volumes': today_volumes }) today_first_open = today_chart.get('opens')[0] return (today_first_open - last_close) / last_close def getNumNewHighs(self, chart, index): print 'getNumNewHighs()' highs = chart.get('highs') times = chart.get('timestamps') volumes = chart.get('volumes') count = 0 abvVolCount = 0 blwVolCount = 0 amCount = 0 pmCount = 0 avgVol = chart.get('avg_volume') max_high = 0 for key, high in enumerate(highs): if high and high > max_high: max_high = high count += 1 if volumes[key] > avgVol: abvVolCount += 1 else: blwVolCount += 1 if utils.isMorning(times[key]): amCount += 1 else: pmCount += 1 self.observations.at(index).set('num_new_highs', count) self.observations.at(index).set('num_highs_abv_avg_vol', abvVolCount) self.observations.at(index).set('num_highs_blw_avg_vol', blwVolCount) self.observations.at(index).set('num_new_highs_am', amCount) self.observations.at(index).set('num_new_highs_pm', pmCount) return count def setNumNewHighs(self): print 'setNumNewHighs()' self.charts.each(self.getNumNewHighs) def getNumNewLows(self, chart, index): print 'getNumNewLows()' lows = chart.get('lows') times = chart.get('timestamps') volumes = chart.get('volumes') count = 0 abvVolCount = 0 blwVolCount = 0 amCount = 0 pmCount = 0 avgVol = chart.get('avg_volume') min_low = float('inf') for key, low in enumerate(lows): if low and low < min_low: min_low = low count += 1 if volumes[key] > avgVol: abvVolCount += 1 else: blwVolCount += 1 if utils.isMorning(times[key]): amCount += 1 else: pmCount += 1 self.observations.at(index).set('num_new_lows', count) self.observations.at(index).set('num_lows_abv_avg_vol', abvVolCount) self.observations.at(index).set('num_lows_blw_avg_vol', blwVolCount) self.observations.at(index).set('num_new_lows_am', amCount) self.observations.at(index).set('num_new_lows_pm', pmCount) return count def setNumNewLows(self): print 'setNumNewLows()' self.charts.each(self.getNumNewLows) def getTallestCandles(self, chart, index): print 'getTallestCandles()' candles = chart.get('candlesticks').toJSON() tallestGreen = 0.0 tallestRed = 0.0 for candle in candles: if candle['color'] == 'green' and candle[ 'relative_height'] > tallestGreen: tallestGreen = candle['relative_height'] if candle['color'] == 'red' and candle[ 'relative_height'] > tallestRed: tallestRed = candle['relative_height'] self.observations.at(index).set('tallest_green_candlestick', tallestGreen) self.observations.at(index).set('tallest_red_candlestick', tallestRed) def setTallestCandles(self): print 'setTallestCandles()' self.charts.each(self.getTallestCandles) def fetchNewsitems(self, chart, index): print 'getNewsitems()' newsitems = News({'symbol': chart.get('symbol')}) news = newsitems.fetch() chart.set('news', newsitems.get('data')) self.observations.at(index).set('age_recent_news', chart.get('news_age')) def setNews(self): print 'getNews()' self.charts.each(self.fetchNewsitems)
def get_industry_data(company): screener = Screener(company) related = screener.get_related_companies() industry = screener.industry if company not in related: related.append(company) industry_dict = {} indicator_list = [ "MCAP", "EPS5", "ROA", "CROCI", "DIVYLD", "EPS1", "PEG", "PE", "PBV", "LTDE", "ROE", "ROCE", "NETPCT", "CRATIO", "SOLRATIO", "LTDE", "IC", "DEBT_ASSETS", ] for comp in related: comp_dict = {} indicator_set = get_indicator_set(comp, indicator_list) if indicator_set: std = indicator_set["standalone"] con = indicator_set["consolidated"] for i in indicator_list: val = con[i] if con[i] else std[i] if val: comp_dict.update({i: val}) else: comp_dict.update({i: np.nan}) industry_dict.update({comp: comp_dict}) adf = pd.DataFrame.from_dict(industry_dict, orient="index") adf["Weights"] = adf.MCAP / adf.MCAP.sum() * 100 indicators = adf.columns.tolist() indicators.remove("Weights") indicators.remove("MCAP") industry_dict = {"companies": related} industry_dict.update({"industry": industry}) for i in indicators: c = adf[i].loc[company] a = adf[i].mean() wa = (adf[i] * adf["Weights"]).sum() / adf["Weights"].sum() if not c or np.isnan(c): c = 0 if not a or np.isnan(a): a = 0 if not wa or np.isnan(wa): wa = 0 industry_dict.update({i: {"company": c, "average": a, "weighed_average": wa}}) nifty_pe_str = r_full.get("NIFTY_PE").decode("UTF-8") industry_dict.update({"nifty_pe": float(nifty_pe_str)}) industry_dict.update({"cash_saving_rate": 5.0}) return industry_dict
def get_indicator_set(comp, indicator_list): return_set = {} obj_consolidated = r_full.get("{}_A_CONSOLIDATED".format(comp)) obj_standalone = r_full.get("{}_A_STANDALONE".format(comp)) if not obj_consolidated and not obj_standalone: print("Company {} not found in the database.".format(comp)) return return_set if obj_consolidated: df_c = pd.read_json(obj_consolidated) if obj_standalone: df_s = pd.read_json(obj_standalone) screenerFailed = False try: screener = Screener(comp) indicator_set_consolidated = {} indicator_set_standalone = {} # get PE and PBV from screener pe = screener.get_pe() pe_standalone = screener.get_pe(standalone=True) indicator_set_standalone.update({"PE": pe_standalone}) indicator_set_consolidated.update({"PE": pe}) bv = screener.get_book_value() bv_standalone = screener.get_book_value(standalone=True) current_price = screener.get_current_price() pbv = np.nan pbv_standalone = np.nan if bv: pbv = current_price / bv if bv_standalone: pbv_standalone = current_price / bv_standalone indicator_set_standalone.update({"PBV": pbv_standalone}) indicator_set_consolidated.update({"PBV": pbv}) except: print("Error while getting data from screener") traceback.print_exc(file=sys.stdout) screenerFailed = True other_indicators = indicator_list if screenerFailed: other_indicators += ["PE", "PBV"] for ind in other_indicators: try: if not df_c.empty: result_c = df_c[ind].loc[df_c[ind].last_valid_index()] else: result_c = np.nan indicator_set_consolidated.update({ind: result_c}) except: indicator_set_consolidated.update({ind: np.nan}) try: if not df_s.empty: result_s = df_s[ind].loc[df_s[ind].last_valid_index()] else: result_s = np.nan indicator_set_standalone.update({ind: result_s}) except: indicator_set_standalone.update({ind: np.nan}) return_set.update( {"company": comp, "standalone": indicator_set_standalone, "consolidated": indicator_set_consolidated} ) return return_set
def get_data(company, indicator, get_quarterly=False): print("Company : {}, Indicator : {}".format(company, indicator)) if not get_quarterly: df_standalone_full = pd.read_json(r_full.get(company + "_A_STANDALONE")) df_consolidated_full = pd.read_json(r_full.get(company + "_A_CONSOLIDATED")) else: df_standalone_full = pd.read_json(r_full.get(company + "_Q_STANDALONE")) df_consolidated_full = pd.read_json(r_full.get(company + "_Q_CONSOLIDATED")) indicatorList = [] if isinstance(indicator, str): indicatorList.append(indicator) else: indicatorList = indicator seriesStandalone = [] seriesConsolidated = [] for indicator in indicatorList: try: df_standalone = df_standalone_full[indicator].dropna() df_consolidated = df_consolidated_full[indicator].dropna() if indicator == "PE": pe_standalone = None pe_consolidated = None screener = None try: screener = Screener(company) except: print("Error while initializing screener") if screener: try: pe_standalone = screener.get_pe(standalone=True) except: print("Error while getting standalone PE from screener") try: pe_consolidated = screener.get_pe() except: print("Error while getting consolidated PE from screener") if pe_standalone: new_df_standalone = pd.Series([pe_standalone], index=[datetime.datetime.now().date()]) df_standalone = df_standalone.append(new_df_standalone) if pe_consolidated: new_df_consolidated = pd.Series([pe_consolidated], index=[datetime.datetime.now().date()]) df_consolidated = df_consolidated.append(new_df_consolidated) seriesStandalone.append( { "Metric": indicator, "Index": df_standalone.index.tolist(), "Data": df_standalone.fillna(0).tolist(), "PctChange1Y": getPercentChange(df_standalone), "PctChange5Y": getPercentChange(df_standalone, step=5), "Mean": getMean(df_standalone), "Success": "True", "Message": "", "Mean_5_YR": getMean(df_standalone, tail=5), } ) seriesConsolidated.append( { "Metric": indicator, "Index": df_consolidated.index.tolist(), "Data": df_consolidated.fillna(0).tolist(), "PctChange1Y": getPercentChange(df_consolidated), "PctChange5Y": getPercentChange(df_consolidated, step=5), "Mean": getMean(df_consolidated), "Mean_5_YR": getMean(df_consolidated, tail=5), "Success": "True", "Message": "", } ) except: seriesStandalone.append( {"Metric": indicator, "Success": "False", "Message": "Error while getting indicator : " + indicator} ) seriesConsolidated.append( {"Metric": indicator, "Success": "False", "Message": "Error while getting indicator : " + indicator} ) my_dict = {} my_dict.update({"Company": "{}".format(company)}) my_dict.update({"Standalone": seriesStandalone}) my_dict.update({"Consolidated": seriesConsolidated}) return my_dict
'peer': peer, 'mailfrom': mailfrom, 'rcpttos': rcpttos, 'data': data, 'status': 'UNVERIFIED', }) if __name__ == '__main__': # Initialize components GLOBAL_ENV['db'] = MongoClient().emails sender = Sender(GLOBAL_ENV) enumerator = Enumerator(GLOBAL_ENV) parser = Parser(GLOBAL_ENV) screener = Screener(GLOBAL_ENV) processor = Processor(GLOBAL_ENV) sendlist_server = SendlistServer(GLOBAL_ENV, IP_ADDRESS) # Initialize global environment GLOBAL_ENV['sender'] = sender GLOBAL_ENV['enumerator'] = enumerator GLOBAL_ENV['parser'] = parser GLOBAL_ENV['screener'] = screener GLOBAL_ENV['logger'] = open(LOG_FILE, 'a', 0) GLOBAL_ENV['processor'] = processor GLOBAL_ENV['sendlist_server'] = sendlist_server server = SimpleSMTPServer((IP_ADDRESS, SMTP_PORT), None) # Start new thread for the SMTP server.