def handle(self, *args, **options): print 'starting script' event_log = DataHarvestEventLog() event_log.data_type = DATA_HARVEST_TYPE_ARTICLES event_log.notes = 'running' event_log.save() script_start_time = datetime.datetime.now() try: status_message = get_articles() log_notes = status_message except Exception as e: print "error getting articles.", str(e) log_notes = str(e) script_end_time = datetime.datetime.now() total_seconds = (script_end_time - script_start_time).total_seconds() print 'time elapsed: %d seconds' % total_seconds event_log.notes = log_notes event_log.save() print 'finished script'
def handle(self, *args, **options): print "Getting tier status" event_log = DataHarvestEventLog() event_log.data_type = DATA_HARVEST_TYPE_IMPORT_TIERS event_log.notes = 'importing tier information' event_log.save() script_start_time = datetime.datetime.now() notes = '' print 'start delete' for t in Ticker.objects.all(): t.tier_status = '' t.tier = 0 t.save() print 'end delete' print 'starting tier import' print 'pre-import: number of tickers in SOL with tier 1: %d' % get_count_of_tier_one_tickers() #columns = defaultdict(list) # each value in each column is appended to a list with open(TIERS_CSV, 'rU') as f: reader = csv.DictReader(f) # read rows into a dictionary format for row in reader: # read a row as {column1: value1, column2: value2,...} ticker_symbol = str(row['ticker']).upper() print ticker_symbol service = str(row['service']) try: ticker = Ticker.objects.get(ticker_symbol = ticker_symbol) ticker.tier = 1 if len(ticker.tier_status): ticker.tier_status += ", %s" % service else: ticker.tier_status = service print ticker_symbol, ticker.tier_status ticker.save() except: print '%s tier status cannot be updated' % ticker_symbol """ EB work-in-progress note: the above solution does not convert tickers to U/C and remove commas like the old code did. However, I plan to maintain the tiers spreadsheet so that there will only ever be one U/C ticker per field. So I will leave that problem for now. """ notes = 'updated tier status' script_end_time = datetime.datetime.now() total_seconds = (script_end_time - script_start_time).total_seconds() print 'time elapsed: %d seconds' % total_seconds event_log.notes = notes event_log.save() print 'finished script' print 'post-import: number of tickers in SOL with tier 1: %d' % get_count_of_tier_one_tickers()
def handle(self, *args, **options): print 'starting script' event_log = DataHarvestEventLog() event_log.data_type = DATA_HARVEST_TYPE_EARNINGS_DATES event_log.notes = 'running' event_log.save() script_start_time = datetime.datetime.now() tickers_symbols_that_errored = set() tickers = Ticker.objects.all().order_by('ticker_symbol') for ticker in tickers: ticker.promised_coverage = None ticker_symbol = ticker.ticker_symbol if '-' in ticker_symbol: ticker_symbol = ticker_symbol.replace('-','.') print ticker_symbol try: earnings_announcement_date = get_earnings_announcement_date(ticker_symbol) print ticker_symbol, earnings_announcement_date ticker.earnings_announcement = earnings_announcement_date ticker.save() except Exception as e: ticker.earnings_announcement = None ticker.save() print "couldn't set earnings date", ticker_symbol, str(e), ticker.earnings_announcement tickers_symbols_that_errored.add(ticker_symbol) if ticker.earnings_announcement == None: print ticker.promised_coverage ticker.promised_coverage = 'Earnings date pending' print ticker.promised_coverage ticker.save() else: continue script_end_time = datetime.datetime.now() total_seconds = (script_end_time - script_start_time).total_seconds() print 'time elapsed: %d seconds' % total_seconds if tickers_symbols_that_errored: event_log.notes = 'errors: ' + ', '.join(tickers_symbols_that_errored) else: event_log.notes = 'no errors' event_log.save() print 'finished script' print 'tickers that errored: %d' % len(tickers_symbols_that_errored) print ', '.join(tickers_symbols_that_errored)
def handle(self, *args, **options): print 'starting script' event_log = DataHarvestEventLog() event_log.data_type = DATA_HARVEST_TYPE_MARKET_DATA event_log.notes = 'running' event_log.save() script_start_time = datetime.datetime.now() tickers = Ticker.objects.all().order_by('ticker_symbol') tickers_symbols_that_errored = set() count_tickers_successfully_updated = 0 # let's process 25 tickers at a time. one implementation: use a for loop, where each loop # processes tickers of index x up to index x+25 batch_size = 25 start_idx = 0 while start_idx < len(tickers): tickers_to_process = tickers[start_idx: start_idx+batch_size] symbols_as_list = ['\"'+t.ticker_symbol+'\"' for t in tickers_to_process] percent_changes_keyed_by_ticker_symbol = get_daily_percent_change(symbols_as_list) for ticker_to_process in tickers_to_process: try: ticker_to_process.daily_percent_change = percent_changes_keyed_by_ticker_symbol[ticker_to_process.ticker_symbol] ticker_to_process.save() count_tickers_successfully_updated += 1 except Exception as e: print "couldn't set daily percent change", ticker_to_process.ticker_symbol, str(e) tickers_symbols_that_errored.add(ticker_to_process.ticker_symbol) start_idx += batch_size script_end_time = datetime.datetime.now() total_seconds = (script_end_time - script_start_time).total_seconds() print 'time elapsed: %d seconds' % total_seconds notes = 'tickers updated: %d; ' % count_tickers_successfully_updated if tickers_symbols_that_errored: notes += 'errors: ' + ', '.join(tickers_symbols_that_errored) else: notes += 'no errors' event_log.notes = notes event_log.save() print 'finished script' print 'tickers that errored: %d' % len(tickers_symbols_that_errored) print ', '.join(tickers_symbols_that_errored)
def handle(self, *args, **options): print 'starting script' event_log = DataHarvestEventLog() event_log.data_type = DATA_HARVEST_TYPE_MARKET_DATA event_log.notes = 'running' event_log.save() tickers_symbols_that_errored = set() count_tickers_successfully_updated = 0 script_start_time = datetime.now() for ticker in Ticker.objects.all(): try: ticker.daily_percent_change = 0 ticker.save() count_tickers_successfully_updated += 1 except Exception as e: print "couldn't reset daily percent change", ticker.ticker_symbol, str(e) tickers_symbols_that_errored.add(ticker.ticker_symbol) script_end_time = datetime.now() total_seconds = (script_end_time - script_start_time).total_seconds() print 'time elapsed: %d seconds' % total_seconds notes = 'tickers reset: %d; ' % count_tickers_successfully_updated if tickers_symbols_that_errored: notes += 'errors: ' + ', '.join(tickers_symbols_that_errored) else: notes += 'no errors' event_log.notes = notes event_log.save() print 'finished script' print 'tickers that errored: %d' % len(tickers_symbols_that_errored) print ', '.join(tickers_symbols_that_errored)
def handle(self, *args, **options): print "Let's do this" event_log = DataHarvestEventLog() event_log.data_type = DATA_HARVEST_TYPE_SCORECARD_RECS event_log.notes = 'running' event_log.save() script_start_time = datetime.datetime.now() notes = '' # delete all previous ServiceTakes ServiceTake.objects.all().delete() for scorecard in Scorecard.objects.all(): print scorecard scorecard_name = scorecard.name url = base_url + scorecard_name print url response = urllib.urlopen(url).read() json_resp = json.loads(response) op = json_resp['OpenPositions'] for o in op: ticker_symbol = o['UnderlyingTickerSymbol'] if ticker_symbol=='': ticker_symbol=o['TickerSymbol'] print ticker_symbol # create a Ticker for this symbol if it doesn't exist matches = Ticker.objects.filter(ticker_symbol=ticker_symbol) if len(matches)==0: t = Ticker() t.ticker_symbol = ticker_symbol t.instrument_id = o['InstrumentId'] t.exchange_symbol = o['ExchangeSymbol'] t.percent_change_historical = 0.0 t.company_name = o['CompanyName'] t.save() else: t = matches[0] # create a ServiceTake st = ServiceTake() st.is_core = o['IsCore'] st.is_first = o['IsFirst'] st.is_newest = o['IsNewest'] st.action = o['Action'] st.is_present = True st.ticker = t st.scorecard = scorecard temp = o['OpenDate'] temp = temp.split('T')[0] st.open_date = datetime.datetime.strptime(temp, '%Y-%m-%d') st.save() #create scorecards_for_ticker and services_for_ticker service_takes_on_this_ticker = ServiceTake.objects.filter(ticker=t) scorecards_for_ticker = list() for st in service_takes_on_this_ticker: scorecards_for_ticker.append(st.scorecard.pretty_name) scorecards_for_ticker = set(scorecards_for_ticker) t.scorecards_for_ticker = ", ".join(scorecards_for_ticker) services_for_ticker = list() for st in service_takes_on_this_ticker: services_for_ticker.append(st.scorecard.service.pretty_name) services_for_ticker = set(services_for_ticker) t.services_for_ticker = ", ".join(services_for_ticker) t.save() notes = 'updated Ticker objects' script_end_time = datetime.datetime.now() total_seconds = (script_end_time - script_start_time).total_seconds() print 'time elapsed: %d seconds' % total_seconds event_log.notes = notes event_log.save() print 'finished script'
def handle(self, *args, **options): print 'starting script' event_log = DataHarvestEventLog() event_log.data_type = DATA_HARVEST_TYPE_BYLINE_META_DATA event_log.notes = 'running' event_log.save() script_start_time = datetime.now() notes = '' try: service_names_set_keyed_by_author_name, ticker_symbols_set_keyed_by_author_name = get_dictionaries_of_service_names_and_ticker_symbols_keyed_by_byline() for byline in service_names_set_keyed_by_author_name.keys(): services_for_byline = service_names_set_keyed_by_author_name[byline] tickers_for_byline = ticker_symbols_set_keyed_by_author_name[byline] services_for_byline = list(services_for_byline) tickers_for_byline = list(tickers_for_byline) services_for_byline.sort() tickers_for_byline.sort() services_for_byline = ', '.join(services_for_byline) tickers_for_byline = ', '.join(tickers_for_byline) matches_by_byline = BylineMetaData.objects.filter(byline=byline) if matches_by_byline: for m in matches_by_byline: m.services = services_for_byline m.tickers = tickers_for_byline m.save() else: b = BylineMetaData() b.byline = byline b.services = services_for_byline b.tickers = tickers_for_byline b.save() for bylineMetaData in BylineMetaData.objects.all(): if bylineMetaData.byline not in service_names_set_keyed_by_author_name.keys(): bylineMetaData.services = "nothing in the last year" bylineMetaData.tickers = "nothing in the last year" bylineMetaData.save() continue notes = 'updated %d BylineMetaData objects' % len(service_names_set_keyed_by_author_name.keys()) except Exception as e: print 'error', str(e) notes = "error: %s" % str(e) script_end_time = datetime.now() total_seconds = (script_end_time - script_start_time).total_seconds() print 'time elapsed: %d seconds' % total_seconds event_log.notes = notes event_log.save() print 'finished script'