def handle(self, *args, **options): print 'starting script' event_log = DataHarvestEventLog() event_log.data_type = DATA_HARVEST_TYPE_ARTICLES event_log.notes = 'running' event_log.save() script_start_time = datetime.datetime.now() try: status_message = get_articles() log_notes = status_message except Exception as e: print "error getting articles.", str(e) log_notes = str(e) script_end_time = datetime.datetime.now() total_seconds = (script_end_time - script_start_time).total_seconds() print 'time elapsed: %d seconds' % total_seconds event_log.notes = log_notes event_log.save() print 'finished script'
def handle(self, *args, **options):
print "Getting tier status"
event_log = DataHarvestEventLog()
event_log.data_type = DATA_HARVEST_TYPE_IMPORT_TIERS
event_log.notes = 'importing tier information'
event_log.save()
script_start_time = datetime.datetime.now()
notes = ''
print 'start delete'
for t in Ticker.objects.all():
t.tier_status = ''
t.tier = 0
t.save()
print 'end delete'
print 'starting tier import'
print 'pre-import: number of tickers in SOL with tier 1: %d' % get_count_of_tier_one_tickers()
#columns = defaultdict(list) # each value in each column is appended to a list
with open(TIERS_CSV, 'rU') as f:
reader = csv.DictReader(f) # read rows into a dictionary format
for row in reader: # read a row as {column1: value1, column2: value2,...}
ticker_symbol = str(row['ticker']).upper()
print ticker_symbol
service = str(row['service'])
try:
ticker = Ticker.objects.get(ticker_symbol = ticker_symbol)
ticker.tier = 1
if len(ticker.tier_status):
ticker.tier_status += ", %s" % service
else:
ticker.tier_status = service
print ticker_symbol, ticker.tier_status
ticker.save()
except:
print '%s tier status cannot be updated' % ticker_symbol
"""
EB work-in-progress note: the above solution does not convert tickers to U/C and remove commas like the old code did.
However, I plan to maintain the tiers spreadsheet so that there will only ever be one U/C ticker per field.
So I will leave that problem for now.
"""
notes = 'updated tier status'
script_end_time = datetime.datetime.now()
total_seconds = (script_end_time - script_start_time).total_seconds()
print 'time elapsed: %d seconds' % total_seconds
event_log.notes = notes
event_log.save()
print 'finished script'
print 'post-import: number of tickers in SOL with tier 1: %d' % get_count_of_tier_one_tickers()
def handle(self, *args, **options):
print 'starting script'
event_log = DataHarvestEventLog()
event_log.data_type = DATA_HARVEST_TYPE_EARNINGS_DATES
event_log.notes = 'running'
event_log.save()
script_start_time = datetime.datetime.now()
tickers_symbols_that_errored = set()
tickers = Ticker.objects.all().order_by('ticker_symbol')
for ticker in tickers:
ticker.promised_coverage = None
ticker_symbol = ticker.ticker_symbol
if '-' in ticker_symbol:
ticker_symbol = ticker_symbol.replace('-','.')
print ticker_symbol
try:
earnings_announcement_date = get_earnings_announcement_date(ticker_symbol)
print ticker_symbol, earnings_announcement_date
ticker.earnings_announcement = earnings_announcement_date
ticker.save()
except Exception as e:
ticker.earnings_announcement = None
ticker.save()
print "couldn't set earnings date", ticker_symbol, str(e), ticker.earnings_announcement
tickers_symbols_that_errored.add(ticker_symbol)
if ticker.earnings_announcement == None:
print ticker.promised_coverage
ticker.promised_coverage = 'Earnings date pending'
print ticker.promised_coverage
ticker.save()
else:
continue
script_end_time = datetime.datetime.now()
total_seconds = (script_end_time - script_start_time).total_seconds()
print 'time elapsed: %d seconds' % total_seconds
if tickers_symbols_that_errored:
event_log.notes = 'errors: ' + ', '.join(tickers_symbols_that_errored)
else:
event_log.notes = 'no errors'
event_log.save()
print 'finished script'
print 'tickers that errored: %d' % len(tickers_symbols_that_errored)
print ', '.join(tickers_symbols_that_errored)
def handle(self, *args, **options):
print 'starting script'
event_log = DataHarvestEventLog()
event_log.data_type = DATA_HARVEST_TYPE_MARKET_DATA
event_log.notes = 'running'
event_log.save()
script_start_time = datetime.datetime.now()
tickers = Ticker.objects.all().order_by('ticker_symbol')
tickers_symbols_that_errored = set()
count_tickers_successfully_updated = 0
# let's process 25 tickers at a time. one implementation: use a for loop, where each loop
# processes tickers of index x up to index x+25
batch_size = 25
start_idx = 0
while start_idx < len(tickers):
tickers_to_process = tickers[start_idx: start_idx+batch_size]
symbols_as_list = ['\"'+t.ticker_symbol+'\"' for t in tickers_to_process]
percent_changes_keyed_by_ticker_symbol = get_daily_percent_change(symbols_as_list)
for ticker_to_process in tickers_to_process:
try:
ticker_to_process.daily_percent_change = percent_changes_keyed_by_ticker_symbol[ticker_to_process.ticker_symbol]
ticker_to_process.save()
count_tickers_successfully_updated += 1
except Exception as e:
print "couldn't set daily percent change", ticker_to_process.ticker_symbol, str(e)
tickers_symbols_that_errored.add(ticker_to_process.ticker_symbol)
start_idx += batch_size
script_end_time = datetime.datetime.now()
total_seconds = (script_end_time - script_start_time).total_seconds()
print 'time elapsed: %d seconds' % total_seconds
notes = 'tickers updated: %d; ' % count_tickers_successfully_updated
if tickers_symbols_that_errored:
notes += 'errors: ' + ', '.join(tickers_symbols_that_errored)
else:
notes += 'no errors'
event_log.notes = notes
event_log.save()
print 'finished script'
print 'tickers that errored: %d' % len(tickers_symbols_that_errored)
print ', '.join(tickers_symbols_that_errored)
def handle(self, *args, **options): print 'starting script' event_log = DataHarvestEventLog() event_log.data_type = DATA_HARVEST_TYPE_MARKET_DATA event_log.notes = 'running' event_log.save() tickers_symbols_that_errored = set() count_tickers_successfully_updated = 0 script_start_time = datetime.now() for ticker in Ticker.objects.all(): try: ticker.daily_percent_change = 0 ticker.save() count_tickers_successfully_updated += 1 except Exception as e: print "couldn't reset daily percent change", ticker.ticker_symbol, str(e) tickers_symbols_that_errored.add(ticker.ticker_symbol) script_end_time = datetime.now() total_seconds = (script_end_time - script_start_time).total_seconds() print 'time elapsed: %d seconds' % total_seconds notes = 'tickers reset: %d; ' % count_tickers_successfully_updated if tickers_symbols_that_errored: notes += 'errors: ' + ', '.join(tickers_symbols_that_errored) else: notes += 'no errors' event_log.notes = notes event_log.save() print 'finished script' print 'tickers that errored: %d' % len(tickers_symbols_that_errored) print ', '.join(tickers_symbols_that_errored)
def handle(self, *args, **options):
print "Let's do this"
event_log = DataHarvestEventLog()
event_log.data_type = DATA_HARVEST_TYPE_SCORECARD_RECS
event_log.notes = 'running'
event_log.save()
script_start_time = datetime.datetime.now()
notes = ''
# delete all previous ServiceTakes
ServiceTake.objects.all().delete()
for scorecard in Scorecard.objects.all():
print scorecard
scorecard_name = scorecard.name
url = base_url + scorecard_name
print url
response = urllib.urlopen(url).read()
json_resp = json.loads(response)
op = json_resp['OpenPositions']
for o in op:
ticker_symbol = o['UnderlyingTickerSymbol']
if ticker_symbol=='':
ticker_symbol=o['TickerSymbol']
print ticker_symbol
# create a Ticker for this symbol if it doesn't exist
matches = Ticker.objects.filter(ticker_symbol=ticker_symbol)
if len(matches)==0:
t = Ticker()
t.ticker_symbol = ticker_symbol
t.instrument_id = o['InstrumentId']
t.exchange_symbol = o['ExchangeSymbol']
t.percent_change_historical = 0.0
t.company_name = o['CompanyName']
t.save()
else:
t = matches[0]
# create a ServiceTake
st = ServiceTake()
st.is_core = o['IsCore']
st.is_first = o['IsFirst']
st.is_newest = o['IsNewest']
st.action = o['Action']
st.is_present = True
st.ticker = t
st.scorecard = scorecard
temp = o['OpenDate']
temp = temp.split('T')[0]
st.open_date = datetime.datetime.strptime(temp, '%Y-%m-%d')
st.save()
#create scorecards_for_ticker and services_for_ticker
service_takes_on_this_ticker = ServiceTake.objects.filter(ticker=t)
scorecards_for_ticker = list()
for st in service_takes_on_this_ticker:
scorecards_for_ticker.append(st.scorecard.pretty_name)
scorecards_for_ticker = set(scorecards_for_ticker)
t.scorecards_for_ticker = ", ".join(scorecards_for_ticker)
services_for_ticker = list()
for st in service_takes_on_this_ticker:
services_for_ticker.append(st.scorecard.service.pretty_name)
services_for_ticker = set(services_for_ticker)
t.services_for_ticker = ", ".join(services_for_ticker)
t.save()
notes = 'updated Ticker objects'
script_end_time = datetime.datetime.now()
total_seconds = (script_end_time - script_start_time).total_seconds()
print 'time elapsed: %d seconds' % total_seconds
event_log.notes = notes
event_log.save()
print 'finished script'
def handle(self, *args, **options): print 'starting script' event_log = DataHarvestEventLog() event_log.data_type = DATA_HARVEST_TYPE_BYLINE_META_DATA event_log.notes = 'running' event_log.save() script_start_time = datetime.now() notes = '' try: service_names_set_keyed_by_author_name, ticker_symbols_set_keyed_by_author_name = get_dictionaries_of_service_names_and_ticker_symbols_keyed_by_byline() for byline in service_names_set_keyed_by_author_name.keys(): services_for_byline = service_names_set_keyed_by_author_name[byline] tickers_for_byline = ticker_symbols_set_keyed_by_author_name[byline] services_for_byline = list(services_for_byline) tickers_for_byline = list(tickers_for_byline) services_for_byline.sort() tickers_for_byline.sort() services_for_byline = ', '.join(services_for_byline) tickers_for_byline = ', '.join(tickers_for_byline) matches_by_byline = BylineMetaData.objects.filter(byline=byline) if matches_by_byline: for m in matches_by_byline: m.services = services_for_byline m.tickers = tickers_for_byline m.save() else: b = BylineMetaData() b.byline = byline b.services = services_for_byline b.tickers = tickers_for_byline b.save() for bylineMetaData in BylineMetaData.objects.all(): if bylineMetaData.byline not in service_names_set_keyed_by_author_name.keys(): bylineMetaData.services = "nothing in the last year" bylineMetaData.tickers = "nothing in the last year" bylineMetaData.save() continue notes = 'updated %d BylineMetaData objects' % len(service_names_set_keyed_by_author_name.keys()) except Exception as e: print 'error', str(e) notes = "error: %s" % str(e) script_end_time = datetime.now() total_seconds = (script_end_time - script_start_time).total_seconds() print 'time elapsed: %d seconds' % total_seconds event_log.notes = notes event_log.save() print 'finished script'