def update_artists(count=queries_per_run): refresh_date = datetime.date.today() - datetime.timedelta(artist_refresh_days) artists = Artist.select( AND(Artist.q.approved != None, OR(Artist.q.batch_updated == None, Artist.q.batch_updated < refresh_date)) ) count = min(artists.count(), count) email_txt = "" for artist in artists[:count]: try: lastfm_artist_update(artist) mbz_artist_update(artist) recording_artist_update(artist) artist.batch_updated = datetime.datetime.now() except: import traceback email_txt += "artist error: '%s'\n" % artist.name email_txt += traceback.format_exc() time.sleep(1) if email_txt: util.email("*****@*****.**", "BandRadar <*****@*****.**>", "artist errors", email_txt) return count
def submit(self, login, newpass, verpass, email): session=cherrypy.request.db if data.User.get_by_login(session, login) != None: cherrypy.session['message']="User with this login already exists." raise cherrypy.InternalRedirect("/login/newuser/") old_user=data.User.get_by_email(session, email) if old_user != None and old_user.status != '' and old_user.status is not None: cherrypy.session['message']="User with this e-mail already exists." raise cherrypy.InternalRedirect("/login/newuser/") if newpass!=verpass: cherrypy.session['message']="Passwords do not match." raise cherrypy.InternalRedirect("/login/newuser/") date=time.strftime("%Y-%m-%d %H:%M:%S") uid=uuid.uuid1().hex if old_user!=None: user=old_user user.login=login user.status='NEW' user.uuid=uid user.date=date user.update_passwd(newpass) else: user=data.User(email,login,newpass,'NEW',uid,date) session.add(user) util.email(email,'system/email/newuser.genshi',newlogin=login, uuid=uid) return util.render('system/msg/newuser.genshi',newlogin=login)
def nightly_task(): log.info("batch started") hub.threadingLocal = threading_local() hub.begin() last = BatchRecord.select(orderBy=BatchRecord.q.last_handled).reversed() if last.count(): last_rec = last[0] from_when = last_rec.last_handled else: from_when = datetime.date.today() last_handled = datetime.datetime.now() current = BatchRecord(first_handled=from_when, last_handled=last_handled) hub.commit() try: current.artists_updated = update_artists(queries_per_run) current.venues_updated = update_venues() cleanup_db() current.email_sent, current.artist_pings, current.venue_pings = send_email(from_when, last_handled) current.finished = datetime.datetime.now() hub.commit() except Exception, inst: import traceback hub.rollback() for admin in Group.by_group_name("admin").users: util.email( admin.email_address, "BandRadar <*****@*****.**>", "batch error", "Batch failed, Andy is on it!\n\n" + traceback.format_exc(), )
def process_fills(self): subj = "" if self.buys + abs(self.sells) > 200000 or self.buys+abs(self.sells) < 10: subj = "WARNING: " subj += "Fill Monitor: %d/%d Total: %d/%d " % (self.buys, self.sells, self.tot_buys, self.tot_sells) util.email(subj, self.fills_batch) self.reset_fills()
def handle_close(self): subject = "Sending halt message to all trade servers" msg = "Most likely cause: Trade server " + str(self.host) + ":" + str( self.port) + " went down." util.email(subject, msg) for chn in self.gtc.chns: if self == chn: self.gtc.chns.remove(chn) break self.close() self.gtc.halt()
def run(self): """Execute each piece""" if not self.skip: for piece in self.pieces: finish_file = self.batch_path(piece['name'], 'finish') if not os.path.isfile(finish_file): cmdline = '%s -nosplash < %s >& %s' piece_mfile = self.piece_path(piece) piece_log = self.log_path(piece) strf = '%Y%m%d %H:%M:%S' beg_time = time.strftime(strf) print('%s:%s:%s: begin %s' % (self.par_name, self.id, piece['name'], beg_time)) return_val = util.run_cmdline(cmdline % (self.mlab_path, piece_mfile, piece_log)) end_time = time.strftime(strf) print('%s:%s:%s: end %s' % (self.par_name, self.id, piece['name'], end_time)) v = 'Piece:%s\nBegan: %s\nEnded: %s\n' email_text = v % (piece['name'], beg_time, end_time) orig_file = self.piece_orig_path(piece) pdf_file = self.piece_pdf_path(piece) if return_val == 0: email_text += "Success\n" if os.path.isfile(orig_file): _, ext = os.path.splitext(orig_file) if ext == '.jpg': self.jpg2pdf(orig_file, pdf_file) if ext == '.ps': self.ps2pdf(orig_file, pdf_file) if return_val == 1: email_text += "Success, no .ps file was created\n" if return_val == 2: email_text += "Success, couldn't copy .ps file" if return_val == 3: email_text += "Error(s)\n" #TODO rescue ps if return_val in [0, 1, 2]: self.touch(finish_file) if os.path.isfile(piece_log): with open(piece_log, 'r') as f: email_text += f.read() else: email_text += "Couldn't open log file.\n" if self.email: subject_line = '%s:%s %s' % (self.project['name'], self.par_name, self.id) util.email(self.email['address'], self.email['to'], subject_line, self.email['server'], self.email['pw'], email_text, pdf_file) else: print("%s:%s:%s: skipping" % (self.par_name, self.id, piece['name']))
def submit_int(self, status, messages, pars, nobase=False): if status!='VALID': return render('system/form/error.genshi',app=self, errors=[m.message for m in messages],nobase=nobase) else: session=cherrypy.request.db req = self.store_query(session, pars) cherrypy.request.req_sub=True if(req.user.id<=0): email=False else: email=True util.email(req.user.e_mail,'system/email/scheduled.genshi',app=self,uuid=req.uuid) return render('system/msg/scheduled.genshi',app=self,uuid=req.uuid,email=email,nobase=nobase)
def submit(self, login, email): session=cherrypy.request.db user=data.User.get_by_login(session, login) if not( user != None and user.e_mail==email and (user.status=='OK' or user.status=='FORGOTTEN')): cherrypy.session['message']="User with these credentials does not exist." raise cherrypy.InternalRedirect("/login/forgotten/") user.status='FORGOTTEN' user.uuid=uuid.uuid1().hex # user.password='' util.email(email,'system/email/forgotten.genshi',newlogin=login, uuid=user.uuid) return util.render('system/msg/forgotten.genshi',newlogin=login)
def hourly_task(): hub.threadingLocal = threading_local() hub.begin() # notify admins of pending events added by users events_pending = Event.select(Event.q.approved == None) pending_count = events_pending.count() unnotified = events_pending.filter(Event.q.admin_notified == False) if unnotified.count(): for event in unnotified: event.admin_notified = True for admin in Group.by_group_name("admin").users: util.email( admin.email_address, "BandRadar <*****@*****.**>", "%d events in queue" % pending_count, "There are events in the pending queue.", ) hub.commit()
def _sendmail(ui, sender, recipients, msg): """send mail using sendmail.""" program = ui.config("email", "method") cmdline = "%s -f %s %s" % (program, util.email(sender), " ".join(map(util.email, recipients))) ui.note(_("sending mail: %s\n") % cmdline) fp = util.popen(cmdline, "w") fp.write(msg) ret = fp.close() if ret: raise util.Abort("%s %s" % (os.path.basename(program.split(None, 1)[0]), util.explainexit(ret)[0]))
def _sendmail(ui, sender, recipients, msg): '''send mail using sendmail.''' program = ui.config('email', 'method') cmdline = '%s -f %s %s' % (program, util.email(sender), ' '.join( map(util.email, recipients))) ui.note(_('sending mail: %s\n') % cmdline) fp = util.popen(cmdline, 'w') fp.write(msg) ret = fp.close() if ret: raise util.Abort('%s %s' % (os.path.basename( program.split(None, 1)[0]), util.explain_exit(ret)[0]))
def _sendmail(ui, sender, recipients, msg): '''send mail using sendmail.''' program = ui.config('email', 'method') cmdline = '%s -f %s %s' % (program, util.email(sender), ' '.join(map(util.email, recipients))) ui.note(_('sending mail: %s\n') % cmdline) fp = util.popen(cmdline, 'w') fp.write(msg) ret = fp.close() if ret: raise util.Abort('%s %s' % ( os.path.basename(program.split(None, 1)[0]), util.explain_exit(ret)[0]))
def email(report, date): subject = "Back office report for {}".format(date) body1 = ["Breaks:"] body2 = [ "Suggested cap adjustments:", "*WARNING* Verify correctness and make sure they are not due to splits or other reason." ] for line in report.split("\n"): if len(line) == 0: continue if line.startswith("#LBAD") or line.startswith("#RBAD"): continue tokens = line.strip().split("|") if tokens[2] != "00": body1.append(line) body2.append("FILL|{}|{}|{}|ABSOLUTE|{}, missing fill".format( tokens[1], tokens[5], int(tokens[4]) - int(tokens[3]), tokens[0])) body = "\n".join(body1) + "\n\n" + "\n".join(body2) util.email(subject, body)
fromDate, toDate)) #as a bonus, see which universe securities are unmapped for xx in newMappings: secid = int(xx[0]) died = xx[4] if (died is None or died > util.now()) and secid in uni: uni.remove(secid) if len(uni) > 0: report.append('') report.append("Orphan secids") for secid in uni: report.append( str(secid) + " " + database.getXrefFromSecid( "TIC", secid, util.now(), "compustat_idhist")) if args.email and (len(report) > 0): util.email("Compustat to Reuters mapping changes", "\n".join(report)) #print "\n".join(report) except: database.rollback() raise else: #database.rollback() database.commit() finally: if locked: database.releaseProcessedFilesLock()
p = subprocess.Popen('mkdir -p ' + output_dir, env=os.environ, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) retcode = p.wait() command = "{} -C {} ".format(tracker_bin, configuration_file) # add the debug-stream arguments command += "--debug-stream.path {} ".format(output_dir) # add DM's arguments command += "--dm.outfile {} --dm.symbol-file {} ".format( output_dir + "/cl_output.log", os.environ["RUN_DIR"] + "/universe") # add price-tracker's arguments command += "--out-file {} ".format(output_files_prefix) p = subprocess.Popen(command_line, env=os.environ, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) output = p.stdout.read() retcode = p.wait() if retcode != 0: util.email( "WARNING. Failed to run the following command line on {}:\n{}\nThe output:\n{}\nThe return code:{}" .format(os.environ['HOSTNAME'], command, output, retcode), 'price-tracker returned a non 0 return code') exit(-1)
def email(text): """:email: Any text. Extracts the first string that looks like an email address. Example: ``User <*****@*****.**>`` becomes ``[email protected]``. """ return util.email(text)
def idle(self): if MKT_CLOSE_TIME / 1000 < time.time(): util.debug("Exceeded market close time.") self.close() return now = datetime.datetime.utcnow() if now - self.last_idle_time < IDLE_TIME: return self.last_idle_time = now #sleep the first few times to make sure that guillotine initializes if self.guillotine_wait_times < NUM_WAIT_TIMES_IN_IDLE: self.guillotine_wait_times += 1 util.log('waiting for guillotine to initialize') return if len(self.gtc.symmap) == 0: util.log('not submitting, not connected') return #self._check_trading() listing = self.fs.list('orders\.[0-9_]+\.txt') if len(listing) == 0: return listing.sort(key=lambda x: x[0], reverse=True) orderfilename = listing[0][0] orderfileStat = os.stat(os.environ['RUN_DIR'] + '/orders/' + orderfilename) if math.trunc(orderfileStat.st_ctime) <= self.last_submitted_time: return util.log("Submitting trades in {}".format(orderfilename)) orderfile = open(os.environ['RUN_DIR'] + '/orders/' + orderfilename, 'r') reader = util.csvdict(orderfile) trades = set() allSymbols = filter(lambda key: self.gtc.symmap[key] != None, self.gtc.symmap.keys()) tradeSymbols = [] #gather trades currentSecid = None currentTicker = None buyDollars = 0 sellDollars = 0 try: for row in reader: #save this info for error messages currentSecid = None currentTicker = None qty = int(float(row['shares'])) if qty == 0: continue sec = int(row['secid']) currentSecid = sec ticker = sec2tic[sec] currentTicker = ticker if qty > 0: buyDollars += qty * float(row['oprice']) else: sellDollars += qty * float(row['oprice']) aggr = float(row['aggr']) orderID = long(row['orderid']) self.gtc.test_trade(ticker, qty, aggr, orderID) trades.add((sec, ticker, qty, aggr, orderID)) tradeSymbols.append(ticker) except KeyError: msg = 'submission failed, key error ' + str( currentSecid ) + ' ' + str( currentTicker ) + ". If the ticker is None, the problem happened when translating the secid to ticker. Else, the ticker was not in the guilottine symbol map" util.log(msg) util.email("[ERROR] livetrader " + orderfilename, msg) return #now actually send the trades. not that if we have a super=unexpected error, we will mark the file as done, so as not to screw things even more #by resending report = [] report.append("Buy $ = {:,.2f}".format(buyDollars)) report.append("Sell $ = {:,.2f}".format(sellDollars)) report.append("secid|ticker|qty|aggr|orderID") setZeroQtySymbols = [ sym for sym in allSymbols if sym not in tradeSymbols ] setZeroQtySymbols = filter(lambda key: key in tic2sec, setZeroQtySymbols) try: for trade in trades: report.append("{}|{}|{}|{}|{}".format(trade[0], trade[1], trade[2], trade[3], trade[4])) self.gtc.trade(trade[1], trade[2], trade[3], trade[4]) for sym in setZeroQtySymbols: report.append("{}|{}|{}|{}|{}".format(tic2sec[sym], sym, 0, 1.0, -1)) self.gtc.trade(sym, 0, 1.0, -1) except KeyError: msg = 'Submission half-done due to a super-unexpected key error. I have not clue why we reached this point' util.log(msg) util.email("[ERROR] livetrader " + orderfilename, msg) else: util.email( "livetrader submitted successfully {} trades from file {}". format(str(len(trades)), orderfilename), "\n".join(report)) # self.last_submitted = tradefilename self.last_submitted_time = math.trunc(orderfileStat.st_ctime) self.last_trade_time = datetime.datetime.utcnow() util.log('submitted %s' % orderfilename)
def handle_error(self, err): subject = "ERROR: Guillotine Error Reported" msg = "ERR %(error)s = %(reason)s (msg %(message-name)s | field %(field)s | sym %(symbol)s" % err util.email(subject, msg)
def process_quotes(self): secs = sorted(self.quotes.keys()) quote_cnt = 0 bad_sprd = 0 bad_mid = 0 bad_sz = 0 sprd_tot = 0 sprd_bps_tot = 0 bid_sz_tot = 0 ask_sz_tot = 0 error_lines = "" for sec in secs: msg = self.quotes[sec] # print "%s|%.4f|%.4f|%d|%d" % (sec, msg['bid'], msg['ask'], msg['bid-size'], msg['ask-size']) quote_cnt += 1 mid = (msg['ask'] + msg['bid']) / 2 if mid <= 0 or mid > 1000: bad_mid += 1 error_lines += "ERROR: bad mid: %s %.2f\n" % (sec, mid) continue spread = msg['ask'] - msg['bid'] if spread <= 0: bad_sprd += 1 error_lines += "ERROR: bad spread %s %.4f\n" % (sec, spread) continue sprd_bps = 10000 * (spread / mid) if sprd_bps > 50: bad_sprd += 1 error_lines += "ERROR: bad spread %s %.2f\n" % (sec, sprd_bps) continue sprd_tot += spread sprd_bps_tot += sprd_bps if msg['bid-size'] < 0 or msg['ask-size'] < 0: bad_sz += 1 error_lines += "ERROR: bad bid/ask size %s %d %d\n" % ( sec, msg['bid-size'], msg['ask-size']) continue bid_sz_tot += msg['bid-size'] ask_sz_tot += msg['ask-size'] email_msg = "" email_subj = "" if quote_cnt < 1000 or (bad_mid + bad_sprd + bad_sz) > (0.05 * quote_cnt): email_subj = "ERROR: " email_subj += "Quote Monitor: %d quotes" % quote_cnt email_msg += "BAD mid/spread/size: %d/%d/%d\n" % (bad_mid, bad_sprd, bad_sz) if (quote_cnt - bad_sprd - bad_mid > 0): email_msg += "AVG SPREAD: %.4f/%.2f (bps)\n" % ( (sprd_tot / (quote_cnt - bad_sprd - bad_mid)), (sprd_bps_tot / (quote_cnt - bad_sprd - bad_mid))) if quote_cnt > 0: email_msg += "AVG bidsz/asksz: %d/%d\n" % ( (bid_sz_tot / quote_cnt), (ask_sz_tot / quote_cnt)) email_msg += "\n" email_msg += error_lines util.email(email_subj, email_msg)
ex_open = long(output.split("|")[0]) ex_close = long(output.split("|")[1]) #exchange is closed, or before 9.36am if ex_open == 0 or ex_close == 0 or nowMillis < ex_open + 6L * 60L * 1000L or nowMillis >= ex_close: exit(0) #read the bar file fname = "all.txt.live" if args.bars else "bars_v2.txt.live" file = open("{}/bars/{}/{}".format(os.environ["DATA_DIR"], nowDate, fname), "r") bars = file.readlines() file.close() close_ts = 0L for bar in bars: tokens = bar.strip().split("|") ts = long(tokens[2]) close_ts = max(close_ts, ts) lag = (nowMillis - close_ts) if lag > 6 * 60 * 1000: if close_ts > 0: msg = "WARNING: We have not received new {} in {:.1f} minutes!".format( "bars" if args.bars else "superbars", lag / 60.0 / 1000.0) else: msg = "WARNING: We have not received new {} in, like, forever!".format( "bars" if args.bars else "superbars") util.email(msg, "You heard me. Go fix this!")
def send_email(start, finish): conn = hub.getConnection() users_to_email = set() # # Gather results for tracked artists. Notify if events added since last update # artist_email = {} results = conn.queryAll( """ select u.id, e.name, e.date, v.name from user_acct u, artist_user_acct au, artist a, artist_event ae, event e, venue v where u.event_email is true and u.id = au.user_acct_id and a.id = au.artist_id and a.id = ae.artist_id and ae.event_id = e.id and e.venue_id = v.id and e.created >= '%s' and e.created < '%s' and e.date >= CURRENT_DATE """ % (start, finish) ) for user_id, name, date, venue_name in results: evt_list = artist_email.get(user_id, list()) evt_list.append((unicode(name, "utf-8"), date, unicode(venue_name, "utf-8"))) artist_email[user_id] = evt_list users_to_email.add(user_id) # # Gather results for tracked events. Notify if they're today # event_email = {} results = conn.queryAll( """ select e.id, e.name, att.user_id, v.name from event e, attendance att, venue v where e.date = CURRENT_DATE and e.id = att.event_id and e.venue_id = v.id """ ) for event_id, event_name, user_id, venue_name in results: evt_list = event_email.get(user_id, list()) evt_list.append((unicode(event_name, "utf-8"), venue_name)) event_email[user_id] = evt_list users_to_email.add(user_id) # Gather results for tracked venues. Once a week. # why do this here, instead of having a separate scheduled function called? # because we want to put both artist and venue notifications in the same email. venue_email = {} if finish.isoweekday() == 4: results = conn.queryAll( """ select u.id, e.name, e.date, v.name from user_acct u, venue v, event e, user_acct_venue uv where u.event_email is true and e.venue_id = v.id and u.id = uv.user_acct_id and uv.venue_id = v.id and e.date >= CURRENT_DATE order by e.date """ ) for user_id, event_name, date, venue_name in results: venue_dict = venue_email.get(user_id, dict()) venue = venue_dict.get(venue_name, list()) venue.append((unicode(event_name, "utf-8"), date)) venue_dict[venue_name] = venue venue_email[user_id] = venue_dict users_to_email.add(user_id) for id in users_to_email: import pkg_resources u = UserAcct.get(id) event_text = "" for event_name, venue_name in event_email.get(id, list()): event_text += u"%s, at %s\n" % (event_name, venue_name) if event_text: hdr_txt = "These events you want to go to are TONIGHT!\n\n" event_text = hdr_txt + event_text + "\n" artist_text = "" for event_name, date, venue_name in artist_email.get(id, list()): artist_text += u"%s, %s at %s\n" % (event_name, date, venue_name) if artist_text: hdr_txt = "Newly added shows featuring artists you are tracking:\n\n" artist_text = hdr_txt + artist_text + "\n" venue_text = "" for venue_name, event_list in venue_email.get(id, dict()).iteritems(): venue_text += venue_name + "\n" + ("-" * len(venue_name)) + "\n" for name, date in event_list: venue_text += u"%s: %s\n" % (date, name) venue_text += "\n" if venue_text: hdr_txt = "Upcoming shows at the venues you are tracking:\n\n" venue_text = hdr_txt + venue_text text = event_text + artist_text + venue_text user_url = "http://bandradar.com/users/%s" % u.user_name msg_to = u.email_address msg_from = "BandRadar Events <*****@*****.**>" body = pkg_resources.resource_string(__name__, "templates/new_event_email.txt") body = body % {"text": text, "user_url": user_url} util.email(msg_to, msg_from, "BandRadar upcoming events", body) return (len(users_to_email), len(artist_email), len(venue_email))
xx[3])).strftime("%Y%m%d") if xx[3] is not None else None report.append("{} {}, {}, {}, {}, {}, {}".format( highlight, operation, secid, ticker, barraid, fromDate, toDate)) #as a bonus, see which universe securities are unmapped for xx in newMappings: secid = int(xx[0]) died = xx[3] if (died is None or died > util.now()) and secid in uni: uni.remove(secid) if len(uni) > 0: report.append('') report.append("Orphan secids") for secid in uni: report.append( str(secid) + " " + database.getXrefFromSecid( "TIC", secid, util.now(), "compustat_idhist")) if args.email and (len(report) > 0): util.email("Compustat to Barra mapping changes", "\n".join(report)) except: database.rollback() raise else: database.commit() finally: if locked: database.releaseProcessedFilesLock()
lines_to_mail = "" filenames = os.popen("ls {}/*.log".format(os.environ["LOG_DIR"])).readlines() for filename in filenames: filename = filename[0:-1] if os.path.basename(filename).startswith("."): continue new_size = os.path.getsize(filename) if not sizes.has_key(filename): if first_loop: sizes[filename] = new_size else: sizes[filename] = 0 if new_size != sizes[filename]: f = file(filename) f.seek(sizes[filename]) found_errors = False for line in f: if re.search("SEVERE", line): lines_to_mail += filename+": "+line found_errors = True f.close() sizes[filename] = new_size if found_errors: lines_to_mail += "\n" if len(lines_to_mail) > 0: util.email("Log Checker", lines_to_mail) lines_to_mail= "" time.sleep(5 * 60) first_loop = False if time.time() - start > 60*60*12: sys.exit();
parser.add_argument("--qcc", action="store_const", const=True, dest="qcc", default=False) args = parser.parse_args() if not args.debug: util.set_log_file("na") newdb.init_db() database = newdb.get_db() if args.qcc: qcc() if args.xref_changes: report = xrefChanges2() if args.debug: print report elif report is not None: util.email("Xref changes", report) if args.processed_files: if args.debug: print newProcessedFiles() else: output = newProcessedFiles() if output is not None: util.email("New processed files", output)