def __init__(self, dir, dumpDir, logName): self.buggy = False self.fields = None self.dir = dir self.dumpDir = dumpDir self.logger = createLogger(logName) self.logger.debug("FSObject ctor") self.newfile = None self.updFile = None self.notNulls = [] self.primaryKey = None self.primaryKeyOnCSV = None self.newCSVLoad = None self.updSQLLoad = None self.updSQLDiscardFields = [] self.modTable = None self.managingFSField = None self.parentFSField = None self.dumpFile = None self.updErrs = [] self.addErrs = [] self.mloaded = {} self.csvBase = None self.changedIDs = None self.postBaseLoadErrs = []
def main(runtype, csvfile, url, userid, password, ): logger = utils.createLogger("p2.py") if runtype == 1: rt = fpn_type_1 elif runtype == 2: rt = fpn_type_2 else: r = utils.getValidFPNs() rt = [x for x in r if x not in fpn_type_1 and x not in fpn_type_2] logger.debug(str(rt)) w = " and FPN_Subscriptions__c INCLUDES ("+",".join(["'%s'"%x for x in rt])+")" where = " where FPN_Subscriptions__c != '' and FPN_Subscriptions__c != 'Does Not Want' and FPN_Subscriptions__c != 'Not Authorized - see JMC'"+w logger.debug("WHERE: "+where) fout, tfile = utils.openTempFile() fout.close() logger.debug("TEMP FILE "+tfile) p1.main(tfile, url, userid, password, where) splitter.main(rt, tfile, csvfile) return 0
def convert(xlsfile, csvfile, fields, validate = True, badRecs = None): logger = createLogger("xls2csv") logger.debug("Entered convert") xls = load(xlsfile, fields) if validate: validateAllRFieldTypes(xls) writer = csv.writer(open(csvfile, 'w'), delimiter=',',\ quotechar='"', quoting=csv.QUOTE_MINIMAL) names = xls.fields.names() writer.writerow(names) ndx = 0 for record in xls.records(): lr = [] for m in names: lr.append(csvFix(record[m])) ndx = ndx + 1 if badRecs is None or (ndx+1 not in badRecs): try: writer.writerow(lr) except UnicodeEncodeError: print lr print ndx raise else: logger.debug("Discarding %d"%ndx)
def load(xlsname, fields): global logger logger = createLogger("xlsloader") logger.debug("Entered xlsloader.load") xls = XLSDataSource(xlsname) xls.initialize() xls.fields = FieldList(fields) return xls
def __init__(self): b = utils.get_sp_cred() self.logger = utils.createLogger("spsoapclient.py") self.soap_client = suds.client.Client(b[0], location=b[1]) auth = self.soap_client.factory.create("tns:authentication") auth.username = b[2] auth.password = b[3] self.export_job_id = b[4] self.soap_client.set_options(soapheaders=auth)
def main(err_file, loginURL, username, password): logger = utils.createLogger("probe.py") sf.setLogger(logger) df = open(err_file, "a") try: sf.login(username, password, loginURL, True) except sf.SFException, e: df.write(str(e)+"\n") sys.exit(1)
def csvloader(fin): logger = utils.createLogger("csvloader") logger.debug("Entered csvloader") m = {} rdr = csv.reader(open(fin), delimiter=',', quotechar='"') kdebug = 0 for row in rdr: kdebug = kdebug + 1 if row[1].strip().lower() != "": m[row[1].strip().lower()] = [row[4].strip(), 0] return m
def main(oper, fn, fo, loginURL, username, password): print oper, fn, fo rcount = 0 fcount = 0 res_msg = "" logger = utils.createLogger("upsert") sf.setLogger(logger) op = open(fo, "w") lx = sf.login(username, password, loginURL) if not lx: return site = lx[0] sessionid = lx[1] jobid = sf.createJob(oper, "Contact", "CSV", site, sessionid) logger.debug( "JOBID: "+jobid) state = sf.checkJobStatus(jobid, site, sessionid) logger.debug("JOB state: "+state) batchid = sf.submitUpdate(jobid, site, sessionid, fn) logger.debug("BATCH "+batchid) while True: # loop till we complete or error out bat = sf.checkBatch(jobid, batchid, site, sessionid) logger.debug("BATCH state "+ str(bat)) batchState = str(bat["state"]) if batchState == "Completed": rcount = int(str(bat["numberRecordsProcessed"])) fcount = int(str(bat["numberRecordsFailed"])) logger.info("NUMBER OF RECORDS PROCESSED %d"%rcount) logger.info("NUMBER OF RECORDS FAILED %d"%fcount) r = sf.getUpdateResults(jobid, batchid, site, sessionid) op.write(r) break if batchState == "Failed": logger.error("QUERY FAILED") res_msg = str(bat["stateMessage"]) logger.error(res_msg) break import time time.sleep(sleepTime) logger.debug("JOB state AFTER: "+state) sf.closeJob(jobid, site, sessionid) logger.debug("JOB state AFTER CLOSE: "+state) return res_msg, rcount, fcount
def __init__(self, name, port=""): super().__init__() self.name = name self.port = port self.connected = False self.className = type(self).__name__ loggerName = self.className + ' (' + self.name + ')' self.logger = utils.createLogger(loggerName) self.createConnectBox() self.createDataReceiveBoxes() mainLayout = QHBoxLayout() mainLayout.addWidget(self.connectBox) mainLayout.addWidget(self.dataReceiveBox) self.setLayout(mainLayout) self.setTitle(loggerName)
def main(fpns, inname, outname): logger = utils.createLogger("splitter.py") logger.debug("inname: "+inname+" outname "+outname) fout = csv.writer(open(outname, "w"), delimiter=',', quotechar='"',quoting=csv.QUOTE_ALL) rdr = csv.reader(open(inname), delimiter=',', quotechar='"') k = 0 for row in rdr: rt = [x.strip() for x in row] if len(rt) == 5: if k > 0: rt[4] = ";".join([x.strip() for x in rt[4].strip().split(";") if x in fpns]) # logger.debug(rt); if (rt[4] != ""): # VERY DIRTY HACK for FPN Digest called as EU Free rt[4] = rt[4].replace("EU Digest", "EU Free") fout.writerow(rt) k = k + 1
def main(fn, loginURL, username, password, whereClause = None): logger = utils.createLogger("p1.py") sf.setLogger(logger) lx = sf.login(username, password, loginURL) if not lx: return site = lx[0] sessionid = lx[1] jobid = sf.createJob("query", "Contact", "CSV", site, sessionid) logger.debug( "JOBID: "+jobid) sfSQL = "select id, Email, firstname, lastname, FPN_Subscriptions__c,Stormpost_Reason_Date__c,Stormpost_Reason__c from contact" if whereClause is not None: sfSQL = sfSQL + " " + whereClause logger.debug("SQL : "+sfSQL) batchid = sf.submitSelect(jobid, site, sessionid, sfSQL) logger.debug("BATCH "+batchid) state = sf.checkJobStatus(jobid, site, sessionid) logger.debug("JOB state: "+state) while True: # loop till we complete or error out bat = sf.checkBatch(jobid, batchid, site, sessionid) logger.debug("BATCH state "+ str(bat)) batchState = str(bat["state"]) if batchState == "Completed": rcount = int(str(bat["numberRecordsProcessed"])) logger.info("NUMBER OF RECORDS %d"%rcount) r = sf.getResults(jobid, batchid, site, sessionid) op = open(fn, "w").write(r) break if batchState == "Failed": logger.error("QUERY FAILED") break import time time.sleep(sleepTime) logger.debug("JOB state AFTER: "+state) sf.closeJob(jobid, site, sessionid) logger.debug("JOB state AFTER CLOSE: "+state)
def diff(now, prev): global logger logger = utils.createLogger("diff") logger.debug("Entered diff") validFPNs = getValidFPNs() ### VERY DIRTY HACK FOR FPN Free validFPNs.append("EU Free") mnow = csvloader.csvloader(now) logger.debug("LEN of now: %d"%len(mnow)) mprev = csvloader.csvloader(prev) logger.debug("LEN of prev: %d"%len(mprev)) adds = {} deletes = {} for k in mnow.keys(): try: p = mprev[k] p[1] = 1 n = mnow[k] n[1] = 1 prevFPNS = p[0] nowFPNS = n[0] if prevFPNS != nowFPNS: logger.debug("DIFFERENT "+k) logger.debug("PREV "+prevFPNS) logger.debug("NOW "+nowFPNS) newFPNS = getInANotInB(nowFPNS, prevFPNS) xFPNS = getInANotInB(prevFPNS, nowFPNS) if len(newFPNS) > 0: putInMap(validFPNs, adds, k, newFPNS) if len(xFPNS) > 0: putInMap(validFPNs, deletes, k, xFPNS) except KeyError, e: fpns = mnow[k][0].strip() if fpns != "": logger.debug("contact %s with %s is new"%(k, fpns)) putInMap(validFPNs, adds, k, fpns.split(";"))
def __init__(self, channel_objs): super().__init__() self.channels = channel_objs self.numChannels = len(self.channels) className = type(self).__name__ self.logger = utils.createLogger(className) self.createMainSettingsBox() self.createChannelSettingsBox() self.createDataFileBox() self.createChannelGroupBox() w = max(self.mainSettingsBox.sizeHint().width(), self.channelSettingsBox.sizeHint().width(), self.dataFileBox.sizeHint().width()) + 5 self.mainSettingsBox.setFixedWidth(w) self.channelSettingsBox.setFixedWidth(w) self.dataFileBox.setFixedWidth(w) col1 = QVBoxLayout() col1.addWidget(self.mainSettingsBox) col1.addWidget(self.channelSettingsBox) col1.addWidget(self.dataFileBox) self.mainLayout = QHBoxLayout() self.mainLayout.addLayout(col1) self.mainLayout.addWidget(self.channelGroupBox) self.setWindowTitle('Olfactometer Prototype & Accessory Control') self.central_widget = QWidget() self.central_widget.setLayout(self.mainLayout) self.setCentralWidget(self.central_widget) # Menu Bar self.createMenuBar() self.resize(self.sizeHint())
def __init__(self, name, port=""): super().__init__() self.name = name self.port = port className = type(self).__name__ loggerName = className + ' (' + self.name + ')' self.logger = utils.createLogger(loggerName) self.logger.debug('Creating module for %s', loggerName) self.createConnectBox() self.createSettingsBox() col1 = QVBoxLayout() col1.addWidget(self.connectBox) col1.addWidget(self.settingsBox) self.createDataReceiveBoxes() mainLayout = QHBoxLayout() mainLayout.addLayout(col1) #mainLayout.addWidget(self.connectBox) mainLayout.addWidget(self.dataReceiveBox) self.setLayout(mainLayout) self.setTitle(self.name)
import re import warnings from pathlib import Path import numpy as np import scipy.sparse as sp import spacy from sklearn.feature_extraction.text import CountVectorizer, ENGLISH_STOP_WORDS from utils import createDirs, createLogger INPUT_DIR = Path(r'../data/aclImdb/') OUTPUT_DIR = Path(r'../data/clean/') LOG_DIR = Path(r'../logs/clean/') logger = createLogger(LOG_DIR, "clean") logger.info("Logger created, logging to %s" % LOG_DIR.absolute()) def getData(path): """ Get IMDb movie reviews and corresponding labels. Inputs: - path (str): IMDb movie review directory Outputs: ([str], [int]): list of movie reviews, list of sentiment labels """ texts, labels = [], []
import utils import pytz import sys import random from datetime import datetime logger = utils.createLogger("scheduling") config = utils.loadConfig() scheduling_config = config["scheduling"] timezone_name = scheduling_config["local_timezone"] try: timezone = pytz.timezone(timezone_name) except pytz.UnknownTimeZoneError: logger.error("timezone {} not found in tz data. make sure it is in the list - https://en.wikipedia.org/wiki/List_of_tz_database_time_zones") sys.exit(1) working_days = scheduling_config["working_days"] min_days_per_week = scheduling_config.get("min_days_per_week", 1) if min_days_per_week == -1: min_days_per_week = len(working_days) if min_days_per_week < 0 or min_days_per_week > len(working_days): logger.error("min_days_per_work should be in [0, len(working_days)] or -1") sys.exit(1) working_hours = scheduling_config["working_hours"] if len(working_hours) == 0: logger.error("working hours must contain atleast one start-end period") sys.exit(1) def getRandomHour(seed): total_duration = sum([(period["end"] - period["start"]) for period in working_hours]) rng = random.Random(seed) random_offset = rng.random() * total_duration
import utils import csv import os from mailer import sendMail send_emails_to = ["*****@*****.**"]#, # "*****@*****.**", # "*****@*****.**"] def usage(): print "usage spclearlist_and_add listname export_list_job_id csvtoadd tmpdir" # csvtoremove is Recips.Address,Recips.SourceDesc, # csvtoadd is email logger = utils.createLogger("spclearlist_and_add") def main(listname, job_id, csvtoadd, tmpdir): spf = spsoapclient.spsoapclient() m = spf.get_lists_as_map() try: list_id = m[listname] except KeyError: logger.error("Cannot find list "+listname) return -1 logger.debug("LIST id "+str(list_id)) # dump all users - we need this info so we know if we have to create a recip for the adds
from joblib import dump, load from sklearn.decomposition import LatentDirichletAllocation from sklearn.model_selection import train_test_split from sklearn.pipeline import make_pipeline from sklearn.preprocessing import StandardScaler from evaluate import evaluate from logistic_regression import LogisticRegressionVal from utils import createLogger, createDirs, loadClean, writeResults, \ preprocessClfParser INPUT_DIR = Path(r'../data/clean') OUTPUT_DIR = Path(r'../logs/models') LOG_DIR = Path("../logs/pipeline") logger = createLogger(LOG_DIR, "lda_clf") logger.info("Logger created, logging to %s" % LOG_DIR.absolute()) K = np.arange(1, 31) * 5 def LDA(train_size, random_state): """ Classification pipeline with LDA preprocessing. Inputs: - train_size (int): number of training samples. - random_state (int): seed for random number generators Output: (None)
from base64 import b64encode import re import json import hmac import hashlib import requests import time import utils import sys logger = utils.createLogger("zoom") ZOOM_USERS_API_URL = "https://api.zoom.us/v2/users/" ZOOM_CREATE_MEETING_API_URL = "https://api.zoom.us/v2/users/{userId}/meetings" config = utils.loadConfig() if "zoom" not in config: logger.error("zoom key not found in the config json") sys.exit(1) zoom_conf = config["zoom"] if "api_key" not in zoom_conf: logger.error("api_key not found in zoom config") sys.exit(1) ZOOM_API_KEY = zoom_conf["api_key"] if "api_secret" not in zoom_conf: logger.error("api_secret not found in zoom config") sys.exit(1) ZOOM_API_SECRET = zoom_conf["api_secret"] def encode(text):
import tornado.locks import json import jwt import aiomysql import bcrypt import datetime from wx.sign import Sign from wechatpy.utils import check_signature from wechatpy.exceptions import InvalidSignatureException from utils import createLogger, config from auth import authenticated from qiniu import Auth # Configure logging logger = createLogger(__name__) class NoResultError(Exception): pass async def maybe_create_tables(db): try: async with db.acquire() as conn: async with conn.cursor() as cur: await cur.execute("SELECT COUNT(*) FROM kt_posts LIMIT 1") await cur.fetchone() except aiomysql.ProgrammingError: with open("schema.sql") as f: schema = f.read()
import sys import os.path import utils import spsfsync_runner import datetime import spsoapclient import diff logger =utils.createLogger("spsfsync_starter") def main(): fpns_to_sync = [x for x in [x.strip() for x in open(os.path.join(utils.getScriptPath(), "fpns_to_sync.txt")).read().\ split("\n")] if x[0] != "#"] # there! # logger.debug("WILL SYNC "+",".join(fpns_to_sync)) if len(fpns_to_sync) == 0: logger.error("Nothing to sync") return -1 # validate legacy list -- used by diff validFPNs = diff.getValidFPNs() #logger.debug("Valid FPNS "+str(validFPNs)) logger.debug("connecting to sp") spsoap = spsoapclient.spsoapclient() sp_list_map = spsoap.get_lists_as_map() def _check_lists(): bad = False for ed in fpns_to_sync:
def main(fin, fout, manifest_file, rmap, debug_max = None): logger = utils.createLogger("sp_get_status.py") if debug_max is not None: debug_max = int(debug_max) logger.debug("debug_max "+str(debug_max)) logger.debug("OUTFILE "+fout) csv_fileo = open(fout, 'w') csvw = csv.writer(csv_fileo, delimiter=',', quotechar='"', quoting=csv.QUOTE_NONNUMERIC) manifest_csv = csv.writer(open(manifest_file, "w"), delimiter=',', quotechar='"', quoting=csv.QUOTE_NONNUMERIC) manifest_csv.writerow(["ID", "Email", "From", "To"]) csvw.writerow(["ID", "Stormpost_Reason__c", "Stormpost_Reason_Date__c"]) date = datetime.datetime.now() sdate = "%02d-%02d-%04d"%(date.month, date.day, date.year) if recips_exp_csv_file is None: logger.debug("exporting recips") recips_exp_csv_file = os.path.join(tempdir, "recips_file.csv") rc = spsoap.runExportRecipList(recips_exp_csv_file) if rc: return rc fle = csv.reader(open(fin), delimiter=",", quotechar='"') first = True logger.debug("START") k = 0 upds = 0 for row in fle: if first: first = False continue k = k + 1 if debug_max is not None: # logger.debug("count: %d - %d"%(k, debug_max)) if k > debug_max: break if k%100 == 0: logger.debug("RECNO %d"%k) id = row[0] email = row[1] oldstat = row[6] try: sp_id, sval = rmap[email] if sval not in valid_status: str = "bad status for %s -- %s"%(email, sval) logger.error(str) raise Exception("bad status for %s -- %s"%(email, sval)) else: if oldstat != sval: # logger.debug("email: %s changed status from: %s to %s"%\ # (email, oldstat, sval)) csvw.writerow([id, sval, sdate]) manifest_csv.writerow([id, email, oldstat, sval]) upds = upds + 1 csv_fileo.flush() except KeyError: logger.info("Email %s is not in SP"%email) logger.debug("DONE") return (0, upds)
from smtplib import SMTP import os from email.MIMEMultipart import MIMEMultipart from email.MIMEBase import MIMEBase from email.MIMEText import MIMEText from email.Utils import COMMASPACE, formatdate from email import Encoders import utils smtphost = "ad-vm-data1" smtpport = 25 #admin_email = ["*****@*****.**", "*****@*****.**", "*****@*****.**"] admin_email = ["*****@*****.**", "*****@*****.**"] logger = utils.createLogger("mailer.py") def sendMail(rcpt, subject, body, files=[]): send_from = "*****@*****.**" msg = MIMEMultipart() msg['From'] = send_from if rcpt is None: rcpt = admin_email msg['To'] = COMMASPACE.join(rcpt) msg['Date'] = formatdate(localtime=True) msg['Subject'] = subject msg.attach( MIMEText(body) )
#!/usr/bin/env python3 # should be added to a cronjob firing everyday # e.g at 10 AM IST everyday # 30 4 * * * python3 path/main.py >> path/cronjob.log 2>&1 import time import random import utils import bot import scheduling logger = utils.createLogger("main") if __name__ == "__main__": logger.info("starting water cooler bot...") shouldScheduleToday, delay_seconds = scheduling.getTodaySchedule() if shouldScheduleToday: logger.info("sleeping for {:.1f} seconds".format(delay_seconds)) time.sleep(delay_seconds) bot.ping() logger.info("sent message to slack channel successfully!")
def main(sf_csv_file, sp_email_map, maxupds, sendto): logger = utils.createLogger("spsfsync.py") b = utils.getSFSiteCred() workdir = os.path.join(tempdir, utils.temp_name("sfupdworkdir")) os.mkdir(workdir) rc,upds, splits, manifest_map = \ __create_sf_updates(sf_csv_file, workdir, sp_email_map, maxupds) if rc != 0 or upds == 0: if rc != 0: logger.error("sp_get_status returned with error %d"%rc) else: logger.info("There were no updates") return rc logger.debug("THERE ARE %d updates and %d splits"%(upds, len(splits))) # loop splits here # then cat the logfile cgood = 0 cbad = 0 goodfile = os.path.join(tempdir, utils.temp_name("goodups.txt")) badfile = os.path.join(tempdir, utils.temp_name("badupds.txt")) logger.debug("GOODFILE "+goodfile) good = open(goodfile, "w") bad = open(badfile, "w") updlogfile = os.path.join(tempdir, utils.temp_name("spsynclogfile.csv")) # yes reuse ktest = 0 for spl in splits: rc = sfupdspstatus.main(b[0], b[1], b[2], spl, updlogfile) csv_spl = csv.reader(open(spl)) up_file = open(updlogfile) csv_log = csv.reader(up_file) first = True for row_upd in csv_spl: row_log = csv_log.next() if first: first = False continue lid = row_upd[0] try: (email, fromv) = manifest_map[lid] cto = row_upd[1] success = row_log[1].lower() error = row_log[3] if success == "true": cgood = cgood + 1 good.write(email+" changed from "+fromv+" to "+cto+"\n") good.flush() else: cbad = cbad + 1 bad.write(email+" failed to update from "+fromv+" to "+cto+" "+error+"\n") bad.flush() except KeyError: logger.error("ID not found "+lid) up_file.close() ktest = ktest + 1 # if ktest > 4: break ## remove this good.close() bad.close() att = [] att.append((goodfile, "goodups.txt")) if cbad > 0: att.append((badfile, "badupds.txt")) logger.debug(str(att)) sendMail(sendto, "%d SF Stormpost_Reason__c updated, %d failed"%(cgood, cbad), "spsfcync", att) return rc
def main(save_dir, work_dir, src_fname, mail_file): global logger doesnotwant = "Does Not Want" logger = utils.createLogger("fpn_unsubs") logger.debug("main started") if not os.path.isdir(save_dir): r,c = commands.getstatusoutput("mkdir -p "+save_dir) if r: logger.error(c) mail_file_o = open(mail_file, "w") mail_file_o.write("ERROR\n") mail_file_o.write(c) sys.exit(1) # save_dir has timestamp:email_add donefile = os.path.join(save_dir, "donefile.dat") grep1 = "grep -n \"%%s\" %s"%donefile open(donefile, "a").close() # wtf consider = {} for p in [l for l in os.listdir(work_dir) if l.find(src_fname) != -1]: for line in open(os.path.join(work_dir, p)).read().split("\n"): if len(line): parts = line.split(delim) if len(parts) == 3: addy = parts[1].strip().lower() eds = parts[2] token = parts[0] blk = "%s:%s"%(token, addy) cmd = grep1%blk # logger.debug(cmd) r,c = commands.getstatusoutput(cmd) if r == 0: logger.debug("Already done "+line) else: logger.debug("Will consider "+line) try: tup = consider[addy] except KeyError: tup = ([], []) consider[addy] = tup tup[0].append(token) for ed in eds.split(";"): edx = ed.split(",")[0] if edx not in tup[1]: tup[1].append(edx) if len(consider) == 0: logger.info("Nothing to process") return 0 (sfurl, username, password) = utils.getSFSiteCred(os.path.join(sf_home, "sfrunner.cfg")) where = None for key in consider.keys(): def escapeQ(x): return x.replace("'", "\\'") if not where: where = "where Email in ('%s'"%escapeQ(key) else: where = where + ",'%s'"%escapeQ(key) where = where + ") and FPN_Subscriptions__c != '' and FPN_Subscriptions__c != '%s' and FPN_Subscriptions__c != 'Not Authorized - see JMC'"%doesnotwant queryRes = os.path.join(work_dir, "fpnunsubsw.csv") if os.path.isfile(queryRes): try: os.unlink(queryRes) except OSError: mail_file_o = open(mail_file, "w") mail_file_o.write("ERROR") msg = "*** cannot delete file "+queryRes mail_file_o.write(msg) logger.error(msg) return 1 p1.main(queryRes, sfurl, username, password, where) if not os.path.isfile(queryRes): msg = "ERROR\n**** query for unsubs did not generate output file\n Query: "+where logger.error(msg); mail_file_o = open(mail_file, "w") mail_file_o.write(msg) return 1 # diff what we got from the server and what is currently in sf sfmap = {} first = True for r in csv.reader(open(queryRes, 'r')): if first: first = False else: key = r[1].strip().lower() sfmap[key] = (r[0], r[4].split(";")) # create the input file csv_file = os.path.join(work_dir, "update.csv") logger.debug("LOADER CSV "+csv_file) csv_fileo = open(csv_file, 'w') csvw = csv.writer(csv_fileo, delimiter=',', quotechar='"', quoting=csv.QUOTE_NONNUMERIC) csvw.writerow(["ID", "FPN_Subscriptions__c"]) id_to_unsub_map = {} has_one = False for key in consider.keys(): try: insf_tup = sfmap[key] except KeyError: # if this is the case then it means the contact was deleted from SF continue to_remove = consider[key][1] logger.debug("CONTACT: "+key) logger.debug("SF val: "+str(insf_tup)) logger.debug("toRemove: "+str(to_remove)) has_one = True new_val = "" for i in insf_tup[1]: i1 = i.split(" ")[0] if i1 not in to_remove: if new_val == "": new_val = i else: new_val = new_val + ";" + i if new_val == "": new_val = doesnotwant csvw.writerow([insf_tup[0], new_val]) id_to_unsub_map[insf_tup[0]] = (key, to_remove) csv_fileo.close() if has_one: logger.debug("id_to_unsub_map "+str(id_to_unsub_map)) stat_file = os.path.join(work_dir, "fpnunsubs.out") logger.debug("STAT FILE "+stat_file) try: rmsg, rcount, fcount = upsert.main("update", csv_file, stat_file, sfurl, username, password) if rmsg != "": emsg = "Error at update: %s"%rmsg mail_file_o = open(mail_file, "w") mail_file_o.write(emsg) return 1 except sf.SFException, e: mail_file_o = open(mail_file, "w") mail_file_o.write("ERROR") mail_file_o.write(str(e)) logger.error(str(e)) return 1 mail_file_o = open(mail_file, "w") mail_file_o.write("SF Updated: Records Processed: %d\nFailed: %d\n"%(rcount, fcount)) mail_file_o.write("Successful updates\nEmail Address\t\tFPN Unsubscribes\n") stat_file_o = open(stat_file) first = True fail_rec = [] for r in csv.reader(stat_file_o): if first: first = False else: id = r[0] success = r[1] create = r[2] reason = r[3] try: email, unsub = id_to_unsub_map[id] if success.lower() == "true": mail_file_o.write("%s\t\t%s\n"%(email, ";".join(unsub))) else: fail_rec.append((email, ";".join(unsub), reason)) except KeyError: fail_rec.append(("", "", reason)) if len(fail_rec): mail_file_o.write("Failed updates\nEmail Address\t\tFPN Unsubscribes\tReason\n") for z in fail_rec: mail_file_o.write("%s\t\t%s\t%s\n"%(z[0], z[1], z[2]))
from utils import getSFSiteCred, createLogger, getScriptPath import sys import sf import os.path import datetime import commands import uu import curl import fcntl mailhost = "mail.advisen.com" administrators = "[email protected],[email protected]" mailsender = "*****@*****.**" logger = createLogger("pushreports") sleepTime = 5 configFile = "pushreports.cfg" configPath = os.path.join(getScriptPath(), configFile) pid_file = os.path.join(getScriptPath(), 'program.pid') fp = open(pid_file, 'w') try: fcntl.lockf(fp, fcntl.LOCK_EX | fcntl.LOCK_NB) except IOError: logger.error("Another instance is already running") sys.exit(0) class PRException(sf.SFException): def __init__(self, msg): self.msg = msg
# save current olfa state - based on the last shit you sent to the Arduinos self.inst_drivers[0].instrument_widget.save_arduino_variables() self.inst_drivers[ 0].instrument_widget.btn_overwrite_olfaConfig_file() # TODO: add popup that shows where it saved it to # THIS ONLY WORKS NOW when you have just the single instrument self.logger.info('~~~~~~closing program~~~~~~~~') event.accept() if __name__ == "__main__": # Create logger mainLogger = utils.createLogger(__name__) mainLogger.info('~~~~~~~~~~~~~~~~~~~~~~~~~~~~~') app1 = QApplication(sys.argv) # Default Channel objects channelObjs = [] channelObjs.append( channelObj(name='olfa prototype', instrument='olfactometer')) channelObjs.append(channelObj(name='PID reading', instrument='NI-DAQ')) # Open main window mainWindow = mainGUI(channelObjs) size = mainWindow.sizeHint() mainWindow.resize(size) #mainLogger.debug('Done creating interface')
import requests import sys import json import utils import zoom import topics as topicGenerator logger = utils.createLogger("bot") config = utils.loadConfig() if "slack" not in config: logger.error("slack key not present in config json") sys.exit(1) slack_conf = config["slack"] if "webhook_url" not in slack_conf or "channel" not in slack_conf: logger.error("webhook_url or channel not present in slack config") sys.exit(1) WEBHOOK_URL = slack_conf["webhook_url"] CHANNEL = slack_conf["channel"] BOT_USERNAME = slack_conf.get("username", "water-cooler-bot") BOT_ICON_EMOJI = slack_conf.get("icon_emoji", ":water-cooler:") def ping(num_topics=3): topics = [topicGenerator.generateTopic() for _ in range(num_topics)] meetingUrl = zoom.getMeetingUrl() msg = "You must be really thirsty, its time to get a glass of water! :glass_of_milk:\n" +\ "Gather around the water cooler ya folks - {}\n\n".format(meetingUrl) +\ "While you're at it, here are some topics that you may find interesting to discuss -\n" +\ "\n".join(["{}. {}".format(i+1, topic) for i, topic in enumerate(topics)])
import signal import asyncio from os import environ from wsapi import readbcv, websockets from utils import createLogger def stop_handler(sig, loop, fatherlog): log = fatherlog.getChild('stop_handler') log.info(f'收到信号: {sig}, 正在关闭......') loop.stop() loop.remove_signal_handler(signal.SIGTERM) loop.add_signal_handler(signal.SIGINT, lambda: None) loop = asyncio.get_event_loop() fatherlog = createLogger('ReadBCV') signal.signal(2, lambda: None) loop.add_signal_handler(15, stop_handler, 15, loop, fatherlog) loop.add_signal_handler(2, stop_handler, 2, loop, fatherlog) addr = environ.get('APP_LISTEN', '0.0.0.0') port = int(environ.get('APP_PORT', '8765')) path = environ.get('APP_PATH', '/') fatherlog.info('Listen %s:%s At Path = %s' % (addr, port, path)) start_server = websockets.serve(readbcv, addr, port) loop.run_until_complete(start_server) loop.run_forever()
def main(source, dest, runtype): # runtype 1 - the one run at midnight # runtype 2 - the final day's run global logger logger = utils.createLogger("runner") if not os.path.isdir(source): print "invalid directory "+source sys.exit(1) if not prepareDestDir(dest): sys.exit(1) import datetime today = datetime.datetime(2012, 12, 21).now() pfile = os.path.join(source, "prev_%s.csv"%runtype) nfile = os.path.join(source, "now_%s.csv"%runtype) backup(source, today, runtype) # abort if there is no prev file if not os.path.isfile(pfile): logger.error("There is no previous file "+pfile) sendMail(None, "**** Stormpost SF Error", "There is no previous file "+pfile) sys.exit(1) import p2 bx = utils.getSFSiteCred() p2.main(int(runtype), nfile, bx[0], bx[1], bx[2]) if not os.path.isfile(nfile): restorePrev(source, runtype) return import diff adds, deletes = diff.diff(nfile, pfile) print adds, deletes tosend = [] jlt_to_old = { "JLT Pharma":"Pharma Global", "JLT Constr":"Constr Global", "JLT Insur":"Insur Global", "JLT Bank":"Bank Global", "JLT Telecom":"Telecom Global", "JLT Asset & Wealth":"Asset & Wealth Global", "JLT Lawyers":"Lawyers Global", "JLT Media":"Media Global"} global_to_new = { "Pharma Global":"Pharma Global Advisen", "Constr Global":"Constr Global Advisen", "Insur Global":"Insur Global Advisen", "Bank Global":"Bank Global Advisen", "Telecom Global":"Telecom Global Advisen", "Asset & Wealth Global":"Asset & Wealth Global Advisen", "Lawyers Global":"Lawyers Global Advisen", "Media Global":"Media Global Advisen"} def createSPFiles(which, decor): import string for w in which.keys(): if w in jlt_to_old.keys(): k = jlt_to_old[w] else: if w in global_to_new: k = global_to_new[w] else: k = w if decor == "": fn = k.replace(" ", "_").replace("/", "_") + "_%04d_%02d_%02d.txt"%(today.year, today.month, today.day) else: fn = k.replace(" ", "_").replace("/", "_") + "_%s_%04d_%02d_%02d.txt"%(decor,today.year, today.month, today.day) fn = os.path.join(dest, fn) tosend.append(fn) op = open(fn, "w").write(string.join(which[w], "\n")) createSPFiles(adds, "") createSPFiles(deletes, "remove") logger.debug("Files to send "+str(tosend)) sys.exit(0)
def __create_sf_updates(fin, workdir, rmap, debug_max = None): splits = [] split_size = 500 logger = utils.createLogger("sp_get_status.py") date = datetime.datetime.now() sdate = "%02d-%02d-%04d"%(date.month, date.day, date.year) if debug_max is not None: debug_max = int(debug_max) logger.debug("debug_max "+str(debug_max)) logger.debug("*** WORKDIR "+workdir) logger.debug("START") k = 0 upds = 0 ksplit = 0 upd_fileo = None upd_csvw = None manifest_map = {} def __create_split_file(): fname = os.path.join(workdir, "spupd%03d"%(len(splits)+1)) logger.debug("creating SPLIT file "+fname) fileo = open(fname, "w") csvw = csv.writer(fileo, delimiter=',', quotechar='"', quoting=csv.QUOTE_NONNUMERIC) csvw.writerow(["ID", "Stormpost_Reason__c", "Stormpost_Reason_Date__c"]) splits.append(fname) return fileo, csvw first = True for row in csv.reader(open(fin), delimiter=",", quotechar='"'): if first: first = False continue k = k + 1 if k%100 == 0: logger.debug("RECNO %d"%k) id = row[0] email = row[1] oldstat = row[6] try: sp_id, sval, m_id = rmap[email] if sval not in valid_status: err = "bad status for %s -- %s"%(email, sval) logger.error(err) raise Exception("bad status for %s -- %s"%(email, sval)) else: if oldstat != sval: if upd_fileo is None: logger.debug("creating init") upd_fileo, upd_csvw = __create_split_file() ksplit = 0 logger.debug("SIZE %d"%(len(splits))) if ksplit > split_size: logger.debug("SPLITTING ANEW") ksplit = 0 upd_fileo.close() upd_fileo, upd_csvw = __create_split_file() logger.debug("SIZE %d"%(len(splits))) upd_csvw.writerow([id, sval, sdate]) upd_fileo.flush() manifest_map[id] = (email, oldstat) # memory max? upds = upds + 1 ksplit = ksplit + 1 if debug_max is not None: if upds > debug_max: break except KeyError: logger.info("Email %s is not in SP"%email) logger.debug("DONE") return (0, upds, splits, manifest_map)
# spsfsync_runner.py import sys import os.path import utils import p1_a from mailer import sendMail import datetime import spsfsync import spsoapclient logger =utils.createLogger("spsfsync_runner") tempdir = utils.get_temp_dir() send_emails_to = ["*****@*****.**", "*****@*****.**", "*****@*****.**"] def __dumpSF(nowfile): b = utils.getSFSiteCred() where = "where FPN_Subscriptions__c != '' and FPN_Subscriptions__c != 'Does Not Want'" p1_a.main(nowfile, b[0], b[1], b[2], where) def main(archive_dir, debug = 0): if not os.path.isdir(archive_dir): print "archive_dir "+archive_dir+" not found" return -1