def main(fout):
import p1
bx = utils.getSFSiteCred()
prepareDestDir(fout)
tmpfile = "/tmp/getFPUsers.csv"
p1.main(tmpfile, bx[0], bx[1], bx[2], " where FPN_Subscriptions__c != '' and FPN_Subscriptions__c != 'Does Not Want' and FPN_Subscriptions__c != 'Not Authorized - see JMC'")
map = {}
# fo = open(fout, "w")
first = True
for r in csv.reader(open(tmpfile, 'r')):
if first: first = False
else:
if r[4] != "":
key = r[1][:lflen].lower()
if key == "": key = "NOEMAILADDRESS"
try:
m = map[key]
except KeyError:
m = []
map[key] = m
str = "%s|%s|%s|%s\n"%(r[1],r[2], r[3], r[4])
m.append(str)
for k in map.keys():
p = os.path.join(fout, k)
print fout+" MAKING "+k+" -- "+p
try:
os.makedirs(p)
f = os.path.join(p, "dat.dat")
map[k].sort()
of = open(f, "w").write("".join(map[k]))
f = os.path.join(p, "map.dat")
of = open(f, "w")
for z in map[k]:
of.write(z.split("|")[0]+"\n")
except OSError:
print "BAD EMAIL ADDRESS "+k
rt = [x.strip() for x in row]
if len(rt) == 5:
if k > 0:
rt[4] = ";".join([x.strip() for x in rt[4].strip().split(";") if x in fpns])
# logger.debug(rt);
if (rt[4] != ""):
# VERY DIRTY HACK for FPN Digest called as EU Free
rt[4] = rt[4].replace("EU Digest", "EU Free")
fout.writerow(rt)
k = k + 1
def usage():
print "usage splitter.py runtype infile outfile"
if __name__ == "__main__":
if len(sys.argv) != 4:
usage()
sys.exit(1)
b = utils.getSFSiteCred()
fpn_type_1 = ["EU Comp", "EU Contract", "EU Paid", "EU Digest"]
fpn_type_2 = ["Pharma Global", "Constr Global", "Insur Global", "Bank Global"]
runtype = int(sys.argv[1])
if runtype == 1: rt = fpn_type_1
elif runtype == 2: rt = fpn_type_2
else:
r = utils.getValidFPNs()
rt = [x for x in r if x not in fpn_type_1 and x not in fpn_type_2]
main(rt,
sys.argv[2], sys.argv[3])
def main(workDir):
import ConfigParser
config = ConfigParser.RawConfigParser()
config.read(configPath)
advisenRestURL = config.get('main', 'advisenRestURL')
advisenUser = config.get('main', 'advisenUser')
advisenPassword = config.get('main', 'advisenPassword')
ftpURL = config.get('main', 'ftpURL')
ftpUser = config.get('main', 'ftpUser')
ftpPassword = config.get('main', 'ftpPassword')
sfurl = config.get('main', 'sfurl')
logger.debug("Starting %s - %s - %s"%(advisenRestURL, advisenUser, advisenPassword))
lr = submitAdvRequest(advisenRestURL, "login",
{"username": advisenUser, "password": advisenPassword})
logger.debug("After advisen login rc = "+str(lr["error"]))
if lr["error"] != 0:
logger.error("Cannot login to advisen REST server "+str(lr["error"]))
mailError("Cannot login to advisen REST server "+str(lr["error"]))
return lr["error"];
lr = submitAdvRequest(advisenRestURL, "getsfpushreports")
if lr["error"] != 0:
logger.error("Cannot get list of records to work on "+str(lr["error"]))
mailError("Cannot get list of records to work on "+str(lr["error"]))
return lr["error"];
aList = lr["result"]
lr = submitAdvRequest(advisenRestURL, "logout")
# drop all files from the FTP server
if len(aList) == 0:
logger.debug("Nothing to do")
return 0
mapFTPOK = {}
mapBad = {}
mapBadSFUpload = {}
listEmail = []
def genReportName(row):
return row["reportname"]+".pdf"
# i need 2 loops - to prevent downloading duplicates
for r in aList:
report = genReportName(r)
if report not in mapFTPOK:
logger.debug("Need to retrieve "+r["reportname"]+".pdf")
cmd = "ncftpget -u%s -p%s %s %s /sfreports/%s"%(ftpUser, ftpPassword, ftpURL, workDir, report)
rc,txt = commands.getstatusoutput(cmd)
logger.debug("After retrieve %d"%rc)
if rc:
mapBad[report] = r
else:
mapFTPOK[report] = r
## FIXME - should be the SF account of the target
loginURL, username, password = getSFSiteCred(configFile)
sf.setLogger(logger)
lx = sf.login(username, password, loginURL)
if not lx:
logger.error("Unable to login to SF")
mailError("Unable to login to SF")
else:
site = lx[0]
sessionid = lx[1]
logger.debug("FIRST SF site: %s sess: %s"%(site, sessionid))
# inGood is the collector for the DB update IN clause
inGood = []
for r in aList:
report = genReportName(r)
# this is correct
# we only upload what we got from FTP server
if report in mapFTPOK:
accountid = r["accountid"]
description = r["description"]
companyname = r["companyname"]
logger.debug("UPLOADING TO SF account %s - %s"%(accountid, report))
rc = uploadFile(site, sessionid, workDir, report, accountid, workDir, description)
if not rc:
inGood.append(r["reportid"])
sql = "select email, lastname, firstname from user where id = '%s'"%(str(r["userid"]))
uset = runSFQuery(site, sessionid, "User", sql)
logger.debug(str(uset))
if uset[0] == 1:
sql = "SELECT Id FROM Attachment where parentid = '%s' ORDER BY CreatedDate DESC NULLS FIRST"%(str(accountid))
aset = runSFQuery(site, sessionid, "Attachment", sql)
logger.debug(str(aset))
if aset[0]:
# take the top
link = sfurl + str(aset[1][0][0])
logger.debug(link)
addy = uset[1][0][0]
lastname = uset[1][0][1]
firstname = uset[1][0][2]
listEmail.append((addy, description, link, companyname, lastname, firstname))
else:
logger.error("Unable to get the ID of the attachment")
mailError("Unable to get the ID of the attachment")
else:
logger.error("USER: %s not found in SF"%(r["userid"]))
mailError("USER: %s not found in SF"%(r["userid"]))
else:
mapBadSFUpload[report] = r
if len(inGood):
lr = submitAdvRequest(advisenRestURL, "login",
{"username": advisenUser, "password": advisenPassword})
logger.debug("After advisen login rc = "+str(lr["error"]))
if lr["error"] != 0:
logger.error("Cannot login to advisen REST server "+str(lr["error"]))
mailError("Cannot login to advisen REST server "+str(lr["error"]))
return lr["error"];
lr = submitAdvRequest(advisenRestURL, "updsfreportstat", inGood)
if lr["error"] != 0:
logger.error("FAILED to update status "+lr["message"])
mailError("FAILED to update status "+lr["message"])
if len(listEmail):
mm = Mailer(mailhost, 25, logger, administrators, mailsender)
for r in listEmail:
body = "%s is now ready for review. <a href=%s>Click here</a> or sign-on to your SalesForce account"%(r[1], r[2])
msg = mm.mailAttach("SFPUSH REPORTS TEST CONAME: "+r[3], body, "html")
mm.doMail(r[0], mailsender, msg)
msg = mm.mailAttach("*** DEBUG *** SFPUSH REPORTS TEST CONAME: %s was sent to %s"%(r[3], r[0]), body, "html")
mm.doMail("*****@*****.**", mailsender, msg)
mm.doMail("*****@*****.**", mailsender, msg)
if len(mapBad):
logger.error("FAILED TO RETRIEVE FILES from FTP")
msg = "Failed to retrieve the following files from FTP "
for k in mapBad.keys():
logger.error(k)
msg = msg + k
mailError(msg)
if len(mapBadSFUpload):
logger.error("FAILED TO UPLOAD FILES TO SF")
msg = "Failed to upload the following files from SF "
for k in mapBadSFUpload.keys():
logger.error(k)
msg = msg + k
mailError(msg)
return 0
def main(sf_csv_file, sp_email_map, maxupds, sendto):
logger = utils.createLogger("spsfsync.py")
b = utils.getSFSiteCred()
workdir = os.path.join(tempdir, utils.temp_name("sfupdworkdir"))
os.mkdir(workdir)
rc,upds, splits, manifest_map = \
__create_sf_updates(sf_csv_file, workdir, sp_email_map,
maxupds)
if rc != 0 or upds == 0:
if rc != 0:
logger.error("sp_get_status returned with error %d"%rc)
else:
logger.info("There were no updates")
return rc
logger.debug("THERE ARE %d updates and %d splits"%(upds, len(splits)))
# loop splits here
# then cat the logfile
cgood = 0
cbad = 0
goodfile = os.path.join(tempdir, utils.temp_name("goodups.txt"))
badfile = os.path.join(tempdir, utils.temp_name("badupds.txt"))
logger.debug("GOODFILE "+goodfile)
good = open(goodfile, "w")
bad = open(badfile, "w")
updlogfile = os.path.join(tempdir, utils.temp_name("spsynclogfile.csv")) # yes reuse
ktest = 0
for spl in splits:
rc = sfupdspstatus.main(b[0], b[1], b[2], spl, updlogfile)
csv_spl = csv.reader(open(spl))
up_file = open(updlogfile)
csv_log = csv.reader(up_file)
first = True
for row_upd in csv_spl:
row_log = csv_log.next()
if first:
first = False
continue
lid = row_upd[0]
try:
(email, fromv) = manifest_map[lid]
cto = row_upd[1]
success = row_log[1].lower()
error = row_log[3]
if success == "true":
cgood = cgood + 1
good.write(email+" changed from "+fromv+" to "+cto+"\n")
good.flush()
else:
cbad = cbad + 1
bad.write(email+" failed to update from "+fromv+" to "+cto+" "+error+"\n")
bad.flush()
except KeyError:
logger.error("ID not found "+lid)
up_file.close()
ktest = ktest + 1
# if ktest > 4: break ## remove this
good.close()
bad.close()
att = []
att.append((goodfile, "goodups.txt"))
if cbad > 0: att.append((badfile, "badupds.txt"))
logger.debug(str(att))
sendMail(sendto, "%d SF Stormpost_Reason__c updated, %d failed"%(cgood, cbad), "spsfcync", att)
return rc
def main(save_dir, work_dir, src_fname, mail_file):
global logger
doesnotwant = "Does Not Want"
logger = utils.createLogger("fpn_unsubs")
logger.debug("main started")
if not os.path.isdir(save_dir):
r,c = commands.getstatusoutput("mkdir -p "+save_dir)
if r:
logger.error(c)
mail_file_o = open(mail_file, "w")
mail_file_o.write("ERROR\n")
mail_file_o.write(c)
sys.exit(1)
# save_dir has timestamp:email_add
donefile = os.path.join(save_dir, "donefile.dat")
grep1 = "grep -n \"%%s\" %s"%donefile
open(donefile, "a").close() # wtf
consider = {}
for p in [l for l in os.listdir(work_dir) if l.find(src_fname) != -1]:
for line in open(os.path.join(work_dir, p)).read().split("\n"):
if len(line):
parts = line.split(delim)
if len(parts) == 3:
addy = parts[1].strip().lower()
eds = parts[2]
token = parts[0]
blk = "%s:%s"%(token, addy)
cmd = grep1%blk
# logger.debug(cmd)
r,c = commands.getstatusoutput(cmd)
if r == 0:
logger.debug("Already done "+line)
else:
logger.debug("Will consider "+line)
try:
tup = consider[addy]
except KeyError:
tup = ([], [])
consider[addy] = tup
tup[0].append(token)
for ed in eds.split(";"):
edx = ed.split(",")[0]
if edx not in tup[1]:
tup[1].append(edx)
if len(consider) == 0:
logger.info("Nothing to process")
return 0
(sfurl, username, password) = utils.getSFSiteCred(os.path.join(sf_home, "sfrunner.cfg"))
where = None
for key in consider.keys():
def escapeQ(x):
return x.replace("'", "\\'")
if not where:
where = "where Email in ('%s'"%escapeQ(key)
else:
where = where + ",'%s'"%escapeQ(key)
where = where + ") and FPN_Subscriptions__c != '' and FPN_Subscriptions__c != '%s' and FPN_Subscriptions__c != 'Not Authorized - see JMC'"%doesnotwant
queryRes = os.path.join(work_dir, "fpnunsubsw.csv")
if os.path.isfile(queryRes):
try:
os.unlink(queryRes)
except OSError:
mail_file_o = open(mail_file, "w")
mail_file_o.write("ERROR")
msg = "*** cannot delete file "+queryRes
mail_file_o.write(msg)
logger.error(msg)
return 1
p1.main(queryRes, sfurl, username, password, where)
if not os.path.isfile(queryRes):
msg = "ERROR\n**** query for unsubs did not generate output file\n Query: "+where
logger.error(msg);
mail_file_o = open(mail_file, "w")
mail_file_o.write(msg)
return 1
# diff what we got from the server and what is currently in sf
sfmap = {}
first = True
for r in csv.reader(open(queryRes, 'r')):
if first: first = False
else:
key = r[1].strip().lower()
sfmap[key] = (r[0], r[4].split(";"))
# create the input file
csv_file = os.path.join(work_dir, "update.csv")
logger.debug("LOADER CSV "+csv_file)
csv_fileo = open(csv_file, 'w')
csvw = csv.writer(csv_fileo, delimiter=',',
quotechar='"', quoting=csv.QUOTE_NONNUMERIC)
csvw.writerow(["ID", "FPN_Subscriptions__c"])
id_to_unsub_map = {}
has_one = False
for key in consider.keys():
try:
insf_tup = sfmap[key]
except KeyError:
# if this is the case then it means the contact was deleted from SF
continue
to_remove = consider[key][1]
logger.debug("CONTACT: "+key)
logger.debug("SF val: "+str(insf_tup))
logger.debug("toRemove: "+str(to_remove))
has_one = True
new_val = ""
for i in insf_tup[1]:
i1 = i.split(" ")[0]
if i1 not in to_remove:
if new_val == "":
new_val = i
else:
new_val = new_val + ";" + i
if new_val == "": new_val = doesnotwant
csvw.writerow([insf_tup[0], new_val])
id_to_unsub_map[insf_tup[0]] = (key, to_remove)
csv_fileo.close()
if has_one:
logger.debug("id_to_unsub_map "+str(id_to_unsub_map))
stat_file = os.path.join(work_dir, "fpnunsubs.out")
logger.debug("STAT FILE "+stat_file)
try:
rmsg, rcount, fcount = upsert.main("update", csv_file, stat_file, sfurl,
username, password)
if rmsg != "":
emsg = "Error at update: %s"%rmsg
mail_file_o = open(mail_file, "w")
mail_file_o.write(emsg)
return 1
except sf.SFException, e:
mail_file_o = open(mail_file, "w")
mail_file_o.write("ERROR")
mail_file_o.write(str(e))
logger.error(str(e))
return 1
mail_file_o = open(mail_file, "w")
mail_file_o.write("SF Updated: Records Processed: %d\nFailed: %d\n"%(rcount, fcount))
mail_file_o.write("Successful updates\nEmail Address\t\tFPN Unsubscribes\n")
stat_file_o = open(stat_file)
first = True
fail_rec = []
for r in csv.reader(stat_file_o):
if first: first = False
else:
id = r[0]
success = r[1]
create = r[2]
reason = r[3]
try:
email, unsub = id_to_unsub_map[id]
if success.lower() == "true":
mail_file_o.write("%s\t\t%s\n"%(email, ";".join(unsub)))
else:
fail_rec.append((email, ";".join(unsub), reason))
except KeyError:
fail_rec.append(("", "", reason))
if len(fail_rec):
mail_file_o.write("Failed updates\nEmail Address\t\tFPN Unsubscribes\tReason\n")
for z in fail_rec:
mail_file_o.write("%s\t\t%s\t%s\n"%(z[0], z[1], z[2]))
def main(source, dest, runtype):
# runtype 1 - the one run at midnight
# runtype 2 - the final day's run
global logger
logger = utils.createLogger("runner")
if not os.path.isdir(source):
print "invalid directory "+source
sys.exit(1)
if not prepareDestDir(dest):
sys.exit(1)
import datetime
today = datetime.datetime(2012, 12, 21).now()
pfile = os.path.join(source, "prev_%s.csv"%runtype)
nfile = os.path.join(source, "now_%s.csv"%runtype)
backup(source, today, runtype)
# abort if there is no prev file
if not os.path.isfile(pfile):
logger.error("There is no previous file "+pfile)
sendMail(None, "**** Stormpost SF Error", "There is no previous file "+pfile)
sys.exit(1)
import p2
bx = utils.getSFSiteCred()
p2.main(int(runtype), nfile, bx[0], bx[1], bx[2])
if not os.path.isfile(nfile):
restorePrev(source, runtype)
return
import diff
adds, deletes = diff.diff(nfile, pfile)
print adds, deletes
tosend = []
jlt_to_old = {
"JLT Pharma":"Pharma Global",
"JLT Constr":"Constr Global",
"JLT Insur":"Insur Global",
"JLT Bank":"Bank Global",
"JLT Telecom":"Telecom Global",
"JLT Asset & Wealth":"Asset & Wealth Global",
"JLT Lawyers":"Lawyers Global",
"JLT Media":"Media Global"}
global_to_new = {
"Pharma Global":"Pharma Global Advisen",
"Constr Global":"Constr Global Advisen",
"Insur Global":"Insur Global Advisen",
"Bank Global":"Bank Global Advisen",
"Telecom Global":"Telecom Global Advisen",
"Asset & Wealth Global":"Asset & Wealth Global Advisen",
"Lawyers Global":"Lawyers Global Advisen",
"Media Global":"Media Global Advisen"}
def createSPFiles(which, decor):
import string
for w in which.keys():
if w in jlt_to_old.keys(): k = jlt_to_old[w]
else:
if w in global_to_new: k = global_to_new[w]
else:
k = w
if decor == "":
fn = k.replace(" ", "_").replace("/", "_") + "_%04d_%02d_%02d.txt"%(today.year, today.month, today.day)
else:
fn = k.replace(" ", "_").replace("/", "_") + "_%s_%04d_%02d_%02d.txt"%(decor,today.year, today.month, today.day)
fn = os.path.join(dest, fn)
tosend.append(fn)
op = open(fn, "w").write(string.join(which[w], "\n"))
createSPFiles(adds, "")
createSPFiles(deletes, "remove")
logger.debug("Files to send "+str(tosend))
sys.exit(0)
def __dumpSF(nowfile):
b = utils.getSFSiteCred()
where = "where FPN_Subscriptions__c != '' and FPN_Subscriptions__c != 'Does Not Want'"
p1_a.main(nowfile, b[0], b[1], b[2], where)
