def csv_Send_To_House_With_Exclusions(csvfile='demo-dataz.csv', messagefile="noCispaMessage.txt", statfile='csv_Send_To_House.log', dryrun=False, excludedDists=[]): ''' Parse from the csv file ''' writeYourRep = WriteYourRep() reader = csv.DictReader(open(csvfile, 'rb'), delimiter='\t') genderassigner = GenderLookup() (subject, message) = parseMessageFile(messagefile) for row in reader: state = 'unknown' status = "" try: i = csv_To_Data(row, writeYourRep, genderassigner, subject, message) alldists = writeYourRep.getWyrDistricts(i.zip5) culleddists = [ dist for dist in alldists if dist not in excludedDists ] distListStr = ' '.join(alldists) if dryrun or len(culleddists) == 0: status += distListStr + " " + ": Not attempted with " + i.__str__( ) + "\n" else: (dist, q) = writeYourRep.write_reps_in_dists(i, culleddists) status += dist + " " + writeYourRep.getStatus(q) + ", " except Exception, e: import traceback traceback.print_exc() status = status + ' failed: ' + e.__str__() file(statfile, 'a').write('%s %s, %s, "%s"\n' % (i.fname, i.lname, i.state, status))
def csv_Send_To_House(csvfile='demo-dataz.csv', messagefile="noCispaMessage.txt", statfile='csv_Send_To_House.log', dryrun=False, onedistrict=None, jsoninput=False): ''' Parse from the csv file ''' writeYourRep = WriteYourRep() if not jsoninput: # default is csv reader = csv.DictReader(open(csvfile, 'rb'), delimiter='\t') else: try: reader = json.load(open(csvfile, 'r')) except: reader = load_json(csvfile) genderassigner = GenderLookup() (subject, message) = parseMessageFile(messagefile) for row in reader: i = None state='unknown' status = "" try: i = row_dict_to_data(row, writeYourRep, genderassigner, subject, message) if onedistrict!=None: if DEBUG: print "Feature of specifying one district is not supported yet" return if dryrun: distListStr=' '.join(writeYourRep.getWyrDistricts(i.zip5)) status += distListStr + " " + ": Not attempted with "+ i.__str__()+"\n" else: q = writeYourRep.writerep(i) status += i.dist + ": " status += writeYourRep.getStatus(q) except Exception, e: traceback.print_exc() status=status + ' failed: ' + e.__str__() if i is not None: file(statfile, 'a').write('%s %s, %s, "%s"\n' % (i.fname.encode('utf-8', 'ignore'), i.lname.encode('utf-8', 'ignore'), i.state, status))
def csv_Send_To_House(csvfile='demo-dataz.csv', messagefile="noCispaMessage.txt", statfile='csv_Send_To_House.log', dryrun=False, onedistrict=None): ''' Parse from the csv file ''' writeYourRep = WriteYourRep() reader = csv.DictReader(open(csvfile, 'rb')) genderassigner = GenderLookup() (subject, message) = parseMessageFile(messagefile) for row in reader: state = 'unknown' status = "" try: i = csv_To_Data(row, writeYourRep, genderassigner, subject, message) if onedistrict != None: print "Feature of specifying one district is not supported yet" return if dryrun: distListStr = ' '.join(writeYourRep.getWyrDistricts(i.zip5)) status += distListStr + " " + ": Not attempted with " + i.__str__( ) + "\n" else: q = writeYourRep.writerep(i) status += writeYourRep.getStatus(q) + ", " except Exception, e: import traceback traceback.print_exc() status = status + ' failed: ' + e.__str__() file(statfile, 'a').write('%s %s, %s, "%s"\n' % (i.fname, i.lname, i.state, status))
def csv_Send_To_Senate(csvfile='demo-dataz.csv', messagefile="noCispaMessage.txt", statfile='csv_Send_To_Senate.log', dryrun=False, onesenator=None): ''' Parse from the csv file Problem forms: www.vitter.senate.gov Problem with link to contact page on senators website www.mccain.senate.gov unknown url type, need to look into this one further levin.senate.gov urllib2.URLError: <urlopen error [Errno 61] Connection refused> www.lieberman.senate.gov Unclear whether it worked or not. franken.senate.gov Unclear whether it worked or not. www.toomey.senate.gov CAPTCHA www.sessions.senate.gov CAPTCHA www.shelby.senate.gov CAPTCHA www.coburn.senate.gov CAPTCHA www.crapo.senate.gov CAPTCHA www.moran.senate.gov CAPTCHA www.roberts.senate.gov CAPTCHA www.paul.senate.gov CAPTCHA ''' import csv from ZipLookup import ZipLookup from GenderLookup import GenderLookup writeYourRep = WriteYourRep() reader = csv.DictReader(open(csvfile, 'rb')) genderassigner = GenderLookup() (subject, message) = parseMessageFile(messagefile) zipLookup = ZipLookup() for row in reader: state = 'unknown' status = "" try: i = csv_To_Data(row, writeYourRep, genderassigner, subject, message) sens = writeYourRep.getSenators(i.state) for sen in sens: if onesenator != None and onesenator not in sen: status += "%s: not attempted, " % (sen) continue print "Writing to senator %s" % sen senname = web.lstrips( web.lstrips(web.lstrips(sen, 'http://'), 'https://'), 'www.').split('.')[0] captchaforms = [ 'toomey', 'sessions', 'shelby', 'coburn', 'crapo', 'roberts', 'paul' ] if senname in captchaforms: status += senname + " has captcha. skipping. " elif dryrun: status += sen + " " + senname + ": Not attempted with " + i.__str__( ) + "\n" else: status += senname + ": " q = writeYourRep.writerep_general(sen, i) status += writeYourRep.getStatus(q) + ", " except Exception, e: import traceback traceback.print_exc() status = status + ' failed: ' + e.__str__() file(statfile, 'a').write('%s %s, %s, "%s"\n' % (i.fname, i.lname, i.state, status))
def housetest(distToEmail=None): ''' Test every house form, or just check the particular distToEmail form. Todo: add the check-by-eye option ''' broken = set(['AR-01','AZ-05','AZ-07','CA-21','CA-25','CA-30','CA-34','CA-44','CA-49','FL-12','FL-14','FL-17','IA-05','IL-16','IN-05','MI-14','MT-00','NC-03','NC-10','NJ-04','NJ-11','NY-20','NY-23','OH-14','OH-15','SC-01','SC-05','SD-00','TN-07','TX-23','TX-24','TX-32','UT-02','WI-05',]) # 38 judiciary members judiciary=set(['TX-21', 'WI-05', 'NC-06', 'CA-24', 'VA-06', 'CA-03', 'OH-01', 'IN-06', 'VA-04', 'IA-05', 'AZ-02', 'TX-01', 'OH-04', 'TX-02', 'UT-03', 'AR-02', 'PA-10', 'SC-04', 'FL-12', 'FL-24', 'AZ-03', 'NV-02', 'MI-14', 'CA-28', 'NY-08', 'VA-03', 'NC-12', 'CA-16', 'TX-18', 'CA-35', 'TN-09', 'GA-04', 'PR-00', 'IL-05', 'CA-32', 'FL-19', 'CA-39']) allCaptcha =set(['CA-44', 'CA-49', 'TX-32', 'IN-05', 'NC-03', 'MI-14', 'CA-21', 'FL-14']) # judiciary members with captchas judCaptcha=set(['CA-49', 'MI-14' ]) #n = set() #n.update(correction); #n.update(err); # Speed up test. # To check the return pages using pattern matching # rather than by eye, set this flag to false checkByEye=False fh = file('results.log', 'a') writeYourRep = WriteYourRep() for dist in sorted(dist_zip_dict.keys()): #if dist not in allCaptcha: continue #if dist not in broken: continue #if dist in h_working or dist in n: continue #if dist not in judiciary: continue if distToEmail and dist != distToEmail: continue print "\n------------\n", dist,"\n" q=None try: q = writeYourRep.writerep(writeYourRep.prepare_i(dist)) errorString = None if checkByEye: subprocess.Popen(['open', 'house/%s.html' % dist]) print result = raw_input('%s? ' % dist) else: confirmations=[cstr for cstr in confirmationStrings if cstr in q.lower()] if len(confirmations) > 0: result='thanked' else: result = 'err' errorString = getError(q) print "ErrorString: ", errorString print result + '.add(%s)' % repr(dist) if not errorString: fh.write('%s.add(%s)\n' % (result, repr(dist))) else: #if thanked, but still have error, we assume we have an error (see FL-14 for example) result = 'err' fh.write('%s.add(%s) %s\n' % (result, repr(dist), errorString)) except Exception, detail: #print "DETAIL: ", detail q=detail.__str__() errorString = detail.__str__() import traceback; traceback.print_exc() print 'err.add(%s)' % (repr(dist)) fh.write('%s.add(%s) %s\n' % ('err', repr(dist),errorString)) fh.flush() if q: file('house/%s.html' % dist, 'w').write(q)
#!/usr/bin/env python # # File to call WriteYourRep class. # from WriteYourRep import * from DataForWriteYourRep import * writeYourRep = WriteYourRep() def senatetest(): ''' Creates a file sen/schumer.html with schumers contact page ''' sendb = get_senate_offices() statfile = open("senate-test-out.txt", "w") for state in sendb: for member in sendb[state]: sen = web.lstrips(web.lstrips(web.lstrips(member, 'http://'), 'https://'), 'www.').split('.')[0] print repr(sen) q = None try: q = writeYourRep.writerep_general(member, writeYourRep.prepare_i(state)) file('sen/%s.html' % sen, 'w').write('<base href="%s"/>' % member + q) status = writeYourRep.getStatus(q) except Exception, e: print "Caught exception on senator: %s " % member status="Failed: exception occurred %s" % e.__str__()
def setUp(self): self.writer = WriteYourRep()