def main(): args = argsFetch() logger = loggerFetch(args.get('log_level')) logger.info('args: %s', str(args)) logger.info("BEGIN PROCESSING...") db = dbInitialize( db="surguja", charset="utf8") # The rest is updated automatically in the function if args['fetch']: # For fetch where only download push and parse options can be ignored fetchMusterDetails(logger, db, "FETCH MUSTER DETAILS", args['directory'], args['url'], args['parse'], args['push'], args['visible']) elif args['parse']: # parseDownloadedMuster(logger, db, "PARSE JOBCARDS", args['directory'], args['url'], ) logger.info( "Requested Parse") # If you are splitting download and parse logic fetchMusterDetails(logger, db, "PARSE MUSTER DETAILS", args['directory'], args['url'], args['parse'], args['push'], args['visible']) else: logger.info("Default fetch which includes download, parse & populate" ) # Controlled by options fetchMusterDetails(logger, db, "FETCH MUSTER DETAILS", args['directory'], args['url'], args['parse'], args['push'], args['visible']) dbFinalize( db) # Make sure you put this if there are other exit paths or errors logger.info("...END PROCESSING") exit(0)
def main(): from firebase import firebase firebase = firebase.FirebaseApplication('https://libtech-app.firebaseio.com/', None) args = argsFetch() logger = loggerFetch(args.get('log_level')) logger.info('args: %s', str(args)) if args['write']: d={} d=[] d.append({'make':'nd','model':'readyToEat','price':95}) d.append({'make':'sahaja','model':'vegetable','price':55}) d.append({'make':'tvs','model':'fruit','price':195}) with open('first.json', 'w') as outfile: json.dump(d, outfile) if args['populate']: jsonName="districts.json" json_data=open(jsonName,encoding='utf-8-sig').read() d = json.loads(json_data) myDict={} for key,values in d.items(): code=key name=values['name'] parentCode=values['stateCode'] logger.info(key) dictKey="%s_%s_%s" % (code,name,parentCode) myDict[dictKey] = {'name': name, 'code': code, 'parentCode': parentCode, 'slug': dictKey } logger.info(dictKey) result = firebase.patch('https://libtech-app.firebaseio.com/medicines/', myDict) logger.info("...END PROCESSING") exit(0)
def main(): args = argsFetch() logger = loggerFetch(args.get('log_level')) logger.info('args: %s', str(args)) logger.info("BEGIN PROCESSING...") myLibtechTag=LibtechTag.objects.filter(name="August 2017 Hearing").first() myStates=["18","17","27","24","33","15","16","31","34","05","32"] myStates=["29"] for eachState in myStates: myobjs=District.objects.filter(state__isNIC=True,state__code=eachState) if args['limit']: limit=int(args['limit']) else: limit=1 for eachDistrict in myobjs: logger.info(eachDistrict.name) myPanchayats=list(Panchayat.objects.filter(block__district__code=eachDistrict.code,crawlRequirement='NONE')) shuffle(myPanchayats) myPanchayats=myPanchayats[:limit] for eachPanchayat in myPanchayats: logger.info(eachPanchayat.name) eachPanchayat.crawlRequirement="FULL" eachPanchayat.libtechTag.add(myLibtechTag) eachPanchayat.save() logger.info("...END PROCESSING") exit(0)
def main(): args = argsFetch() logger = loggerFetch(args.get('log_level')) logger.info('args: %s', str(args)) logger.info("BEGIN PROCESSING...") db = dbInitialize(db="biharPDS", charset="utf8") # The rest is updated automatically in the function cur=db.cursor() db.autocommit(True) #Query to set up Database to read Hindi Characters query="SET NAMES utf8" cur.execute(query) inyear=args['year'] logger.info(inyear) display = displayInitialize(args['visible']) driver = driverInitialize(args['browser']) #Start Program here url="http://www.google.com" driver.get(url) myhtml=driver.page_source print myhtml # End program here driverFinalize(driver) displayFinalize(display) dbFinalize(db) # Make sure you put this if there are other exit paths or errors logger.info("...END PROCESSING") exit(0)
def runTestSuite(): logger = loggerFetch("info") logger.info("BEGIN PROCESSING...") db = dbInitialize(db="surguja", charset="utf8") cur = db.cursor() cur.execute("SET NAMES utf8") jobcard = 'CH-05-005-015-001/187' query = 'select headOfFamily from jobcardRegister where jobcard="' + jobcard + '"' logger.info("query[%s]" % query) cur.execute(query) head = cur.fetchall()[0][0] logger.info("HeadOfFamily[%s]" % head) query = 'select applicantName, accountNo from jobcardDetails where jobcard="' + jobcard + '"' logger.info("query[%s]" % query) cur.execute(query) names = cur.fetchall() #print names.encode('UTF-8') logger.info("Names[%s]" % str(names).encode('UTF-8')) dbFinalize(db) logger.info("...END PROCESSING")
def runTestSuite(): logger = loggerFetch("info") logger.info("BEGIN PROCESSING...") display = displayInitialize(1) driver = driverInitialize(path='/home/mayank/.mozilla/firefox/4s3bttuq.default/', timeout=timeout) driver.set_page_load_timeout(timeout) for version in versions: url = url_base + str(version) try: logger.info('Fetching URL[%s]' % url) driver.get(url) logger.info('After Fetch[%s]' % url) except Exception as e: logger.info("Warning %s", e) if os.path.exists(csv_file): version_file = 'CSVs/' + str(version) + '.csv' logger.info('Writing %s' % version_file) os.rename(csv_file, version_file) else: logger.error('Missed file[%s] from URL[%s]' % (version_file, url)) logger.info("CSV Fetched From [%s]" % url) driverFinalize(driver) displayFinalize(display) logger.info("...END PROCESSING")
def main(): args = argsFetch() logger = loggerFetch(args.get('log_level')) logger.info('args: %s', str(args)) logger.info("BEGIN PROCESSING...") if args['limit']: limit = int(args['limit']) else: limit =1 if args['crawlRequirement']: crawlRequirement=args['crawlRequirement'] else: crawlRequirement='STAT' finyear=args['finyear'] fullfinyear=getFullFinYear(finyear) myBlocks=Block.objects.filter(crawlRequirement=crawlRequirement)[:limit] for eachBlock in myBlocks: if eachBlock.district.state.stateCode != '02': logger.info("BlockName: %s BlockCode: %s " % (eachBlock.name,eachBlock.fullBlockCode)) reportTypes=["I","R"] for rtype in reportTypes: url="http://%s/netnrega/FTO/rejection.aspx?lflag=eng&state_code=%s&state_name=%s&district_code=%s&page=d&Block_code=%s&Block_name=%s&district_name=%s&fin_year=%s&typ=%s&linkr=X&" %(eachBlock.district.state.crawlIP,eachBlock.district.state.stateCode,eachBlock.district.state.name,eachBlock.district.fullDistrictCode,eachBlock.fullBlockCode,eachBlock.name,eachBlock.district.name,fullfinyear,rtype) logger.info(url) logger.info("...END PROCESSING") exit(0)
def main(): regex=re.compile(r'<input+.*?"\s*/>+',re.DOTALL) args = argsFetch() finyear=args['finyear'] if args['limit']: limit = int(args['limit']) else: limit =50000 if args['musterID']: mid=args['musterID'] else: mid=None if args['maxProcess']: maxProcess=int(args['maxProcess']) else: maxProcess=1 additionalFilters='' if args['district']: additionalFilters+= " and b.districtName='%s' " % args['district'] if args['block']: additionalFilters+= " and b.blockName='%s' " % args['block'] fullfinyear=getFullFinYear(finyear) logger = loggerFetch(args.get('log_level')) logger.info('args: %s', str(args)) logger.info("BEGIN PROCESSING...") db = dbInitialize(db=nregaDB, charset="utf8") # The rest is updated automatically in the function cur=db.cursor() db.autocommit(True) #Query to set up Database to read Hindi Characters query="SET NAMES utf8" cur.execute(query) tasks = multiprocessing.JoinableQueue() results = multiprocessing.Queue() if mid is None: query="select m.id from musters m,blocks b where m.fullBlockCode=b.fullBlockCode and m.finyear='%s' and (m.isDownloaded=0 or (m.wdComplete=0 and TIMESTAMPDIFF(HOUR, m.downloadAttemptDate, now()) > 48 )) %s order by isDownloaded,m.downloadAttemptDate limit %s" % (finyear,additionalFilters,str(limit)) else: query="select m.id from musters m where m.id=%s " % str(mid) logger.info(query) cur.execute(query) noOfTasks=cur.rowcount results1=cur.fetchall() for row in results1: musterID=row[0] tasks.put(Task(musterID)) for i in range(maxProcess): tasks.put(None) myProcesses=[musterProcess(tasks, results) for i in range(maxProcess)] for eachProcess in myProcesses: eachProcess.start() while noOfTasks: result = results.get() logger.info(result) noOfTasks -= 1 dbFinalize(db) # Make sure you put this if there are other exit paths or errors
def runTestSuite(): args = argsFetch() logger = loggerFetch(args.get('log_level')) logger.info('args: %s', str(args)) logger.info("BEGIN PROCESSING...") display = displayInitialize(args['visible']) driver = driverInitialize(args['browser']) # Mynk to use personal profile driver = driverInitialize(browser=args['browser'] , path='/home/mayank/.mozilla/firefox/4s3bttuq.default/') if args['cookie_dump']: cookieDump(driver) logger.info("Fetching [%s]" % driver.current_url) logger.info(wdTest(driver, args['url'])) logger.info("Fetched [%s]" % driver.current_url) if args['cookie_dump']: cookieDump(driver) driverFinalize(driver) displayFinalize(display) ''' display = vDisplayInitialize(visible) driver = driverInitialize(browser) logger.info(wdTest(driver)) driverFinalize(driver) vDisplayFinalize(display) ''' logger.info("...END PROCESSING")
def main(): alwaysTag=LibtechTag.objects.filter(name="Always") regex=re.compile("^[0-9]{4}-[0-9]{4}$") benchMark = datetime.strptime(telanganaThresholdDate, "%Y-%m-%d") telanganaStateCode='36' args = argsFetch() logger = loggerFetch(args.get('log_level')) logger.info('args: %s', str(args)) if args['limit']: limit = int(args['limit']) else: limit =1 if args['enumerate']: logger.info("Will enumerate and put all the District and Block Codes in place") enumerateCodes(logger) if args['download']: logger.info("Attempting to Download Postal P ayments") myPanchayats=Panchayat.objects.filter(crawlRequirement="FULL",block__district__state__code=telanganaStateCode) # for eachPanchayat in myPanchayats: # downloadPage(logger,eachPanchayat) queueManager(logger,myPanchayats,'download') if args['process']: logger.info("Attempting to Process Postal P ayments") curfinyear=getCurrentFinYear() today_min = datetime.combine(date.today(), time.min) today_max = datetime.combine(date.today(), time.max) myPanchayatReports=PanchayatReport.objects.filter(finyear=curfinyear,panchayat__crawlRequirement="FULL",panchayat__block__district__state__code=telanganaStateCode,reportType=reportType,updateDate__range=(today_min, today_max)) queueManager(logger,myPanchayatReports,'process') # for eachReport in myPanchayatReports: # logger.info(eachReport.panchayat.code) # processPostalReport(logger,eachReport) logger.info("...END PROCESSING") exit(0)
def main(): args = argsFetch() logger = loggerFetch(args.get('log_level')) db = dbInitialize(db=nregaDB, charset="utf8") # The rest is updated automatically in the function cur=db.cursor() db.autocommit(True) #Query to set up Database to read Hindi Characters query="SET NAMES utf8" cur.execute(query) if args['updateMusterStats']: updateMusterStats(cur,logger) if args['selectRandomDistricts']: selectRandomDistricts(cur,logger) if args['updatePanchayatStats']: updatePanchayatStats(cur,logger) if args['genMusterURL']: genMusterURLs(cur,logger,args['musterID']) if args['downloadRejectedPaymentReport']: if args['finyear']: finyear=args['finyear'] else: finyear='17' downloadRejectedPaymentReport(cur,logger,finyear) dbFinalize(db) # Make sure you put this if there are other exit paths or errors logger.info("...END PROCESSING") exit(0)
def setUp(self): self.logger = loggerFetch('info') self.db = dbInitialize(host='localhost', user='******', passwd='lt123', db='libtech') self.logger.info('BEGIN PROCESSING...')
def runTestSuite(): logger = loggerFetch("info") logger.info("BEGIN PROCESSING...") db = dbInitialize(db="surguja", charset="utf8") cur = db.cursor() cur.execute("SET NAMES utf8") jobcard = 'CH-05-005-015-001/187' query = 'select headOfFamily from jobcardRegister where jobcard="' + jobcard + '"' logger.info("query[%s]" % query) cur.execute(query) head = cur.fetchall()[0][0] logger.info("HeadOfFamily[%s]" % head) query = 'select applicantName, accountNo from jobcardDetails where jobcard="' + jobcard + '"' logger.info("query[%s]" % query) cur.execute(query) names = cur.fetchall() #print names.encode('UTF-8') logger.info("Names[%s]" % str(names).encode('UTF-8')) dbFinalize(db) logger.info("...END PROCESSING")
def runTestSuite(): logger = loggerFetch("info") logger.info("BEGIN PROCESSING...") display = displayInitialize(0) driver = driverInitialize() for mandal in mandals: report = generate_report(logger, driver, mandal, districts[mandal]) logger.info('Finally: \n%s' % report) filename = './mandals/' +mandal + '.csv' with open(filename, 'wb') as csv_file: logger.info("Writing to [%s]" % filename) csv_file.write(report.encode('utf-8')) if final_report[0][0] == '': rows = report.split('|') driverFinalize(driver) displayFinalize(display) logger.info("...END PROCESSING") return def main(): runTestSuite() exit(0) if __name__ == '__main__': main()
def main(): args = argsFetch() logger = loggerFetch(args.get('log_level')) if args['singleExecute']: myTask=Task.objects.filter(isComplete=False).order_by("modified").first() logger.info("Running Task %s Task ID %s" % (myTask.crawlState.name,str(myTask.id))) funcName=myTask.crawlState.name objID=myTask.objID finyear=myTask.finyear isError=getattr(crawlFunctions1,funcName)(logger,objID,finyear=finyear) if isError==False: isComplete=True myTask.isComplete=isComplete myTask.save() if args['run']: while True: myTask=Task.objects.filter(isComplete=False).order_by("modified").first() if myTask is not None: logger.info("Running Task %s Task ID %s" % (myTask.crawlState.name,str(myTask.id))) funcName=myTask.crawlState.name objID=myTask.objID finyear=myTask.finyear isError=getattr(crawlFunctions1,funcName)(logger,objID,finyear=finyear) if isError==False: isComplete=True myTask.isComplete=isComplete myTask.save() else: logger.debug("No tasks to run") time.sleep(10) logger.info("...END PROCESSING") exit(0)
def main(): args = argsFetch() logger = loggerFetch(args.get('log_level')) logger.info('args: %s', str(args)) db = dbInitialize(db="libtech", charset="utf8") # The rest is updated automatically in the function cur=db.cursor() db.autocommit(True) #Query to set up Database to read Hindi Characters query="SET NAMES utf8" cur.execute(query) query="select phone,bid from callLogs where DATE(callStartTime) = CURDATE() group by phone " cur.execute(query) results=cur.fetchall() for row in results: #rowid=str(row[0]) phone=row[0] logger.info(" Phone %s " % (phone)) query="select count(*) from callLogs where phone='%s'" % (phone) totalCalls=singleRowQuery(cur,query) query="select count(*) from callLogs where phone='%s' and status='pass'" % (phone) totalSuccessCalls=singleRowQuery(cur,query) if totalCalls > 0: logger.info("Calculating Percentage") successP=math.trunc(totalSuccessCalls*100/totalCalls) else: successP=0 logger.info("Total Calls %s Success Calls %s Success Percentage %s " % (str(totalCalls),str(totalSuccessCalls),str(successP))) query="update addressbook set totalCalls='%s',successPercentage='%s' where phone='%s' " % (str(totalCalls),str(successP),phone) cur.execute(query) dbFinalize(db) # Make sure you put this if there are other exit paths or errors logger.info("...END PROCESSING") exit(0)
def main(): args = argsFetch() logger = loggerFetch(args.get('log_level')) if args['populate']: panchayatCode=args['panchayatCode'] blockCode=args['blockCode'] if panchayatCode is not None: eachPanchayat=Panchayat.objects.filter(code=panchayatCode).first() CrawlQueue.objects.create(panchayat=eachPanchayat) elif blockCode is not None: eachBlock=Block.objects.filter(code=blockCode).first() CrawlQueue.objects.create(block=eachBlock,priority=500) if args['execute']: stage=args['step'] limit=args['limit'] qid=args['qid'] if qid is not None: libtechCrawler(logger,stage,qid=qid) else: libtechCrawler(logger,stage) logger.info("...END PROCESSING") exit(0)
def main(): logger = loggerFetch("info") logger.info("BEGIN PROCESSING...") phone='9845155447' #Logic to upload the wave_file for the first time - wave_file = '1629_RSCD281016.wav' # '1431_CgbcstChakeri010316.wav' # This should be done only once for each new file filename = wave_file.strip('.wav') if wave_file: logger.info(awaazdeUpload(logger, wave_file)) if False: vendorcallid = awaazdePlaceCall(logger, phone, wave_file) time.sleep(10) else: vendorcallid = '71503' # '70239' # '70221' # '57897' logger.info("Call ID [%s]" % vendorcallid) callreturn = awaazdeStatusCheck(logger, vendorcallid) logger.info(callreturn) logger.info("...END PROCESSING")
def runTestSuite(): args = argsFetch() logger = loggerFetch(args.get('log_level')) logger.info('args: %s', str(args)) logger.info("BEGIN PROCESSING...") display = displayInitialize(args['visible']) driver = driverInitialize(args['browser']) # Mynk to use personal profile driver = driverInitialize(browser=args['browser'] , path='/home/mayank/.mozilla/firefox/4s3bttuq.default/') if args['cookie_dump']: cookieDump(driver) logger.info("Fetching [%s]" % driver.current_url) logger.info(wdTest(driver, args['url'])) logger.info("Fetched [%s]" % driver.current_url) if args['cookie_dump']: cookieDump(driver) driverFinalize(driver) displayFinalize(display) ''' display = vDisplayInitialize(visible) driver = driverInitialize(browser) logger.info(wdTest(driver)) driverFinalize(driver) vDisplayFinalize(display) ''' logger.info("...END PROCESSING")
def main(): args = argsFetch() logger = loggerFetch(args.get('log_level')) logger.info('args: %s', str(args)) if args['limit']: limit = int(args['limit']) else: limit =1 finyear=args['finyear'] stateCodes=['33','34','16','27','24','15','18','35'] stateCodes=['33','34','27','24','15','18',] stateCodes=['16','31','05','17'] stateCodes=['15'] stateCodes=[args['stateCode']] for stateCode in stateCodes: # if stateCode is not None: # logger.info("StateCode is %s" % stateCode) myPanchayats=Panchayat.objects.filter(crawlRequirement='FULL',block__district__state__code=stateCode) reportType="workPayment" # else: # myPanchayats=Panchayat.objects.filter(crawlRequirement='FULL')[:limit] for eachPanchayat in myPanchayats: logger.info("**********************************************************************************") panchayatReport=PanchayatReport.objects.filter(reportType=reportType,finyear=finyear,panchayat=eachPanchayat).first() if panchayatReport is None: logger.info("Createing work Payment report for panchayat: %s panchayatCode: %s ID: %s" % (eachPanchayat.name,eachPanchayat.code,str(eachPanchayat.id))) outcsv='' outcsv+="jobcard,name,musterNo,workCode,workName,dateFrom,dateTo,daysWorked,totalWage,accountNo,musterStatus,creditedDate,secondSignatoryDate,wagelistNo" outcsv+="\n" workRecords=WorkDetail.objects.filter(muster__block__panchayat__id=eachPanchayat.id,muster__finyear=finyear) workRecords=WorkDetail.objects.filter(muster__panchayat=eachPanchayat,muster__finyear=finyear) logger.info("Total Work Records: %s " %str(len(workRecords))) for wd in workRecords: workName=wd.muster.workName.replace(","," ") applicantName=wd.zname.replace(",","") wagelistArray=wd.wagelist.all() if len(wagelistArray) > 0: wagelist=wagelistArray[len(wagelistArray) -1 ] else: wagelist='' if wd.applicant: fatherHusbandName=wd.applicant.fatherHusbandName.replace(",","") else: fatherHusbandName=None outcsv+="%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s" % (wd.zjobcard,applicantName,wd.muster.musterNo,wd.muster.workCode,workName,str(wd.muster.dateFrom),str(wd.muster.dateTo),str(wd.daysWorked),str(wd.totalWage),wd.zaccountNo,wd.musterStatus,str(wd.creditedDate),str(wd.muster.paymentDate),wagelist) outcsv+="\n" try: outcsv=outcsv.encode("UTF-8") except: outcsv=outcsv csvfilename=eachPanchayat.slug+"_"+finyear+"_wp.csv" savePanchayatReport(logger,eachPanchayat,finyear,reportType,csvfilename,outcsv) logger.info("...END PROCESSING") exit(0)
def setUp(self): self.args = argsFetch() self.logger = loggerFetch('info') self.logger.info('BEGIN PROCESSING') self.display = displayInitialize(self.args['visible']) self.driver = driverInitialize(browser=self.args['browser'] , path='/home/mayank/.mozilla/firefox/4s3bttuq.default/') self.cmd = 'nohup youtube-dl --username %s --password %s -o %s ' % (user, password, '"%(title)s.%(ext)s"') self.url = self.args['url']
def setUp(self): self.logger = loggerFetch('info') self.logger.info('BEGIN PROCESSING...') self.display = displayInitialize(0) # self.driver = driverInitialize(timeout=3) self.driver = driverInitialize( timeout=3, options='--headless' ) # driverInitialize(path='/opt/firefox/', timeout=3)
def setUp(self): self.logger = loggerFetch('info') self.logger.info('BEGIN PROCESSING') # Pass 0 for headless self.display = displayInitialize(1) #, path='/path/to/firefox/') self.driver = driverInitialize(browser='firefox') self.url = URL
def main(): args = argsFetch() logger = loggerFetch(args.get('log_level')) logger.info('args: %s', str(args)) logger.info("BEGIN PROCESSING...") if args['district']: districtName=args['district'].lower() if args['limit']: limitString=" limit %s " % (str(args['limit'])) else: limitString=" " if args['reportID']: reportIDFilter= " and id = %s " % args['reportID'] else: reportIDFilter= " " db = dbInitialize(db=districtName.lower(), charset="utf8") # The rest is updated automatically in the function cur=db.cursor() db.autocommit(True) #Query to set up Database to read Hindi Characters query="SET NAMES utf8" cur.execute(query) crawlIP,stateName,stateCode,stateShortCode,districtCode=getDistrictParams(cur,districtName) htmlDir=nregaDir.replace("districtName",districtName.lower()) #block Reports query="select b.name,b.blockCode from blocks b where b.isRequired=1 %s" % limitString #query="select b.name,b.blockCode from blocks b where b.isRequired=1 limit 1" #query="select b.name,b.blockCode,p.name,p.panchayatCode from panchayats p, blocks b where b.blockCode=p.blockCode and p.isRequired=1 limit 1" #query="select b.name,b.blockCode,p.name,p.panchayatCode from panchayats p, blocks b where b.blockCode=p.blockCode and p.isRequired=1 and b.blockCode='005' " # query="select b.name,b.blockCode,p.name,p.panchayatCode from panchayats p, blocks b where b.blockCode=p.blockCode and p.isRequired=1 and b.blockCode='003' and panchayatCode='013'" cur.execute(query) results=cur.fetchall() for row in results: blockName=row[0] blockCode=row[1] genReport(cur,logger,1,htmlDir,'16',districtName,blockCode,blockName,'','',reportIDFilter) genReport(cur,logger,1,htmlDir,'17',districtName,blockCode,blockName,'','',reportIDFilter) genReport(cur,logger,1,htmlDir,'all',districtName,blockCode,blockName,'','',reportIDFilter) query="select b.name,b.blockCode,p.name,p.panchayatCode from panchayats p, blocks b where b.blockCode=p.blockCode and p.isRequired=1 %s" % limitString #query="select b.name,b.blockCode,p.name,p.panchayatCode from panchayats p, blocks b where b.blockCode=p.blockCode and p.isRequired=1 limit 1" cur.execute(query) results=cur.fetchall() for row in results: blockName=row[0] blockCode=row[1] panchayatName=row[2] panchayatCode=row[3] finyear='16' genReport(cur,logger,0,htmlDir,'16',districtName,blockCode,blockName,panchayatCode,panchayatName,reportIDFilter) genReport(cur,logger,0,htmlDir,'17',districtName,blockCode,blockName,panchayatCode,panchayatName,reportIDFilter) genReport(cur,logger,0,htmlDir,'all',districtName,blockCode,blockName,panchayatCode,panchayatName,reportIDFilter) dbFinalize(db) # Make sure you put this if there are other exit paths or errors logger.info("...END PROCESSING") exit(0)
def main(): args = argsFetch() logger = loggerFetch(args.get('log_level')) logger.info('args: %s', str(args)) logger.info("...END PROCESSING") exit(0)
def main(): args = argsFetch() logger = loggerFetch(args.get('log_level')) logger.info('args: %s', str(args)) url = "http://164.100.129.6/netnrega/dynamic_account_details.aspx" if args['limit']: limit = int(args['limit']) else: limit =1 stateCode=args['stateCode'] if stateCode is not None: myPanchayats=Panchayat.objects.filter(Q (crawlRequirement="FULL") & (Q (jobcardCrawlDate__lt = jobCardRegisterTimeThreshold) | Q(jobcardCrawlDate__isnull = True ) ) & Q (block__district__state__code=stateCode)).order_by('jobcardCrawlDate')[:limit] else: myPanchayats=Panchayat.objects.filter(Q (crawlRequirement="FULL") & (Q (jobcardCrawlDate__lt = jobCardRegisterTimeThreshold) | Q(jobcardCrawlDate__isnull = True ) ) ).order_by('jobcardCrawlDate')[:limit] # myPanchayats=Panchayat.objects.filter(fullPanchayatCode='3405003010').order_by('jobcardCrawlDate')[:limit] for eachPanchayat in myPanchayats: logger.info("Processing : panchayat: %s " % (eachPanchayat.name)) stateCode=eachPanchayat.block.district.state.code fullDistrictCode=eachPanchayat.block.district.code fullBlockCode=eachPanchayat.block.code fullPanchayatCode=eachPanchayat.code districtName=eachPanchayat.block.district.name blockName=eachPanchayat.block.name stateName=eachPanchayat.block.district.state.name logger.info("Processing StateCode %s, fullDistrictCode : %s, fullBlockCode : %s, fullPanchayatCode: %s " % (stateCode,fullDistrictCode,fullBlockCode,fullPanchayatCode)) cur='' htmlresponse,myhtml=getWorkerList(logger,url,stateCode,fullDistrictCode,fullBlockCode,fullPanchayatCode) if htmlresponse['status'] == '200': logger.info("File Downloaded SuccessFully") status,outhtml,outcsv=alterWorkerList(cur,logger,myhtml,stateName,districtName,blockName,eachPanchayat.name) if status is not None: filename=eachPanchayat.slug+"_jr.html" filenamecsv=eachPanchayat.slug+"_jr.csv" try: outhtml=outhtml.encode("UTF-8") except: outhtml=outhtml try: outcsv=outcsv.encode("UTF-8") except: outcsv=outcsv logger.info("Processing StateCode %s, fullDistrictCode : %s, fullBlockCode : %s, fullPanchayatCode: %s " % (stateCode,fullDistrictCode,fullBlockCode,fullPanchayatCode)) finyear=getCurrentFinYear() reportType="jobcardRegisterHTML" savePanchayatReport(logger,eachPanchayat,finyear,reportType,filename,outhtml) reportType="jobcardRegisterCSV" savePanchayatReport(logger,eachPanchayat,finyear,reportType,filenamecsv,outcsv) # eachPanchayat.jobcardRegisterFile.save(filename, ContentFile(outhtml)) eachPanchayat.jobcardCrawlDate=timezone.now() eachPanchayat.status='1' eachPanchayat.save() logger.info("Processing StateCode %s, fullDistrictCode : %s, fullBlockCode : %s, fullPanchayatCode: %s " % (stateCode,fullDistrictCode,fullBlockCode,fullPanchayatCode)) logger.info("...END PROCESSING") exit(0)
def main(): args = argsFetch() logger = loggerFetch(args.get('log_level')) logger.info('args: %s', str(args)) logger.info("BEGIN PROCESSING...") #This is a Kludge to remove all the input tags from the html because for some reason Beautiful Soup does not parse the html correctly regex=re.compile(r'<input+.*?"\s*/>+',re.DOTALL) if args['district']: districtName=args['district'] logger.info("DistrictName "+districtName) if args['finyear']: finyear=args['finyear'] logger.info("finyear "+finyear) fullFinyear=getFullFinYear(finyear) db = dbInitialize(db=districtName.lower(), charset="utf8") # The rest is updated automatically in the function cur=db.cursor() db.autocommit(True) #Query to set up Database to read Hindi Characters query="SET NAMES utf8" cur.execute(query) crawlIP,stateName,stateCode,stateShortCode,districtCode=getDistrictParams(cur,districtName) #Query to get all the blocks query="select stateCode,districtCode,blockCode,name from blocks where isRequired=1" cur.execute(query) results = cur.fetchall() for row in results: stateCode=row[0] districtCode=row[1] blockCode=row[2] blockName=row[3] fullBlockCode=stateCode+districtCode+blockCode logger.info(stateCode+districtCode+blockCode+blockName) url="http://"+crawlIP+"/netnrega/FTO/fto_reprt_detail.aspx?lflag=local&flg=W&page=b&state_name="+stateName.upper()+"&state_code="+stateCode+"&district_name="+districtName.upper()+"&district_code="+stateCode+districtCode+"&block_name="+blockName+"&block_code="+stateCode+districtCode+blockCode+"&fin_year="+fullFinyear+"&typ=pb&mode=b" logger.info(url) r = requests.get(url) htmlsource=r.text htmlsoup=BeautifulSoup(htmlsource,"html.parser") for fto in htmlsoup.find_all('a'): ftoNo=fto.contents[0] #ftoURL="http://164.100.112.66/netnrega/FTO/"+fto.get('href') if fullBlockCode in ftoNo: query="insert into ftoDetails (ftoNo,stateCode,districtCode,blockCode,finyear) values ('"+ftoNo+"','"+stateCode+"','"+districtCode+"','"+blockCode+"','"+finyear+"');" try: logger.info(query) cur.execute(query) except MySQLdb.IntegrityError,e: errormessage=(time.strftime("%d/%m/%Y %H:%M:%S "))+str(e)+"\n" errorfile.write(errormessage) continue
def main(): args = argsFetch() logger = loggerFetch(args.get('log_level')) logger.info('args: %s', str(args)) logger.info("BEGIN PROCESSING...") limitString='' if args['limit']: limitString=' limit '+args['limit'] if args['district']: districtName=args['district'] logger.info("DistrictName "+districtName) finyear=args['finyear'] db = dbInitialize(db=districtName.lower(), charset="utf8") # The rest is updated automatically in the function cur=db.cursor() db.autocommit(True) #Query to set up Database to read Hindi Characters query="SET NAMES utf8" cur.execute(query) logger.info("finyear "+finyear) fullFinYear=getFullFinYear(finyear) crawlIP,stateName,stateCode,stateShortCode,districtCode=getDistrictParams(cur,districtName) #Query to get all the blocks ftorawfilepath=nregaRawDataDir.replace("districtName",districtName.lower()) query="select id,blockCode,name from blocks order by rejectedPaymentDownloadDate DESC" cur.execute(query) results=cur.fetchall() for row in results: rowid=str(row[0]) blockCode=row[1] blockName=row[2] fullBlockCode=stateCode+districtCode+blockCode fullDistrictCode=stateCode+districtCode url="http://164.100.129.4/netnrega/FTO/rejection.aspx?lflag=eng&state_code=%s&state_name=%s&district_code=%s&page=d&Block_code=%s&Block_name=%s&district_name=%s&fin_year=%s&typ=I&linkr=X" % (stateCode,stateName.upper(),fullDistrictCode,fullBlockCode,blockName.upper(),districtName.upper(),fullFinYear) logger.info(url) r=requests.get(url) time.sleep(300) inhtml=r.text ftorawfilename=ftorawfilepath+blockName.upper()+"/FTO/"+fullFinYear+"/rejected.html" writeFile(ftorawfilename,inhtml) query="update blocks set rejectedPaymentDownloadDate=NOW() where id=%s" % rowid logger.info(query) cur.execute(query) dbFinalize(db) # Make sure you put this if there are other exit paths or errors logger.info("...END PROCESSING") exit(0)
def setUp(self): self.args = argsFetch() self.logger = loggerFetch('info') self.logger.info('BEGIN PROCESSING') self.display = displayInitialize(self.args['visible']) self.driver = driverInitialize( browser=self.args['browser'], path='/home/mayank/.mozilla/firefox/4s3bttuq.default/') self.cmd = 'nohup youtube-dl --username %s --password %s -o %s ' % ( user, password, '"%(title)s.%(ext)s"') self.url = self.args['url']
def main(): alwaysTag=LibtechTag.objects.filter(name="Always") args = argsFetch() logger = loggerFetch(args.get('log_level')) logger.info('args: %s', str(args)) finyear=getCurrentFinYear() fullfinyear=getFullFinYear(finyear) url = "http://164.100.129.6/netnrega/dynamic_account_details.aspx" if args['limit']: limit = int(args['limit']) else: limit =1 stateCode=args['stateCode'] panchayatCode=args['panchayatCode'] if panchayatCode is not None: myPanchayats=Panchayat.objects.filter(code=panchayatCode) elif stateCode is not None: myPanchayats=Panchayat.objects.filter(Q (crawlRequirement="FULL") & (Q (applicationRegisterCrawlDate__lt = jobCardRegisterTimeThreshold) | Q(applicationRegisterCrawlDate__isnull = True ) ) & Q (block__district__state__code=stateCode)).order_by('applicationRegisterCrawlDate')[:limit] else: myPanchayats=Panchayat.objects.filter(Q (crawlRequirement="FULL") & Q(block__district__state__isNIC=True) & (Q (applicationRegisterCrawlDate__lt = jobCardRegisterTimeThreshold) | Q(applicationRegisterCrawlDate__isnull = True ) ) ).order_by('applicationRegisterCrawlDate')[:limit] # myPanchayats=Panchayat.objects.filter(fullPanchayatCode='3405003010').order_by('applicationRegisterCrawlDate')[:limit] for eachPanchayat in myPanchayats: myhtml=getJobcardRegister(logger,eachPanchayat) myhtml=str(myhtml).replace("</nobr><br>",",") myhtml=myhtml.replace("bordercolor='#111111'>","bordercolor='#111111'><tr>") # myhtml = myhtml.replace('\n', ' ').replace('\r', '') # myhtml=str(myhtml).replace("</td> <tr>","</td></tr><tr>") # driver.get(panchayatPageURL) # elem = driver.find_element_by_link_text("Registration Application Register") # elem.click() # myhtml = driver.page_source # logger.info(myhtml) # with open("/tmp/abcd.html","w") as f: # f.write(myhtml) error,myTable=validateApplicationRegister(logger,myhtml,eachPanchayat.block) if error is None: logger.info("No error") outhtml='' outhtml+=stripTableAttributes(myTable,"myTable") title="Jobcard Register: state:%s District:%s block:%s panchayat: %s finyear:%s " % (eachPanchayat.block.district.state.name,eachPanchayat.block.district.name,eachPanchayat.block.name,eachPanchayat.name,fullfinyear) outhtml=htmlWrapperLocal(title=title, head='<h1 aling="center">'+title+'</h1>', body=outhtml) try: outhtml=outhtml.encode("UTF-8") except: outhtml=outhtml filename="applicationRegister_%s_%s.html" % (eachPanchayat.slug,finyear) reportType="applicationRegister" savePanchayatReport(logger,eachPanchayat,finyear,reportType,filename,outhtml) eachPanchayat.applicationRegisterCrawlDate=timezone.now() eachPanchayat.save() else: logger.info("Error") logger.info("...END PROCESSING") exit(0)
def main(): args = argsFetch() logger = loggerFetch(args.get('log_level')) logger.info('args: %s', str(args)) logger.info("BEGIN PROCESSING...") if args['district']: districtName=args['district'].lower() finyear=args['finyear'] db = dbInitialize(db=districtName, charset="utf8") # The rest is updated automatically in the function cur=db.cursor() db.autocommit(True) #Query to set up Database to read Hindi Characters query="SET NAMES utf8" cur.execute(query) limitString='' if args['limit']: limitString="limit "+str(args['limit']) # if args['copyPhones']: # logger.info("Copying Phones from libtech to Surguja Database") # copyPhones(cur,logger) # if args['updateRejectionReason']: # logger.info("This loop will update RejectionReason") # updateRejectionReason(cur,logger,districtName) # if args['updateFinYear']: # logger.info("This loop will update financial year in workDetails") # updateFinYear(cur,logger,districtName,limitString) if args['revertFTOInformation']: logger.info("This loop will update FTO Information") if args['revertWorkID']: wdID=args['revertWorkID'] revertFTOInformation(cur,logger,wdID) else: query="select id from workDetails where finyear='%s' order by id DESC %s" % (finyear,limitString) cur.execute(query) results=cur.fetchall() for row in results: wdID=row[0] revertFTOInformation(cur,logger,wdID) if args['updateJCNumber']: logger.info("This will update jcNumber") updateJCNumber(cur,logger,districtName,finyear,limitString) # if args['updatePrimaryAccountHolder']: # logger.info("This loop will update Primary Account Holder") # updatePrimaryAccountHolder(cur,logger,districtName) dbFinalize(db) # Make sure you put this if there are other exit paths or errors logger.info("...END PROCESSING") exit(0)
def main(): args = argsFetch() logger = loggerFetch(args.get('log_level')) logger.info('args: %s', str(args)) myPanchayats=State.objects.all() for eachPanchayat in myPanchayats: logger.info("Panchayt Code: %s, PanchayatName: %s " % (eachPanchayat.code,eachPanchayat.name)) eachPanchayat.save() logger.info("...END PROCESSING") exit(0)
def setUp(self): args = argsFetch() level = args['log_level'] if not level: level = 'info' self.logger = loggerFetch(level) self.logger.info('args: %s', str(args)) self.logger.info('BEGIN PROCESSING...') self.parse = args['parse'] dirname = args['directory'] if not self.parse: self.display = displayInitialize(args['visible']) self.driver = driverInitialize(timeout=3, options='--headless') # driverInitialize(path='/opt/firefox/', timeout=3)
def main(): args = argsFetch() logger = loggerFetch(args.get('log_level')) if args['initiateCrawl']: logger.debug("This script is going to initiate crawl") if args['step']: crawlStates=CrawlState.objects.filter(name=args['step']) else: #crawlStates=CrawlState.objects.all() crawlStates=CrawlState.objects.filter(isActive=True) for eachState in crawlStates: crawlProcessTimeThreshold=eachState.threshold curStateName=eachState.name logfile="%s/%s.log" % (logDir,curStateName) debuglogfile="/%s/debug/%s_%s.log" % (logDir,curStateName,str(int(time.time()))) logger.debug("Curent state name is %s " % curStateName) curhour=datetime.now().hour nicTimeBand=False if (curhour >=6) and (curhour <= 20): nicTimeBand=True scriptDir='%s/custom/crawlScripts/' % djangoDir if ((eachState.nicHourRestriction==False) or ((eachState.nicHourRestriction==True) and (nicTimeBand==True))): cmd="python %s/crawlMain.py -e -l debug -step %s " % (scriptDir,curStateName) # cmd=scriptName p1 = Popen(['pgrep', '-f', cmd], stdout=PIPE) # logger.debug(p1.communicate()) mypid = p1.communicate()[0].decode("utf-8").lstrip().rstrip() logger.debug("Exsiting PID for this command %s is %s " % (curStateName,str(mypid))) if (mypid == ""): logger.debug("We are going to launch this program %s" % cmd) proc = Popen([cmd], shell=True,stdin=None, stdout=None, stderr=None, close_fds=True) else: mycmd='ps -o etimes= -p %s ' % mypid p1 = Popen([mycmd], stdout=PIPE,shell=True) output = p1.communicate()[0].decode("utf-8").lstrip().rstrip() logger.debug(output) if int(output) > crawlProcessTimeThreshold: mycmd="kill -9 %s " % mypid mycmd="pkill -P %s " % mypid logger.debug(mycmd) p1 = Popen([mycmd], stdout=PIPE,shell=True) output = p1.communicate()[0].decode("utf-8").lstrip().rstrip() logger.debug(output) mycmd="cp %s %s " % (logfile,debuglogfile) p1 = Popen([mycmd], stdout=PIPE,shell=True) output = p1.communicate()[0].decode("utf-8").lstrip().rstrip() logger.debug(output) logger.info("...END PROCESSING") exit(0)
def main(): args = argsFetch() logger = loggerFetch(args.get('log_level')) logger.info('args: %s', str(args)) logger.info("BEGIN PROCESSING...") limitString='' if args['limit']: limitString=" limit %s " % args['limit'] additionalFilters='' if args['district']: additionalFilters+= " and p.districtName='%s' " % args['district'] if args['fullPanchayatCode']: additionalFilters= " and p.fullPanchayatCode='%s' " % args['fullPanchayatCode'] db = dbInitialize(db=nregaDB, charset="utf8") # The rest is updated automatically in the function cur=db.cursor() db.autocommit(True) #Query to set up Database to read Hindi Characters query="SET NAMES utf8" cur.execute(query) query="select p.stateCode,p.districtCode,p.blockCode,p.panchayatCode,p.stateName,p.districtName,p.rawBlockName,p.panchayatName,p.fullPanchayatCode,p.stateShortCode,p.crawlIP,p.fullBlockCode from panchayats p,panchayatStatus ps where p.fullPanchayatCode=ps.fullPanchayatCode and p.isRequired=1 and ( (TIMESTAMPDIFF(DAY, ps.jobcardCrawlDate, now()) > 7) or ps.jobcardCrawlDate is NULL) %s order by ps.jobcardCrawlDate,fullPanchayatCode %s" % (additionalFilters,limitString) cur.execute(query) results=cur.fetchall() for row in results: [stateCode,districtCode,blockCode,panchayatCode,stateName,districtName,blockName,panchayatName,fullPanchayatCode,stateShortCode,crawlIP,fullBlockCode]=row logger.info("Processing statName: %s, districtName: %s, blockName: %s, panchayatName : %s " % (stateName,districtName,blockName,panchayatName)) filepath=nregaWebDir.replace("stateName",stateName.upper()).replace("districtName",districtName.upper()) filename=filepath+blockName.upper()+"/%s/%s_MISjobcardRegister.html" % (panchayatName.upper(),panchayatName.upper()) fullDistrictCode=stateCode+districtCode htmlresponse,myhtml=getWorkerList(cur,logger,stateCode,fullDistrictCode,fullBlockCode,fullPanchayatCode) if htmlresponse['status'] == '200': logger.info("File Downloaded SuccessFully") status,outhtml=alterWorkerList(cur,logger,myhtml,stateName,districtName,blockName,panchayatName) if status == 1: if not os.path.exists(os.path.dirname(filename)): os.makedirs(os.path.dirname(filename)) myfile = open(filename, "wb") myfile.write(outhtml.encode("UTF-8")) query="update panchayatStatus set jobcardCrawlDate=NOW() where fullPanchayatCode='%s' " % fullPanchayatCode cur.execute(query) #myfile.write(outhtml) else: logger.info("Not Downloaded") logger.info(filename) dbFinalize(db) # Make sure you put this if there are other exit paths or errors logger.info("...END PROCESSING") exit(0)
def main(): args = argsFetch() logger = loggerFetch(args.get('log_level')) logger.info('args: %s', str(args)) logger.info("BEGIN PROCESSING...") limitString='' if args['limit']: limitString=" limit %s " % args['limit'] additionalFilters='' if args['district']: additionalFilters+= " and p.districtName='%s' " % args['district'] if args['fullPanchayatCode']: additionalFilters= " and p.fullPanchayatCode='%s' " % args['fullPanchayatCode'] db = dbInitialize(db=nregaDB, charset="utf8") # The rest is updated automatically in the function cur=db.cursor() db.autocommit(True) #Query to set up Database to read Hindi Characters query="SET NAMES utf8" cur.execute(query) query="select p.stateCode,p.districtCode,p.blockCode,p.panchayatCode,p.stateName,p.districtName,p.rawBlockName,p.panchayatName,p.fullPanchayatCode,p.stateShortCode,p.crawlIP,p.fullBlockCode from panchayats p,panchayatStatus ps where p.fullPanchayatCode=ps.fullPanchayatCode and p.isRequired=1 and ( (TIMESTAMPDIFF(DAY, ps.jobcardCrawlDate, now()) > 7) or ps.jobcardCrawlDate is NULL) %s order by ps.jobcardCrawlDate,fullPanchayatCode %s" % (additionalFilters,limitString) cur.execute(query) results=cur.fetchall() for row in results: [stateCode,districtCode,blockCode,panchayatCode,stateName,districtName,blockName,panchayatName,fullPanchayatCode,stateShortCode,crawlIP,fullBlockCode]=row logger.info("Processing statName: %s, districtName: %s, blockName: %s, panchayatName : %s " % (stateName,districtName,blockName,panchayatName)) fullDistrictCode=stateCode+districtCode filepath=nregaWebDir.replace("stateName",stateName.upper()).replace("districtName",districtName.upper()) filename=filepath+blockName.upper()+"/%s/%s_stats.html" % (panchayatName.upper(),panchayatName.upper()) logger.info("filename: %s " % filename) url="http://mnregaweb4.nic.in/netnrega/all_lvl_details_new.aspx?panchayat_code=%s&panchayat_name=%s&block_code=%s&block_name=%s&district_code=%s&district_name=%s&state_code=%s&state_name=%s&page=p&fin_year=2015-2016" % (fullPanchayatCode,panchayatName,fullBlockCode,blockName,fullDistrictCode,districtName,stateCode,stateName) logger.info(url) # htmlresponse,myhtml=getStatus(cur,logger,glanceURL,stateCode,fullDistrictCode,fullBlockCode,fullPanchayatCode) httplib2.debuglevel = 1 h = httplib2.Http('.cache') htmlresponse, myhtml = h.request(url) if htmlresponse['status'] == '200': logger.info("File Downloaded SuccessFully") if not os.path.exists(os.path.dirname(filename)): os.makedirs(os.path.dirname(filename)) myfile = open(filename, "wb") try: myfile.write(myhtml.encode("UTF-8")) except: myfile.write(myhtml) dbFinalize(db) # Make sure you put this if there are other exit paths or errors logger.info("...END PROCESSING") exit(0)
def main(): args = argsFetch() logger = loggerFetch(args.get('log_level')) logger.info('args: %s', str(args)) if args['populateWeights']: db = firestore.Client() logger.info("Going to Populate the database") if args['customer']: customer=args['customer'] jsonName="%s.json" % (customer) json_data=open(jsonName,encoding='utf-8-sig').read() d = json.loads(json_data) i=0 for item in d: i=i+1 weight=item logger.info(weight) doc_ref = db.collection(customer).document(str(i)) doc_ref.set({ u'weight': weight }) if args['populate']: db = firestore.Client() logger.info("Going to Populate the database") jsonName="sample_data.json" json_data=open(jsonName,encoding='utf-8-sig').read() d = json.loads(json_data) i=0 for item in d: i=i+1 name=item[0] value=item[1] logger.info(name) doc_ref = db.collection(u'products').document(str(i)) doc_ref.set({ u'name': name, u'value': value }) # Project ID is determined by the GCLOUD_PROJECT environment variable # # doc_ref = db.collection(u'users').document(u'alovelace') # doc_ref.set({ # u'first': u'Ada', # u'last': u'Lovelace', # u'born': 1815 # }) logger.info("...END PROCESSING") exit(0)
def runTestSuite(): logger = loggerFetch("info") logger.info("BEGIN PROCESSING...") display = displayInitialize(1) driver = driverInitialize(path='/opt/firefox/') driver.get(url) logger.info("Fetching...[%s]" % url) driver.get(url) # A double refresh required for the page to load logger.info("Refreshing...[%s]" % url) html_source = driver.page_source.replace('<head>', '<head><meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>') logger.debug("HTML Fetched [%s]" % html_source) bs = BeautifulSoup(html_source, "html.parser") tr_list = bs.findAll('tr', attrs={'class':['normalRow', 'alternateRow']}) logger.debug(str(tr_list)) for tr in tr_list: td = tr.find('td') td = td.findNext('td') panchayat = td.text.strip() logger.info("Panchayat[%s]", panchayat) elem = driver.find_element_by_link_text(panchayat) elem.click() filename="/tmp/%s.html" % panchayat with open(filename, 'wb') as html_file: logger.info("Writing [%s]" % filename) html_file.write(driver.page_source.encode('utf-8')) driver.back() driverFinalize(driver) displayFinalize(display) logger.info("...END PROCESSING")
def runTestSuite(): logger = loggerFetch("info") logger.info("BEGIN PROCESSING...") display = displayInitialize(0) driver = driverInitialize() report = generate_report(logger, driver) logger.info('Finally: \n%s' % report) with open(filename, 'wb') as csv_file: logger.info("Writing to [%s]" % filename) csv_file.write(report.encode('utf-8')) driverFinalize(driver) displayFinalize(display) logger.info("...END PROCESSING") return
def setUp(self): self.logger = loggerFetch('info') self.logger.info('BEGIN PROCESSING...') self.display = displayInitialize(1) self.driver = driverInitialize(path='/opt/firefox/', timeout=3)
def setUp(self): self.logger = loggerFetch('info') self.logger.info('BEGIN PROCESSING...')
def main(): thresholdDate = datetime.strptime('Oct 15 2017', '%b %d %Y').date() memberIndex = 1 productNameIndex = 0 startCol = 6 args = argsFetch() logger = loggerFetch(args.get('log_level')) logger.info('args: %s', str(args)) print("test") if args['limit']: limit = int(args['limit']) else: limit = 1 #Lets Process Grocery Order Form if args['sheetID']: myOrderForms = ConsolidatedOrderForm.objects.filter( isProcessed=0, id=args['sheetID'])[:limit] else: myOrderForms = ConsolidatedOrderForm.objects.filter( isProcessed=0)[:limit] for eachOrderForm in myOrderForms: isBeforeSoftware = False productCodeIndex = 2 priceIndex = 4 orderDate = eachOrderForm.orderDate if orderDate < thresholdDate: isBeforeSoftware = True productCodeIndex = 1 priceIndex = 2 creditIndex = eachOrderForm.creditIndex - 1 creditSheetIndex = eachOrderForm.creditSheetIndex sheetIndex = eachOrderForm.sheetNo balanceSheetIndex = eachOrderForm.balanceSheetIndex maxOrderRow = eachOrderForm.maxRow myFile = eachOrderForm.orderFile.read() book = xlrd.open_workbook(file_contents=myFile) #Balance Sheet Index worksheet = book.sheet_by_index(balanceSheetIndex) num_rows = worksheet.nrows - 1 curr_row = 0 vegetableCostIndex = 11 creditIndex = 10 adjustmentIndex = 11 adjustmentReasonIndex = 14 nameIndex = 4 emailIndex = 5 boxIDIndex = 1 password = "******" while curr_row < num_rows: curr_row = curr_row + 1 memberCode = str(worksheet.cell_value(curr_row, 0)).replace( "-", "").lstrip().rstrip() vegetableCost = int( worksheet.cell_value(curr_row, vegetableCostIndex)) if isBeforeSoftware == True: memberCode = str(worksheet.cell_value(curr_row, 0)).replace( "-", "").lstrip().rstrip() myMember = Member.objects.filter( user__username=memberCode).first() else: memberCode = int(worksheet.cell_value(curr_row, 0)) if memberCode != 0: boxID = str(worksheet.cell_value( curr_row, boxIDIndex)).lstrip().rstrip() email = str(worksheet.cell_value( curr_row, emailIndex)).lstrip().rstrip() name = str(worksheet.cell_value( curr_row, nameIndex)).lstrip().rstrip() myMember = Member.objects.filter( user__username=memberCode).first() logger.info(str(int(memberCode))) if myMember is None: myUser = User.objects.create_user(username=memberCode, email=email, password=password) Member.objects.create(user=myUser, boxID=boxID, name=name) else: myUser = myMember.user myUser.email = email myUser.save() myMember = Member.objects.filter( user__username=memberCode).first() #myMember=Member.objects.filter(boxID=memberCode).first() if myMember is None: allMembersFound = 0 logger.info("Member not found %s " % (memberCode)) else: if vegetableCost != 0: error = addMemberTransaction(logger, myMember, "VEGBOX", 1, vegetableCost, "DR", orderDate) #Making the credit and Adjustment if isBeforeSoftware == True: if worksheet.cell_type(curr_row, creditIndex) != 0: credit = str( worksheet.cell_value(curr_row, creditIndex)).replace( "-", "").lstrip().rstrip() error = addMemberTransaction(logger, myMember, "CRR", 1, credit, "CR", orderDate) if worksheet.cell_type(curr_row, adjustmentIndex) != 0: adjustment = str( worksheet.cell_value( curr_row, adjustmentIndex)).lstrip().rstrip() logger.info(adjustment) adjustmentReason = str( worksheet.cell_value( curr_row, adjustmentReasonIndex)).replace( "-", "").lstrip().rstrip() if "-" in adjustment: logger.info("Adjustment is Negative") adjustmentTransactionType = "CR" adjustment = adjustment[1:] else: adjustmentTransactionType = "DR" error = addMemberTransaction(logger, myMember, "ADJST", 1, adjustment, adjustmentTransactionType, orderDate, remarks=adjustmentReason) #This is for calcuating Credits if creditSheetIndex != 0: worksheet = book.sheet_by_index(creditSheetIndex) num_cols = worksheet.ncols - 1 num_rows = worksheet.nrows - 1 curr_row = 0 while curr_row < num_rows: curr_row += 1 memberCode = int(worksheet.cell_value(curr_row, 0)) logger.info("Credit memberCode %s " % str(memberCode)) Amount = worksheet.cell_value(curr_row, 1) creditDateExcel = worksheet.cell_value(curr_row, 2) creditDate = datetime( *xlrd.xldate_as_tuple(creditDateExcel, 0)).date() logger.info(str(memberCode) + str(Amount) + str(creditDate)) myMember = Member.objects.filter( user__username=memberCode).first() if myMember is not None: error = addMemberTransaction(logger, myMember, "CRR", 1, Amount, "CR", creditDate) #This is for calculating Products if eachOrderForm.isItemizedOrderFormat == False: worksheet = book.sheet_by_index(sheetIndex) isComplete = 1 firstRow = worksheet.row_values(0) logger.info(str(firstRow)) num_cols = worksheet.ncols - 1 curr_col = startCol allMembersFound = 1 while curr_col < num_cols: curr_col += 1 if worksheet.cell_type(memberIndex, curr_col) != 0: memberCode = str( worksheet.cell_value(memberIndex, curr_col)).replace("-", "") memberCode = memberCode[2:] # logger.info(memberCode) if isBeforeSoftware == True: myMember = Member.objects.filter( user__username=memberCode).first() else: myMember = Member.objects.filter( user__username=memberCode).first() #myMember=Member.objects.filter(boxID=memberCode).first() if myMember is None: allMembersFound = 0 logger.info("Member not found %s " % (memberCode)) else: curr_row = 2 while curr_row < maxOrderRow: curr_row += 1 if worksheet.cell_type(curr_row, curr_col) != 0: if worksheet.cell_type(curr_row, priceIndex) != 0: productName = worksheet.cell_value( curr_row, productNameIndex).lstrip().rstrip() productCode = worksheet.cell_value( curr_row, productCodeIndex).lstrip().rstrip() price = worksheet.cell_value( curr_row, priceIndex) quantity = worksheet.cell_value( curr_row, curr_col) if float(quantity) != 0: logger.info( "Product Code: %s Price : %s Quantity %s Member %s " % (productCode, price, str(int(quantity)), memberCode)) error = addMemberTransaction( logger, myMember, productCode, quantity, price, "DR", orderDate) if error is not None: isComplete = False else: logger.info("I am in itemized biling") productCodeIndex = 15 priceIndex = 12 quantityIndex = 11 orderStatusIndex = 19 worksheet = book.sheet_by_index(sheetIndex) isComplete = 1 num_rows = worksheet.nrows - 1 num_cols = worksheet.ncols - 1 curr_col = startCol allMembersFound = 1 row = 0 while row < num_rows: row = row + 1 rowValues = worksheet.row_values(row) # logger.info(str(rowValues)) logger.info(row) logger.info(worksheet.cell_value(row, 1)) memberCode = int(worksheet.cell_value(row, 1)) myMember = Member.objects.filter( user__username=memberCode).first() productCode = worksheet.cell_value( row, productCodeIndex).lstrip().rstrip() orderStatus = worksheet.cell_value( row, orderStatusIndex).lstrip().rstrip() quantity = worksheet.cell_value(row, quantityIndex) price = worksheet.cell_value(row, priceIndex) error = addMemberTransaction(logger, myMember, productCode, quantity, price, "DR", orderDate, orderStatus=orderStatus) if (isComplete == True) and (allMembersFound == 1): logger.info("Sheet is Completely Processed") logger.info("...END PROCESSING") exit(0)
def runTestSuite(): logger = loggerFetch("info") logger.info("BEGIN PROCESSING...") display = displayInitialize(1) driver = driverInitialize() ''' content = csv.reader(open('./gats.csv', 'r'), delimiter=',', quotechar='"') for (gat, d) in content: ''' for gat in gat_list: logger.info('Fetching gat[%s]...' % gat) driver.get(url) try: driver.find_element_by_xpath( "//form[@id='aspnetForm']/div[3]/div/div/div[3]/a[3]/p").click( ) except: logger.error('Cant find element for [%s]' % gat) continue Select( driver.find_element_by_id("distSelect")).select_by_visible_text(dn) Select( driver.find_element_by_id("talSelect")).select_by_visible_text(tn) Select( driver.find_element_by_id("vilSelect")).select_by_visible_text(vn) # Select(driver.find_element_by_id("vilSelect")).select_by_visible_text(u"सार्पिली") # driver.find_element_by_css_selector("option[value=\"string:273200030399810000\"]").click() driver.find_element_by_id("rbsryno").click() driver.find_element_by_xpath("//input[@type='number']").clear() driver.find_element_by_xpath("//input[@type='number']").send_keys(gat) driver.find_element_by_css_selector("input[type=\"button\"]").click() ''' from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support import expected_conditions as EC from selenium.common.exceptions import TimeoutException try: WebDriverWait(driver, 5).until(EC.alert_is_present(), 'Waiting for alert timed out') alert = driver.switch_to_alert() alert.accept() logger.warning("alert accepted") except: logger.info("Yippie!") ''' if len(driver.window_handles) > 1: logger.info("Dialog Box Window [" + str(driver.window_handles) + "]") # self.assertEqual(u"सर्वे नंबर / गट नंबर तपासुन पहा.", self.close_alert_and_get_its_text()) # driver.find_element_by_xpath("//div[@id='block-desktop']/div[3]/div[2]/div").click() driver.switch_to_alert().accept() continue html_source = driver.page_source.replace( '<head>', '<head><meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>' ) logger.debug("HTML Fetched [%s]" % html_source) soup = BeautifulSoup(driver.page_source, "html.parser") #dict_from_json = soup.find(attribute).text sno_select = soup.find( "select", {"ng-model": "selectedSno"}) # ng-model="selectedSno" sno_options = sno_select.findAll("option") snos = [sno_option.text for sno_option in sno_options] logger.debug("Found [%s]" % str(snos)) snos = snos[1:] logger.info("SNO List [%s]" % str(snos)) #time.sleep(5) for sno in snos: logger.info('Processing [%s]' % sno) filename = '/home/mayank/wd/SaatBaara/remaining/%s.html' % sno.replace( '/', '_') if os.path.exists(filename): #time.sleep(1) continue Select( driver.find_element_by_xpath( "//form[@id='aspnetForm']/div[3]/div/div/div[3]/div/div[3]/table/tbody/tr[3]/td/select" )).select_by_visible_text(sno) #logger.info(driver.find_element_by_link_text(sno)) driver.find_element_by_css_selector( "td.last-rows > input[type=\"button\"]").click() time.sleep(5) parent_handle = driver.current_window_handle logger.info("Handles : %s Number : %s" % (driver.window_handles, len(driver.window_handles))) if len(driver.window_handles) == 2: driver.switch_to_window(driver.window_handles[-1]) else: logger.error("Handlers gone wrong [" + str(driver.window_handles) + "]") driver.save_screenshot('z.png') html_source = driver.page_source.replace( '<head>', '<head><meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>' ).encode('utf-8') logger.debug("HTML Fetched [%s]" % html_source) if (driver.title != '७/१२'): logger.error(driver.title) driver.close() driver.switch_to_window(parent_handle) continue bs = BeautifulSoup(html_source, "html.parser") body = bs.find('tbody') try: body = body.findNext('tbody') except: logger.error('Empty body for [%s]' % sno) driver.close() driver.switch_to_window(parent_handle) continue body = body.findNext('tbody') logger.debug(body) td = body.find('td') td = td.findAll('td') logger.info("Checking [%s]" % td[2].text) if (sno != td[2].text): logger.error('sno[%s] != td.text[%s]' % (sno, td[2].text)) driver.close() driver.switch_to_window(parent_handle) continue with open(filename, 'wb') as html_file: logger.info('Writing [%s]' % filename) html_file.write(html_source) driver.close() driver.switch_to_window(parent_handle) time.sleep(1) time.sleep(1) driverFinalize(driver) displayFinalize(display) logger.info("...END PROCESSING")
def main(): args = argsFetch() logger = loggerFetch(args.get('log_level')) logger.info('args: %s', str(args)) if args['limit']: limit = int(args['limit']) else: limit = 1 if args['misc']: logger.info("Doing misc tasks") myobjs = MemberTransaction.objects.all() for obj in myobjs: obj.orderQuantity = obj.quantity obj.save() if args['memberSpend']: logger.info("Member wise spend") curmonth = 2 s = "" myTransactions = MemberTransaction.objects.filter( orderDate__month=curmonth, transactionType='DR') for eachTransaction in myTransactions: total = str(eachTransaction.price * eachTransaction.quantity) s += "%s,%s,%s\n" % (eachTransaction.member.user.username, str(eachTransaction.orderDate), total) with open("/tmp/ds.csv", "w") as f: f.write(s) if args['monthlySpend']: logger.info("Calculating Monthly Spend") myMonths = [4, 5, 6, 7, 8, 9, 10, 11, 12, 1, 2, 3] s = '' for curmonth in myMonths: total = 0 myTransactions = MemberTransaction.objects.filter( orderDate__month=curmonth, transactionType='DR') for eachTransaction in myTransactions: total = total + eachTransaction.price * eachTransaction.quantity logger.info(total) eachLine = "%s,%s\n" % (str(curmonth), str(total)) s += eachLine with open("/tmp/ms.csv", "w") as f: f.write(s) if args['dumpMonthlyBalance']: logger.info("Going to Dump Monthly Balance") myMembers = Member.objects.all() curmonth = 5 curyear = 2018 s = "MemberName,AprilBal,APrilCredit,MayBal,MayCredit,JuneBal,JuneCredit,JulyBal,JulyCredit,AUgustBal,AugustCredit,SeptemberBal,SeptemberCredit,OctoberBal,OctoberCredit,NovemberBal,NovemberCredit,DecemberBal,DecemberCredit,JanuaryBalance,JanuaryCredit,FebuaryBalance,FebuaryCredit\n" s = "memberCode,memberName,April,May,June,July,August,Sepetember,October,November,December,January,Feb,March\n" for eachMember in myMembers: s += str(eachMember.user.username) s += "," memberID = "%s-%s," % (eachMember.user.username, eachMember.name) s += memberID curyear = 2017 myMonths = [4, 5, 6, 7, 8, 9, 10, 11, 12] startmonth = 5 startyear = 2017 endmonth = 4 endyear = 2018 myDates = [ datetime.date(m // 12, m % 12 + 1, 1) for m in range(startyear * 12 + startmonth - 1, endyear * 12 + endmonth) ] for eachDate in myDates: lastTransaction = MemberTransaction.objects.filter( member=eachMember, orderDate__lt=eachDate).order_by("-orderDate", "-id").first() if lastTransaction is not None: lastBalance = lastTransaction.balance else: lastBalance = 0 s += str(lastBalance) s += "," # for curmonth in myMonths: # lastTransaction=MemberTransaction.objects.filter(member=eachMember,orderDate__year__lte=curyear,orderDate__month__lte=curmonth).order_by("-orderDate","id").first() # if lastTransaction is not None: # lastBalance=lastTransaction.balance # else: # lastBalance=0 # s+=str(lastBalance) # s+="," # myTransactions=MemberTransaction.objects.filter(member=eachMember,orderDate__year=curyear,orderDate__month=curmonth,product__id=181) # totalCredit=0 # for eachTransaction in myTransactions: # totalCredit+=eachTransaction.price # s+=str(totalCredit) # s+="," # curyear=2018 # myMonths=[1,2,3] # for curmonth in myMonths: # lastTransaction=MemberTransaction.objects.filter(member=eachMember,orderDate__year__lte=curyear,orderDate__month__lte=curmonth).order_by("-orderDate","id").first() # if lastTransaction is not None: # lastBalance=lastTransaction.balance # else: # lastBalance=0 # s+=str(lastBalance) # s+="," # # myTransactions=MemberTransaction.objects.filter(member=eachMember,orderDate__year=curyear,orderDate__month=curmonth,product__id=181) # totalCredit=0 # for eachTransaction in myTransactions: # totalCredit+=eachTransaction.price # s+=str(totalCredit) # s+="," s += "\n" with open("/tmp/mb.csv", "w") as f: f.write(s) if args['changeMemberCode']: logger.info("Changing Member Code") myUsers = User.objects.all() for eachUser in myUsers: username = eachUser.username areaCode = username[0:2] eachMember = Member.objects.filter(user=eachUser).first() if eachMember is not None: boxID = eachMember.boxID else: boxID = "abcd" boxID = boxID[2:] if boxID.isdigit(): if int(boxID) != 0 and int(boxID) != 1: logger.info('username : %s boxID : %s ' % (username, boxID)) eachUser.username = str(boxID) eachUser.save() else: a = 1 # myMembers=Member.objects.all()[:1] # for eachMember in myMembers: # logger.info(eachMember.boxID) if args['print']: orderDateString = args['orderDate'] orderDate = datetime.datetime.strptime(orderDateString, '%d-%m-%Y').date() myTransactions = MemberTransaction.objects.filter( orderDate=orderDate).values("member__boxID").annotate( dcount=Count('pk')) for eachTransaction in myTransactions: logger.info(str(eachTransaction)) boxID = eachTransaction['member__boxID'] count = eachTransaction['dcount'] logger.info(boxID + "-" + str(count)) # for eachTransaction in myTransactions: # logger.info("Prodct: %s Quantity: %s " % (eachTransaction.product.name,str(eachTransaction.quantity))) if args['deleteTransactions']: orderDateString = args['orderDate'] orderDate = datetime.datetime.strptime(orderDateString, '%d-%m-%Y').date() myTransactions = MemberTransaction.objects.filter(orderDate=orderDate) for eachTransaction in myTransactions: logger.info(eachTransaction.member) eachTransaction.delete() if args['dumpBalance']: myMembers = Member.objects.all() for eachMember in myMembers: lastTransaction = MemberTransaction.objects.filter( member=eachMember).order_by("-orderDate", "-id").first() if lastTransaction is not None: lastBalance = lastTransaction.balance else: lastBalance = 0 print(eachMember.user.username + "," + str(eachMember.boxID) + ",", str(lastBalance)) if args['calculateBalance']: myMembers = Member.objects.all()[:limit] for eachMember in myMembers: logger.info(eachMember.name) memberTransactions = MemberTransaction.objects.filter( member=eachMember).order_by('orderDate', 'id') curBalance = eachMember.startBalance curBalance = 0 logger.info("Current Balance %s " % str(curBalance)) for eachTransaction in memberTransactions: logger.info("%s-%s" % (str(eachTransaction.orderDate), str(eachTransaction.product.name))) transactionType = eachTransaction.transactionType price = eachTransaction.price quantity = eachTransaction.quantity if transactionType == "DR": amount = price * (-1) * quantity else: amount = price * 1 * quantity # logger.info("Price : %s, Amount : %s " % (str(price),str(amount))) curBalance = curBalance + amount eachMember.balance = curBalance # logger.info("Current Balance %s " % str(curBalance)) eachTransaction.balance = curBalance eachTransaction.save() eachMember.balance = curBalance eachMember.save() if args['deleteZeroTransaction']: myTransactions = MemberTransaction.objects.filter(quantity=0) for eachTransaction in myTransactions: logger.info(eachTransaction.quantity) eachTransaction.delete() myTransactions = MemberTransaction.objects.filter(price=0) for eachTransaction in myTransactions: logger.info(eachTransaction.price) eachTransaction.delete() if args['updateBoxID']: logger.info("We are going to update BOx IDs") with open('/tmp/mm.csv', 'r') as fp: for line in fp: line = line.lstrip().rstrip() if (line != ''): lineArray = str(line).split(",") username = lineArray[0] boxid = lineArray[1] logger.info("Old Username: %s box id %s " % (username, boxid)) myUser = User.objects.filter(username=username).first() myMember = Member.objects.filter(user=myUser).first() if myMember is not None: print("Member Found") myMember.boxID = boxid myMember.save() logger.info("...END PROCESSING") exit(0)
def main(): args = argsFetch() logger = loggerFetch(args.get('log_level')) logger.info('args: %s', str(args)) logger.info("BEGIN PROCESSING...") display = displayInitialize(args['visible']) driver = driverInitialize(browser=args['browser'] , path='/home/mayank/.mozilla/firefox/4s3bttuq.default/') base_url="https://www.skillshare.com/login" driver.get(base_url) logger.info('Fetching URL[%s]' % base_url) # driver.find_element_by_link_text("Sign In").click() try: driver.find_element_by_name("LoginForm[email]").clear() driver.find_element_by_name("LoginForm[email]").send_keys("*****@*****.**") driver.find_element_by_name("LoginForm[password]").clear() # driver.find_element_by_name("LoginForm[password]").send_keys("#######") # driver.find_element_by_xpath("//input[@value='Sign In']").click() time.sleep(100) # If you want to manually log in except Exception as e: logger.info('Already signed in [%s]', e) time.sleep(10) filename = "./z.csv" content = csv.reader(open(filename, 'r'), delimiter=',', quotechar='"') for (title, url) in content: # driver.get('https://www.skillshare.com/classes/Sketchbook-Practice-Bring-watercolour-to-Life-with-Line-Drawing/1053382271/classroom/discuss') logger.info('Fetching URL[%s]' % url) driver.get(url) time.sleep(10) escaped_title = re.sub(r"[^A-Za-z 0-9]+", '', title).replace(' ', '_') dirname = 'SkillsShare/' + escaped_title cmd = 'mkdir -p ' + dirname logger.info(cmd) os.system(cmd) els = driver.find_elements_by_class_name("session-item") for i, el in enumerate(els): logger.debug(str(el)) bs = BeautifulSoup(el.get_attribute('innerHTML'), "html.parser") p = bs.find('p') name = p.text name = "%02d" % (i+1) + '_' + re.sub(r"[^A-Za-z 0-9]+", '', name).replace(' ', '_') + '.mp4' logger.info(str(p) + name) el.click() time.sleep(10) html_source = driver.page_source bs = BeautifulSoup(html_source, "html.parser") html = bs.findAll('video', attrs={'class':['vjs-tech']}) str_html = str(html) logger.info(str_html) url = str_html[str_html.find("src=")+5:] fetch_url = url[:url.find("?pubId")] logger.info(fetch_url) if os.path.exists(dirname + '/' + name): continue cmd = 'cd %s && curl -s %s -o %s' % (dirname, fetch_url, name) logger.info(cmd) # os.system(cmd) driverFinalize(driver) displayFinalize(display) logger.info("...END PROCESSING") exit(0)
def main(): args = argsFetch() logger = loggerFetch(args.get('log_level')) logger.info('args: %s', str(args)) logger.info("This is module to Manage Transactions") if args['limit']: limit = args['limit'] else: limit=50000 if args['sendOrder']: if args['dateFrom']: dateFrom=datetime.strptime(args['dateFrom'], '%d-%m-%Y').date() if args['dateTo']: dateTo=datetime.strptime(args['dateTo'], '%d-%m-%Y').date() myMembers=Member.objects.all()[:limit] myMembers=Member.objects.filter(user__username="******") for eachMember in myMembers: if eachMember.user.username.isdigit(): firstTransaction=MemberTransaction.objects.filter(member=eachMember,orderDate__lte=dateFrom).order_by("-orderDate","-id").first() startBalance=firstTransaction.balance table=getHTMLHeader() table+='<body><table class=" table table-striped" ><tbody>' table+="<tr><th>Product</th><th>orderQuantity</th><th>Status</th><th>type</th><th>SuppliedQuantity</th><th>price</th><th>value</th><th>balance</th></tr>" tableLine="<tr><td colspan=7>%s</td><td>%s</td></tr>" % ("Start Balance", str(startBalance)) table+=tableLine lastTransaction=MemberTransaction.objects.filter(member=eachMember,orderDate=dateTo).order_by("-id").first() lastBalance=lastTransaction.balance myTransactions=MemberTransaction.objects.filter(member=eachMember,orderDate__gt=dateFrom,orderDate__lte=dateTo) for eachTransaction in myTransactions: orderStatus=eachTransaction.orderStatus lineValue=eachTransaction.price*eachTransaction.quantity tableLine="<tr><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td></tr>" % (eachTransaction.product.name,str(eachTransaction.orderQuantity),orderStatus,eachTransaction.transactionType,str(eachTransaction.quantity),str(eachTransaction.price),str(lineValue),str(eachTransaction.balance)) table+=tableLine tableLine="<tr><td colspan=7>%s</td><td>%s</td></tr>" % ("Last Balance", str(lastBalance)) table+=tableLine table+='</tbody></table ></body>' toEmail="*****@*****.**" body=table subject="In Todays Box" with open("/tmp/aaa.html","w") as f: f.write(table) genericSendEmail(toEmail,subject,body) if args['sendBalance']: d=date.today() dateString=str(datetime.strftime(d, "%b-%d")) baseURL="https://s3.ap-south-1.amazonaws.com/ndcsa/memberData/" logger.info("Sending the balance") myMembers=Member.objects.filter(isActive=True)[:limit] for eachMember in myMembers: if eachMember.user.username.isdigit(): lastTransaction=MemberTransaction.objects.filter(member=eachMember).order_by("-orderDate","-id").first() if lastTransaction is not None: lastBalance=lastTransaction.balance logger.info("%s-%s-%s" % (str(eachMember.user.username),eachMember.name,str(lastBalance))) balanceURL="%s%s.csv" % (baseURL,str(eachMember.user.username)) logger.info(balanceURL) memberEmail=eachMember.user.email if args['test']: memberEmail="*****@*****.**" logger.info(memberEmail) sendMail(memberEmail,eachMember.name,lastBalance,balanceURL,dateString) if args['upload']: logger.info("Will upload everyone transaction History") myMembers=Member.objects.all()[:limit] for eachMember in myMembers: logger.info(eachMember.user.username) logger.info(eachMember.name) memID=str(eachMember.user.username)+"-"+eachMember.name s='' balance=0 s+="member,orderDate,product,quantity,price,total,type,balance\n" myTransactions=MemberTransaction.objects.filter(member=eachMember).order_by("orderDate","id") for eachTransaction in myTransactions: logger.info(eachTransaction.product.name) logger.info(eachTransaction.orderDate) total=eachTransaction.quantity*eachTransaction.price if eachTransaction.transactionType == 'DR': balance=balance-total else: balance=balance+total s+="%s,%s,%s,%s,%s,%s,%s,%s\n" % (memID,str(eachTransaction.orderDate),eachTransaction.product.name,str(eachTransaction.quantity),str(eachTransaction.price),str(total),eachTransaction.transactionType,str(balance)) logger.info(s) cloudFilename="memberData/%s.csv" % (str(eachMember.user.username)) session = boto3.session.Session(aws_access_key_id=LIBTECH_AWS_ACCESS_KEY_ID, aws_secret_access_key=LIBTECH_AWS_SECRET_ACCESS_KEY) s3 = session.resource('s3',config=Config(signature_version='s3v4')) s3.Bucket(AWS_STORAGE_BUCKET_NAME).put_object(ACL='public-read',Key=cloudFilename, Body=s) logger.info("...END PROCESSING") exit(0)