def main():
args = argsFetch()
logger = loggerFetch(args.get('log_level'))
logger.info('args: %s', str(args))
logger.info("BEGIN PROCESSING...")
db = dbInitialize(
db="surguja",
charset="utf8") # The rest is updated automatically in the function
if args['fetch']:
# For fetch where only download push and parse options can be ignored
fetchMusterDetails(logger, db, "FETCH MUSTER DETAILS",
args['directory'], args['url'], args['parse'],
args['push'], args['visible'])
elif args['parse']:
# parseDownloadedMuster(logger, db, "PARSE JOBCARDS", args['directory'], args['url'], )
logger.info(
"Requested Parse") # If you are splitting download and parse logic
fetchMusterDetails(logger, db, "PARSE MUSTER DETAILS",
args['directory'], args['url'], args['parse'],
args['push'], args['visible'])
else:
logger.info("Default fetch which includes download, parse & populate"
) # Controlled by options
fetchMusterDetails(logger, db, "FETCH MUSTER DETAILS",
args['directory'], args['url'], args['parse'],
args['push'], args['visible'])
dbFinalize(
db) # Make sure you put this if there are other exit paths or errors
logger.info("...END PROCESSING")
exit(0)
def main():
from firebase import firebase
firebase = firebase.FirebaseApplication('https://libtech-app.firebaseio.com/', None)
args = argsFetch()
logger = loggerFetch(args.get('log_level'))
logger.info('args: %s', str(args))
if args['write']:
d={}
d=[]
d.append({'make':'nd','model':'readyToEat','price':95})
d.append({'make':'sahaja','model':'vegetable','price':55})
d.append({'make':'tvs','model':'fruit','price':195})
with open('first.json', 'w') as outfile:
json.dump(d, outfile)
if args['populate']:
jsonName="districts.json"
json_data=open(jsonName,encoding='utf-8-sig').read()
d = json.loads(json_data)
myDict={}
for key,values in d.items():
code=key
name=values['name']
parentCode=values['stateCode']
logger.info(key)
dictKey="%s_%s_%s" % (code,name,parentCode)
myDict[dictKey] = {'name': name, 'code': code, 'parentCode': parentCode, 'slug': dictKey }
logger.info(dictKey)
result = firebase.patch('https://libtech-app.firebaseio.com/medicines/', myDict)
logger.info("...END PROCESSING")
exit(0)
def main():
args = argsFetch()
logger = loggerFetch(args.get('log_level'))
logger.info('args: %s', str(args))
logger.info("BEGIN PROCESSING...")
myLibtechTag=LibtechTag.objects.filter(name="August 2017 Hearing").first()
myStates=["18","17","27","24","33","15","16","31","34","05","32"]
myStates=["29"]
for eachState in myStates:
myobjs=District.objects.filter(state__isNIC=True,state__code=eachState)
if args['limit']:
limit=int(args['limit'])
else:
limit=1
for eachDistrict in myobjs:
logger.info(eachDistrict.name)
myPanchayats=list(Panchayat.objects.filter(block__district__code=eachDistrict.code,crawlRequirement='NONE'))
shuffle(myPanchayats)
myPanchayats=myPanchayats[:limit]
for eachPanchayat in myPanchayats:
logger.info(eachPanchayat.name)
eachPanchayat.crawlRequirement="FULL"
eachPanchayat.libtechTag.add(myLibtechTag)
eachPanchayat.save()
logger.info("...END PROCESSING")
exit(0)
def main():
args = argsFetch()
logger = loggerFetch(args.get('log_level'))
logger.info('args: %s', str(args))
logger.info("BEGIN PROCESSING...")
db = dbInitialize(db="biharPDS", charset="utf8") # The rest is updated automatically in the function
cur=db.cursor()
db.autocommit(True)
#Query to set up Database to read Hindi Characters
query="SET NAMES utf8"
cur.execute(query)
inyear=args['year']
logger.info(inyear)
display = displayInitialize(args['visible'])
driver = driverInitialize(args['browser'])
#Start Program here
url="http://www.google.com"
driver.get(url)
myhtml=driver.page_source
print myhtml
# End program here
driverFinalize(driver)
displayFinalize(display)
dbFinalize(db) # Make sure you put this if there are other exit paths or errors
logger.info("...END PROCESSING")
exit(0)
def runTestSuite():
logger = loggerFetch("info")
logger.info("BEGIN PROCESSING...")
db = dbInitialize(db="surguja", charset="utf8")
cur = db.cursor()
cur.execute("SET NAMES utf8")
jobcard = 'CH-05-005-015-001/187'
query = 'select headOfFamily from jobcardRegister where jobcard="' + jobcard + '"'
logger.info("query[%s]" % query)
cur.execute(query)
head = cur.fetchall()[0][0]
logger.info("HeadOfFamily[%s]" % head)
query = 'select applicantName, accountNo from jobcardDetails where jobcard="' + jobcard + '"'
logger.info("query[%s]" % query)
cur.execute(query)
names = cur.fetchall()
#print names.encode('UTF-8')
logger.info("Names[%s]" % str(names).encode('UTF-8'))
dbFinalize(db)
logger.info("...END PROCESSING")
def runTestSuite():
logger = loggerFetch("info")
logger.info("BEGIN PROCESSING...")
display = displayInitialize(1)
driver = driverInitialize(path='/home/mayank/.mozilla/firefox/4s3bttuq.default/', timeout=timeout)
driver.set_page_load_timeout(timeout)
for version in versions:
url = url_base + str(version)
try:
logger.info('Fetching URL[%s]' % url)
driver.get(url)
logger.info('After Fetch[%s]' % url)
except Exception as e:
logger.info("Warning %s", e)
if os.path.exists(csv_file):
version_file = 'CSVs/' + str(version) + '.csv'
logger.info('Writing %s' % version_file)
os.rename(csv_file, version_file)
else:
logger.error('Missed file[%s] from URL[%s]' % (version_file, url))
logger.info("CSV Fetched From [%s]" % url)
driverFinalize(driver)
displayFinalize(display)
logger.info("...END PROCESSING")
def main():
args = argsFetch()
logger = loggerFetch(args.get('log_level'))
logger.info('args: %s', str(args))
logger.info("BEGIN PROCESSING...")
if args['limit']:
limit = int(args['limit'])
else:
limit =1
if args['crawlRequirement']:
crawlRequirement=args['crawlRequirement']
else:
crawlRequirement='STAT'
finyear=args['finyear']
fullfinyear=getFullFinYear(finyear)
myBlocks=Block.objects.filter(crawlRequirement=crawlRequirement)[:limit]
for eachBlock in myBlocks:
if eachBlock.district.state.stateCode != '02':
logger.info("BlockName: %s BlockCode: %s " % (eachBlock.name,eachBlock.fullBlockCode))
reportTypes=["I","R"]
for rtype in reportTypes:
url="http://%s/netnrega/FTO/rejection.aspx?lflag=eng&state_code=%s&state_name=%s&district_code=%s&page=d&Block_code=%s&Block_name=%s&district_name=%s&fin_year=%s&typ=%s&linkr=X&" %(eachBlock.district.state.crawlIP,eachBlock.district.state.stateCode,eachBlock.district.state.name,eachBlock.district.fullDistrictCode,eachBlock.fullBlockCode,eachBlock.name,eachBlock.district.name,fullfinyear,rtype)
logger.info(url)
logger.info("...END PROCESSING")
exit(0)
def main():
regex=re.compile(r'<input+.*?"\s*/>+',re.DOTALL)
args = argsFetch()
finyear=args['finyear']
if args['limit']:
limit = int(args['limit'])
else:
limit =50000
if args['musterID']:
mid=args['musterID']
else:
mid=None
if args['maxProcess']:
maxProcess=int(args['maxProcess'])
else:
maxProcess=1
additionalFilters=''
if args['district']:
additionalFilters+= " and b.districtName='%s' " % args['district']
if args['block']:
additionalFilters+= " and b.blockName='%s' " % args['block']
fullfinyear=getFullFinYear(finyear)
logger = loggerFetch(args.get('log_level'))
logger.info('args: %s', str(args))
logger.info("BEGIN PROCESSING...")
db = dbInitialize(db=nregaDB, charset="utf8") # The rest is updated automatically in the function
cur=db.cursor()
db.autocommit(True)
#Query to set up Database to read Hindi Characters
query="SET NAMES utf8"
cur.execute(query)
tasks = multiprocessing.JoinableQueue()
results = multiprocessing.Queue()
if mid is None:
query="select m.id from musters m,blocks b where m.fullBlockCode=b.fullBlockCode and m.finyear='%s' and (m.isDownloaded=0 or (m.wdComplete=0 and TIMESTAMPDIFF(HOUR, m.downloadAttemptDate, now()) > 48 )) %s order by isDownloaded,m.downloadAttemptDate limit %s" % (finyear,additionalFilters,str(limit))
else:
query="select m.id from musters m where m.id=%s " % str(mid)
logger.info(query)
cur.execute(query)
noOfTasks=cur.rowcount
results1=cur.fetchall()
for row in results1:
musterID=row[0]
tasks.put(Task(musterID))
for i in range(maxProcess):
tasks.put(None)
myProcesses=[musterProcess(tasks, results) for i in range(maxProcess)]
for eachProcess in myProcesses:
eachProcess.start()
while noOfTasks:
result = results.get()
logger.info(result)
noOfTasks -= 1
dbFinalize(db) # Make sure you put this if there are other exit paths or errors
def runTestSuite():
args = argsFetch()
logger = loggerFetch(args.get('log_level'))
logger.info('args: %s', str(args))
logger.info("BEGIN PROCESSING...")
display = displayInitialize(args['visible'])
driver = driverInitialize(args['browser'])
# Mynk to use personal profile driver = driverInitialize(browser=args['browser'] , path='/home/mayank/.mozilla/firefox/4s3bttuq.default/')
if args['cookie_dump']:
cookieDump(driver)
logger.info("Fetching [%s]" % driver.current_url)
logger.info(wdTest(driver, args['url']))
logger.info("Fetched [%s]" % driver.current_url)
if args['cookie_dump']:
cookieDump(driver)
driverFinalize(driver)
displayFinalize(display)
'''
display = vDisplayInitialize(visible)
driver = driverInitialize(browser)
logger.info(wdTest(driver))
driverFinalize(driver)
vDisplayFinalize(display)
'''
logger.info("...END PROCESSING")
def main():
alwaysTag=LibtechTag.objects.filter(name="Always")
regex=re.compile("^[0-9]{4}-[0-9]{4}$")
benchMark = datetime.strptime(telanganaThresholdDate, "%Y-%m-%d")
telanganaStateCode='36'
args = argsFetch()
logger = loggerFetch(args.get('log_level'))
logger.info('args: %s', str(args))
if args['limit']:
limit = int(args['limit'])
else:
limit =1
if args['enumerate']:
logger.info("Will enumerate and put all the District and Block Codes in place")
enumerateCodes(logger)
if args['download']:
logger.info("Attempting to Download Postal P ayments")
myPanchayats=Panchayat.objects.filter(crawlRequirement="FULL",block__district__state__code=telanganaStateCode)
# for eachPanchayat in myPanchayats:
# downloadPage(logger,eachPanchayat)
queueManager(logger,myPanchayats,'download')
if args['process']:
logger.info("Attempting to Process Postal P ayments")
curfinyear=getCurrentFinYear()
today_min = datetime.combine(date.today(), time.min)
today_max = datetime.combine(date.today(), time.max)
myPanchayatReports=PanchayatReport.objects.filter(finyear=curfinyear,panchayat__crawlRequirement="FULL",panchayat__block__district__state__code=telanganaStateCode,reportType=reportType,updateDate__range=(today_min, today_max))
queueManager(logger,myPanchayatReports,'process')
# for eachReport in myPanchayatReports:
# logger.info(eachReport.panchayat.code)
# processPostalReport(logger,eachReport)
logger.info("...END PROCESSING")
exit(0)
def main():
args = argsFetch()
logger = loggerFetch(args.get('log_level'))
db = dbInitialize(db=nregaDB, charset="utf8") # The rest is updated automatically in the function
cur=db.cursor()
db.autocommit(True)
#Query to set up Database to read Hindi Characters
query="SET NAMES utf8"
cur.execute(query)
if args['updateMusterStats']:
updateMusterStats(cur,logger)
if args['selectRandomDistricts']:
selectRandomDistricts(cur,logger)
if args['updatePanchayatStats']:
updatePanchayatStats(cur,logger)
if args['genMusterURL']:
genMusterURLs(cur,logger,args['musterID'])
if args['downloadRejectedPaymentReport']:
if args['finyear']:
finyear=args['finyear']
else:
finyear='17'
downloadRejectedPaymentReport(cur,logger,finyear)
dbFinalize(db) # Make sure you put this if there are other exit paths or errors
logger.info("...END PROCESSING")
exit(0)
def setUp(self):
self.logger = loggerFetch('info')
self.db = dbInitialize(host='localhost',
user='******',
passwd='lt123',
db='libtech')
self.logger.info('BEGIN PROCESSING...')
def runTestSuite():
logger = loggerFetch("info")
logger.info("BEGIN PROCESSING...")
db = dbInitialize(db="surguja", charset="utf8")
cur = db.cursor()
cur.execute("SET NAMES utf8")
jobcard = 'CH-05-005-015-001/187'
query = 'select headOfFamily from jobcardRegister where jobcard="' + jobcard + '"'
logger.info("query[%s]" % query)
cur.execute(query)
head = cur.fetchall()[0][0]
logger.info("HeadOfFamily[%s]" % head)
query = 'select applicantName, accountNo from jobcardDetails where jobcard="' + jobcard + '"'
logger.info("query[%s]" % query)
cur.execute(query)
names = cur.fetchall()
#print names.encode('UTF-8')
logger.info("Names[%s]" % str(names).encode('UTF-8'))
dbFinalize(db)
logger.info("...END PROCESSING")
def runTestSuite():
logger = loggerFetch("info")
logger.info("BEGIN PROCESSING...")
display = displayInitialize(0)
driver = driverInitialize()
for mandal in mandals:
report = generate_report(logger, driver, mandal, districts[mandal])
logger.info('Finally: \n%s' % report)
filename = './mandals/' +mandal + '.csv'
with open(filename, 'wb') as csv_file:
logger.info("Writing to [%s]" % filename)
csv_file.write(report.encode('utf-8'))
if final_report[0][0] == '':
rows = report.split('|')
driverFinalize(driver)
displayFinalize(display)
logger.info("...END PROCESSING")
return
def main():
runTestSuite()
exit(0)
if __name__ == '__main__':
main()
def main():
args = argsFetch()
logger = loggerFetch(args.get('log_level'))
if args['singleExecute']:
myTask=Task.objects.filter(isComplete=False).order_by("modified").first()
logger.info("Running Task %s Task ID %s" % (myTask.crawlState.name,str(myTask.id)))
funcName=myTask.crawlState.name
objID=myTask.objID
finyear=myTask.finyear
isError=getattr(crawlFunctions1,funcName)(logger,objID,finyear=finyear)
if isError==False:
isComplete=True
myTask.isComplete=isComplete
myTask.save()
if args['run']:
while True:
myTask=Task.objects.filter(isComplete=False).order_by("modified").first()
if myTask is not None:
logger.info("Running Task %s Task ID %s" % (myTask.crawlState.name,str(myTask.id)))
funcName=myTask.crawlState.name
objID=myTask.objID
finyear=myTask.finyear
isError=getattr(crawlFunctions1,funcName)(logger,objID,finyear=finyear)
if isError==False:
isComplete=True
myTask.isComplete=isComplete
myTask.save()
else:
logger.debug("No tasks to run")
time.sleep(10)
logger.info("...END PROCESSING")
exit(0)
def main():
args = argsFetch()
logger = loggerFetch(args.get('log_level'))
logger.info('args: %s', str(args))
db = dbInitialize(db="libtech", charset="utf8") # The rest is updated automatically in the function
cur=db.cursor()
db.autocommit(True)
#Query to set up Database to read Hindi Characters
query="SET NAMES utf8"
cur.execute(query)
query="select phone,bid from callLogs where DATE(callStartTime) = CURDATE() group by phone "
cur.execute(query)
results=cur.fetchall()
for row in results:
#rowid=str(row[0])
phone=row[0]
logger.info(" Phone %s " % (phone))
query="select count(*) from callLogs where phone='%s'" % (phone)
totalCalls=singleRowQuery(cur,query)
query="select count(*) from callLogs where phone='%s' and status='pass'" % (phone)
totalSuccessCalls=singleRowQuery(cur,query)
if totalCalls > 0:
logger.info("Calculating Percentage")
successP=math.trunc(totalSuccessCalls*100/totalCalls)
else:
successP=0
logger.info("Total Calls %s Success Calls %s Success Percentage %s " % (str(totalCalls),str(totalSuccessCalls),str(successP)))
query="update addressbook set totalCalls='%s',successPercentage='%s' where phone='%s' " % (str(totalCalls),str(successP),phone)
cur.execute(query)
dbFinalize(db) # Make sure you put this if there are other exit paths or errors
logger.info("...END PROCESSING")
exit(0)
def main():
args = argsFetch()
logger = loggerFetch(args.get('log_level'))
if args['populate']:
panchayatCode=args['panchayatCode']
blockCode=args['blockCode']
if panchayatCode is not None:
eachPanchayat=Panchayat.objects.filter(code=panchayatCode).first()
CrawlQueue.objects.create(panchayat=eachPanchayat)
elif blockCode is not None:
eachBlock=Block.objects.filter(code=blockCode).first()
CrawlQueue.objects.create(block=eachBlock,priority=500)
if args['execute']:
stage=args['step']
limit=args['limit']
qid=args['qid']
if qid is not None:
libtechCrawler(logger,stage,qid=qid)
else:
libtechCrawler(logger,stage)
logger.info("...END PROCESSING")
exit(0)
def main():
logger = loggerFetch("info")
logger.info("BEGIN PROCESSING...")
phone='9845155447'
#Logic to upload the wave_file for the first time -
wave_file = '1629_RSCD281016.wav' # '1431_CgbcstChakeri010316.wav' # This should be done only once for each new file
filename = wave_file.strip('.wav')
if wave_file:
logger.info(awaazdeUpload(logger, wave_file))
if False:
vendorcallid = awaazdePlaceCall(logger, phone, wave_file)
time.sleep(10)
else:
vendorcallid = '71503' # '70239' # '70221' # '57897'
logger.info("Call ID [%s]" % vendorcallid)
callreturn = awaazdeStatusCheck(logger, vendorcallid)
logger.info(callreturn)
logger.info("...END PROCESSING")
def runTestSuite():
args = argsFetch()
logger = loggerFetch(args.get('log_level'))
logger.info('args: %s', str(args))
logger.info("BEGIN PROCESSING...")
display = displayInitialize(args['visible'])
driver = driverInitialize(args['browser'])
# Mynk to use personal profile driver = driverInitialize(browser=args['browser'] , path='/home/mayank/.mozilla/firefox/4s3bttuq.default/')
if args['cookie_dump']:
cookieDump(driver)
logger.info("Fetching [%s]" % driver.current_url)
logger.info(wdTest(driver, args['url']))
logger.info("Fetched [%s]" % driver.current_url)
if args['cookie_dump']:
cookieDump(driver)
driverFinalize(driver)
displayFinalize(display)
'''
display = vDisplayInitialize(visible)
driver = driverInitialize(browser)
logger.info(wdTest(driver))
driverFinalize(driver)
vDisplayFinalize(display)
'''
logger.info("...END PROCESSING")
def main():
args = argsFetch()
logger = loggerFetch(args.get('log_level'))
logger.info('args: %s', str(args))
if args['limit']:
limit = int(args['limit'])
else:
limit =1
finyear=args['finyear']
stateCodes=['33','34','16','27','24','15','18','35']
stateCodes=['33','34','27','24','15','18',]
stateCodes=['16','31','05','17']
stateCodes=['15']
stateCodes=[args['stateCode']]
for stateCode in stateCodes:
# if stateCode is not None:
# logger.info("StateCode is %s" % stateCode)
myPanchayats=Panchayat.objects.filter(crawlRequirement='FULL',block__district__state__code=stateCode)
reportType="workPayment"
# else:
# myPanchayats=Panchayat.objects.filter(crawlRequirement='FULL')[:limit]
for eachPanchayat in myPanchayats:
logger.info("**********************************************************************************")
panchayatReport=PanchayatReport.objects.filter(reportType=reportType,finyear=finyear,panchayat=eachPanchayat).first()
if panchayatReport is None:
logger.info("Createing work Payment report for panchayat: %s panchayatCode: %s ID: %s" % (eachPanchayat.name,eachPanchayat.code,str(eachPanchayat.id)))
outcsv=''
outcsv+="jobcard,name,musterNo,workCode,workName,dateFrom,dateTo,daysWorked,totalWage,accountNo,musterStatus,creditedDate,secondSignatoryDate,wagelistNo"
outcsv+="\n"
workRecords=WorkDetail.objects.filter(muster__block__panchayat__id=eachPanchayat.id,muster__finyear=finyear)
workRecords=WorkDetail.objects.filter(muster__panchayat=eachPanchayat,muster__finyear=finyear)
logger.info("Total Work Records: %s " %str(len(workRecords)))
for wd in workRecords:
workName=wd.muster.workName.replace(","," ")
applicantName=wd.zname.replace(",","")
wagelistArray=wd.wagelist.all()
if len(wagelistArray) > 0:
wagelist=wagelistArray[len(wagelistArray) -1 ]
else:
wagelist=''
if wd.applicant:
fatherHusbandName=wd.applicant.fatherHusbandName.replace(",","")
else:
fatherHusbandName=None
outcsv+="%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s" % (wd.zjobcard,applicantName,wd.muster.musterNo,wd.muster.workCode,workName,str(wd.muster.dateFrom),str(wd.muster.dateTo),str(wd.daysWorked),str(wd.totalWage),wd.zaccountNo,wd.musterStatus,str(wd.creditedDate),str(wd.muster.paymentDate),wagelist)
outcsv+="\n"
try:
outcsv=outcsv.encode("UTF-8")
except:
outcsv=outcsv
csvfilename=eachPanchayat.slug+"_"+finyear+"_wp.csv"
savePanchayatReport(logger,eachPanchayat,finyear,reportType,csvfilename,outcsv)
logger.info("...END PROCESSING")
exit(0)
def setUp(self):
self.args = argsFetch()
self.logger = loggerFetch('info')
self.logger.info('BEGIN PROCESSING')
self.display = displayInitialize(self.args['visible'])
self.driver = driverInitialize(browser=self.args['browser'] , path='/home/mayank/.mozilla/firefox/4s3bttuq.default/')
self.cmd = 'nohup youtube-dl --username %s --password %s -o %s ' % (user, password, '"%(title)s.%(ext)s"')
self.url = self.args['url']
def setUp(self):
self.logger = loggerFetch('info')
self.logger.info('BEGIN PROCESSING...')
self.display = displayInitialize(0)
# self.driver = driverInitialize(timeout=3)
self.driver = driverInitialize(
timeout=3, options='--headless'
) # driverInitialize(path='/opt/firefox/', timeout=3)
def setUp(self):
self.logger = loggerFetch('info')
self.logger.info('BEGIN PROCESSING')
# Pass 0 for headless
self.display = displayInitialize(1)
#, path='/path/to/firefox/')
self.driver = driverInitialize(browser='firefox')
self.url = URL
def main():
args = argsFetch()
logger = loggerFetch(args.get('log_level'))
logger.info('args: %s', str(args))
logger.info("BEGIN PROCESSING...")
if args['district']:
districtName=args['district'].lower()
if args['limit']:
limitString=" limit %s " % (str(args['limit']))
else:
limitString=" "
if args['reportID']:
reportIDFilter= " and id = %s " % args['reportID']
else:
reportIDFilter= " "
db = dbInitialize(db=districtName.lower(), charset="utf8") # The rest is updated automatically in the function
cur=db.cursor()
db.autocommit(True)
#Query to set up Database to read Hindi Characters
query="SET NAMES utf8"
cur.execute(query)
crawlIP,stateName,stateCode,stateShortCode,districtCode=getDistrictParams(cur,districtName)
htmlDir=nregaDir.replace("districtName",districtName.lower())
#block Reports
query="select b.name,b.blockCode from blocks b where b.isRequired=1 %s" % limitString
#query="select b.name,b.blockCode from blocks b where b.isRequired=1 limit 1"
#query="select b.name,b.blockCode,p.name,p.panchayatCode from panchayats p, blocks b where b.blockCode=p.blockCode and p.isRequired=1 limit 1"
#query="select b.name,b.blockCode,p.name,p.panchayatCode from panchayats p, blocks b where b.blockCode=p.blockCode and p.isRequired=1 and b.blockCode='005' "
# query="select b.name,b.blockCode,p.name,p.panchayatCode from panchayats p, blocks b where b.blockCode=p.blockCode and p.isRequired=1 and b.blockCode='003' and panchayatCode='013'"
cur.execute(query)
results=cur.fetchall()
for row in results:
blockName=row[0]
blockCode=row[1]
genReport(cur,logger,1,htmlDir,'16',districtName,blockCode,blockName,'','',reportIDFilter)
genReport(cur,logger,1,htmlDir,'17',districtName,blockCode,blockName,'','',reportIDFilter)
genReport(cur,logger,1,htmlDir,'all',districtName,blockCode,blockName,'','',reportIDFilter)
query="select b.name,b.blockCode,p.name,p.panchayatCode from panchayats p, blocks b where b.blockCode=p.blockCode and p.isRequired=1 %s" % limitString
#query="select b.name,b.blockCode,p.name,p.panchayatCode from panchayats p, blocks b where b.blockCode=p.blockCode and p.isRequired=1 limit 1"
cur.execute(query)
results=cur.fetchall()
for row in results:
blockName=row[0]
blockCode=row[1]
panchayatName=row[2]
panchayatCode=row[3]
finyear='16'
genReport(cur,logger,0,htmlDir,'16',districtName,blockCode,blockName,panchayatCode,panchayatName,reportIDFilter)
genReport(cur,logger,0,htmlDir,'17',districtName,blockCode,blockName,panchayatCode,panchayatName,reportIDFilter)
genReport(cur,logger,0,htmlDir,'all',districtName,blockCode,blockName,panchayatCode,panchayatName,reportIDFilter)
dbFinalize(db) # Make sure you put this if there are other exit paths or errors
logger.info("...END PROCESSING")
exit(0)
def main():
args = argsFetch()
logger = loggerFetch(args.get('log_level'))
logger.info('args: %s', str(args))
logger.info("...END PROCESSING")
exit(0)
def main():
args = argsFetch()
logger = loggerFetch(args.get('log_level'))
logger.info('args: %s', str(args))
url = "http://164.100.129.6/netnrega/dynamic_account_details.aspx"
if args['limit']:
limit = int(args['limit'])
else:
limit =1
stateCode=args['stateCode']
if stateCode is not None:
myPanchayats=Panchayat.objects.filter(Q (crawlRequirement="FULL") & (Q (jobcardCrawlDate__lt = jobCardRegisterTimeThreshold) | Q(jobcardCrawlDate__isnull = True ) ) & Q (block__district__state__code=stateCode)).order_by('jobcardCrawlDate')[:limit]
else:
myPanchayats=Panchayat.objects.filter(Q (crawlRequirement="FULL") & (Q (jobcardCrawlDate__lt = jobCardRegisterTimeThreshold) | Q(jobcardCrawlDate__isnull = True ) ) ).order_by('jobcardCrawlDate')[:limit]
# myPanchayats=Panchayat.objects.filter(fullPanchayatCode='3405003010').order_by('jobcardCrawlDate')[:limit]
for eachPanchayat in myPanchayats:
logger.info("Processing : panchayat: %s " % (eachPanchayat.name))
stateCode=eachPanchayat.block.district.state.code
fullDistrictCode=eachPanchayat.block.district.code
fullBlockCode=eachPanchayat.block.code
fullPanchayatCode=eachPanchayat.code
districtName=eachPanchayat.block.district.name
blockName=eachPanchayat.block.name
stateName=eachPanchayat.block.district.state.name
logger.info("Processing StateCode %s, fullDistrictCode : %s, fullBlockCode : %s, fullPanchayatCode: %s " % (stateCode,fullDistrictCode,fullBlockCode,fullPanchayatCode))
cur=''
htmlresponse,myhtml=getWorkerList(logger,url,stateCode,fullDistrictCode,fullBlockCode,fullPanchayatCode)
if htmlresponse['status'] == '200':
logger.info("File Downloaded SuccessFully")
status,outhtml,outcsv=alterWorkerList(cur,logger,myhtml,stateName,districtName,blockName,eachPanchayat.name)
if status is not None:
filename=eachPanchayat.slug+"_jr.html"
filenamecsv=eachPanchayat.slug+"_jr.csv"
try:
outhtml=outhtml.encode("UTF-8")
except:
outhtml=outhtml
try:
outcsv=outcsv.encode("UTF-8")
except:
outcsv=outcsv
logger.info("Processing StateCode %s, fullDistrictCode : %s, fullBlockCode : %s, fullPanchayatCode: %s " % (stateCode,fullDistrictCode,fullBlockCode,fullPanchayatCode))
finyear=getCurrentFinYear()
reportType="jobcardRegisterHTML"
savePanchayatReport(logger,eachPanchayat,finyear,reportType,filename,outhtml)
reportType="jobcardRegisterCSV"
savePanchayatReport(logger,eachPanchayat,finyear,reportType,filenamecsv,outcsv)
# eachPanchayat.jobcardRegisterFile.save(filename, ContentFile(outhtml))
eachPanchayat.jobcardCrawlDate=timezone.now()
eachPanchayat.status='1'
eachPanchayat.save()
logger.info("Processing StateCode %s, fullDistrictCode : %s, fullBlockCode : %s, fullPanchayatCode: %s " % (stateCode,fullDistrictCode,fullBlockCode,fullPanchayatCode))
logger.info("...END PROCESSING")
exit(0)
def main():
args = argsFetch()
logger = loggerFetch(args.get('log_level'))
logger.info('args: %s', str(args))
logger.info("BEGIN PROCESSING...")
#This is a Kludge to remove all the input tags from the html because for some reason Beautiful Soup does not parse the html correctly
regex=re.compile(r'<input+.*?"\s*/>+',re.DOTALL)
if args['district']:
districtName=args['district']
logger.info("DistrictName "+districtName)
if args['finyear']:
finyear=args['finyear']
logger.info("finyear "+finyear)
fullFinyear=getFullFinYear(finyear)
db = dbInitialize(db=districtName.lower(), charset="utf8") # The rest is updated automatically in the function
cur=db.cursor()
db.autocommit(True)
#Query to set up Database to read Hindi Characters
query="SET NAMES utf8"
cur.execute(query)
crawlIP,stateName,stateCode,stateShortCode,districtCode=getDistrictParams(cur,districtName)
#Query to get all the blocks
query="select stateCode,districtCode,blockCode,name from blocks where isRequired=1"
cur.execute(query)
results = cur.fetchall()
for row in results:
stateCode=row[0]
districtCode=row[1]
blockCode=row[2]
blockName=row[3]
fullBlockCode=stateCode+districtCode+blockCode
logger.info(stateCode+districtCode+blockCode+blockName)
url="http://"+crawlIP+"/netnrega/FTO/fto_reprt_detail.aspx?lflag=local&flg=W&page=b&state_name="+stateName.upper()+"&state_code="+stateCode+"&district_name="+districtName.upper()+"&district_code="+stateCode+districtCode+"&block_name="+blockName+"&block_code="+stateCode+districtCode+blockCode+"&fin_year="+fullFinyear+"&typ=pb&mode=b"
logger.info(url)
r = requests.get(url)
htmlsource=r.text
htmlsoup=BeautifulSoup(htmlsource,"html.parser")
for fto in htmlsoup.find_all('a'):
ftoNo=fto.contents[0]
#ftoURL="http://164.100.112.66/netnrega/FTO/"+fto.get('href')
if fullBlockCode in ftoNo:
query="insert into ftoDetails (ftoNo,stateCode,districtCode,blockCode,finyear) values ('"+ftoNo+"','"+stateCode+"','"+districtCode+"','"+blockCode+"','"+finyear+"');"
try:
logger.info(query)
cur.execute(query)
except MySQLdb.IntegrityError,e:
errormessage=(time.strftime("%d/%m/%Y %H:%M:%S "))+str(e)+"\n"
errorfile.write(errormessage)
continue
def main():
args = argsFetch()
logger = loggerFetch(args.get('log_level'))
logger.info('args: %s', str(args))
logger.info("BEGIN PROCESSING...")
limitString=''
if args['limit']:
limitString=' limit '+args['limit']
if args['district']:
districtName=args['district']
logger.info("DistrictName "+districtName)
finyear=args['finyear']
db = dbInitialize(db=districtName.lower(), charset="utf8") # The rest is updated automatically in the function
cur=db.cursor()
db.autocommit(True)
#Query to set up Database to read Hindi Characters
query="SET NAMES utf8"
cur.execute(query)
logger.info("finyear "+finyear)
fullFinYear=getFullFinYear(finyear)
crawlIP,stateName,stateCode,stateShortCode,districtCode=getDistrictParams(cur,districtName)
#Query to get all the blocks
ftorawfilepath=nregaRawDataDir.replace("districtName",districtName.lower())
query="select id,blockCode,name from blocks order by rejectedPaymentDownloadDate DESC"
cur.execute(query)
results=cur.fetchall()
for row in results:
rowid=str(row[0])
blockCode=row[1]
blockName=row[2]
fullBlockCode=stateCode+districtCode+blockCode
fullDistrictCode=stateCode+districtCode
url="http://164.100.129.4/netnrega/FTO/rejection.aspx?lflag=eng&state_code=%s&state_name=%s&district_code=%s&page=d&Block_code=%s&Block_name=%s&district_name=%s&fin_year=%s&typ=I&linkr=X" % (stateCode,stateName.upper(),fullDistrictCode,fullBlockCode,blockName.upper(),districtName.upper(),fullFinYear)
logger.info(url)
r=requests.get(url)
time.sleep(300)
inhtml=r.text
ftorawfilename=ftorawfilepath+blockName.upper()+"/FTO/"+fullFinYear+"/rejected.html"
writeFile(ftorawfilename,inhtml)
query="update blocks set rejectedPaymentDownloadDate=NOW() where id=%s" % rowid
logger.info(query)
cur.execute(query)
dbFinalize(db) # Make sure you put this if there are other exit paths or errors
logger.info("...END PROCESSING")
exit(0)
def setUp(self):
self.args = argsFetch()
self.logger = loggerFetch('info')
self.logger.info('BEGIN PROCESSING')
self.display = displayInitialize(self.args['visible'])
self.driver = driverInitialize(
browser=self.args['browser'],
path='/home/mayank/.mozilla/firefox/4s3bttuq.default/')
self.cmd = 'nohup youtube-dl --username %s --password %s -o %s ' % (
user, password, '"%(title)s.%(ext)s"')
self.url = self.args['url']
def main():
alwaysTag=LibtechTag.objects.filter(name="Always")
args = argsFetch()
logger = loggerFetch(args.get('log_level'))
logger.info('args: %s', str(args))
finyear=getCurrentFinYear()
fullfinyear=getFullFinYear(finyear)
url = "http://164.100.129.6/netnrega/dynamic_account_details.aspx"
if args['limit']:
limit = int(args['limit'])
else:
limit =1
stateCode=args['stateCode']
panchayatCode=args['panchayatCode']
if panchayatCode is not None:
myPanchayats=Panchayat.objects.filter(code=panchayatCode)
elif stateCode is not None:
myPanchayats=Panchayat.objects.filter(Q (crawlRequirement="FULL") & (Q (applicationRegisterCrawlDate__lt = jobCardRegisterTimeThreshold) | Q(applicationRegisterCrawlDate__isnull = True ) ) & Q (block__district__state__code=stateCode)).order_by('applicationRegisterCrawlDate')[:limit]
else:
myPanchayats=Panchayat.objects.filter(Q (crawlRequirement="FULL") & Q(block__district__state__isNIC=True) & (Q (applicationRegisterCrawlDate__lt = jobCardRegisterTimeThreshold) | Q(applicationRegisterCrawlDate__isnull = True ) ) ).order_by('applicationRegisterCrawlDate')[:limit]
# myPanchayats=Panchayat.objects.filter(fullPanchayatCode='3405003010').order_by('applicationRegisterCrawlDate')[:limit]
for eachPanchayat in myPanchayats:
myhtml=getJobcardRegister(logger,eachPanchayat)
myhtml=str(myhtml).replace("</nobr><br>",",")
myhtml=myhtml.replace("bordercolor='#111111'>","bordercolor='#111111'><tr>")
# myhtml = myhtml.replace('\n', ' ').replace('\r', '')
# myhtml=str(myhtml).replace("</td> <tr>","</td></tr><tr>")
# driver.get(panchayatPageURL)
# elem = driver.find_element_by_link_text("Registration Application Register")
# elem.click()
# myhtml = driver.page_source
# logger.info(myhtml)
# with open("/tmp/abcd.html","w") as f:
# f.write(myhtml)
error,myTable=validateApplicationRegister(logger,myhtml,eachPanchayat.block)
if error is None:
logger.info("No error")
outhtml=''
outhtml+=stripTableAttributes(myTable,"myTable")
title="Jobcard Register: state:%s District:%s block:%s panchayat: %s finyear:%s " % (eachPanchayat.block.district.state.name,eachPanchayat.block.district.name,eachPanchayat.block.name,eachPanchayat.name,fullfinyear)
outhtml=htmlWrapperLocal(title=title, head='<h1 aling="center">'+title+'</h1>', body=outhtml)
try:
outhtml=outhtml.encode("UTF-8")
except:
outhtml=outhtml
filename="applicationRegister_%s_%s.html" % (eachPanchayat.slug,finyear)
reportType="applicationRegister"
savePanchayatReport(logger,eachPanchayat,finyear,reportType,filename,outhtml)
eachPanchayat.applicationRegisterCrawlDate=timezone.now()
eachPanchayat.save()
else:
logger.info("Error")
logger.info("...END PROCESSING")
exit(0)
def main():
args = argsFetch()
logger = loggerFetch(args.get('log_level'))
logger.info('args: %s', str(args))
logger.info("BEGIN PROCESSING...")
if args['district']:
districtName=args['district'].lower()
finyear=args['finyear']
db = dbInitialize(db=districtName, charset="utf8") # The rest is updated automatically in the function
cur=db.cursor()
db.autocommit(True)
#Query to set up Database to read Hindi Characters
query="SET NAMES utf8"
cur.execute(query)
limitString=''
if args['limit']:
limitString="limit "+str(args['limit'])
# if args['copyPhones']:
# logger.info("Copying Phones from libtech to Surguja Database")
# copyPhones(cur,logger)
# if args['updateRejectionReason']:
# logger.info("This loop will update RejectionReason")
# updateRejectionReason(cur,logger,districtName)
# if args['updateFinYear']:
# logger.info("This loop will update financial year in workDetails")
# updateFinYear(cur,logger,districtName,limitString)
if args['revertFTOInformation']:
logger.info("This loop will update FTO Information")
if args['revertWorkID']:
wdID=args['revertWorkID']
revertFTOInformation(cur,logger,wdID)
else:
query="select id from workDetails where finyear='%s' order by id DESC %s" % (finyear,limitString)
cur.execute(query)
results=cur.fetchall()
for row in results:
wdID=row[0]
revertFTOInformation(cur,logger,wdID)
if args['updateJCNumber']:
logger.info("This will update jcNumber")
updateJCNumber(cur,logger,districtName,finyear,limitString)
# if args['updatePrimaryAccountHolder']:
# logger.info("This loop will update Primary Account Holder")
# updatePrimaryAccountHolder(cur,logger,districtName)
dbFinalize(db) # Make sure you put this if there are other exit paths or errors
logger.info("...END PROCESSING")
exit(0)
def main():
args = argsFetch()
logger = loggerFetch(args.get('log_level'))
logger.info('args: %s', str(args))
myPanchayats=State.objects.all()
for eachPanchayat in myPanchayats:
logger.info("Panchayt Code: %s, PanchayatName: %s " % (eachPanchayat.code,eachPanchayat.name))
eachPanchayat.save()
logger.info("...END PROCESSING")
exit(0)
def setUp(self):
args = argsFetch()
level = args['log_level']
if not level:
level = 'info'
self.logger = loggerFetch(level)
self.logger.info('args: %s', str(args))
self.logger.info('BEGIN PROCESSING...')
self.parse = args['parse']
dirname = args['directory']
if not self.parse:
self.display = displayInitialize(args['visible'])
self.driver = driverInitialize(timeout=3, options='--headless') # driverInitialize(path='/opt/firefox/', timeout=3)
def main():
args = argsFetch()
logger = loggerFetch(args.get('log_level'))
if args['initiateCrawl']:
logger.debug("This script is going to initiate crawl")
if args['step']:
crawlStates=CrawlState.objects.filter(name=args['step'])
else:
#crawlStates=CrawlState.objects.all()
crawlStates=CrawlState.objects.filter(isActive=True)
for eachState in crawlStates:
crawlProcessTimeThreshold=eachState.threshold
curStateName=eachState.name
logfile="%s/%s.log" % (logDir,curStateName)
debuglogfile="/%s/debug/%s_%s.log" % (logDir,curStateName,str(int(time.time())))
logger.debug("Curent state name is %s " % curStateName)
curhour=datetime.now().hour
nicTimeBand=False
if (curhour >=6) and (curhour <= 20):
nicTimeBand=True
scriptDir='%s/custom/crawlScripts/' % djangoDir
if ((eachState.nicHourRestriction==False) or ((eachState.nicHourRestriction==True) and (nicTimeBand==True))):
cmd="python %s/crawlMain.py -e -l debug -step %s " % (scriptDir,curStateName)
# cmd=scriptName
p1 = Popen(['pgrep', '-f', cmd], stdout=PIPE)
# logger.debug(p1.communicate())
mypid = p1.communicate()[0].decode("utf-8").lstrip().rstrip()
logger.debug("Exsiting PID for this command %s is %s " % (curStateName,str(mypid)))
if (mypid == ""):
logger.debug("We are going to launch this program %s" % cmd)
proc = Popen([cmd], shell=True,stdin=None, stdout=None, stderr=None, close_fds=True)
else:
mycmd='ps -o etimes= -p %s ' % mypid
p1 = Popen([mycmd], stdout=PIPE,shell=True)
output = p1.communicate()[0].decode("utf-8").lstrip().rstrip()
logger.debug(output)
if int(output) > crawlProcessTimeThreshold:
mycmd="kill -9 %s " % mypid
mycmd="pkill -P %s " % mypid
logger.debug(mycmd)
p1 = Popen([mycmd], stdout=PIPE,shell=True)
output = p1.communicate()[0].decode("utf-8").lstrip().rstrip()
logger.debug(output)
mycmd="cp %s %s " % (logfile,debuglogfile)
p1 = Popen([mycmd], stdout=PIPE,shell=True)
output = p1.communicate()[0].decode("utf-8").lstrip().rstrip()
logger.debug(output)
logger.info("...END PROCESSING")
exit(0)
def main():
args = argsFetch()
logger = loggerFetch(args.get('log_level'))
logger.info('args: %s', str(args))
logger.info("BEGIN PROCESSING...")
limitString=''
if args['limit']:
limitString=" limit %s " % args['limit']
additionalFilters=''
if args['district']:
additionalFilters+= " and p.districtName='%s' " % args['district']
if args['fullPanchayatCode']:
additionalFilters= " and p.fullPanchayatCode='%s' " % args['fullPanchayatCode']
db = dbInitialize(db=nregaDB, charset="utf8") # The rest is updated automatically in the function
cur=db.cursor()
db.autocommit(True)
#Query to set up Database to read Hindi Characters
query="SET NAMES utf8"
cur.execute(query)
query="select p.stateCode,p.districtCode,p.blockCode,p.panchayatCode,p.stateName,p.districtName,p.rawBlockName,p.panchayatName,p.fullPanchayatCode,p.stateShortCode,p.crawlIP,p.fullBlockCode from panchayats p,panchayatStatus ps where p.fullPanchayatCode=ps.fullPanchayatCode and p.isRequired=1 and ( (TIMESTAMPDIFF(DAY, ps.jobcardCrawlDate, now()) > 7) or ps.jobcardCrawlDate is NULL) %s order by ps.jobcardCrawlDate,fullPanchayatCode %s" % (additionalFilters,limitString)
cur.execute(query)
results=cur.fetchall()
for row in results:
[stateCode,districtCode,blockCode,panchayatCode,stateName,districtName,blockName,panchayatName,fullPanchayatCode,stateShortCode,crawlIP,fullBlockCode]=row
logger.info("Processing statName: %s, districtName: %s, blockName: %s, panchayatName : %s " % (stateName,districtName,blockName,panchayatName))
filepath=nregaWebDir.replace("stateName",stateName.upper()).replace("districtName",districtName.upper())
filename=filepath+blockName.upper()+"/%s/%s_MISjobcardRegister.html" % (panchayatName.upper(),panchayatName.upper())
fullDistrictCode=stateCode+districtCode
htmlresponse,myhtml=getWorkerList(cur,logger,stateCode,fullDistrictCode,fullBlockCode,fullPanchayatCode)
if htmlresponse['status'] == '200':
logger.info("File Downloaded SuccessFully")
status,outhtml=alterWorkerList(cur,logger,myhtml,stateName,districtName,blockName,panchayatName)
if status == 1:
if not os.path.exists(os.path.dirname(filename)):
os.makedirs(os.path.dirname(filename))
myfile = open(filename, "wb")
myfile.write(outhtml.encode("UTF-8"))
query="update panchayatStatus set jobcardCrawlDate=NOW() where fullPanchayatCode='%s' " % fullPanchayatCode
cur.execute(query)
#myfile.write(outhtml)
else:
logger.info("Not Downloaded")
logger.info(filename)
dbFinalize(db) # Make sure you put this if there are other exit paths or errors
logger.info("...END PROCESSING")
exit(0)
def main():
args = argsFetch()
logger = loggerFetch(args.get('log_level'))
logger.info('args: %s', str(args))
logger.info("BEGIN PROCESSING...")
limitString=''
if args['limit']:
limitString=" limit %s " % args['limit']
additionalFilters=''
if args['district']:
additionalFilters+= " and p.districtName='%s' " % args['district']
if args['fullPanchayatCode']:
additionalFilters= " and p.fullPanchayatCode='%s' " % args['fullPanchayatCode']
db = dbInitialize(db=nregaDB, charset="utf8") # The rest is updated automatically in the function
cur=db.cursor()
db.autocommit(True)
#Query to set up Database to read Hindi Characters
query="SET NAMES utf8"
cur.execute(query)
query="select p.stateCode,p.districtCode,p.blockCode,p.panchayatCode,p.stateName,p.districtName,p.rawBlockName,p.panchayatName,p.fullPanchayatCode,p.stateShortCode,p.crawlIP,p.fullBlockCode from panchayats p,panchayatStatus ps where p.fullPanchayatCode=ps.fullPanchayatCode and p.isRequired=1 and ( (TIMESTAMPDIFF(DAY, ps.jobcardCrawlDate, now()) > 7) or ps.jobcardCrawlDate is NULL) %s order by ps.jobcardCrawlDate,fullPanchayatCode %s" % (additionalFilters,limitString)
cur.execute(query)
results=cur.fetchall()
for row in results:
[stateCode,districtCode,blockCode,panchayatCode,stateName,districtName,blockName,panchayatName,fullPanchayatCode,stateShortCode,crawlIP,fullBlockCode]=row
logger.info("Processing statName: %s, districtName: %s, blockName: %s, panchayatName : %s " % (stateName,districtName,blockName,panchayatName))
fullDistrictCode=stateCode+districtCode
filepath=nregaWebDir.replace("stateName",stateName.upper()).replace("districtName",districtName.upper())
filename=filepath+blockName.upper()+"/%s/%s_stats.html" % (panchayatName.upper(),panchayatName.upper())
logger.info("filename: %s " % filename)
url="http://mnregaweb4.nic.in/netnrega/all_lvl_details_new.aspx?panchayat_code=%s&panchayat_name=%s&block_code=%s&block_name=%s&district_code=%s&district_name=%s&state_code=%s&state_name=%s&page=p&fin_year=2015-2016" % (fullPanchayatCode,panchayatName,fullBlockCode,blockName,fullDistrictCode,districtName,stateCode,stateName)
logger.info(url)
# htmlresponse,myhtml=getStatus(cur,logger,glanceURL,stateCode,fullDistrictCode,fullBlockCode,fullPanchayatCode)
httplib2.debuglevel = 1
h = httplib2.Http('.cache')
htmlresponse, myhtml = h.request(url)
if htmlresponse['status'] == '200':
logger.info("File Downloaded SuccessFully")
if not os.path.exists(os.path.dirname(filename)):
os.makedirs(os.path.dirname(filename))
myfile = open(filename, "wb")
try:
myfile.write(myhtml.encode("UTF-8"))
except:
myfile.write(myhtml)
dbFinalize(db) # Make sure you put this if there are other exit paths or errors
logger.info("...END PROCESSING")
exit(0)
def main():
args = argsFetch()
logger = loggerFetch(args.get('log_level'))
logger.info('args: %s', str(args))
if args['populateWeights']:
db = firestore.Client()
logger.info("Going to Populate the database")
if args['customer']:
customer=args['customer']
jsonName="%s.json" % (customer)
json_data=open(jsonName,encoding='utf-8-sig').read()
d = json.loads(json_data)
i=0
for item in d:
i=i+1
weight=item
logger.info(weight)
doc_ref = db.collection(customer).document(str(i))
doc_ref.set({
u'weight': weight
})
if args['populate']:
db = firestore.Client()
logger.info("Going to Populate the database")
jsonName="sample_data.json"
json_data=open(jsonName,encoding='utf-8-sig').read()
d = json.loads(json_data)
i=0
for item in d:
i=i+1
name=item[0]
value=item[1]
logger.info(name)
doc_ref = db.collection(u'products').document(str(i))
doc_ref.set({
u'name': name,
u'value': value
})
# Project ID is determined by the GCLOUD_PROJECT environment variable
#
# doc_ref = db.collection(u'users').document(u'alovelace')
# doc_ref.set({
# u'first': u'Ada',
# u'last': u'Lovelace',
# u'born': 1815
# })
logger.info("...END PROCESSING")
exit(0)
def runTestSuite():
logger = loggerFetch("info")
logger.info("BEGIN PROCESSING...")
display = displayInitialize(1)
driver = driverInitialize(path='/opt/firefox/')
driver.get(url)
logger.info("Fetching...[%s]" % url)
driver.get(url) # A double refresh required for the page to load
logger.info("Refreshing...[%s]" % url)
html_source = driver.page_source.replace('<head>',
'<head><meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>')
logger.debug("HTML Fetched [%s]" % html_source)
bs = BeautifulSoup(html_source, "html.parser")
tr_list = bs.findAll('tr', attrs={'class':['normalRow', 'alternateRow']})
logger.debug(str(tr_list))
for tr in tr_list:
td = tr.find('td')
td = td.findNext('td')
panchayat = td.text.strip()
logger.info("Panchayat[%s]", panchayat)
elem = driver.find_element_by_link_text(panchayat)
elem.click()
filename="/tmp/%s.html" % panchayat
with open(filename, 'wb') as html_file:
logger.info("Writing [%s]" % filename)
html_file.write(driver.page_source.encode('utf-8'))
driver.back()
driverFinalize(driver)
displayFinalize(display)
logger.info("...END PROCESSING")
def runTestSuite():
logger = loggerFetch("info")
logger.info("BEGIN PROCESSING...")
display = displayInitialize(0)
driver = driverInitialize()
report = generate_report(logger, driver)
logger.info('Finally: \n%s' % report)
with open(filename, 'wb') as csv_file:
logger.info("Writing to [%s]" % filename)
csv_file.write(report.encode('utf-8'))
driverFinalize(driver)
displayFinalize(display)
logger.info("...END PROCESSING")
return
def setUp(self):
self.logger = loggerFetch('info')
self.logger.info('BEGIN PROCESSING...')
self.display = displayInitialize(1)
self.driver = driverInitialize(path='/opt/firefox/', timeout=3)
def setUp(self):
self.logger = loggerFetch('info')
self.logger.info('BEGIN PROCESSING...')
def main():
thresholdDate = datetime.strptime('Oct 15 2017', '%b %d %Y').date()
memberIndex = 1
productNameIndex = 0
startCol = 6
args = argsFetch()
logger = loggerFetch(args.get('log_level'))
logger.info('args: %s', str(args))
print("test")
if args['limit']:
limit = int(args['limit'])
else:
limit = 1
#Lets Process Grocery Order Form
if args['sheetID']:
myOrderForms = ConsolidatedOrderForm.objects.filter(
isProcessed=0, id=args['sheetID'])[:limit]
else:
myOrderForms = ConsolidatedOrderForm.objects.filter(
isProcessed=0)[:limit]
for eachOrderForm in myOrderForms:
isBeforeSoftware = False
productCodeIndex = 2
priceIndex = 4
orderDate = eachOrderForm.orderDate
if orderDate < thresholdDate:
isBeforeSoftware = True
productCodeIndex = 1
priceIndex = 2
creditIndex = eachOrderForm.creditIndex - 1
creditSheetIndex = eachOrderForm.creditSheetIndex
sheetIndex = eachOrderForm.sheetNo
balanceSheetIndex = eachOrderForm.balanceSheetIndex
maxOrderRow = eachOrderForm.maxRow
myFile = eachOrderForm.orderFile.read()
book = xlrd.open_workbook(file_contents=myFile)
#Balance Sheet Index
worksheet = book.sheet_by_index(balanceSheetIndex)
num_rows = worksheet.nrows - 1
curr_row = 0
vegetableCostIndex = 11
creditIndex = 10
adjustmentIndex = 11
adjustmentReasonIndex = 14
nameIndex = 4
emailIndex = 5
boxIDIndex = 1
password = "******"
while curr_row < num_rows:
curr_row = curr_row + 1
memberCode = str(worksheet.cell_value(curr_row, 0)).replace(
"-", "").lstrip().rstrip()
vegetableCost = int(
worksheet.cell_value(curr_row, vegetableCostIndex))
if isBeforeSoftware == True:
memberCode = str(worksheet.cell_value(curr_row, 0)).replace(
"-", "").lstrip().rstrip()
myMember = Member.objects.filter(
user__username=memberCode).first()
else:
memberCode = int(worksheet.cell_value(curr_row, 0))
if memberCode != 0:
boxID = str(worksheet.cell_value(
curr_row, boxIDIndex)).lstrip().rstrip()
email = str(worksheet.cell_value(
curr_row, emailIndex)).lstrip().rstrip()
name = str(worksheet.cell_value(
curr_row, nameIndex)).lstrip().rstrip()
myMember = Member.objects.filter(
user__username=memberCode).first()
logger.info(str(int(memberCode)))
if myMember is None:
myUser = User.objects.create_user(username=memberCode,
email=email,
password=password)
Member.objects.create(user=myUser,
boxID=boxID,
name=name)
else:
myUser = myMember.user
myUser.email = email
myUser.save()
myMember = Member.objects.filter(
user__username=memberCode).first()
#myMember=Member.objects.filter(boxID=memberCode).first()
if myMember is None:
allMembersFound = 0
logger.info("Member not found %s " % (memberCode))
else:
if vegetableCost != 0:
error = addMemberTransaction(logger, myMember, "VEGBOX", 1,
vegetableCost, "DR",
orderDate)
#Making the credit and Adjustment
if isBeforeSoftware == True:
if worksheet.cell_type(curr_row, creditIndex) != 0:
credit = str(
worksheet.cell_value(curr_row,
creditIndex)).replace(
"-",
"").lstrip().rstrip()
error = addMemberTransaction(logger, myMember, "CRR",
1, credit, "CR",
orderDate)
if worksheet.cell_type(curr_row, adjustmentIndex) != 0:
adjustment = str(
worksheet.cell_value(
curr_row, adjustmentIndex)).lstrip().rstrip()
logger.info(adjustment)
adjustmentReason = str(
worksheet.cell_value(
curr_row, adjustmentReasonIndex)).replace(
"-", "").lstrip().rstrip()
if "-" in adjustment:
logger.info("Adjustment is Negative")
adjustmentTransactionType = "CR"
adjustment = adjustment[1:]
else:
adjustmentTransactionType = "DR"
error = addMemberTransaction(logger,
myMember,
"ADJST",
1,
adjustment,
adjustmentTransactionType,
orderDate,
remarks=adjustmentReason)
#This is for calcuating Credits
if creditSheetIndex != 0:
worksheet = book.sheet_by_index(creditSheetIndex)
num_cols = worksheet.ncols - 1
num_rows = worksheet.nrows - 1
curr_row = 0
while curr_row < num_rows:
curr_row += 1
memberCode = int(worksheet.cell_value(curr_row, 0))
logger.info("Credit memberCode %s " % str(memberCode))
Amount = worksheet.cell_value(curr_row, 1)
creditDateExcel = worksheet.cell_value(curr_row, 2)
creditDate = datetime(
*xlrd.xldate_as_tuple(creditDateExcel, 0)).date()
logger.info(str(memberCode) + str(Amount) + str(creditDate))
myMember = Member.objects.filter(
user__username=memberCode).first()
if myMember is not None:
error = addMemberTransaction(logger, myMember, "CRR", 1,
Amount, "CR", creditDate)
#This is for calculating Products
if eachOrderForm.isItemizedOrderFormat == False:
worksheet = book.sheet_by_index(sheetIndex)
isComplete = 1
firstRow = worksheet.row_values(0)
logger.info(str(firstRow))
num_cols = worksheet.ncols - 1
curr_col = startCol
allMembersFound = 1
while curr_col < num_cols:
curr_col += 1
if worksheet.cell_type(memberIndex, curr_col) != 0:
memberCode = str(
worksheet.cell_value(memberIndex,
curr_col)).replace("-", "")
memberCode = memberCode[2:]
# logger.info(memberCode)
if isBeforeSoftware == True:
myMember = Member.objects.filter(
user__username=memberCode).first()
else:
myMember = Member.objects.filter(
user__username=memberCode).first()
#myMember=Member.objects.filter(boxID=memberCode).first()
if myMember is None:
allMembersFound = 0
logger.info("Member not found %s " % (memberCode))
else:
curr_row = 2
while curr_row < maxOrderRow:
curr_row += 1
if worksheet.cell_type(curr_row, curr_col) != 0:
if worksheet.cell_type(curr_row,
priceIndex) != 0:
productName = worksheet.cell_value(
curr_row,
productNameIndex).lstrip().rstrip()
productCode = worksheet.cell_value(
curr_row,
productCodeIndex).lstrip().rstrip()
price = worksheet.cell_value(
curr_row, priceIndex)
quantity = worksheet.cell_value(
curr_row, curr_col)
if float(quantity) != 0:
logger.info(
"Product Code: %s Price : %s Quantity %s Member %s "
% (productCode, price,
str(int(quantity)), memberCode))
error = addMemberTransaction(
logger, myMember, productCode,
quantity, price, "DR", orderDate)
if error is not None:
isComplete = False
else:
logger.info("I am in itemized biling")
productCodeIndex = 15
priceIndex = 12
quantityIndex = 11
orderStatusIndex = 19
worksheet = book.sheet_by_index(sheetIndex)
isComplete = 1
num_rows = worksheet.nrows - 1
num_cols = worksheet.ncols - 1
curr_col = startCol
allMembersFound = 1
row = 0
while row < num_rows:
row = row + 1
rowValues = worksheet.row_values(row)
# logger.info(str(rowValues))
logger.info(row)
logger.info(worksheet.cell_value(row, 1))
memberCode = int(worksheet.cell_value(row, 1))
myMember = Member.objects.filter(
user__username=memberCode).first()
productCode = worksheet.cell_value(
row, productCodeIndex).lstrip().rstrip()
orderStatus = worksheet.cell_value(
row, orderStatusIndex).lstrip().rstrip()
quantity = worksheet.cell_value(row, quantityIndex)
price = worksheet.cell_value(row, priceIndex)
error = addMemberTransaction(logger,
myMember,
productCode,
quantity,
price,
"DR",
orderDate,
orderStatus=orderStatus)
if (isComplete == True) and (allMembersFound == 1):
logger.info("Sheet is Completely Processed")
logger.info("...END PROCESSING")
exit(0)
def runTestSuite():
logger = loggerFetch("info")
logger.info("BEGIN PROCESSING...")
display = displayInitialize(1)
driver = driverInitialize()
'''
content = csv.reader(open('./gats.csv', 'r'), delimiter=',', quotechar='"')
for (gat, d) in content:
'''
for gat in gat_list:
logger.info('Fetching gat[%s]...' % gat)
driver.get(url)
try:
driver.find_element_by_xpath(
"//form[@id='aspnetForm']/div[3]/div/div/div[3]/a[3]/p").click(
)
except:
logger.error('Cant find element for [%s]' % gat)
continue
Select(
driver.find_element_by_id("distSelect")).select_by_visible_text(dn)
Select(
driver.find_element_by_id("talSelect")).select_by_visible_text(tn)
Select(
driver.find_element_by_id("vilSelect")).select_by_visible_text(vn)
# Select(driver.find_element_by_id("vilSelect")).select_by_visible_text(u"सार्पिली")
# driver.find_element_by_css_selector("option[value=\"string:273200030399810000\"]").click()
driver.find_element_by_id("rbsryno").click()
driver.find_element_by_xpath("//input[@type='number']").clear()
driver.find_element_by_xpath("//input[@type='number']").send_keys(gat)
driver.find_element_by_css_selector("input[type=\"button\"]").click()
'''
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException
try:
WebDriverWait(driver, 5).until(EC.alert_is_present(), 'Waiting for alert timed out')
alert = driver.switch_to_alert()
alert.accept()
logger.warning("alert accepted")
except:
logger.info("Yippie!")
'''
if len(driver.window_handles) > 1:
logger.info("Dialog Box Window [" + str(driver.window_handles) +
"]")
# self.assertEqual(u"सर्वे नंबर / गट नंबर तपासुन पहा.", self.close_alert_and_get_its_text())
# driver.find_element_by_xpath("//div[@id='block-desktop']/div[3]/div[2]/div").click()
driver.switch_to_alert().accept()
continue
html_source = driver.page_source.replace(
'<head>',
'<head><meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>'
)
logger.debug("HTML Fetched [%s]" % html_source)
soup = BeautifulSoup(driver.page_source, "html.parser")
#dict_from_json = soup.find(attribute).text
sno_select = soup.find(
"select", {"ng-model": "selectedSno"}) # ng-model="selectedSno"
sno_options = sno_select.findAll("option")
snos = [sno_option.text for sno_option in sno_options]
logger.debug("Found [%s]" % str(snos))
snos = snos[1:]
logger.info("SNO List [%s]" % str(snos))
#time.sleep(5)
for sno in snos:
logger.info('Processing [%s]' % sno)
filename = '/home/mayank/wd/SaatBaara/remaining/%s.html' % sno.replace(
'/', '_')
if os.path.exists(filename):
#time.sleep(1)
continue
Select(
driver.find_element_by_xpath(
"//form[@id='aspnetForm']/div[3]/div/div/div[3]/div/div[3]/table/tbody/tr[3]/td/select"
)).select_by_visible_text(sno)
#logger.info(driver.find_element_by_link_text(sno))
driver.find_element_by_css_selector(
"td.last-rows > input[type=\"button\"]").click()
time.sleep(5)
parent_handle = driver.current_window_handle
logger.info("Handles : %s Number : %s" %
(driver.window_handles, len(driver.window_handles)))
if len(driver.window_handles) == 2:
driver.switch_to_window(driver.window_handles[-1])
else:
logger.error("Handlers gone wrong [" +
str(driver.window_handles) + "]")
driver.save_screenshot('z.png')
html_source = driver.page_source.replace(
'<head>',
'<head><meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>'
).encode('utf-8')
logger.debug("HTML Fetched [%s]" % html_source)
if (driver.title != '७/१२'):
logger.error(driver.title)
driver.close()
driver.switch_to_window(parent_handle)
continue
bs = BeautifulSoup(html_source, "html.parser")
body = bs.find('tbody')
try:
body = body.findNext('tbody')
except:
logger.error('Empty body for [%s]' % sno)
driver.close()
driver.switch_to_window(parent_handle)
continue
body = body.findNext('tbody')
logger.debug(body)
td = body.find('td')
td = td.findAll('td')
logger.info("Checking [%s]" % td[2].text)
if (sno != td[2].text):
logger.error('sno[%s] != td.text[%s]' % (sno, td[2].text))
driver.close()
driver.switch_to_window(parent_handle)
continue
with open(filename, 'wb') as html_file:
logger.info('Writing [%s]' % filename)
html_file.write(html_source)
driver.close()
driver.switch_to_window(parent_handle)
time.sleep(1)
time.sleep(1)
driverFinalize(driver)
displayFinalize(display)
logger.info("...END PROCESSING")
def main():
args = argsFetch()
logger = loggerFetch(args.get('log_level'))
logger.info('args: %s', str(args))
if args['limit']:
limit = int(args['limit'])
else:
limit = 1
if args['misc']:
logger.info("Doing misc tasks")
myobjs = MemberTransaction.objects.all()
for obj in myobjs:
obj.orderQuantity = obj.quantity
obj.save()
if args['memberSpend']:
logger.info("Member wise spend")
curmonth = 2
s = ""
myTransactions = MemberTransaction.objects.filter(
orderDate__month=curmonth, transactionType='DR')
for eachTransaction in myTransactions:
total = str(eachTransaction.price * eachTransaction.quantity)
s += "%s,%s,%s\n" % (eachTransaction.member.user.username,
str(eachTransaction.orderDate), total)
with open("/tmp/ds.csv", "w") as f:
f.write(s)
if args['monthlySpend']:
logger.info("Calculating Monthly Spend")
myMonths = [4, 5, 6, 7, 8, 9, 10, 11, 12, 1, 2, 3]
s = ''
for curmonth in myMonths:
total = 0
myTransactions = MemberTransaction.objects.filter(
orderDate__month=curmonth, transactionType='DR')
for eachTransaction in myTransactions:
total = total + eachTransaction.price * eachTransaction.quantity
logger.info(total)
eachLine = "%s,%s\n" % (str(curmonth), str(total))
s += eachLine
with open("/tmp/ms.csv", "w") as f:
f.write(s)
if args['dumpMonthlyBalance']:
logger.info("Going to Dump Monthly Balance")
myMembers = Member.objects.all()
curmonth = 5
curyear = 2018
s = "MemberName,AprilBal,APrilCredit,MayBal,MayCredit,JuneBal,JuneCredit,JulyBal,JulyCredit,AUgustBal,AugustCredit,SeptemberBal,SeptemberCredit,OctoberBal,OctoberCredit,NovemberBal,NovemberCredit,DecemberBal,DecemberCredit,JanuaryBalance,JanuaryCredit,FebuaryBalance,FebuaryCredit\n"
s = "memberCode,memberName,April,May,June,July,August,Sepetember,October,November,December,January,Feb,March\n"
for eachMember in myMembers:
s += str(eachMember.user.username)
s += ","
memberID = "%s-%s," % (eachMember.user.username, eachMember.name)
s += memberID
curyear = 2017
myMonths = [4, 5, 6, 7, 8, 9, 10, 11, 12]
startmonth = 5
startyear = 2017
endmonth = 4
endyear = 2018
myDates = [
datetime.date(m // 12, m % 12 + 1, 1)
for m in range(startyear * 12 + startmonth - 1, endyear * 12 +
endmonth)
]
for eachDate in myDates:
lastTransaction = MemberTransaction.objects.filter(
member=eachMember,
orderDate__lt=eachDate).order_by("-orderDate",
"-id").first()
if lastTransaction is not None:
lastBalance = lastTransaction.balance
else:
lastBalance = 0
s += str(lastBalance)
s += ","
# for curmonth in myMonths:
# lastTransaction=MemberTransaction.objects.filter(member=eachMember,orderDate__year__lte=curyear,orderDate__month__lte=curmonth).order_by("-orderDate","id").first()
# if lastTransaction is not None:
# lastBalance=lastTransaction.balance
# else:
# lastBalance=0
# s+=str(lastBalance)
# s+=","
# myTransactions=MemberTransaction.objects.filter(member=eachMember,orderDate__year=curyear,orderDate__month=curmonth,product__id=181)
# totalCredit=0
# for eachTransaction in myTransactions:
# totalCredit+=eachTransaction.price
# s+=str(totalCredit)
# s+=","
# curyear=2018
# myMonths=[1,2,3]
# for curmonth in myMonths:
# lastTransaction=MemberTransaction.objects.filter(member=eachMember,orderDate__year__lte=curyear,orderDate__month__lte=curmonth).order_by("-orderDate","id").first()
# if lastTransaction is not None:
# lastBalance=lastTransaction.balance
# else:
# lastBalance=0
# s+=str(lastBalance)
# s+=","
#
# myTransactions=MemberTransaction.objects.filter(member=eachMember,orderDate__year=curyear,orderDate__month=curmonth,product__id=181)
# totalCredit=0
# for eachTransaction in myTransactions:
# totalCredit+=eachTransaction.price
# s+=str(totalCredit)
# s+=","
s += "\n"
with open("/tmp/mb.csv", "w") as f:
f.write(s)
if args['changeMemberCode']:
logger.info("Changing Member Code")
myUsers = User.objects.all()
for eachUser in myUsers:
username = eachUser.username
areaCode = username[0:2]
eachMember = Member.objects.filter(user=eachUser).first()
if eachMember is not None:
boxID = eachMember.boxID
else:
boxID = "abcd"
boxID = boxID[2:]
if boxID.isdigit():
if int(boxID) != 0 and int(boxID) != 1:
logger.info('username : %s boxID : %s ' %
(username, boxID))
eachUser.username = str(boxID)
eachUser.save()
else:
a = 1
# myMembers=Member.objects.all()[:1]
# for eachMember in myMembers:
# logger.info(eachMember.boxID)
if args['print']:
orderDateString = args['orderDate']
orderDate = datetime.datetime.strptime(orderDateString,
'%d-%m-%Y').date()
myTransactions = MemberTransaction.objects.filter(
orderDate=orderDate).values("member__boxID").annotate(
dcount=Count('pk'))
for eachTransaction in myTransactions:
logger.info(str(eachTransaction))
boxID = eachTransaction['member__boxID']
count = eachTransaction['dcount']
logger.info(boxID + "-" + str(count))
# for eachTransaction in myTransactions:
# logger.info("Prodct: %s Quantity: %s " % (eachTransaction.product.name,str(eachTransaction.quantity)))
if args['deleteTransactions']:
orderDateString = args['orderDate']
orderDate = datetime.datetime.strptime(orderDateString,
'%d-%m-%Y').date()
myTransactions = MemberTransaction.objects.filter(orderDate=orderDate)
for eachTransaction in myTransactions:
logger.info(eachTransaction.member)
eachTransaction.delete()
if args['dumpBalance']:
myMembers = Member.objects.all()
for eachMember in myMembers:
lastTransaction = MemberTransaction.objects.filter(
member=eachMember).order_by("-orderDate", "-id").first()
if lastTransaction is not None:
lastBalance = lastTransaction.balance
else:
lastBalance = 0
print(eachMember.user.username + "," + str(eachMember.boxID) + ",",
str(lastBalance))
if args['calculateBalance']:
myMembers = Member.objects.all()[:limit]
for eachMember in myMembers:
logger.info(eachMember.name)
memberTransactions = MemberTransaction.objects.filter(
member=eachMember).order_by('orderDate', 'id')
curBalance = eachMember.startBalance
curBalance = 0
logger.info("Current Balance %s " % str(curBalance))
for eachTransaction in memberTransactions:
logger.info("%s-%s" % (str(eachTransaction.orderDate),
str(eachTransaction.product.name)))
transactionType = eachTransaction.transactionType
price = eachTransaction.price
quantity = eachTransaction.quantity
if transactionType == "DR":
amount = price * (-1) * quantity
else:
amount = price * 1 * quantity
# logger.info("Price : %s, Amount : %s " % (str(price),str(amount)))
curBalance = curBalance + amount
eachMember.balance = curBalance
# logger.info("Current Balance %s " % str(curBalance))
eachTransaction.balance = curBalance
eachTransaction.save()
eachMember.balance = curBalance
eachMember.save()
if args['deleteZeroTransaction']:
myTransactions = MemberTransaction.objects.filter(quantity=0)
for eachTransaction in myTransactions:
logger.info(eachTransaction.quantity)
eachTransaction.delete()
myTransactions = MemberTransaction.objects.filter(price=0)
for eachTransaction in myTransactions:
logger.info(eachTransaction.price)
eachTransaction.delete()
if args['updateBoxID']:
logger.info("We are going to update BOx IDs")
with open('/tmp/mm.csv', 'r') as fp:
for line in fp:
line = line.lstrip().rstrip()
if (line != ''):
lineArray = str(line).split(",")
username = lineArray[0]
boxid = lineArray[1]
logger.info("Old Username: %s box id %s " %
(username, boxid))
myUser = User.objects.filter(username=username).first()
myMember = Member.objects.filter(user=myUser).first()
if myMember is not None:
print("Member Found")
myMember.boxID = boxid
myMember.save()
logger.info("...END PROCESSING")
exit(0)
def main():
args = argsFetch()
logger = loggerFetch(args.get('log_level'))
logger.info('args: %s', str(args))
logger.info("BEGIN PROCESSING...")
display = displayInitialize(args['visible'])
driver = driverInitialize(browser=args['browser'] , path='/home/mayank/.mozilla/firefox/4s3bttuq.default/')
base_url="https://www.skillshare.com/login"
driver.get(base_url)
logger.info('Fetching URL[%s]' % base_url)
# driver.find_element_by_link_text("Sign In").click()
try:
driver.find_element_by_name("LoginForm[email]").clear()
driver.find_element_by_name("LoginForm[email]").send_keys("*****@*****.**")
driver.find_element_by_name("LoginForm[password]").clear()
# driver.find_element_by_name("LoginForm[password]").send_keys("#######")
# driver.find_element_by_xpath("//input[@value='Sign In']").click()
time.sleep(100) # If you want to manually log in
except Exception as e:
logger.info('Already signed in [%s]', e)
time.sleep(10)
filename = "./z.csv"
content = csv.reader(open(filename, 'r'), delimiter=',', quotechar='"')
for (title, url) in content:
# driver.get('https://www.skillshare.com/classes/Sketchbook-Practice-Bring-watercolour-to-Life-with-Line-Drawing/1053382271/classroom/discuss')
logger.info('Fetching URL[%s]' % url)
driver.get(url)
time.sleep(10)
escaped_title = re.sub(r"[^A-Za-z 0-9]+", '', title).replace(' ', '_')
dirname = 'SkillsShare/' + escaped_title
cmd = 'mkdir -p ' + dirname
logger.info(cmd)
os.system(cmd)
els = driver.find_elements_by_class_name("session-item")
for i, el in enumerate(els):
logger.debug(str(el))
bs = BeautifulSoup(el.get_attribute('innerHTML'), "html.parser")
p = bs.find('p')
name = p.text
name = "%02d" % (i+1) + '_' + re.sub(r"[^A-Za-z 0-9]+", '', name).replace(' ', '_') + '.mp4'
logger.info(str(p) + name)
el.click()
time.sleep(10)
html_source = driver.page_source
bs = BeautifulSoup(html_source, "html.parser")
html = bs.findAll('video', attrs={'class':['vjs-tech']})
str_html = str(html)
logger.info(str_html)
url = str_html[str_html.find("src=")+5:]
fetch_url = url[:url.find("?pubId")]
logger.info(fetch_url)
if os.path.exists(dirname + '/' + name):
continue
cmd = 'cd %s && curl -s %s -o %s' % (dirname, fetch_url, name)
logger.info(cmd)
# os.system(cmd)
driverFinalize(driver)
displayFinalize(display)
logger.info("...END PROCESSING")
exit(0)
def main():
args = argsFetch()
logger = loggerFetch(args.get('log_level'))
logger.info('args: %s', str(args))
logger.info("This is module to Manage Transactions")
if args['limit']:
limit = args['limit']
else:
limit=50000
if args['sendOrder']:
if args['dateFrom']:
dateFrom=datetime.strptime(args['dateFrom'], '%d-%m-%Y').date()
if args['dateTo']:
dateTo=datetime.strptime(args['dateTo'], '%d-%m-%Y').date()
myMembers=Member.objects.all()[:limit]
myMembers=Member.objects.filter(user__username="******")
for eachMember in myMembers:
if eachMember.user.username.isdigit():
firstTransaction=MemberTransaction.objects.filter(member=eachMember,orderDate__lte=dateFrom).order_by("-orderDate","-id").first()
startBalance=firstTransaction.balance
table=getHTMLHeader()
table+='<body><table class=" table table-striped" ><tbody>'
table+="<tr><th>Product</th><th>orderQuantity</th><th>Status</th><th>type</th><th>SuppliedQuantity</th><th>price</th><th>value</th><th>balance</th></tr>"
tableLine="<tr><td colspan=7>%s</td><td>%s</td></tr>" % ("Start Balance", str(startBalance))
table+=tableLine
lastTransaction=MemberTransaction.objects.filter(member=eachMember,orderDate=dateTo).order_by("-id").first()
lastBalance=lastTransaction.balance
myTransactions=MemberTransaction.objects.filter(member=eachMember,orderDate__gt=dateFrom,orderDate__lte=dateTo)
for eachTransaction in myTransactions:
orderStatus=eachTransaction.orderStatus
lineValue=eachTransaction.price*eachTransaction.quantity
tableLine="<tr><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td></tr>" % (eachTransaction.product.name,str(eachTransaction.orderQuantity),orderStatus,eachTransaction.transactionType,str(eachTransaction.quantity),str(eachTransaction.price),str(lineValue),str(eachTransaction.balance))
table+=tableLine
tableLine="<tr><td colspan=7>%s</td><td>%s</td></tr>" % ("Last Balance", str(lastBalance))
table+=tableLine
table+='</tbody></table ></body>'
toEmail="*****@*****.**"
body=table
subject="In Todays Box"
with open("/tmp/aaa.html","w") as f:
f.write(table)
genericSendEmail(toEmail,subject,body)
if args['sendBalance']:
d=date.today()
dateString=str(datetime.strftime(d, "%b-%d"))
baseURL="https://s3.ap-south-1.amazonaws.com/ndcsa/memberData/"
logger.info("Sending the balance")
myMembers=Member.objects.filter(isActive=True)[:limit]
for eachMember in myMembers:
if eachMember.user.username.isdigit():
lastTransaction=MemberTransaction.objects.filter(member=eachMember).order_by("-orderDate","-id").first()
if lastTransaction is not None:
lastBalance=lastTransaction.balance
logger.info("%s-%s-%s" % (str(eachMember.user.username),eachMember.name,str(lastBalance)))
balanceURL="%s%s.csv" % (baseURL,str(eachMember.user.username))
logger.info(balanceURL)
memberEmail=eachMember.user.email
if args['test']:
memberEmail="*****@*****.**"
logger.info(memberEmail)
sendMail(memberEmail,eachMember.name,lastBalance,balanceURL,dateString)
if args['upload']:
logger.info("Will upload everyone transaction History")
myMembers=Member.objects.all()[:limit]
for eachMember in myMembers:
logger.info(eachMember.user.username)
logger.info(eachMember.name)
memID=str(eachMember.user.username)+"-"+eachMember.name
s=''
balance=0
s+="member,orderDate,product,quantity,price,total,type,balance\n"
myTransactions=MemberTransaction.objects.filter(member=eachMember).order_by("orderDate","id")
for eachTransaction in myTransactions:
logger.info(eachTransaction.product.name)
logger.info(eachTransaction.orderDate)
total=eachTransaction.quantity*eachTransaction.price
if eachTransaction.transactionType == 'DR':
balance=balance-total
else:
balance=balance+total
s+="%s,%s,%s,%s,%s,%s,%s,%s\n" % (memID,str(eachTransaction.orderDate),eachTransaction.product.name,str(eachTransaction.quantity),str(eachTransaction.price),str(total),eachTransaction.transactionType,str(balance))
logger.info(s)
cloudFilename="memberData/%s.csv" % (str(eachMember.user.username))
session = boto3.session.Session(aws_access_key_id=LIBTECH_AWS_ACCESS_KEY_ID,
aws_secret_access_key=LIBTECH_AWS_SECRET_ACCESS_KEY)
s3 = session.resource('s3',config=Config(signature_version='s3v4'))
s3.Bucket(AWS_STORAGE_BUCKET_NAME).put_object(ACL='public-read',Key=cloudFilename, Body=s)
logger.info("...END PROCESSING")
exit(0)
