def getLeads(self):
titleDivs = self.driver.find_elements_by_xpath("//h3[not(ancestor::div[@id='scholarship_intro_859'])]")
for i in range(len(titleDivs)):
title = titleDivs[i].get_attribute('textContent')
requirements = ''
sourceWebsite = ''
description = ''
if title != 'Quick Links' and title != 'About Us':
if i == 0:
description = self.driver.find_element_by_xpath("//div[@class='intro']/p").get_attribute(
'textContent')
sourceWebsite = self.driver.find_element_by_xpath("//div[@class='intro']/p/a").get_attribute('href')
requirements = self.driver.find_element_by_xpath(
"//div[@class='intro']/following-sibling::*[1][self::ul]").get_attribute('textContent')
else:
j = i + 1
if self.checkIfElementExists(
"//h3[not(ancestor::div[@id='scholarship_intro_859'])][%s]/following-sibling::p[1]" % j):
description = self.driver.find_element_by_xpath(
"//h3[not(ancestor::div[@id='scholarship_intro_859'])][%s]/following-sibling::p[1]" % j).get_attribute(
'textContent')
if self.checkIfElementExists(
"//h3[not(ancestor::div[@id='scholarship_intro_859'])][%s]/following-sibling::p[2][(preceding-sibling::*[1][self::p])]" % j):
requirements = self.driver.find_element_by_xpath(
"//h3[not(ancestor::div[@id='scholarship_intro_859'])][%s]/following-sibling::p[2][(preceding-sibling::*[1][self::p])]" % j).get_attribute(
'textContent')
if self.checkIfElementExists(
"//h3[not(ancestor::div[@id='scholarship_intro_859'])][%s]/following-sibling::p[1]/a" % j):
sourceWebsite = self.driver.find_element_by_xpath(
"//h3[not(ancestor::div[@id='scholarship_intro_859'])][%s]/following-sibling::p[1]/a" % j).get_attribute(
'href')
elif self.checkIfElementExists(
"//h3[not(ancestor::div[@id='scholarship_intro_859'])][%s]/following-sibling::p[2][(preceding-sibling::*[1][self::p])]" % j):
if self.checkIfElementExists(
"//h3[not(ancestor::div[@id='scholarship_intro_859'])][%s]/following-sibling::p[2][(preceding-sibling::*[1][self::p])]/a" % j):
sourceWebsite = self.driver.find_element_by_xpath(
"//h3[not(ancestor::div[@id='scholarship_intro_859'])][%s]/following-sibling::p[2][(preceding-sibling::*[1][self::p])]/a" % j).get_attribute(
'href')
sourceText = RipPage.getPageSource(sourceWebsite)
title = CleanText.cleanALLtheText(title)
description = CleanText.cleanALLtheText(description)
requirements = CleanText.cleanALLtheText(requirements)
sourceText = CleanText.cleanALLtheText(sourceText)
leadArray = [title, description, requirements, sourceWebsite, sourceText]
self.teacherDotOrgLeadArrays.append(leadArray)
self.driver.close()
return self.teacherDotOrgLeadArrays
def getTitleAbstractList(self):
wholeList = []
titles = self.getTitles()
abstracts = self.getAbstracts()
for i in range(len(abstracts)):
abstract = CleanText.cleanALLtheText(abstracts[i])
title = CleanText.cleanALLtheText(titles[i])
listOfItems = [title, abstract]
wholeList.append(listOfItems)
return wholeList
def getResultPageInfo(self):
sponsor = CleanText.cleanALLtheText(self.driver.find_element_by_xpath(
"//div/p/strong[text() = 'Awarded By']/../../following-sibling::div/p").get_attribute('textContent'))
awardAmount = CleanText.cleanALLtheText(self.driver.find_element_by_xpath(
"//div/p/strong[text() = 'Award Amount']/../../following-sibling::div/p").get_attribute('textContent'))
recipients = CleanText.cleanALLtheText(self.driver.find_element_by_xpath(
"//div/p/strong[text() = 'Recipients']/../../following-sibling::div/p").get_attribute('textContent'))
requirements = CleanText.cleanALLtheText(self.driver.find_element_by_xpath(
"//div/p/strong[text() = 'Requirements']/../../following-sibling::div").get_attribute('textContent'))
additionalInfo = CleanText.cleanALLtheText(self.driver.find_element_by_xpath(
"//div/p/strong[text() = 'Additional Information']/../../following-sibling::div/p").get_attribute(
'textContent'))
contact = CleanText.cleanALLtheText(self.driver.find_element_by_xpath(
"//div/p/strong[text() = 'Contact']/../../following-sibling::div/p").get_attribute('textContent'))
address = CleanText.cleanALLtheText(self.driver.find_element_by_xpath(
"//div/p/strong[text() = 'Address']/../../following-sibling::div").get_attribute('textContent'))
if self.checkIfElementExists("//a[@class='button secondary']"):
sourceWebsite = self.driver.find_element_by_xpath("//a[@class='button secondary']").get_attribute('href')
sourceText = CleanText.cleanALLtheText(RipPage.getPageSource(sourceWebsite))
else:
sourceWebsite = ''
sourceText = ''
resultPageArray = [sponsor, awardAmount, recipients, requirements, additionalInfo, contact, address,
sourceWebsite, sourceText]
return resultPageArray
def getTitleAbstractList(self):
wholeList = []
titles = self.getTitles()
abstracts = self.getAbstracts()
for i in range(len(abstracts)):
abstract = CleanText.cleanALLtheText(abstracts[i])
title = CleanText.cleanALLtheText(titles[i])
listOfItems = [title, abstract]
wholeList.append(listOfItems)
return wholeList
def getResultPageInfo(self):
url = self.driver.current_url
sponsor = ''
awardAmount = ''
recipients = ''
requirements = ''
additionalInfo = ''
contact = ''
address = ''
deadlineInformation = ''
if self.checkIfElementExists("//div/p/strong[text() = 'Awarded By']/../../following-sibling::div/p"):
sponsor = CleanText.cleanALLtheText(self.driver.find_element_by_xpath(
"//div/p/strong[text() = 'Awarded By']/../../following-sibling::div/p").get_attribute(
'textContent'))
sponsor = re.sub('» More Info', '', sponsor)
if self.checkIfElementExists("//div/p/strong[text() = 'Award Amount']/../../following-sibling::div/p"):
awardAmount = CleanText.cleanALLtheText(self.driver.find_element_by_xpath(
"//div/p/strong[text() = 'Award Amount']/../../following-sibling::div/p").get_attribute(
'textContent'))
if self.checkIfElementExists("//div/p/strong[text() = 'Recipients']/../../following-sibling::div/p"):
recipients = CleanText.cleanALLtheText(self.driver.find_element_by_xpath(
"//div/p/strong[text() = 'Recipients']/../../following-sibling::div/p").get_attribute(
'textContent'))
if self.checkIfElementExists("//div/p/strong[text() = 'Requirements']/../../following-sibling::div"):
requirements = CleanText.cleanALLtheText(self.driver.find_element_by_xpath(
"//div/p/strong[text() = 'Requirements']/../../following-sibling::div").get_attribute(
'textContent'))
if self.checkIfElementExists(
"//div/p/strong[text() = 'Additional Information']/../../following-sibling::div/p"):
additionalInfo = CleanText.cleanALLtheText(self.driver.find_element_by_xpath(
"//div/p/strong[text() = 'Additional Information']/../../following-sibling::div/p").get_attribute(
'textContent'))
if self.checkIfElementExists("//div/p/strong[text() = 'Contact']/../../following-sibling::div/p"):
contact = CleanText.cleanALLtheText(self.driver.find_element_by_xpath(
"//div/p/strong[text() = 'Contact']/../../following-sibling::div/p").get_attribute('textContent'))
if self.checkIfElementExists("//div/p/strong[text() = 'Address']/../../following-sibling::div"):
address = CleanText.cleanALLtheText(self.driver.find_element_by_xpath(
"//div/p/strong[text() = 'Address']/../../following-sibling::div").get_attribute('textContent'))
if self.checkIfElementExists(
"//strong[text() ='Deadline Information']/following-sibling::span[@class='smalltext']"):
deadlineInformation = CleanText.cleanALLtheText(self.driver.find_element_by_xpath(
"//strong[text() ='Deadline Information']/following-sibling::span[@class='smalltext']").get_attribute(
'textContent'))
if self.checkIfElementExists("//a[@class='button cta']"):
sourceWebsite = self.driver.find_element_by_xpath("//a[@class='button cta']").get_attribute('href')
sourceText = CleanText.cleanALLtheText(RipPage.getPageSource(sourceWebsite))
else:
sourceWebsite = ''
sourceText = ''
resultPageArray = [url, sponsor, awardAmount, recipients, requirements, additionalInfo, contact, address,
deadlineInformation, sourceWebsite, sourceText]
return resultPageArray
def test_getListConcatenatedDescriptionEligibility(self):
# set up
db = SUDBConnect()
keyword = 'East Asian Studies'
testListConcatenatedDescriptionEligibility = GrantForwardItemsGetDatabaseInfo(
keyword=keyword).getListStringConcatenatedDescriptionEligibility()
firstCombo = testListConcatenatedDescriptionEligibility[0]
# test
rows = db.getRowsDB("select * from dbo.GrantForwardItems where Keyword='" + keyword + "'")
description = CleanText.cleanALLtheText(rows[0].Description)
eligibility = CleanText.cleanALLtheText(rows[0].Eligibility)
testCombo = '%s %s' % (description, eligibility)
self.assertEqual(testCombo, firstCombo)
def getListofListofItems():
titles = GetPivotTagsTitleAbstractEligibility.getTitles()
abstracts = GetPivotTagsTitleAbstractEligibility.getAbstracts()
eligibilities = GetPivotTagsTitleAbstractEligibility.getEligibilities()
wholeList = []
for i in range(len(abstracts)):
abstract = CleanText.cleanALLtheText(abstracts[i])
eligibility = CleanText.cleanALLtheText(eligibilities[i])
title = CleanText.cleanALLtheText(titles[i])
listOfItems = [title, abstract, eligibility]
wholeList.append(listOfItems)
return wholeList
def getLeads(self):
self.expandSeeMore()
arrayOfAmountObjects = self.driver.find_elements_by_xpath(
"//div[@class='amount']/span[@data-bind='text: Aequitas.toCurrency(DollarAmount)']")
arrayOfTitleObjects = self.driver.find_elements_by_xpath(
"//h4[@data-bind='text: $parent.resultLayout ? shortTitle : Title']")
arrayOfDeadlineObjects = self.driver.find_elements_by_xpath(
"//h4[@data-bind='text: $parent.resultLayout ? shortTitle : Title']")
titlesList = self.getTitlesList(arrayOfTitleObjects)
amountsList = self.getAmountsList(arrayOfAmountObjects)
deadlinesList = self.getDeadlinesList(arrayOfDeadlineObjects)
for i in range(len(titlesList)):
title = CleanText.cleanALLtheText(titlesList[i])
amount = CleanText.cleanALLtheText(amountsList[i])
deadline = CleanText.cleanALLtheText(deadlinesList[i])
self.driver.get(self.base_url + 'match/scholarshipresult')
self.driver.implicitly_wait(2)
self.expandSeeMore()
arrayOfClickResultObjects = self.driver.find_elements_by_xpath(
"//a[@data-bind='click: function(scholarship, event) { $parent.showScholarshipDetail(scholarship, event) }']")
if arrayOfClickResultObjects[i]:
objectToClick = arrayOfClickResultObjects[i]
objectToClick.click()
self.driver.implicitly_wait(2)
resultPageArray = self.getResultPageInfo()
sponsor = resultPageArray[0]
awardAmount = resultPageArray[1]
recipients = resultPageArray[2]
requirements = resultPageArray[3]
additionalInfo = resultPageArray[4]
contact = resultPageArray[5]
address = resultPageArray[6]
sourceWebsite = resultPageArray[7]
sourceText = resultPageArray[8]
leadArray = [title, amount, deadline, sponsor, awardAmount, recipients, requirements, additionalInfo,
contact, address, sourceWebsite, sourceText]
self.unigoLeadsArray.append(leadArray)
self.driver.quit()
return self.unigoLeadsArray
def __init__(self, iefaLeadArray, fundingClassification,
badScholarshipClassification):
self.iefaLeadArray = iefaLeadArray
self.fundingClassification = fundingClassification
self.badScholarshipClassificaion = badScholarshipClassification
self.db = SUDBConnect()
self.fileSystemDB = SUDBConnect(destination='filesystem')
self.name = self.iefaLeadArray[0]
self.url = self.iefaLeadArray[1]
self.url = CleanText.replaceSingleQuotesWithTwoSingleQuotes(self.url)
self.sponsor = self.iefaLeadArray[2]
self.submissionDeadline = self.iefaLeadArray[3]
self.majors = self.iefaLeadArray[4]
self.amount = self.iefaLeadArray[5]
self.description = self.iefaLeadArray[6]
self.otherCriteria = self.iefaLeadArray[7]
self.numberAwards = self.iefaLeadArray[8]
self.hostInstitution = self.iefaLeadArray[9]
self.includes = self.iefaLeadArray[10]
self.nationalityRequired = self.iefaLeadArray[11]
self.hostCountries = self.iefaLeadArray[12]
self.sourceWebsite = self.iefaLeadArray[13]
self.sourceText = self.iefaLeadArray[14]
self.date = time.strftime('%Y%m%d')
def doSingleArraysForUnevenNumberElements(self):
stringOfMatchedDivParts = self.driver.find_elements_by_xpath(
"//h3[contains(concat(' ', @class, ' '), 'r')]/following-sibling::div/div"
)
for element in stringOfMatchedDivParts:
elementParts = element.text.split('\n', 2)
elementTitle = ''
elementLink = ''
elementDescription = ''
if len(elementParts) == 3:
elementLink = elementParts[0]
elementTitle = elementParts[1]
elementDescription = elementParts[2]
elif len(elementParts) == 2:
elementLink = elementParts[0]
elementDescription = elementParts[1]
if not re.search('^https?://', elementLink):
elementLink = 'http://' + elementLink
singleResultArray = [
elementTitle, elementLink,
CleanText.replaceSingleQuotesWithTwoSingleQuotes(
elementDescription)
]
self.arrayOfGoogleLeads.append(singleResultArray)
def getTitlesList(self):
titleDivs = self.driver.find_elements_by_xpath("//h2[@class='col-xs-12']")
titlesList = [titleDiv.get_attribute('textContent') for titleDiv in titleDivs]
titlesList = [CleanText.cleanALLtheText(title) for title in titlesList]
return titlesList
def goToResultPageAndPullInformation(self, resultPageLink):
self.driver.get(resultPageLink)
self.driver.implicitly_wait(2)
description = ''
sponsor = ''
amount = ''
eligibility = ''
submissionInfo = ''
categories = ''
sourceWebsite = ''
sourceText = ''
deadline = ''
if self.checkIfElementExists("//div[@id = 'field-description']/div[@class = 'content-collapsed']"):
description = self.driver.find_element_by_xpath(
"//div[@id = 'field-description']/div[@class = 'content-collapsed']").get_attribute('textContent')
description = CleanText.cleanALLtheText(description)
if self.checkIfElementExists("//div[@class = 'sponsor-content']/div/a"):
sponsor = self.driver.find_element_by_xpath("//div[@class = 'sponsor-content']/div/a").get_attribute(
'textContent')
sponsor = CleanText.cleanALLtheText(sponsor)
if self.checkIfElementExists("//div[@id = 'field-amount_info']/div[@class = 'content-collapsed']"):
amount = self.driver.find_element_by_xpath(
"//div[@id = 'field-amount_info']/div[@class = 'content-collapsed']").get_attribute('textContent')
amount = CleanText.cleanALLtheText(amount)
if self.checkIfElementExists("//div[@id = 'field-eligibility']/div[@class = 'content-collapsed']"):
eligibility = self.driver.find_element_by_xpath(
"//div[@id = 'field-eligibility']/div[@class = 'content-collapsed']").get_attribute('textContent')
eligibility = CleanText.cleanALLtheText(eligibility)
if self.checkIfElementExists("//div[@id = 'field-submission_info']/div[@class = 'content-collapsed']"):
submissionInfo = self.driver.find_element_by_xpath(
"//div[@id = 'field-submission_info']/div[@class = 'content-collapsed']").get_attribute('textContent')
submissionInfo = CleanText.cleanALLtheText(submissionInfo)
if self.checkIfElementExists("//div[@id = 'field-subjects']/ul"):
categories = self.driver.find_element_by_xpath("//div[@id = 'field-subjects']/ul").get_attribute(
'textContent')
categories = CleanText.cleanALLtheText(categories)
if self.checkIfElementExists("//a[@class = 'source-link btn btn-warning']"):
sourceWebsite = self.driver.find_element_by_xpath(
"//a[@class = 'source-link btn btn-warning']").get_attribute('href')
sourceText = CleanText.cleanALLtheText(RipPage.getPageSource(sourceWebsite))
if self.checkIfElementExists("//div[@class='table-responsive deadline-tables']/table/tbody"):
deadline = self.driver.find_element_by_xpath(
"//div[@class='table-responsive deadline-tables']/table/tbody").get_attribute('textContent')
deadline = CleanText.cleanALLtheText(deadline)
resultPageInfo = [description, sponsor, amount, eligibility, submissionInfo, categories, sourceWebsite,
sourceText, deadline]
return resultPageInfo
def test_ListOfItemsList(self):
# set up
db = SUDBConnect()
keyword = 'Accounting'
testListTitleAbstractEligibilityPivotId = PivotLeadsGetDatabaseInfo(
keyword).getTitleAbstractList()
firstList = testListTitleAbstractEligibilityPivotId[0]
testTitle = firstList[0]
testAbstract = firstList[1]
# test
rows = db.getRowsDB("select * from dbo.PivotLeads where Keyword='" + keyword + "'")
title = CleanText.cleanALLtheText(rows[0].Name)
abstract = CleanText.cleanALLtheText(rows[0].Abstract)
self.assertEqual(title, testTitle)
self.assertEqual(abstract, testAbstract)
def getAmountsList(self):
amountsList = []
amountsDivs = self.driver.find_elements_by_xpath("//div[@class='amount']")
for amountDiv in amountsDivs:
amountsList.append(CleanText.cleanALLtheText(re.sub('Amount', '', amountDiv.get_attribute('textContent'))))
return amountsList
def getTitlesList(self):
titlesList = []
titlesDivs = self.driver.find_elements_by_xpath("//div[@class='main-details clearfix']/h2/a")
for title in titlesDivs:
titlesList.append(CleanText.cleanALLtheText(title.get_attribute('textContent')))
return titlesList
def test_getListConcatenatedDescriptionEligibility(self):
# set up
db = SUDBConnect()
keyword = 'East Asian Studies'
testListConcatenatedDescriptionEligibility = GrantForwardItemsGetDatabaseInfo(
keyword=keyword).getListStringConcatenatedDescriptionEligibility()
firstCombo = testListConcatenatedDescriptionEligibility[0]
# test
rows = db.getRowsDB(
"select * from dbo.GrantForwardItems where Keyword='" + keyword +
"'")
description = CleanText.cleanALLtheText(rows[0].Description)
eligibility = CleanText.cleanALLtheText(rows[0].Eligibility)
testCombo = '%s %s' % (description, eligibility)
self.assertEqual(testCombo, firstCombo)
def test_eligibilitiesList(self):
dbinfo = GetDatabaseInfoScholarshipsWithClassStatuses('Senior')
self.assertIsNotNone(dbinfo)
eligibilitesList = dbinfo.getEligibilitiesList()
self.assertIsNotNone(eligibilitesList)
testEligibility = eligibilitesList[0]
testCleanText = CleanText.cleanALLtheText(testEligibility)
self.assertIsNotNone(testCleanText)
def test_ListOfItemsList(self):
# set up
db = SUDBConnect()
keyword = 'Accounting'
testListTitleDescriptionEligibilityPivotId = GrantForwardItemsGetDatabaseInfo(
keyword).getTitleDescriptionList()
firstList = testListTitleDescriptionEligibilityPivotId[0]
testTitle = firstList[0]
testDescription = firstList[1]
# test
rows = db.getRowsDB("select * from dbo.GrantForwardItems where Keyword='" + keyword + "'")
title = CleanText.cleanALLtheText(rows[0].Name)
description = CleanText.cleanALLtheText(rows[0].Description)
self.assertEqual(title, testTitle)
self.assertEqual(description, testDescription)
def test_scholarshipsDescriptionsList(self):
dbinfo = GetDatabaseInfoScholarshipsWithClassStatuses('Junior')
self.assertIsNotNone(dbinfo)
descriptionsList = dbinfo.getScholarshipDescriptionsList()
self.assertIsNotNone(descriptionsList)
testDescription = descriptionsList[0]
testCleanText = CleanText.cleanALLtheText(testDescription)
self.assertIsNotNone(testCleanText)
def getEligibilitiesList(self):
eligibilityDivs = self.driver.find_elements_by_xpath("//div[@class='col-md-10 col-md-offset-1']/div[1]/p[2]")
eligbilitiesList = [eligibilityDiv.get_attribute('textContent') for eligibilityDiv in eligibilityDivs]
eligbilitiesList = [re.sub('Who can apply\? ', '', eligibility) for eligibility in eligbilitiesList]
eligbilitiesList = [CleanText.cleanALLtheText(eligibility) for eligibility in eligbilitiesList]
return eligbilitiesList
def getSourceWebsitesAndSourceTexts(self):
sourceWebsiteDivs = self.driver.find_elements_by_xpath("//div[@class='col-xs-8 col-xs-offset-2']/a")
sourceWebsitesList = [sourceWebsiteDiv.get_attribute('href') for sourceWebsiteDiv in sourceWebsiteDivs]
sourceTextsList = [RipPage.getPageSource(sourceWebsite) for sourceWebsite in sourceWebsitesList]
sourceTextsList = [CleanText.cleanALLtheText(sourceText) for sourceText in sourceTextsList]
return sourceWebsitesList, sourceTextsList
def getAwardsList(self):
awardDivs = self.driver.find_elements_by_xpath("//div[@class='col-md-10 col-md-offset-1']/div[2]/p[1]")
awardsList = [awardDiv.get_attribute('textContent') for awardDiv in awardDivs]
awardsList = [re.sub('Awards: ', '', award) for award in awardsList]
awardsList = [CleanText.cleanALLtheText(award) for award in awardsList]
return awardsList
def getDescriptionsList(self):
descriptionDivs = self.driver.find_elements_by_xpath("//div[@class='col-md-10 col-md-offset-1']/div[1]/p[1]")
descriptionsList = [descriptionDiv.get_attribute('textContent') for descriptionDiv in descriptionDivs]
descriptionsList = [re.sub('Details: ', '', description) for description in descriptionsList]
descriptionsList = [CleanText.cleanALLtheText(description) for description in descriptionsList]
return descriptionsList
def test_ListOfItemsList(self):
# set up
db = SUDBConnect()
keyword = 'Accounting'
testListTitleAbstractEligibilityPivotId = PivotLeadsGetDatabaseInfo(
keyword).getTitleAbstractList()
firstList = testListTitleAbstractEligibilityPivotId[0]
testTitle = firstList[0]
testAbstract = firstList[1]
# test
rows = db.getRowsDB("select * from dbo.PivotLeads where Keyword='" +
keyword + "'")
title = CleanText.cleanALLtheText(rows[0].Name)
abstract = CleanText.cleanALLtheText(rows[0].Abstract)
self.assertEqual(title, testTitle)
self.assertEqual(abstract, testAbstract)
def getNumAwardsList(self):
numAwardsList = []
numAwardsDivs = self.driver.find_elements_by_xpath("//div[@class='award-count']")
for awardDiv in numAwardsDivs:
numAwardsList.append(
CleanText.cleanALLtheText(re.sub('# Awards', '', awardDiv.get_attribute('textContent'))))
return numAwardsList
def getAmountsList(self):
amountsList = []
amountsDivs = self.driver.find_elements_by_xpath("//div[@class='award']/p[not (@class='label')]")
for amount in amountsDivs:
amountsList.append(amount.get_attribute('textContent'))
amountsList = [CleanText.cleanALLtheText(amount) for amount in amountsList]
return amountsList
def getDeadlinesList(self):
deadlinesList = []
deadlinesDiv = self.driver.find_elements_by_xpath("//div[@class='deadline']/p[not (@class='label')]")
for deadline in deadlinesDiv:
deadlinesList.append(deadline.get_attribute('textContent'))
deadlinesList = [CleanText.cleanALLtheText(deadline) for deadline in deadlinesList]
return deadlinesList
def getTitlesList(self):
titlesList = []
titlesDivs = self.driver.find_elements_by_xpath("//h3/a")
for title in titlesDivs:
titlesList.append(title.get_attribute('textContent'))
titlesList = [CleanText.cleanALLtheText(title) for title in titlesList]
return titlesList
def getSponsorsList(self):
sponsorsList = []
sponsorsDivs = self.driver.find_elements_by_xpath("//div[@class='provided_by']/p[not (@class='label')]")
for sponsor in sponsorsDivs:
sponsorsList.append(sponsor.get_attribute('textContent'))
sponsorsList = [CleanText.cleanALLtheText(sponsor) for sponsor in sponsorsList]
return sponsorsList
def doSingleArraysForSameNumberElements(self):
for i in range(len(self.arrayOfTitles)):
elementTitle = self.arrayOfTitles[i].text
elementLink = self.arrayOfLinks[i]
elementDescription = self.arrayOfDescriptions[i].text
singleResultArray = [elementTitle, elementLink,
CleanText.replaceSingleQuotesWithTwoSingleQuotes(elementDescription)]
self.arrayOfGoogleLeads.append(singleResultArray)
def test_GetPivotTagsTitleAbstractEligibilityListItems(self):
# set up
db = SUDBConnect()
testListItems = GetPivotTagsTitleAbstractEligibility.getListofListofItems()
firstList = testListItems[0]
testtitle = firstList[0]
testabstract = firstList[1]
testeligibility = firstList[2]
# test
rows = db.getRowsDB("select * from dbo.PivotTags")
title = CleanText.cleanALLtheText(rows[0].Name)
abstract = CleanText.cleanALLtheText(rows[0].Abstract)
eligibility = CleanText.cleanALLtheText(rows[0].Eligibility)
self.assertEqual(title, testtitle)
self.assertEqual(abstract, testabstract)
self.assertEqual(eligibility, testeligibility)
def getInfoFromScholarshipPage(self, url):
self.driver.get(url)
self.driver.implicitly_wait(2)
description = ''
eligibility = ''
amountInfo = ''
deadlineInfo = ''
sourceWebsite = ''
sourceText = ''
if self.checkIfElementExists("//div[@class='entry-content']/p[1]"):
description = self.driver.find_element_by_xpath("//div[@class='entry-content']/p[1]").get_attribute(
'textContent')
description = CleanText.cleanALLtheText(description)
if self.checkIfElementExists(
"//div[@class='entry-content']/p/strong[text() = 'Who is eligible to apply?']/../following-sibling::ul[1]"):
eligibility = self.driver.find_element_by_xpath(
"//div[@class='entry-content']/p/strong[text() = 'Who is eligible to apply?']/../following-sibling::ul[1]").get_attribute(
'textContent')
eligibility = CleanText.cleanALLtheText(eligibility)
if self.checkIfElementExists(
"//div[@class='entry-content']/p/strong[text() = 'How much is each scholarship worth?']/../following-sibling::p[1]"):
amountInfo = self.driver.find_element_by_xpath(
"//div[@class='entry-content']/p/strong[text() = 'How much is each scholarship worth?']/../following-sibling::p[1]").get_attribute(
'textContent')
amountInfo = CleanText.cleanALLtheText(amountInfo)
if self.checkIfElementExists(
"//div[@class='entry-content']/p/strong[text() = 'When is the deadline to apply?']/../following-sibling::ul[1]"):
deadlineInfo = self.driver.find_element_by_xpath(
"//div[@class='entry-content']/p/strong[text() = 'When is the deadline to apply?']/../following-sibling::ul[1]").get_attribute(
'textContent')
deadlineInfo = CleanText.cleanALLtheText(deadlineInfo)
if self.checkIfElementExists("//span[@class='apply']/a"):
sourceWebsite = self.driver.find_element_by_xpath("//span[@class='apply']/a").get_attribute('href')
sourceText = RipPage.getPageSource(sourceWebsite)
sourceText = CleanText.cleanALLtheText(sourceText)
scholarshipPageInfoArray = [description, eligibility, amountInfo, deadlineInfo, sourceWebsite, sourceText]
return scholarshipPageInfoArray
def getScholarshipDescriptionsList(self):
scholarshipDescriptionsList = []
for row in self.rows:
scholarshipDescriptionsList.append(row.ScholarshipDescription)
scholarshipDescriptionsList = [CleanText.cleanALLtheText(description) for description in
scholarshipDescriptionsList if type(description) == str]
return scholarshipDescriptionsList
def getLeads(self):
arrayOfTitleLinkDivs = self.driver.find_elements_by_xpath(
"//td[@class='f']/../preceding-sibling::tr[1]/td[@class='a']/a")
arrayOfDateDescriptionDivs = self.driver.find_elements_by_xpath("//tr/td[@class='f']/../td")
titlesList = self.getTitlesList(arrayOfTitleLinkDivs)
linksList = self.getLinksList(arrayOfTitleLinkDivs)
dueDatesList = self.getDueDates(arrayOfDateDescriptionDivs)
descriptionsList = self.getDescriptionsList(arrayOfDateDescriptionDivs)
for i in range(len(titlesList)):
title = CleanText.cleanALLtheText(titlesList[i])
link = linksList[i]
dueDate = dueDatesList[i]
description = CleanText.cleanALLtheText(descriptionsList[i])
sourceText = CleanText.cleanALLtheText(RipPage.getPageSource(link))
scholarshipArray = [title, description, dueDate, link, sourceText]
self.fatomeiLeadsArray.append(scholarshipArray)
def getAmountsList(self):
amountsList = []
amountsDivs = self.driver.find_elements_by_xpath("//div[@class='scholarship__amount']")
for amount in amountsDivs:
amountsList.append(amount.get_attribute('textContent'))
amountsList = [CleanText.cleanALLtheText(amount) for amount in amountsList]
return amountsList
def getTitlesList(self):
titlesList = []
titlesDivs = self.driver.find_elements_by_xpath(
"//div[@class='main-details clearfix']/h2/a")
for title in titlesDivs:
titlesList.append(
CleanText.cleanALLtheText(title.get_attribute('textContent')))
return titlesList
def getTitlesList(self):
titleDivs = self.driver.find_elements_by_xpath(
"//h2[@class='col-xs-12']")
titlesList = [
titleDiv.get_attribute('textContent') for titleDiv in titleDivs
]
titlesList = [CleanText.cleanALLtheText(title) for title in titlesList]
return titlesList
def test_ListOfItemsList(self):
# set up
db = SUDBConnect()
keyword = 'Accounting'
testListTitleDescriptionEligibilityPivotId = GrantForwardItemsGetDatabaseInfo(
keyword).getTitleDescriptionList()
firstList = testListTitleDescriptionEligibilityPivotId[0]
testTitle = firstList[0]
testDescription = firstList[1]
# test
rows = db.getRowsDB(
"select * from dbo.GrantForwardItems where Keyword='" + keyword +
"'")
title = CleanText.cleanALLtheText(rows[0].Name)
description = CleanText.cleanALLtheText(rows[0].Description)
self.assertEqual(title, testTitle)
self.assertEqual(description, testDescription)
def getDeadlinesList(self):
deadlinesList = []
deadlinesDivs = self.driver.find_elements_by_xpath("//span[@class='due']")
for deadline in deadlinesDivs:
deadlinesList.append(deadline.get_attribute('textContent'))
deadlinesList = [re.sub('Due:', '', deadline) for deadline in deadlinesList]
deadlinesList = [CleanText.cleanALLtheText(deadline) for deadline in deadlinesList]
return deadlinesList
def getDeadlinesList(self):
deadlinesList = []
deadlinesDivs = self.driver.find_elements_by_xpath("//div[@class='scholarship__deadline']")
for deadline in deadlinesDivs:
deadlinesList.append(deadline.get_attribute('textContent'))
deadlinesList = [CleanText.cleanALLtheText(deadline) for deadline in deadlinesList]
return deadlinesList
def getEligibilitiesList(self):
eligibilitiesList = []
for row in self.rows:
eligibilitiesList.append(row.Eligibility)
eligibilitiesList = [CleanText.cleanALLtheText(eligibility) for eligibility in eligibilitiesList if
type(eligibility) == str]
return eligibilitiesList
def test_GetPivotTagsTitleAbstractEligibilityListItems(self):
# set up
db = SUDBConnect()
testListItems = GetPivotTagsTitleAbstractEligibility.getListofListofItems(
)
firstList = testListItems[0]
testtitle = firstList[0]
testabstract = firstList[1]
testeligibility = firstList[2]
# test
rows = db.getRowsDB("select * from dbo.PivotTags")
title = CleanText.cleanALLtheText(rows[0].Name)
abstract = CleanText.cleanALLtheText(rows[0].Abstract)
eligibility = CleanText.cleanALLtheText(rows[0].Eligibility)
self.assertEqual(title, testtitle)
self.assertEqual(abstract, testabstract)
self.assertEqual(eligibility, testeligibility)
def getAmountsList(self):
amountsList = []
amountsDivs = self.driver.find_elements_by_xpath("//span[@class='amount']")
for amount in amountsDivs:
amountsList.append(amount.get_attribute('textContent'))
amountsList = [re.sub('Amount:', '', amount) for amount in amountsList]
amountsList = [CleanText.cleanALLtheText(amount) for amount in amountsList]
return amountsList
def getLeads(self):
arrayOfTitleLinkDivs = self.driver.find_elements_by_xpath(
"//td[@class='f']/../preceding-sibling::tr[1]/td[@class='a']/a")
arrayOfDateDescriptionDivs = self.driver.find_elements_by_xpath(
"//tr/td[@class='f']/../td")
titlesList = self.getTitlesList(arrayOfTitleLinkDivs)
linksList = self.getLinksList(arrayOfTitleLinkDivs)
dueDatesList = self.getDueDates(arrayOfDateDescriptionDivs)
descriptionsList = self.getDescriptionsList(arrayOfDateDescriptionDivs)
for i in range(len(titlesList)):
title = CleanText.cleanALLtheText(titlesList[i])
link = linksList[i]
dueDate = dueDatesList[i]
description = CleanText.cleanALLtheText(descriptionsList[i])
sourceText = CleanText.cleanALLtheText(RipPage.getPageSource(link))
scholarshipArray = [title, description, dueDate, link, sourceText]
self.fatomeiLeadsArray.append(scholarshipArray)
def getAmountsList(self):
amountsList = []
amountsDivs = self.driver.find_elements_by_xpath(
"//div[@class='amount']")
for amountDiv in amountsDivs:
amountsList.append(
CleanText.cleanALLtheText(
re.sub('Amount', '',
amountDiv.get_attribute('textContent'))))
return amountsList
def getAwardsList(self):
awardDivs = self.driver.find_elements_by_xpath(
"//div[@class='col-md-10 col-md-offset-1']/div[2]/p[1]")
awardsList = [
awardDiv.get_attribute('textContent') for awardDiv in awardDivs
]
awardsList = [re.sub('Awards: ', '', award) for award in awardsList]
awardsList = [CleanText.cleanALLtheText(award) for award in awardsList]
return awardsList
def doSingleArraysForSameNumberElements(self):
for i in range(len(self.arrayOfTitles)):
elementTitle = self.arrayOfTitles[i].text
elementLink = self.arrayOfLinks[i]
elementDescription = self.arrayOfDescriptions[i].text
singleResultArray = [
elementTitle, elementLink,
CleanText.replaceSingleQuotesWithTwoSingleQuotes(
elementDescription)
]
self.arrayOfGoogleLeads.append(singleResultArray)
def getTitlesListEditors(self):
titlesList = []
titlesDivs = self.driver.find_elements_by_xpath(
"//section[@class='scholarships']/div[@class='tab h-results']/div[@class='h-result js-has-toggle active']/div[@class='h-result-header js-toggle active']/h3"
)
for title in titlesDivs:
titlesList.append(title.get_attribute('textContent'))
titlesList = [CleanText.cleanALLtheText(title) for title in titlesList]
return titlesList
def getTitlesListScholarships(self):
titlesList = []
titlesDivs = self.driver.find_elements_by_xpath(
"//section[@class='scholarships']/div[@class='tab']/section/div/div/h3"
)
for title in titlesDivs:
titlesList.append(title.get_attribute('textContent'))
titlesList = [CleanText.cleanALLtheText(title) for title in titlesList]
return titlesList