def run(url):
soup = get_javascript_soup_delayed(url, 'current-openings-item')
jobs_list = soup.find_all('div', {'class':'current-openings-details'})
for job_entry in jobs_list:
globals.job_title = job_entry.find('span', {'class':'current-opening-title'}).text.strip()
if job_entry.find('span', {'class':'current-opening-location-item'}):
globals.job_location = job_entry.find('span', {'class':'current-opening-location-item'}).text.strip()
# Calculate post date relative to current date and store it
posted_ago = job_entry.find('span', {'class':'current-opening-post-date'}).text.split(' ')
if posted_ago[0] == 'a':
globals.job_post_date = date_ago(1, posted_ago[1])
elif posted_ago[0].lower() == 'yesterday':
globals.job_post_date = date_ago(1, 'day')
elif posted_ago[0] == '30+':
# over 30 days ago
globals.job_post_date = date_ago(31, posted_ago[1])
else:
globals.job_post_date = date_ago(int(posted_ago[0]), posted_ago[1])
if job_entry.find('span', {'class':'current-opening-worker-catergory'}):
globals.full_or_part = job_entry.find('span', {'class':'current-opening-worker-catergory'}).text.strip()
globals.info_link = 'https://workforcenow.adp.com/mascsr/default/mdf/recruitment/recruitment.html?cid=b4842dc2-cd32-4f0f-88d3-b259fbc96f09&ccId=19000101_000001&type=MP&lang'
globals.job_summary = globals.info_link
update_db(organization)
reset_vars()
def run(url):
globals.job_post_date = ''
next_page_url = url
soup = get_javascript_soup_delayed(next_page_url, 'job-table-title')
while soup:
job_table = soup.find('tbody')
for job_row in job_table.find_all('tr'):
globals.job_title = job_row.find('td', {
'class': 'job-table-title'
}).a.text.strip()
globals.info_link = 'https://www.governmentjobs.com' + job_row.find(
'td', {
'class': 'job-table-title'
}).a['href']
globals.salary = job_row.find('td', {
'class': 'job-table-salary'
}).text
globals.full_or_part = job_row.find('td', {
'class': 'job-table-type'
}).text
# Get soup for job listing to get more info
job_soup = get_soup(globals.info_link)
info_container = job_soup.find('div',
{'class': 'summary container'})
globals.job_location = clean_location(
info_container.find('div', {
'id': 'location-label-id'
}).parent.find_all('div')[2].text)
globals.job_zip_code = city_to_zip(globals.job_location)
globals.job_summary = job_soup.find('div', {
'id': 'details-info'
}).find('p').text
update_db(organization)
reset_vars()
if not 'disabled' in soup.find('li', {
'class': 'PagedList-skipToNext'
}).get("class"):
next_page_url = 'https://www.governmentjobs.com/careers/lahsa?' + soup.find(
'li', {
'class': 'PagedList-skipToNext'
}).a['href'].split('?')[1]
soup = get_javascript_soup_delayed(next_page_url,
'job-table-title')
else:
soup = False
def run(url):
globals.job_post_date = ''
soup = get_soup(url)
jobs_table = soup.find('table',{'id':'job-result-table'})
for job_row in jobs_table.find_all('tr',{'class':'job-result'}):
job_title_cell = job_row.find('td',{'class':'job-result-title-cell'})
globals.job_title = job_title_cell.a.text.strip()
globals.info_link = 'https://pennylanecenters.jobs.net' + job_title_cell.a['href']
globals.job_summary = globals.info_link
globals.job_location = clean_location(job_row.find('div',{'class':'job-location-line'}).text)
globals.job_zip_code = city_to_zip(globals.job_location)
# Get Job Soup
job_soup = get_soup(globals.info_link)
globals.full_or_part = job_soup.find('li',{'class':'job-employee-type'}).find('div',{'class':'secondary-text-color'}).text
globals.job_post_date = string_to_date(job_soup.find('li',{'class':'job-date-posted'}).find('div',{'class':'secondary-text-color'}).text)
update_db(organization)
reset_vars()
def run(url):
soup = get_soup(url)
article = soup.find('article')
jobs_list = article.find_all('p')
for job_entry in jobs_list:
if 'Posted ' in job_entry.text:
job_element = job_entry.find('a')
globals.job_title = job_element.text
globals.info_link = job_element['href']
globals.job_summary = globals.info_link
date = job_entry.text.split('Posted ')[1].split('/')
month = int(date[0])
day = int(date[1])
year = int(date[2])
globals.job_post_date = datetime(year, month, day)
update_db(organization)
reset_vars()
from os.path import basename
# CONSTANTS
pdf_message = 'See PDF document.'
# SQL CONNECTION
globals.db = sqlite3.connect("jobs_for_hope.db")
globals.c = globals.db.cursor()
# Clear and recreate SQL schema
globals.drop_table_jobs()
globals.create_table_jobs()
globals.reset_vars()
# set the active scraper if one is passed in
if len(sys.argv) - 1 == 1:
globals.active_scrapers = [basename(sys.argv[1])]
# load and run scrapers
for i in scraperloader.getScrapers():
try:
# filter to run target scrapers
if len(globals.active_scrapers
) > 0 and not i['name'] in globals.active_scrapers:
continue
scraper = scraperloader.loadScraper(i)
organization = scraper.organization
print organization