Example #1
0
def run(url):
    soup = get_javascript_soup(url)

    job_listings = soup.find_all('div', {'class': 'job-listing-job-item'})
    job_class = Job(organization, "")
    job_class.organization_id = organization_id
    insert_count = 0
    for job_listing in job_listings:
        job_class.title = job_listing.find('span', {
            'class': 'job-item-title'
        }).a.text.strip()
        job_class.info_link = 'https://recruiting.paylocity.com' + \
            job_listing.find('span', {'class': 'job-item-title'}).a['href']
        details = get_soup(job_class.info_link)
        location = details.find('div', {'class': 'preview-location'})
        if location.a:
            job_class.location = location.a.text
            zipcode = location.a['href'].split('+')[-1]
            try:
                job_class.zip_code = int(zipcode)
            except ValueError:
                # generate a zip code if one is not available
                job_class.zip_code = city_to_zip(job_class.location)
        else:
            job_class.location = ''
            job_class.zip_code = ''
        job_class.post_date = string_to_date(
            job_listing.find('div', {
                'class': 'job-title-column'
            }).find_all('span')[1].text.split(' - ')[0])
        insert_count += job_insert(job_class)
    return insert_count
def run(url):
    soup = get_soup(url)

    jobs_list = soup.select('div[class*="JobGrid-"]')[0]
    job_class= Job(organization, "")
    job_class.organization_id= organization_id
    insert_count= 0
    for job_entry in jobs_list.find_all('a'):
        job_class.info_link = 'https://path.catsone.com' + job_entry['href']
        job_row = job_entry.find('div', {'class': 'row'})
        job_divs = job_row.find_all('div')
        job_class.title = job_divs[0].text.strip()
        job_class.location = clean_location(job_divs[2].text.strip())
        job_class.zip_code = city_to_zip(job_class.location)
        insert_count+= job_insert(job_class)
        # Possible to get more info by scraping each job link, but the listings are extremely poorly written/standardized; scraper below works for most of the listings, but a few poorly written listings break the scraper
        # job_soup = get_soup(info_link)
        # job_description = job_soup.find('div',{'class':'Job__StyledDescription-s1h17u0t-0'})
        # if '\n' in job_description.find_all('strong')[0].text:
        #     full_or_part = job_description.find_all('strong')[0].text.split('\n')[1].strip()
        #     salary = job_description.find_all('strong')[0].text.split('\n')[2].strip().split(': ')[1]
        # else:
        #     full_or_part = job_description.find_all('strong')[1].text.strip()
        #     salary = job_description.find_all('strong')[2].text.split('\n')[0].split(':')[1].strip()
    return insert_count
def run(url):
    soup = get_javascript_soup_delayed_and_click(url, 'hrmSearchButton')

    job_listings = soup.find_all('tr', {'class': 'ReqRowClick'})
    job_class = Job(organization, "")
    job_class.organization_id = organization_id
    insert_count = 0
    for job_row in job_listings:
        job_class.title = job_row.find('td', {
            'class': 'posTitle'
        }).text.strip()
        job_class.info_link = 'http://mhala.hrmdirect.com/employment/' + \
            job_row.find('td', {'class': 'posTitle'}).a['href']
        job_class.location = job_row.find('td', {'class': 'cities'}).text
        job_class.zip_code = globals.city_to_zip(job_class.location)
        job_soup = get_soup(job_class.info_link)
        summary = job_soup.find(string=["Summary:", "Summary: "])
        if summary:
            summary_parent = summary.parent
            summary_parent.clear()
            job_class.summary = summary_parent.find_parent("p").text.strip()
        else:
            job_class.summary = ''
        insert_count += job_insert(job_class)
    return insert_count
Example #4
0
def run(url):
    soup = get_javascript_soup(url)
    jobs_list = soup.find(
        'table', {'class': 'srJobList'}).tbody.find_all('tr')[1:]
    job_class= Job(organization, "")
    job_class.organization_id= organization_id
    insert_count= 0
    for job_entry in jobs_list:
        job_class.title = job_entry.find(
            'td', {'class': 'srJobListJobTitle'}).text.strip()
        onClickLink = job_entry['onclick']
        job_class.info_link = onClickLink[13:len(onClickLink) - 3]
        job_class.full_or_part = job_entry.find(
            'td', {'class': 'srJobListTypeOfEmployment'}).text
        job_class.location = job_entry.find(
            'td', {'class': 'srJobListLocation'}).text
        location_parts = job_class.location.split(',')
        if len(location_parts) > 1 and len(
                location_parts[-1]) and location_parts[-1].strip().lower() != 'ca':
            # skip job if state is not CA
            print('Skip location: %s' % job_class.location)
            continue
        job_class.zip_code = city_to_zip(location_parts[0])
        insert_count+= job_insert(job_class)
    return insert_count
Example #5
0
def run(url):
    soup = get_soup(url)
    jobs_table = soup.find('table', {'id': 'job-result-table'})
    job_class = Job(organization, "")
    job_class.post_date = ""
    job_class.organization_id = organization_id
    insert_count = 0
    for job_row in jobs_table.find_all('tr', {'class': 'job-result'}):
        job_title_cell = job_row.find('td', {'class': 'job-result-title-cell'})
        job_class.title = job_title_cell.a.text.strip()
        job_class.info_link = 'https://pennylanecenters.jobs.net' + \
            job_title_cell.a['href']
        job_class.location = clean_location(
            job_row.find('div', {
                'class': 'job-location-line'
            }).text)
        job_class.zip_code = city_to_zip(job_class.location)
        # Get Job Soup
        job_soup = get_soup(job_class.info_link)
        job_class.full_or_part = job_soup.find('li', {
            'class': 'job-employee-type'
        }).find('div', {
            'class': 'secondary-text-color'
        }).text
        job_class.post_date = string_to_date(
            job_soup.find('li', {
                'class': 'job-date-posted'
            }).find('div', {
                'class': 'secondary-text-color'
            }).text)
        insert_count += job_insert(job_class)
    return insert_count
Example #6
0
def run(url):
    soup = get_soup(url)

    jobs_list = soup.find_all('div', {'class': 'list-data'})
    job_class = Job(organization, "")
    job_class.organization_id = organization_id
    insert_count = 0
    for job_entry in jobs_list:
        job_info = job_entry.find('div', {'class': 'job-info'})
        job_class.title = job_info.find('span', {
            'class': 'job-title'
        }).text.strip()
        job_class.info_link = job_info.h4.a['href']
        job_class.full_or_part = job_entry.find('div', {
            'class': 'job-type'
        }).text.strip()
        job_class.location = clean_location(
            job_entry.find('div', {
                'class': 'job-location'
            }).text.strip())
        job_class.zip_code = city_to_zip(job_class.location)
        relative_date = job_entry.find('div', {
            'class': 'job-date'
        }).text.strip().split(' ')
        job_class.post_date = date_ago(int(relative_date[1]), relative_date[2])
        job_class.summary = job_entry.find('div', {
            'class': 'job-description'
        }).p.text.strip()
        insert_count += job_insert(job_class)
    return insert_count
def run(url):
    soup = get_javascript_soup(url)
    jobs_list = soup.find('table',{'class':'srJobList'}).tbody.find_all('tr')[1:]

    for job_entry in jobs_list:
        globals.job_title = job_entry.find('td',{'class':'srJobListJobTitle'}).text.strip()
        onClickLink = job_entry['onclick']
        globals.info_link = onClickLink[13:len(onClickLink)-3]
        globals.full_or_part = job_entry.find('td',{'class':'srJobListTypeOfEmployment'}).text
        globals.job_location = clean_location(job_entry.find('td',{'class':'srJobListLocation'}).text)
        globals.job_zip_code = city_to_zip(globals.job_location)
        update_db(organization)
def run(url):
    soup = get_soup(url)

    jobs_list = soup.select('div[class*="JobGrid-"]')[0]

    for job_entry in jobs_list.find_all('a'):
        globals.info_link = 'https://path.catsone.com' + job_entry['href']
        job_row = job_entry.find('div', {'class':'row'})
        job_divs = job_row.find_all('div')
        globals.job_title = job_divs[0].text.strip()
        globals.job_location = clean_location(job_divs[2].text.strip())
        globals.job_zip_code = city_to_zip(globals.job_location)
        update_db(organization)
Example #9
0
def run(url):
    soup = get_javascript_soup(url)

    current_openings = soup.findAll(attrs={"data-tn-element": "jobLink[]"})
    job_class = Job(organization, "")
    job_class.organization_id = organization_id
    insert_count = 0
    for current_opening in current_openings:

        detail_page_link = current_opening.find('a')['href']
        detail_page_soup = get_soup(detail_page_link)
        detail_page_desc = detail_page_soup.find(
            'div', {"data-tn-component": "jobDescription"})

        job_class.title = detail_page_desc.find('h1').text.strip()

        job_summary_parts = detail_page_desc.findAll(['p', 'li'])
        job_class.summary = ' '.join(
            map(lambda a: a.getText(), job_summary_parts[1:-1])).strip()

        job_class.location = detail_page_desc.find(
            'dt', string="Location").findNext().get_text()

        location_parts = job_class.location.split(',')
        if len(location_parts) > 1 and len(
                location_parts[-1]
        ) and location_parts[-1].strip().lower() != 'ca':
            # skip job if state is not CA
            print('Skip location: %s' % job_class.location)
            continue
        job_class.zip_code = city_to_zip(location_parts[0])

        posted_ago = job_summary_parts[-1].get_text().split(' ')
        length = posted_ago[1]
        if (length[-1:] == '+'):
            length = length[:1]
        length = int(length)
        unit = posted_ago[2]
        job_class.post_date = date_ago(length, unit)

        job_class.full_or_part = detail_page_desc.find(
            'dt', string="Job Type").findNext().get_text()

        salary_search = detail_page_desc.find('dt', string="Salary")
        if (salary_search is not None):
            job_class.salary = salary_search.findNext().get_text()

        job_class.info_link = detail_page_link

        insert_count += job_insert(job_class)
    return insert_count
def run(url):
    soup = get_javascript_soup(url)
    job_listings = soup.find_all('div', {'class': 'job-listing-job-item'})
    job_class = Job(organization, "")
    job_class.organization_id = organization_id
    insert_count = 0
    for job_listing in job_listings:
        job_description = job_listing.find_all('span')
        # Get job title and link
        job_class.title = job_description[0].a.text
        job_class.info_link = 'https://recruiting.paylocity.com' + \
            job_description[0].a['href']
        # Get date as string
        date = job_description[1].text
        # Clean up date string by removing trailing -'s, then split and convert
        # to datetime object
        if date[len(date) - 2] == '-':
            date = date[0:len(date) - 3]
        date = date.strip().split('/')
        month = int(date[0])
        day = int(date[1])
        year = int(date[2])
        job_class.post_date = datetime(year, month, day)
        # Get Location
        job_class.location = job_listing.find('div', {
            'class': 'location-column'
        }).span.text
        # Get soup of job listing to scrape more info
        listing_soup = get_soup(job_class.info_link)
        listing_body = listing_soup.find('body').find_all('p')
        # Retrieve Full/Part-time and Salary info if available
        if 'Location' in listing_body[0].text:
            location_string = listing_body[0].text.split(':')[1].lstrip()
            zip_code_result = re.search(r'(\d{5})', location_string)
            if zip_code_result is not None:
                job_class.zip_code = zip_code_result.group(1)
            # can't get city since there's no standard. It could be
            # "Hollywood", "Koreatown, Los angeles, California", or even
            # "Multiple Locations"
        if len(job_class.zip_code) == 0:
            job_class.zip_code = globals.city_to_zip(job_class.location)
        if 'Status' in listing_body[1].text:
            job_class.full_or_part = listing_body[1].text[8:]
        if 'Salary' in listing_body[2].text:
            job_class.salary = listing_body[2].text[14:]
        insert_count += job_insert(job_class)
    return insert_count
Example #11
0
def run(url):
    globals.job_post_date = ''
    next_page_url = url
    soup = get_javascript_soup_delayed(next_page_url, 'job-table-title')

    while soup:
        job_table = soup.find('tbody')
        for job_row in job_table.find_all('tr'):
            globals.job_title = job_row.find('td', {
                'class': 'job-table-title'
            }).a.text.strip()
            globals.info_link = 'https://www.governmentjobs.com' + job_row.find(
                'td', {
                    'class': 'job-table-title'
                }).a['href']
            globals.salary = job_row.find('td', {
                'class': 'job-table-salary'
            }).text
            globals.full_or_part = job_row.find('td', {
                'class': 'job-table-type'
            }).text
            # Get soup for job listing to get more info
            job_soup = get_soup(globals.info_link)
            info_container = job_soup.find('div',
                                           {'class': 'summary container'})
            globals.job_location = clean_location(
                info_container.find('div', {
                    'id': 'location-label-id'
                }).parent.find_all('div')[2].text)
            globals.job_zip_code = city_to_zip(globals.job_location)
            globals.job_summary = job_soup.find('div', {
                'id': 'details-info'
            }).find('p').text
            update_db(organization)
            reset_vars()
        if not 'disabled' in soup.find('li', {
                'class': 'PagedList-skipToNext'
        }).get("class"):
            next_page_url = 'https://www.governmentjobs.com/careers/lahsa?' + soup.find(
                'li', {
                    'class': 'PagedList-skipToNext'
                }).a['href'].split('?')[1]
            soup = get_javascript_soup_delayed(next_page_url,
                                               'job-table-title')
        else:
            soup = False
def run(url):
    globals.job_post_date = ''
    soup = get_soup(url)
    jobs_table = soup.find('table',{'id':'job-result-table'})

    for job_row in jobs_table.find_all('tr',{'class':'job-result'}):
        job_title_cell = job_row.find('td',{'class':'job-result-title-cell'})
        globals.job_title = job_title_cell.a.text.strip()
        globals.info_link = 'https://pennylanecenters.jobs.net' + job_title_cell.a['href']
        globals.job_summary = globals.info_link
        globals.job_location = clean_location(job_row.find('div',{'class':'job-location-line'}).text)
        globals.job_zip_code = city_to_zip(globals.job_location)
        # Get Job Soup
        job_soup = get_soup(globals.info_link)
        globals.full_or_part = job_soup.find('li',{'class':'job-employee-type'}).find('div',{'class':'secondary-text-color'}).text
        globals.job_post_date = string_to_date(job_soup.find('li',{'class':'job-date-posted'}).find('div',{'class':'secondary-text-color'}).text)
        update_db(organization)
        reset_vars()
def run(url):
    next_page_url = url
    soup = get_javascript_soup_delayed(next_page_url, 'job-table-title')
    job_class = Job(organization, "")
    job_class.organization_id = organization_id
    insert_count = 0
    while soup:
        job_table = soup.find('tbody')
        for job_row in job_table.find_all('tr'):
            job_class.title = job_row.find('td', {
                'class': 'job-table-title'
            }).a.text.strip()
            job_class.info_link = 'https://www.governmentjobs.com' + \
                job_row.find('td', {'class': 'job-table-title'}).a['href']
            job_class.salary = job_row.find('td', {
                'class': 'job-table-salary'
            }).text
            job_class.full_or_part = job_row.find('td', {
                'class': 'job-table-type'
            }).text
            # Get soup for job listing to get more info
            job_soup = get_soup(job_class.info_link)
            info_container = job_soup.find('div',
                                           {'class': 'summary container'})
            job_class.location = clean_location(
                info_container.find('div', {
                    'id': 'location-label-id'
                }).parent.find_all('div')[2].text)
            job_class.zip_code = city_to_zip(job_class.location)
            job_class.summary = job_soup.find('div', {
                'id': 'details-info'
            }).find('p').text
            insert_count += job_insert(job_class)
        if not 'disabled' in soup.find('li', {
                'class': 'PagedList-skipToNext'
        }).get("class"):
            next_page_url = 'https://www.governmentjobs.com/careers/lahsa?' + \
                soup.find('li', {'class': 'PagedList-skipToNext'}
                          ).a['href'].split('?')[1]
            soup = get_javascript_soup_delayed(next_page_url,
                                               'job-table-title')
        else:
            soup = False
    return insert_count
def run(url):
    soup = get_javascript_soup(url)

    jobs_list = soup.find(
        'table', {'id': 'cws-search-results'}).find_all('tr')[1:]
    job_class= Job(organization, "")
    job_class.organization_id= organization_id
    insert_count= 0
    for job_entry in jobs_list:
        row_cells = job_entry.find_all('td')
        job_class.title = row_cells[1].a.text.strip()
        job_class.info_link = row_cells[1].a['href']
        job_class.location = clean_location(row_cells[2].text)
        job_class.zip_code = city_to_zip(job_class.location)
        job_soup = get_soup(job_class.info_link)
        job_class.full_or_part = job_soup.find(
            text="Employment Duration:").parent.parent.b.text.strip()
        insert_count+= job_insert(job_class)
    return insert_count
Example #15
0
def run(url):
    # Use Selenium browser to click on all positions button and get soup
    options = webdriver.ChromeOptions()
    options.add_argument('window-size=800x841')
    options.add_argument('headless')
    browser = webdriver.Chrome('./chromedriver', chrome_options=options)
    browser.get(url)
    python_button = browser.find_element_by_id('AllJobs')
    python_button.click()
    innerHTML = browser.execute_script("return document.body.innerHTML")
    soup = BeautifulSoup(innerHTML, "lxml")
    browser.quit()

    jobs_list = soup.find_all('tr')[2:]
    job_class = Job(organization, "")
    job_class.organization_id = organization_id
    insert_count = 0
    for job_entry in jobs_list:
        job_details = job_entry.find_all('td')
        job_class.title = job_details[0].text.strip()
        job_class.info_link = 'https://www5.recruitingcenter.net/Clients/HathawaySycamores/PublicJobs/' + \
            job_details[0].a['href']
        job_class.full_or_part = job_details[4].text.strip()
        location_details = job_details[2].text.strip()
        if len(location_details) > 0:
            if location_details.isdigit():
                job_class.zip_code = int(location_details)
                job_class.location = zip_to_city(job_class.zip_code)
            else:
                job_class.location = location_details
                job_class.zip_code = city_to_zip(job_class.location)
        else:
            job_class.location = ''
            job_class.zip_code = ''
        insert_count += job_insert(job_class)
    return insert_count
Example #16
0
def run(url):
    soup = get_javascript_soup(url)

    job_listings = soup.find_all('div', {'class': 'jobInfo'})
    job_class = Job(organization, "")
    job_class.organization_id = organization_id
    insert_count = 0
    for job_listing in job_listings:
        job_class.title = job_listing.find('span', {
            'class': 'jobTitle'
        }).a.text.strip()
        job_class.info_link = 'https://www.paycomonline.net' + \
            job_listing.find('span', {'class': 'jobTitle'}).a['href']
        if job_listing.find('span', {'class': 'jobLocation'}).text:
            job_class.location = globals.clean_location(
                job_listing.find('span', {
                    'class': 'jobLocation'
                }).text.split(' - ')[1])
            job_class.zip_code = globals.city_to_zip(job_class.location)
        if job_listing.find('span', {'class': 'jobDescription'}).text:
            job_class.summary = job_listing.find('span', {
                'class': 'jobDescription'
            }).text.strip()
        if job_listing.find('span', {'class': 'jobType'}).text:
            if ('ft' in str(
                    job_listing.find('span', {
                        'class': 'jobType'
                    }).text).lower()) or ('full' in str(
                        job_listing.find('span', {
                            'class': 'jobType'
                        }).text).lower()):
                job_class.full_or_part = 'full'
            else:
                job_class.full_or_part = 'part'
        insert_count += job_insert(job_class)
    return insert_count