def multiprocess_pages(query_URL, job_title, job_location, page_num): """Grab the URLs and other relevant info. from job postings on the page. The ZipRecruiter URL used for job searching takes an additional parameter, `page`, that allows you to start the job search at page 0-20 (20 is the max). Use this to grab job results from multiple pages at once, and then pass jobs on to threads to grab relevant info. Args: ---- base_URL: str job_title: str job_location: str page_start: int """ url = query_URL + '&page=' + str(page_num) #print (url) html = get_html(url) rows = html.select('.job_content') #print (rows) threads = [] mongo_update_lst = [] for row in rows: # print (row) thread = RequestInfoThread(row, job_title, job_location) thread.start() threads.append(thread) for thread in threads: thread.join() mongo_update_lst.append(thread.json_dct) #store_in_mongo(mongo_update_lst, 'job_postings', 'ziprecruiter') #print (mongo_update_lst) store_in_mongo(mongo_update_lst, 'job_postings', 'ziprecruiter_final')
def multiprocess_pages(base_URL, job_title, job_location, page_number): """Grab the URLS and other relevant info. from job postings on the page. The Simply Hired URL used for job searching takes another parameter, `pn`, that allows you to start the job search at jobs 11-20, 21-30, etc. Use this to grab job results from multiple pages at once, and then feed the jobs from each page to threads for further parsing. Args: ---- base_URL: str job_title: str job_location: str page_number: int """ url = base_URL + '&pn=' + str(page_number) html = get_html(url) # Each row corresponds to a job. jobs = html.select('.js-job') threads = [] mongo_update_lst = [] for job in jobs: thread = RequestInfoThread(job, job_title, job_location) thread.start() threads.append(thread) for thread in threads: thread.join() mongo_update_lst.append(thread.json_dct) store_in_mongo(mongo_update_lst, 'job_postings', 'simplyhired')
def multiprocess_pages(base_URL, job_title, job_location, page_start): """Grab the URLS and other relevant info. from job postings on the page. The Indeed URL used for job searching takes another parameter, `start`, that allows you to start the job search at jobs 10-20, 20-30, etc. I can use this to grab job results from multiple pages at once. This function takes in the base_URL and then adds that start={page_start} parameter to the URL, and then queries it. It passes the results on to a thread to grab the details from each job posting. Args: base_URL: String that holds the base URL to add the page_start parameter to. job_title: String holding the job title used for the search job_location: String holding the job location used for the search page_start: Integer of what the `start` parameter in the URL should be set to. """ url = base_URL + '&start=' + str(page_start) html = get_html(url) # Each row corresponds to a job. rows = html.select('.row') threads = [] mongo_update_lst = [] for row in rows: thread = RequestInfoThread(row, job_title, job_location) thread.start() threads.append(thread) for thread in threads: thread.join() mongo_update_lst.append(thread.json_dct) store_in_mongo(mongo_update_lst, 'job_postings', 'indeed')
def multiprocess_pages(base_URL, job_title, job_location, page_num): """Grab the URLs and other relevant info. from job postings on the page. The ZipRecruiter URL used for job searching takes an additional parameter, `page`, that allows you to start the job search at page 0-20 (20 is the max). I can use this to grab job results from multiple pages at once. This function here takes in the base_URL, and then adds that page={page_num} parameter to the URL, and then queries it. It passes the results on to a thread to grab the details from each job posting. Args: base_URL: String that holds the base URL to add the page_num parameter to. job_title: String holding the job title used for the search job_location: String holding the job location used for the search page_num: Integer of what the `page` paramter in the URL should be set to. """ url = query_URL + '&page=' + str(page_num) html = get_html(url) rows = html.select('.job_result') threads = [] mongo_update_lst = [] for row in rows: thread = RequestInfoThread(row, job_title, job_location) thread.start() threads.append(thread) for thread in threads: thread.join() mongo_update_lst.append(thread.json_dct) store_in_mongo(mongo_update_lst, 'job_postings', 'ziprecruiter')
def multiprocess_pages(base_URL, job_title, job_location, page_start): """Grab the URLS and other relevant info. from job postings on the page. The Indeed URL used for job searching takes another parameter, `start`, that allows you to start the job search at jobs 10-20, 20-30, etc. Use this to grab job results from multiple pages at once, passing the result from a page on to a thread to grab the details from each job posting. Args: ---- base_URL: str job_title: str job_location: str page_start: int """ url = base_URL + '&start=' + str(page_start) html = get_html(url) # Each row corresponds to a job. rows = html.select('.row') threads = [] mongo_update_lst = [] for row in rows: thread = RequestInfoThread(row, job_title, job_location) thread.start() threads.append(thread) for thread in threads: thread.join() mongo_update_lst.append(thread.json_dct) store_in_mongo(mongo_update_lst, 'job_postings', 'indeed')