def manage_get_ITJW_top_30_menu(self):
print('Please select from the below menu options\n')
print(
'1. Print top 30 to downloads folder with default name (No Headers)'
)
print(
'2. Print top 30 to downloads folder with default name (with Headers)'
)
print('3. Return to Main Menu')
print('or type exit to quit program\n')
print('Please select option:\n')
option_selected = input()
if option_selected.lower() == 'exit':
exit()
elif option_selected == '':
self.manage_get_ITJW_top_30_menu()
elif int(option_selected) == 1:
Top30CSVGenerator().generate_top_30_csv(
ItJobsWatchHomePageTop30(itjobswatch_home_page_url()).
get_top_30_table_elements_into_array())
print('Please check your downloads folder')
self.manage_get_ITJW_top_30_menu()
elif int(option_selected) == 2:
top_30 = ItJobsWatchHomePageTop30(itjobswatch_home_page_url())
Top30CSVGenerator().generate_top_30_csv(
top_30.get_top_30_table_elements_into_array(),
os.path.expanduser('~/Downloads/'), 'ItJobsWatchTop30.csv',
top_30.get_table_headers_array())
elif int(option_selected) == 3:
self.menu_control()
else:
print('Please select an option from the menu or type exit')
self.manage_get_ITJW_top_30_menu()
def generate_top_30_html_object(self):
if get_test_env_setting() == 'live':
return itjobswatch_home_page_top_30.ItJobsWatchHomePageTop30(
itjobswatch_home_page_url())
else:
return itjobswatch_home_page_top_30.ItJobsWatchHomePageTop30(
itjobswatch_home_page_test_file())
def run_automation_head(self):
top_30 = ItJobsWatchHomePageTop30(itjobswatch_home_page_url())
Top30CSVGenerator().generate_top_30_csv(
top_30.get_top_30_table_elements_into_array(),
os.path.expanduser('~/Downloads/'), 'ItJobsWatchTop30.csv',
top_30.get_table_headers_array())
print('Please check your downloads folder')
def no_headers():
# If there is a pre-existing csv file this will remove it
# This is to ensure the app always uses the newest data present
if os.path.exists(os.getcwd() + "/ItJobsWatchTop30s3.csv"):
os.remove(os.getcwd() + "/ItJobsWatchTop30s3.csv")
live_response = requests.get("https://www.itjobswatch.co.uk/")
if live_response.status_code:
cwd = os.getcwd() + "/"
top_30 = ItJobsWatchHomePageTop30(itjobswatch_home_page_url())
Top30CSVGenerator().generate_top_30_csv(
top_30.get_top_30_table_elements_into_array(), cwd,
'ItJobsWatchTop30s3.csv', top_30.get_table_headers_array())
try:
upload_file("ItJobsWatchTop30s3.csv", "eng74-final-project-bucket")
download_file("eng74-final-project-bucket",
"ItJobsWatchTop30s3.csv")
except:
pass
# # If not then just download the s3 bucket data
else:
try:
download_file("eng74-final-project-bucket", "ItJobsWatchTop30.csv")
except:
pass
filename = "ItJobsWatchTop30s3.csv"
data = pandas.read_csv(filename, header=0)
joblist = list(data.values)
return render_template('list.html', joblist=joblist)
def html_manager_object(self):
if get_test_env_setting() == 'live':
return HttpManager(itjobswatch_home_page_url())
else:
return HttpManager(itjobswatch_home_page_test_file())
from config_manager import itjobswatch_home_page_url
from src.itjobswatch_html_readers.itjobswatch_home_page_top_30 import ItJobsWatchHomePageTop30
from src.csv_generators.top_30_csv_generator import * # importing file that will create our csv file from the specified url
import os
from aws_upload import upload_to_aws # importing the function that will upload file to aws
# the below functions will be run when we run our python main.py in the shell
if __name__ == '__main__':
# This function will run an API scrape that will grab the top 30 IT jobs and place it in a Downloads folder within our project in a CSV format
top_30 = ItJobsWatchHomePageTop30(itjobswatch_home_page_url())
Top30CSVGenerator().generate_top_30_csv(
top_30.get_top_30_table_elements_into_array(),
os.path.expanduser('Downloads/'), 'ItJobsWatchTop30.csv',
top_30.get_table_headers_array())
# this function will take our csv file that will be stored in our downloads folder and push it to our s3 bucket, the details are specified below
upload_to_aws()
def run_automation_head_none(self):
Top30CSVGenerator().generate_top_30_csv(
ItJobsWatchHomePageTop30(itjobswatch_home_page_url()).
get_top_30_table_elements_into_array())
print('Please check your downloads folder')
import requests
import os.path
from config_manager import itjobswatch_home_page_test_file, itjobswatch_home_page_url
class HtmlObjectManager():
def __init__(self, file_or_url_location):
self.url_response = None
self.html = None
self._get_html_file_or_url(file_or_url_location)
def _get_html_file_or_url(self, file_location_or_url_location):
if os.path.exists(file_location_or_url_location):
self.html = open(file_location_or_url_location).read()
elif 'http' in file_location_or_url_location:
response = requests.get(file_location_or_url_location)
self.url_response = response
self.html = response.content.decode("utf-8")
else:
raise NameError("Please ensure your file or url have the correct path")
def get_response_code(self):
return self.url_response.status_code
if __name__ == '__main__':
print(HtmlObjectManager(itjobswatch_home_page_url()).html)
