def test_error_in_loading_page():
with requests_mock.mock() as m:
address = 'https://hexlet.io/courses'
m.get(address, reason='Not Found', status_code=404)
with tempfile.TemporaryDirectory() as tmpdirname:
with pytest.raises(ConnectionError) as excinfo:
load_page(address, tmpdirname, logging.DEBUG)
assert str(excinfo.value) == (
f"Request to {address} returned: 404 Not Found")
def test_no_permission_to_save_file():
with requests_mock.mock() as m:
address = 'https://hexlet.io/courses'
m.get(address, text='')
with tempfile.TemporaryDirectory() as tmpdirname:
result_path = f"{tmpdirname}/hexlet-io-courses.html"
os.chmod(tmpdirname, stat.S_IREAD)
with pytest.raises(PermissionError) as excinfo:
load_page(address, tmpdirname, logging.DEBUG)
assert str(excinfo.value) == (
f"No permission to save file: {result_path}")
def test_invalid_save_path():
with requests_mock.mock() as m:
address = 'https://hexlet.io/courses'
m.get(address, text='')
with tempfile.TemporaryDirectory() as tmpdirname:
wrong_tmpdirname = tmpdirname + 'xxxxxxxx'
result_path = f"{wrong_tmpdirname}/hexlet-io-courses.html"
with pytest.raises(FileNotFoundError) as excinfo:
load_page(address, wrong_tmpdirname, logging.DEBUG)
assert str(
excinfo.value) == (f"Invalid path to save file: {result_path}")
def test_load_page():
with requests_mock.mock() as m:
address = 'https://hexlet.io/courses'
response_text = '<div>\n test\n</div>'
m.get(address, text=response_text)
with tempfile.TemporaryDirectory() as tmpdirname:
load_page(address, tmpdirname, logging.DEBUG)
result_path = os.path.join(tmpdirname, 'hexlet-io-courses.html')
with open(result_path, 'r') as f:
assert f.read() == response_text
def test_load_page_with_local_resources():
with requests_mock.mock() as m:
host = 'https://hexlet.io'
address = host + '/courses'
with open('tests/fixtures/index.html', 'r') as f:
m.get(address, text=f.read())
with open('tests/fixtures/index.css', 'r') as f:
request_address = host + '/assets/index.css'
result_css_content = f.read()
m.get(request_address, text=result_css_content)
with open('tests/fixtures/index.js', 'r') as f:
request_address = host + '/assets/index.js'
result_js_content = f.read()
m.get(request_address, text=result_js_content)
with open('tests/fixtures/index.jpg', 'rb') as f:
request_address = host + '/assets/index.jpg'
result_bin_content = f.read()
m.get(request_address, content=result_bin_content)
with open('tests/fixtures/result.html', 'r') as f:
result_html_content = f.read()
with tempfile.TemporaryDirectory() as tmpdirname:
load_page(address, tmpdirname, logging.DEBUG)
result_html_path = os.path.join(tmpdirname,
'hexlet-io-courses.html')
with open(result_html_path, 'r') as f:
assert f.read() == result_html_content
result_css_path = os.path.join(
tmpdirname, 'hexlet-io-courses_files/assets-index.css')
with open(result_css_path, 'r') as f:
assert f.read() == result_css_content
result_js_path = os.path.join(
tmpdirname, 'hexlet-io-courses_files/assets-index.js')
with open(result_js_path, 'r') as f:
assert f.read() == result_js_content
result_bin_path = os.path.join(
tmpdirname, 'hexlet-io-courses_files/assets-index.jpg')
with open(result_bin_path, 'rb') as f:
assert f.read() == result_bin_content
def test_error_in_loading_local_resource():
with requests_mock.mock() as m:
host = 'https://hexlet.io'
address = host + '/courses'
with open('tests/fixtures/index.html', 'r') as f:
m.get(address, text=f.read())
with open('tests/fixtures/index.css', 'r') as f:
request_address = host + '/assets/index.css'
m.get(request_address,
reason='Internal Server Error',
status_code=500)
with tempfile.TemporaryDirectory() as tmpdirname:
with pytest.raises(ConnectionError) as excinfo:
load_page(address, tmpdirname, logging.DEBUG)
assert str(
excinfo.value) == (f"Request to {request_address} returned: "
"500 Internal Server Error")
def main():
parser = argparse.ArgumentParser(description=(
'Downloads the internet page and put it in the specified folder.'))
parser.add_argument('address')
parser.add_argument('-o',
'--output',
help='set output path',
type=str,
default=os.getcwd())
parser.add_argument('-l',
'--logging',
help='set logging level',
type=logging_level,
default=WARNING)
args = parser.parse_args()
try:
load_page(args.address, args.output, args.logging)
except FileNotFoundError:
sys.exit(errno.ENOENT)
except PermissionError:
sys.exit(errno.EACCES)
def save_individual_trades(element_list, individual_code):
for element in element_list:
insider_trade = IndividualInsiderTrades()
link = element[0]
stock_code = link.replace('https://www.nasdaq.com/symbol/', '')
stock_code = stock_code.replace('/insider-trades',
'').strip(' \t\n\r').upper()
print(stock_code)
try:
Stock().get_by_id(stock_code)
print('success')
insider_trade.company = stock_code
except Exception:
stock = Stock()
stock_url = make_url('main', stock_code)
stock_page = load_page(stock_url)
stock_name = scrape_stock_name(stock_page)
stock.name = stock_name
stock.code = stock_code
save_stock(stock)
insider_trade.company = Stock().get_by_id(stock_code)
insider_trade.insider = individual_code
insider_trade.relation = element[2]
trade_last_date = datetime.date(
datetime.strptime(element[3], '%m/%d/%Y'))
insider_trade.last_date = trade_last_date
insider_trade.tran = element[5]
insider_trade.owner_type = element[6]
shares_traded = element[7].replace(',', '')
if shares_traded == '':
shares_traded = 0
insider_trade.shares_traded = shares_traded
last_price = element[8]
if last_price == '':
last_price = 0.0
insider_trade.last_price = last_price
shares_held = element[9].replace(',', '')
if shares_held == '':
shares_held = 0
insider_trade.shares_held = shares_held
try:
insider_trade.save()
logger.info(
f'Trade object of {individual_code} on last date {element[3]} saved in db'
)
except Exception as exception:
print(exception)
logger.error(f'{individual_code} is allready exist: {exception}')
db.rollback()