def get_file(normalized_url, folder, changed_url):
'''
Download file
:param normalized_url: URL for downloads
:param folder: Destination folder
:param changed_url: File name
:return: Path to downloaded file
'''
filename, file_extension = os.path.splitext(normalized_url)
if filename.startswith('data:'):
return normalized_url
changed_filename = change_symbols(filename)
logger.debug('New filename for {}{} is {}{}'.format(filename, file_extension,
changed_filename,
file_extension))
try:
get_files = requests.get(normalized_url)
get_files.raise_for_status()
except requests.exceptions.RequestException as error:
logger.critical(error)
raise requests.exceptions.RequestException(3)
file = get_files.content
logger.debug('{} is downloaded'.format(normalized_url))
file_path = os.path.join(folder, changed_filename + file_extension)
save_file(file_path, file)
return os.path.join(changed_url + POSTFIX, changed_filename + file_extension)
def create_directory(folder):
'''
Create directory
:param folder: Path to folder
:return:
'''
try:
os.makedirs(folder)
logger.warning('Created folder {}'.format(folder))
except OSError as error:
logger.critical(error)
raise OSError(4)
def get_page(url):
'''
Download page
:param url: URL
:return: BeautifulSoup object
'''
try:
page = requests.get(url)
page.raise_for_status()
return BeautifulSoup(page.text, "html.parser")
except requests.exceptions.RequestException as error:
logger.critical(error)
raise requests.exceptions.RequestException(2)
def save_page(path, page):
'''
Created page from BeautifulSoup object
:param path: Destination directory
:param page: BeautifulSoup object
:return: None
'''
try:
with open(path, 'w') as file:
file.write(page.prettify())
logger.info('Modified page created')
except OSError as error:
logger.critical(error)
raise OSError(6)
def save_file(file_path, file):
'''
Save file from request
:param file_path: Path to new file
:param file: File from request
:return: None
'''
try:
with open(file_path, 'wb') as received_file:
received_file.write(file)
logger.debug('{} is created'.format(file_path))
except OSError as error:
logger.critical(error)
raise OSError(5)