def ospackages(pkg_list):
"""Summary
Install OS Package Prerequisites
Returns:
Success | Failure, TYPE: bool
"""
try:
for pkg in pkg_list:
if is_installed(pkg):
logger.info(f'{pkg} binary is already installed - skip')
continue
elif which('yum'):
cmd = 'sudo yum install ' + pkg + ' 2>/dev/null'
print(subprocess.getoutput(cmd))
elif which('dnf'):
cmd = 'sudo dnf install ' + pkg + ' 2>/dev/null'
print(subprocess.getoutput(cmd))
else:
logger.warning(
'%s: Dependent OS binaries not installed - package manager not identified'
% inspect.stack()[0][3])
except OSError as e:
logger.exception('{}: Problem installing os package {}'.format(
inspect.stack()[0][3], pkg))
return False
return True
def main():
logger.info("Program started")
parser = XMLParser()
link = parser.get_first_link("data/select.xml")
if not link:
logger.warning("No link exists")
response = requests.get(link)
with open('data/zip/xml_download.zip', 'wb') as output:
output.write(response.content)
logger.info("File download successful")
with zipfile.ZipFile('data/zip/xml_download.zip', 'r') as zip_ref:
zip_ref.extractall('data/unzip')
logger.info("File unzipped successful")
for filename in os.listdir('data/unzip'):
logger.info(f"Professing file {filename}")
if filename.endswith(".xml"):
filepath = os.path.join('data/unzip', filename)
destination_file = '.'.join(filename.split('.')[:-1]) + '.csv'
destination_path = os.path.join('data/csv', destination_file)
converter = XMLToCSVConverter(filepath, destination_path)
converter.convert()
logger.info(f"File created at path {destination_path}")
logger.info('Task completed .... :)')
def create_directories(self, path_to_create):
'''The function creates the same named directories as were parsed from the beginning.'''
try:
os.mkdir(path_to_create)
except FileExistsError as fee:
logger.warning(fee)
except OSError as ose:
logger.warning(ose)
def build_package(build_root, builddir):
"""
Summary.
Creates final os installable package for current build, build version
Returns:
Success | Failure, TYPE: bool
"""
try:
pwd = os.getcwd()
os.chdir(build_root)
if os.path.exists(builddir):
cmd = 'dpkg-deb --build ' + builddir + ' 2>/dev/null'
stdout_message('Building {}... '.format(bn + builddir + rst))
stdout_message(subprocess.getoutput(cmd))
os.chdir(pwd)
else:
logger.warning(
'Build directory {} not found. Failed to create .deb package'.
format(builddir))
os.chdir(pwd)
return False
except OSError as e:
logger.exception('{}: Error during os package creation: {}'.format(
inspect.stack()[0][3], e))
return False
except Exception as e:
logger.exception(
'{}: Unknown Error during os package creation: {}'.format(
inspect.stack()[0][3], e))
return False
return True
def init_cli():
"""Collect parameters and call main """
try:
parser = argparse.ArgumentParser(add_help=False)
args = options(parser)
except Exception as e:
help_menu()
stdout_message(str(e), 'ERROR')
return exit_codes['E_MISC']['Code']
if args.debug:
stdout_message(message='forceVersion:\t{}'.format(args.set),
prefix='DBUG',
severity='WARNING')
stdout_message(message='build:\t{}'.format(args.build),
prefix='DBUG',
severity='WARNING')
stdout_message(message='debug flag:\t{}'.format(args.debug),
prefix='DBUG',
severity='WARNING')
if len(sys.argv) == 1:
help_menu()
return exit_codes['EX_OK']['Code']
elif args.help:
help_menu()
return exit_codes['EX_OK']['Code']
elif args.build:
if valid_version(args.set) and prebuild(
TMPDIR, VOLMNT,
git_root() + '/' + PACKAGE_CONFIG):
package_path = main(setVersion=args.set,
environment=args.distro,
force=args.force,
debug=args.debug)
if package_path:
path, package = os.path.split(package_path)
stdout_message(
f'{PROJECT} build package created: {lk + path + rst}/{bn + package + rst}'
)
stdout_message(
f'Debian build process completed successfully. End',
prefix='OK')
return exit_codes['EX_OK']['Code']
else:
stdout_message(
'{}: Problem creating os installation package. Exit'.
format(inspect.stack()[0][3]),
prefix='WARN',
severity='WARNING')
return exit_codes['E_MISC']['Code']
elif not valid_version(args.set):
stdout_message(
'You must enter a valid version when using --set-version parameter. Ex: 1.6.3',
prefix='WARN',
severity='WARNING')
return exit_codes['E_DEPENDENCY']['Code']
else:
logger.warning('{} Failure in prebuild stage'.format(
inspect.stack()[0][3]))
return exit_codes['E_DEPENDENCY']['Code']
return True
def main(self):
'''This function splits data into pieces and prepares it for saving in directories.'''
if not os.path.exists(self.PATH):
return 'Filename is not correct, please check the spelling!'
for directory in os.listdir(self.PATH):
try:
with open(self.PATH + directory + '/messages.html',
'r') as messages:
soup = BS(messages, 'html.parser')
except FileNotFoundError as fee:
logger.warning(fee)
body_blocks = soup.find_all('div', class_='body')
if not os.path.exists(
os.path.dirname(__file__) + f'/data/{directory}/persons/'):
os.makedirs(
os.path.dirname(__file__) + f'/data/{directory}/persons/')
for block in body_blocks[1:]:
media_block = block.find('div', class_='media_wrap clearfix')
main_div = block.find('div', class_='text')
if main_div:
# In order to save text as in indictment.txt example we take it as it is.
div_with_class_text = BS(main_div.prettify(),
'html.parser')
if not div_with_class_text.get_text().strip().split(
'\n')[0].startswith(garbish[0]):
raw_name = div_with_class_text.get_text().strip(
).split('\n')[0].replace(' ', '_').strip(',')
if raw_name.count('_') == 0:
raw_name = ''.join(
div_with_class_text.get_text().strip().split(
'\n')[:4]).replace(' ', '_').replace(
'__', '_').strip(',')
# This code is responsible for checking the name of person.
# If name starts from forbidden words which do not have any information the name will be skipped.
if garbish[1] not in raw_name.lower(
) and garbish[2] not in raw_name.lower():
if '-' in raw_name:
name = '_' + translit(raw_name.split('-')[0],
'ru',
reversed=True).title()
else:
name = '_' + translit(
raw_name, 'ru', reversed=True).title()
self.PERSON_PATH = os.path.dirname(
__file__
) + f'/data/{directory}/persons/' + name
if len(name) > 50:
self.PERSON_PATH = os.path.dirname(
__file__
) + f'/data/{directory}/persons/' + name[:60]
self.create_directories(self.PERSON_PATH)
'''When the data is ready to be saved, this part gets both a path and data to save which.'''
if media_block and self.PATH is not None:
try:
# This code creates "indictment.txt" in every person's folder.
with open(self.PERSON_PATH + '/indictment.txt',
'w') as indictment:
indictment.write(
div_with_class_text.get_text().strip(
'\n').replace('\n\n', ''))
# This code creates "info.txt" in every person's folder
with open(self.PERSON_PATH + '/info.html',
'w') as info:
info.write(str(div_with_class_text))
# This code extracts both photos and videos that related to the person
with open(
self.PATH + directory + '/' +
media_block.a["href"], 'rb') as media_file1:
media_file_data = media_file1.read()
# This code extracts only images that related to the person
with open(
self.PATH + directory + '/' +
media_block.a.img["src"], 'rb') as media_file2:
media_file_data = media_file2.read()
# This code saves both photos and videos that related to the person
with open(
self.PERSON_PATH + '/' +
media_block.a["href"].split('/')[1],
'wb') as person_media_file1:
person_media_file1.write(media_file_data)
# This code saves only images that related to the person
with open(
self.PERSON_PATH + '/' +
media_block.a.img["src"].split('/')[1],
'wb') as person_media_file2:
person_media_file2.write(media_file_data)
except FileNotFoundError as fee:
logger.warning(fee)
except OSError as ose:
logger.warning(ose)