def ospackages(pkg_list): """Summary Install OS Package Prerequisites Returns: Success | Failure, TYPE: bool """ try: for pkg in pkg_list: if is_installed(pkg): logger.info(f'{pkg} binary is already installed - skip') continue elif which('yum'): cmd = 'sudo yum install ' + pkg + ' 2>/dev/null' print(subprocess.getoutput(cmd)) elif which('dnf'): cmd = 'sudo dnf install ' + pkg + ' 2>/dev/null' print(subprocess.getoutput(cmd)) else: logger.warning( '%s: Dependent OS binaries not installed - package manager not identified' % inspect.stack()[0][3]) except OSError as e: logger.exception('{}: Problem installing os package {}'.format( inspect.stack()[0][3], pkg)) return False return True
def main(): logger.info("Program started") parser = XMLParser() link = parser.get_first_link("data/select.xml") if not link: logger.warning("No link exists") response = requests.get(link) with open('data/zip/xml_download.zip', 'wb') as output: output.write(response.content) logger.info("File download successful") with zipfile.ZipFile('data/zip/xml_download.zip', 'r') as zip_ref: zip_ref.extractall('data/unzip') logger.info("File unzipped successful") for filename in os.listdir('data/unzip'): logger.info(f"Professing file {filename}") if filename.endswith(".xml"): filepath = os.path.join('data/unzip', filename) destination_file = '.'.join(filename.split('.')[:-1]) + '.csv' destination_path = os.path.join('data/csv', destination_file) converter = XMLToCSVConverter(filepath, destination_path) converter.convert() logger.info(f"File created at path {destination_path}") logger.info('Task completed .... :)')
def create_directories(self, path_to_create): '''The function creates the same named directories as were parsed from the beginning.''' try: os.mkdir(path_to_create) except FileExistsError as fee: logger.warning(fee) except OSError as ose: logger.warning(ose)
def build_package(build_root, builddir): """ Summary. Creates final os installable package for current build, build version Returns: Success | Failure, TYPE: bool """ try: pwd = os.getcwd() os.chdir(build_root) if os.path.exists(builddir): cmd = 'dpkg-deb --build ' + builddir + ' 2>/dev/null' stdout_message('Building {}... '.format(bn + builddir + rst)) stdout_message(subprocess.getoutput(cmd)) os.chdir(pwd) else: logger.warning( 'Build directory {} not found. Failed to create .deb package'. format(builddir)) os.chdir(pwd) return False except OSError as e: logger.exception('{}: Error during os package creation: {}'.format( inspect.stack()[0][3], e)) return False except Exception as e: logger.exception( '{}: Unknown Error during os package creation: {}'.format( inspect.stack()[0][3], e)) return False return True
def init_cli(): """Collect parameters and call main """ try: parser = argparse.ArgumentParser(add_help=False) args = options(parser) except Exception as e: help_menu() stdout_message(str(e), 'ERROR') return exit_codes['E_MISC']['Code'] if args.debug: stdout_message(message='forceVersion:\t{}'.format(args.set), prefix='DBUG', severity='WARNING') stdout_message(message='build:\t{}'.format(args.build), prefix='DBUG', severity='WARNING') stdout_message(message='debug flag:\t{}'.format(args.debug), prefix='DBUG', severity='WARNING') if len(sys.argv) == 1: help_menu() return exit_codes['EX_OK']['Code'] elif args.help: help_menu() return exit_codes['EX_OK']['Code'] elif args.build: if valid_version(args.set) and prebuild( TMPDIR, VOLMNT, git_root() + '/' + PACKAGE_CONFIG): package_path = main(setVersion=args.set, environment=args.distro, force=args.force, debug=args.debug) if package_path: path, package = os.path.split(package_path) stdout_message( f'{PROJECT} build package created: {lk + path + rst}/{bn + package + rst}' ) stdout_message( f'Debian build process completed successfully. End', prefix='OK') return exit_codes['EX_OK']['Code'] else: stdout_message( '{}: Problem creating os installation package. Exit'. format(inspect.stack()[0][3]), prefix='WARN', severity='WARNING') return exit_codes['E_MISC']['Code'] elif not valid_version(args.set): stdout_message( 'You must enter a valid version when using --set-version parameter. Ex: 1.6.3', prefix='WARN', severity='WARNING') return exit_codes['E_DEPENDENCY']['Code'] else: logger.warning('{} Failure in prebuild stage'.format( inspect.stack()[0][3])) return exit_codes['E_DEPENDENCY']['Code'] return True
def main(self): '''This function splits data into pieces and prepares it for saving in directories.''' if not os.path.exists(self.PATH): return 'Filename is not correct, please check the spelling!' for directory in os.listdir(self.PATH): try: with open(self.PATH + directory + '/messages.html', 'r') as messages: soup = BS(messages, 'html.parser') except FileNotFoundError as fee: logger.warning(fee) body_blocks = soup.find_all('div', class_='body') if not os.path.exists( os.path.dirname(__file__) + f'/data/{directory}/persons/'): os.makedirs( os.path.dirname(__file__) + f'/data/{directory}/persons/') for block in body_blocks[1:]: media_block = block.find('div', class_='media_wrap clearfix') main_div = block.find('div', class_='text') if main_div: # In order to save text as in indictment.txt example we take it as it is. div_with_class_text = BS(main_div.prettify(), 'html.parser') if not div_with_class_text.get_text().strip().split( '\n')[0].startswith(garbish[0]): raw_name = div_with_class_text.get_text().strip( ).split('\n')[0].replace(' ', '_').strip(',') if raw_name.count('_') == 0: raw_name = ''.join( div_with_class_text.get_text().strip().split( '\n')[:4]).replace(' ', '_').replace( '__', '_').strip(',') # This code is responsible for checking the name of person. # If name starts from forbidden words which do not have any information the name will be skipped. if garbish[1] not in raw_name.lower( ) and garbish[2] not in raw_name.lower(): if '-' in raw_name: name = '_' + translit(raw_name.split('-')[0], 'ru', reversed=True).title() else: name = '_' + translit( raw_name, 'ru', reversed=True).title() self.PERSON_PATH = os.path.dirname( __file__ ) + f'/data/{directory}/persons/' + name if len(name) > 50: self.PERSON_PATH = os.path.dirname( __file__ ) + f'/data/{directory}/persons/' + name[:60] self.create_directories(self.PERSON_PATH) '''When the data is ready to be saved, this part gets both a path and data to save which.''' if media_block and self.PATH is not None: try: # This code creates "indictment.txt" in every person's folder. with open(self.PERSON_PATH + '/indictment.txt', 'w') as indictment: indictment.write( div_with_class_text.get_text().strip( '\n').replace('\n\n', '')) # This code creates "info.txt" in every person's folder with open(self.PERSON_PATH + '/info.html', 'w') as info: info.write(str(div_with_class_text)) # This code extracts both photos and videos that related to the person with open( self.PATH + directory + '/' + media_block.a["href"], 'rb') as media_file1: media_file_data = media_file1.read() # This code extracts only images that related to the person with open( self.PATH + directory + '/' + media_block.a.img["src"], 'rb') as media_file2: media_file_data = media_file2.read() # This code saves both photos and videos that related to the person with open( self.PERSON_PATH + '/' + media_block.a["href"].split('/')[1], 'wb') as person_media_file1: person_media_file1.write(media_file_data) # This code saves only images that related to the person with open( self.PERSON_PATH + '/' + media_block.a.img["src"].split('/')[1], 'wb') as person_media_file2: person_media_file2.write(media_file_data) except FileNotFoundError as fee: logger.warning(fee) except OSError as ose: logger.warning(ose)