def create_main_log_file(self): path = self.logs_dir_path + 'xegtor.log' file = open(path ,'w') file.close() # create log header logger = Logger(log_file=path) logger.add_log_header()
def __check_for_update(self) -> None: ''' Check if any new updates are available. ''' self.core_framework_location = os.path.abspath( os.path.join(os.path.dirname(__file__), '../../../')) self.last_update_file_location = self.core_framework_location + '/source/core/python/update_logs/last_update_date.log' self.versions_file_location = self.core_framework_location + '/source/core/python/update_logs/versions.json' if not os.path.isfile(self.last_update_file_location): with open(self.last_update_file_location, 'w'): Logger().log('>>> Created empty last_updated.log') if not os.path.isfile(self.versions_file_location): with open(self.versions_file_location, 'w'): Logger().log('>>> Created empty versions.json') with open(self.last_update_file_location) as last_update_date: last_update_date = last_update_date.read() if last_update_date == '' or last_update_date != datetime.now( ).strftime('%b %d %Y'): Logger().log('>>> Updating.') if self.__update(): Logger().log('>>> Update was successful.') else: Logger().log('>>> Update date is not reached yet.')
def start( self ): for index, news in enumerate( self.news_list, start=0 ): try: if news['status'] == 'pending': news_content = self.download_news( news ) if news_content: self.news_list[ index ]['status'] = 'completed' self.news.append( news_content ) log.success('[ {nid} ] Dados salvos com sucesso!'.format(nid=news['id'])) print() print() else: error_message = 'Não foi possível fazer o parse dos dados.' log.error( error_message ) self.errors.append( error_message ) self.news_list[ index ]['errors'].append( error_message ) else: log.warning('Dados já adquiridos [ {nid} ]'.format(nid=news['id'])) except Exception as error: log.error('Erro ao baixar a notícia [ {nid} ]'.format(nid=news['id'])) log.error(error) pass finally: helper.create_file( filename=self.dump_file, content=self.news, format='json', mode='w') helper.create_file( filename=self.news_json_file, content=self.news_list, format='json', mode='w')
def __init__(self,logfile,debugmode): global encode_logger if debugmode == True: encode_logger = Logger(logfile,logging.DEBUG,logging.INFO) else: encode_logger = Logger(logfile,logging.INFO,logging.INFO) encode_logger.debug('Loading Encode Module')
def __init__(self, logfile, srcarray, srcuser, srcpasswd, srcscope): global navi_logger global array global user global password global scope navi_logger = Logger(logfile, logging.DEBUG, logging.INFO) navi_logger.debug('Loading NaviCLI Module') array = srcarray user = srcuser password = srcpasswd scope = srcscope
def validate_env_variables(self) -> None: try: os.environ['SUDO_UID'] except KeyError: Logger().log('Please start the application using "sudo -E".') for env_variable in ENVIRONMENTAL_VARIABLES: try: os.environ[env_variable] except KeyError: Logger().log('Environmental variable {} is not set.'.format( env_variable))
def download(type, filename, nid, url): if type == 'image': try: response = requests.get(url, stream=True) with open(filename, 'wb') as image: shutil.copyfileobj(response.raw, image) log.success('Imagem baixada com sucesso [{url}]'.format(url=url)) return True except Exception as error: log.error(error) pass
def create_file(filename, content, mode='a+', encoding='utf-8', format=None): os.chdir(sys.path[0]) try: if format == 'json': content = json.dumps(content, indent=4, sort_keys=True) _file = open(filename, mode=mode, encoding=encoding) _file.write(content) _file.close() except Exception as error: log.error( error ) pass
class MonthToIntegerConverter(object): def __init__(self): self.logger = Logger() self.months = [ 'NoneToMakeNumberNotStupid', 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec' ] def get_month(self, month): try: return self.months.index(month) except Exception as e: self.logger.error( 'Error converting month string to integer value: {}'.format(e))
class Applications(object): def __init__(self, user, password): self.logger = Logger() self.xml_parser = xml_parser.XmlParser() self.user = user self.password = password return def get_apps(self): apps = [] try: self.logger.info('Attempting to get applications!!!') resp = requests.post(endpoints.REQUEST_APPS, auth=HTTPBasicAuth(self.user, self.password)) if resp.status_code == 200: response = self.xml_parser.parse_xml(resp.text) for item in response: if item.attrib['app_id'] is not None and not item.attrib[ 'app_id'] in apps: apps.append({ 'app_id': item.attrib['app_id'], 'app_name': item.attrib['app_name'], 'sandboxes': [] }) self.logger.info('Successfully retrieved app list!') else: self.logger.error('Authentication Issue: {}'.format( resp.status_code)) except Exception as e: self.logger.exception('Error retrieving app list: {}'.format(e)) return apps
def run_openface(configs, dataset): text_file = open("./logs/bp4d_failure_20200724193802.txt", "r") lines = text_file.readlines() lines = [i.split(':')[3].strip(' ') for i in lines] ts = time.time() st = datetime.datetime.fromtimestamp(ts).strftime('%Y%m%d%H%M%S') d = configs['paths']['input'] logger = Logger(f'./logs/{dataset}_success_{st}.txt', f'./logs/{dataset}_failure_{st}.txt') for dir, subdir, files in os.walk(d): if not subdir: if configs['params']['type'] == 'frames': subject = dir.split('/')[-2] task = dir.split('/')[-1] fname = f"{subject}_{task}" if fname in lines: print( f" extracting for Subject {dir.split('/')[-2]} task {dir.split('/')[-1]}..." ) status, msg = extract_data(configs, dir) if status == -1: logger.log_failure( dir.split('/')[-2], 'frames', dir.split('/')[-1], msg) print("failed!") else: logger.log_success( dir.split('/')[-2], 'frames', dir.split('/')[-1]) print("done!") else: continue elif configs['params']['type'] == 'videos': for file in files: if os.path.splitext(file)[-1] == configs['params']['ext']: subject = os.path.join(dir, file) print( f"extracting for Subject {subject.split('/')[-1]}...." ) status, msg = extract_data(configs, subject) if status == -1: logger.log_failure( dir.split('/')[-1], 'videos', msg) print("failed!") else: logger.log_success(dir.split('/')[-1], 'videos') print("done!") print("openface extraction complete!")
def __init__(self,logfile,debugmode,global_XMS_IP,global_XMS_USER,global_XMS_PASS): global rest_logger global XMS_IP global XMS_USERID global XMS_PASS if debugmode == True: rest_logger = Logger(logfile,logging.DEBUG,logging.INFO) else: rest_logger = Logger(logfile,logging.INFO,logging.INFO) rest_logger.debug('Loading Restful Module') XMS_IP = global_XMS_IP XMS_USERID = global_XMS_USER XMS_PASS = global_XMS_PASS
class LCS(object): """main class""" logger = Logger() op_sys = OperatingSystem() networking = Networking() services = Services() printers = Printers() environments = Environments() def __init__(self): self.main() def main(self): """ main function for starting up scanner """ self.logger.normal_output("Running linux security check") self.run() def run(self): """ default run method """ self.op_sys.run() self.networking.run() self.services.run() self.environments.run()
class ExecuteCommand(object): """ Command execution object """ logger = Logger() def __init__(self): return def execute_command(self, command_text): """ Execute a command using subprocess.check_output """ try: return check_output(command_text.split(" ")) except (OSError, CalledProcessError) as error: self.logger.error("Error processing command: " + command_text + " " + error.output) return def thing(self): """ test """ return self
def start_thread(mode, db_name, qid, task, info, tq_rank, log_dir=None): config = ConfigParser.RawConfigParser(allow_no_value=True) config.read('config.ini') db = Database(config.get('database', 'user'), config.get('database', 'pw'), config.get('database', 'host'), db_name, config.get('database', 'cache_dir'), timeout=config.get('database', 'timeout'), buffer_pool_size=config.get('database', 'buffer_pool_size')) parser = SQLParser(db_name, config.get('parser', 'cache_dir')) # only load aig if info includes range aig = None if (mode == 'greedybb' or mode == 'greedyfirst') and info == 'range': aig = AIG(db, os.path.join(config.get('aig', 'dir'), db_name + '.aig')) data = json.loads(task) task_cleaned = { 'cqs': {}, 'ans': [] } for cqid, cq in data['cqs'].items(): task_cleaned['cqs'][int(cqid)] = cq for cqid in data['ans']: task_cleaned['ans'].append(int(cqid)) task = task_cleaned if log_dir: log_path = os.path.join(log_dir, str(qid) + '.log') with Logger(log_path): return run_task(mode, db, parser, qid, task, info, aig, tq_rank) else: return run_task(mode, db, parser, qid, task, info, aig, tq_rank)
def set_image( news, index, link ): images_file = 'data/images.json' images = helper.read_file( images_file, format='json' ) if os.path.isfile( images_file ) else [] try: images.append({ 'catalog': news['catalog'], 'notice': news['id'], 'downloaded': False, 'original_path': link, 'new_path': set_image_link( news, index, link ) }) helper.create_file(images_file, images, mode='w', format='json') log.success('Imagem adicionada para a lista de downloads [ {image_link} ]'.format(image_link=set_image_link( news, index, link ))) except Exception as error: log.error( error )
def read_file(filename, format=None, mode='r', encoding='utf-8'): os.chdir(sys.path[0]) if not os.path.isfile(filename): return None else: try: _file = open( filename, mode=mode, encoding=encoding ) _content = _file.read() _file.close() if format == 'json': return json.loads( _content ) else: return _content except Exception as error: log.error( error.args[0] ) pass
def __init__(self,logfile,srcarray,srcuser,srcpasswd,srcscope): global navi_logger global array global user global password global scope navi_logger = Logger( logfile, logging.DEBUG, logging.INFO ) navi_logger.debug( 'Loading NaviCLI Module' ) array = srcarray user = srcuser password = srcpasswd scope = srcscope
def __init__(self): self.logger = Logger() user = os.getenv('VC_USER') password = os.getenv('VC_PASS') if not self.should_run(user, password): self.logger.error('User or password is not defined, exiting...') sys.exit() self.sandboxes = Sandboxes(user, password) self.applications = Applications(user, password) self.builds = Builds(user, password) self.spreadsheet_creator = SpreadsheetCreator() apps = self.applications.get_apps() self.get_app_sandboxes(apps) self.get_app_sandbox_builds(apps) self.get_builds(apps) self.create_spreadsheet(apps) self.send_message()
def main(self) -> None: try: if len(sys.argv) == 1: Logger().log('File requires at least one console argument.') else: Logger().log('Checking for updates...') self.__check_for_update() if sys.argv[1].lower() == 'gui': if sys.argv[2].lower() == '--normal': gui_elements.show_gui(False) else: gui_elements.show_gui(True) elif sys.argv[1].lower() == 'nogui': usb_detector = USBHotplugDetector() usb_detector.start() except KeyboardInterrupt: Logger().log('Keyboard interrupt detected.')
def checkLog(connector): # (↓) [-get forgotten biases-] biases = Logger().getLog(LogTypes.FORGOTTEN_BIASES) # (↓) [-delete all forgotten biases-] for biasId in biases: # (↓) [-delete forgotten bias-] try: connector.manager(actionType=ActionTypes.DELETE_BIAS, biasId=biasId) except: checkLog(connector)
def get_value(fields, field_id, warn=False, prod_fields=None): prod_fields = prod_fields or fields if field_id in fields: return fields[field_id] else: if warn: warning_text = prod_fields["ARTNR"] warning_text += ": Kein Wert für das Feld '" warning_text += field_id warning_text += "'. Das Feld in der Tabelle bleibt leer." Logger().log(warning_text) return ""
def get_welcome_text(prod_fields, ilugg_fields): max_placeholders = 5 welcome_text = "" if not "ARTWELCOMESTATE" in prod_fields: Logger().log("[WARNUNG] {} hat kein ARTWELCOMESTATE Feld".format( prod_fields["ARTNR"])) return welcome_text welcome_state = prod_fields["ARTWELCOMESTATE"] if welcome_state == "0": if not "WELCOMETHISTEXT" in prod_fields: Logger().log( "[WARNUNG] {} mit ARTWELCOMESTATE 0 hat kein WELCOMETHISTEXT Feld" .format(prod_fields["ARTNR"])) return welcome_text welcome_text = prod_fields["WELCOMETHISTEXT"] if welcome_state == "1": index = 1 while index <= max_placeholders: welcome_text = welcome_text + placeholder(index) index += 1 if welcome_state == "2": if not "WELCOMETEXT" in prod_fields: Logger().log( "[WARNUNG] {} mit ARTWELCOMESTATE 2 hat kein WELCOMETEXT Feld". format(prod_fields["ARTNR"])) return welcome_text welcome_text = prod_fields["WELCOMETEXT"] index = 1 while index <= max_placeholders: welcome_text = welcome_text.replace(placeholder(index), replacement(ilugg_fields, index)) index += 1 return welcome_text
def __init__(self, logfile, debugmode): global encode_logger if debugmode == True: encode_logger = Logger(logfile, logging.DEBUG, logging.INFO) else: encode_logger = Logger(logfile, logging.INFO, logging.INFO) encode_logger.debug('Loading Encode Module')
class SpreadsheetCreator(object): def __init__(self): self.have_data = False self.logger = Logger() self.filename = '{}VeracodeSandboxResults-{}.xlsx'.format( os.getenv('VC_ATTACHMENT_BASE_PATH'), date.today()) return def write_spreadsheet(self, apps): try: workbook = xlsxwriter.Workbook(self.filename) worksheet = workbook.add_worksheet('Aggregated Build Data') bold = workbook.add_format({'bold': True}) worksheet.write('A1', 'User', bold) worksheet.write('B1', 'Total Builds', bold) row = 1 col = 0 data = {} if len(apps) > 0: for app in apps: for sandbox in app['sandboxes']: for build in sandbox['builds']: if not build['submitter'] in data: data[build['submitter']] = 0 data[build['submitter']] += 1 if (len(data) > 0): self.have_data = True for user, number_of_builds in data.items(): worksheet.write(row, col, user) worksheet.write(row, col + 1, number_of_builds) row += 1 else: raise 'Unable to create spreadsheet, no apps found in data' workbook.close() except Exception as e: self.logger.exception('Error creating spreadsheet: {}'.format(e))
def get_factor(prod_fields, prod_field, ilugg_fields, ilugg_field): prod_definition = prod_fields[prod_field] ilugg_definition = ilugg_fields[ilugg_field] factor_category = prod_definition.split(":")[0] factor_definitions = ilugg_definition.split("§") factor = None for factor_definition in factor_definitions: if factor_definition.startswith("{}:".format(factor_category)): factor = factor_definition.split(":")[1] break if factor == None: logger = Logger() factor = prod_definition if ":" in factor: factor = factor.split(":")[1] log_text = "{}: Faktor zur Preisberechnung".format( prod_fields[ARTICLE_NUMBER]) log_text += " konnte nicht bestimmt werden." log_text += " {} in PROD ist '{}',".format(prod_field, prod_definition) log_text += " {} in ILUGG ist '{}'.".format(ilugg_field, ilugg_definition) log_text += " {} wird als Faktor angenommen.".format(factor) logger.log(log_text) return get_number(factor)
class Environments(object): """ Environment object """ logger = Logger() executor = ExecuteCommand() def __init__(self): self.env = {} def run(self): """ generic run method """ self.get_environment() def get_environment(self): """ get system environment variables """ self.logger.normal_output("Grabbing environment variables") self.logger.normal_output("Grabbing profile") self.env.update( {"profile": self.executor.execute_command("cat /etc/profile")}) self.logger.normal_output("Grabbing bashrc") self.env.update( {"bashrc": self.executor.execute_command("cat /etc/bashrc")}) self.logger.normal_output("Grabbing bash profile") self.env.update({ "bashprofile": self.executor.execute_command("cat ~/.bash_profile") }) self.logger.normal_output("Grabbing bashrc") self.env.update( {"bashrc": self.executor.execute_command("cat ~/.bashrc")}) self.logger.normal_output("Grabbing logout") self.env.update({ "bash logout": self.executor.execute_command("cat ~/.bash_logout") }) self.logger.normal_output("Grabbing env") self.env.update({"env": self.executor.execute_command("env")}) self.logger.normal_output("Grabbing set") self.env.update({"set": self.executor.execute_command("set")})
def download_news( self, news ): init_crawling = '= Iniciando crawling, alvo: [ {nid} ] {link}'.format(nid=news['id'], link=os.path.basename( news['link'] )) print() log.success( '=' * len( init_crawling ) ) log.success( init_crawling ) log.success( '=' * len( init_crawling ) ) print() request = requests.get( news['link'] ) document = BeautifulSoup( request.text, 'html.parser' ) if request.status_code == 200: return parser.parse_news( news, document ) else: error_message = 'Erro ao acessar a página: Status {status_code}'.format(status_code=request.status_code) self.errors.append( error_message ) log.error( error_message )
def parse_download(download_content, download_field = None, article = None): download_parts = download_content.split("][") if (len(download_parts) < 3): warning_text = "[ACHTUNG] Unbekanntes Download-Format " if download_field != None and article != None: warning_text += "von " warning_text += download_field warning_text += " in " warning_text += article else: warning_text += "({})".format(download_content) warning_text += ". Der Download wird übersprungen." Logger().log(warning_text) return None return { "path": build_download_path(download_parts[0]).strip(), "product": download_parts[1].strip(), "type": download_parts[2].strip() }
class FileWriter(object): """ filewriter object """ logger = Logger() def __init__(self, filename): self.filename = filename def write_output(self, command_output): """ output """ try: with open(self.filename, "w") as text_file: for output in command_output: text_file.write("Command: {0}\nOutput: {1}\n\n".format(output, str(command_output[output]).rstrip())) except OSError as error: self.logger.error("Error writing output file{0}".format(error.message))
def __init__( self ): super( Images, self ).__init__() self.images_file = 'data/images.json' self.images_folder = 'data/news/' self.dump_file = 'data/news/dump.json' if os.path.isfile( self.images_file ): images = helper.read_file( self.images_file, format='json' ) for index, image in enumerate(images, start=0): try: if not image['downloaded']: path = 'data/{image_path}'.format(image_path=image['new_path'].replace('https://static.weg.net/', '')) filename = os.path.basename( path ) folder = path.split('/') folder.pop() folder = '/'.join( folder ) base_url = 'http://www.weg.net' download_url = image['original_path'] if not os.path.isdir( folder ): os.makedirs(folder, exist_ok=True) if not download_url.startswith('http'): download_url = '{base_url}/{path}'.format(base_url=base_url, path=download_url) if helper.download(type='image', filename=path, nid=index, url=download_url): images[ index ]['downloaded'] = True log.success('Imagem baixada com sucesso [ {path} ]'.format(path=path)) else: log.warning('Imagem já baixada [ {url} ]'.format(url=image['new_path'])) except Exception as error: log.error( error ) finally: helper.create_file(self.images_file, images, mode='w', format='json') else: log.error('[!] Dump de imagens não existe')
def __init__( self ): super( Data, self ).__init__() self.news_list_file = 'data/notices.list' self.news_json_file = 'data/notices.json' self.dump_file = 'data/dump.json' self.proccess = os.getpid() self.errors = [] self.news_id_length = 4 init_message = 'Iniciando processo: {proccess}'.format(proccess=self.proccess) log.success( '=' * len( init_message ) ) log.success( init_message ) log.success( '=' * len( init_message ) ) print()
def __init__(self, parent=None): super(MyWin, self).__init__(parent) QWidget.__init__(self, parent) self.ui = Ui_IPSca() self.ui.setupUi(self) self.setWindowIcon(QIcon('ipsca.jpg')) self.logger = Logger() self.Scan = None self.IoTOnly = config.IoTOnly self.dead_counter = 0 self.successful_counter = 0 self.alive_counter = 0 self.report = {} self.ui.scan_btn.clicked.connect(self.click_scan) self.ui.import_btn.clicked.connect(self.file_open) self.ui.stop_btn.clicked.connect(self.click_stop) self.ui.shuffle_btn.clicked.connect(self.shuffle_hosts) self.ui.iot_chbx.stateChanged.connect(self.iot_only_change) self.ui.brute_chbx.stateChanged.connect(self.brute_enable_change) self.ui.submit_export.clicked.connect(self.export) self.scan = QThread()
class Printers(object): """ Printer checking object """ logger = Logger() executor = ExecuteCommand() def __init__(self): self.printers = {} def run(self): """ default run method """ self.logger.normal_output("Running Printers") self.get_printers() def get_printers(self): """ get any printers """ self.logger.normal_output("Grabbing printers") self.printers.update( {"printers lpstat": self.executor.execute_command("lpstat -a")})
def __init__(self, logfile, debugmode, global_XMS_IP, global_XMS_USER, global_XMS_PASS, clus_name=None): global rest_logger global XMS_IP global XMS_USERID global XMS_PASS self.clus_name = clus_name if debugmode == True: rest_logger = Logger(logfile, logging.DEBUG, logging.INFO) else: rest_logger = Logger(logfile, logging.INFO, logging.INFO) rest_logger.debug('Loading Restful Module') XMS_IP = global_XMS_IP XMS_USERID = global_XMS_USER XMS_PASS = global_XMS_PASS
def ScriptSet(log,array,user,passwd,scope,options): scriptset_logger = Logger(log,logging.DEBUG,logging.INFO) scriptset_logger.debug('Starting ScriptSet module.') for x in xrange(len(array)): cli = NaviCLI(log,array[x],user[x],passwd[x],scope[x]) var_array = array[x] var_user=user[x] var_passwd = passwd[x] var_scope = scope[x] bol_Analyzer = True mod = 0 scriptset_logger.info('************************************************') scriptset_logger.info('Getting Current Settings for '+var_array+' :') SerialNo,SNerrcode = cli._GetSerialNumber() if SNerrcode != '0': scriptset_logger.critical(SerialNo) else: scriptset_logger.info('Serial Number:\t'+SerialNo) NARInterval,NIerrcode = cli._GetNarInterval() SPcollectstat,SPstaterrcode = cli._GetSPCollectStatus() scriptset_logger.info('SPCollect AutoExecution:\t'+SPcollectstat) if SPcollectstat != 'Enabled': SPcollectAEstat,SPcollectAEerrcode = cli._SetSPcollectAE() scriptset_logger.info('Enabling SPCollect AutoExecution....') scriptset_logger.info('Archive Interval:\t'+NARInterval) RTInterval,RTerrcode = cli._GetRealtimeInterval() if RTerrcode != '0': bol_Analyzer = False scriptset_logger.info('**** Analyzer is not installed ****') else: scriptset_logger.info('Real Time Interval:\t'+RTInterval) LogDays,LDerrcode = cli._GetLogPeriod() scriptset_logger.info('Current Logging Period:\t'+LogDays) if bol_Analyzer is False: scriptset_logger.info('**** Skipping Periodic Archive Setting, Analyzer is not installed ****') else: PeriodicArchive,PAerrcode = cli._GetPeriodicArchive() scriptset_logger.info('Periodic Archiving:\t'+PeriodicArchive) if options['--ni'] != None: if NARInterval == options['--ni']: scriptset_logger.info('NAR Polling Interval is correctly set to:\t' + NARInterval) else: NARInterval = options['--ni'] scriptset_logger.info('****Modifying the Archive Poll interval to:\t' + NARInterval) mod = mod + 1 if LogDays != 'nonstop': if options['--ns'] == True: LogDays = 'nonstop' scriptset_logger.info('****Logging Period is being modified to:\t' + LogDays) mod = mod + 1 elif LogDays != '7': LogDays = '7' scriptset_logger.info('****Logging Period is being modified to:\t' + LogDays) mod = mod + 1 else: scriptset_logger.info('Logging Period is correctly set to:\t' + LogDays) else: scriptset_logger.info('Logging Period is correctly set to:\t' + LogDays) if bol_Analyzer is True: if PeriodicArchive != 'Yes': PeriodicArchive = 'Yes' scriptset_logger.info('****Periodic Archiving is Being Enabled') mod = mod + 1 else: scriptset_logger.info('Periodic Archiving is correctly set to:\t' +PeriodicArchive) else: scriptset_logger.info('Skipping Periodic Archiving Modification - No Analyzer') NaviStat = cli._GetAnalyzerStatus() if mod > 0: if 'Stopped' in NaviStat: scriptset_logger.debug('*****Analyzer Logging is NOT Running *****') else: StopLogStat,SLSerrcode = cli._StopLogging() time.sleep(10) if 'stopped' in StopLogStat: scriptset_logger.info('*****Analyzer Logging is Already Stopped *****') if bol_Analyzer is True: SetLogsStat,SetLogserrcode = cli._SetNaviAnalyzerOptionsLicensed(NARInterval,LogDays) if SetLogserrcode != '0': scriptset_logger.debug('*****Error Setting Analyzer Logging and Archiving Options *****') else: SetLogsStat,SetLogserrcode = cli._SetNaviAnalyzerOptions(NARInterval,LogDays) if SetLogserrcode != '0': scriptset_logger.debug('*****Error Setting Analyzer Logging and Archiving Options *****') StartLogStat,Starterrcode = cli._StartLogging() if Starterrcode != '0': time.sleep(5) scriptset_logger.debug('*****Error Starting Analyzer Logging *****') NaviStat = cli._GetAnalyzerStatus() if 'Stopped' in NaviStat: scriptset_logger.debug('*****Analyzer Logging is NOT Running *****') time.sleep(5) StartLogStat,Starterrcode = cli._StartLogging() if Starterrcode != '0': scriptset_logger.info('*****Error Starting Analyzer Logging *****') NARInterval, NIerrcode = cli._GetNarInterval() if bol_Analyzer is True: RTInterval,RTerrcode = cli._GetRealtimeInterval() PeriodicArchive,PAerrcode = cli._GetPeriodicArchive() else: RTInterval = '' PeriodicArchive = '' LogDays, LDerrcode = cli._GetLogPeriod() scriptset_logger.info('\tSerial Number:\t' + SerialNo) scriptset_logger.info('\tNAR Interval:\t' + NARInterval) scriptset_logger.info('\tRealTime Interval:\t' + RTInterval) scriptset_logger.info('\tLog Days:\t' + LogDays) scriptset_logger.info('\tPeriodic Archive:\t' + PeriodicArchive) else: scriptset_logger.info('No Need to Change Analyzer Options. Skipping....') NaviStat = cli._GetAnalyzerStatus() if 'Stopped' in NaviStat: scriptset_logger.debug('*****Analyzer Logging is NOT Running *****') StartLogStat,Starterrcode = cli._StartLogging() if Starterrcode != '0': scriptset_logger.debug('*****Error Starting Analyzer Logging *****') time.sleep(5) NaviStat = cli._GetAnalyzerStatus() if 'Stopped' in NaviStat: scriptset_logger.debug('*****Analyzer Logging is NOT Running *****') time.sleep(5) StartLogStat,Starterrcode = cli._StartLogging() if Starterrcode != '0': scriptset_logger.info('*****Error Starting Analyzer Logging *****') time.sleep(5) NaviStat = cli._GetAnalyzerStatus() scriptset_logger.info(var_array + ' Analyzer Status = ' + NaviStat) if 'Stopped' in NaviStat: scriptset_logger.info('*****Analyzer Logging is NOT Running *****') scriptset_logger.debug('Exiting def_ScriptSet module.') scriptset_logger.info('Script is finished setting options.')
def ScriptGet(log,array,user,passwd,scope,options): scriptget_logger = Logger(log,logging.DEBUG,logging.INFO) scriptget_logger.debug('Starting def_ScriptGet module.') timenow = datetime.datetime.now().hour + datetime.datetime.now().minute + datetime.datetime.now().second + datetime.datetime.now().microsecond for x in xrange(len(array)): cli = NaviCLI(log,array[x],user[x],passwd[x],scope[x]) var_array = array[x] var_user=user[x] var_passwd = passwd[x] var_scope = scope[x] bool_32plus = False scriptget_logger.info('************************************************') scriptget_logger.info('Getting Serial Number for ' + var_array + ' ....') SerialNo, SNerror = cli._GetSerialNumber() if SNerror != '0': scriptget_logger.critical(SerialNo) else: scriptget_logger.info('Getting OE Version for ' + var_array + ' ....') Version, Verror = cli._GetVersion() scriptget_logger.info(' Running OE Version ' + Version) if '7.32.' in Version: bool_32plus = True if '7.33.' in Version: bool_32plus = True outPath = ''.join([var_cwd,'\\',SerialNo]) if not os.path.exists(outPath): scriptget_logger.info('Creating Local Directory ' + outPath + ' ....') os.makedirs(outPath) if bool_32plus == True: skew_outpath = outPath+'_SkewData' scriptget_logger.info('Creating Directory for Skew Data ' + skew_outpath + ' ....') os.makedirs(skew_outpath) else: if os.path.isdir(outPath): scriptget_logger.info('Found Existing Folder: ' + outPath) outPath = outPath +'_'+str(date.today())+'_'+str(timenow) if not os.path.exists(outPath): scriptget_logger.info('Creating Local Directory ' + outPath + ' ....') os.makedirs(outPath) if bool_32plus == True: skew_outpath = outPath+'_SkewData' scriptget_logger.info('Creating Directory for Skew Data ' + skew_outpath + ' ....') os.makedirs(skew_outpath) scriptget_logger.info('Using ' + outPath + ' to save files for ' + SerialNo) scriptget_logger.info('Retrieving SPcollect for '+SerialNo +' ....') SPColStat = cli._GetSPCollectList(outPath) scriptget_logger.debug(SPColStat) l,narstring,oldnar,ol = cli._GetNARList2(SerialNo,options['--nnf']) scriptget_logger.info('Retrieving ' + str(l) + ' NAR/NAZ files from ' + SerialNo + ' ....') stat = cli._GetNarFiles(narstring,outPath) scriptget_logger.info(stat) if bool_32plus == True: scriptget_logger.info('Retrieving skew files from ' + SerialNo + ' ....') stat = cli._GetArrayconfigXML(skew_outpath) scriptget_logger.debug(stat) stat = cli._GetFASTData(skew_outpath) scriptget_logger.debug(stat) os.chdir(var_cwd) scriptget_logger.info('**** File Retrieval Complete for ' + SerialNo + ' ****') scriptget_logger.info('**** Creating zip of Data for ' + SerialNo + ' ****') shutil.make_archive(outPath, 'zip', outPath) if bool_32plus == True: shutil.make_archive(skew_outpath, 'zip', skew_outpath) scriptget_logger.info('**** Cleaning Up Local Directory ****') shutil.rmtree(outPath, ignore_errors=True) if bool_32plus == True: shutil.rmtree(skew_outpath, ignore_errors=True) if options['--c'] == True: scriptget_logger.info('**** Cleaning Up '+str(ol)+' NAR/NAZ files from '+SerialNo+' ****') cleanupstat = cli._CleanupOldNar(oldnar) scriptget_logger.debug(cleanupstat) scriptget_logger.debug('Exiting def_ScriptGet module.') scriptget_logger.debug('Script is finished retrieving files.')
# coding: utf-8 import csv import os import numpy as np import pandas as pd from modules.logger import Logger, DEBUG, INFO, WARNING, ERROR, CRITICAL from modules.exceptions import ColumnNotFoundException logfile = os.path.join(Logger.get_logfile_dir(__file__), '..', 'logs', 'datareader.log') _ = Logger.create(__file__, logfile, DEBUG) _.addHandler(Logger.create_file_handler(logfile, DEBUG)) _.addHandler(Logger.create_stream_handler(WARNING)) class DataReader(object): """ Reads the CSV data file then cleans it """ def __init__(self, fname, delim=","): self.delim = delim self.header = [] self.data = [] self.rows = 0 self.cols = 0 _.debug("Reading %s" % fname) with open(fname, 'r') as rf:
def OnlyCleanupOldNar(log, array, user, password, scope, options): onlycleannarfiles_logger = Logger(log, logging.DEBUG, logging.INFO) onlycleannarfiles_logger.debug('Staring def_OnlyCleanupOldNar module.') for x in xrange(len(array)): cli = NaviCLI(log, array[x], user[x], passwd[x], scope[x]) var_array = array[x] var_user = user[x] var_passwd = passwd[x] var_scope = scope[x] onlycleannarfiles_logger.info( '************************************************') onlycleannarfiles_logger.info('Getting Serial Number for ' + var_array + ' ....') ## SerialNo,SNerror = def_GetSerialNumber(var_array,var_user,var_passwd,var_scope) SerialNo, SNerror = cli._GetSerialNumber() if SNerror != '0': onlycleannarfiles_logger.info(SerialNo) else: ## l,narstring,oldnar,ol = def_GetNARList2(SerialNo,var_array,var_user,var_passwd,var_scope) l, narstring, oldnar, ol = cli._GetNARList2( SerialNo, options['--nnf']) onlycleannarfiles_logger.info( '**** Cleaning Up ' + str(ol) + ' NAR/NAZ files, older than 7 days, from ' + SerialNo + ' ****') ## def_CleanupOldNar(var_array,var_user,var_passwd,var_scope,oldnar) cli._CleanupOldNar(oldnar)
def get_content( news, content ): if not content[0]: return '' allowed_images_extension = ['.jpeg', '.jpg', '.png', '.gif', '.bmp', '.tif'] document = BeautifulSoup( content[0].encode('utf-8'), 'html.parser' ) to_remove = ['comparison', 'bgdark', 'bglight', 'default', 'clr', 'novaJanela'] link = news['link'] catalog = news['catalog'] nid = news['id'] for item in to_remove: if document.select('.{selector}'.format(selector=item)): for element in document.select('.{selector}'.format(selector=item)): index = element['class'].index( item ) del element['class'][ index ] if document.select('.center'): for center in document.select('.center'): center['class'] = 'text-center' if document.select('p'): paragraphs = document.select('p') for paragraph in paragraphs: for content in paragraph.contents: if content == '\xa0' or not content: paragraph.decompose() if document.select('table'): tables = document.select('table') tablefilename = 'logs/weg/tables.list' link = link if isinstance( link, str ) else link.attrs['href'] table_log = '[ {nid} ]: {link}\n'.format(link=link, nid=nid) for table in tables: to_remove = ['cellpadding', 'border', 'cellspacing', 'width', 'height'] responsive = document.new_tag('div') responsive['class'] = 'table-responsive' table.wrap( responsive ) table['class'].append('table table-bordered table-hover') for item in to_remove: del table[ item ] if os.path.isfile( tablefilename ): content = helper.read_file( tablefilename ) if link not in content: helper.create_file(tablefilename, table_log) else: log.warning('Tabela já adicionada para a lista [ {url} ]'.format(url=link)) else: helper.create_file(tablefilename, table_log) log.success('Log de tabelas criado.') if document.select('a'): for index, link in enumerate( document.select('a'), start=0 ): if 'href' in link.attrs: filename, file_extension = os.path.splitext( link.attrs['href'] ) if link.attrs['href'] == 'javascript:void();': link.attrs['href'] = '#{nid}'.format(nid=news['id']) link.attrs['data-prevent-default'] = 'true' if file_extension in allowed_images_extension: set_image( news, index, link.attrs['href'] ) link.attrs['href'] = set_image_link( news, index, link.attrs['href'] ) if document.select('img'): for index, image in enumerate( document.select('img'), start=0 ): filename, file_extension = os.path.splitext( image.attrs['src'] ) responsive = True if file_extension in allowed_images_extension: set_image( news, index, image.attrs['src'] ) image.attrs['src'] = set_image_link( news, index, image.attrs['src'] ) # for parent in image.parents: # if 'class' in parent.attrs: # if 'coluna6' in parent.attrs['class']: # responsive = False # if responsive: # if 'class' in image.attrs: # image.attrs['class'].append('img-responsive') # else: # image.attrs['class'] = 'img-responsive' if document.select('.coluna6'): columns = document.select('.coluna6') for column in columns: column['class'] = 'xtt-gallery pull-right' if document.select('ul'): for ul in document.select('ul'): ul['class'] = 'xtt-list-style' for li in ul.select('> li'): span = document.new_tag('span') span.string = li.contents[0] li.string = '' li.append( span ) return str( document ).strip()
class App(object): def __init__(self): self.logger = Logger() user = os.getenv('VC_USER') password = os.getenv('VC_PASS') if not self.should_run(user, password): self.logger.error('User or password is not defined, exiting...') sys.exit() self.sandboxes = Sandboxes(user, password) self.applications = Applications(user, password) self.builds = Builds(user, password) self.spreadsheet_creator = SpreadsheetCreator() apps = self.applications.get_apps() self.get_app_sandboxes(apps) self.get_app_sandbox_builds(apps) self.get_builds(apps) self.create_spreadsheet(apps) self.send_message() def should_run(self, user, password): return user != None and password != None and os.getenv( 'VC_ATTACHMENT_BASE_PATH') != None and os.getenv( 'VC_ADMIN_ADDRESS') != None and os.getenv( 'VC_SMTP') != None and os.getenv( 'VC_SEND_ADDRESS') != None and os.getenv( 'VC_RECIPIENTS') != None def get_app_sandboxes(self, apps): for app in apps: app['sandboxes'] = self.sandboxes.get_sandboxes(app['app_id']) def get_app_sandbox_builds(self, apps): for app in apps: for sandbox in app['sandboxes']: sandbox['builds'] = self.builds.get_builds( app['app_id'], sandbox['sandbox_id']) def get_builds(self, apps): for app in apps: for sandbox in app['sandboxes']: for build in sandbox['builds']: build['submitter'] = self.builds.get_build( app['app_id'], build['build_id']) def create_spreadsheet(self, apps): self.spreadsheet_creator.write_spreadsheet(apps) def send_message(self): if self.spreadsheet_creator.have_data: Mailer( os.getenv('VC_SEND_ADDRESS'), os.getenv('VC_RECIPIENTS'), constants.MESSAGE_SUBJECT, os.getenv('VC_SMTP'), 25, constants.BUILDS_FOUND_MESSAGE_TEXT.format( os.getenv('VC_ADMIN_ADDRESS')), self.spreadsheet_creator.filename) else: Mailer( os.getenv('VC_SEND_ADDRESS'), os.getenv('VC_RECIPIENTS'), constants.MESSAGE_SUBJECT, os.getenv('VC_SMTP'), 25, constants.BUILDS_NOT_FOUND_MESSAGE_TEXT.format( os.getenv('VC_ADMIN_ADDRESS')))
class Services(object): """ Object for service enumeration """ logger = Logger() executor = ExecuteCommand() def __init__(self): self.services = {} self.services_as_root = {} self.installed_applications = {} self.plaintext_passwords = {} self.scheduled_jobs = {} self.service_configurations = {} def run(self): """ generic run method """ self.logger.normal_output("Running Service Checks") self.get_services() self.get_services_as_root() self.get_installed_applications() self.get_for_plaintext_passwords() self.get_scheduled_jobs() self.get_service_configurations() def get_services(self): """ get services """ self.logger.normal_output("Running services") self.services.update( {"ps aux": self.executor.execute_command("ps aux")}) self.services.update( {"ps aux -ef": self.executor.execute_command("ps -ef")}) # self.services.update({"top": self.executor.execute_command("top")}) self.services.update({ "cat /etc/services": self.executor.execute_command("cat /etc/services") }) def get_services_as_root(self): """ get root services """ self.logger.normal_output("Running services as root") self.services_as_root.update({ "ps aux |grep root": self.executor.execute_command("ps aux | grep root") }) self.services_as_root.update({ "ps -ef | grep root": self.executor.execute_command("ps -ef | grep root") }) def get_installed_applications(self): """ get installed applications """ self.logger.normal_output("Running installed apps") self.installed_applications.update({ "ls -alh /usr/bin": self.executor.execute_command("ls -alh /usr/bin/") }) self.installed_applications.update({ "ls -alh /sbin/": self.executor.execute_command("ls -alh /sbin/") }) self.installed_applications.update( {"dpkg -l": self.executor.execute_command("dpkg -l")}) self.installed_applications.update( {"rpm -qa": self.executor.execute_command("rpm -qa")}) self.installed_applications.update({ "ls -alh /var/cache/apt/archives0": self.executor.execute_command("ls -alh /var/cache/apt/archivesO") }) self.installed_applications.update({ "ls -alh /var/cache/yum": self.executor.execute_command("ls -alh /var/cache/yum/") }) def get_service_configurations(self): """ get service configurations """ self.logger.normal_output("Running service configs") self.service_configurations.update( {"syslog": self.executor.execute_command("cat /etc/syslog.conf")}) self.service_configurations.update( {"chhtp": self.executor.execute_command("cat /etc/chttp.conf")}) self.service_configurations.update({ "lighthttpd": self.executor.execute_command("cat /etc/lighttpd.conf") }) self.service_configurations.update({ "cupsd": self.executor.execute_command("cat /etc/cups/cupsd.conf") }) self.service_configurations.update( {"inetd": self.executor.execute_command("cat /etc/inetd.conf")}) self.service_configurations.update({ "apache2": self.executor.execute_command("cat /etc/apache2/apache2.conf") }) self.service_configurations.update( {"my.conf": self.executor.execute_command("cat /etc/my.conf")}) self.service_configurations.update({ "httpd": self.executor.execute_command("cat /etc/httpd/conf/httpd.conf") }) self.service_configurations.update({ "lamphttpd": self.executor.execute_command("cat /opt/lampp/etc/httpd.conf") }) self.service_configurations.update({ "/etc/": self.executor.execute_command("ls -aRl /etc/ | awk '$1 ~ /^.*r.*/") }) def get_scheduled_jobs(self): """ get scheduled jobs """ self.logger.normal_output("Running scheduled jobs") self.scheduled_jobs.update( {"crontab -l": self.executor.execute_command("crontab -l")}) self.scheduled_jobs.update({ "ls alh cron": self.executor.execute_command("ls -alh /var/spool/cron") }) self.scheduled_jobs.update({ "/etc cron": self.executor.execute_command("ls -al /etc/ | grep cron") }) self.scheduled_jobs.update({ "ls -al /etc/cron*": self.executor.execute_command("ls -al /etc/cron*") }) self.scheduled_jobs.update( {"etc/cron": self.executor.execute_command("cat /etc/cron*")}) self.scheduled_jobs.update({ "etc/at.allow": self.executor.execute_command("cat /etc/at.allow") }) self.scheduled_jobs.update( {"etc/at.deny": self.executor.execute_command("cat /etc/at.deny")}) self.scheduled_jobs.update({ "etc/cron.allow": self.executor.execute_command("cat /etc/cron.allow") }) self.scheduled_jobs.update({ "etc/cron.deny": self.executor.execute_command("cat /etc/cron.deny") }) self.scheduled_jobs.update( {"crontab": self.executor.execute_command("cat /etc/crontab")}) self.scheduled_jobs.update( {"acrontab": self.executor.execute_command("cat /etc/anacrontab")}) self.scheduled_jobs.update({ "rootcrontab": self.executor.execute_command("cat /var/spool/cron/crontabs/root") }) def get_for_plaintext_passwords(self): """ check for different config files containing passwords """ self.logger.normal_output("Running plaintext passwords and users") self.plaintext_passwords.update({ "grep for user": self.executor.execute_command("grep -i user /*") }) self.plaintext_passwords.update({ "grep for pass": self.executor.execute_command("grep -i pass /*") }) self.plaintext_passwords.update({ "grep -c 5": self.executor.execute_command("grep -C 5 \"password\" /*") }) self.plaintext_passwords.update({ "find php $password": self.executor.execute_command( "find . -name \"*.php\" -print0 | xargs -0 grep -i -n \"var $password\"" # Joomla ) })
from modules.logger import Logger logger = Logger() def grouping(value, format_option): try: numeric_value = float(value.replace(",", ".")) matching_threshold = None group_thresholds = format_option["thresholds"] largest_threshold = max(group_thresholds) digits = len(str(largest_threshold)) for threshold in group_thresholds: if numeric_value <= threshold: matching_threshold = threshold break if matching_threshold != None: indicator = "bis" else: indicator = "über" matching_threshold = largest_threshold unit = format_option["unit"] return "{} {}{}".format(indicator, str(matching_threshold).zfill(digits), unit) except: logger.log( "Der Wert '{}' kann nicht gruppiert werden, da er nicht numerisch ist." .format(value)) return value
log.error(error) pass finally: helper.create_file( filename=self.dump_file, content=self.news, format='json', mode='w') helper.create_file( filename=self.news_json_file, content=self.news_list, format='json', mode='w') if __name__ == '__main__': scrapper = Scrapper() try: scrapper.start() except Exception as error: print() error_message = 'Erro ao iniciar processo: {proccess}'.format(proccess=scrapper.proccess) log.error('=' * len( error_message )) log.error( error_message ) log.error(error) log.error('=' * len( error_message )) print() finally: finished_with_errors = 'Finalizado com {errors} erro{suffix}'.format(errors=len( scrapper.errors ), suffix='s' if len( scrapper.errors ) > 1 else '') finished_without_errors = 'Finalizado sem erros' if scrapper.errors: print() log.warning( '=' * len( finished_with_errors ) ) log.warning( finished_with_errors ) else: print() log.success( '=' * len( finished_without_errors ) )
def __init__(self): self.have_data = False self.logger = Logger() self.filename = '{}VeracodeSandboxResults-{}.xlsx'.format( os.getenv('VC_ATTACHMENT_BASE_PATH'), date.today()) return
def main(): log = options['--l'] if options['--debug'] == True: main_logger = Logger( log, logging.DEBUG, logging.INFO ) else: main_logger = Logger( log, logging.INFO, logging.INFO ) rd_csv = ReadCSV( log, options ) if options['--set'] == True: main_logger.debug( 'Set argument is set' ) ScriptSet( log, rd_csv.host, rd_csv.uname, rd_csv.passwd, rd_csv.scope, options ) if options['--get'] == True: main_logger.debug( 'Get argument is set' ) ScriptGet( log, rd_csv.host, rd_csv.uname, rd_csv.passwd, rd_csv.scope, options ) if options['--c'] == True: main_logger.debug( 'Cleanup argument is set with the Get argument' ) if options['--get'] == False: main_logger.debug( 'Cleanup argument is set by itself' ) OnlyCleanupOldNar( log, rd_csv.host, rd_csv.uname, rd_csv.passwd, rd_csv.scope, options ) main_logger.debug( 'Exiting.' ) main_logger.info( 'Script Complete.' ) sys.exit(0)
# pip install MySQL-python sqlalchemy import sys import time import ConfigParser from modules.config_reader import ConfigReader from modules.stock import Stock from modules.mysqlclient import MySQLClient from modules.logger import Logger CONF_FILE_PATH = 'config/stock.conf' if __name__ == '__main__': log = Logger.get_instance() log.info('------------------ START TO FETCH STOCK HISTORY ------------------') try: conf = ConfigReader(ConfigParser.ConfigParser(), CONF_FILE_PATH) except Exception as e: print e log.error('Failed to read config file: %s', e) sys.exit(1) db = MySQLClient(conf.user, conf.password, conf.host, conf.dbname) db.create_tables() Stock(None, from_date=conf.from_date, db=db, log=log)
def __init__(self,logfile,options): global options_logger if options['--debug'] == True: options_logger = Logger(logfile,logging.DEBUG,logging.INFO) else: options_logger = Logger(logfile,logging.INFO,logging.INFO) options_logger.debug('Loading Options Module') encoder = Encode(options['--l'],options['--debug']) if options['--encode'] is True: options_logger.info('Encoding user id and password') encode_user = encoder._encodeuser(options['XMS_USER']) encode_pass = encoder._encodepass(options['XMS_PASS']) print '' print 'Encoded User ID = ' + encode_user print 'Encoded Password = '******'' options_logger.info('Use the above, encoded, user id and password with the --e option') options_logger.info('to execute the tool without using the plain text usename and password') sys.exit(0) elif options['--e']: options_logger.debug('Using an encoded username and password') XMS_USER = encoder._decodeuser(options['XMS_USER']) XMS_PASS = encoder._decodepass(options['XMS_PASS']) else: options_logger.debug('Username and password are not encoded') XMS_USER = options['XMS_USER'] XMS_PASS = options['XMS_PASS'] self.XMS_USER = XMS_USER self.XMS_PASS = XMS_PASS if options['--schedule'] == 'hourly': options_logger.debug('Using the hourly schedule') self.schedule = 'hourly' elif options['--schedule'] == 'daily': options_logger.debug('Using the daily schedule') self.schedule = 'daily' elif options['--schedule'] == 'weekly': options_logger.debug('Using the weekly schedule') self.schedule = 'weekly' else: options_logger.critical('No schedule, or incorrect option Specified. Exiting...') sys.exit(1) SnapFolder = '_Snapshots' self.var_snap_tgt_folder = options['--tf'] ## this should be an optional variable, if the customer wants to organize snapshots under a folder hierarchy if self.var_snap_tgt_folder == None: options_logger.debug('No snapshot target folder specified') self.var_snap_tgt_folder ='' self.snap_tgt_folder = SnapFolder options_logger.debug('Using '+self.snap_tgt_folder+' as the snapshot target') else: self.snap_tgt_folder = self.var_snap_tgt_folder+'/'+SnapFolder options_logger.debug('Using '+self.snap_tgt_folder+' as the snapshot target')
def __init__(self,logfile,options): global csv_logger csvfile = options['--csv'] csv_logger = Logger(logfile,logging.DEBUG,logging.INFO) csv_logger.debug('Loading ReadCSV Module') csv_logger.debug(csvfile) arr_host = [] arr_uname = [] arr_passwd = [] arr_scope = [] result = _FileCheck(csvfile) if result is 0: if options['LDAP_USER'] != None: f = open(csvfile, 'r') r = csv.reader(f) for row in _skip_blank(r): arr_host.append(row[0]) arr_uname.append(options['LDAP_USER']) arr_passwd.append(options['LDAP_PASS']) arr_scope.append('2') else: try: f = open(csvfile, 'r') r = csv.reader(f) for row in _skip_blank(r): arr_host.append(row[0]) arr_uname.append(row[1]) arr_passwd.append(row[2]) arr_scope.append(row[3]) except: csv_logger.critical('There was an error reading the username and/or password info, is it specified in the csv?') csv_logger.critical('exiting...') sys.exit(1) self.uname = arr_uname self.passwd = arr_passwd self.host = arr_host self.scope = arr_scope csv_logger.debug('Exiting ReadCSV module.') else: sys.exit(1)
def main(): log = options['--l'] if options['--debug'] == True: main_logger = Logger( options['--l'], logging.DEBUG, logging.INFO ) else: main_logger = Logger( options['--l'], logging.INFO, logging.INFO ) main_logger.info( 'Running XtremIO Snap Script' ) main_options = Options( log, options ) XMS_IP = options['XMS_IP'] XMS_USER = main_options.XMS_USER XMS_PASS = main_options.XMS_PASS snap_tgt_folder = main_options.snap_tgt_folder parent_folder_id = main_options.var_snap_tgt_folder num_snaps = options['--n'] snap_src = options['--snap'] SnapFolder = '_Snapshots' bool_create_folder = True ## will change to False if the /_Snapshots folder already exists rest = Restful( log, options['--debug'], XMS_IP, XMS_USER, XMS_PASS ) folder_list = rest._get( '/api/json/types/volume-folders' ).json()['folders'] for folder_list_rs in folder_list: if folder_list_rs['name'] == '/'+ snap_tgt_folder: bool_create_folder = False main_logger.info( 'The target snapshot folder, '+folder_list_rs['name']+' already exists.' ) if bool_create_folder is True: cf_payload = '{\ "caption": \"'+SnapFolder+'\" , \ "parent-folder-id": \"/'+main_options.var_snap_tgt_folder+'\" \ }' cf_resp = rest._post( '/api/json/types/volume-folders', cf_payload ) if cf_resp.status_code == 201: main_logger.warn( 'Created folder: '+main_options.snap_tgt_folder ) if options['--f'] ==True: newsnapsuffix = '.folder.'+main_options.schedule+'.' folder_vol_list = rest._get( '/api/json/types/volume-folders/?name=/'+snap_src ).json()['content']['direct-list'] arr_folder_vol_list_component = [] for folder_vol_list_rs in folder_vol_list: if '/_Snapshots' in folder_vol_list_rs[1]: pass else: arr_folder_vol_list_component.append(folder_vol_list_rs[1]) main_logger.info( 'Will retain '+num_snaps+' snapshots for volume: '+folder_vol_list_rs [1] ) vol_snap_list = rest._get( '/api/json/types/volumes/?name='+folder_vol_list_rs [1] ).json()['content']['dest-snap-list']##<--Initial list of snapshots arr_vol_snap_list_component = [] for vol_snap_list_rs in vol_snap_list: if newsnapsuffix in vol_snap_list_rs[1]: arr_vol_snap_list_component.append(vol_snap_list_rs[1]) arr_vol_snap_list_component.sort(reverse=True) for y in range(len(arr_vol_snap_list_component)): ##<--shifting the suffix of each matchin snap by 1 if newsnapsuffix in arr_vol_snap_list_component[y]: list_snapname = [] try: list_snapname = arr_vol_snap_list_component[y].split('.',3) except: pass rename_to =list_snapname[0]+newsnapsuffix+str(y+1) rename_payload = '{"vol-name": \"'+rename_to+'\"}' rename_resp = rest._put( '/api/json/types/volumes/?name='+arr_vol_snap_list_component[y], rename_payload ) if rename_resp.status_code == 200: main_logger.info( 'Snapshot: '+arr_vol_snap_list_component[y]+' was renamed to '+rename_to ) timestamp = datetime.datetime.now() timestamp = timestamp.isoformat() arr_timestamp = timestamp.split('.',2) ##<--stripping the microseconds from the timestamp for aesthetics fullsuffix = '_'+arr_timestamp[0]+newsnapsuffix+'0' fs_payload = '{\ "source-folder-id": \"/'+snap_src+'\" , \ "suffix": \"'+fullsuffix+'\" ,\ "folder-id": \"/'+snap_tgt_folder+'\" \ }' vol_snap = rest._post( '/api/json/types/snapshots', fs_payload ) folder_vol_list = rest._get( '/api/json/types/volume-folders/?name=/'+snap_src ).json()['content']['direct-list'] arr_folder_vol_list_component = [] for folder_vol_list_rs in folder_vol_list: if '/_Snapshots' in folder_vol_list_rs[1]: pass else: vol_snap_list = rest._get( '/api/json/types/volumes/?name='+folder_vol_list_rs [1] ).json()['content']['dest-snap-list']##<--Refresh the snap list arr_vol_snap_list_component = [] for vol_snap_list_rs in vol_snap_list: if newsnapsuffix in vol_snap_list_rs[1]: arr_vol_snap_list_component.append(vol_snap_list_rs[1]) arr_vol_snap_list_component.sort(reverse=False) for x in xrange(len(arr_vol_snap_list_component)-(int(num_snaps))): if newsnapsuffix in arr_vol_snap_list_component[x]: main_logger.debug( str(x)+': '+ arr_vol_snap_list_component[x] ) get_snap_details = rest._get( '/api/json/types/snapshots/?name='+arr_vol_snap_list_component[x] ) arr_ancestor_vol_id = get_snap_details.json()['content']['ancestor-vol-id'] snap_parent_name = arr_ancestor_vol_id[1] snap_creation_time = get_snap_details.json()['content']['creation-time'] snap_space_consumed = get_snap_details.json()['content']['logical-space-in-use'] arr_snap_lun_mapping = get_snap_details.json()['content']['lun-mapping-list'] main_logger.warn( 'Parent Volume of '+arr_vol_snap_list_component[x]+' = '+snap_parent_name ) main_logger.warn( 'Snapshot: ' +arr_vol_snap_list_component[x] ) main_logger.warn( ' Snap was created on '+snap_creation_time ) main_logger.warn( ' Snap is using '+ str((float(snap_space_consumed)/1024)/1024)+' GB' ) arr_lun_mapping_component = [] if len(arr_snap_lun_mapping) > 0:##<--checking to see if an active LUN mapping exists for rs in arr_snap_lun_mapping: arr_lun_mapping_component = [[y] for y in rs[0]] arr_lun_mapping_component =str(arr_lun_mapping_component[1]) arr_lun_mapping_component = arr_lun_mapping_component.replace('[u\'','') arr_lun_mapping_component = arr_lun_mapping_component.replace('\']','') main_logger.critical( 'Snapshot: '+arr_vol_snap_list_component[x]+' is currently mapped to '+arr_lun_mapping_component+', it will not be deleted.' ) else: main_logger.warn( ' No hosts mapped to '+arr_vol_snap_list_component[x]+', it will be deleted.' ) delete_status = rest._delete( '/api/json/types/volumes/?name='+arr_vol_snap_list_component[x] ) elif options['--v'] ==True: main_logger.info( 'Will retain '+num_snaps+' snapshots for volume: '+snap_src ) newsnapsuffix = '.'+main_options.schedule+'.' vol_snap_list = rest._get( '/api/json/types/volumes/?name='+snap_src ).json()['content']['dest-snap-list'] arr_vol_snap_list_component = [] for vol_snap_list_rs in vol_snap_list: if newsnapsuffix in vol_snap_list_rs[1]: if '.folder.' in vol_snap_list_rs[1]: pass else: arr_vol_snap_list_component.append(vol_snap_list_rs[1]) arr_vol_snap_list_component.sort(reverse=True) for y in range(len(arr_vol_snap_list_component)): ##<--shifting the suffix of each matchin snap by 1 if newsnapsuffix in arr_vol_snap_list_component[y]: list_snapname = [] try: list_snapname = arr_vol_snap_list_component[y].split('.',3) except: pass rename_to =list_snapname[0]+newsnapsuffix+str(y+1) rename_payload = '{"vol-name": \"'+rename_to+'\"}' rename_resp = rest._put( '/api/json/types/volumes/?name='+arr_vol_snap_list_component[y], rename_payload ) if rename_resp.status_code == 200: main_logger.info( 'Snapshot: '+arr_vol_snap_list_component[y]+' was renamed to '+rename_to ) timestamp = datetime.datetime.now() timestamp = timestamp.isoformat() arr_timestamp = timestamp.split('.',2) ##<--stripping the microseconds from the timestamp for aesthetics fullsuffix = '_'+arr_timestamp[0]+newsnapsuffix newsnap = snap_src+fullsuffix+'0'##<--sets the newly created snapshot to always be .0 vol_snap_payload = '{\ "ancestor-vol-id": \"'+snap_src+'\" , \ "snap-vol-name": \"'+newsnap+'\" ,\ "folder-id": \"/'+snap_tgt_folder+'\" \ }' vol_snap_resp = rest._post( '/api/json/types/snapshots', vol_snap_payload ) vol_snap_list = rest._get( '/api/json/types/volumes/?name='+snap_src ).json()['content']['dest-snap-list'] arr_vol_snap_list_component = [] for vol_snap_list_rs in vol_snap_list: if newsnapsuffix in vol_snap_list_rs[1]: if '.folder.' in vol_snap_list_rs[1]: pass else: arr_vol_snap_list_component.append(vol_snap_list_rs[1]) arr_vol_snap_list_component.sort(reverse=False) for x in xrange(len(arr_vol_snap_list_component)-(int(num_snaps))): if newsnapsuffix in arr_vol_snap_list_component[x]: main_logger.debug(str(x)+': '+ arr_vol_snap_list_component[x]) get_snap_details = rest._get( '/api/json/types/snapshots/?name='+arr_vol_snap_list_component[x] ) arr_ancestor_vol_id = get_snap_details.json()['content']['ancestor-vol-id'] snap_parent_name = arr_ancestor_vol_id[1] snap_creation_time = get_snap_details.json()['content']['creation-time'] snap_space_consumed = get_snap_details.json()['content']['logical-space-in-use'] arr_snap_lun_mapping = get_snap_details.json()['content']['lun-mapping-list'] main_logger.warn( 'Parent Volume of '+arr_vol_snap_list_component[x]+' = '+snap_parent_name ) main_logger.warn( 'Snapshot: '+arr_vol_snap_list_component[x] ) main_logger.warn( ' Snap was created on '+snap_creation_time ) main_logger.warn( ' Snap is using '+ str((float(snap_space_consumed)/1024)/1024)+' GB' ) arr_lun_mapping_component = [] if len(arr_snap_lun_mapping) > 0:##<--checking to see if an active LUN mapping exists for rs in arr_snap_lun_mapping: arr_lun_mapping_component = [[y] for y in rs[0]] arr_lun_mapping_component =str(arr_lun_mapping_component[1]) arr_lun_mapping_component = arr_lun_mapping_component.replace('[u\'','') arr_lun_mapping_component = arr_lun_mapping_component.replace('\']','') main_logger.critical( 'Snapshot '+arr_vol_snap_list_component[x]+' is currently mapped to '+arr_lun_mapping_component+', it will not be deleted.' ) else: main_logger.warn( ' No hosts mapped to '+arr_vol_snap_list_component[x]+', it will be deleted.' ) delete_status = rest._delete( '/api/json/types/volumes/?name='+arr_vol_snap_list_component[x] ) else: print 'NO FOLDER OR VOLUME OPTION SPECIFIED' sys.exit(1) main_logger.info('Complete!') sys.exit(0)