def read_obs_tabular(gnss: str, logger: logging.Logger) -> pd.DataFrame: """ read_obs_tabular reads the observation data into a dataframe """ cFuncName = colored(os.path.basename(__file__), 'yellow') + ' - ' + colored( sys._getframe().f_code.co_name, 'green') # check that th erequested OBSTAB file is present gnss_obstab = os.path.join(amc.dRTK['options']['rnx_dir'], 'gfzrnx', amc.dRTK['json']['rnx']['gnss'][gnss]['marker'], amc.dRTK['json']['rnx']['gnss'][gnss]['obstab']) print('gnss_obstab = {!s}'.format(gnss_obstab)) # df = pd.read_csv('gnss_obstab') logger.info( '{func:s}: reading observation tabular file {obstab:s} (be patient)'. format(obstab=colored(gnss_obstab, 'green'), func=cFuncName)) try: df = pd.read_csv(gnss_obstab, parse_dates=[['DATE', 'TIME']], delim_whitespace=True) except FileNotFoundError as e: logger.critical('{func:s}: Error = {err!s}'.format(err=e, func=cFuncName)) sys.exit(amc.E_FILE_NOT_EXIST) return df
def fatal_error_exit_or_backtrace(err: Exception, msg: str, logger: logging.Logger, *logargs, **logkwargs): """ standard handling of fatal errors. Logs a critical error, then, if debug mode is enabled, rethrows the error (to get a backtrace or debug), and if not, exits the program with return code 1 arguments: err: the exception that caused this situation. Can be None, in which case will not be re-raised msg: the message you want to log logger: the logger to log to. Can be None, in which case a default logger will be obtained logargs, logkwargs: arguments to pass on to the logging function """ if logger is None: logger = logging.getLogger("pushfish-api") logger.critical(msg, *logargs, **logkwargs) logger.critical("exiting...") if Config.GLOBAL_BACKTRACE_ENABLE: if err is not None: raise err sys.exit(1)
def getAuthToken(tracer: logging.Logger, resource: Optional[str] = None, msiClientId: Optional[str] = None, msiResourceId: Optional[str] = None) -> Tuple[str, str]: tracer.info("getting auth token for resource=%s%s" % (resource, ", msiClientId=%s" % msiClientId if msiClientId else "")) authToken = None if not resource: resource = AzureInstanceMetadataService.resource requestParams = {"resource": resource} if (msiResourceId): requestParams['mi_res_id'] = msiResourceId if (msiClientId): requestParams['client_id'] = msiClientId try: result = AzureInstanceMetadataService._sendRequest( tracer, "identity/oauth2/token", params=requestParams) authToken, msiClientId = result["access_token"], result[ "client_id"] except Exception as e: tracer.critical("could not get auth token (%s)" % e) sys.exit(ERROR_GETTING_AUTH_TOKEN) return authToken, msiClientId
def test_logger_propagation(logger: logging.Logger): msg = f"TESTING %s log with {logger}" logger.critical(msg, "critical") logger.error(msg, "error") logger.info(msg, "info") logger.warning(msg, "warning") logger.debug(msg, "debug")
def attempt_database_upgrade(self, oLogHandler=None): """Attempt to upgrade the database, going via a temporary memory copy.""" oTempConn = connectionForURI("sqlite:///:memory:") oLogger = Logger('attempt upgrade') if oLogHandler: oLogger.addHandler(oLogHandler) (bOK, aMessages) = self.create_memory_copy(oTempConn, oLogHandler) if bOK: oLogger.info("Copied database to memory, performing upgrade.") if aMessages: oLogger.info("Messages reported: %s", aMessages) (bOK, aMessages) = self.create_final_copy(oTempConn, oLogHandler) if bOK: oLogger.info("Everything seems to have gone OK") if aMessages: oLogger.info("Messages reported %s", aMessages) return True oLogger.critical("Unable to perform upgrade.") if aMessages: oLogger.error("Errors reported: %s", aMessages) oLogger.critical("!!YOUR DATABASE MAY BE CORRUPTED!!") else: oLogger.error( "Unable to create memory copy. Database not upgraded.") if aMessages: oLogger.error("Errors reported %s", aMessages) return False
def attempt_database_upgrade(oLogHandler=None): """Attempt to upgrade the database, going via a temporary memory copy.""" oTempConn = connectionForURI("sqlite:///:memory:") oLogger = Logger('attempt upgrade') if oLogHandler: oLogger.addHandler(oLogHandler) (bOK, aMessages) = create_memory_copy(oTempConn, oLogHandler) if bOK: oLogger.info("Copied database to memory, performing upgrade.") if len(aMessages) > 0: oLogger.info("Messages reported: %s", aMessages) (bOK, aMessages) = create_final_copy(oTempConn, oLogHandler) if bOK: oLogger.info("Everything seems to have gone OK") if len(aMessages) > 0: oLogger.info("Messages reported %s", aMessages) return True else: oLogger.critical("Unable to perform upgrade.") if len(aMessages) > 0: oLogger.error("Errors reported: %s", aMessages) oLogger.critical("!!YOUR DATABASE MAY BE CORRUPTED!!") else: oLogger.error("Unable to create memory copy. Database not upgraded.") if len(aMessages) > 0: oLogger.error("Errors reported %s", aMessages) return False
def write_message(logger: logging.Logger) -> None: """ help method to write test messages to specified logger """ logger.debug('debug message') logger.info('info message') logger.warning('warn message') logger.error('error message') logger.critical('critical message')
def download_prices_from_s3(bucket: ServiceResource, dir_prices: Path, remote_dir_prices: Path, missing_rics: List[str], logger: logging.Logger) -> None: dir_prices.mkdir(parents=True, exist_ok=True) for ric in missing_rics: remote_filename = ric2filename(remote_dir_prices, ric, 'csv.gz') basename = remote_filename.name dest_parent = dir_prices dest = dest_parent / Path(basename) if dest.is_file(): logger.debug('skip downloading {}'.format(basename)) else: logger.debug('start downloading {}'.format(basename)) try: bucket.download_file(Key=str(remote_filename), Filename=str(dest)) except ClientError as e: code = e.response.get('Error', {}).get('Code', '') if str(code) == str(HTTPStatus.NOT_FOUND.value): logger.critical('{} is not found'.format( str(remote_filename))) logger.debug('end downloading {}'.format(basename))
def log(self, inner_logger: logging.Logger, response, is_post: bool, subject=None): method = 'POST' if is_post else 'GET' if int(response.status_code / 100) == 2: inner_logger.info( f'Successfully signed up on {subject} subject, {response.status_code}' ) self.write_logs_html(response.text, response.status_code) return True elif int(response.status_code / 100) == 4: inner_logger.error( f'Client error {response.status_code}, {method}, subject: {subject}' ) self.write_logs_html(response.text, response.status_code) elif int(response.status_code / 100) == 5: inner_logger.warning( f'Server error {response.status_code}, POST, subject: {subject}' ) time.sleep(self.RESEND_DELAY) inner_logger.info( f'Resending POST request for subject: {subject}, attempts left: {self.resend_attempts}' ) self.write_logs_html(response.text, response.status_code, 'error') return self._signup(subject) else: inner_logger.critical(f"Lol, it's {response.status_code}") self.write_logs_html(response.text, response.status_code, 'critical')
def publish_model(args: argparse.Namespace, backend: StorageBackend, log: logging.Logger): """ Pushes the model to Google Cloud Storage and updates the index file. :param args: :class:`argparse.Namespace` with "model", "backend", "args", "force" and \ "update_default". :return: None if successful, 1 otherwise. """ path = os.path.abspath(args.model) try: model = GenericModel(source=path, dummy=True) except ValueError as e: log.critical('"model" must be a path: %s', e) return 1 except Exception as e: log.critical("Failed to load the model: %s: %s" % (type(e).__name__, e)) return 1 meta = model.meta with backend.lock(): model_url = backend.upload_model(path, meta, args.force) log.info("Uploaded as %s", model_url) log.info("Updating the models index...") index = backend.fetch_index() index["models"].setdefault(meta["model"], {})[meta["uuid"]] = \ extract_index_meta(meta, model_url) if args.update_default: index["models"][meta["model"]][Model.DEFAULT_NAME] = meta["uuid"] backend.upload_index(index)
def validate_config_file( config: configobj.ConfigObj, logger: logging.Logger ) -> configobj.ConfigObj: """Validate config file, i.e. check that everything is set properly. This also sets all default values.""" logger.debug("Validating config file.") # 'copy' parameter of config.validate: Also copy all comments and default # values from the configspecs to the config file (to be written out later)? # Note: The config object might be completely empty, because only the first # Run of this methods sets them from the defaults if the user did not # specify a config file. # However, this method gets called 2 times (one time before and one time # after the handling of the --parameter options), # Unfortunately, if you do copy one time, an additional copy=False won't # bring it back. So we start with copy=False and if r.config.copy_all # is set to true, copy it the second time this function is called. try: copy_all = config["r"]["config"]["copy_all"] except KeyError: copy_all = False valid = config.validate( validate.Validator(), preserve_errors=True, copy=copy_all ) # adapted from https://stackoverflow.com/questions/14345879/ # answer from user sgt_pats 2017 # todo: This might need some better handling. for entry in configobj.flatten_errors(config, valid): [path, key, error] = entry if not error: msg = "The parameter {} was not in the config file\n".format(key) msg += ( "Please check to make sure this parameter is present and " "there are no mis-spellings." ) logger.critical(msg) raise ValueError(msg) if key is not None: if isinstance(error, validate.VdtValueError): optionString = config.configspec[key] msg = ( "The parameter {} was set to {} which is not one of " "the allowed values\n".format(key, config[key]) ) msg += "Please set the value to be in {}".format(optionString) logger.critical(msg) raise ValueError(msg) elif error: msg = "Validation error (section='{}', key={}): {}".format( "/".join(path), key, error ) logger.error(msg) raise ValueError(msg) return config
def load_health_tests_response(logger: Logger) -> str: response_file_path = os.getenv("RESPONSE_FILE_PATH") if not response_file_path: logger.critical('Environment variable "RESPONSE_FILE_PATH" is required') sys.exit(1) logger.info(f'Serving mesh test results from "{response_file_path}"') with open(response_file_path, mode="r") as f: return f.read()
def write_json_file(filename: str, r: requests.models.Response, logger: logging.Logger) -> None: try: with open(filename, "w", encoding="utf-8") as output_file: json.dump(r.json(), output_file) except Exception as error: logger.critical(f"Writing output file failed: {error}. Nothing to do.") sys.exit(1)
def critical(self, moduleName, className='', functionOrmethod='', message=''): DefaultLogger.critical( self, Logger.format(moduleName, className, functionOrmethod, message))
def sample_logging(logger: logging.Logger): """Sample function on how logging works""" # Add messages now! logger.debug("A debug message") logger.info("An info message") logger.warning("Something is not right.") logger.error("A Major error has happened.") logger.critical("Fatal error. Cannot continue")
def handle_signal( signal: int, frame: FrameType, self: TaskContainer = self, logger: logging.Logger = logger, ) -> None: logger.critical( "received termination signal {}".format(signal)) self._terminate = True
def to_restic_option(self, log: logging.Logger): if self.type == "file": return '--password-file "{}"'.format(self.arg) if self.type == "command": return '--password-command "{}"'.format(self.arg) log.critical( "PasswordProvider::to_restic_option(): unknown password provider type: {}" .format(self.type)) raise TypeError("unknown password provider type: {}".format(self.type))
async def run_ffmpeg(logger: logging.Logger, args: list, print_every: int = 30): logger.debug('CMD: ffmpeg %s', ' '.join(args)) p = await asyncio.create_subprocess_exec('ffmpeg', *args, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE) try: await asyncio.gather(watch_ffmpeg(logger, p.stdout, print_every), watch_ffmpeg(logger, p.stderr, print_every)) except Exception as e: logger.critical('stdout/err critical failure: %s', str(e)) await p.wait() if p.returncode != 0: raise Exception(f'ffmpeg exit code: {p.returncode}')
def create_backend_noexc(log: logging.Logger, git_index: GitIndex, name: str=None, args: str=None): try: return create_backend(name, git_index, args) except KeyError: log.critical("No such backend: %s (looked in %s)", name, list(__registry__.keys())) return None except ValueError: log.critical("Invalid backend arguments: %s", args) return None
def _load_generic_model(source: str, backend: StorageBackend, log: logging.Logger) -> Optional[GenericModel]: try: return GenericModel(source, backend=backend) except ValueError as e: log.critical('"input" must be a path: %s', e) return None except Exception as e: log.critical("Failed to load the model: %s: %s" % (type(e).__name__, e)) return None
def check_console_input_config(config: SharedConfig, log: logging.Logger = None) -> bool: log = log or logging.getLogger(__package__.split(".", maxsplit=1)[0]) # Check if config is valid if not config.is_valid: for prop, msg in config.validation_errors: log.critical("[!] '%s' property %s" % (prop, msg)) return False return True
def start_live_recording(bucket_name: str, order_name: str, start_time: str, end_time: str, camera_address: str, camera_username: str = 'xames3', camera_password: str = 'iamironman', camera_port: Union[int, str] = 554, camera_timeout: Union[float, int] = 30.0, timestamp_format: str = '%H:%M:%S', log: logging.Logger = None) -> Optional[str]: """Saves videos based on time duration.""" log = _log(__file__) if log is None else log run_date = datetime.now().strftime('%Y-%m-%d') start_time, end_time = f'{run_date} {start_time}', f'{run_date} {end_time}' duration = calculate_duration(start_time, end_time, timestamp_format, True) force_close = datetime.strptime( end_time, '%Y-%m-%d %H:%M:%S').replace(tzinfo=timezone.utc).timestamp() vid_type = video_type(True, True, True) temp_path = os.path.join(live, f'{bucket_name}{order_name}_{timestamp_dirname()}') if not os.path.isdir(temp_path): os.mkdir(temp_path) temp_file = os.path.join(temp_path, f'{bucket_name}{order_name}{vid_type}.mp4') url = configure_camera_url(camera_address, camera_username, camera_password, int(camera_port)) slept_duration, idx = 0, 1 if duration != 0: try: while True: if camera_live(camera_address, camera_port, camera_timeout, log): file = filename(temp_file, idx) log.info('Recording started for selected camera.') os.system(ffmpeg_str(url, file, duration, camera_timeout)) stop_utc = now().replace(tzinfo=timezone.utc).timestamp() stop_secs = now().second _old_file = file_size(file) old_duration = stop_secs if _old_file == '300.0 bytes' else drn(file) duration = duration - old_duration - slept_duration slept_duration = 0 idx += 1 if (force_close <= stop_utc) or (duration <= 0): output = concate_videos(temp_path, delete_old_files=True) if output: return output else: log.warning('Unable to record because of poor network connectivity.') slept_duration += camera_timeout log.warning('Compensating lost time & attempting after 30 secs.') time.sleep(camera_timeout) except Exception as error: log.critical(f'Something went wrong because of {error}')
def create_backend_noexc(log: logging.Logger, name: str=None, git_index: GitIndex=None, args: str=None) -> Optional[StorageBackend]: """Initialize a new Backend, return None if there was a known problem.""" try: return create_backend(name, git_index, args) except KeyError: log.critical("No such backend: %s (looked in %s)", name, list(__registry__.keys())) return None except ValueError: log.critical("Invalid backend arguments: %s", args) return None
def _log(logger: logging.Logger, level: int, message: str): if level == logging.DEBUG: logger.debug(message) elif level == logging.INFO: logger.info(message) elif level == logging.WARNING: logger.warning(message) elif level == logging.ERROR: logger.error(message) elif level == logging.CRITICAL: logger.critical(message) else: raise AssertionError('wrong level')
def log(self, logger: Logger = logger, level: int = INFO) -> 'ChainIter': """ Just print the content. """ if level == INFO: logger.info(self.data) elif level == WARNING: logger.warning(self.data) elif level == ERROR: logger.error(self.data) elif level == CRITICAL: logger.critical(self.data) return cast(ChainIter, self)
async def run_cmd(cmd: str, logger: logging.Logger): exec_name = cmd.split(' ')[0] p = await asyncio.create_subprocess_shell(cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE) try: await asyncio.gather( watch(p.stdout, exec_name, logger, prefix='STDOUT:'), watch(p.stderr, exec_name, logger, prefix='STDERR:')) except Exception as e: logger.critical(f'stdout/err critical failure: {str(e)}') await p.wait() if p.returncode != 0: raise Exception(F"{exec_name} exit code: {p.returncode}")
async def run_exec(exec_name, args: list, logger: logging.Logger): p = await asyncio.create_subprocess_exec(exec_name, *args, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE) try: await asyncio.gather( watch(p.stdout, exec_name, logger, prefix='STDOUT:'), watch(p.stderr, exec_name, logger, prefix='STDERR:')) except Exception as e: logger.critical(f'stdout/err critical failure: {str(e)}') await p.wait() if p.returncode != 0: raise Exception(F"{exec_name} exit code: {p.returncode}")
def __init__(self): logger = logging.getLogger('antivirus') hdlr = logging.FileHandler('antivirus.log') formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s') hdlr.setFormatter(formatter) logger.addHandler(hdlr) logger.setLevel(logging.WARNING) logger.setLevel(logging.INFO) try: self.fisier = raw_input('What file do you want to scan>?') if (magic.from_file(self.fisier)[0:3] in 'PE32') or (magic.from_file(self.fisier)[0:3] in 'ELF'): if os.path.getsize(self.fisier) > 524288000: logger.warning('file bigger than 500 mb') else: conn = sqlite3.connect( '/home/cyberghostkid/Desktop/antivirus/hash_signature.db' ) c = conn.cursor() c.fetchone() try: fisiez = open('virus_signature.txt', 'r') #if md5.new(self.fisier).hexdigest() in c.execute('SELECT * FROM hashsignature where hash=?',(str(md5.new(self.fisier).hexdigest()), )): if md5.new(self.fisier).hexdigest() in fisiez.read(): print 'we may have found a possible thread!' logger.critical('Possible thread found') else: print "sorry we weren't' able to see if it was a thread,so we say it's safe " fisiez = open('virus_signature.txt', 'w+r') c.executemany( 'INSERT INTO hashsignature(hash) VALUES (?)', str(md5.new(self.fisier).hexdigest())) conn.commit() fisiez.write(str(md5.new(self.fisier).hexdigest())) fisiez.seek(0) fisiez.close() logger.info( 'updated database after scanning {} programms name' .format(self.fisier)) except NameError: print 'error 404 file not found' logger.warning('app not found') else: pass print 'file not an executable' logger.warning('file not an executable') except IOError: logger.error('IOError: [Errno 9] Bad file descriptor')
def retry(stage_name: str, current_try: int, max_tries: int, err: Exception or str, logger: logging.Logger) -> None: """ Compares current run number with max run number and creates delay before the rerun. If max retries number is reached it exits the program. """ if current_try <= max_tries: logger.error( f"{stage_name} - Try #{current_try} - Occurred error '{err}'. Rerunning after delay." ) timer(logger=logger) else: logger.critical( f"{stage_name} - Try #{current_try}. Exiting the program.") sys.exit() # no point in running further if no results in N tries
def _assert_fail(self, logger: logging.Logger): if self.show_on_standard is False: if self.out: print(self.out, file=sys.stdout) if self.err: print(Fore.RED, file=sys.stderr, end="") print(self.err, file=sys.stderr, end="") print(Fore.RESET, file=sys.stderr) logger.critical( "exit code %d for %s: %s in %s", self.exit_code, self.request.cwd, self.shell_cmd, self.elapsed, ) raise SystemExit(self.exit_code)
def publish_model(args: argparse.Namespace, backend: StorageBackend, log: logging.Logger): """ Push the model to Google Cloud Storage and updates the index file. :param args: :class:`argparse.Namespace` with "model", "backend", "args", "force", "meta" \ "update_default", "username", "password", "remote_repo", "template_model", \ "template_readme" and "log_level". :param backend: Backend which is responsible for working with model files. :param log: Logger supplied by supply_backend :return: None if successful, 1 otherwise. """ path = os.path.abspath(args.model) try: model = GenericModel(source=path, dummy=True) except ValueError as e: log.critical('"model" must be a path: %s', e) return 1 except Exception as e: log.critical("Failed to load the model: %s: %s" % (type(e).__name__, e)) return 1 base_meta = model.meta try: model_url = backend.upload_model(path, base_meta, args.force) except ModelAlreadyExistsError: return 1 log.info("Uploaded as %s", model_url) with open(os.path.join(args.meta), encoding="utf-8") as _in: extra_meta = json.load(_in) model_type, model_uuid = base_meta["model"], base_meta["uuid"] meta = extract_model_meta(base_meta, extra_meta, model_url) log.info("Updating the models index...") try: template_model = backend.index.load_template(args.template_model) template_readme = backend.index.load_template(args.template_readme) except ValueError: return 1 backend.index.add_model(model_type, model_uuid, meta, template_model, args.update_default) backend.index.update_readme(template_readme) try: backend.index.upload("add", {"model": model_type, "uuid": model_uuid}) except ValueError: # TODO: replace with PorcelainError, see related TODO in index.py:181 return 1 log.info("Successfully published.")
def critical(self, msg, *args, **kwargs): msg = "%s# %s" % (MyLogger.indent, msg) Logger.critical(self, msg, *args, **kwargs)
def newline(self): Logger.critical(self, "")
if __name__ == "__main__": parser = ArgumentParser() parser.add_argument('--host', help = 'Server IP address e.g., 107.170.251.142', default = '128.199.191.249') parser.add_argument('--port', help = 'Port on the server (usually 80)', default = 80) parser.add_argument('--baud','-b', help = 'Serial port baud rate (default 57600)', default = 57600) parser.add_argument('--queue_size','-q', help = 'The size of the queue that functions as a buffer between Serial-to-Internet (default 10000)', default = 10000) parser.add_argument('--upload_interval', '-u', help = 'Interval in seconds between uploading to the server (default 60)', default = 60) parser.add_argument('--debug_level', '-l', help = 'Debug level', default = 'INFO', choices = ('DEBUG', 'INFO', 'WARNING', 'ERROR')) parser.add_argument('--serial_device', '-d', help = 'On Linux/OSX this is typically /dev/ttyUSB-something, and on Windows COM-something.') args = parser.parse_args() try: QUEUE_MAXSIZE = int(args.queue_size) except ValueError: logger.critical("Please specify an integer for the queue size") os._exit(1) HOST = args.host try: PORT = int(args.port) except ValueError: logger.critical("Please specify a port with an integer") os._exit(1) try: BAUDRATE = int(args.baud) except ValueError: logger.critical("Please specify the baudrate of the serial port with an integer") os._exit(1)