def invoke(self, args): """ Invocation of command. Args: args (Namespace): parsed arguments to hold customized parameters. """ error = None try: self.update_settings(args) except MindInsightException as e: error = e self.console = setup_logger('mindinsight', 'console', console=True, logfile=False, formatter='%(message)s') if error is not None: self.console.error(error.message) sys.exit(1) self.logfile = setup_logger('scripts', self.name, console=False, logfile=True) self.run(args)
def start(): """Start web service.""" gunicorn_conf_file = os.path.join(WEB_CONFIG_DIR, "gunicorn_conf.py") cmd = "gunicorn " \ "-b {host}:{port} {app_module} " \ "-c {conf_file} " \ "--logger-class {logger_class} " \ "--access-logformat {log_format}"\ .format(host=settings.HOST, port=settings.PORT, conf_file=gunicorn_conf_file, app_module=MINDBOARD_APP_MODULE, logger_class=GUNICORN_LOGGER, log_format=settings.GUNICORN_ACCESS_FORMAT ) error_log_abspath = _get_error_log_path() log_size = _get_file_size(error_log_abspath) # Init the logger file setup_logger('gunicorn', 'error') log_handler = open(error_log_abspath, 'a+') # start server process = subprocess.Popen( shlex.split(cmd), shell=False, # Change stdout to DEVNULL to prevent broken pipe error when creating new processes. stdin=subprocess.DEVNULL, stdout=log_handler, stderr=subprocess.STDOUT) # sleep 1 second for gunicorn appplication to load modules time.sleep(1) # check if gunicorn application is running console = setup_logger('mindinsight', 'console', console=True, logfile=False, formatter='%(message)s') if process.poll() is not None: console.error( "Start MindInsight failed. See log for details, log path: %s.", error_log_abspath) sys.exit(1) else: state_result = _check_server_start_stat(error_log_abspath, log_size) # print gunicorn start state to stdout label = 'Web address:' for ip in _get_all_ip_addresses(settings.HOST): format_args = label, ip, str( settings.PORT), settings.URL_PATH_PREFIX console.info('%s http://%s:%s%s', *format_args) label = '.' * len(label) for line in state_result["prompt_message"]: console.info(line) if state_result["state"] == ServerStateEnum.FAILED.value: sys.exit(1)
def __init__(self, cfg): self.access_log = setup_logger('gunicorn', 'access') self.error_log = setup_logger('gunicorn', 'error') super(GunicornLogger, self).__init__(cfg) access_log_path = _get_access_log_path() error_log_path = _get_error_log_path() os.chmod(access_log_path, stat.S_IREAD | stat.S_IWRITE) os.chmod(error_log_path, stat.S_IREAD | stat.S_IWRITE)
def __init__(self, cfg): self.cfg = cfg self.access_log = setup_logger('gunicorn', 'access', formatter='%(message)s') self.error_log = setup_logger('gunicorn', 'error', formatter=self.error_fmt) access_log_path = _get_access_log_path() error_log_path = _get_error_log_path() os.chmod(access_log_path, stat.S_IREAD | stat.S_IWRITE) os.chmod(error_log_path, stat.S_IREAD | stat.S_IWRITE) super(GunicornLogger, self).__init__(cfg)
def __init__(self): self.logger = setup_logger("mindconverter", "mindconverter", console=True, sub_log_name="logger_console") self.console = setup_logger("mindconverter", "mindconverter", console=True, sub_log_name="logger_only_console", logfile=False)
def create_app(): """Set flask APP config, and start the data manager.""" gunicorn_logger = setup_logger("gunicorn", "error") gunicorn_logger.info("create_app starts.") static_url_path = settings.URL_PATH_PREFIX + "/static" static_folder_path = os.path.realpath(os.path.join(os.path.dirname(__file__), os.pardir, 'ui', 'dist', 'static')) app = Flask(__name__, static_url_path=static_url_path, static_folder=static_folder_path) app.config['JSON_SORT_KEYS'] = False if settings.ENABLE_CORS: CORS(app, supports_credentials=True) app.before_request(before_request) app.register_error_handler(HTTPException, error_handler.handle_http_exception_error) app.register_error_handler(MindInsightException, error_handler.handle_mindinsight_error) app.register_error_handler(Exception, error_handler.handle_unknown_error) app.response_class = CustomResponse _init_app_module(app) gunicorn_logger.info("create_app ends.") return app
def __init__(self, max_processes_cnt, exit_callback, exit_check_fn): self._backend = futures.ProcessPoolExecutor( max_workers=max_processes_cnt, mp_context=_MP_CONTEXT) self._exit_callback = exit_callback self._task_slots = threading.Semaphore(value=max_processes_cnt) self._exit_check_fn = exit_check_fn self._logger = setup_logger("utils", "utils")
def __init__(self, mgr: ComputingResourceManager, executor_id, available_workers): self._mgr = mgr self.closed = False self._available_workers = available_workers self._effective_workers = self._calc_effective_workers( self._available_workers) self._slots = threading.Semaphore(value=self._effective_workers) self._id = executor_id self._futures = set() self._lock = threading.Lock() self.logger = setup_logger("utils", "utils") self.logger.debug("Available workers: %s.", available_workers)
def invoke(self, args): """ Invocation of command. Args: args (Namespace): parsed arguments to hold customized parameters. """ try: self.update_settings(args) except MindInsightException as error: print(error.message) else: self.logger = setup_logger('scripts', self.name) self.run(args)
def start(): """Start web service.""" errorlog_abspath = _get_error_log_path() gunicorn_conf_file = os.path.join(WEB_CONFIG_DIR, "gunicorn_conf.py") cmd = "gunicorn " \ "-b {host}:{port} {app_module} " \ "-c {conf_file} " \ "--logger-class {logger_class} " \ "--access-logformat {log_format}"\ .format(host=settings.HOST, port=settings.PORT, conf_file=gunicorn_conf_file, app_module=MINDBOARD_APP_MODULE, logger_class=GUNICORN_LOGGER, log_format=settings.GUNICORN_ACCESS_FORMAT ) log_size = _get_file_size(errorlog_abspath) console = setup_logger('mindinsight', 'console', console=True, logfile=False, formatter='%(message)s') # start server process = subprocess.Popen( shlex.split(cmd), shell=False, # Change stdout to DEVNULL to prevent broken pipe error when creating new processes. stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) # sleep 1 second for gunicorn appplication to load modules time.sleep(1) # check if gunicorn application is running if process.poll() is not None: console.error("Start MindInsight failed. See log for details.") else: state_result = _check_server_start_stat(errorlog_abspath, log_size) # print gunicorn start state to stdout console.info('Web address: http://%s:%s%s', settings.HOST, settings.PORT, settings.URL_PATH_PREFIX) for line in state_result["prompt_message"]: console.info(line)
def start(): """Start web service.""" errorlog_abspath = _get_error_log_path() gunicorn_conf_file = os.path.join(WEB_CONFIG_DIR, "gunicorn_conf.py") cmd = "gunicorn " \ "-b {host}:{port} {app_module} " \ "-c {conf_file} " \ "--logger-class {logger_class} " \ "--access-logformat {log_format}"\ .format(host=settings.HOST, port=settings.PORT, conf_file=gunicorn_conf_file, app_module=MINDBOARD_APP_MODULE, logger_class=GUNICORN_LOGGER, log_format=settings.GUNICORN_ACCESS_FORMAT ) log_size = _get_file_size(errorlog_abspath) console = setup_logger('mindinsight', 'console', console=True, logfile=False, formatter='%(message)s') # start server process = subprocess.Popen(shlex.split(cmd), shell=False, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) # sleep 1 second for gunicorn appplication to load modules time.sleep(1) # check if gunicorn application is running if process.poll() is not None: _, stderr = process.communicate() for line in stderr.decode().split('\n'): console.error(line) else: state_result = _check_server_start_stat(errorlog_abspath, log_size) # print gunicorn start state to stdout console.info('Web address: http://%s:%s', settings.HOST, settings.PORT) for line in state_result["prompt_message"]: console.info(line)
def _init_app_module(app): """ Init app module. Args: app (Flask): An instance of Flask. """ packages = find_app_package() gunicorn_logger = setup_logger("gunicorn", "error") for package in packages: try: app_module = import_module(package) gunicorn_logger.info("[%s].init_module starts.", package) app_module.init_module(app) gunicorn_logger.info("[%s].init_module ends.", package) except AttributeError: logger.debug('[%s].init_module not exists.', package)
def __init__(self, executors_cnt=1, max_processes_cnt=4): self._max_processes_cnt = max_processes_cnt self._executors_cnt = executors_cnt self._lock = threading.Lock() self._executors = { ind: Executor(self, executor_id=ind, available_workers=fractions.Fraction( self._max_processes_cnt, self._executors_cnt)) for ind in range(self._executors_cnt) } self._remaining_executors = len(self._executors) self._backend = futures.ProcessPoolExecutor( max_workers=max_processes_cnt, mp_context=_MP_CONTEXT) self.logger = setup_logger("utils", "utils") self.logger.info( "Initialized ComputingResourceManager with executors_cnt=%s, max_processes_cnt=%s.", executors_cnt, max_processes_cnt)
# Copyright 2020 Huawei Technologies Co., Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Logger""" from mindinsight.utils.log import setup_logger logger = setup_logger("optimizer", "optimizer")
# Copyright 2020 Huawei Technologies Co., Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Log module""" from mindinsight.utils.log import setup_logger logger = setup_logger(sub_module="conditionmgr", log_name="conditionmgr")
def __init__(self): self._executors = {} self._executor_id_counter = 1 self._lock = threading.Lock() self._exiting = False self._logger = setup_logger("utils", "utils")
# Copyright 2020 Huawei Technologies Co., Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Create a logger for explain data.""" from mindinsight.utils.log import setup_logger logger = setup_logger("explainer", "explainer") restful_logger = setup_logger("restful_api", "restful_api")
# Copyright 2019 Huawei Technologies Co., Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Create a logger.""" from mindinsight.utils.log import setup_logger logger = setup_logger("datavisual", "datavisual") restful_logger = setup_logger("restful_api", "restful_api")
# Copyright 2020 Huawei Technologies Co., Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Import mindinsight unified log module.""" from mindinsight.utils.log import setup_logger LOG_NAME = "debugger" LOG_MODULE = "debugger" LOGGER = setup_logger(sub_module=LOG_MODULE, log_name=LOG_NAME)
def __init__(self, executor, original_future: futures.Future): self._original_future = original_future self._executor = executor self.logger = setup_logger("utils", "utils")
# Copyright 2019 Huawei Technologies Co., Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Create a logger.""" from mindinsight.utils.log import setup_logger logger = setup_logger("datavisual", "datavisual") restful_logger = setup_logger("restful_api", "restful_api") parse_summary_logger = setup_logger("parse_summary", "parse_summary", console=True, formatter='[%(levelname)s]%(message)s')
def main(): """Entry point for mindinsight CLI.""" console = setup_logger('mindinsight', 'console', console=True, logfile=False, formatter='%(message)s') if (sys.version_info.major, sys.version_info.minor) < (3, 7): console.error('Python version should be at least 3.7') sys.exit(1) permissions = os.R_OK | os.W_OK | os.X_OK # set umask to 0o077 os.umask(permissions << 3 | permissions) parser = argparse.ArgumentParser( prog='mindinsight', description='MindInsight CLI entry point (version: {})'.format( mindinsight.__version__)) parser.add_argument('--version', action='version', version='%(prog)s ({})'.format( mindinsight.__version__)) subparsers = parser.add_subparsers( dest='cli', title='subcommands', description='the following subcommands are supported', ) commands = {} scripts_path = os.path.realpath( os.path.join(__file__, os.pardir, os.pardir, 'scripts')) files = os.listdir(scripts_path) files.sort() for file in files: if file.startswith('_') or not file.endswith('.py'): continue module = import_module('mindinsight.scripts.{}'.format( file[:-len('.py')])) command_cls = getattr(module, 'Command', None) if command_cls is None or not issubclass(command_cls, BaseCommand): continue command = command_cls() command_parser = subparsers.add_parser(command.name, help=command.description) command.add_arguments(command_parser) commands[command.name] = command argv = sys.argv[1:] if not argv or argv[0] == 'help': argv = ['-h'] args = parser.parse_args(argv) cli = args.__dict__.pop('cli') command = commands[cli] command.invoke(args)
# You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Create a logger.""" __all__ = ["logger", "logger_console"] from mindinsight.utils.log import setup_logger logger = setup_logger("mindconverter", "mindconverter", console=False) class MindConverterLogger: """MindConverter logger for stdout.""" def __init__(self): self.logger = setup_logger("mindconverter", "mindconverter", console=True, sub_log_name="logger_console") self.console = setup_logger("mindconverter", "mindconverter", console=True, sub_log_name="logger_only_console", logfile=False)
def get_statistics_from_tensor(tensors): """ Calculates statistics data of tensor. Args: tensors (numpy.ndarray): An numpy.ndarray of tensor data. Returns: Statistics, an instance of Statistics. """ ma_value = np.ma.masked_invalid(tensors) total, valid = tensors.size, ma_value.count() invalids = [] for isfn in np.isnan, np.isposinf, np.isneginf: if total - valid > sum(invalids): count = np.count_nonzero(isfn(tensors)) invalids.append(count) else: invalids.append(0) nan_count, pos_inf_count, neg_inf_count = invalids logger = setup_logger("utils", "utils") if not valid: logger.warning( 'There are no valid values in the tensors(size=%d, shape=%s)', total, tensors.shape) statistics = Statistics({ 'max_value': 0, 'min_value': 0, 'avg_value': 0, 'count': total, 'nan_count': nan_count, 'neg_inf_count': neg_inf_count, 'pos_inf_count': pos_inf_count }) return statistics # BUG: max of a masked array with dtype np.float16 returns inf # See numpy issue#15077 if issubclass(tensors.dtype.type, np.floating): tensor_min = ma_value.min(fill_value=np.PINF) tensor_max = ma_value.max(fill_value=np.NINF) if tensor_min < F32_MIN or tensor_max > F32_MAX: logger.warning( 'Values(%f, %f) are too large, you may encounter some undefined ' 'behaviours hereafter.', tensor_min, tensor_max) else: tensor_min = ma_value.min() tensor_max = ma_value.max() tensor_sum = ma_value.sum(dtype=np.float64) with np.errstate(invalid='ignore'): neg_zero_count = np.sum(ma_value < 0) with np.errstate(invalid='ignore'): pos_zero_count = np.sum(ma_value > 0) with np.errstate(invalid='ignore'): zero_count = np.sum(ma_value == 0) statistics = Statistics({ 'is_bool': tensors.dtype == np.bool, 'max_value': tensor_max, 'min_value': tensor_min, 'avg_value': tensor_sum / valid, 'count': total, 'neg_zero_count': neg_zero_count, 'pos_zero_count': pos_zero_count, 'zero_count': zero_count, 'nan_count': nan_count, 'neg_inf_count': neg_inf_count, 'pos_inf_count': pos_inf_count }) return statistics
# Copyright 2020 Huawei Technologies Co., Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ """Create a logger.""" from mindinsight.utils.log import setup_logger logger = setup_logger("mindconverter", "mindconverter", console=False) logger_console = setup_logger("mindconverter", "mindconverter", console=True, sub_log_name="logger_console", formatter="%(message)s")