def test_emit_counter(self):
with self.assertLogs() as cm:
metrics.emit_counter('test', 1)
logs = [r.msg for r in cm.records]
self.assertEqual(logs, [
'[Metric][Counter] test: 1',
'[Test][Counter] test - 1'])
def create_app(config):
# format logging
logging.config.dictConfig(LOGGING_CONFIG)
before_hook_path = os.getenv('FEDLEARNER_WEBCONSOLE_BEFORE_APP_START')
if before_hook_path:
module_path, func_name = before_hook_path.split(':')
module = importlib.import_module(module_path)
# Dynamically run the function
getattr(module, func_name)()
app = Flask('fedlearner_webconsole')
app.config.from_object(config)
db.init_app(app)
migrate.init_app(app, db)
jwt.init_app(app)
# Error handlers
app.register_error_handler(400, _handle_bad_request)
app.register_error_handler(404, _handle_not_found)
app.register_error_handler(WebConsoleApiException, make_response)
app.register_error_handler(Exception, _handle_uncaught_exception)
api = Api(prefix='/api/v2')
initialize_auth_apis(api)
initialize_project_apis(api)
initialize_workflow_template_apis(api)
initialize_workflow_apis(api)
initialize_job_apis(api)
initialize_dataset_apis(api)
initialize_setting_apis(api)
initialize_mmgr_apis(api)
if os.environ.get('FLASK_ENV') != 'production':
initialize_debug_apis(api)
# A hack that use our customized error handlers
# Ref: https://github.com/flask-restful/flask-restful/issues/280
handle_exception = app.handle_exception
handle_user_exception = app.handle_user_exception
api.init_app(app)
app.handle_exception = handle_exception
app.handle_user_exception = handle_user_exception
# Inits k8s related stuff first since something in composer
# may depend on it
if Envs.FLASK_ENV == 'production' or Envs.K8S_CONFIG_PATH is not None:
k8s_watcher.start()
if app.config.get('START_GRPC_SERVER', True):
rpc_server.stop()
rpc_server.start(app)
if app.config.get('START_SCHEDULER', True):
scheduler.stop()
scheduler.start(app)
if app.config.get('START_COMPOSER', True):
with app.app_context():
composer.run(db_engine=db.get_engine())
metrics.emit_counter('create_app', 1)
return app
def is_peer_job_inheritance_matched(workflow):
# TODO: Move it to workflow service
if workflow.forked_from is None:
return True
job_flags = workflow.get_create_job_flags()
peer_job_flags = workflow.get_peer_create_job_flags()
job_defs = workflow.get_config().job_definitions
project = workflow.project
if project is None:
return True
project_config = project.get_config()
# TODO: Fix for multi-peer
client = RpcClient(project_config, project_config.participants[0])
parent_workflow = db.session.query(Workflow).get(workflow.forked_from)
resp = client.get_workflow(parent_workflow.name)
if resp.status.code != common_pb2.STATUS_SUCCESS:
emit_counter('get_workflow_failed', 1)
raise InternalException(resp.status.msg)
peer_job_defs = resp.config.job_definitions
for i, job_def in enumerate(job_defs):
if job_def.is_federated:
for j, peer_job_def in enumerate(peer_job_defs):
if job_def.name == peer_job_def.name:
if job_flags[i] != peer_job_flags[j]:
return False
return True
def create_app(config):
# format logging
logging.config.dictConfig(LOGGING_CONFIG)
app = Flask('fedlearner_webconsole')
app.config.from_object(config)
jwt.init_app(app)
# Error handlers
app.register_error_handler(400, _handle_bad_request)
app.register_error_handler(404, _handle_not_found)
app.register_error_handler(WebConsoleApiException, make_response)
app.register_error_handler(Exception, _handle_uncaught_exception)
# TODO(wangsen.0914): This will be removed sooner!
db.init_app(app)
api = Api(prefix='/api/v2')
initialize_auth_apis(api)
initialize_project_apis(api)
initialize_workflow_template_apis(api)
initialize_workflow_apis(api)
initialize_job_apis(api)
initialize_dataset_apis(api)
initialize_setting_apis(api)
initialize_mmgr_apis(api)
initialize_sparkapps_apis(api)
if os.environ.get('FLASK_ENV') != 'production' or Envs.DEBUG:
initialize_debug_apis(api)
# A hack that use our customized error handlers
# Ref: https://github.com/flask-restful/flask-restful/issues/280
handle_exception = app.handle_exception
handle_user_exception = app.handle_user_exception
api.init_app(app)
app.handle_exception = handle_exception
app.handle_user_exception = handle_user_exception
# Inits k8s related stuff first since something in composer
# may depend on it
if Envs.FLASK_ENV == 'production' or Envs.K8S_CONFIG_PATH is not None:
k8s_watcher.start()
if app.config.get('START_GRPC_SERVER', True):
rpc_server.stop()
rpc_server.start(app)
if app.config.get('START_SCHEDULER', True):
scheduler.stop()
scheduler.start(app)
if app.config.get('START_COMPOSER', True):
with app.app_context():
composer.run(db_engine=db.get_engine())
metrics.emit_counter('create_app', 1)
return app
def is_peer_ready(job: Job) -> bool:
project_config = job.project.get_config()
for party in project_config.participants:
client = RpcClient(project_config, party)
resp = client.check_job_ready(job.name)
if resp.status.code != common_pb2.STATUS_SUCCESS:
emit_counter('check_peer_ready_failed', 1)
return True
if not resp.is_ready:
return False
return True
def update_running_state(self, job_name):
job = self._session.query(Job).filter_by(name=job_name).first()
if job is None:
emit_counter('[JobService]job_not_found', 1)
return
if not job.state == JobState.STARTED:
emit_counter('[JobService]wrong_job_state', 1)
return
if job.is_flapp_complete():
job.complete()
logging.debug('[JobService]change job %s state to %s', job.name,
JobState(job.state))
elif job.is_flapp_failed():
job.fail()
logging.debug('[JobService]change job %s state to %s', job.name,
JobState(job.state))
