def schedule_logger(job_id=None, delete=False): if not job_id: return getLogger("fate_flow_schedule") else: if delete: with LoggerFactory.lock: try: for key in LoggerFactory.schedule_logger_dict.keys(): if job_id in key: del LoggerFactory.schedule_logger_dict[key] except: pass return True key = job_id + 'schedule' if key in LoggerFactory.schedule_logger_dict: return LoggerFactory.schedule_logger_dict[key] return get_job_logger(job_id, "schedule")
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # import numpy as np from fate_arch import storage from fate_arch.abc import StorageTableABC, StorageTableMetaABC, AddressABC from fate_arch.common import log, EngineType from fate_arch.computing import ComputingEngine from fate_arch.storage import StorageTableMeta, StorageEngine, Relationship from fate_flow.entity.metric import MetricMeta from fate_flow.utils import job_utils, data_utils from fate_flow.components.component_base import ComponentBase LOGGER = log.getLogger() MAX_NUM = 10000 class Reader(ComponentBase): def __init__(self): super(Reader, self).__init__() self.parameters = None def run(self, component_parameters=None, args=None): self.parameters = component_parameters["ReaderParam"] output_storage_address = args["job_parameters"].engines_address[ EngineType.STORAGE] table_key = [key for key in self.parameters.keys()][0] computing_engine = args["job_parameters"].computing_engine output_table_namespace, output_table_name = data_utils.default_output_table_info(
# See the License for the specific language governing permissions and # limitations under the License. # import io import os from typing import Iterable from pyarrow import fs from fate_arch.common import hdfs_utils from fate_arch.common.log import getLogger from fate_arch.storage import StorageEngine, LocalFSStoreType from fate_arch.storage import StorageTableBase LOGGER = getLogger() class StorageTable(StorageTableBase): def __init__( self, address=None, name: str = None, namespace: str = None, partitions: int = 1, storage_type: LocalFSStoreType = LocalFSStoreType.DISK, options=None, ): super(StorageTable, self).__init__( name=name, namespace=namespace,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # import hashlib import time import typing import beautifultable from fate_arch.common.log import getLogger import inspect from functools import wraps from fate_arch.abc import CTableABC profile_logger = getLogger("PROFILING") _PROFILE_LOG_ENABLED = False _START_TIME = None _END_TIME = None class _TimerItem(object): def __init__(self): self.count = 0 self.total_time = 0.0 self.max_time = 0.0 def union(self, other: '_TimerItem'): self.count += other.count self.total_time += other.total_time if self.max_time < other.max_time:
JOB_START_TIMEOUT = 60 * 1000 # ms END_STATUS_JOB_SCHEDULING_TIME_LIMIT = 5 * 60 * 1000 # ms END_STATUS_JOB_SCHEDULING_UPDATES = 1 # Endpoint FATE_FLOW_MODEL_TRANSFER_ENDPOINT = "/v1/model/transfer" FATE_MANAGER_GET_NODE_INFO_ENDPOINT = "/fate-manager/api/site/secretinfo" FATE_MANAGER_NODE_CHECK_ENDPOINT = "/fate-manager/api/site/checksite" FATE_BOARD_DASHBOARD_ENDPOINT = "/index.html#/dashboard?job_id={}&role={}&party_id={}" # Logger log.LoggerFactory.LEVEL = 10 # {CRITICAL: 50, FATAL:50, ERROR:40, WARNING:30, WARN:30, INFO:20, DEBUG:10, NOTSET:0} log.LoggerFactory.set_directory( os.path.join(file_utils.get_project_base_directory(), "logs", "fate_flow")) stat_logger = log.getLogger("fate_flow_stat") detect_logger = log.getLogger("fate_flow_detect") access_logger = log.getLogger("fate_flow_access") data_manager_logger = log.getLogger("fate_flow_data_manager") peewee_logger = log.getLogger("peewee") # Switch UPLOAD_DATA_FROM_CLIENT = True USE_AUTHENTICATION = False PRIVILEGE_COMMAND_WHITELIST = [] CHECK_NODES_IDENTITY = False DEFAULT_FEDERATED_STATUS_COLLECT_TYPE = get_base_config( FATEFLOW_SERVICE_NAME, {}).get("default_federated_status_collect_type", "PUSH") # Init
######################################################## # Copyright 2019-2021 program was created VMware, Inc. # # SPDX-License-Identifier: Apache-2.0 # ######################################################## import logging import json import requests from requests.adapters import HTTPAdapter from urllib3.util.retry import Retry from fate_arch.common.log import getLogger from fate_arch.federation.pulsar._mq_channel import DEFAULT_SUBSCRIPTION_NAME logger = getLogger() MAX_RETRIES = 10 MAX_REDIRECT = 5 BACKOFF_FACTOR = 1 # sleep time equips to {BACKOFF_FACTOR} * (2 ** ({NUMBER_OF_TOTALRETRIES} - 1)) CLUSTER = 'clusters/{}' TENANT = 'tenants/{}' # APIs are refer to https://pulsar.apache.org/admin-rest-api/?version=2.7.0&apiversion=v2 class PulsarManager(): def __init__(self, host: str, port: str, runtime_config: dict = {}): self.service_url = "http://{}:{}/admin/v2/".format(host, port)