def backend_compatibility(cls, job_parameters: RunParameters):
# compatible with previous 1.5 versions
if job_parameters.computing_engine is None or job_parameters.federation_engine is None:
if job_parameters.work_mode is None or job_parameters.backend is None:
raise RuntimeError("unable to find compatible backend engines")
work_mode = WorkMode(job_parameters.work_mode)
backend = Backend(job_parameters.backend)
if backend == Backend.EGGROLL:
if work_mode == WorkMode.CLUSTER:
job_parameters.computing_engine = ComputingEngine.EGGROLL
job_parameters.federation_engine = FederationEngine.EGGROLL
job_parameters.storage_engine = StorageEngine.EGGROLL
else:
job_parameters.computing_engine = ComputingEngine.STANDALONE
job_parameters.federation_engine = FederationEngine.STANDALONE
job_parameters.storage_engine = StorageEngine.STANDALONE
elif backend == Backend.SPARK:
job_parameters.computing_engine = ComputingEngine.SPARK
job_parameters.federation_engine = FederationEngine.RABBITMQ
job_parameters.storage_engine = StorageEngine.HDFS
# add mq info
federation_info = {}
federation_info['union_name'] = string_utils.random_string(4)
federation_info['policy_id'] = string_utils.random_string(10)
job_parameters.federation_info = federation_info
if job_parameters.federated_mode is None:
if job_parameters.computing_engine in [ComputingEngine.EGGROLL, ComputingEngine.SPARK]:
job_parameters.federated_mode = FederatedMode.MULTIPLE
elif job_parameters.computing_engine in [ComputingEngine.STANDALONE]:
job_parameters.federated_mode = FederatedMode.SINGLE
def backend_compatibility(work_mode: typing.Union[WorkMode,
int] = WorkMode.STANDALONE,
backend: typing.Union[Backend,
int] = Backend.EGGROLL,
**kwargs):
# Compatible with previous 1.5 versions
if kwargs.get("computing_engine") is None or kwargs.get(
"federation_engine") is None or kwargs.get(
"federation_mode") is None:
if work_mode is None or backend is None:
raise RuntimeError("unable to find compatible engines")
if isinstance(work_mode, int):
work_mode = WorkMode(work_mode)
if isinstance(backend, int):
backend = Backend(backend)
if backend == Backend.EGGROLL:
if work_mode == WorkMode.CLUSTER:
return ComputingEngine.EGGROLL, FederationEngine.EGGROLL, FederatedMode.MULTIPLE
else:
return ComputingEngine.STANDALONE, FederationEngine.STANDALONE, FederatedMode.SINGLE
if backend == Backend.SPARK_RABBITMQ:
return ComputingEngine.SPARK, FederationEngine.RABBITMQ, FederatedMode.MULTIPLE
if backend == Backend.SPARK_PULSAR:
return ComputingEngine.SPARK, FederationEngine.PULSAR, FederatedMode.MULTIPLE
else:
return kwargs["computing_engine"], kwargs["federation_engine"], kwargs[
"federated_mode"]
def federation_cleanup(job, task):
from fate_arch.common import Backend
from fate_arch.common import Party
runtime_conf = json_loads(job.f_runtime_conf_on_party)
job_parameters = runtime_conf['job_parameters']
backend = Backend(job_parameters.get('backend', 0))
store_engine = StoreEngine(job_parameters.get('store_engine', 0))
if backend.is_spark() and store_engine.is_hdfs():
runtime_conf['local'] = {'role': job.f_role, 'party_id': job.f_party_id}
parties = [Party(k, p) for k,v in runtime_conf['role'].items() for p in v ]
from fate_arch.session.spark import Session
ssn = Session(session_id=task.f_task_id)
ssn.init_federation(federation_session_id=task.f_task_id, runtime_conf=runtime_conf)
ssn._get_federation().generate_mq_names(parties=parties)
ssn._get_federation().cleanup()
def create(backend: typing.Union[Backend, int] = Backend.EGGROLL,
work_mode: typing.Union[WorkMode, int] = WorkMode.CLUSTER):
if isinstance(work_mode, int):
work_mode = WorkMode(work_mode)
if isinstance(backend, int):
backend = Backend(backend)
if backend == Backend.EGGROLL:
if work_mode == WorkMode.CLUSTER:
return Session(ComputingEngine.EGGROLL, FederationEngine.EGGROLL)
else:
return Session(ComputingEngine.STANDALONE, FederationEngine.STANDALONE)
if backend == Backend.SPARK:
return Session(ComputingEngine.SPARK, FederationEngine.RABBITMQ)
def init(job_id=None,
mode: typing.Union[int, WorkMode] = WorkMode.STANDALONE,
backend: typing.Union[int, Backend] = Backend.EGGROLL,
options: dict = None,
**kwargs):
if kwargs:
LOGGER.warning(f"{kwargs} not used, check!")
if isinstance(mode, int):
mode = WorkMode(mode)
if isinstance(backend, int):
backend = Backend(backend)
if job_id is None:
job_id = str(uuid.uuid1())
if options is None:
options = {}
sess = session.Session.create(backend, mode)
sess.init_computing(computing_session_id=job_id, options=options)
sess.as_default()
return sess