def startup(cls, **kwargs): cls.__host = kwargs.pop("host", "127.0.0.1") cls.__port = kwargs.pop("port", GflConf.get_property("api.http.port")) while True: cmd = input("> ") if "EXIT".lower() == cmd.lower(): cls.exit() break if cmd.startswith("SHOWCONF"): key = cmd[9:].strip() print(GflConf.get_property(key))
def run(cls, role, console, **kwargs): sys.stderr = open(os.devnull, "w") cls.logger = logging.getLogger("gfl") with Daemonizer() as (is_setup, daemonizer): main_pid = None if is_setup: main_pid = os.getpid() pid_file = PathUtils.join(GflConf.home_dir, "proc.lock") stdout_file = PathUtils.join(GflConf.logs_dir, "console_out") stderr_file = PathUtils.join(GflConf.logs_dir, "console_err") is_parent = daemonizer(pid_file, stdout_goto=stdout_file, stderr_goto=stderr_file) if is_parent: if console and main_pid == os.getpid(): Shell.startup() GflNode.load_node() if GflConf.get_property("net.mode") == "standalone": client_number = GflConf.get_property( "net.standalone.client_number") for _ in range(len(GflNode.standalone_nodes), client_number): GflNode.add_standalone_node() ManagerHolder.default_manager = NodeManager( node=GflNode.default_node, role="server") for i in range(client_number): client_manager = NodeManager(node=GflNode.standalone_nodes[i], role="client") ManagerHolder.standalone_managers.append(client_manager) else: ManagerHolder.default_manager = NodeManager( node=GflNode.default_node, role=role) # cls.__startup_node_managers() HttpListener.start() while HttpListener.is_alive(): time.sleep(2)
def run(cls, **kwargs): daemon = kwargs.pop("daemon", False) if daemon: print("DAEMON") with Daemonizer() as (is_setup, daemonizer): if is_setup: pass pid_file = "proc.lock" stdout_file = PathUtils.join(GflConf.logs_dir, "console_out") stderr_file = PathUtils.join(GflConf.logs_dir, "console_err") is_parent = daemonizer(pid_file, stdout_goto=stdout_file, stderr_goto=stderr_file) if is_parent: pass GflConf.reload() GflNode.load_node() if GflConf.get_property("standalone.enabled"): server_number = GflConf.get_property("standalone.server_number") client_number = GflConf.get_property("standalone.client_number") for _ in range(len(GflNode.standalone_nodes), server_number + client_number): GflNode.add_standalone_node() for i in range(0, server_number): node_manager = NodeManager(node=GflNode.standalone_nodes[i], role="server") cls.node_managers.append(node_manager) for i in range(server_number, server_number + client_number): node_manager = NodeManager(node=GflNode.standalone_nodes[i], role="client") cls.node_managers.append(node_manager) else: role = kwargs.pop("role") print(role) node_manager = NodeManager(node=GflNode.default_node, role=role) cls.node_managers.append(node_manager) cls.__startup_node_managers()
def load_dataset_zip(cls, dataset_id: str) -> File: """ Load dataset from a ZIP file. :param dataset_id: dataset ID """ dataset_path = DatasetPath(dataset_id) file_obj = BytesIO() ZipUtils.compress(dataset_path.metadata_file, file_obj) if GflConf.get_property("ipfs.enabled"): file_obj.seek(0) ipfs_hash = Ipfs.put(file_obj.read()) return File(ipfs_hash=ipfs_hash, file=None) else: return File(ipfs_hash=None, file=file_obj)
def load_job_zip(job_id: str) -> File: """ Load job from a ZIP file. :param job_id: dataset ID """ job_path = JobPath(job_id) file_obj = BytesIO() ZipUtils.compress([job_path.metadata_file, job_path.config_dir], file_obj) if GflConf.get_property("ipfs.enabled"): file_obj.seek(0) ipfs_hash = Ipfs.put(file_obj.read()) return File(ipfs_hash == ipfs_hash) else: return File(file=file_obj)
__all__ = ["NetBroadcast", "NetFetch", "NetReceive", "NetSend", "NetCallback"] import warnings from gfl.conf import GflConf net_mode = GflConf.get_property("net.mode") if net_mode == "standalone": from gfl.core.net.standalone import NetBroadcast, NetFetch, NetReceive, NetSend, NetCallback elif net_mode == "http": from gfl.core.net.http import NetBroadcast, NetFetch, NetReceive, NetSend, NetCallback elif net_mode == "eth": from gfl.core.net.eth import NetBroadcast, NetFetch, NetReceive, NetSend, NetCallback elif net_mode is None: from gfl.core.net.abstract import NetBroadcast, NetFetch, NetReceive, NetSend, NetCallback warnings.warn("net mode has not set.") else: raise ValueError("unknown net mode.")
def run(cls) -> None: bind_ip = GflConf.get_property("api.http.bind_ip") port = int(GflConf.get_property("api.http.port")) app.logger.disabled = True app.run(host=bind_ip, port=port)
from pathlib import PurePath import ipfshttpclient from gfl.conf import GflConf from gfl.utils.path_utils import PathUtils ipfs_addr = GflConf.get_property("ipfs.addr") tmp_dir = GflConf.get_property("dir.tmp") class Ipfs(object): @classmethod def put(cls, file_bytes: bytes): """ Adds file bytes to IPFS and return hash of the added IPFS object. :param file_bytes: file bytes :return: hash of the added IPFS object """ client = ipfshttpclient.connect(ipfs_addr) res = client.add_bytes(file_bytes) return res["Hash"] @classmethod def get(cls, ipfs_hash: str): """ Downloads a file from IPFS and read from the file. :param ipfs_hash: hash of the IPFS object :return: content read from the downloaded file
def get_property(key, default=None): return GflConf.get_property(key, default)