def startup(cls, **kwargs):
cls.__host = kwargs.pop("host", "127.0.0.1")
cls.__port = kwargs.pop("port", GflConf.get_property("api.http.port"))
while True:
cmd = input("> ")
if "EXIT".lower() == cmd.lower():
cls.exit()
break
if cmd.startswith("SHOWCONF"):
key = cmd[9:].strip()
print(GflConf.get_property(key))
def init_gfl(force: bool):
if os.path.exists(GflConf.home_dir):
if force:
shutil.rmtree(GflConf.home_dir)
else:
raise ValueError("home dir not empty.")
# 创建节点根目录
os.makedirs(GflConf.home_dir)
# 初始化节点地址和密钥对
GflNode.init_node()
# 创建默认配置文件
GflConf.init_conf()
# 初始化节点数据目录
init_lfs()
def init(cls, force):
if os.path.exists(GflConf.home_dir):
if force:
logging.shutdown()
shutil.rmtree(GflConf.home_dir)
else:
raise ValueError("homedir not empty.")
# create home dir
os.makedirs(GflConf.home_dir)
# generate config file
GflConf.generate_config(PathUtils.join(GflConf.home_dir,
"config.yaml"))
# generate node address and key
GflNode.init_node()
# create data directories
Lfs.init()
def run(cls, role, console, **kwargs):
sys.stderr = open(os.devnull, "w")
cls.logger = logging.getLogger("gfl")
with Daemonizer() as (is_setup, daemonizer):
main_pid = None
if is_setup:
main_pid = os.getpid()
pid_file = PathUtils.join(GflConf.home_dir, "proc.lock")
stdout_file = PathUtils.join(GflConf.logs_dir, "console_out")
stderr_file = PathUtils.join(GflConf.logs_dir, "console_err")
is_parent = daemonizer(pid_file,
stdout_goto=stdout_file,
stderr_goto=stderr_file)
if is_parent:
if console and main_pid == os.getpid():
Shell.startup()
GflNode.load_node()
if GflConf.get_property("net.mode") == "standalone":
client_number = GflConf.get_property(
"net.standalone.client_number")
for _ in range(len(GflNode.standalone_nodes), client_number):
GflNode.add_standalone_node()
ManagerHolder.default_manager = NodeManager(
node=GflNode.default_node, role="server")
for i in range(client_number):
client_manager = NodeManager(node=GflNode.standalone_nodes[i],
role="client")
ManagerHolder.standalone_managers.append(client_manager)
else:
ManagerHolder.default_manager = NodeManager(
node=GflNode.default_node, role=role)
# cls.__startup_node_managers()
HttpListener.start()
while HttpListener.is_alive():
time.sleep(2)
def run(cls, **kwargs):
daemon = kwargs.pop("daemon", False)
if daemon:
print("DAEMON")
with Daemonizer() as (is_setup, daemonizer):
if is_setup:
pass
pid_file = "proc.lock"
stdout_file = PathUtils.join(GflConf.logs_dir, "console_out")
stderr_file = PathUtils.join(GflConf.logs_dir, "console_err")
is_parent = daemonizer(pid_file,
stdout_goto=stdout_file,
stderr_goto=stderr_file)
if is_parent:
pass
GflConf.reload()
GflNode.load_node()
if GflConf.get_property("standalone.enabled"):
server_number = GflConf.get_property("standalone.server_number")
client_number = GflConf.get_property("standalone.client_number")
for _ in range(len(GflNode.standalone_nodes),
server_number + client_number):
GflNode.add_standalone_node()
for i in range(0, server_number):
node_manager = NodeManager(node=GflNode.standalone_nodes[i],
role="server")
cls.node_managers.append(node_manager)
for i in range(server_number, server_number + client_number):
node_manager = NodeManager(node=GflNode.standalone_nodes[i],
role="client")
cls.node_managers.append(node_manager)
else:
role = kwargs.pop("role")
print(role)
node_manager = NodeManager(node=GflNode.default_node, role=role)
cls.node_managers.append(node_manager)
cls.__startup_node_managers()
def load_dataset_zip(cls, dataset_id: str) -> File:
"""
Load dataset from a ZIP file.
:param dataset_id: dataset ID
"""
dataset_path = DatasetPath(dataset_id)
file_obj = BytesIO()
ZipUtils.compress(dataset_path.metadata_file, file_obj)
if GflConf.get_property("ipfs.enabled"):
file_obj.seek(0)
ipfs_hash = Ipfs.put(file_obj.read())
return File(ipfs_hash=ipfs_hash, file=None)
else:
return File(ipfs_hash=None, file=file_obj)
def load_job_zip(job_id: str) -> File:
"""
Load job from a ZIP file.
:param job_id: dataset ID
"""
job_path = JobPath(job_id)
file_obj = BytesIO()
ZipUtils.compress([job_path.metadata_file, job_path.config_dir], file_obj)
if GflConf.get_property("ipfs.enabled"):
file_obj.seek(0)
ipfs_hash = Ipfs.put(file_obj.read())
return File(ipfs_hash == ipfs_hash)
else:
return File(file=file_obj)
def detect_homedir(args_home):
if args_home is not None:
return args_home
env = os.getenv("GFL_HOME")
if env is not None:
return env
return GflConf.home_dir
if __name__ == "__main__":
args = parser.parse_args()
GflConf.home_dir = detect_homedir(args.home)
if args.action == "attach":
Shell.attach(args.host, args.port)
else:
GflConf.load()
Shell.welcome()
if args.action == "init":
Application.init(args.force)
elif args.action == "run":
props = {}
if args.props:
for p in args.props:
kv = p.split("=")
props[kv[0]] = kv[1]
Application.run(role=args.role, console=args.console, **props)
else:
print("unknown action.")
__all__ = ["NetBroadcast", "NetFetch", "NetReceive", "NetSend", "NetCallback"]
import warnings
from gfl.conf import GflConf
net_mode = GflConf.get_property("net.mode")
if net_mode == "standalone":
from gfl.core.net.standalone import NetBroadcast, NetFetch, NetReceive, NetSend, NetCallback
elif net_mode == "http":
from gfl.core.net.http import NetBroadcast, NetFetch, NetReceive, NetSend, NetCallback
elif net_mode == "eth":
from gfl.core.net.eth import NetBroadcast, NetFetch, NetReceive, NetSend, NetCallback
elif net_mode is None:
from gfl.core.net.abstract import NetBroadcast, NetFetch, NetReceive, NetSend, NetCallback
warnings.warn("net mode has not set.")
else:
raise ValueError("unknown net mode.")
def run(cls) -> None:
bind_ip = GflConf.get_property("api.http.bind_ip")
port = int(GflConf.get_property("api.http.port"))
app.logger.disabled = True
app.run(host=bind_ip, port=port)
from pathlib import PurePath
import ipfshttpclient
from gfl.conf import GflConf
from gfl.utils.path_utils import PathUtils
ipfs_addr = GflConf.get_property("ipfs.addr")
tmp_dir = GflConf.get_property("dir.tmp")
class Ipfs(object):
@classmethod
def put(cls, file_bytes: bytes):
"""
Adds file bytes to IPFS and return hash of the added IPFS object.
:param file_bytes: file bytes
:return: hash of the added IPFS object
"""
client = ipfshttpclient.connect(ipfs_addr)
res = client.add_bytes(file_bytes)
return res["Hash"]
@classmethod
def get(cls, ipfs_hash: str):
"""
Downloads a file from IPFS and read from the file.
:param ipfs_hash: hash of the IPFS object
:return: content read from the downloaded file
def set_property(key, value):
GflConf.set_property(key, value)
def get_property(key, default=None):
return GflConf.get_property(key, default)