def job_generate(): #generate a fl job which is the same as fl_master inputs = [fluid.layers.data( \ name=str(slot_id), shape=[5], dtype="float32") for slot_id in range(3)] label = fluid.layers.data( \ name="label", shape=[1], dtype='int64') model = Model() model.mlp(inputs, label) job_generator = JobGenerator() optimizer = fluid.optimizer.SGD(learning_rate=0.1) job_generator.set_optimizer(optimizer) job_generator.set_losses([model.loss]) job_generator.set_startup_program(model.startup_program) job_generator.set_infer_feed_and_target_names([x.name for x in inputs], [model.predict.name]) build_strategy = FLStrategyFactory() build_strategy.fed_avg = True build_strategy.inner_step = 10 strategy = build_strategy.create_fl_strategy() # endpoints will be collected through the cluster # in this example, we suppose endpoints have been collected server_ip = ["{}".format(ip_list[0])] output = "job_config" job_generator.generate_fl_job( strategy, server_endpoints=server_ip, worker_num=int(default_dict["worker_nodes"]), output=output) file_list = os.listdir(output) for file in file_list: tar = tarfile.open('{}/{}.tar.gz'.format(output, file), 'w:gz') for root, dir, files in os.walk("{}/{}".format(output, file)): for f in files: fullpath = os.path.join(root, f) tar.add(fullpath) tar.close()
def init_env(self): from paddle_fl.paddle_fl.core.scheduler.agent_master import FLScheduler import paddle.fluid as fluid # import paddle_fl as fl from paddle_fl.paddle_fl.core.master.job_generator import JobGenerator from paddle_fl.paddle_fl.core.strategy.fl_distribute_transpiler import FLDistributeTranspiler from paddle_fl.paddle_fl.core.strategy.fl_strategy_base import FLStrategyFactory, FedAvgStrategy if self.config['parameter']['model'] == 'resnet': model = ResNet18() # inputs = np.array([np.zeros((3, 224, 224)).astype('float32')]).astype('float32') inputs = fluid.layers.data(name='x', shape=[1, 3, 224, 224], dtype='float32') labels = np.array([0]).astype('float32').reshape(-1, 1) labels = fluid.layers.data(name='label', shape=[1, 1], dtype='float32') model.resnet(inputs, labels) else: inputs = [fluid.layers.data( name=str(slot_id), shape=[5], dtype="float32") for slot_id in range(3)] label = fluid.layers.data( name="label", shape=[1], dtype='int64') model = Model() model.mlp(inputs, label) job_generator = JobGenerator() optimizer = fluid.optimizer.SGD(learning_rate=0.1) job_generator.set_optimizer(optimizer) job_generator.set_losses([model.loss]) job_generator.set_startup_program(model.startup_program) job_generator.set_infer_feed_and_target_names( [x.name for x in inputs], [model.predict.name]) build_strategy = FLStrategyFactory() build_strategy.fed_avg = True build_strategy.inner_step = 1 strategy = build_strategy.create_fl_strategy() endpoints = ['{}:{}'.format(self.config['server']['ip'], self.config['server']['port'])] output = self.config['path']['job_path'] job_generator.generate_fl_job( strategy, server_endpoints=endpoints, worker_num=int(self.config['parameter']['num_users']), output=output) QMessageBox.information(self, 'compile ok', 'compile env done', QMessageBox.Ok) print('finish!') self.worker_num = int(self.config['parameter']['num_users']) self.server_num = 1 # Define the number of worker/server and the port for scheduler self.scheduler = MFLScheduler(self.worker_num, self.server_num, port=int(self.config['scheduler']['port'])) self.scheduler.set_sample_worker_num(self.worker_num) # self.scheduler.set_sample_worker_num(max(1, int(float(self.config['parameter']['frac']) * self.worker_num))) import paddle_fl as fl import paddle.fluid as fluid from paddle_fl.paddle_fl.core.server.fl_server import FLServer from paddle_fl.paddle_fl.core.master.fl_job import FLRunTimeJob self.server = FLServer() server_id = self.id job_path = self.config['path']['job_path'] print(job_path) self.job = FLRunTimeJob() self.job.load_server_job(job_path, server_id) self.job._scheduler_ep = '{}:{}'.format(self.config['scheduler']['ip'], self.config['scheduler']['port']) print(self.job._scheduler_ep) self.server.set_server_job(self.job) self.server._current_ep = '{}:{}'.format(self.config['server']['ip'], self.config['server']['port']) print(self.server._current_ep) self.processLabel.setText('waiting for agents') # self.server.start() self.initThread = threading.Thread(target=self.scheduler.init_env) self.waitThread = threading.Thread(target=self.wait_agent) self.servThread = threading.Thread(target=self.server.start) self.initThread.start() self.servThread.start() self.waitThread.start() # self.servThread.join() # self.waitThread.join() print("init env done.")
# You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import math import paddle_fl.paddle_fl as fl import paddle.fluid as fluid from paddle_fl.paddle_fl.core.master.job_generator import JobGenerator from paddle_fl.paddle_fl.core.strategy.fl_strategy_base import FLStrategyFactory # Fedavg build_strategy = FLStrategyFactory() build_strategy.fed_avg = True build_strategy.inner_step = 5 strategy = build_strategy.create_fl_strategy() endpoints = ["127.0.0.1:8181"] output = "fl_job_config" program_file = "faster_rcnn_program" job_generator = JobGenerator() job_generator.generate_fl_job_from_program(strategy=strategy, endpoints=endpoints, worker_num=2, program_input=program_file, output=output)