def job_generate():
#generate a fl job which is the same as fl_master
inputs = [fluid.layers.data( \
name=str(slot_id), shape=[5],
dtype="float32")
for slot_id in range(3)]
label = fluid.layers.data( \
name="label",
shape=[1],
dtype='int64')
model = Model()
model.mlp(inputs, label)
job_generator = JobGenerator()
optimizer = fluid.optimizer.SGD(learning_rate=0.1)
job_generator.set_optimizer(optimizer)
job_generator.set_losses([model.loss])
job_generator.set_startup_program(model.startup_program)
job_generator.set_infer_feed_and_target_names([x.name for x in inputs],
[model.predict.name])
build_strategy = FLStrategyFactory()
build_strategy.fed_avg = True
build_strategy.inner_step = 10
strategy = build_strategy.create_fl_strategy()
# endpoints will be collected through the cluster
# in this example, we suppose endpoints have been collected
server_ip = ["{}".format(ip_list[0])]
output = "job_config"
job_generator.generate_fl_job(
strategy,
server_endpoints=server_ip,
worker_num=int(default_dict["worker_nodes"]),
output=output)
file_list = os.listdir(output)
for file in file_list:
tar = tarfile.open('{}/{}.tar.gz'.format(output, file), 'w:gz')
for root, dir, files in os.walk("{}/{}".format(output, file)):
for f in files:
fullpath = os.path.join(root, f)
tar.add(fullpath)
tar.close()
def init_env(self):
from paddle_fl.paddle_fl.core.scheduler.agent_master import FLScheduler
import paddle.fluid as fluid
# import paddle_fl as fl
from paddle_fl.paddle_fl.core.master.job_generator import JobGenerator
from paddle_fl.paddle_fl.core.strategy.fl_distribute_transpiler import FLDistributeTranspiler
from paddle_fl.paddle_fl.core.strategy.fl_strategy_base import FLStrategyFactory, FedAvgStrategy
if self.config['parameter']['model'] == 'resnet':
model = ResNet18()
# inputs = np.array([np.zeros((3, 224, 224)).astype('float32')]).astype('float32')
inputs = fluid.layers.data(name='x', shape=[1, 3, 224, 224], dtype='float32')
labels = np.array([0]).astype('float32').reshape(-1, 1)
labels = fluid.layers.data(name='label', shape=[1, 1], dtype='float32')
model.resnet(inputs, labels)
else:
inputs = [fluid.layers.data(
name=str(slot_id), shape=[5],
dtype="float32")
for slot_id in range(3)]
label = fluid.layers.data(
name="label",
shape=[1],
dtype='int64')
model = Model()
model.mlp(inputs, label)
job_generator = JobGenerator()
optimizer = fluid.optimizer.SGD(learning_rate=0.1)
job_generator.set_optimizer(optimizer)
job_generator.set_losses([model.loss])
job_generator.set_startup_program(model.startup_program)
job_generator.set_infer_feed_and_target_names(
[x.name for x in inputs], [model.predict.name])
build_strategy = FLStrategyFactory()
build_strategy.fed_avg = True
build_strategy.inner_step = 1
strategy = build_strategy.create_fl_strategy()
endpoints = ['{}:{}'.format(self.config['server']['ip'], self.config['server']['port'])]
output = self.config['path']['job_path']
job_generator.generate_fl_job(
strategy, server_endpoints=endpoints, worker_num=int(self.config['parameter']['num_users']), output=output)
QMessageBox.information(self, 'compile ok', 'compile env done', QMessageBox.Ok)
print('finish!')
self.worker_num = int(self.config['parameter']['num_users'])
self.server_num = 1
# Define the number of worker/server and the port for scheduler
self.scheduler = MFLScheduler(self.worker_num, self.server_num, port=int(self.config['scheduler']['port']))
self.scheduler.set_sample_worker_num(self.worker_num)
# self.scheduler.set_sample_worker_num(max(1, int(float(self.config['parameter']['frac']) * self.worker_num)))
import paddle_fl as fl
import paddle.fluid as fluid
from paddle_fl.paddle_fl.core.server.fl_server import FLServer
from paddle_fl.paddle_fl.core.master.fl_job import FLRunTimeJob
self.server = FLServer()
server_id = self.id
job_path = self.config['path']['job_path']
print(job_path)
self.job = FLRunTimeJob()
self.job.load_server_job(job_path, server_id)
self.job._scheduler_ep = '{}:{}'.format(self.config['scheduler']['ip'], self.config['scheduler']['port'])
print(self.job._scheduler_ep)
self.server.set_server_job(self.job)
self.server._current_ep = '{}:{}'.format(self.config['server']['ip'], self.config['server']['port'])
print(self.server._current_ep)
self.processLabel.setText('waiting for agents')
# self.server.start()
self.initThread = threading.Thread(target=self.scheduler.init_env)
self.waitThread = threading.Thread(target=self.wait_agent)
self.servThread = threading.Thread(target=self.server.start)
self.initThread.start()
self.servThread.start()
self.waitThread.start()
# self.servThread.join()
# self.waitThread.join()
print("init env done.")
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import math
import paddle_fl.paddle_fl as fl
import paddle.fluid as fluid
from paddle_fl.paddle_fl.core.master.job_generator import JobGenerator
from paddle_fl.paddle_fl.core.strategy.fl_strategy_base import FLStrategyFactory
# Fedavg
build_strategy = FLStrategyFactory()
build_strategy.fed_avg = True
build_strategy.inner_step = 5
strategy = build_strategy.create_fl_strategy()
endpoints = ["127.0.0.1:8181"]
output = "fl_job_config"
program_file = "faster_rcnn_program"
job_generator = JobGenerator()
job_generator.generate_fl_job_from_program(strategy=strategy,
endpoints=endpoints,
worker_num=2,
program_input=program_file,
output=output)