def post(self, request, pk):
serializer = StudentSerializer(data=request.data)
if serializer.is_valid():
course = Course.objects.get(pk=pk)
email = serializer.validated_data['email']
try:
student = User.objects.get(email=email)
if StudentOnCourse.objects.filter(student=student,
course=course):
return Response(
{
"message":
f"Student {student.email} is already on course"
}, status.HTTP_100_CONTINUE)
StudentOnCourse.objects.create(
student=student,
course=course,
course_is_paid=True,
)
students = course.students.all()
user_serializer = StudentSerializer(students, many=True)
return Response(user_serializer.data, status.HTTP_201_CREATED)
except User.DoesNotExist:
logging.error("Student does not exists")
return Response({"error": "Student does not exists"},
status=status.HTTP_400_BAD_REQUEST)
logging.error(serializer.errors)
return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
def patch(self, request, pk):
course = get_object_or_404(Course, pk=pk)
serializer = CourseSerializer(instance=course, data=request.data)
if serializer.is_valid():
serializer.save()
return Response(request.data)
logging.error(serializer.errors)
return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
def post(self, request):
serializer = CourseSerializer(data=request.data)
data = request.data
if serializer.is_valid():
if not data or data['start_date']:
return Response({'message': "start_date is cannot be empty"},
status=status.HTTP_400_BAD_REQUEST)
serializer.save()
return Response(serializer.data, status=status.HTTP_201_CREATED)
logging.error(serializer.errors)
return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
def init_env(self, local_shard_num=1):
if (local_shard_num > 250):
logging.error("shard num must be less than or equal to 250")
exit()
self.free_ports = self._get_free_endpoints(local_shard_num, 40000)
import socket
ip = socket.gethostbyname(socket.gethostname())
self.free_endpoints = ["{}:{}".format(ip, x) for x in self.free_ports]
self._comm.barrier()
self.node_type = 1
if self._comm.Get_rank() == 0:
self.node_type = 0
self.group_comm = self._comm.Split(self.node_type)
self.all_endpoints = self._comm.allgather(self.free_endpoints)
def user_is_registered_email(email):
""" Send letter to new users """
site = Site.objects.get_current()
try:
user = User.objects.get(email=email)
subject = f'You are registered on {site.name}'
message = f'{user.full_name}, welcome to our site <a href="{site.domain}">{site.name}</a><br>' \
f'We are glad to see you here, please, write us if you need help'
send_mail(
subject,
message,
settings.EMAIL_HOST_USER,
[email],
html_message=message
)
except User.DoesNotExist:
logging.error(f'User with email {email} not found')
def load_vocab(filename):
with open(filename) as f:
return json.loads(f.read())
if os.path.exists(VOCAB_PATH):
logging.info("load vocab form: {}".format(VOCAB_PATH))
VOCAB = load_vocab(VOCAB_PATH)
else:
#TODO: singleton
logging.info("build vocab form: {}".format(TRAIN_DATA_PATH))
VOCAB = build_vocab(TRAIN_DATA_PATH)
logging.info("save vocab into: {}".format(VOCAB_PATH))
save_vocab(VOCAB_PATH, VOCAB)
if VOCAB is None:
logging.error("load vocab error")
raise Exception("load vocab error")
def train_reader(lines):
def local_iter():
seg_id = 0
for line in lines:
assert (len(line.split("\t")) == 3)
uid, _, input_str = line.split("\t")
data = json.loads(input_str)
data_x = data["x"]
data_y = data["y"]
data_mask = data["mask"]
input_data, input_length = process_x(data_x, VOCAB)
def _test(self, date):
if self.trainer.infer_one_user_func is None:
pass
logging.info("doing test...")
if self.test_sampler is None:
logging.error("self.test_sampler should not be None when testing")
sim_idx = self.role_maker.simulator_idx()
sim_num = self.role_maker.simulator_num()
user_info_dict = self.test_sampler.sample_user_list(
self.scheduler_client,
date,
sim_idx,
len(self.data_client.stub_list),
sim_num, )
if self.do_profile:
print("test user info_dict: ", user_info_dict)
global_param_dict = self.scheduler_client.get_global_params()
def divide_chunks(l, n):
for i in range(0, len(l), n):
yield l[i:i + n]
# at most 50 process for testing
chunk_size = 50
# at most 100 uid for testing
max_test_uids = 100
uid_chunks = divide_chunks(user_info_dict.keys(), chunk_size)
os.system("rm -rf _test_result")
os.system("mkdir _test_result")
tested_uids = 0
for uids in uid_chunks:
if tested_uids >= max_test_uids:
break
processes = []
for user in uids:
arg_dict = {
"uid": str(user),
"date": date,
"data_endpoints":
self.role_maker.get_data_server_endpoints(),
"global_params": global_param_dict,
"user_param_names": self.trainer.get_user_param_names(),
"global_param_names":
self.trainer.get_global_param_names(),
"infer_result_dir": "_test_result/uid-%s" % user,
}
p = Process(
target=self.trainer.infer_one_user_func,
args=(arg_dict, self.trainer.trainer_config))
p.start()
processes.append(p)
if self.do_profile:
logging.info("wait test processes to close")
for i, p in enumerate(processes):
processes[i].join()
tested_uids += chunk_size
infer_results = []
# only support one test metric now
for uid in os.listdir("_test_result"):
with open("_test_result/" + uid + "/res", 'r') as f:
sample_cout, metric = f.readlines()[0].strip('\n').split('\t')
infer_results.append((int(sample_cout), float(metric)))
if sum([x[0] for x in infer_results]) == 0:
logging.info("infer results: 0.0")
else:
count = sum([x[0] for x in infer_results])
metric = sum([x[0] * x[1] for x in infer_results]) / count
logging.info("infer results: %f" % metric)
def train_one_user(arg_dict, trainer_config):
show_metric = trainer_config["show_metric"]
shuffle = trainer_config["shuffle"]
max_training_steps = trainer_config["max_training_steps"]
batch_size = trainer_config["batch_size"]
# logging.info("training one user...")
main_program = fluid.Program.parse_from_string(
trainer_config["main_program_desc"])
startup_program = fluid.Program.parse_from_string(
trainer_config["startup_program_desc"])
place = fluid.CPUPlace()
exe = fluid.Executor(place)
scope = fluid.global_scope()
if (startup_program is None):
logging.error("startup_program is None")
exit()
exe.run(startup_program)
feeder = fluid.DataFeeder(feed_list=trainer_config["input_names"],
place=place,
program=main_program)
data_server_endpoints = arg_dict["data_endpoints"]
# create data clients
data_client = DataClient()
data_client.set_data_server_endpoints(data_server_endpoints)
uid = arg_dict["uid"]
date = arg_dict["date"]
global_param_dict = arg_dict["global_params"]
user_data = data_client.get_data_by_uid(uid, date)
train_reader = reader.train_reader(user_data)
if shuffle == True:
train_reader = paddle.reader.shuffle(train_reader, buf_size=10000)
train_reader = paddle.batch(train_reader, batch_size=batch_size)
# get user param
# logging.debug("do not need to get user params")
set_global_param_dict(arg_dict["global_param_names"],
arg_dict["global_params"], scope)
if (main_program is None):
logging.error("main_program is None")
exit()
epoch = trainer_config["epoch"]
max_steps_in_epoch = trainer_config.get("max_steps_in_epoch", -1)
metrics = trainer_config["metrics"]
fetch_list = []
for var in trainer_config["target_names"]:
fetch_list.append(var)
for ei in range(epoch):
fetch_res_list = []
trained_sample_num = 0
step = 0
num_layers = trainer_config["num_layers"]
hidden_size = trainer_config["n_hidden"]
tot_loss, tot_correct = 0, 0
tot_samples = 0
init_hidden, init_cell = generate_init_data(batch_size, num_layers,
hidden_size)
for data in train_reader():
feed_data, input_lengths = prepare_input(batch_size, data,
init_hidden, init_cell)
fetch_res = exe.run(main_program,
feed=feeder.feed(feed_data),
fetch_list=fetch_list)
loss, last_hidden, last_cell, correct = fetch_res
init_hidden = np.array(last_hidden)
init_cell = np.array(last_cell)
tot_loss += np.array(loss)
tot_correct += np.array(correct)
tot_samples += np.sum(input_lengths)
step += 1
trained_sample_num += len(data)
fetch_res_list.append([np.array(loss), np.array(correct)])
if max_steps_in_epoch != -1 and step >= max_steps_in_epoch:
break
if show_metric and trained_sample_num > 0:
loss = tot_loss / step
acc = float(tot_correct) / tot_samples
print("loss: {}, acc: {}".format(loss, acc))
local_updated_param_dict = {}
# update user param
# logging.debug("do not need to update user params")
data_client.set_param_by_uid(uid, local_updated_param_dict)
# global_updated_param_dict = {}
write_global_param_file = arg_dict["write_global_param_file"]
#os.makedirs("%s/params" % write_global_param_file)
for var_name in arg_dict["global_param_names"]:
var = scope.var(var_name).get_tensor().__array__().astype(np.float32)
filename = os.path.join(write_global_param_file, "params", var_name)
#logging.info("filename: {}".format(filename))
dirname = os.path.dirname(filename)
if not os.path.exists(dirname):
os.makedirs(dirname)
with open(filename, "w") as f:
np.save(f, var)
with open("%s/_info" % write_global_param_file, "w") as f:
pickle.dump([uid, trained_sample_num], file=f)
def infer_one_user(arg_dict, trainer_config):
"""
infer a model with global_param and user params
input:
global_param
user_params
infer_program
user_data
output:
[sample_cout, top1]
"""
# run startup program, set params
uid = arg_dict["uid"]
batch_size = trainer_config["batch_size"]
startup_program = fluid.Program.parse_from_string(
trainer_config["startup_program_desc"])
infer_program = fluid.Program.parse_from_string(
trainer_config["infer_program_desc"])
place = fluid.CPUPlace()
exe = fluid.Executor(place)
scope = fluid.global_scope()
if (startup_program is None):
logging.error("startup_program is None")
exit()
if (infer_program is None):
logging.error("infer_program is None")
exit()
exe.run(startup_program)
data_client = DataClient()
data_client.set_data_server_endpoints(arg_dict["data_endpoints"])
# get user param
# logging.debug("do not need to get user params")
set_global_param_dict(arg_dict["global_param_names"],
arg_dict["global_params"], scope)
# reader
date = arg_dict["date"]
global_param_dict = arg_dict["global_params"]
user_data = data_client.get_data_by_uid(uid, date)
infer_reader = reader.infer_reader(user_data)
infer_reader = paddle.batch(infer_reader, batch_size=batch_size)
# run infer program
os.mkdir(arg_dict["infer_result_dir"])
#pred_file = open(arg_dict["infer_result_dir"] + '/' + "pred_file", "w")
feeder = fluid.DataFeeder(feed_list=trainer_config["input_names"],
place=place,
program=infer_program)
fetch_list = trainer_config["target_names"]
#logging.info("fetch_list: {}".format(fetch_list))
fetch_res = []
sample_count = 0
num_layers = trainer_config["num_layers"]
hidden_size = trainer_config["n_hidden"]
tot_correct, tot_loss = 0, 0
tot_samples, tot_batches = 0, 0
init_hidden, init_cell = generate_init_data(batch_size, num_layers,
hidden_size)
for data in infer_reader():
feed_data, input_lengths = prepare_input(batch_size, data, init_hidden,
init_cell)
fetch_res = exe.run(infer_program,
feed=feeder.feed(feed_data),
fetch_list=fetch_list)
loss, last_hidden, last_cell, correct = fetch_res
cost_eval = np.array(loss)
init_hidden = np.array(last_hidden)
init_cell = np.array(last_cell)
correct_val = np.array(correct)
tot_loss += cost_eval
tot_correct += correct_val
tot_samples += np.sum(input_lengths)
tot_batches += 1
loss = tot_loss / tot_batches
acc = float(tot_correct) / tot_samples
logging.info("infer acc: {}".format(acc))
with open(arg_dict["infer_result_dir"] + "/res", "w") as f:
f.write("%d\t%f\n" % (1, acc))