def _reply(self):
url = self._get_reply_url()
data = {
"message": self._msg,
"posttime": self._get_args("posttime"),
"formhash": self._get_args("formhash"),
"sesig": self._get_args("usesig"),
"subject": self._get_args("subject"),
}
# if the self._msg is exist in the page, do nothing
if self._exist():
Recorder.info(self, self._user, "reply exist already")
return
while self._max_retry:
try:
Recorder.debug(self, self._user, "try to submit comment")
r = self._session.post(url, data=data)
r.raise_for_status()
Recorder.info(self, self._user, "submit succ")
break
except Exception as e:
msg = "_reply failed[{}]".format(self._max_retry)
Recorder.warn(self, self._user, msg, e)
else:
Recorder.error(self, self._user,
"run out of reply request retries")
def _get_text(self):
while True:
try:
r = self._session.get(self._page_url)
r.raise_for_status()
self._text = r.text
return
except Exception as e:
Recorder.error(self, self._user, "_get_text failed", e)
def monit(self):
try:
auth = Auth(self._session, self._user, self._config)
auth.confirm()
topic = Topic(self._session, self._user, self._config)
nt = Post(self._session, self._user, self._config,
topic.locate_post_url())
nt.reply()
except Exception as e:
Recorder.error(self, self._user, "init the watchdog", e)
return None
def _locate(self):
try:
r = self._session.get(self._page_url)
url = (self._domain + re.search(self._post_regex, r.text).group(1))
return url_trim(url)
except IndexError as e:
pass
except Exception as e:
Recorder.error(self, self._user, "_run failed", e)
return False
def locate_post_url(self):
while True:
url = self._locate()
if not url:
msg = "mismatch and sleep for {}".format(self._interval)
Recorder.debug(self, self._user, msg)
time.sleep(self._interval)
continue
msg = "hit the pattern: {}".format(url)
Recorder.info(self, self._user, msg)
return url
def __init__(self, output_dir, dataloaders):
self.trainloader, self.validloader, self.testloader = dataloaders
self.train_num = int(cfg.Total.node_num * cfg.Data.frac)
self.test_num = cfg.Total.node_num - self.train_num
self.train_clients = [node.Client(k, self.trainloader[k], self.validloader[k]) for k in range(self.train_num)]
self.test_list = [node.Client(j + self.train_num, self.trainloader[j], self.validloader[j]) for j in
range(self.test_num)]
self.model_dir = os.path.join(output_dir, 'Model')
self.log_dir = os.path.join(output_dir, 'Log')
self.server = node.Server(self.testloader)
mkdir_p(self.model_dir)
mkdir_p(self.log_dir)
self.summary_writer = FileWriter(self.log_dir)
self.recorder = Recorder()
def __init__(self, session, user, config):
try:
self._session = session
self._user = user
self._domain = config["domain"]
self._page_url = config["topic_page_url"]
self._interval = config["topic_interval"]
self._post_regex = config["post_regex"]
Recorder.info(self, self._user, "init")
except Exception as e:
Recorder.error(self, self._user, "init failed", e)
raise Exception(e)
def dispatch(fpath):
Recorder.info("<dispatch>", "main", "ready to dispatch")
configs = configparser.ConfigParser()
configs.read(fpath)
dog_ls = []
for user, config in configs.iteritems():
if user == "DEFAULT":
continue
dog = WatchDog(user, config)
dog_ls.append(gevent.spawn(dog.monit))
gevent.joinall(dog_ls)
Recorder.info("<dispatch>", "main", "all is finished")
def __init__(self, session, user, config, post_url):
try:
self._session = session
self._user = user
self._page_url = post_url
self._domain = config["domain"]
self._name = config["name"]
self._msg = config["msg"]
self._max_retry = config["post_max_retry"]
self._reply_args_regex = config["post_reply_args_regex"]
self._reply_url_regex = config["post_reply_url_regex"]
Recorder.info(self, self._user, "init")
except Exception as e:
Recorder.error(self, self._user, "init failed", e)
raise Exception(e)
def __init__(self, session, user, config):
try:
self._session = session
self._user = user
self._name = config["name"]
self._passwd = config["passwd"]
self._page_url = config["login_page_url"]
self._submit_url = config["login_submit_url"]
self._referer_url = config["login_referer_url"]
self._succ_flag = config["login_succ_flag"]
self._formhash_regex = config["formhash_regex"]
Recorder.info(self, self._user, "init")
except Exception as e:
Recorder.error(self, self._user, "init failed", e)
raise Exception(e)
def confirm(self):
try:
r = self._session.get(self._page_url)
data = {
"formhash": (re.search(self._formhash_regex, r.text).group(1)),
"referer": self._referer_url,
"loginfield": "username",
"username": self._name,
"password": self._passwd,
"questionid": 0,
"answer": '',
"cookietime": 2592000,
}
r = self._session.post(self._submit_url, data=data)
# check if login successed
# XXX can't handle Verification Code
if self._succ_flag not in r.text:
raise Exception("mismatch key words in the page")
Recorder.info(self, self._user, "confirm succ")
except Exception as e:
Recorder.error(self, self._user, "confirm failed", e)
raise Exception(e)
def __init__(self, user, config):
self._session = requests.Session()
self._user = user
self._config = config
Recorder.info(self, self._user, "init")
class Trainer(object):
def __init__(self, output_dir, dataloaders):
self.trainloader, self.validloader, self.testloader = dataloaders
self.train_num = int(cfg.Total.node_num * cfg.Data.frac)
self.test_num = cfg.Total.node_num - self.train_num
self.train_clients = [node.Client(k, self.trainloader[k], self.validloader[k]) for k in range(self.train_num)]
self.test_list = [node.Client(j + self.train_num, self.trainloader[j], self.validloader[j]) for j in
range(self.test_num)]
self.model_dir = os.path.join(output_dir, 'Model')
self.log_dir = os.path.join(output_dir, 'Log')
self.server = node.Server(self.testloader)
mkdir_p(self.model_dir)
mkdir_p(self.log_dir)
self.summary_writer = FileWriter(self.log_dir)
self.recorder = Recorder()
def train(self):
# start training
for rounds in range(cfg.Total.R):
print(f'==================The {rounds + 1}-th round==================')
training_client = random.sample(self.train_clients, 4)
# print(f' The training clients is {training_client}')
data_sum = 0
datalist = []
for client in training_client:
data_sum += len(client.train_dataloader.dataset)
data = len(client.train_dataloader.dataset)
datalist.append(data)
for idx, client in enumerate(training_client):
client.fork(self.server)
agreement = datalist[idx] / data_sum
metric = client.compute_gradients(cfg.Total.E, agreement)
self.recorder.validate(client)
self.recorder.printer(client)
self.server.merge(training_client)
print(f'Start personalization testing')
for idx, client in enumerate(self.test_list):
client.fork(self.server)
print(f'Start adapatation process')
client.adapt(cfg.Total.E)
print(f'Start validation')
self.recorder.validate(client)
self.recorder.printer(client)
self.recorder.validate(self.server)
self.recorder.printer(self.server)
self.recorder.finish()
Summary()