async def handle_message(self, message):
"""Check if each incoming message is a command or something to be operated on
:param message:
"""
# Check if message is a new pokemon and if so catch it
if self.pokecord.pokecord_check(message):
await self.pokecord.find_pokemon(message)
return
# Check if message is a command
owners = self.shared_config["owners"]
if message.author.id not in owners and message.author.id != self.client.user.id:
return
if message.content.startswith("|"):
to_repeat = message.content[1:]
await message.channel.send(to_repeat)
if self.shared_config["logging"]:
utils.log(f"Repeated {to_repeat}")
return
await self.client.process_commands(
message
) # https://discordpy.readthedocs.io/en/latest/faq.html#id18
async def fishy(self):
"""Send messages to add fishies to the configured person at a configured interval"""
await self.client.wait_until_ready()
# If disabled in configuration, don"t proceed
if not self.config["fishyfarming"]:
return
if self.client.shared["logging"]:
utils.log("Sushii fishy farming enabled")
recipients = utils.list_generator(self.config["fishyrecipients"])
while not self.client.is_closed():
channel = self.client.get_channel(self.config["channel"])
# Get a random recipient from one of the configured ones
random_recipient = next(recipients)
# Human typing delay
delay = self.rand.randint(1, 3)
# Give the random recipient fishies
outbound = outbound_message.Outbound_Message(
f"-fishy <@{random_recipient}>", channel, delay)
await outbound.send()
if self.client.shared["logging"]:
utils.log(f"Gave sushii fishies to {random_recipient}")
# Delay the loop if configured
await asyncio.sleep(
utils.get_delay(self.config["fishydelay"], self.rand))
def collaborativeUpdate(peer, t):
w_ref = peer.get_model_params()
accepted = []
rejected = []
for j, w_j in peer.V[t]:
angle, ED = angular_metric(w_ref.view(1, -1), w_j.view(1, -1),
"euclidean")
# if peer.id == 0 and t % 10 == 0:
# log('result', f"{peer}, ref={torch.sum(w_ref)}, w_j={torch.sum(w_j)}, angle={angle}, ED={ED}")
# if j not in peer.params.Wi:
# peer.params.Wi[j] = 0
# peer.params.Wi[j] = max((1 - peer.params.beta) * peer.params.Wi[j] + peer.params.beta * ED, 0)
# divergence filter
if ED <= peer.params.delta or True:
peer.params.n_accept += 1
accepted.append(w_j)
else:
peer.params.n_reject += 1
rejected.append(w_j)
if accepted:
w_gar = peer.params.mu * w_ref + (1 - peer.params.mu) * GAR(
peer, accepted)
else:
log('log', f"{peer}: No gradients accepted")
w_gar = w_ref
# update beta
peer.params.beta = 1 - np.mean([*peer.params.Wi.values()])
return w_gar
def get(self):
# First, check that the logged in user is a student
student = utils.check_privilege(model.Role.student)
if not student:
# Redirect home if not a student
return self.redirect('/home')
# end
# Otherwise, log which student made the get
utils.log('Student logged in: ' + str(student))
# And grab the key for the section
section_key = self.request.get('section')
# Make sure that it isn't null
if not section_key:
# Error if so, and redirect home
utils.error('Section_key is null')
self.redirect('/home')
else:
# And then grab the section from the key
section = ndb.Key(urlsafe=section_key).get()
# Making sure it's not null
if not section:
# Error if so
utils.error('Section is null')
else:
# Now check if the current round is 0
if section.current_round == 0:
# And redirect to an error if so
self.redirect('/error?code=103')
else:
# Otherwise, we need to set our template values
self.render_template(student, section)
def discussion_view_template(self, student, section, round_number, template_values):
student_info = utils.get_student_info(student.email, section.students)
if student_info:
template_values['alias'] = student_info.alias
group_id = student_info.group
group = model.Group.get_by_id(group_id, parent=section.key)
# Double check that it was found
if group:
# Depending on round number, we have to grab from
if round_number == 1:
previous_round = model.Round.get_by_id(1, parent=section.key)
else:
previous_round = model.Round.get_by_id(round_number - 1, parent=section.key)
# end
# Now grab all the group comments for the previous round
comments, did_not_participate = group_comments(group, section, previous_round)
# Set the template value for all the group comments
template_values['comments'] = comments
template_values['did_not_participate'] = did_not_participate
# Grab the requested round
requested_round = model.Round.get_by_id(round_number, parent=section.key)
# Grab the discussion description
template_values['description'] = requested_round.description
# And grab the logged in student's response
stu_response = model.Response.get_by_id(student.email, parent=requested_round.key)
# Check that they actually answered
if stu_response:
# And set template values to show their previous response
template_values['comment'] = stu_response.comment
utils.log('Comment = {0}'.format(str(stu_response.comment)))
template_values['response'] = ','.join(str(item) for item in stu_response.response)
def disconnect(self) -> None:
"""Disconnect the Client created by `connect`."""
utils.log("Disconnecting MQTT client...")
self.__client.loop_stop()
self.__client.disconnect()
self.__client.connected_flag = False
utils.log("MQTT client disonnected")
def random_graph(models, rho=0.2, cluster_enabled=False, k=2, data=None):
if rho and rho < 0:
log('warning', f"Generating a negative similarity matrix.")
# prob_edge = 1, rnd_state = None
nb_nodes = len(models)
if cluster_enabled:
clusters = cluster_peers(nb_nodes, k)
else:
clusters = {0: np.arange(nb_nodes)}
if data:
adjacency, similarities = similarity_matrix(models,
clusters,
rho,
data=data)
else:
adjacency, similarities = similarity_matrix(nb_nodes,
clusters,
rho,
data=data)
return {
'clusters': clusters,
'similarities': similarities,
'adjacency': adjacency
}
def dataparse(self):
u"""将读取到的参数列表的第一项，即excel中的第二行，解析为参数的type，把所有的参数按照这个对应的type做处理。
1.将所有为‘null’的字段都解析为None
2.将type为‘int’的字段强制转换成int类型
3.将type为‘str’的字段直接赋值
4.将type为‘password’的字段进行加密，zhigou的加密方式为SHA1
5.将type为‘best’的字段，加上盐，与id拼接并加密，zhigou的方式为MD5
处理完之后，返回处理后的参数列表"""
params_list = list()
for case in self.cases:
params = dict()
mes = ''
for key in self.types.keys():
if case[key] in CELLS:
params[key] = self.do_cell(case[key])
elif self.types[key] in TYPES:
if self.types[key] == 'int':
params[key] = self.do_int(case[key])
elif self.types[key] == 'str':
params[key] = self.do_str(case[key])
elif self.types[key] == 'password':
params[key] = self.do_password(case[key])
elif self.types[key] == 'best':
params[key] = self.do_best(case[key])
print key, params[key]
mes = mes + key + str(params[key]).decode('utf-8')
log(mes, 'Case Data', 'info')
params_list.append(params)
return params_list
def start_rounds(self, instructor):
# So first we need to get at the course and section
course, section = utils.get_course_and_section_objs(
self.request, instructor)
# And grab all of the rounds for this section
rounds = model.Round.query(ancestor=section.key).fetch()
# The view requires at least a initial question to add rounds, but check
if not rounds:
# Send an error if no rounds exist for this section
utils.error('Error! No initial question exists; cannot start yet.',
handler=self)
# And redirect
return self.redirect('/')
# end
# Now simply turn on the first round
section.current_round = 1
section.put()
# Add the dummy read only round if it's a rounds based discussion
if section.has_rounds:
self.add_rounds(num_of_rounds=1,
duration=0,
instructor=instructor,
type=model.Round.get_round_type('readonly'),
buffer_bw_rounds=0,
quiet=True)
# And send a success message
utils.log('Successfully started the first round.',
type='Success!',
handler=self)
def run_cmd_with_perf_stat_subshell(cmd: str,
stat_file_path,
stdout_file_path='/dev/null',
log_on=False):
test_cmd = 'sudo perf stat ' + cmd + ' > ' + stdout_file_path + ' 2> ' + stat_file_path
log(test_cmd, LogType.RUN_CMD, _on=log_on)
return system(test_cmd)
def remove_student(self, section, email):
"""
Removes a specific students from the given section.
Args:
section (object):
Section from which the student is to be removed.
email (str):
Email (ID) of the student to be removed.
"""
# First, grab the student from the db by the email passed in
student = model.Student.get_by_id(email)
# Check that there is actually a record for that email
if not student:
# And error if not
utils.error('Student does not exist!', handler=self)
else:
# Create a new list for the section removing the student
section.students = [s for s in section.students if s.email != email] # TODO better? use remove?
# Check if the student is enrolled in this section
if section.key in student.sections:
# And remove them if so
student.sections.remove(section.key)
# end
# And save both the student and section back to the db and log it
student.put()
section.put()
utils.log(
'Student {0} has been removed from Section {1}'.format(str(student),
str(section)), handler=self, type='Success!')
def log_round(peer, epoch, history):
peer_loss = history[peer.id][-1]['val_loss']
peer_acc = history[peer.id][-1]['val_acc']
log(
'',
f"Round [{epoch}], {peer}, loss: {peer_loss:.4f}, val_acc: {peer_acc:.4f}"
)
async def rep(self):
"""Automate farming Tatsumaki rep with a configured recipient and interval"""
if self.client.shared["logging"]:
utils.log("Tatsumaki rep farming enabled")
recipients = utils.list_generator(self.config["recipients"])
channel = self.client.get_channel(self.config["channel"])
while not self.client.is_closed():
# Get a random recipient from one of the configured ones
random_recipient = next(recipients)
# Human typing delay
delay = self.rand.randint(1, 3)
# If configured, get the delay before deleting the message
silent_delay = utils.get_delay(self.config["silent"], self.rand)
# Give the recipient rep in the configured channel
outbound = outbound_message.Outbound_Message(
f"t!rep <@{random_recipient}>", channel, delay, silent_delay)
await outbound.send()
if self.client.shared["logging"]:
utils.log(f"Gave tatsumaki rep to {random_recipient}")
# Delay the loop if configured
await asyncio.sleep(
utils.get_delay(self.config["delay"], self.rand))
def centralized_training(args, inference=True):
t = time.time()
log('warning', f'ML engine: {ML_ENGINE}')
log('event', 'Centralized training ...')
args.num_users = 1
args.iid = 1
args.unequal = 0
args.rounds = 0
train_ds, test_ds, user_groups = get_dataset(args)
train, val, test = train_val_test(train_ds, user_groups[0], args)
data = {'train': train, 'val': val, 'test': test, 'inference': test_ds}
log('info', f"Initializing {args.model} model.")
models = initialize_models(args, same=True)
server = Node(0, models[0], data, [], False, {}, args)
server.inference = inference_ds(server, args)
log(
'info',
f"Start server training on {len(server.train.dataset)} samples ..."
)
history = server.fit(inference)
server.stop()
t = time.time() - t
log("success", f"Centralized training finished in {t:.2f} seconds.")
return [history]
def _get_performance(self, mode=0):
# _ = run_cmd_with_perf_stat_subshell(cmd=test_cmd, stat_file_path=stat_file)
# run multiprocesses to test the running time, then return average
sub_ps = []
total_task_clocks = 0.0
log(
str(self.test_times) + ' subprocesses: perf stat ' +
self.test_info.test_cmd, LogType.RUN_CMD)
for stat_file in self._stat_files:
sub_ps.append(
run_cmd_with_perf_stat(cmd=self.test_info.test_cmd_args,
stat_file_path=stat_file,
repeat=10))
meets_error = False
for sub_p in sub_ps:
try:
outs, errs = sub_p.communicate(timeout=50.0)
if errs:
log(errs, LogType.ERROR, True)
meets_error = True
except Exception:
sub_p.kill()
#sub_p.communicate()
meets_error = True
if meets_error:
# this iteration is useless, kill all children and return -1
for sub_p in sub_ps:
if sub_p.poll() is None:
sub_p.kill()
# sub_p.communicate()
return -1
# make sure all children have terminated
time_count = 0
while time_count < 51:
has_alive = False
for sub_p in sub_ps:
if sub_p.poll() is None:
has_alive = True
break
if has_alive:
time_count += 1
sys.sleep(1)
else:
break
if time_count == 52:
for sub_p in sub_ps:
if sub_p.poll() is None:
sub_p.kill()
# sub_p.communicate()
min_task_clock = 10000000.0
for stat_file in self._stat_files:
task_clock = get_perf_stat_result(stat_file)
min_task_clock = min(min_task_clock, task_clock)
total_task_clocks += task_clock
if mode == 0:
return total_task_clocks / self.test_times
elif mode == 1:
return min_task_clock
def dataparse(self):
u"""将读取到的参数列表的第一项，即excel中的第二行，解析为参数的type，把所有的参数按照这个对应的type做处理。
1.将所有为‘null’的字段都解析为None
2.将type为‘int’的字段强制转换成int类型
3.将type为‘str’的字段直接赋值
4.将type为‘password’的字段进行加密，zhigou的加密方式为SHA1
5.将type为‘best’的字段，加上盐，与id拼接并加密，zhigou的方式为MD5
处理完之后，返回处理后的参数列表"""
params_list = list()
for case in self.cases:
params = dict()
mes = ''
for key in self.types.keys():
if case[key] in CELLS:
params[key] = self.do_cell(case[key])
elif self.types[key] in TYPES:
if self.types[key] == 'int':
params[key] = self.do_int(case[key])
elif self.types[key] == 'str':
params[key] = self.do_str(case[key])
elif self.types[key] == 'password':
params[key] = self.do_password(case[key])
elif self.types[key] == 'best':
params[key] = self.do_best(case[key])
print key,params[key]
mes = mes + key + str(params[key]).decode('utf-8')
log(mes, 'Case Data', 'info')
params_list.append(params)
return params_list
def players_turn(G, SCREEN):
inp = expect_key()
if inp in DIRECTION_MAP:
player.move(G, DIRECTION_MAP[inp])
if inp == KEY_MAP["stairs"]:
stack = get(G["DUNGEON"][G["FLOOR"]], player.PLAYER["POS"])
if "upstairs" in stack: G["FLOOR"] += 1
elif "downstairs" in stack: G["FLOOR"] -= 1
if inp == KEY_MAP[
"equipt"]: # TODO: make this better, choose from menu and everything
idx = "0123456789abcdefghijklmnopqrstuvwxyz"
dr.draw_INV(SCREEN, (0, 0), player.PLAYER, search="EQUIPTABLE")
key = expect_key(KEYBOARD_MAP.keys())
i = idx.index(KEYBOARD_MAP[key])
if i < len(player.PLAYER["INV"]):
item = player.PLAYER["INV"][i]
items.equipt(player.PLAYER, item, G)
else:
log(G, "That didn't work.")
if inp == KEY_MAP["INV"]:
dr.draw_INV(SCREEN, (0, 0), player.PLAYER)
players_turn(G, SCREEN)
def connect(self, neighbor: Node):
try:
if neighbor.id in [n.neighbor_id for n in self.neighbors]:
log('log', f"{self}, neighbor {neighbor} already connected.")
return True
if self.mp:
sock = create_tcp_socket()
sock.settimeout(SOCK_TIMEOUT)
sock.connect((neighbor.host, neighbor.port))
neighbor_conn = NodeConnection(self, neighbor.id, sock)
neighbor_conn.start()
neighbor_conn.send(
protocol.connect(sock.getsockname(), self.id))
self.neighbors.append(neighbor_conn)
else:
slink = NodeLink(self, neighbor, None)
dlink = NodeLink(neighbor, self, slink)
slink.link = dlink
self.neighbors.append(slink)
neighbor.neighbors.append(dlink)
return True
except Exception as e:
log('error', f"{self}: Can't connect to {neighbor} -- {e}")
return False
def show_neighbors(self, verbose=False):
log('info', "Neighbors list")
for peer in self.peers:
if isinstance(peer, Node):
log('', f"{peer} has: {len(peer.neighbors)} neighbors.")
else:
log(
'',
f"{peer} has: {len(peer.neighbors)} neighbors: {peer.neighbors}"
)
if verbose:
# log('', f"{peer} neighbors: {peer.neighbors}")
log(
'',
f"{peer} has: {len(peer.train.dataset)} train samples / {len(peer.val.dataset)} validation samples "
f"/ {len(peer.test.dataset)} test samples / {len(peer.inference.dataset)} inference samples"
)
iterator = iter(peer.train)
x_batch, y_batch = iterator.next()
log(
'',
f"{peer} has: [{len(peer.train.dataset)}] {set(y_batch.numpy())}"
)
print()
def _cp_next_file2test_dir(self):
cp_cmd = self.test_info.get_cmd_cp_output2test_dir()
log(cp_cmd, LogType.RUN_CMD)
count = 0
while os.system(cp_cmd) != 0 and count < 600:
sleep(1)
count += 1
def add_cookies(self):
cookies = self.headers.get('Cookie', '')
kvs = cookies.split('; ')
log('cookie', kvs)
for kv in kvs:
if '=' in kv:
k, v = kv.split('=')
self.cookies[k] = v
def collaborative_training(self, learner, args):
t = time.time()
log('event',
f'Starting collaborative training using {learner.name} ...')
collab_logs = learner.collaborate(self, args)
t = time.time() - t
log("success", f"Collaborative training finished in {t:.2f} seconds.")
return collab_logs
def run(self):
address = self.socket.getsockname()
log('Starting checker (%s, %s) ' % (address[0], address[1]))
while True:
time.sleep(SECONDS_TO_WAIT_FOR_CHECK)
if (not self.sharedData['failProcess']):
self.communicationThread.checkIfLeaderIsAlive()
def __event_detected(self, pin_returned):
sensor = self.__configs.sensor_list[pin_returned]
topic = self.__configs.root_topic + sensor.topic
state = sensor.determine_state(self.__gpio.input)
event = Event(topic, state, utils.timestamp())
utils.log(event.log())
self.__mqtt.publish(topic, event.as_json())
def disconnect(self, neighbor_conn: NodeConnection):
if not neighbor_conn.terminate:
neighbor_conn.send(protocol.disconnect(self.id))
neighbor_conn.terminate = True
if neighbor_conn in self.neighbors:
self.neighbors.remove(neighbor_conn)
log('log',
f"{self} disconnected from {neighbor_conn.neighbor_name}")
def train_stop(peer):
inference_eval(peer)
acceptance_rate = peer.params.n_accept / peer.params.exchanges * 100
log(
'info',
f"{peer} Acceptance rate for alpha_max=({peer.params.alpha_max}): {acceptance_rate} %"
)
peer.stop()
return
def __new__(cls, *args, **kwargs):
if not cls.instance:
log("No Clipper instance found, creating new instance")
obj = object.__new__(cls)
cls.instance = obj
return obj
else:
log("Instance of Clipper already exists, return it")
return cls.instance
def get(driver, url, *args, **kwargs):
"""打开网址"""
try:
driver.get(url)
except:
log(u'open url: {0}'.format(url), 'failed : UnknowError')
raise UnknowError(msg='未知错误')
log(u'open url: {0}'.format(url), 'success.', 'info')
return driver
def run_cmd_with_perf_stat(cmd: list, stat_file_path, repeat=1, stdout_file_path=None, log_on=False):
# need root to run
perf_stat = ['/usr/bin/perf', 'stat', '-e', 'task-clock',
'--repeat', str(repeat), '-o', stat_file_path]
#perf_stat = ['/usr/bin/chrt', '-f', '99', '/usr/bin/perf', 'stat',
# '-e', 'task-clock', '--repeat', str(repeat), '-o', stat_file_path]
log(' '.join(cmd), LogType.RUN_CMD, _on=log_on)
p = Popen(perf_stat + cmd, stdout=DEVNULL, stderr=PIPE)
return p
def __getitem__(self, index):
if self.train is True:
return self.train_x_set[index], self.train_y_set[index]
elif self.train is False:
return self.test_x_set[index], self.test_y_set[index]
else:
log('error',
"Iterating MNIST dataset with Train=None is not supported")
exit()
def get(driver, url, *args, **kwargs):
"""打开网址"""
try:
driver.get(url)
except:
log(u'open url: {0}'.format(url), 'failed : UnknowError')
raise UnknowError(msg='未知错误')
log(u'open url: {0}'.format(url), 'success.', 'info')
return driver
def sig_handler(signo: int, _frame) -> None:
"""Log signal & pass handling to App.quit().
Simple callback to be given to signal.signal().
"""
utils.log(
"sig_handler processing {signo} signal"
.format(signo=signo))
APP.quit()
def _signandpost(self, param, name):
u"""签名并发送"""
sig = Encrypt().sign(param)
param['sign'] = sig
params_json = json.dumps(param)
# print params_json
log(name, params_json, 'info')
url = ReadXML(Config().get('data', 'url_xml')).get_url(name)
return self._header().post(url, params_json).content
def find(driver, by, value, *args, **kwargs):
if by.lower() in by_dict.keys():
try:
element = driver.find_element(by_dict[by.lower()], value)
except (NoSuchElementException, ElementNotVisibleException, ElementNotSelectableException) as e:
log(u'find by {0}: {1}'.format(by, value), e)
raise e
except Exception, e:
print e
log(u'find by {0}: {1}'.format(by, value), 'failed : UnkownError')
raise UnknowError(msg='未知错误')
def open_browser(browser='firefox', profile=None, *args, **kwargs):
"""打开浏览器，允许三种类型，firefox,chrome,IE,否则会抛出异常。
默认火狐浏览器，火狐浏览器可以使用指定的配置文件"""
if browser.lower() == 'firefox':
if profile is None:
driver = webdriver.Firefox()
else:
driver = webdriver.Firefox(profile)
elif browser.lower() == 'chrome':
driver = webdriver.Chrome(TOOLS_PATH + 'browsertools\\chromedriver.exe')
elif browser.lower() == 'ie':
driver = webdriver.Ie(TOOLS_PATH + 'browsertools\\IEDriverServer.exe')
else:
log(u'open browser {0}'.format(browser), 'failed : UnsupportBrowser')
raise UnsupportBrowser(msg='不支持的浏览器类型')
driver.maximize_window()
driver.implicitly_wait(30)
log(u'open browser {0}'.format(browser), 'success.', 'info')
return driver
def check(fun):
"""检查是否能够获取到元素，如果可以返回该元素，否则抛出异常"""
def find(driver, by, value, *args, **kwargs):
if by.lower() in by_dict.keys():
try:
element = driver.find_element(by_dict[by.lower()], value)
except (NoSuchElementException, ElementNotVisibleException, ElementNotSelectableException) as e:
log(u'find by {0}: {1}'.format(by, value), e)
raise e
except Exception, e:
print e
log(u'find by {0}: {1}'.format(by, value), 'failed : UnkownError')
raise UnknowError(msg='未知错误')
else:
log(u'find by {0}: {1}'.format(by, value), 'failed : InvalidSelectorException')
raise InvalidSelectorException(msg='没有匹配的定位方式')
fun(driver, by, value, *args, **kwargs)
return driver
return find
@ check
def click(driver, by, value, *args, **kwargs):
driver.find_element(by_dict[by.lower()], value).click()
log(u'click by {0}: {1}'.format(by, value), 'success.', 'info')
@ check
def send_keys(driver, by, value, text, *args, **kwargs):
driver.find_element(by_dict[by.lower()], value).send_keys(text)
def quitdriver(driver, *args, **kwargs):
driver.quit()
log(u'quit driver {0}'.format(driver), 'success.', 'info')
def close(driver, *args, **kwargs):
driver.close()
log(u'close driver {0}'.format(driver), 'success.', 'info')
def clear(driver, by, value, *args, **kwargs):
driver.find_element(by_dict[by.lower()], value).clear()
log(u'clear by {0}: {1}'.format(by, value), 'success.', 'info')
def send_keys(driver, by, value, text, *args, **kwargs):
driver.find_element(by_dict[by.lower()], value).send_keys(text)
log(u'send keys by {0}: {1}, text="{2}"'.format(by, value, text), 'success.', 'info')
