def get_tasks(self):
packages = self.get_packages_and_dependencies()
tasks = Tasks()
for package, dependencies in packages.iteritems():
tasks.add(package, dependencies)
return tasks
def handle_spam(post, reasons, why):
datahandling.append_to_latest_questions(post.post_site, post.post_id, post.title if not post.is_answer else "")
if len(reasons) == 1 and ("all-caps title" in reasons or
"repeating characters in title" in reasons or
"repeating characters in body" in reasons or
"repeating characters in answer" in reasons or
"repeating words in title" in reasons or
"repeating words in body" in reasons or
"repeating words in answer" in reasons):
datahandling.add_auto_ignored_post((post.post_id, post.post_site, datetime.utcnow()))
if why is not None and why != "":
datahandling.add_why(post.post_site, post.post_id, why)
if post.is_answer and post.post_id is not None and post.post_id != "":
datahandling.add_post_site_id_link((post.post_id, post.post_site, "answer"), post.parent.post_id)
try:
post_url = parsing.to_protocol_relative(parsing.url_to_shortlink(post.post_url))
poster_url = parsing.to_protocol_relative(parsing.user_url_to_shortlink(post.user_url))
if not post.user_name.strip() or (not poster_url or poster_url.strip() == ""):
username = ""
else:
username = post.user_name.strip()
Tasks.do(metasmoke.Metasmoke.send_stats_on_post,
post.title_ignore_type, post_url, reasons, post.body, username,
post.user_link, why, post.owner_rep, post.post_score,
post.up_vote_count, post.down_vote_count)
offensive_mask = 'offensive title detected' in reasons
message = build_message(post, reasons)
if offensive_mask:
post.title = "(potentially offensive title -- see MS for details)"
clean_message = build_message(post, reasons)
log('debug', GlobalVars.parser.unescape(message).encode('ascii', errors='replace'))
GlobalVars.deletion_watcher.subscribe(post_url)
without_roles = tuple(["no-" + reason for reason in reasons]) + ("site-no-" + post.post_site,)
if set(reasons) - GlobalVars.experimental_reasons == set() and \
not why.startswith("Post manually "):
chatcommunicate.tell_rooms(message, ("experimental-all-sites", "experimental-site-" + post.post_site),
without_roles, notify_site=post.post_site, report_data=(post_url, poster_url))
else:
if offensive_mask:
chatcommunicate.tell_rooms(message, ("all-sites", "site-" + post.post_site),
without_roles + ("offensive-mask",), notify_site=post.post_site,
report_data=(post_url, poster_url))
chatcommunicate.tell_rooms(clean_message, ("all-sites", "site-" + post.post_site),
without_roles + ("no-offensive-mask",), notify_site=post.post_site,
report_data=(post_url, poster_url))
else:
chatcommunicate.tell_rooms(message, ("all-sites", "site-" + post.post_site),
without_roles, notify_site=post.post_site,
report_data=(post_url, poster_url))
except Exception as e:
excepthook.uncaught_exception(*sys.exc_info())
def processMain(matrixIdx):
global mainPidProcess
threads = []
actualTasks = Tasks(os.getpid() - mainPidProcess, matrixIdx)
lockErrors = threading.Lock()
lockCompleted = threading.Lock()
# for i in range(numthreads):
# threads.append(Thread(target=threadsVerify, args=(matrixIdx,actualTasks)))
# for i in range(numthreads):
# threads[i].start()
# for i in range(numthreads):
# threads[i].join()
with ThreadPoolExecutor(max_workers=numthreads) as pool:
for i in range(9):
pool.submit(checkRow, matrixIdx, actualTasks, lockErrors,
lockCompleted)
pool.submit(checkCollum, matrixIdx, actualTasks, lockErrors,
lockCompleted)
pool.submit(checkArea, matrixIdx, actualTasks, lockErrors,
lockCompleted)
with lock:
actualTasks.setFinalLog()
def __init__(self):
if GlobalVars.no_edit_watcher:
self.socket = None
return
# posts is a dict with the WebSocket action as keys {site_id}-question-{question_id} as keys
# with each value being: (site_id, hostname, question_id, max_time)
self.posts = {}
self.posts_lock = threading.Lock()
self.save_handle = None
self.save_handle_lock = threading.Lock()
try:
self.socket = websocket.create_connection(
"wss://qa.sockets.stackexchange.com/")
except websocket.WebSocketException:
self.socket = None
log('error', 'EditWatcher failed to create a websocket connection')
if datahandling.has_pickle(PICKLE_FILENAME):
pickle_data = datahandling.load_pickle(PICKLE_FILENAME)
now = time.time()
new_posts = {
action: value
for action, value in pickle_data if value[-1] > now
}
with self.posts_lock:
self.posts = new_posts
for action in new_posts.keys():
Tasks.do(self._subscribe, action)
self._schedule_save()
threading.Thread(name="edit watcher", target=self._start,
daemon=True).start()
def compareFixed():
t = Tasks()
x_test, y_test = t.sequence_type_1(100)
add_params, mul_params = torch.load('program_memory/add.pt'), torch.load(
'program_memory/mul.pt')
hnm = HNM(10, 20, add_params, mul_params)
hnm.load_state_dict(torch.load("learned_params/hnm_arch_2.pt"))
ntm = NTM(10, 20)
ntm.load_state_dict(torch.load("learned_params/ntm.pt"))
lstm = LSTM(14, 256, 325, 1)
lstm.load_state_dict(torch.load("learned_params/lstm.pt"))
hnm_diff, lstm_diff, ntm_diff = 0, 0, 0
for i in range(len(x_test)):
hnm_out = hnm.recurrent_forward(x_test[i:i + 1])
ntm_out = ntm.recurrent_forward(x_test[i:i + 1])
lstm_out = lstm.recurrent_forward(x_test[i:i + 1])
answer = np.argmax(y_test[i:i + 1].detach().numpy())
hnm_diff += abs(answer - np.argmax(hnm_out.detach().numpy()))
ntm_diff += abs(answer - np.argmax(ntm_out.detach().numpy()))
lstm_diff += abs(answer - np.argmax(lstm_out.detach().numpy()))
print(hnm_diff / len(y_test), ntm_diff / len(y_test),
lstm_diff / len(y_test))
def handle(cls, content):
for k, d in content.items():
if k in cls._callbacks:
for cb in cls._callbacks[k]:
cb(d)
if "metasmoke_state" in content:
if content["metasmoke_state"] == "down":
log(
'info',
"{} says metasmoke is down, switching to active ping monitoring."
.format(content["location"]))
GlobalVars.metasmoke_down = True
Tasks.later(SocketScience.check_recent_pings, after=90)
if content["metasmoke_state"] == "up":
log(
'info',
'{} says metasmoke is up, disabling ping monitoring.'.
format(content["location"]))
GlobalVars.metasmoke_down = False
if "ping" in content:
cls._pings.append({
"timestamp": content["ping"],
"location": content["location"]
})
if cls._switch_task is not None:
cls._switch_task.cancel()
def do_c(self, line):
"""
Task completati
"""
task_ids = line.split(',')
for task_id in task_ids:
Tasks.updateTaskToComplete(task_id)
def _start(self):
while True:
msg = self.socket.recv()
if msg:
msg = json.loads(msg)
action = msg["action"]
if action == "hb":
self.socket.send("hb")
else:
data = json.loads(msg["data"])
if data["a"] == "post-deleted":
try:
post_id, _, post_type, post_url, callbacks = self.posts[
action]
del self.posts[action]
if not post_type == "answer" or (
"aId" in data
and str(data["aId"]) == post_id):
self.socket.send("-" + action)
Tasks.do(
metasmoke.Metasmoke.
send_deletion_stats_for_post, post_url,
True)
for callback, max_time in callbacks:
if not max_time or time.time() < max_time:
callback()
except KeyError:
pass
def _start(self):
while True:
msg = self.socket.recv()
if msg:
msg = json.loads(msg)
action = msg["action"]
if action == "hb":
self.socket.send("hb")
else:
data = json.loads(msg["data"])
if data["a"] == "post-deleted":
try:
post_id, _, post_type, post_url, callbacks = self.posts[action]
del self.posts[action]
if not post_type == "answer" or ("aId" in data and str(data["aId"]) == post_id):
self.socket.send("-" + action)
Tasks.do(metasmoke.Metasmoke.send_deletion_stats_for_post, post_url, True)
for callback, max_time in callbacks:
if not max_time or time.time() < max_time:
callback()
except KeyError:
pass
def countTasks(self):
"""Count tasks statistics divided by projects"""
self._data = DataManager.getByRange(self._fromDate, self._toDate)
res = {}
for date, task, projectName in self._data:
if task == "__start__":
self.timings.setPrevDate(None)
spentSeconds = self.timings.count(date, Tasks.taskType(task))
if Tasks.taskType(task) != "work":
continue
if spentSeconds:
if projectName not in res:
res[projectName] = {}
if task not in res[projectName]:
res[projectName][task] = spentSeconds
else:
res[projectName][task] += spentSeconds
self._countAttrib([v for k in res for v in res[k].values()])
if res:
ret = {}
for k in res.keys():
ret[k] = sorted(res[k].iteritems(), key=lambda item: item[1], reverse=True)
return ret
else:
return {}
def dispatch_reply_command(msg, reply, full_cmd, comment=True):
command_parts = full_cmd.split(" ", 1)
if len(command_parts) == 2:
cmd, args = command_parts
else:
cmd, = command_parts
args = ""
cmd = cmd.lower()
quiet_action = cmd[-1] == "-"
cmd = regex.sub(r"\W*$", "", cmd)
if cmd in _reply_commands:
func, (min_arity, max_arity) = _reply_commands[cmd]
assert min_arity == 1
if max_arity == 1:
return func(msg, original_msg=reply, alias_used=cmd, quiet_action=quiet_action)
elif max_arity == 2:
return func(msg, args, original_msg=reply, alias_used=cmd, quiet_action=quiet_action)
else:
args = args.split()
args.extend([None] * (max_arity - len(args)))
return func(msg, *args, original_msg=reply, alias_used=cmd, quiet_action=quiet_action)
elif comment and is_privileged(reply.owner, reply.room):
post_data = get_report_data(msg)
if post_data:
Tasks.do(metasmoke.Metasmoke.post_auto_comment, full_cmd, reply.owner, url=post_data[0])
def do_start(self, line):
"""
Avvia sessione pomodoro
"""
task_ids = line.split(',')
if not task_ids[0]:
print "Devi inserire un task da cui vuoi partire"
return
firstTodayTask = Tasks.getNextTask(task_ids.pop(0))
numberOfPomodoro = 1
while True:
pomodoroTimer = TimerCountdown(1, firstTodayTask['task'], 15)
pomodoroTimer.daemon = True
pomodoroTimer.start()
pomodoroTimer.join()
print
user_input = raw_input("Ti serve altro tempo per completare il task? [s/n]")
if (user_input == 's'):
numberOfPomodoro += 1
else:
Tasks.updatePomodoroByTodayTask(firstTodayTask['taskid'], numberOfPomodoro)
break
def get_tasks(self):
packages = self.get_packages_and_dependencies()
tasks = Tasks()
for package, dependencies in packages.iteritems():
tasks.add(package, dependencies)
return tasks
def handle(content):
global _pings
global _switch_task
if "metasmoke_state" in content:
if content["metasmoke_state"] == "down":
log(
'info',
"{} says metasmoke is down, switching to active ping monitoring."
.format(content["location"]))
GlobalVars.metasmoke_down = True
Tasks.later(SocketScience.check_recent_pings, after=90)
if content["metasmoke_state"] == "up":
log(
'info',
'{} says metasmoke is up, disabling ping monitoring.'.
format(content["location"]))
GlobalVars.metasmoke_down = False
if "ping" in content:
_pings.append({
"timestamp": content["ping"],
"location": content["location"]
})
if _switch_task is not None:
_switch_task.cancel()
def subscribe(self, post_url, callback=None, pickle=True, timeout=None):
post_id, post_site, post_type = fetch_post_id_and_site_from_url(
post_url)
if post_site not in GlobalVars.site_id_dict:
log(
"warning", "unknown site {} when subscribing to {}".format(
post_site, post_url))
return
if post_type == "answer":
question_id = datahandling.get_post_site_id_link(
(post_id, post_site, post_type))
if question_id is None:
return
else:
question_id = post_id
site_id = GlobalVars.site_id_dict[post_site]
action = "{}-question-{}".format(site_id, question_id)
max_time = (time.time() + timeout) if timeout else None
if action in self.posts and callback:
_, _, _, _, callbacks = self.posts[action]
callbacks.append((callback, max_time))
else:
self.posts[action] = (post_id, post_site, post_type, post_url,
[(callback, max_time)] if callback else [])
self.socket.send(action)
if pickle:
Tasks.do(self._save)
def subscribe(self, post_url=None, hostname=None, site_id=None, question_id=None,
pickle=True, timeout=DEFAULT_TIMEOUT, max_time=None, from_time=None):
if GlobalVars.no_edit_watcher:
return
if post_url and not ((hostname or site_id) and question_id):
post_id, hostname, post_type = fetch_post_id_and_site_from_url(post_url)
if post_type == "answer":
question_id = datahandling.get_post_site_id_link((post_id, hostname, post_type))
if question_id is None:
log("warning", "Unable to get question ID when subscribing to: hostname: "
"{} :: post ID:{} when subscribing to {}".format(hostname, post_id, post_url))
return
else:
question_id = post_id
if post_type != "question":
log("warning", "tried to edit-watch non-question: hostname: "
"{} :: post ID:{} when subscribing to {}".format(hostname, question_id, post_url))
return
if not site_id or not hostname:
with GlobalVars.site_id_dict_lock:
if not site_id and hostname:
site_id = GlobalVars.site_id_dict.get(hostname)
if site_id and not hostname:
hostname = GlobalVars.site_id_dict_by_id.get(site_id)
if not site_id or not hostname:
log("warning", "unable to determine a valid site ID or hostname when subscribing to question ID "
"{}:: site_id:{}:: hostname:{}:: post_url:{}".format(question_id, site_id, hostname,
post_url))
return
question_ids = question_id
if type(question_ids) != list:
question_ids = [question_id]
now = time.time()
if from_time:
now = from_time
if not max_time:
max_time = now + timeout
updated = None
to_subscribe = []
with self.posts_lock:
for question_id in question_ids:
action = "{}-question-{}".format(site_id, question_id)
if action not in self.posts:
self.posts[action] = (site_id, hostname, question_id, max_time)
to_subscribe.append(action)
else:
old_max_time = self.posts[action][2]
if max_time > old_max_time:
self.posts[action] = (site_id, hostname, question_id, max_time)
elif updated is None:
updated = False
for action in to_subscribe:
print('scheduling subscription to action:', action)
Tasks.do(self._subscribe, action)
if updated and pickle:
self._schedule_save()
def dispatch_reply_command(msg, reply, full_cmd, comment=True):
command_parts = full_cmd.split(" ", 1)
if len(command_parts) == 2:
cmd, args = command_parts
else:
cmd, = command_parts
args = ""
cmd = cmd.lower()
quiet_action = cmd[-1] == "-"
cmd = regex.sub(r"\W*$", "", cmd)
if cmd in _reply_commands:
func, (min_arity, max_arity) = _reply_commands[cmd]
assert min_arity == 1
if max_arity == 1:
return func(msg, original_msg=reply, alias_used=cmd, quiet_action=quiet_action)
elif max_arity == 2:
return func(msg, args, original_msg=reply, alias_used=cmd, quiet_action=quiet_action)
else:
args = args.split()
args.extend([None] * (max_arity - len(args)))
return func(msg, *args, original_msg=reply, alias_used=cmd, quiet_action=quiet_action)
elif comment and is_privileged(reply.owner, reply.room):
post_data = get_report_data(msg)
if post_data:
Tasks.do(metasmoke.Metasmoke.post_auto_comment, full_cmd, reply.owner, url=post_data[0])
def _start(self):
while True:
msg = self.socket.recv()
if msg:
msg = json.loads(msg)
action = msg["action"]
if action == "hb":
self.socket.send("hb")
else:
data = json.loads(msg["data"])
if data["a"] == "post-deleted":
try:
with self.posts_lock:
post_id, _, _, post_url, callbacks = self.posts[
action]
if post_id == str(data["aId"] if "aId" in
data else data["qId"]):
with self.posts_lock:
del self.posts[action]
Tasks.do(self._unsubscribe, action)
Tasks.do(
metasmoke.Metasmoke.
send_deletion_stats_for_post, post_url,
True)
for callback, max_time in callbacks:
if not max_time or time.time() < max_time:
callback()
except KeyError:
pass
def _countObject(self, objType, targetAction):
"""Generic function for calculating projects data or slacking statistics"""
self._data = DataManager.getByRange(self._fromDate, self._toDate)
res = {}
for date, task, projectName in self._data:
if task == "__start__":
self.timings.setPrevDate(None)
objKey = projectName if objType == "project" else task
spentSeconds = self.timings.count(date, Tasks.taskType(task))
if Tasks.taskType(task) != targetAction:
self.timings.setPrevDate(date)
continue
if spentSeconds:
if objKey not in res:
res[objKey] = spentSeconds
else:
res[objKey] += spentSeconds
self._countAttrib(res.values())
if res:
return sorted(res.iteritems(), key=lambda item:item[1], reverse=True)
else:
return []
def do_add(self, line):
"""
Aggiunge un nuovo task add <task> <numero_pomodori> <tags>
"""
words = line.split(' ')
Tasks.addTasks(words)
Tasks.writeTaskWithTags()
def __init__(self, runtime_env, ini_file="config.ini", os=None):
self.logger = Logger(log_level=0, console_level=3)
self.sts = Settings(self.logger, runtime_env, ini_file=ini_file) # sts
self.logger.set_console_level(self.sts.debug_level)
self.logger.set_log_level(0)
self.comfun = CommonFunctions(self.logger)
if os is None:
self.os = platform.system()
if self.os == "Windows":
self.os = "w"
elif self.os == "Linux":
self.os = "l"
else:
self.os = None
# abbreviation for very often used variables, helping with identification the main modules
self.usr = Users(self) # usr
self.prj = Projects(self) # prj
self.sch = Schemas(self) # sch
self.tsk = Tasks(self) # tsk
self.que = Queue(self) # que
self.nod = SimNodes(self) # nod
self.dfn = Definitions(self) # dfn
self.sio = StorageInOut(self) # sio
# abbreviation END
self.logger.inf("SimBatch started")
def trainNTM():
t = Tasks()
x_train, y_train = t.sequence_type_1(2000)
ntm = NTM(10, 20)
ntm.train(x_train, y_train, 1, maxEpoch=25, learning_rate=0.0006)
def subscribe(self, post_url, callback=None, pickle=True, timeout=None):
post_id, post_site, post_type = fetch_post_id_and_site_from_url(post_url)
if post_site not in GlobalVars.site_id_dict:
log("warning", "unknown site {} when subscribing to {}".format(post_site, post_url))
return
if post_type == "answer":
question_id = datahandling.get_post_site_id_link((post_id, post_site, post_type))
if question_id is None:
return
else:
question_id = post_id
site_id = GlobalVars.site_id_dict[post_site]
action = "{}-question-{}".format(site_id, question_id)
max_time = (time.time() + timeout) if timeout else None
if action not in self.posts:
self.posts[action] = (post_id, post_site, post_type, post_url, [(callback, max_time)] if callback else [])
try:
self.socket.send(action)
except websocket.WebSocketException:
log('error', 'DeletionWatcher failed on sending {}'.format(action))
elif callback:
_, _, _, _, callbacks = self.posts[action]
callbacks.append((callback, max_time))
else:
return
if pickle:
Tasks.do(self._save)
def main():
import argparse
parser = argparse.ArgumentParser(
description='Exhaustively try project-toolchain combinations')
parser.add_argument('--project',
default=None,
nargs="+",
help='run given project only (default: all)')
parser.add_argument('--toolchain',
default=None,
nargs="+",
help='run given toolchain only (default: all)')
parser.add_argument('--out-prefix',
default='build',
help='output directory prefix (default: build)')
parser.add_argument(
'--build_type',
default='generic',
help=
'Type of build that is performed (e.g. regression test, multiple options, etc.)'
)
parser.add_argument('--build', default=None, help='Build number')
parser.add_argument('--parameters',
default=None,
help='Tool parameters json file')
parser.add_argument('--fail', action='store_true', help='fail on error')
parser.add_argument('--verbose',
action='store_true',
help='verbose output')
args = parser.parse_args()
tasks = Tasks(src_dir)
args_dict = {"project": args.project, "toolchain": args.toolchain}
task_list = tasks.get_tasks(args_dict)
params_file = args.parameters
params_strings = [None]
if params_file:
params_strings = []
assert len(
args.toolchain
) == 1, "A single toolchain can be selected when running multiple params."
params_helper = ToolParametersHelper(args.toolchain[0], params_file)
for params in params_helper.get_all_params_combinations():
params_strings.append(" ".join(params))
runner = Runner(task_list, args.verbose, args.out_prefix, root_dir,
args.build_type, args.build, params_strings)
runner.run()
runner.collect_results()
result = print_summary_table(args.out_prefix, args.build_type, args.build)
if not result and args.fail:
print("ERROR: some tests have failed.")
exit(1)
def on_msg(msg, client):
global _room_roles
if not isinstance(msg, events.MessagePosted) and not isinstance(
msg, events.MessageEdited):
return
message = msg.message
room_ident = (client.host, message.room.id)
with _room_roles_lock:
if message.owner.id == client._br.user_id:
if 'direct' in _room_roles and room_ident in _room_roles['direct']:
SocketScience.receive(
message.content_source.replace("\u200B",
"").replace("\u200C", ""))
return
if message.content.startswith("<div class='partial'>"):
message.content = message.content[21:]
if message.content.endswith("</div>"):
message.content = message.content[:-6]
if message.parent:
try:
if message.parent.owner.id == client._br.user_id:
strip_mention = regex.sub(
"^(<span class=(\"|')mention(\"|')>)?@.*?(</span>)? ", "",
message.content)
cmd = GlobalVars.parser.unescape(strip_mention)
result = dispatch_reply_command(message.parent, message, cmd)
send_reply_if_not_blank(room_ident, message.id, result)
except ValueError:
pass
elif message.content.lower().startswith("sd "):
result = dispatch_shorthand_command(message)
send_reply_if_not_blank(room_ident, message.id, result)
elif message.content.startswith(
"!!/") or message.content.lower().startswith("sdc "):
result = dispatch_command(message)
send_reply_if_not_blank(room_ident, message.id, result)
elif classes.feedback.FEEDBACK_REGEX.search(message.content) \
and is_privileged(message.owner, message.room) and datahandling.last_feedbacked:
ids, expires_in = datahandling.last_feedbacked
if time.time() < expires_in:
Tasks.do(metasmoke.Metasmoke.post_auto_comment,
message.content_source,
message.owner,
ids=ids)
else:
with _room_roles_lock:
if 'direct' in _room_roles and room_ident in _room_roles['direct']:
SocketScience.receive(
message.content_source.replace("\u200B",
"").replace("\u200C", ""))
def __init__(self, fileName=None):
fileName = 'tasks.json' if fileName is None else fileName
# get the args object
self.args = self.createArgsObj()
self.conf = self.getDefaultConf()
tasks_dir = self.args.task_dir
self.tasks = Tasks(fileName, tasks_dir)
def do_mv(self, line):
"""
Sposta i task dall'inventario al todaySheet (rappresenta i task presi in carico)
"""
tasksid = line.split(',')
for taskid in tasksid:
self._todaySheet = TodaySheet(taskid)
Tasks.writeTasksInTodaySheet(self._todaySheet.makeRecordToWriteOnDB())
Tasks.deleteSoftFromTasksInventory(taskid)
def trainHarvard():
t1 = Tasks()
x_train, y_train = t1.sequence_type_1(2000)
add_params, mul_params = torch.load('program_memory/add.pt'), torch.load(
'program_memory/mul.pt')
hnm = HNM(10, 20, add_params, mul_params)
hnm.train(x_train, y_train, 1)
def on_msg(msg, client):
global _room_roles
if not isinstance(msg, events.MessagePosted) and not isinstance(msg, events.MessageEdited):
return
message = msg.message
room_ident = (client.host, message.room.id)
room_data = _rooms[room_ident]
if message.owner.id == client._br.user_id:
if 'direct' in _room_roles and room_ident in _room_roles['direct']:
SocketScience.receive(message.content_source.replace("\u200B", "").replace("\u200C", ""))
return
if message.content.startswith("<div class='partial'>"):
message.content = message.content[21:]
if message.content.endswith("</div>"):
message.content = message.content[:-6]
if message.parent:
try:
if message.parent.owner.id == client._br.user_id:
strip_mention = regex.sub("^(<span class=(\"|')mention(\"|')>)?@.*?(</span>)? ", "", message.content)
cmd = GlobalVars.parser.unescape(strip_mention)
result = dispatch_reply_command(message.parent, message, cmd)
if result:
s = ":{}\n{}" if "\n" not in result and len(result) >= 488 else ":{} {}"
_msg_queue.put((room_data, s.format(message.id, result), None))
except ValueError:
pass
elif message.content.lower().startswith("sd "):
result = dispatch_shorthand_command(message)
if result:
s = ":{}\n{}" if "\n" not in result and len(result) >= 488 else ":{} {}"
_msg_queue.put((room_data, s.format(message.id, result), None))
elif message.content.startswith("!!/"):
result = dispatch_command(message)
if result:
s = ":{}\n{}" if "\n" not in result and len(result) >= 488 else ":{} {}"
_msg_queue.put((room_data, s.format(message.id, result), None))
elif classes.feedback.FEEDBACK_REGEX.search(message.content) \
and is_privileged(message.owner, message.room) and datahandling.last_feedbacked:
ids, expires_in = datahandling.last_feedbacked
if time.time() < expires_in:
Tasks.do(metasmoke.Metasmoke.post_auto_comment, message.content_source, message.owner, ids=ids)
elif 'direct' in _room_roles and room_ident in _room_roles['direct']:
SocketScience.receive(message.content_source.replace("\u200B", "").replace("\u200C", ""))
def do_ta(self, line):
"""
Visualizza tutti i task completati nei giorni precedenti
"""
strSearch = ""
find_tag = line.split(' ')
if (len(find_tag[0]) > 0):
if ('+' not in find_tag[0]):
strSearch = '+' + find_tag[0]
Tasks.findAllTodaySheets(strSearch)
def on_msg(msg, client):
if not isinstance(msg, events.MessagePosted) and not isinstance(
msg, events.MessageEdited):
return
message = msg.message
if message.owner.id == client._br.user_id:
return
if message.content.startswith("<div class='partial'>"):
message.content = message.content[21:]
if message.content.endswith("</div>"):
message.content = message.content[:-6]
room_data = _rooms[(client.host, message.room.id)]
if message.parent:
try:
if message.parent.owner.id == client._br.user_id:
strip_mention = regex.sub(
"^(<span class=(\"|')mention(\"|')>)?@.*?(</span>)? ", "",
message.content)
cmd = GlobalVars.parser.unescape(strip_mention)
result = dispatch_reply_command(message.parent, message, cmd)
if result:
_msg_queue.put(
(room_data, ":{} {}".format(message.id, result), None))
except ValueError:
pass
elif message.content.lower().startswith("sd "):
result = dispatch_shorthand_command(message)
if result:
_msg_queue.put((room_data, ":{} {}".format(message.id,
result), None))
elif message.content.startswith("!!/"):
result = dispatch_command(message)
if result:
_msg_queue.put((room_data, ":{} {}".format(message.id,
result), None))
elif classes.feedback.FEEDBACK_REGEX.search(message.content) \
and is_privileged(message.owner, message.room) and datahandling.last_feedbacked:
ids, expires_in = datahandling.last_feedbacked
if time.time() < expires_in:
Tasks.do(metasmoke.Metasmoke.post_auto_comment,
message.content_source,
message.owner,
ids=ids)
def _load_tasks(cls, input_path, args, label_config_file):
with io.open(label_config_file, encoding='utf8') as f:
label_config = f.read()
task_loader = Tasks()
if args.input_format == 'json':
return task_loader.from_json_file(input_path)
if args.input_format == 'json-dir':
return task_loader.from_dir_with_json_files(input_path)
input_data_tags = cls.get_input_data_tags(label_config)
if len(input_data_tags) > 1:
val = ",".join(tag.attrib.get("name") for tag in input_data_tags)
print('Warning! Multiple input data tags found: ' + val +
'. Only first one is used.')
elif len(input_data_tags) == 0:
raise ValueError(
'You\'ve specified input format "{fmt}" which requires label config being explicitly defined. '
'Please specify --label-config=path/to/config.xml or use --format=json or format=json_dir'
.format(fmt=args.input_format))
input_data_tag = input_data_tags[0]
data_key = input_data_tag.attrib.get('value').lstrip('$')
if args.input_format == 'text':
return task_loader.from_text_file(input_path, data_key)
if args.input_format == 'text-dir':
return task_loader.from_dir_with_text_files(input_path, data_key)
if args.input_format == 'image-dir':
return task_loader.from_dir_with_image_files(input_path, data_key)
if args.input_format == 'audio-dir':
return task_loader.from_dir_with_audio_files(input_path, data_key)
raise RuntimeError('Can\'t load tasks for input format={}'.format(
args.input_format))
def sum_weight(reasons: list):
if not GlobalVars.reason_weights:
datahandling.update_reason_weights()
now = datetime.utcnow() - timedelta(minutes=15)
if now.date() != GlobalVars.reason_weights['last_updated'] and now.hour >= 1:
Tasks.do(datahandling.update_reason_weights)
s = 0
weights = GlobalVars.reason_weights
for r in reasons:
try:
if "(" in r:
r = regex.sub(r"\s*\(.*$", "", r)
s += weights[r.lower()]
except KeyError:
pass # s += 0
return s
def sum_weight(reasons: list):
if not GlobalVars.reason_weights:
datahandling.update_reason_weights()
now = datetime.utcnow() - timedelta(minutes=15)
if now.date() != GlobalVars.reason_weights['last_updated'] and now.hour >= 1:
Tasks.do(datahandling.update_reason_weights)
s = 0
weights = GlobalVars.reason_weights
for r in reasons:
try:
if "(" in r:
r = regex.sub(r"\s*\(.*$", "", r)
s += weights[r.lower()]
except KeyError:
pass # s += 0
return s
def tell_rooms(msg, has, hasnt, notify_site="", report_data=None):
global _rooms
msg = msg.rstrip()
target_rooms = set()
# Go through the list of properties in "has" and add all rooms which have any of those properties
# to the target_rooms set. _room_roles contains a list of rooms for each property.
for prop_has in has:
if isinstance(prop_has, tuple):
# If the current prop_has is a tuple, then it's assumed to be a descriptor of a specific room.
# The format is: (_client.host, room.id)
target_rooms.add(prop_has)
if prop_has not in _room_roles:
# No rooms have this property.
continue
for room in _room_roles[prop_has]:
if all(map(lambda prop: prop not in _room_roles or room not in _room_roles[prop], hasnt)):
if room not in _rooms:
# If SD is not already in the room, then join the room.
site, roomid = room
deletion_watcher = room in _watcher_rooms
new_room = _clients[site].get_room(roomid)
new_room.join()
_rooms[room] = RoomData(new_room, -1, deletion_watcher)
target_rooms.add(room)
for room_id in target_rooms:
room = _rooms[room_id]
if notify_site:
pings = datahandling.get_user_names_on_notification_list(room.room._client.host,
room.room.id,
notify_site,
room.room._client)
msg_pings = datahandling.append_pings(msg, pings)
else:
msg_pings = msg
timestamp = time.time()
if room.block_time < timestamp and _global_block < timestamp:
if report_data and "delay" in _room_roles and room_id in _room_roles["delay"]:
def callback(room=room, msg=msg_pings):
post = fetch_post_id_and_site_from_url(report_data[0])[0:2]
if not datahandling.is_false_positive(post) and not datahandling.is_ignored_post(post):
_msg_queue.put((room, msg, report_data))
task = Tasks.later(callback, after=300)
GlobalVars.deletion_watcher.subscribe(report_data[0], callback=task.cancel)
else:
_msg_queue.put((room, msg_pings, report_data))
def tell_rooms(msg, has, hasnt, notify_site="", report_data=None):
global _rooms
msg = msg.rstrip()
target_rooms = set()
for prop_has in has:
if isinstance(prop_has, tuple):
target_rooms.add(prop_has)
if prop_has not in _room_roles:
continue
for room in _room_roles[prop_has]:
if all(
map(
lambda prop: prop not in _room_roles or room not in
_room_roles[prop], hasnt)):
if room not in _rooms:
site, roomid = room
deletion_watcher = room in _watcher_rooms
new_room = _clients[site].get_room(roomid)
new_room.join()
_rooms[room] = RoomData(new_room, -1, deletion_watcher)
target_rooms.add(room)
for room_id in target_rooms:
room = _rooms[room_id]
if notify_site:
pings = datahandling.get_user_names_on_notification_list(
room.room._client.host, room.room.id, notify_site,
room.room._client)
msg_pings = datahandling.append_pings(msg, pings)
else:
msg_pings = msg
timestamp = time.time()
if room.block_time < timestamp and _global_block < timestamp:
if report_data and "delay" in _room_roles and room_id in _room_roles[
"delay"]:
def callback(room=room, msg=msg_pings):
post = fetch_post_id_and_site_from_url(report_data[0])[0:2]
if not datahandling.is_false_positive(
post) and not datahandling.is_ignored_post(post):
_msg_queue.put((room, msg, report_data))
task = Tasks.later(callback, after=300)
GlobalVars.deletion_watcher.subscribe(report_data[0],
callback=task.cancel)
else:
_msg_queue.put((room, msg_pings, report_data))
def schedule_store_recently_scanned_posts():
global recently_scanned_posts_save_handle
with recently_scanned_posts_save_handle_lock:
if recently_scanned_posts_save_handle:
recently_scanned_posts_save_handle.cancel()
recently_scanned_posts_save_handle = Tasks.do(
store_recently_scanned_posts)
def create_tasks(self):
tasks = Tasks()
tasks.add(self.check_gamepad_timeout)
tasks.add_periodic(300, self.check_gamepad_battery)
if self._tv:
tasks.add_periodic(60, self.check_tv_power_status)
return tasks
def __init__(self):
""" HTTP Server
"""
self.RECEIVED_DTMF_MSG = u"Bien reçu! Au revoir et bonne journée!"
self.GENERATE_TOKEN = False
self.COLLECT_SUBJECTIVE_FEEDBACK = False
self.COLLECT_TASK = False
self.host = 'localhost'
self.port = 8082
self.TOKENSERVERURL = 'http://www.camdial.org/~djv27/mt-multiDomain/receive-token.py'
# DTMF prompt given over voip call at dialogue's end
self.COLLECT_TASKID_PROMPT = '. Please now enter the 5 digit task number followed by the hash key'
#The hash key is only for multiple digits
self.COLLECT_SUBJECTIVE_PROMPT = u" Avez-vous bien obtenu toutes les informations dont vous aviez besoin? Répondez simplement par oui ou non, s'il vous plaît."
self.ood_count = 0
self.OOD_THRESHOLD = 1
# Speech Settings:
self.allowbargein = False # TODO - config this or remove it -- OR SHOULD IT BE SOMETHING SENT FROM VOICEBROKER EACH TURN?
if Settings.config.has_option("dialogueserver", "tokenserverurl"):
self.TOKENSERVERURL = Settings.config.get("dialogueserver",
"tokenserverurl")
self.TOKENSERVERURL = self.TOKENSERVERURL.strip('"')
if Settings.config.has_option("dialogueserver", "generatetoken"):
self.GENERATE_TOKEN = Settings.config.getboolean(
"dialogueserver", "generatetoken")
if Settings.config.has_option("dialogueserver", "collecttask"):
self.COLLECT_TASK = Settings.config.getboolean(
"dialogueserver", "collecttask")
if Settings.config.has_option("dialogueserver", "subjectivefeedback"):
self.COLLECT_SUBJECTIVE_FEEDBACK = Settings.config.getboolean(
"dialogueserver", "subjectivefeedback")
if Settings.config.has_option("dialogueserver",
"subjectivefeedbackprompt"):
self.COLLECT_SUBJECTIVE_PROMPT = Settings.config.get(
"dialogueserver", "subjectivefeedbackprompt")
if Settings.config.has_option("dialogueserver", "dialhost"):
self.host = Settings.config.get("dialogueserver", "dialhost")
self.host = self.host.strip('"')
if Settings.config.has_option("dialogueserver", "dialport"):
self.port = Settings.config.getint("dialogueserver", "dialport")
self.issueConsent = False
if Settings.config.has_option("dialogueserver", "issueConsent"):
self.issueConsent = Settings.config.getboolean(
"dialogueserver", "issueConsent")
# Dialogue agent:
self.agent_factory = Agent.AgentFactory(hub_id='dialogueserver')
self.tasks = None
if Settings.config.has_option("dialogueserver", "tasksfile"):
from tasks import Tasks
self.tasks = Tasks.TaskReader(
taskfile=Settings.config.get("dialogueserver", "tasksfile"))
if self.COLLECT_TASK and self.tasks is None:
logger.error(
"Must provide a tasks file if you want to collect task")
def __init__(self):
""" HTTP Server
"""
self.RECEIVED_DTMF_MSG = "Got it, thanks."
self.GENERATE_TOKEN = True
self.COLLECT_SUBJECTIVE_FEEDBACK = False
self.COLLECT_TASK = False
self.host = "localhost"
self.port = 8082
self.TOKENSERVERURL = "http://www.camdial.org/~djv27/mt-multiDomain/receive-token.py"
# DTMF prompt given over voip call at dialogue's end
self.COLLECT_TASKID_PROMPT = ". Please now enter the 5 digit task number followed by the hash key"
#The hash key is only for multiple digits
self.COLLECT_SUBJECTIVE_PROMPT = " Have you found all the information you were looking for? Please enter one for yes, and zero for no."
self.ood_count = 0
self.OOD_THRESHOLD = 1
# Speech Settings:
self.allowbargein = False # TODO - config this or remove it -- OR SHOULD IT BE SOMETHING SENT FROM VOICEBROKER EACH TURN?
if Settings.config.has_option("dialogueserver", "tokenserverurl"):
self.TOKENSERVERURL = Settings.config.get("dialogueserver",
"tokenserverurl")
self.TOKENSERVERURL = self.TOKENSERVERURL.strip('"')
if Settings.config.has_option("dialogueserver", "generatetoken"):
self.GENERATE_TOKEN = Settings.config.getboolean(
"dialogueserver", "generatetoken")
if Settings.config.has_option("dialogueserver", "collecttask"):
self.COLLECT_TASK = Settings.config.getboolean(
"dialogueserver", "collecttask")
if Settings.config.has_option("dialogueserver", "subjectivefeedback"):
self.COLLECT_SUBJECTIVE_FEEDBACK = Settings.config.getboolean(
"dialogueserver", "subjectivefeedback")
if Settings.config.has_option("dialogueserver",
"subjectivefeedbackprompt"):
self.COLLECT_SUBJECTIVE_PROMPT = Settings.config.get(
"dialogueserver", "subjectivefeedbackprompt")
if Settings.config.has_option("dialogueserver", "dialhost"):
self.host = Settings.config.get("dialogueserver", "dialhost")
self.host = self.host.strip('"')
if Settings.config.has_option("dialogueserver", "dialport"):
self.port = Settings.config.getint("dialogueserver", "dialport")
self.issueConsent = False
if Settings.config.has_option("dialogueserver", "issueConsent"):
self.issueConsent = Settings.config.getboolean(
"dialogueserver", "issueConsent")
# Dialogue agent:
self.agent_factory = Agent.AgentFactory(hub_id='dialogueserver')
self.tasks = None
if Settings.config.has_option("dialogueserver", "tasksfile"):
from tasks import Tasks
self.tasks = Tasks.TaskReader(
taskfile=Settings.config.get("dialogueserver", "tasksfile"))
if self.COLLECT_TASK and self.tasks is None:
logger.error(
"Must provide a tasks file if you want to collect task")
def __importTasksFromFile(self):
try:
in_file = open("/Users/luigi/Dropbox/todo/td.txt","r")
pass
except IOError:
return
if in_file:
for line in in_file.readlines():
splitRecord = line.split('$')
taskid = splitRecord[0].strip()
task = splitRecord[1].strip()
Tasks.updateTaskFromTodoTxtFile(taskid, task)
in_file.close()
def __init__(self):
Thread.__init__(self)
self._run = True
self.success = 0 # 登录成功1or失败-1or正在登录中0
self._http = HttpClient.getInstance()
self._task = Tasks()
self._event = Event()
self._event.setDaemon(True)
def manage_tasks(self, tasks):
"""Receives completed tasks from the clients and updates the tasks
file.
"""
start_time = datetime.now()
params = self.get_params()
tasks_file = path(params["tasks_path"])
completed_file = tasks_file.dirname().joinpath("completed.json")
num_tasks = len(tasks)
task_queue = self.get_task_queue()
done_queue = self.get_done_queue()
try:
num_remaining = task_queue.qsize()
except NotImplementedError:
num_remaining = num_tasks
num_processed = 0
def running():
return (not task_queue.empty() or task_queue.join()
or not done_queue.empty())
# Main task loop. Stopping criteria: no more tasks and all
# clients have signaled task done.
while running():
# Wait for a done task to arrive.
task_name = done_queue.get()
completed = Tasks.load(completed_file)
completed[task_name] = True
# Save task to disk.
completed.save(completed_file)
# Report progress.
num_processed += 1
try:
num_remaining = task_queue.qsize()
except NotImplementedError:
num_remaining -= 1
num_finished = num_tasks - num_remaining
progress = 100 * float(num_finished) / num_tasks
dt = datetime.now() - start_time
avg_dt = timedelta(seconds=(dt.total_seconds() /
float(num_processed + 1e-5)))
time_left = timedelta(seconds=(avg_dt.total_seconds() *
num_remaining))
logger.info("-" * 60)
logger.info("Task `%s` complete", task_name)
logger.info("Progress: %d/%d (%.2f%%)", num_finished, num_tasks,
progress)
logger.info("Time elapsed : %s", str(dt))
logger.info("Time per task : %s", str(avg_dt))
logger.info("Time remaining: %s", str(time_left))
def __exportTasksToFile(self):
out_file = open("/Users/luigi/Dropbox/todo/td.txt","w+")
tasks = Tasks.findAllTasks()
if tasks:
for task in tasks:
taskRecord = str(task['_id']) + " $ " + task['todo'] + "\n"
out_file.write(taskRecord)
out_file.close()
def do_list(self, line):
"""
Elenca tutti i task inseriti
"""
sub_command = line.split(' ')
if (len(sub_command[0]) > 0):
strSearch = ""
if ('+' not in sub_command[0]):
strSearch = '+' + sub_command[0]
tasks = Tasks.findTaskWithTagname(strSearch)
for task in tasks:
print "%d) %-100s" % (task['_id'],task['todo'])
else:
tasks = Tasks.findAllTasks()
for task in tasks:
print "%d) %-100s %-50s" % (task['_id'],task['todo'], task['tags'])
def check_recent_pings(cls):
recent = cls._pings.sort(key=lambda p: p["timestamp"]).reverse()
if len(recent) >= 1:
most_recent = recent[0]["timestamp"]
now = time.time()
if now - most_recent >= 90 or len(recent) == 0:
# No active Smokeys. Wait a random number of seconds, then switch to active.
sleep = random.randint(0, 30)
cls._switch_task = Tasks.later(SocketScience.switch_to_active, after=sleep)
def tell_rooms(msg, has, hasnt, notify_site="", report_data=None):
global _rooms
msg = msg.rstrip()
target_rooms = set()
for prop_has in has:
if isinstance(prop_has, tuple):
target_rooms.add(prop_has)
if prop_has not in _room_roles:
continue
for room in _room_roles[prop_has]:
if all(map(lambda prop: prop not in _room_roles or room not in _room_roles[prop], hasnt)):
if room not in _rooms:
site, roomid = room
deletion_watcher = room in _watcher_rooms
new_room = _clients[site].get_room(roomid)
new_room.join()
_rooms[room] = RoomData(new_room, -1, deletion_watcher)
target_rooms.add(room)
for room_id in target_rooms:
room = _rooms[room_id]
if notify_site:
pings = datahandling.get_user_names_on_notification_list(room.room._client.host,
room.room.id,
notify_site,
room.room._client)
msg_pings = datahandling.append_pings(msg, pings)
else:
msg_pings = msg
timestamp = time.time()
if room.block_time < timestamp and _global_block < timestamp:
if report_data and "delay" in _room_roles and room_id in _room_roles["delay"]:
def callback(room=room, msg=msg_pings):
post = fetch_post_id_and_site_from_url(report_data[0])[0:2]
if not datahandling.is_false_positive(post) and not datahandling.is_ignored_post(post):
_msg_queue.put((room, msg, report_data))
task = Tasks.later(callback, after=300)
GlobalVars.deletion_watcher.subscribe(report_data[0], callback=task.cancel)
else:
_msg_queue.put((room, msg_pings, report_data))
def handle(cls, content):
for k, d in content.items():
if k in cls._callbacks:
for cb in cls._callbacks[k]:
cb(d)
if "metasmoke_state" in content:
if content["metasmoke_state"] == "down":
log('info', "{} says metasmoke is down, switching to active ping monitoring."
.format(content["location"]))
GlobalVars.metasmoke_down = True
Tasks.later(SocketScience.check_recent_pings, after=90)
if content["metasmoke_state"] == "up":
log('info', '{} says metasmoke is up, disabling ping monitoring.'.format(content["location"]))
GlobalVars.metasmoke_down = False
if "ping" in content:
cls._pings.append({"timestamp": content["ping"], "location": content["location"]})
if cls._switch_task is not None:
cls._switch_task.cancel()
def on_msg(msg, client):
if not isinstance(msg, events.MessagePosted) and not isinstance(msg, events.MessageEdited):
return
message = msg.message
if message.owner.id == client._br.user_id:
return
if message.content.startswith("<div class='partial'>"):
message.content = message.content[21:]
if message.content.endswith("</div>"):
message.content = message.content[:-6]
room_data = _rooms[(client.host, message.room.id)]
if message.parent:
if message.parent.owner.id == client._br.user_id:
strip_mention = regex.sub("^(<span class=(\"|')mention(\"|')>)?@.*?(</span>)? ", "", message.content)
cmd = GlobalVars.parser.unescape(strip_mention)
result = dispatch_reply_command(message.parent, message, cmd)
if result:
_msg_queue.put((room_data, ":{} {}".format(message.id, result), None))
elif message.content.lower().startswith("sd "):
result = dispatch_shorthand_command(message)
if result:
_msg_queue.put((room_data, ":{} {}".format(message.id, result), None))
elif message.content.startswith("!!/"):
result = dispatch_command(message)
if result:
_msg_queue.put((room_data, ":{} {}".format(message.id, result), None))
elif classes.feedback.FEEDBACK_REGEX.search(message.content) \
and is_privileged(message.owner, message.room) and datahandling.last_feedbacked:
ids, expires_in = datahandling.last_feedbacked
if time.time() < expires_in:
Tasks.do(metasmoke.Metasmoke.post_auto_comment, message.content_source, message.owner, ids=ids)
def initControls(self):
"""Init basic controls"""
self.outputArea.setString_("")
self.tasks = Tasks()
if userPrefs.showWorkTill:
self.workTillBox.setHidden_(False)
else:
self.workTillBox.setHidden_(True)
self.pbtnProject.removeAllItems()
self.pbtnProject.addItemsWithTitles_(Projects.get())
self.pbtnProject.selectItemWithTitle_(userPrefs.selectedProject)
self.projectChange_(None)
self.initDoneButton()
self.fillTasks()
self.scrollToEnd()
def set_manager(self, man=None):
old_stream = self.stream
self.stream = False
if self.isRunning():
self.wait(1)
self.manager = man
logging.debug('KidRegistry.set_manager', man)
self.taskswg = Tasks.instance()
self.connect(
self, QtCore.SIGNAL('set_server(PyQt_PyObject)'), self.taskswg.set_server)
if man is not None:
if man.remote:
logging.debug('KidRegistry.set_manager with remote', man.remote)
self.emit(
QtCore.SIGNAL('set_server(PyQt_PyObject)'), man.remote)
else:
logging.debug('KidRegistry.set_manager without remote', man.remote)
else:
logging.debug('KidRegistry.set_manager set to None', man)
if old_stream:
self.stream = True
self.start()
def btnDonePress_(self, sender):
"""On done button press"""
if self.tasks.dayStarted():
if self.cbxInput.stringValue().strip():
taskName = self.cbxInput.stringValue()
self.appendTask(*fh.formatTaskString(
*self.tasks.add(taskName,
self.pbtnProject.titleOfSelectedItem())))
self.readCounters()
self.cbxInput.setStringValue_("")
self.scrollToEnd()
if Tasks.taskType(taskName) == "work":
Projects.addAutocomplete(
self.pbtnProject.titleOfSelectedItem(), taskName)
else:
SlackingAutocompletes.add(taskName)
self.cbxInput.addItemWithObjectValue_(taskName)
else:
if userPrefs.showHelpMessageOnStart:
self.showStartHelpMessage()
taskName = userPrefs.startPlaceholder
self.appendTask(*fh.formatTaskString(*self.tasks.add(taskName)))
self.initDoneButton()
def check_websocket_for_deletion(self, post_site_id, post_url, timeout):
time_to_check = time.time() + timeout
post_id = post_site_id[0]
post_type = post_site_id[2]
if post_type == "answer":
question_id = str(datahandling.get_post_site_id_link(post_site_id))
if question_id is None:
return
else:
question_id = post_id
post_site = post_site_id[1]
if post_site not in GlobalVars.site_id_dict:
return
site_id = GlobalVars.site_id_dict[post_site]
ws = websocket.create_connection("wss://qa.sockets.stackexchange.com/")
ws.send(site_id + "-question-" + question_id)
while time.time() < time_to_check:
ws.settimeout(time_to_check - time.time())
try:
a = ws.recv()
except websocket.WebSocketTimeoutException:
Tasks.do(metasmoke.Metasmoke.send_deletion_stats_for_post, post_url, False)
return False
if a is not None and a != "":
try:
action = json.loads(a)["action"]
if action == "hb":
ws.send("hb")
continue
else:
d = json.loads(json.loads(a)["data"])
except:
continue
if d["a"] == "post-deleted" and str(d["qId"]) == question_id:
if (post_type == "answer" and "aId" in d and str(d["aId"]) == post_id) or post_type == "question":
Tasks.do(metasmoke.Metasmoke.send_deletion_stats_for_post, post_url, True)
return True
Tasks.do(metasmoke.Metasmoke.send_deletion_stats_for_post, post_url, False)
return False
")"
GlobalVars.standby_message = "[ " + GlobalVars.chatmessage_prefix + " ] " \
"SmokeDetector started in [standby mode](" + \
"https://charcoal-se.org/smokey/SmokeDetector-Statuses#standby-mode) " + \
"at [rev " +\
GlobalVars.commit_with_author +\
"](" + GlobalVars.bot_repository + "/commit/" +\
GlobalVars.commit['id'] +\
") (running on " +\
GlobalVars.location +\
")"
GlobalVars.standby_mode = "standby" in sys.argv
chatcommunicate.init(username, password)
Tasks.periodic(Metasmoke.send_status_ping, interval=60)
Tasks.periodic(Metasmoke.check_last_pingtime, interval=30)
if GlobalVars.standby_mode:
chatcommunicate.tell_rooms_with("debug", GlobalVars.standby_message)
Metasmoke.send_status_ping()
while GlobalVars.standby_mode:
time.sleep(3)
chatcommunicate.init(username, password) # to rejoin rooms
# noinspection PyProtectedMember
def check_socket_connections():
for client in chatcommunicate._clients.values():
def handle_spam(post, reasons, why):
post_url = parsing.to_protocol_relative(parsing.url_to_shortlink(post.post_url))
poster_url = parsing.to_protocol_relative(parsing.user_url_to_shortlink(post.user_url))
shortened_site = post.post_site.replace("stackexchange.com", "SE") # site.stackexchange.com -> site.SE
datahandling.append_to_latest_questions(post.post_site, post.post_id, post.title if not post.is_answer else "")
if len(reasons) == 1 and ("all-caps title" in reasons or
"repeating characters in title" in reasons or
"repeating characters in body" in reasons or
"repeating characters in answer" in reasons or
"repeating words in title" in reasons or
"repeating words in body" in reasons or
"repeating words in answer" in reasons):
datahandling.add_auto_ignored_post((post.post_id, post.post_site, datetime.now()))
if why is not None and why != "":
datahandling.add_why(post.post_site, post.post_id, why)
if post.is_answer and post.post_id is not None and post.post_id is not "":
datahandling.add_post_site_id_link((post.post_id, post.post_site, "answer"), post.parent.post_id)
if GlobalVars.reason_weights or GlobalVars.metasmoke_key:
reason_weight = sum_weight(reasons)
if reason_weight >= 1000:
reason_weight_s = " (**{}**)".format(reason_weight)
else:
reason_weight_s = " ({})".format(reason_weight)
else: # No reason weight if neither cache nor MS
reason_weight_s = ""
try:
# If the post is an answer type post, the 'title' is going to be blank, so when posting the
# message contents we need to set the post title to the *parent* title, so the message in the
# chat is properly constructed with parent title instead. This will make things 'print'
# in a proper way in chat messages.
sanitized_title = parsing.sanitize_title(post.title if not post.is_answer else post.parent.title)
sanitized_title = escape_format(sanitized_title).strip()
prefix = u"[ [SmokeDetector](//git.io/vyDZv) ]"
if GlobalVars.metasmoke_key:
prefix_ms = u"[ [SmokeDetector](//git.io/vyDZv) | [MS]({}) ]".format(
to_metasmoke_link(post_url, protocol=False))
else:
prefix_ms = prefix
# We'll insert reason list later
edited = '' if not post.edited else ' \u270F\uFE0F'
if not post.user_name.strip() or (not poster_url or poster_url.strip() == ""):
s = " {{}}{}: [{}]({}){} by a deleted user on `{}`".format(
reason_weight_s, sanitized_title, post_url, edited, shortened_site)
username = ""
else:
username = post.user_name.strip()
escaped_username = escape_format(parsing.escape_markdown(username))
s = " {{}}{}: [{}]({}){} by [{}]({}) on `{}`".format(
reason_weight_s, sanitized_title, post_url, edited, escaped_username, poster_url, shortened_site)
Tasks.do(metasmoke.Metasmoke.send_stats_on_post,
post.title_ignore_type, post_url, reasons, post.body, username,
post.user_link, why, post.owner_rep, post.post_score,
post.up_vote_count, post.down_vote_count)
log('debug', GlobalVars.parser.unescape(s).encode('ascii', errors='replace'))
GlobalVars.deletion_watcher.subscribe(post_url)
reason = message = None
for reason_count in range(5, 0, -1): # Try 5 reasons and all the way down to 1
reason = ", ".join(reasons[:reason_count])
if len(reasons) > reason_count:
reason += ", +{} more".format(len(reasons) - reason_count)
reason = reason.capitalize()
message = prefix_ms + s.format(reason) # Insert reason list
if len(message) <= 500:
break # Problem solved, stop attempting
s = s.format(reason) # Later code needs this variable
if len(message) > 500:
message = (prefix_ms + s)[:500] # Truncate directly and keep MS link
without_roles = tuple(["no-" + reason for reason in reasons]) + ("site-no-" + post.post_site,)
if set(reasons) - GlobalVars.experimental_reasons == set() and \
not why.startswith("Post manually "):
chatcommunicate.tell_rooms(message, ("experimental",),
without_roles, notify_site=post.post_site, report_data=(post_url, poster_url))
else:
chatcommunicate.tell_rooms(message, ("all", "site-" + post.post_site),
without_roles, notify_site=post.post_site, report_data=(post_url, poster_url))
except Exception as e:
excepthook.uncaught_exception(*sys.exc_info())
def stopflagging():
Tasks.do(Metasmoke.stop_autoflagging)
return "Request sent..."
def handle_spam(post, reasons, why):
post_url = parsing.to_protocol_relative(parsing.url_to_shortlink(post.post_url))
poster_url = parsing.to_protocol_relative(parsing.user_url_to_shortlink(post.user_url))
reason = ", ".join(reasons[:5])
if len(reasons) > 5:
reason += ", +{} more".format(len(reasons) - 5)
reason = reason[:1].upper() + reason[1:] # reason is capitalised, unlike the entries of reasons list
shortened_site = post.post_site.replace("stackexchange.com", "SE") # site.stackexchange.com -> site.SE
datahandling.append_to_latest_questions(post.post_site, post.post_id, post.title if not post.is_answer else "")
if len(reasons) == 1 and ("all-caps title" in reasons or
"repeating characters in title" in reasons or
"repeating characters in body" in reasons or
"repeating characters in answer" in reasons or
"repeating words in title" in reasons or
"repeating words in body" in reasons or
"repeating words in answer" in reasons):
datahandling.add_auto_ignored_post((post.post_id, post.post_site, datetime.now()))
if why is not None and why != "":
datahandling.add_why(post.post_site, post.post_id, why)
if post.is_answer and post.post_id is not None and post.post_id is not "":
datahandling.add_post_site_id_link((post.post_id, post.post_site, "answer"), post.parent.post_id)
try:
post._title = parsing.escape_special_chars_in_title(post.title)
if post.is_answer:
# If the post is an answer type post, the 'title' is going to be blank, so when posting the
# message contents we need to set the post title to the *parent* title, so the message in the
# chat is properly constructed with parent title instead. This will make things 'print'
# in a proper way in chat messages.
sanitized_title = regex.sub('(https?://|\n)', '', post.parent.title)
else:
sanitized_title = regex.sub('(https?://|\n)', '', post.title)
sanitized_title = regex.sub(r'([\]*`])', r'\\\1', sanitized_title).replace('\n', u'\u23CE')
prefix = u"[ [SmokeDetector](//goo.gl/eLDYqh) ]"
if GlobalVars.metasmoke_key:
prefix_ms = u"[ [SmokeDetector](//goo.gl/eLDYqh) | [MS](//m.erwaysoftware.com/posts/by-url?url=" + \
post_url + ") ]"
else:
prefix_ms = prefix
if not post.user_name.strip() or (not poster_url or poster_url.strip() == ""):
s = u" {}: [{}]({}) by a deleted user on `{}`".format(reason, sanitized_title.strip(), post_url,
shortened_site)
username = ""
else:
s = u" {}: [{}]({}) by [{}]({}) on `{}`".format(reason, sanitized_title.strip(), post_url,
post.user_name.strip(), poster_url, shortened_site)
username = post.user_name.strip()
Tasks.do(metasmoke.Metasmoke.send_stats_on_post,
post.title_ignore_type, post_url, reasons, post.body, username,
post.user_link, why, post.owner_rep, post.post_score,
post.up_vote_count, post.down_vote_count)
log('debug', GlobalVars.parser.unescape(s).encode('ascii', errors='replace'))
datahandling.append_to_latest_questions(post.post_site, post.post_id, post.title)
message = prefix_ms + s
if len(message) > 500:
message = (prefix + s)[:500]
without_roles = tuple("no-" + reason for reason in reasons)
if set(reason) & GlobalVars.experimental_reasons == {}:
chatcommunicate.tell_rooms(message, ("experimental"),
without_roles, notify_site=post.post_site, report_data=(post_url, poster_url))
else:
chatcommunicate.tell_rooms(message, ("all", "site-" + post.post_site),
without_roles, notify_site=post.post_site, report_data=(post_url, poster_url))
except:
exc_type, exc_obj, exc_tb = sys.exc_info()
excepthook.uncaught_exception(exc_type, exc_obj, exc_tb)
def send_custom(type, url, msg):
Tasks.do(metasmoke.Metasmoke.send_feedback_for_post, url, type, msg.owner.name, msg.owner.id, msg._client.host)