def process_and_save_patches(out_path, all_files, patch_size):
""" Function that processes and saves the patches from the images """
img_queue = Queue(150)
load_proc = Process(target=_load_task, args=(img_queue, all_files))
load_proc.start()
save_pool = ThreadPool()
cont_patches = 0
out_path = Path(out_path)
_verify_out_path(out_path)
dir_format = 'dir{}'
while True:
img = img_queue.get()
if not len(img):
break
patches = extract_img_patches(img, patch_size)
num_patches = len(patches)
begin = cont_patches
end = cont_patches + num_patches
curr_dir = out_path / Path(dir_format.format(begin // 50000))
if not curr_dir.exists():
curr_dir.mkdir()
names = [
curr_dir / ('patch_' + str(n) + '.png')
for n in np.arange(begin, end)
]
save_pool.starmap_async(save_img_from_array, zip(patches, names))
cont_patches = end
save_pool.close()
save_pool.join()
def run(self,
n_games=int(1e4),
initial_temperature=1.0,
n_rollouts=1600,
temperature_decay_factor=0.95,
moves_before_decaying=8,
verbose=0):
pool = ThreadPool()
progress_bar = tqdm.tqdm(total=n_games)
assert verbose == 0
arguments = (self.model.step_model, self.model.step_model, n_rollouts,
self.c_puct, 0, initial_temperature,
temperature_decay_factor, moves_before_decaying,
progress_bar, None, None, None, None, verbose)
iterable_arguments = [arguments] * n_games
res = pool.starmap_async(play_single_game, iterable_arguments)
res.wait()
progress_bar.close()
for (winner, history) in res.get():
save_memory(self.replay_memory, winner, history)
def run_solver(args, single_solver=False):
solver = args[0]
command = args[1]
problems = args[2]
problem_sets = args[3]
finished_instances = []
for problem_set in problem_sets:
problem_set_name = problem_set[10:]
if (len(problem_set_name) > 0):
filename = "%s/%s_%s.csv" % (RESULTS_DIR, solver, problem_set[10:])
exists = os.path.isfile(filename)
if (exists):
file_data = np.genfromtxt(filename,
delimiter=',',
dtype=None,
encoding=None,
names=[
"Instance", "Result", "Time",
"conflict_size", "average_len",
"reduction", "i_uip_reduction",
"mem_use", "core_clause", "lbd"
],
skip_header=1)
op = 'a+'
finished_instances = [
problem_set + '/' + item[0] for item in file_data
]
else:
op = 'w+'
with open(filename, op, buffering=1) as fp:
if not exists:
fp.write(CSV_HEADER)
if single_solver:
lock = multiprocessing.Lock()
number_processes = int(multiprocessing.cpu_count() / 2) - 1
pool = ThreadPool(number_processes)
tasks = []
for problem in problems:
if (problem.startswith(problem_set)
) and problem not in finished_instances:
tasks.append((solver, command, problem, fp, lock))
results = pool.starmap_async(run_problem_and_write_result,
tasks)
pool.close()
pool.join()
else:
for problem in problems:
if (problem.startswith(problem_set)
) and problem not in finished_instances:
result = run_problem(solver, command, problem)
fp.write(
"%s,%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s\n"
% (result.problem, result.result,
result.elapsed, result.conflict_size,
result.average_len, result.reduction,
result.i_uip_reduction, result.mem_use,
result.core_clause, result.lbd,
result.attempt_rate, result.success_rate))
def execute_function_in_parallel(func: Callable, list_args: List, processes: bool = False,
local_pool: bool = False, num_threads: int = NUM_OF_THREADS,
num_processes: int = NUM_OF_PROCESSES) -> List:
"""
Execute a function in parallel using ThreadPool or ProcessPool
:param local_pool: create a new pool of threads/processes
:param num_processes: a num of processes to create
:param num_threads: a num of threads to create
:param processes: execute tasks in separate processes
:param func: a func to call
:param list_args: an array containing calling params
:return: an array with results
"""
if not (func and list_args):
return []
if local_pool:
pool = ThreadPool(num_threads) if not processes else Pool(num_processes)
else:
global process_pool
if processes:
if not process_pool:
process_pool = Pool(NUM_OF_PROCESSES)
pool = process_pool
else:
global thread_pool
if not thread_pool:
thread_pool = ThreadPool(NUM_OF_THREADS)
pool = thread_pool
results_tmp = pool.starmap_async(func, list_args)
results = [result for result in results_tmp.get() if result is not None]
if local_pool:
pool.close()
return results
def PoolByModule():
pool = ThreadPool(processes=cpu_count() << 2)
# for i in range(20):
# pool.apply(func, args=(i,)) # 同步方式，会等待上一个线程执行结束
# # pool.apply_async(func, args=(i,)) # 异步方式，'并行'
# 线程池map操作
# res = pool.map(func, range(20))
# print(res)
# res = pool.map_async(func, range(20))
# print(res.get())
# 线程池imap操作
# res = pool.imap(func, range(20))
# print([i for i in res])
# res = pool.imap_unordered(func, range(20))
# print([i for i in res])
# 多输入参数方法实现调用
print([i for i in enumerate(range(20))])
res = pool.starmap(add, enumerate(range(20)))
print(res)
res = pool.starmap_async(add, enumerate(range(20)))
print(res.get())
def evaluate(self,
model,
n_games=400,
initial_temperature=0.25,
n_rollouts=1600,
new_best_model_threshold=0.55,
temperature_decay_factor=0.95,
moves_before_decaying=8,
eval_iter=0,
verbose=0):
# log_every_n_games = max(2, n_games // 4)
log_n_games = 8
log_n_games_detailed = 1
n_wins = 0
pool = ThreadPool()
progress_bar = tqdm.tqdm(total=n_games)
iterable_arguments = [
(
self.model.train_model,
self.model.step_model,
n_rollouts,
self.c_puct,
(i % 2),
initial_temperature,
temperature_decay_factor,
moves_before_decaying,
progress_bar,
eval_iter,
i,
self.model.sess,
self.model.summary_writer,
# verbose if verbose and i % log_every_n_games == 0 else 0
2 if i < log_n_games_detailed else 1 if i < log_n_games else 0)
for i in range(n_games)
]
res = pool.starmap_async(play_single_game, iterable_arguments)
res.wait()
progress_bar.close()
for (winner, _) in res.get():
if winner == 0:
n_wins += 1
print("win_ratio={}".format(100 * n_wins / n_games), file=sys.stderr)
win_ratio_summary = model.sess.run(self.win_ratio_summary_op,
feed_dict={
self.n_wins_ph: n_wins,
self.n_games_ph: n_games
})
model.summary_writer.add_summary(win_ratio_summary, eval_iter)
return n_wins / n_games > new_best_model_threshold
def list_Terminator(self, Sub):
pi = self.get_play_list_id()
videos, number = self.get_video_urls(pi)
print("there are {} videos in this playlist".format(number))
print("Getting streams, please wait ... ")
pool = ThreadPool(processes=16)
res = pool.map_async(self.title_for_url,
(videos[i] for i in range(len(videos))))
streams = res.get()
for i in range(len(streams)):
print(str(i + 1) + "-" + streams[i])
select = input("Enter Number of videos you want with '-' between:")
create_dir_in_path(self.download_dir, pi)
if (not Sub) and (select == '0'):
#just download all list without sub
e = 0
s = 0
formatt = input("what format do you want:")
ress = input("what resolotion do you want:")
import time
t1 = time.time()
pool = ThreadPool(processes=4)
res = pool.starmap_async(
self.single_video_downloader,
((videos[i], 'video', formatt, ress, pi, i)
for i in range(len(videos))))
res = res.get()
print(res)
print(time.time() - t1)
elif (not Sub) and (select != '0'):
#downloading choosen items without sub
pass
elif (Sub) and (select == '0'):
#download all list with sub
pass
elif (Sub) and (select != '0'):
#downloading choosen items with sub
pass
else:
print(
"the value of sub parameter or the numbers of videos is invalid"
)
def _submit_scan_job(targets: list):
"""
Run a multithreaded scan across all targets specified
"""
# Attempt to randomize as a jury rigged method of scanning across multiple hosts at once
random.shuffle(targets)
pool = ThreadPool(processes=CONFIG['thread_count'])
async_result = pool.starmap_async(_scan_target, targets)
results = async_result.get()
# Un-randomize results
sorted_results = sorted(results, key=lambda k: k['target'])
return sorted_results
def seg(request):
"""View for seg."""
if request.method == 'POST':
data = json.loads(request.body.decode('utf-8'))
source_text = data['source_text']
source_text = source_text.replace('\n', ' ') # remove linebreaks
segmentators = data['segmentators']
segmentators = [i.lower() for i in segmentators]
pool = ThreadPool(processes=5)
res = pool.starmap_async(_segwrap, zip(segmentators, repeat(source_text)))
vals = res.get()
seg_with_diff = segcomp(vals)
return HttpResponse(json.dumps(seg_with_diff), content_type="application/json")
elif request.method == 'GET':
return HttpResponse('')
return Http404
def seg(request):
"""View for seg."""
if request.method == 'POST':
data = json.loads(request.body.decode('utf-8'))
source_text = data['source_text']
source_text = source_text.replace('\n', ' ') # remove linebreaks
segmentators = data['segmentators']
segmentators = [i.lower() for i in segmentators]
pool = ThreadPool(processes=5)
res = pool.starmap_async(_segwrap,
zip(segmentators, repeat(source_text)))
vals = res.get()
seg_with_diff = segcomp(vals)
return HttpResponse(json.dumps(seg_with_diff),
content_type="application/json")
elif request.method == 'GET':
return HttpResponse('')
return Http404
class PiperBot(threading.Thread):
def __init__(self):
super(PiperBot, self).__init__(daemon=True)
self.servers = {}
self.admins = defaultdict(list)
self.users = defaultdict(lambda: defaultdict(User))
self.ops = defaultdict(lambda: defaultdict(set))
self.command_char = "#"
self.in_queue = PriorityQueue()
self.apikeys = {}
self.commands = {}
self.aliases = {}
self.plugins = {}
self.pre_command_exts = []
self.post_command_exts = []
self.pre_event_exts = []
self.post_event_exts = []
self.pre_trigger_exts = []
self.post_trigger_exts = []
self.pre_regex_exts = []
self.post_regex_exts = []
self.worker_pool = ThreadPool(processes=8)
self.message_buffer = defaultdict(lambda: defaultdict(lambda: deque(maxlen=50)))
self.buffer_pattern = re.compile(r"(?:(\w+)|(\s)|^)(?:\^(\d+)|(\^+))")
self.buffer_pattern_escape = re.compile(r"\\\^")
self.running = False
def buffer_replace(self, text, servername, channel, offset=0):
buffer = list(self.message_buffer[servername][channel])[offset:]
def buffer_sub(match_object):
if match_object.group(1):
messgages = [x for x in buffer if x.nick == match_object.group(1)]
if not messgages:
raise Exception("user not in memory")
else:
messgages = buffer
if match_object.group(3):
count = int(match_object.group(3))
else:
count = len(match_object.group(4))
if count <= len(messgages):
return (" " if match_object.group(2) else "") + messgages[count - 1].text
else:
raise Exception("line not in memory")
text = self.buffer_pattern.sub(buffer_sub, text)
text = self.buffer_pattern_escape.sub("^", text)
return text
def connect_to(self, servername, network, port, nick, channels=None, admins=None, password=None, username=None,
ircname=None, ssl=False):
self.servers[servername] = ServerConnection(self.in_queue, servername, network, port, nick, password, username,
ircname, channels, ssl)
self.servers[servername].connect()
if admins:
self.admins[servername] += admins
def run(self):
self.running = True
try:
while self.running:
try:
message = self.in_queue.get(timeout=10)
print("<< " + str(message))
self.handle_message(message)
except Empty:
pass
finally:
self.shutdown()
def shutdown(self):
self.running = False
for plugin in self.plugins.values():
for func in plugin._onUnloads:
func()
for server in self.servers.values():
server.disconnect(message="shutting down")
def load_plugin_from_module(self, plugin):
if "." in plugin:
module = "".join(plugin.split(".")[:-1])
plugin_name = plugin.split(".")[-1]
temp = importlib.machinery.SourceFileLoader(module, os.path.dirname(
os.path.abspath(__file__)) + "/plugins/" + module + ".py").load_module()
found = False
for name, Class in inspect.getmembers(temp, lambda x: inspect.isclass(x) and hasattr(x, "_plugin")):
if name == plugin_name:
self.load_plugin(Class)
found = True
if not found:
raise Exception("no such plugin to load")
else:
temp = importlib.machinery.SourceFileLoader(plugin, os.path.dirname(
os.path.abspath(__file__)) + "/plugins/" + plugin + ".py").load_module()
found = False
for name, Class in inspect.getmembers(temp, lambda x: inspect.isclass(x) and hasattr(x, "_plugin")):
self.load_plugin(Class)
found = True
if not found:
raise Exception("no such plugin to load")
def unload_module(self, module_name):
found = False
plugins = []
for plugin_name, plugin in self.plugins.items():
if plugin_name.startswith(module_name + ".") or plugin_name == module_name:
plugins.append(plugin_name)
found = True
for plugin in plugins:
self.unload_plugin(plugin)
if not found:
raise Exception("no such plugin to unload")
def load_plugin(self, plugin):
if plugin.__module__ + "." + plugin.__name__ in self.plugins:
raise Exception("already loaded!")
plugin_instance = plugin()
plugin_instance._plugin__init__(self)
for func in plugin_instance._onLoads:
func()
self.plugins[plugin_instance.__module__ + "." + plugin_instance.__class__.__name__] = plugin_instance
for (func, args) in plugin_instance._commands:
if args["command"] not in self.commands:
self.commands[args["command"]] = (func, args)
else:
print("command overlap! : " + args["command"])
for (priority, func) in plugin_instance._command_extensions:
if priority < 1:
self.pre_command_exts.append((priority,
plugin_instance.__module__ + "." + plugin_instance.__class__.__name__,
func))
else:
self.post_command_exts.append((priority,
plugin_instance.__module__ + "." + plugin_instance.__class__.__name__,
func))
for (priority, func) in plugin_instance._event_extensions:
if priority < 1:
self.pre_event_exts.append((priority,
plugin_instance.__module__ + "." + plugin_instance.__class__.__name__,
func))
else:
self.post_event_exts.append((priority,
plugin_instance.__module__ + "." + plugin_instance.__class__.__name__,
func))
for (priority, func) in plugin_instance._trigger_extensions:
if priority < 1:
self.pre_trigger_exts.append((priority,
plugin_instance.__module__ + "." + plugin_instance.__class__.__name__,
func))
else:
self.post_trigger_exts.append((priority,
plugin_instance.__module__ + "." + plugin_instance.__class__.__name__,
func))
for (priority, func) in plugin_instance._regex_extensions:
if priority < 1:
self.pre_regex_exts.append((priority,
plugin_instance.__module__ + "." + plugin_instance.__class__.__name__,
func))
else:
self.post_regex_exts.append((priority,
plugin_instance.__module__ + "." + plugin_instance.__class__.__name__,
func))
self.post_command_exts.sort(key=itemgetter(0, 1))
self.pre_command_exts.sort(key=itemgetter(0, 1))
self.post_event_exts.sort(key=itemgetter(0, 1))
self.pre_event_exts.sort(key=itemgetter(0, 1))
self.post_regex_exts.sort(key=itemgetter(0, 1))
self.pre_regex_exts.sort(key=itemgetter(0, 1))
self.post_trigger_exts.sort(key=itemgetter(0, 1))
self.pre_trigger_exts.sort(key=itemgetter(0, 1))
if plugin._plugin_thread:
plugin_instance.start()
def unload_plugin(self, plugin_name):
for func, args in self.plugins[plugin_name]._commands:
if args["command"] in self.commands.keys():
del self.commands[args["command"]]
for lump in self.post_command_exts:
if lump[1] == plugin_name:
self.post_command_exts.remove(lump)
for lump in self.pre_command_exts:
if lump[1] == plugin_name:
self.pre_command_exts.remove(lump)
for lump in self.post_event_exts:
if lump[1] == plugin_name:
self.post_event_exts.remove(lump)
for lump in self.pre_event_exts:
if lump[1] == plugin_name:
self.pre_event_exts.remove(lump)
for lump in self.post_regex_exts:
if lump[1] == plugin_name:
self.post_regex_exts.remove(lump)
for lump in self.pre_regex_exts:
if lump[1] == plugin_name:
self.pre_regex_exts.remove(lump)
for lump in self.post_trigger_exts:
if lump[1] == plugin_name:
self.post_trigger_exts.remove(lump)
for lump in self.pre_trigger_exts:
if lump[1] == plugin_name:
self.pre_trigger_exts.remove(lump)
for func in self.plugins[plugin_name]._onUnloads:
func()
del self.plugins[plugin_name]
def handle_message(self, message):
if message.command == "PING":
response = message.copy()
response.command = "PONG"
self.send(response)
elif message.command == "PRIVMSG":
self.message_buffer[message.server][message.params].appendleft(message)
if message.text.startswith(self.command_char) and message.text[1:]:
first = message.text[1:].split()[0]
if first == "alias":
self.handle_alias_assign(message)
elif first in self.commands or first in self.aliases:
self.worker_pool.apply_async(self.handle_command, args=(message,))
triggered = []
for plugin in self.plugins.values():
if message.command == "PRIVMSG":
for regex, rfunc in plugin._regexes:
matches = regex.findall(message.text)
for groups in matches:
temp = message.copy()
temp.groups = groups
triggered.append((rfunc, temp, self.pre_regex_exts, self.post_trigger_exts))
for trigger, tfunc in plugin._triggers:
if trigger(message, bot):
triggered.append((tfunc, message, self.pre_trigger_exts, self.post_trigger_exts))
for event, efunc in plugin._handlers:
if message.command.lower() == event.lower():
triggered.append((efunc, message, self.pre_event_exts, self.post_event_exts))
self.worker_pool.starmap_async(self.call_triggered, triggered, error_callback=print)
def call_triggered(self, func, message, pre=[], post=[]):
pipe = self.resulter()
for _, _, func_, in post:
pipe = func_(message, pipe)
pipe = func(pipe)
next(pipe)
for _, _, func_ in pre:
pipe = func_(message, pipe)
pipe.send(message)
def handle_alias_assign(self, message):
try:
name, *alias = message.text[len(self.command_char)+5:].split("=")
name = name.strip()
alias = "=".join(alias).strip()
if name in self.commands:
raise Exception("cannot overwrite existing command")
commands = map(lambda x: [(command, " ".join(args)) for command, *args in [x.split(" ")]][0],
map(lambda x: x.strip(), alias.split(" || ")))
funcs = []
args = []
for func, arg in commands:
if func in self.commands:
funcs.append(self.commands[func][1]["command"])
args.append(arg)
elif func in self.aliases:
if func == name:
raise Exception("recursion detected")
funcs_, args_ = self.aliases[func]
funcs.extend(funcs_)
args.extend(args_)
else:
raise Exception("unrecognised command: " + func)
self.aliases[name] = list(zip(funcs, args))
self.send(message.reply("The alias %s has been saved" % name))
except Exception as e:
self.send(message.reply(type(e).__name__ + (": " + str(e)) if str(e) else ""))
exc_type, exc_value, exc_traceback = sys.exc_info()
print("*** print_tb:")
traceback.print_tb(exc_traceback, file=sys.stdout)
def handle_command(self, message):
try:
def prepper(results):
while 1:
x = yield
results.append(x)
results = []
prepipe = prepper(results)
next(prepipe)
for _, _, f in self.pre_command_exts:
prepipe = f(message, prepipe)
prepipe.send(message)
for result in results:
while "$(" in result.text:
result = self.handle_inners(result)
funcs, args = self.funcs_n_args(result)
pipe = self.resulter()
for func, args in chain([(f, message) for _, _, f in self.post_command_exts],
list(zip(funcs[::-1], args[::-1]))):
pipe = func(args, pipe)
pipe.send(None)
pipe.close()
except Exception as e:
self.send(message.reply(type(e).__name__ + (": " + str(e)) if str(e) else ""))
exc_type, exc_value, exc_traceback = sys.exc_info()
print("*** print_tb:")
traceback.print_tb(exc_traceback, file=sys.stdout)
def funcs_n_args(self, message):
commands = map(lambda x: [(command, " ".join(args)) for command, *args in [x.split(" ")]][0],
map(lambda x: x.strip(), message.text[1:].split(" || ")))
funcs = []
args = []
for i, (func, arg) in enumerate(commands):
if func in self.commands:
if self.commands[func][1].get("adminonly", False) and message.nick not in self.admins[message.server]:
raise Exception("admin only command: " + func)
else:
funcs.append(self.commands[func][0])
if self.commands[func][1].get("bufferreplace", True):
arg = self.buffer_replace(arg, message.server, message.params, offset=1)
temp = message.reply(arg)
if self.commands[func][1].get("argparse", False):
text, args_ = self.parse_args(temp, self.commands[func][0], func)
temp = message.reply(text)
temp.args = args_
args.append(temp)
elif func in self.aliases:
funcs.append(self.argpass)
args.append(message.reply(arg))
for func_, arg_ in self.aliases[func]:
if self.commands[func_][1].get("adminonly", False) \
and message.nick not in self.admins[message.server]:
raise Exception("admin only command: " + func)
else:
funcs.append(self.commands[func_][0])
if self.commands[func_][1].get("bufferreplace", True):
arg__ = self.buffer_replace(arg_, message.server, message.params, offset=1)
else:
arg__ = arg_
temp = message.reply(arg__)
if self.commands[func_][1].get("argparse", True):
text, args_ = self.parse_args(temp, self.commands[func_][0], func_)
temp = message.reply(text)
temp.args = args_
args.append(temp)
else:
raise Exception("unrecognised command: " + func)
return funcs, args
def parse_args(self, message, func, funcname):
if hasattr(func, "_args"):
args = dict(func._args)
else:
args = {}
if message.text.startswith('"') and message.text.endswith('"'):
return message.data[1:-1], args
words = message.data.split()
msg = []
for word in words:
if word.startswith("--"):
word = word[2:]
if "=" in word:
arg, _, value = word.partition("=")
if arg not in args:
raise Exception("invalid argument: %s for command %s" % (arg, funcname))
values = {
"True": True,
"False": False,
}
if value in values:
value = values[value]
args[arg] = value
else:
if word not in args:
raise Exception("invalid argument: %s for command %s" % (word, funcname))
args[word] = True
elif word.startswith("-"):
word = word[1:]
for flag in word:
if flag not in args:
raise Exception("invalid flag: %s for command %s" % (flag, funcname))
else:
msg.append(word)
return " ".join(msg), args
def handle_inners(self, message):
first = message.text.find("$(")
left = message.text[:first]
rest = message.text[first+1:]
openbr = 0
prevchar = ""
for i, char in enumerate(rest):
if prevchar+char == "$(":
openbr += 1
if char == "$" and prevchar == ")":
if openbr != 0:
openbr -= 1
else:
final = i
break
prevchar = char
else:
raise Exception("no closing bracket")
middle = rest[:final-1]
right = rest[final+1:]
while "$(" in middle:
middle = self.handle_inners(message.reply(middle)).text
funcs, args = self.funcs_n_args(message.reply(middle))
text = []
cat = self.concater(text)
pipe = cat
for func, args in zip(funcs[::-1], args[::-1]):
pipe = func(args, pipe)
pipe.send(None)
pipe.close()
return message.reply(text=left + " ".join(text) + right)
def send(self, message):
if message.server in self.servers:
print(">> " + str(message))
line = message.to_line()
if not line.endswith("\n"):
line += "\n"
self.servers[message.server].socket.send(line.encode())
if message.command == "PRIVMSG":
temp = message.copy()
temp.nick = self.servers[message.server].nick
self.message_buffer[message.server][message.params].appendleft(temp)
else:
raise Exception("no such server: " + message.server)
@_coroutine
def argpass(self, arg, target):
formats = len(list(string.Formatter().parse(arg.text)))
try:
while 1:
x = yield
if x is None:
if not arg.data:
target.send(None)
else:
if formats:
target.send(arg.reply(data=arg.data.format(*([""] * formats)), args=arg.args))
else:
target.send(arg)
else:
if formats:
if x.data is not None:
target.send(x.reply(data=arg.data.format(*([x.data] * formats)), args=arg.args))
else:
target.send(x.reply(data=arg.data.format(*([""] * formats)), args=arg.args))
else:
target.send(x.reply(data=x.data, args=arg.args))
except GeneratorExit:
target.close()
@_coroutine
def resulter(self):
try:
while 1:
x = yield
self.send(x)
except GeneratorExit:
pass # pipe closed
@_coroutine
def concater(self, results):
try:
while 1:
x = yield
results.append(x.data)
except GeneratorExit:
pass # pipe closed
class ThreadPoolStrategy(ConcurrentStrategy, _PoolRunnableStrategy,
_Resultable):
_Thread_Pool: ThreadPool = None
_Thread_List: List[Union[ApplyResult, AsyncResult]] = None
def __init__(self, pool_size: int):
super().__init__(pool_size=pool_size)
def initialization(self,
queue_tasks: Optional[Union[_BaseQueueTask,
_BaseList]] = None,
features: Optional[Union[_BaseFeatureAdapterFactory,
_BaseList]] = None,
*args,
**kwargs) -> None:
super(ThreadPoolStrategy, self).initialization(queue_tasks=queue_tasks,
features=features,
*args,
**kwargs)
# Initialize and build the Processes Pool.
__pool_initializer: Callable = kwargs.get("pool_initializer", None)
__pool_initargs: IterableType = kwargs.get("pool_initargs", None)
self._Thread_Pool = ThreadPool(processes=self.pool_size,
initializer=__pool_initializer,
initargs=__pool_initargs)
def apply(self,
tasks_size: int,
function: Callable,
args: Tuple = (),
kwargs: Dict = {}) -> None:
self.reset_result()
__process_running_result = None
self._Thread_List = [
self._Thread_Pool.apply(func=function, args=args, kwds=kwargs)
for _ in range(tasks_size)
]
for thread in self._Thread_List:
try:
__process_running_result = thread
__exception = None
__process_run_successful = True
except Exception as e:
__exception = e
__process_run_successful = False
# Save Running result state and Running result value as dict
self._result_saving(successful=__process_run_successful,
result=__process_running_result,
exception=__exception)
def async_apply(self,
tasks_size: int,
function: Callable,
args: Tuple = (),
kwargs: Dict = {},
callback: Callable = None,
error_callback: Callable = None) -> None:
self.reset_result()
self._Thread_List = [
self._Thread_Pool.apply_async(func=function,
args=args,
kwds=kwargs,
callback=callback,
error_callback=error_callback)
for _ in range(tasks_size)
]
for process in self._Thread_List:
_process_running_result = None
_process_run_successful = None
_exception = None
try:
_process_running_result = process.get()
_process_run_successful = process.successful()
except Exception as e:
_exception = e
_process_run_successful = False
# Save Running result state and Running result value as dict
self._result_saving(successful=_process_run_successful,
result=_process_running_result,
exception=_exception)
def apply_with_iter(self,
functions_iter: List[Callable],
args_iter: List[Tuple] = None,
kwargs_iter: List[Dict] = None) -> None:
self.reset_result()
__process_running_result = None
if args_iter is None:
args_iter = [() for _ in functions_iter]
if kwargs_iter is None:
kwargs_iter = [{} for _ in functions_iter]
self._Thread_List = [
self._Thread_Pool.apply(func=_func, args=_args, kwds=_kwargs)
for _func, _args, _kwargs in zip(functions_iter, args_iter,
kwargs_iter)
]
for thread in self._Thread_List:
try:
__process_running_result = thread
__exception = None
__process_run_successful = True
except Exception as e:
__exception = e
__process_run_successful = False
# Save Running result state and Running result value as dict
self._result_saving(successful=__process_run_successful,
result=__process_running_result,
exception=__exception)
def async_apply_with_iter(
self,
functions_iter: List[Callable],
args_iter: List[Tuple] = None,
kwargs_iter: List[Dict] = None,
callback_iter: List[Callable] = None,
error_callback_iter: List[Callable] = None) -> None:
self.reset_result()
if args_iter is None:
args_iter = [() for _ in functions_iter]
if kwargs_iter is None:
kwargs_iter = [{} for _ in functions_iter]
if callback_iter is None:
callback_iter = [None for _ in functions_iter]
if error_callback_iter is None:
error_callback_iter = [None for _ in functions_iter]
self._Thread_List = [
self._Thread_Pool.apply_async(func=_func,
args=_args,
kwds=_kwargs,
callback=_callback)
for _func, _args, _kwargs, _callback in zip(
functions_iter, args_iter, kwargs_iter, callback_iter)
]
for process in self._Thread_List:
_process_running_result = None
_process_run_successful = None
_exception = None
try:
_process_running_result = process.get()
_process_run_successful = process.successful()
except Exception as e:
_exception = e
_process_run_successful = False
# Save Running result state and Running result value as dict
self._result_saving(successful=_process_run_successful,
result=_process_running_result,
exception=_exception)
def map(self,
function: Callable,
args_iter: IterableType = (),
chunksize: int = None) -> None:
self.reset_result()
__process_running_result = None
try:
__process_running_result = self._Thread_Pool.map(
func=function, iterable=args_iter, chunksize=chunksize)
__exception = None
__process_run_successful = True
except Exception as e:
__exception = e
__process_run_successful = False
# Save Running result state and Running result value as dict
for __result in (__process_running_result or []):
self._result_saving(successful=__process_run_successful,
result=__result,
exception=None)
def async_map(self,
function: Callable,
args_iter: IterableType = (),
chunksize: int = None,
callback: Callable = None,
error_callback: Callable = None) -> None:
self.reset_result()
__map_result = self._Thread_Pool.map_async(
func=function,
iterable=args_iter,
chunksize=chunksize,
callback=callback,
error_callback=error_callback)
__process_running_result = __map_result.get()
__process_run_successful = __map_result.successful()
# Save Running result state and Running result value as dict
for __result in (__process_running_result or []):
self._result_saving(successful=__process_run_successful,
result=__result,
exception=None)
def map_by_args(self,
function: Callable,
args_iter: IterableType[IterableType] = (),
chunksize: int = None) -> None:
self.reset_result()
__process_running_result = None
try:
__process_running_result = self._Thread_Pool.starmap(
func=function, iterable=args_iter, chunksize=chunksize)
__exception = None
__process_run_successful = True
except Exception as e:
__exception = e
__process_run_successful = False
# Save Running result state and Running result value as dict
for __result in (__process_running_result or []):
self._result_saving(successful=__process_run_successful,
result=__result,
exception=None)
def async_map_by_args(self,
function: Callable,
args_iter: IterableType[IterableType] = (),
chunksize: int = None,
callback: Callable = None,
error_callback: Callable = None) -> None:
self.reset_result()
__map_result = self._Thread_Pool.starmap_async(
func=function,
iterable=args_iter,
chunksize=chunksize,
callback=callback,
error_callback=error_callback)
__process_running_result = __map_result.get()
__process_run_successful = __map_result.successful()
# Save Running result state and Running result value as dict
for __result in (__process_running_result or []):
self._result_saving(successful=__process_run_successful,
result=__result,
exception=None)
def imap(self,
function: Callable,
args_iter: IterableType = (),
chunksize: int = 1) -> None:
self.reset_result()
__process_running_result = None
try:
imap_running_result = self._Thread_Pool.imap(func=function,
iterable=args_iter,
chunksize=chunksize)
__process_running_result = [
result for result in imap_running_result
]
__exception = None
__process_run_successful = True
except Exception as e:
__exception = e
__process_run_successful = False
# Save Running result state and Running result value as dict
for __result in (__process_running_result or []):
self._result_saving(successful=__process_run_successful,
result=__result,
exception=None)
def imap_unordered(self,
function: Callable,
args_iter: IterableType = (),
chunksize: int = 1) -> None:
self.reset_result()
__process_running_result = None
try:
imap_running_result = self._Thread_Pool.imap_unordered(
func=function, iterable=args_iter, chunksize=chunksize)
__process_running_result = [
result for result in imap_running_result
]
__exception = None
__process_run_successful = True
except Exception as e:
__exception = e
__process_run_successful = False
# Save Running result state and Running result value as dict
for __result in (__process_running_result or []):
self._result_saving(successful=__process_run_successful,
result=__result,
exception=None)
def _result_saving(self, successful: bool, result: List,
exception: Exception) -> None:
_thread_result = {
"successful": successful,
"result": result,
"exception": exception
}
# Saving value into list
self._Thread_Running_Result.append(_thread_result)
def close(self) -> None:
self._Thread_Pool.close()
self._Thread_Pool.join()
def terminal(self) -> None:
self._Thread_Pool.terminate()
def get_result(self) -> List[_ConcurrentResult]:
return self.result()
def _saving_process(self) -> List[_ThreadPoolResult]:
_pool_results = []
for __result in self._Thread_Running_Result:
_pool_result = _ThreadPoolResult()
_pool_result.is_successful = __result["successful"]
_pool_result.data = __result["result"]
_pool_results.append(_pool_result)
return _pool_results
class AgentService(metaclass=ABCMeta):
"""Base class for agent_service"""
def __init__(self, config_path: str = 'settings.conf') -> None:
# Config-based parameters
try:
self._agent_config = ConfigParser()
self._agent_config.read(config_path)
# Meta-data extraction
self.name = self._agent_config.get(META_SECTION_KEY, 'name')
self.redis_name = self._agent_config.get(
META_SECTION_KEY, 'redis_name') or f'{self.name}_agent'
self.description = self._agent_config.get(META_SECTION_KEY,
'description')
self.status = self._agent_config.getint(META_SECTION_KEY, 'status')
self.pool = ThreadPool(NUM_THREADS_PER_SERVICE)
# Subscription data extraction
currency = self._agent_config.get(SUBSCRIPTION_SECTION_KEY,
CURRENCY_KEY)
self.subscriptions = {
SUBSCRIPTION_PLANS_KEY: [
{
SUBSCRIPTION_INTERVAL_KEY:
DAY_KEY,
SUBSCRIPTION_INTERVAL_COUNT_KEY:
self._agent_config.getint(SUBSCRIPTION_SECTION_KEY,
DAY_INTERVAL_KEY),
CAPACITY_KEY:
DAY_CAPACITY_KEY,
PLAN_PRICINGS_KEY: [
# TODO: Somehow read it through loop
{
CURRENCY_KEY:
currency,
SALARY_KEY:
self._agent_config.getfloat(
SUBSCRIPTION_SECTION_KEY, DAY_SALARY_KEY)
}
]
},
{
SUBSCRIPTION_INTERVAL_KEY:
WEEK_KEY,
SUBSCRIPTION_INTERVAL_COUNT_KEY:
self._agent_config.getint(SUBSCRIPTION_SECTION_KEY,
WEEK_INTERVAL_KEY),
CAPACITY_KEY:
self._agent_config.getint(SUBSCRIPTION_SECTION_KEY,
WEEK_CAPACITY_KEY),
PLAN_PRICINGS_KEY: [
# TODO: Somehow read it through loop
{
CURRENCY_KEY:
currency,
SALARY_KEY:
self._agent_config.getfloat(
SUBSCRIPTION_SECTION_KEY, WEEK_SALARY_KEY)
}
]
},
{
SUBSCRIPTION_INTERVAL_KEY:
MONTH_KEY,
SUBSCRIPTION_INTERVAL_COUNT_KEY:
self._agent_config.getint(SUBSCRIPTION_SECTION_KEY,
MONTH_INTERVAL_KEY),
CAPACITY_KEY:
self._agent_config.getint(SUBSCRIPTION_SECTION_KEY,
MONTH_CAPACITY_KEY),
PLAN_PRICINGS_KEY: [
# TODO: Somehow read it through loop
{
CURRENCY_KEY:
currency,
SALARY_KEY:
self._agent_config.getfloat(
SUBSCRIPTION_SECTION_KEY, MONTH_SALARY_KEY)
}
]
},
]
}
# Service data extraction
self.port = str(self._agent_config.get(SERVICE_SECTION_KEY,
'port'))
self.elements_checker_period = self._agent_config.getfloat(
SERVICE_SECTION_KEY, 'elements_checker_period')
self.updating_time_interval = self._agent_config.getfloat(
SERVICE_SECTION_KEY, 'updating_period')
self.new_checker_period = self._agent_config.getfloat(
SERVICE_SECTION_KEY, 'new_checker_period')
# Content data (messages) extraction
self.hello_message = self._agent_config.get(
CONTENT_SECTION_KEY, 'hello_message').format(name=self.name)
self.thread_names = {
'service': f'{self.name}_service_thread',
'websocket': f'{self.name}_websocket_thread',
'updater': f'{self.name}_updater_thread'
}
self.platform_access = None
self.agent_db_key = os.environ[f'{self.name.upper()}_AGENT_DB_KEY']
self.db = Firestore(self.agent_db_key)
self.task_running_listeners = {}
self.task_running_updaters = {}
self.running = True
self.updater_thread_lock = threading.Lock()
self.listener_thread_lock = threading.Lock()
atexit.register(self.stop_running_tasks_on_exit)
for task_file_name in self.get_agent_tasks_names():
task_name = task_file_name.replace('.json', '')
threading.Thread(target=lambda: self._start_periodical_updates(
task_name, DEFAULT_PERIOD_TIME_FOR_UPDATES_CHECKER),
daemon=True).start()
except NoSectionError as no_section_err:
error(ENGINE_NAME, f'Missing config section: {no_section_err}')
except NoOptionError as no_option_err:
error(ENGINE_NAME, f'Missing config option: {no_option_err}')
except KeyError as key_error:
error(ENGINE_NAME, f'Missing parameter in config. {key_error}')
except Exception as excpt:
error(ENGINE_NAME, f'Unexpectedly couldn\'t read config. {excpt}')
else:
debug(self.name, f'New agent-service [{self.name}] is registered')
try:
platform_access = self.__get_platform_access()
self.agents_user_id = platform_access.get_user_id()
except Exception as excpt:
# TODO: Catch all the exceptions
error(ENGINE_NAME, excpt)
@abstractmethod
def identify_and_pass_task(self,
element: Element,
agent_task: Dict,
update: bool = False,
doc_id: str = None) -> Optional[Element]:
"""
Identify a task and pass it to appropriate method
:param element: own_adapter.element.Element on which Company-agent should run
:param agent_task: an agent task to get details from
:param update: an element meant to be updated
:param doc_id: optional id of a document in Firestore to send messages from Agent
:return: Target element or None if task wasn't complete correctly
"""
raise NotImplementedError
def _run_on_element(self,
element: Element,
agent_task: Dict = None,
update: bool = False,
doc_id: str = None) -> Optional[Element]:
"""
Running on a target element...
:param element: own_adapter.element.Element on which the agent should run...
:param agent_task: an agent_task with answers for an agent to process
:param update: if an element is being updated
:param doc_id: optional id of a document in Firestore to send messages from Agent
:return: Target element or None if board or board.element or task are not defined
"""
if not element:
logger.exception(self.name, 'Element was not provided')
return None
board = element.get_board()
if not board:
logger.exception(self.name, f'Board is undefined.')
return None
if agent_task:
return self.identify_and_pass_task(element, agent_task, update)
return None
def communication_handling(self,
doc_ref: firestore.firestore.DocumentReference):
"""
Periodically check messages in Firestore and send them to OWN
:param doc_ref: a reference to document containing messages
:return: Nothing
"""
doc_id = None
try:
doc_id = self.db.get_doc_id(doc_ref)
self.task_running_listeners[doc_id] = doc_ref
platform_access = self.__get_platform_access()
content = self.db.get_doc_content(doc_ref)
if not content:
return
board = Board(platform_access,
identifier=content[BOARD_IDENTIFIER_KEY])
self.retrieve_and_upload_data(doc_ref, board)
finally:
if doc_id:
self.db.finish_monitoring_task(doc_ref, listener=True)
self.task_running_listeners.pop(doc_id)
def retrieve_and_upload_data(
self, doc_ref: firestore.firestore.DocumentReference,
board: Board):
"""
Implements communication functionality between agents and platform for constant monitoring tasks
:param doc_ref: a reference to a document containing info about the task
:param board: constant monitoring task's board
:return: Nothing
"""
self.retrieve_and_upload_messages_to_board(doc_ref, board)
def retrieve_and_upload_messages_to_board(
self, doc_ref: firestore.firestore.DocumentReference,
board: Board):
"""
Method for upload messages on a board
:param doc_ref: a reference to a document containing info about the task
:param board: constant monitoring task's board
:return: Nothing
"""
content = self.db.get_doc_content(doc_ref)
if not content:
return
last_message_read = content[LAST_MESSAGE_READ_KEY]
messages = content[MESSAGES_KEY]
last_index = max(last_message_read,
messages[-1][INDEX_KEY] if messages else 0)
new_messages = list(
filter(lambda message: message[INDEX_KEY] > last_index, messages))
self.db.update_document(doc_ref, {
LAST_MESSAGE_READ_KEY: last_index,
MESSAGES_KEY: new_messages,
})
for message in messages:
if message[INDEX_KEY] <= last_index:
board.put_message(message[MESSAGE_KEY])
def stop_running_tasks_on_exit(self):
"""
Update values in DB for monitoring tasks before exiting
:return: Nothing
"""
self.running = False
with self.listener_thread_lock:
for doc in self.task_running_listeners.copy().values():
self.db.finish_monitoring_task(doc_ref=doc, listener=True)
with self.updater_thread_lock:
for doc in self.task_running_updaters.copy().values():
self.db.finish_monitoring_task(doc_ref=doc, update=True)
def _run_on_element_default(
self,
element: Element,
query_id: str = '',
agent_task: Optional[Dict] = None,
update: bool = False,
constant_monitoring: bool = False) -> Optional[Element]:
"""
Run on element and save task to db
:param query_id: an id of agent task query
:param update: if an element is being updated
:param element: own_adapter.element.Element on which the agent should run
:param agent_task: an agent_task with answers for an agent to process
:param constant_monitoring: whether this task should run in constant monitoring mode
:return: Target element
"""
task_name = self.identify_task(agent_task)
return self._run_on_element_and_save_task(
element,
query_id=query_id,
task_name=task_name,
agent_task=agent_task,
update=update,
constant_monitoring=constant_monitoring)
def _run_on_element_and_save_task(self, element: Element, task_name: str, query_id: str,
agent_task: Dict = None, start_listener: bool = False,
update: bool = False, constant_monitoring: bool = False) \
-> Optional[Element]:
"""
Create document to communicate with agent, run on a target element
:param query_id: an id of agent task query
:param task_name: a name of task which is being executed
:param update: if an element is being updated
:param element: own_adapter.element.Element on which the agent should run
:param agent_task: an agent_task with answers for an agent to process
:param constant_monitoring: whether this task should run in constant monitoring mode
:param start_listener: if a communication listener should be started for this task
:return: Target element
"""
self.db.delete_old_agent_tasks(element.get_id())
doc_ref = self.db.create_new_doc_for_task(
element.get_board().get_id(),
element.get_id(),
query_id,
task_name,
update_period=self.updating_time_interval,
constant_monitoring=constant_monitoring,
agent_task=agent_task)
doc_id = self.db.get_doc_id(doc_ref)
try:
self.task_running_updaters[doc_id] = doc_ref
if start_listener or constant_monitoring:
threading.Thread(
target=lambda: self.communication_handling(doc_ref),
daemon=True).start()
res = self._run_on_element(element, agent_task, update, doc_id)
if not res:
self.db.delete_doc(doc_ref)
return res
except Exception as e:
logger.exception(
self.name,
f'Couldn\'t run updates on element for task: {task_name}. Error: {e}'
)
finally:
self.db.finish_monitoring_task(doc_ref, update=True)
self.task_running_updaters.pop(doc_id)
def _run_update_for_task(self, doc_ref: firestore.firestore.DocumentReference, task_name: str) \
-> Optional[Element]:
"""
Run a periodical update for agent task stored in Firestore
:param doc_ref: a reference to Firestore document with info about the task
:param task_name: a name of the task
:return: Nothing
"""
doc_id = None
try:
task_content = self.db.get_doc_content(doc_ref)
platform_access = self.__get_platform_access()
board = Board(platform_access,
identifier=task_content[BOARD_IDENTIFIER_KEY])
element = Element(platform_access,
identifier=task_content[ELEMENT_ID_KEY],
board=board)
agent_task = task_content[AGENT_TASK_KEY]
doc_id = self.db.get_doc_id(doc_ref)
self.task_running_updaters[doc_id] = doc_ref
except Exception as e:
logger.exception(
self.name,
f'Could not perform periodical update for task {task_name}. Error: {e}'
)
return None
else:
self._run_on_element(element=element,
agent_task=agent_task,
update=True,
doc_id=doc_id)
finally:
self.db.finish_monitoring_task(doc_ref, update=True)
if doc_id:
self.task_running_updaters.pop(doc_id)
# Web-sockets
def on_websocket_message(self, ws: websocket.WebSocketApp,
message: str) -> None:
"""Processes websocket messages"""
message_dict = json.loads(message)
content_type = message_dict['contentType']
message_type = content_type.replace('application/vnd.uberblik.', '')
debug(self.name, message)
if message_type == 'liveUpdateAgentTaskElementAnswersSaved+json':
# Get the data from the LiveUpdate's message
agent_data_id = int(message_dict['agentDataId'])
agent_task_id = int(message_dict['agentTaskId'])
element_id = int(message_dict['elementId'])
board_id = int(message_dict['boardId'])
query_id = str(message_dict['agentQueryId'])
agent = self.get_agent()
agent_task = get_agent_task_answers_by_id(
agent.get_platform_access(),
agent_data_id=agent_data_id,
agent_task_id=agent_task_id,
board_id=board_id,
element_id=element_id)
board = Board.get_board_by_id(board_id,
agent.get_platform_access(),
need_name=False)
element = Element.get_element_by_id(element_id,
agent.get_platform_access(),
board)
if element:
# Run AgentTask on the element
updated_element = self._run_on_element_default(
element, query_id, agent_task)
if updated_element:
element = updated_element
element.set_last_processing_time(datetime.datetime.now())
agent.cache_element_to_redis(element)
elif message_type in [
'liveUpdateElementPermanentlyDeleted+json',
'liveUpdateElementDeleted+json'
]:
element_id = f'/{"/".join(message_dict["path"].split("/")[-4:])}'
self.db.delete_old_agent_tasks(element_id)
elif message_type == 'liveUpdateAgentTaskElementDeleted+json':
# Get the data from the LiveUpdate's message
element_id = int(message_dict['elementId'])
board_id = int(message_dict['boardId'])
full_element_id = f'/boards/{board_id}/elements/{element_id}'
self.db.delete_old_agent_tasks(full_element_id)
elif message_type == 'liveUpdateBoardDeleted+json':
board_id = int(message_dict['path'].split('/')[-1])
self.db.delete_old_agent_tasks('', board_id)
def identify_task(self,
agent_task: Optional[Dict] = None,
agent_task_id: Optional[int] = None) -> str:
"""
Identify a task
:param agent_task_id: an id of the agent task
:param agent_task: an agent task to get details from
:return: a name of the task
"""
if agent_task_id:
agent_task_config_id = agent_task_id
else:
if not agent_task:
return ''
agent_task_answers = agent_task['agentTaskElement']['agentTask']
agent_task_config_id = agent_task_answers['id']
# Get all the agent-tasks
possible_files = self.get_agent_tasks_names()
# Find the AgentTask to run
for filename in possible_files:
if self.get_agent_task_config_from_file(filename)['agentTask'].get(
'id', None) == agent_task_config_id:
return filename.replace('.json', '')
return ''
def on_websocket_error(self, ws, err):
"""Logs websocket errors"""
error(self.name, err)
def on_websocket_open(self, ws):
"""Logs websocket openings"""
info(self.name, f'{self.redis_name}\'s websocket is open')
def on_websocket_close(self, ws):
"""Logs websocket closings"""
info(self.name, f'{self.redis_name}\'s Websocket is closed')
def open_websocket(self):
"""Opens a websocket to receive messages from the boards about events"""
agent = self.get_agent()
query = {}
# getting the service url without protocol name
platform_url_no_protocol = agent.get_platform_access(
).get_platform_url().split('://')[1]
query['token'] = agent.get_platform_access().get_access_token()
url = f'{PROTOCOL}://{platform_url_no_protocol}/opensocket?{urllib.parse.urlencode(query)}'
ws = websocket.WebSocketApp(
url,
on_message=lambda *args: self.pool.starmap_async(
self.on_websocket_message, [args]),
on_error=self.on_websocket_error,
on_open=self.on_websocket_open,
on_close=self.on_websocket_close)
ws.run_forever()
def run(self) -> None:
"""
Runs the agent by starting websocket and updater threads
:return: Nothing
"""
if self.status == STATUS['INACTIVE']:
info(self.name, f'The {self.name}-agent is inactive.')
return
websocket_thread = None
# TODO: Add multi-threaded lock/stop here
while True:
# opening a websocket for catching server messages
if websocket_thread is None or not websocket_thread.is_alive():
try:
websocket_thread = threading.Thread(
target=self.open_websocket,
name=self.thread_names['websocket'],
args={},
daemon=True)
websocket_thread.start()
debug(self.name, f'{self.name} started websocket')
except Exception as excpt:
exception(
self.name,
f'Could not open a websocket. Exception message: {str(excpt)}'
)
# wait until next check
time.sleep(10)
# Util
def get_agent(self) -> [Agent, None]:
"""
Returns the current agent
:return: Object: Instance of own_platform.agent.Agent,
or None with incorrect authentication data
"""
platform_access = self.__get_platform_access()
agent = Agent(platform_access, self.redis_name)
return agent
def __get_platform_access(self) -> [PlatformAccess, None]:
"""Returns PlatformAccess with valid agent's credentials (environ), None otherwise"""
if not self.platform_access:
try:
login = os.environ[f'OWN_{self.name.upper()}_AGENT_LOGIN']
password = os.environ[
f'OWN_{self.name.upper()}_AGENT_PASSWORD']
self.platform_access = PlatformAccess(login, password)
except KeyError as key_error:
exception(
self.name,
f'Failed get credentials for {self.name}-agent. Error message: {str(key_error)}'
)
except Exception as err:
error(
self.name,
f'Some error occurred while establishing connection to the platform: {err}'
)
return self.platform_access
def get_service_id(self) -> int:
"""Returns ID of the agent-service"""
response = get_agent_data_by_user_id(self.__get_platform_access(),
self.agents_user_id)
resp_content = json.loads(response.content.decode())
result = int(
resp_content['agentData']['_links'][0]['href'].split('/')[-1])
return result
def get_agent_task_config_from_file(self, file_name: str) -> Dict:
"""
Return an agent task config stored in a file
:param file_name: a name of the file relative to agent tasks folder
:return: the agent task config
"""
file_location = os.path.join(AGENTS_SERVICES_PATH,
f'{self.name}_service', 'agent_tasks',
file_name)
try:
with open(file_location, 'r') as file:
agent_task_config = json.load(file)
except Exception as e:
error(
self.name,
f'Agent task config in {file_name} not found. Exception {e}.')
raise
return agent_task_config
def update_element_caption(self, element: Element,
update_with_data: str) -> None:
"""
Sets the new caption for the element
"<agent_name>: <new_data>[, <old_query>]"
:param element: Element to update the caption on
:param update_with_data: the data (sentence, or a keyword) to update the element with.
:return: Nothing
"""
# Save the old caption
old_caption = element.get_name()
# Form the new caption
new_caption = f'{self.name}: {update_with_data}'
# Do nothing if they are the same
if old_caption == new_caption:
return
# Check if the old caption matches the pattern
pattern = re.compile(f'{self.name}: [^\n\r]+')
if pattern.match(old_caption):
# If it is, extract the query itself, leaving the sender
old_caption_queries = old_caption.replace(f'{self.name}: ', '')
# Check if last query wasn't the same (i.e., it's not a periodical update
last_old_query = old_caption_queries.split(', ')[0]
if last_old_query != update_with_data:
new_caption += f', {old_caption_queries}'
element.set_name(new_caption)
def get_agent_tasks_names(self) -> List[str]:
"""Returns AgentTasks' names in the agent-service agent_task directory"""
# Form the agent_tasks absolute path
this_service_tasks_path = os.path.join(AGENTS_SERVICES_PATH,
f'{self.name}_service',
'agent_tasks')
if not os.path.exists(this_service_tasks_path):
return []
# Get all the filenames that are .json
possible_files = [
file for file in os.listdir(this_service_tasks_path)
if file.endswith('.json')
]
# TODO: Check if they are really AgentTasks
return possible_files
def send_request_to_agent_handler(self, method: str, url: str, params: Dict = None, data: Dict = None) \
-> Optional[requests.Response]:
"""
Send a task to a free agent handler
:param method: an http method to use
:param url: An URL of endpoint
Example: for http://localhost:5000/handler, 'url' will be 'handler'
Example: for http://localhost:5000/, 'url' will be an empty string ('')
:param params: query params to transfer
:param data: data to transfer
:return: response
"""
MAX_NUMBER_OF_ATTEMPTS = 5
number_of_attempts = 0
black_list_of_agent_handlers = set()
response = None
SLEEP_TIME = 2
while number_of_attempts <= MAX_NUMBER_OF_ATTEMPTS:
handler = self.db.get_agent_handler_with_least_amount_of_tasks(
list(black_list_of_agent_handlers))
if not handler:
break
handler_id, handler_info = handler
handler_ip = handler_info.get(AGENT_HANDLER_ADDRESS_KEY, None)
if not handler_ip:
continue
black_list_of_agent_handlers.add(handler_id)
num_of_tasks_in_handler = handler_info.get(
AGENT_HANDLER_NUM_TASKS_KEY, 0)
if not params:
params = {}
params[NUMBER_OF_TASKS_KEY] = num_of_tasks_in_handler
try:
response = requests.request(method=method,
url=f'{handler_ip}/{url}',
params=params,
data=data)
if response.status_code == HTTPStatus.CONFLICT:
black_list_of_agent_handlers.remove(handler_id)
response.raise_for_status()
return response
except requests.ConnectionError as e:
self.db.increase_handler_number_of_fails(handler_id)
logger.exception(
self.name, f'Agent handler is not reachable. Error {e}.',
response)
except requests.RequestException as e:
logger.exception(
self.name, f'Agent handler responded with an error {e}.',
response)
number_of_attempts += 1
time.sleep(SLEEP_TIME * (number_of_attempts**2))
logger.exception(
self.name,
f'Amount of requests to agents from handler exceeded limit.')
return response
def _start_listeners(self, task_name: str, period_time: float):
"""
Check constant tasks in Firestore and start listeners for them
:param task_name: a name of task which is being executed
:param period_time: amount of seconds to wait between checks
:return: Nothing
"""
SLEEP_TIME = 1
MAX_NUMBER_OF_TASKS_FOR_ONE_PERIOD = 10
docs = {}
while True:
for i in range(MAX_NUMBER_OF_TASKS_FOR_ONE_PERIOD):
with self.listener_thread_lock:
if not self.running:
return
doc = self.db.get_agent_task_from_firestore(
task_name,
runners_dict=self.task_running_listeners,
listener=True,
constant_monitoring=True)
if not doc:
break
docs[self.db.get_doc_id(doc)] = doc
time.sleep(SLEEP_TIME)
for doc_id, doc in docs.copy().items():
docs.pop(doc_id)
threading.Thread(
target=lambda: self.communication_handling(doc),
daemon=True).start()
time.sleep(period_time)
def _start_periodical_updates(self, task_name: str, period_time: float):
"""
Check if there are tasks in Firestore that should be updated
:param task_name: a name of a task to check
:param period_time: amount of seconds to wait between checks
:return: Nothing
"""
while True:
with self.updater_thread_lock:
if not self.running:
return
doc = self.db.get_agent_task_from_firestore(
task_name,
runners_dict=self.task_running_updaters,
constant_monitoring=False,
update=True)
if doc:
threading.Thread(
target=lambda: self._run_update_for_task(doc, task_name),
daemon=True).start()
time.sleep(period_time)
def get_file_from_agent_and_send_to_element(self, file_url: str,
filename: str,
element: Element) -> bool:
"""
Downloads a file from agent and uploads it to an element
:param file_url: a url of the file
:param filename: a name of the file to display in the element
:param element: the element where file should be placed
:return: True if files was uploaded, False otherwise
"""
if not (file_url and filename and element):
return False
try:
file_bytes = requests.get(file_url).content
except Exception as e:
logger.error(
self.name,
f'Agent can\'t download file by url {file_url}. Error {e}')
return False
returned_code = element.put_file(filename, file_bytes)
if returned_code not in [
HTTPStatus.CREATED, AdapterStatus.CONNECTION_ABORTED
]:
logger.error(
self.name,
f'Agent can\'t upload file to element {element.get_id()}.')
return False
else:
return True
class PiperBot(threading.Thread):
def __init__(self):
super(PiperBot, self).__init__(daemon=True)
self.servers = MutableNameSpace({})
self.admins = defaultdict(list)
self.command_char = "#"
self.in_queue = PriorityQueue()
self.apikeys = {}
self.commands = {}
self.aliases = {}
self.plugins = {}
self.pre_dispatch_exts = []
self.pre_command_exts = []
self.post_command_exts = []
self.pre_event_exts = []
self.post_event_exts = []
self.pre_trigger_exts = []
self.post_trigger_exts = []
self.pre_regex_exts = []
self.post_regex_exts = []
self.worker_pool = ThreadPool(processes=4)
self.scheduler = Scheduler()
self.scheduler.start()
self.message_buffer = defaultdict(
lambda: defaultdict(lambda: deque(maxlen=50)))
self.buffer_pattern = re.compile(r"(?:(\w+)|(\s)|^)(?:\^(\d+)|(\^+))")
self.buffer_pattern_escape = re.compile(r"\\\^")
self.running = False
def buffer_replace(self, text, servername, channel, offset=0):
buffer = list(self.message_buffer[servername][channel])[offset:]
def buffer_sub(match_object):
if match_object.group(1):
messgages = [
x for x in buffer if x.nick == match_object.group(1)
]
if not messgages:
raise Exception("user not in memory")
else:
messgages = buffer
if match_object.group(3):
count = int(match_object.group(3))
else:
count = len(match_object.group(4))
if count <= len(messgages):
return (" " if match_object.group(2) else
"") + messgages[count - 1].text
else:
raise Exception("line not in memory")
text = self.buffer_pattern.sub(buffer_sub, text)
text = self.buffer_pattern_escape.sub("^", text)
return text
def connect_to(self,
servername,
network,
port,
nick,
channels=None,
admins=None,
password=None,
username=None,
ircname=None,
ssl=False):
self.servers[servername] = ServerConnection(self.in_queue, servername,
network, port, nick,
password, username,
ircname, channels, ssl)
self.servers[servername].connect()
if admins:
self.admins[servername] += admins
def run(self):
self.running = True
try:
while self.running:
try:
message = self.in_queue.get(timeout=10)
print("<< " + str(message))
self.handle_message(message)
except Empty:
pass
finally:
self.shutdown()
def shutdown(self):
self.running = False
for plugin in self.plugins.values():
for func in plugin._onUnloads:
func()
for server in self.servers.values():
server.disconnect(message="shutting down")
def load_plugin_from_module(self, plugin):
if "." in plugin:
module = "".join(plugin.split(".")[:-1])
plugin_name = plugin.split(".")[-1]
temp = importlib.machinery.SourceFileLoader(
module,
os.path.dirname(os.path.abspath(__file__)) + "/plugins/" +
module + ".py").load_module()
found = False
for name, Class in inspect.getmembers(
temp,
lambda x: inspect.isclass(x) and hasattr(x, "_plugin")):
if name == plugin_name:
self.load_plugin(Class)
found = True
if not found:
raise Exception("no such plugin to load")
else:
temp = importlib.machinery.SourceFileLoader(
plugin,
os.path.dirname(os.path.abspath(__file__)) + "/plugins/" +
plugin + ".py").load_module()
found = False
for name, Class in inspect.getmembers(
temp,
lambda x: inspect.isclass(x) and hasattr(x, "_plugin")):
self.load_plugin(Class)
found = True
if not found:
raise Exception("no such plugin to load")
def unload_module(self, module_name):
found = False
plugins = []
for plugin_name, plugin in self.plugins.items():
if plugin_name.startswith(module_name +
".") or plugin_name == module_name:
plugins.append(plugin_name)
found = True
for plugin in plugins:
self.unload_plugin(plugin)
if not found:
raise Exception("no such plugin to unload")
def load_plugin(self, plugin):
if plugin.__module__ + "." + plugin.__name__ in self.plugins:
raise Exception("already loaded!")
plugin_instance = plugin()
plugin_instance._plugin__init__(self)
for func in plugin_instance._onLoads:
func()
self.plugins[plugin_instance.__module__ + "." +
plugin_instance.__class__.__name__] = plugin_instance
for (func, args) in plugin_instance._commands:
if args["command"] not in self.commands:
self.commands[args["command"]] = (func, args)
else:
print("command overlap! : " + args["command"])
for task in plugin_instance._scheduleds:
self.scheduler.add_task(task)
for (priority, func) in plugin_instance._command_extensions:
if priority < 1:
self.pre_command_exts.append(
(priority, plugin_instance.__module__ + "." +
plugin_instance.__class__.__name__, func))
else:
self.post_command_exts.append(
(priority, plugin_instance.__module__ + "." +
plugin_instance.__class__.__name__, func))
for (priority, func) in plugin_instance._event_extensions:
if priority < 1:
self.pre_event_exts.append(
(priority, plugin_instance.__module__ + "." +
plugin_instance.__class__.__name__, func))
else:
self.post_event_exts.append(
(priority, plugin_instance.__module__ + "." +
plugin_instance.__class__.__name__, func))
for (priority, func) in plugin_instance._trigger_extensions:
if priority < 1:
self.pre_trigger_exts.append(
(priority, plugin_instance.__module__ + "." +
plugin_instance.__class__.__name__, func))
else:
self.post_trigger_exts.append(
(priority, plugin_instance.__module__ + "." +
plugin_instance.__class__.__name__, func))
for (priority, func) in plugin_instance._regex_extensions:
if priority < 1:
self.pre_regex_exts.append(
(priority, plugin_instance.__module__ + "." +
plugin_instance.__class__.__name__, func))
else:
self.post_regex_exts.append(
(priority, plugin_instance.__module__ + "." +
plugin_instance.__class__.__name__, func))
self.post_command_exts.sort(key=itemgetter(0, 1))
self.pre_command_exts.sort(key=itemgetter(0, 1))
self.post_event_exts.sort(key=itemgetter(0, 1))
self.pre_event_exts.sort(key=itemgetter(0, 1))
self.post_regex_exts.sort(key=itemgetter(0, 1))
self.pre_regex_exts.sort(key=itemgetter(0, 1))
self.post_trigger_exts.sort(key=itemgetter(0, 1))
self.pre_trigger_exts.sort(key=itemgetter(0, 1))
if plugin._plugin_thread:
plugin_instance.start()
def unload_plugin(self, plugin_name):
for func, args in self.plugins[plugin_name]._commands:
if args["command"] in self.commands.keys():
del self.commands[args["command"]]
for task in self.plugins[plugin_name]._scheduleds:
self.scheduler.cancel_task(task)
for lump in self.post_command_exts:
if lump[1] == plugin_name:
self.post_command_exts.remove(lump)
for lump in self.pre_command_exts:
if lump[1] == plugin_name:
self.pre_command_exts.remove(lump)
for lump in self.post_event_exts:
if lump[1] == plugin_name:
self.post_event_exts.remove(lump)
for lump in self.pre_event_exts:
if lump[1] == plugin_name:
self.pre_event_exts.remove(lump)
for lump in self.post_regex_exts:
if lump[1] == plugin_name:
self.post_regex_exts.remove(lump)
for lump in self.pre_regex_exts:
if lump[1] == plugin_name:
self.pre_regex_exts.remove(lump)
for lump in self.post_trigger_exts:
if lump[1] == plugin_name:
self.post_trigger_exts.remove(lump)
for lump in self.pre_trigger_exts:
if lump[1] == plugin_name:
self.pre_trigger_exts.remove(lump)
for func in self.plugins[plugin_name]._onUnloads:
func()
del self.plugins[plugin_name]
def handle_message(self, message):
if message.command == "PING":
response = message.copy()
response.command = "PONG"
self.send(response)
elif message.command == "PRIVMSG":
self.message_buffer[message.server][message.params].appendleft(
message)
if message.text.startswith(self.command_char) and message.text[1:]:
first = message.text[1:].split()[0]
if first == "alias":
self.handle_alias_assign(message)
elif first in self.commands or first in self.aliases:
self.worker_pool.apply_async(self.handle_command,
args=(message, ))
triggered = []
for plugin in self.plugins.values():
if message.command == "PRIVMSG":
for regex, rfunc in plugin._regexes:
matches = regex.findall(message.text)
for groups in matches:
temp = message.copy()
temp.groups = groups
triggered.append((rfunc, temp, self.pre_regex_exts,
self.post_trigger_exts))
for trigger, tfunc in plugin._triggers:
if trigger(message, bot):
triggered.append((tfunc, message, self.pre_trigger_exts,
self.post_trigger_exts))
for event, efunc in plugin._handlers:
if message.command.lower() == event.lower():
triggered.append((efunc, message, self.pre_event_exts,
self.post_event_exts))
self.worker_pool.starmap_async(self.call_triggered,
triggered,
error_callback=print)
def call_triggered(self, func, message, pre=[], post=[]):
try:
pipe = self.resulter()
for _, _, func_, in post:
pipe = func_(message, pipe)
pipe = func(pipe)
next(pipe)
for _, _, func_ in pre:
pipe = func_(message, pipe)
pipe.send(message)
except StopIteration as e:
raise e
except Exception as e:
print("Error calling trigger: %s : %s: %s" %
(func.__name__, type(e), e))
exc_type, exc_value, exc_traceback = sys.exc_info()
print("*** print_tb:")
traceback.print_tb(exc_traceback, file=sys.stdout)
def handle_alias_assign(self, message):
try:
name, *alias = message.text[len(self.command_char) + 5:].split("=")
name = name.strip()
alias = "=".join(alias).strip()
if name in self.commands:
raise Exception("cannot overwrite existing command")
commands = map(
lambda x: [(command, " ".join(args))
for command, *args in [x.split(" ")]][0],
map(lambda x: x.strip(), alias.split(" || ")))
funcs = []
args = []
for func, arg in commands:
if func in self.commands:
funcs.append(self.commands[func][1]["command"])
args.append(arg)
elif func in self.aliases:
if func == name:
raise Exception("recursion detected")
funcs_, args_ = self.aliases[func]
funcs.extend(funcs_)
args.extend(args_)
else:
raise Exception("unrecognised command: " + func)
self.aliases[name] = list(zip(funcs, args))
self.send(message.reply("The alias %s has been saved" % name))
except Exception as e:
self.send(
message.reply(
type(e).__name__ + (": " + str(e)) if str(e) else ""))
exc_type, exc_value, exc_traceback = sys.exc_info()
print("*** print_tb:")
traceback.print_tb(exc_traceback, file=sys.stdout)
def handle_command(self, message):
try:
def prepper(results):
while 1:
x = yield
results.append(x)
results = []
prepipe = prepper(results)
next(prepipe)
for _, _, f in self.pre_command_exts:
prepipe = f(message, prepipe)
prepipe.send(message)
for result in results:
while "$(" in result.text:
result = self.handle_inners(result)
funcs, args = self.funcs_n_args(result)
pipe = self.resulter()
for func, args in chain(
[(f, message) for _, _, f in self.post_command_exts],
list(zip(funcs[::-1], args[::-1]))):
pipe = func(args, pipe)
pipe.send(None)
pipe.close()
except Exception as e:
self.send(
message.reply(
type(e).__name__ + (": " + str(e)) if str(e) else ""))
exc_type, exc_value, exc_traceback = sys.exc_info()
print("*** print_tb:")
traceback.print_tb(exc_traceback, file=sys.stdout)
def funcs_n_args(self, message):
commands = map(
lambda x: [(command, " ".join(args))
for command, *args in [x.split(" ")]][0],
map(lambda x: x.strip(), message.text[1:].split(" || ")))
funcs = []
args = []
for i, (func, arg) in enumerate(commands):
if func in self.commands:
if self.commands[func][1].get(
"adminonly", False
) and message.nick not in self.admins[message.server]:
raise Exception("admin only command: " + func)
else:
funcs.append(self.commands[func][0])
if self.commands[func][1].get("bufferreplace", True):
arg = self.buffer_replace(arg,
message.server,
message.params,
offset=1)
temp = message.reply(arg)
if self.commands[func][1].get("argparse", False):
text, args_ = self.parse_args(temp,
self.commands[func][0],
func)
temp = message.reply(text)
temp.args = args_
args.append(temp)
elif func in self.aliases:
funcs.append(self.argpass)
args.append(message.reply(arg))
for func_, arg_ in self.aliases[func]:
if self.commands[func_][1].get("adminonly", False) \
and message.nick not in self.admins[message.server]:
raise Exception("admin only command: " + func)
else:
funcs.append(self.commands[func_][0])
if self.commands[func_][1].get("bufferreplace", True):
arg__ = self.buffer_replace(arg_,
message.server,
message.params,
offset=1)
else:
arg__ = arg_
temp = message.reply(arg__)
if self.commands[func_][1].get("argparse", True):
text, args_ = self.parse_args(
temp, self.commands[func_][0], func_)
temp = message.reply(text)
temp.args = args_
args.append(temp)
else:
raise Exception("unrecognised command: " + func)
return funcs, args
def parse_args(self, message, func, funcname):
if hasattr(func, "_args"):
args = dict(func._args)
else:
args = {}
if message.text.startswith('"') and message.text.endswith('"'):
return message.data[1:-1], args
words = message.data.split()
msg = []
for word in words:
if word.startswith("--"):
word = word[2:]
if "=" in word:
arg, _, value = word.partition("=")
if arg not in args:
raise Exception("invalid argument: %s for command %s" %
(arg, funcname))
values = {
"True": True,
"False": False,
}
if value in values:
value = values[value]
args[arg] = value
else:
if word not in args:
raise Exception("invalid argument: %s for command %s" %
(word, funcname))
args[word] = True
elif word.startswith("-"):
word = word[1:]
for flag in word:
if flag not in args:
raise Exception("invalid flag: %s for command %s" %
(flag, funcname))
else:
msg.append(word)
return " ".join(msg), args
def handle_inners(self, message):
first = message.text.find("$(")
left = message.text[:first]
rest = message.text[first + 1:]
openbr = 0
prevchar = ""
for i, char in enumerate(rest):
if prevchar + char == "$(":
openbr += 1
if char == "$" and prevchar == ")":
if openbr != 0:
openbr -= 1
else:
final = i
break
prevchar = char
else:
raise Exception("no closing bracket")
middle = rest[:final - 1]
right = rest[final + 1:]
while "$(" in middle:
middle = self.handle_inners(message.reply(middle)).text
funcs, args = self.funcs_n_args(message.reply(middle))
text = []
cat = self.concater(text)
pipe = cat
for func, args in zip(funcs[::-1], args[::-1]):
pipe = func(args, pipe)
pipe.send(None)
pipe.close()
return message.reply(text=left + " ".join(text) + right)
def send(self, message):
if message.server in self.servers:
print(">> " + str(message))
line = message.to_line()
if not line.endswith("\n"):
line += "\n"
self.servers[message.server].socket.send(line.encode())
if message.command == "PRIVMSG":
temp = message.copy()
temp.nick = self.servers[message.server].nick
self.message_buffer[message.server][message.params].appendleft(
temp)
else:
raise Exception("no such server: " + message.server)
@_coroutine
def argpass(self, arg, target):
formats = len(list(string.Formatter().parse(arg.text)))
try:
while 1:
x = yield
if x is None:
if not arg.data:
target.send(None)
else:
if formats:
target.send(
arg.reply(data=arg.data.format(*([""] *
formats)),
args=arg.args))
else:
target.send(arg)
else:
if formats:
if x.data is not None:
target.send(
x.reply(data=arg.data.format(*([x.data] *
formats)),
args=arg.args))
else:
target.send(
x.reply(data=arg.data.format(*([""] *
formats)),
args=arg.args))
else:
target.send(x.reply(data=x.data, args=arg.args))
except GeneratorExit:
target.close()
@_coroutine
def resulter(self):
try:
while 1:
x = yield
self.send(x)
except GeneratorExit:
pass # pipe closed
@_coroutine
def concater(self, results):
try:
while 1:
x = yield
results.append(x.data)
except GeneratorExit:
pass # pipe closed
@staticmethod
def from_config(filename):
config = json.load(json_config)
print(json.dumps(config, sort_keys=True, indent=4))
print(list(config.keys()))
bot = PiperBot()
if "apikeys" in config:
for api, key in config["apikeys"]:
bot.apikeys[api] = key
for plugin, config_ in config["plugins"].items():
bot.load_plugin_from_module(plugin)
for server in config["servers"]:
server_name = server["IRCNet"]
network = server["IRCHost"]
name = server["IRCName"]
user = server["IRCUser"]
port = server["IRCPort"]
nick = server["IRCNick"]
password = server["IRCPass"] if "IRCPass" in server else None
autojoin = server["AutoJoin"]
admins = server["Admins"]
usessl = server["UseSSL"]
bot.connect_to(server_name, network, port, nick, autojoin, admins,
password, user, name, usessl)
return bot
