def process_and_save_patches(out_path, all_files, patch_size): """ Function that processes and saves the patches from the images """ img_queue = Queue(150) load_proc = Process(target=_load_task, args=(img_queue, all_files)) load_proc.start() save_pool = ThreadPool() cont_patches = 0 out_path = Path(out_path) _verify_out_path(out_path) dir_format = 'dir{}' while True: img = img_queue.get() if not len(img): break patches = extract_img_patches(img, patch_size) num_patches = len(patches) begin = cont_patches end = cont_patches + num_patches curr_dir = out_path / Path(dir_format.format(begin // 50000)) if not curr_dir.exists(): curr_dir.mkdir() names = [ curr_dir / ('patch_' + str(n) + '.png') for n in np.arange(begin, end) ] save_pool.starmap_async(save_img_from_array, zip(patches, names)) cont_patches = end save_pool.close() save_pool.join()
def run(self, n_games=int(1e4), initial_temperature=1.0, n_rollouts=1600, temperature_decay_factor=0.95, moves_before_decaying=8, verbose=0): pool = ThreadPool() progress_bar = tqdm.tqdm(total=n_games) assert verbose == 0 arguments = (self.model.step_model, self.model.step_model, n_rollouts, self.c_puct, 0, initial_temperature, temperature_decay_factor, moves_before_decaying, progress_bar, None, None, None, None, verbose) iterable_arguments = [arguments] * n_games res = pool.starmap_async(play_single_game, iterable_arguments) res.wait() progress_bar.close() for (winner, history) in res.get(): save_memory(self.replay_memory, winner, history)
def run_solver(args, single_solver=False): solver = args[0] command = args[1] problems = args[2] problem_sets = args[3] finished_instances = [] for problem_set in problem_sets: problem_set_name = problem_set[10:] if (len(problem_set_name) > 0): filename = "%s/%s_%s.csv" % (RESULTS_DIR, solver, problem_set[10:]) exists = os.path.isfile(filename) if (exists): file_data = np.genfromtxt(filename, delimiter=',', dtype=None, encoding=None, names=[ "Instance", "Result", "Time", "conflict_size", "average_len", "reduction", "i_uip_reduction", "mem_use", "core_clause", "lbd" ], skip_header=1) op = 'a+' finished_instances = [ problem_set + '/' + item[0] for item in file_data ] else: op = 'w+' with open(filename, op, buffering=1) as fp: if not exists: fp.write(CSV_HEADER) if single_solver: lock = multiprocessing.Lock() number_processes = int(multiprocessing.cpu_count() / 2) - 1 pool = ThreadPool(number_processes) tasks = [] for problem in problems: if (problem.startswith(problem_set) ) and problem not in finished_instances: tasks.append((solver, command, problem, fp, lock)) results = pool.starmap_async(run_problem_and_write_result, tasks) pool.close() pool.join() else: for problem in problems: if (problem.startswith(problem_set) ) and problem not in finished_instances: result = run_problem(solver, command, problem) fp.write( "%s,%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s\n" % (result.problem, result.result, result.elapsed, result.conflict_size, result.average_len, result.reduction, result.i_uip_reduction, result.mem_use, result.core_clause, result.lbd, result.attempt_rate, result.success_rate))
def execute_function_in_parallel(func: Callable, list_args: List, processes: bool = False, local_pool: bool = False, num_threads: int = NUM_OF_THREADS, num_processes: int = NUM_OF_PROCESSES) -> List: """ Execute a function in parallel using ThreadPool or ProcessPool :param local_pool: create a new pool of threads/processes :param num_processes: a num of processes to create :param num_threads: a num of threads to create :param processes: execute tasks in separate processes :param func: a func to call :param list_args: an array containing calling params :return: an array with results """ if not (func and list_args): return [] if local_pool: pool = ThreadPool(num_threads) if not processes else Pool(num_processes) else: global process_pool if processes: if not process_pool: process_pool = Pool(NUM_OF_PROCESSES) pool = process_pool else: global thread_pool if not thread_pool: thread_pool = ThreadPool(NUM_OF_THREADS) pool = thread_pool results_tmp = pool.starmap_async(func, list_args) results = [result for result in results_tmp.get() if result is not None] if local_pool: pool.close() return results
def PoolByModule(): pool = ThreadPool(processes=cpu_count() << 2) # for i in range(20): # pool.apply(func, args=(i,)) # 同步方式,会等待上一个线程执行结束 # # pool.apply_async(func, args=(i,)) # 异步方式,'并行' # 线程池map操作 # res = pool.map(func, range(20)) # print(res) # res = pool.map_async(func, range(20)) # print(res.get()) # 线程池imap操作 # res = pool.imap(func, range(20)) # print([i for i in res]) # res = pool.imap_unordered(func, range(20)) # print([i for i in res]) # 多输入参数方法实现调用 print([i for i in enumerate(range(20))]) res = pool.starmap(add, enumerate(range(20))) print(res) res = pool.starmap_async(add, enumerate(range(20))) print(res.get())
def evaluate(self, model, n_games=400, initial_temperature=0.25, n_rollouts=1600, new_best_model_threshold=0.55, temperature_decay_factor=0.95, moves_before_decaying=8, eval_iter=0, verbose=0): # log_every_n_games = max(2, n_games // 4) log_n_games = 8 log_n_games_detailed = 1 n_wins = 0 pool = ThreadPool() progress_bar = tqdm.tqdm(total=n_games) iterable_arguments = [ ( self.model.train_model, self.model.step_model, n_rollouts, self.c_puct, (i % 2), initial_temperature, temperature_decay_factor, moves_before_decaying, progress_bar, eval_iter, i, self.model.sess, self.model.summary_writer, # verbose if verbose and i % log_every_n_games == 0 else 0 2 if i < log_n_games_detailed else 1 if i < log_n_games else 0) for i in range(n_games) ] res = pool.starmap_async(play_single_game, iterable_arguments) res.wait() progress_bar.close() for (winner, _) in res.get(): if winner == 0: n_wins += 1 print("win_ratio={}".format(100 * n_wins / n_games), file=sys.stderr) win_ratio_summary = model.sess.run(self.win_ratio_summary_op, feed_dict={ self.n_wins_ph: n_wins, self.n_games_ph: n_games }) model.summary_writer.add_summary(win_ratio_summary, eval_iter) return n_wins / n_games > new_best_model_threshold
def list_Terminator(self, Sub): pi = self.get_play_list_id() videos, number = self.get_video_urls(pi) print("there are {} videos in this playlist".format(number)) print("Getting streams, please wait ... ") pool = ThreadPool(processes=16) res = pool.map_async(self.title_for_url, (videos[i] for i in range(len(videos)))) streams = res.get() for i in range(len(streams)): print(str(i + 1) + "-" + streams[i]) select = input("Enter Number of videos you want with '-' between:") create_dir_in_path(self.download_dir, pi) if (not Sub) and (select == '0'): #just download all list without sub e = 0 s = 0 formatt = input("what format do you want:") ress = input("what resolotion do you want:") import time t1 = time.time() pool = ThreadPool(processes=4) res = pool.starmap_async( self.single_video_downloader, ((videos[i], 'video', formatt, ress, pi, i) for i in range(len(videos)))) res = res.get() print(res) print(time.time() - t1) elif (not Sub) and (select != '0'): #downloading choosen items without sub pass elif (Sub) and (select == '0'): #download all list with sub pass elif (Sub) and (select != '0'): #downloading choosen items with sub pass else: print( "the value of sub parameter or the numbers of videos is invalid" )
def _submit_scan_job(targets: list): """ Run a multithreaded scan across all targets specified """ # Attempt to randomize as a jury rigged method of scanning across multiple hosts at once random.shuffle(targets) pool = ThreadPool(processes=CONFIG['thread_count']) async_result = pool.starmap_async(_scan_target, targets) results = async_result.get() # Un-randomize results sorted_results = sorted(results, key=lambda k: k['target']) return sorted_results
def seg(request): """View for seg.""" if request.method == 'POST': data = json.loads(request.body.decode('utf-8')) source_text = data['source_text'] source_text = source_text.replace('\n', ' ') # remove linebreaks segmentators = data['segmentators'] segmentators = [i.lower() for i in segmentators] pool = ThreadPool(processes=5) res = pool.starmap_async(_segwrap, zip(segmentators, repeat(source_text))) vals = res.get() seg_with_diff = segcomp(vals) return HttpResponse(json.dumps(seg_with_diff), content_type="application/json") elif request.method == 'GET': return HttpResponse('') return Http404
def seg(request): """View for seg.""" if request.method == 'POST': data = json.loads(request.body.decode('utf-8')) source_text = data['source_text'] source_text = source_text.replace('\n', ' ') # remove linebreaks segmentators = data['segmentators'] segmentators = [i.lower() for i in segmentators] pool = ThreadPool(processes=5) res = pool.starmap_async(_segwrap, zip(segmentators, repeat(source_text))) vals = res.get() seg_with_diff = segcomp(vals) return HttpResponse(json.dumps(seg_with_diff), content_type="application/json") elif request.method == 'GET': return HttpResponse('') return Http404
class PiperBot(threading.Thread): def __init__(self): super(PiperBot, self).__init__(daemon=True) self.servers = {} self.admins = defaultdict(list) self.users = defaultdict(lambda: defaultdict(User)) self.ops = defaultdict(lambda: defaultdict(set)) self.command_char = "#" self.in_queue = PriorityQueue() self.apikeys = {} self.commands = {} self.aliases = {} self.plugins = {} self.pre_command_exts = [] self.post_command_exts = [] self.pre_event_exts = [] self.post_event_exts = [] self.pre_trigger_exts = [] self.post_trigger_exts = [] self.pre_regex_exts = [] self.post_regex_exts = [] self.worker_pool = ThreadPool(processes=8) self.message_buffer = defaultdict(lambda: defaultdict(lambda: deque(maxlen=50))) self.buffer_pattern = re.compile(r"(?:(\w+)|(\s)|^)(?:\^(\d+)|(\^+))") self.buffer_pattern_escape = re.compile(r"\\\^") self.running = False def buffer_replace(self, text, servername, channel, offset=0): buffer = list(self.message_buffer[servername][channel])[offset:] def buffer_sub(match_object): if match_object.group(1): messgages = [x for x in buffer if x.nick == match_object.group(1)] if not messgages: raise Exception("user not in memory") else: messgages = buffer if match_object.group(3): count = int(match_object.group(3)) else: count = len(match_object.group(4)) if count <= len(messgages): return (" " if match_object.group(2) else "") + messgages[count - 1].text else: raise Exception("line not in memory") text = self.buffer_pattern.sub(buffer_sub, text) text = self.buffer_pattern_escape.sub("^", text) return text def connect_to(self, servername, network, port, nick, channels=None, admins=None, password=None, username=None, ircname=None, ssl=False): self.servers[servername] = ServerConnection(self.in_queue, servername, network, port, nick, password, username, ircname, channels, ssl) self.servers[servername].connect() if admins: self.admins[servername] += admins def run(self): self.running = True try: while self.running: try: message = self.in_queue.get(timeout=10) print("<< " + str(message)) self.handle_message(message) except Empty: pass finally: self.shutdown() def shutdown(self): self.running = False for plugin in self.plugins.values(): for func in plugin._onUnloads: func() for server in self.servers.values(): server.disconnect(message="shutting down") def load_plugin_from_module(self, plugin): if "." in plugin: module = "".join(plugin.split(".")[:-1]) plugin_name = plugin.split(".")[-1] temp = importlib.machinery.SourceFileLoader(module, os.path.dirname( os.path.abspath(__file__)) + "/plugins/" + module + ".py").load_module() found = False for name, Class in inspect.getmembers(temp, lambda x: inspect.isclass(x) and hasattr(x, "_plugin")): if name == plugin_name: self.load_plugin(Class) found = True if not found: raise Exception("no such plugin to load") else: temp = importlib.machinery.SourceFileLoader(plugin, os.path.dirname( os.path.abspath(__file__)) + "/plugins/" + plugin + ".py").load_module() found = False for name, Class in inspect.getmembers(temp, lambda x: inspect.isclass(x) and hasattr(x, "_plugin")): self.load_plugin(Class) found = True if not found: raise Exception("no such plugin to load") def unload_module(self, module_name): found = False plugins = [] for plugin_name, plugin in self.plugins.items(): if plugin_name.startswith(module_name + ".") or plugin_name == module_name: plugins.append(plugin_name) found = True for plugin in plugins: self.unload_plugin(plugin) if not found: raise Exception("no such plugin to unload") def load_plugin(self, plugin): if plugin.__module__ + "." + plugin.__name__ in self.plugins: raise Exception("already loaded!") plugin_instance = plugin() plugin_instance._plugin__init__(self) for func in plugin_instance._onLoads: func() self.plugins[plugin_instance.__module__ + "." + plugin_instance.__class__.__name__] = plugin_instance for (func, args) in plugin_instance._commands: if args["command"] not in self.commands: self.commands[args["command"]] = (func, args) else: print("command overlap! : " + args["command"]) for (priority, func) in plugin_instance._command_extensions: if priority < 1: self.pre_command_exts.append((priority, plugin_instance.__module__ + "." + plugin_instance.__class__.__name__, func)) else: self.post_command_exts.append((priority, plugin_instance.__module__ + "." + plugin_instance.__class__.__name__, func)) for (priority, func) in plugin_instance._event_extensions: if priority < 1: self.pre_event_exts.append((priority, plugin_instance.__module__ + "." + plugin_instance.__class__.__name__, func)) else: self.post_event_exts.append((priority, plugin_instance.__module__ + "." + plugin_instance.__class__.__name__, func)) for (priority, func) in plugin_instance._trigger_extensions: if priority < 1: self.pre_trigger_exts.append((priority, plugin_instance.__module__ + "." + plugin_instance.__class__.__name__, func)) else: self.post_trigger_exts.append((priority, plugin_instance.__module__ + "." + plugin_instance.__class__.__name__, func)) for (priority, func) in plugin_instance._regex_extensions: if priority < 1: self.pre_regex_exts.append((priority, plugin_instance.__module__ + "." + plugin_instance.__class__.__name__, func)) else: self.post_regex_exts.append((priority, plugin_instance.__module__ + "." + plugin_instance.__class__.__name__, func)) self.post_command_exts.sort(key=itemgetter(0, 1)) self.pre_command_exts.sort(key=itemgetter(0, 1)) self.post_event_exts.sort(key=itemgetter(0, 1)) self.pre_event_exts.sort(key=itemgetter(0, 1)) self.post_regex_exts.sort(key=itemgetter(0, 1)) self.pre_regex_exts.sort(key=itemgetter(0, 1)) self.post_trigger_exts.sort(key=itemgetter(0, 1)) self.pre_trigger_exts.sort(key=itemgetter(0, 1)) if plugin._plugin_thread: plugin_instance.start() def unload_plugin(self, plugin_name): for func, args in self.plugins[plugin_name]._commands: if args["command"] in self.commands.keys(): del self.commands[args["command"]] for lump in self.post_command_exts: if lump[1] == plugin_name: self.post_command_exts.remove(lump) for lump in self.pre_command_exts: if lump[1] == plugin_name: self.pre_command_exts.remove(lump) for lump in self.post_event_exts: if lump[1] == plugin_name: self.post_event_exts.remove(lump) for lump in self.pre_event_exts: if lump[1] == plugin_name: self.pre_event_exts.remove(lump) for lump in self.post_regex_exts: if lump[1] == plugin_name: self.post_regex_exts.remove(lump) for lump in self.pre_regex_exts: if lump[1] == plugin_name: self.pre_regex_exts.remove(lump) for lump in self.post_trigger_exts: if lump[1] == plugin_name: self.post_trigger_exts.remove(lump) for lump in self.pre_trigger_exts: if lump[1] == plugin_name: self.pre_trigger_exts.remove(lump) for func in self.plugins[plugin_name]._onUnloads: func() del self.plugins[plugin_name] def handle_message(self, message): if message.command == "PING": response = message.copy() response.command = "PONG" self.send(response) elif message.command == "PRIVMSG": self.message_buffer[message.server][message.params].appendleft(message) if message.text.startswith(self.command_char) and message.text[1:]: first = message.text[1:].split()[0] if first == "alias": self.handle_alias_assign(message) elif first in self.commands or first in self.aliases: self.worker_pool.apply_async(self.handle_command, args=(message,)) triggered = [] for plugin in self.plugins.values(): if message.command == "PRIVMSG": for regex, rfunc in plugin._regexes: matches = regex.findall(message.text) for groups in matches: temp = message.copy() temp.groups = groups triggered.append((rfunc, temp, self.pre_regex_exts, self.post_trigger_exts)) for trigger, tfunc in plugin._triggers: if trigger(message, bot): triggered.append((tfunc, message, self.pre_trigger_exts, self.post_trigger_exts)) for event, efunc in plugin._handlers: if message.command.lower() == event.lower(): triggered.append((efunc, message, self.pre_event_exts, self.post_event_exts)) self.worker_pool.starmap_async(self.call_triggered, triggered, error_callback=print) def call_triggered(self, func, message, pre=[], post=[]): pipe = self.resulter() for _, _, func_, in post: pipe = func_(message, pipe) pipe = func(pipe) next(pipe) for _, _, func_ in pre: pipe = func_(message, pipe) pipe.send(message) def handle_alias_assign(self, message): try: name, *alias = message.text[len(self.command_char)+5:].split("=") name = name.strip() alias = "=".join(alias).strip() if name in self.commands: raise Exception("cannot overwrite existing command") commands = map(lambda x: [(command, " ".join(args)) for command, *args in [x.split(" ")]][0], map(lambda x: x.strip(), alias.split(" || "))) funcs = [] args = [] for func, arg in commands: if func in self.commands: funcs.append(self.commands[func][1]["command"]) args.append(arg) elif func in self.aliases: if func == name: raise Exception("recursion detected") funcs_, args_ = self.aliases[func] funcs.extend(funcs_) args.extend(args_) else: raise Exception("unrecognised command: " + func) self.aliases[name] = list(zip(funcs, args)) self.send(message.reply("The alias %s has been saved" % name)) except Exception as e: self.send(message.reply(type(e).__name__ + (": " + str(e)) if str(e) else "")) exc_type, exc_value, exc_traceback = sys.exc_info() print("*** print_tb:") traceback.print_tb(exc_traceback, file=sys.stdout) def handle_command(self, message): try: def prepper(results): while 1: x = yield results.append(x) results = [] prepipe = prepper(results) next(prepipe) for _, _, f in self.pre_command_exts: prepipe = f(message, prepipe) prepipe.send(message) for result in results: while "$(" in result.text: result = self.handle_inners(result) funcs, args = self.funcs_n_args(result) pipe = self.resulter() for func, args in chain([(f, message) for _, _, f in self.post_command_exts], list(zip(funcs[::-1], args[::-1]))): pipe = func(args, pipe) pipe.send(None) pipe.close() except Exception as e: self.send(message.reply(type(e).__name__ + (": " + str(e)) if str(e) else "")) exc_type, exc_value, exc_traceback = sys.exc_info() print("*** print_tb:") traceback.print_tb(exc_traceback, file=sys.stdout) def funcs_n_args(self, message): commands = map(lambda x: [(command, " ".join(args)) for command, *args in [x.split(" ")]][0], map(lambda x: x.strip(), message.text[1:].split(" || "))) funcs = [] args = [] for i, (func, arg) in enumerate(commands): if func in self.commands: if self.commands[func][1].get("adminonly", False) and message.nick not in self.admins[message.server]: raise Exception("admin only command: " + func) else: funcs.append(self.commands[func][0]) if self.commands[func][1].get("bufferreplace", True): arg = self.buffer_replace(arg, message.server, message.params, offset=1) temp = message.reply(arg) if self.commands[func][1].get("argparse", False): text, args_ = self.parse_args(temp, self.commands[func][0], func) temp = message.reply(text) temp.args = args_ args.append(temp) elif func in self.aliases: funcs.append(self.argpass) args.append(message.reply(arg)) for func_, arg_ in self.aliases[func]: if self.commands[func_][1].get("adminonly", False) \ and message.nick not in self.admins[message.server]: raise Exception("admin only command: " + func) else: funcs.append(self.commands[func_][0]) if self.commands[func_][1].get("bufferreplace", True): arg__ = self.buffer_replace(arg_, message.server, message.params, offset=1) else: arg__ = arg_ temp = message.reply(arg__) if self.commands[func_][1].get("argparse", True): text, args_ = self.parse_args(temp, self.commands[func_][0], func_) temp = message.reply(text) temp.args = args_ args.append(temp) else: raise Exception("unrecognised command: " + func) return funcs, args def parse_args(self, message, func, funcname): if hasattr(func, "_args"): args = dict(func._args) else: args = {} if message.text.startswith('"') and message.text.endswith('"'): return message.data[1:-1], args words = message.data.split() msg = [] for word in words: if word.startswith("--"): word = word[2:] if "=" in word: arg, _, value = word.partition("=") if arg not in args: raise Exception("invalid argument: %s for command %s" % (arg, funcname)) values = { "True": True, "False": False, } if value in values: value = values[value] args[arg] = value else: if word not in args: raise Exception("invalid argument: %s for command %s" % (word, funcname)) args[word] = True elif word.startswith("-"): word = word[1:] for flag in word: if flag not in args: raise Exception("invalid flag: %s for command %s" % (flag, funcname)) else: msg.append(word) return " ".join(msg), args def handle_inners(self, message): first = message.text.find("$(") left = message.text[:first] rest = message.text[first+1:] openbr = 0 prevchar = "" for i, char in enumerate(rest): if prevchar+char == "$(": openbr += 1 if char == "$" and prevchar == ")": if openbr != 0: openbr -= 1 else: final = i break prevchar = char else: raise Exception("no closing bracket") middle = rest[:final-1] right = rest[final+1:] while "$(" in middle: middle = self.handle_inners(message.reply(middle)).text funcs, args = self.funcs_n_args(message.reply(middle)) text = [] cat = self.concater(text) pipe = cat for func, args in zip(funcs[::-1], args[::-1]): pipe = func(args, pipe) pipe.send(None) pipe.close() return message.reply(text=left + " ".join(text) + right) def send(self, message): if message.server in self.servers: print(">> " + str(message)) line = message.to_line() if not line.endswith("\n"): line += "\n" self.servers[message.server].socket.send(line.encode()) if message.command == "PRIVMSG": temp = message.copy() temp.nick = self.servers[message.server].nick self.message_buffer[message.server][message.params].appendleft(temp) else: raise Exception("no such server: " + message.server) @_coroutine def argpass(self, arg, target): formats = len(list(string.Formatter().parse(arg.text))) try: while 1: x = yield if x is None: if not arg.data: target.send(None) else: if formats: target.send(arg.reply(data=arg.data.format(*([""] * formats)), args=arg.args)) else: target.send(arg) else: if formats: if x.data is not None: target.send(x.reply(data=arg.data.format(*([x.data] * formats)), args=arg.args)) else: target.send(x.reply(data=arg.data.format(*([""] * formats)), args=arg.args)) else: target.send(x.reply(data=x.data, args=arg.args)) except GeneratorExit: target.close() @_coroutine def resulter(self): try: while 1: x = yield self.send(x) except GeneratorExit: pass # pipe closed @_coroutine def concater(self, results): try: while 1: x = yield results.append(x.data) except GeneratorExit: pass # pipe closed
class ThreadPoolStrategy(ConcurrentStrategy, _PoolRunnableStrategy, _Resultable): _Thread_Pool: ThreadPool = None _Thread_List: List[Union[ApplyResult, AsyncResult]] = None def __init__(self, pool_size: int): super().__init__(pool_size=pool_size) def initialization(self, queue_tasks: Optional[Union[_BaseQueueTask, _BaseList]] = None, features: Optional[Union[_BaseFeatureAdapterFactory, _BaseList]] = None, *args, **kwargs) -> None: super(ThreadPoolStrategy, self).initialization(queue_tasks=queue_tasks, features=features, *args, **kwargs) # Initialize and build the Processes Pool. __pool_initializer: Callable = kwargs.get("pool_initializer", None) __pool_initargs: IterableType = kwargs.get("pool_initargs", None) self._Thread_Pool = ThreadPool(processes=self.pool_size, initializer=__pool_initializer, initargs=__pool_initargs) def apply(self, tasks_size: int, function: Callable, args: Tuple = (), kwargs: Dict = {}) -> None: self.reset_result() __process_running_result = None self._Thread_List = [ self._Thread_Pool.apply(func=function, args=args, kwds=kwargs) for _ in range(tasks_size) ] for thread in self._Thread_List: try: __process_running_result = thread __exception = None __process_run_successful = True except Exception as e: __exception = e __process_run_successful = False # Save Running result state and Running result value as dict self._result_saving(successful=__process_run_successful, result=__process_running_result, exception=__exception) def async_apply(self, tasks_size: int, function: Callable, args: Tuple = (), kwargs: Dict = {}, callback: Callable = None, error_callback: Callable = None) -> None: self.reset_result() self._Thread_List = [ self._Thread_Pool.apply_async(func=function, args=args, kwds=kwargs, callback=callback, error_callback=error_callback) for _ in range(tasks_size) ] for process in self._Thread_List: _process_running_result = None _process_run_successful = None _exception = None try: _process_running_result = process.get() _process_run_successful = process.successful() except Exception as e: _exception = e _process_run_successful = False # Save Running result state and Running result value as dict self._result_saving(successful=_process_run_successful, result=_process_running_result, exception=_exception) def apply_with_iter(self, functions_iter: List[Callable], args_iter: List[Tuple] = None, kwargs_iter: List[Dict] = None) -> None: self.reset_result() __process_running_result = None if args_iter is None: args_iter = [() for _ in functions_iter] if kwargs_iter is None: kwargs_iter = [{} for _ in functions_iter] self._Thread_List = [ self._Thread_Pool.apply(func=_func, args=_args, kwds=_kwargs) for _func, _args, _kwargs in zip(functions_iter, args_iter, kwargs_iter) ] for thread in self._Thread_List: try: __process_running_result = thread __exception = None __process_run_successful = True except Exception as e: __exception = e __process_run_successful = False # Save Running result state and Running result value as dict self._result_saving(successful=__process_run_successful, result=__process_running_result, exception=__exception) def async_apply_with_iter( self, functions_iter: List[Callable], args_iter: List[Tuple] = None, kwargs_iter: List[Dict] = None, callback_iter: List[Callable] = None, error_callback_iter: List[Callable] = None) -> None: self.reset_result() if args_iter is None: args_iter = [() for _ in functions_iter] if kwargs_iter is None: kwargs_iter = [{} for _ in functions_iter] if callback_iter is None: callback_iter = [None for _ in functions_iter] if error_callback_iter is None: error_callback_iter = [None for _ in functions_iter] self._Thread_List = [ self._Thread_Pool.apply_async(func=_func, args=_args, kwds=_kwargs, callback=_callback) for _func, _args, _kwargs, _callback in zip( functions_iter, args_iter, kwargs_iter, callback_iter) ] for process in self._Thread_List: _process_running_result = None _process_run_successful = None _exception = None try: _process_running_result = process.get() _process_run_successful = process.successful() except Exception as e: _exception = e _process_run_successful = False # Save Running result state and Running result value as dict self._result_saving(successful=_process_run_successful, result=_process_running_result, exception=_exception) def map(self, function: Callable, args_iter: IterableType = (), chunksize: int = None) -> None: self.reset_result() __process_running_result = None try: __process_running_result = self._Thread_Pool.map( func=function, iterable=args_iter, chunksize=chunksize) __exception = None __process_run_successful = True except Exception as e: __exception = e __process_run_successful = False # Save Running result state and Running result value as dict for __result in (__process_running_result or []): self._result_saving(successful=__process_run_successful, result=__result, exception=None) def async_map(self, function: Callable, args_iter: IterableType = (), chunksize: int = None, callback: Callable = None, error_callback: Callable = None) -> None: self.reset_result() __map_result = self._Thread_Pool.map_async( func=function, iterable=args_iter, chunksize=chunksize, callback=callback, error_callback=error_callback) __process_running_result = __map_result.get() __process_run_successful = __map_result.successful() # Save Running result state and Running result value as dict for __result in (__process_running_result or []): self._result_saving(successful=__process_run_successful, result=__result, exception=None) def map_by_args(self, function: Callable, args_iter: IterableType[IterableType] = (), chunksize: int = None) -> None: self.reset_result() __process_running_result = None try: __process_running_result = self._Thread_Pool.starmap( func=function, iterable=args_iter, chunksize=chunksize) __exception = None __process_run_successful = True except Exception as e: __exception = e __process_run_successful = False # Save Running result state and Running result value as dict for __result in (__process_running_result or []): self._result_saving(successful=__process_run_successful, result=__result, exception=None) def async_map_by_args(self, function: Callable, args_iter: IterableType[IterableType] = (), chunksize: int = None, callback: Callable = None, error_callback: Callable = None) -> None: self.reset_result() __map_result = self._Thread_Pool.starmap_async( func=function, iterable=args_iter, chunksize=chunksize, callback=callback, error_callback=error_callback) __process_running_result = __map_result.get() __process_run_successful = __map_result.successful() # Save Running result state and Running result value as dict for __result in (__process_running_result or []): self._result_saving(successful=__process_run_successful, result=__result, exception=None) def imap(self, function: Callable, args_iter: IterableType = (), chunksize: int = 1) -> None: self.reset_result() __process_running_result = None try: imap_running_result = self._Thread_Pool.imap(func=function, iterable=args_iter, chunksize=chunksize) __process_running_result = [ result for result in imap_running_result ] __exception = None __process_run_successful = True except Exception as e: __exception = e __process_run_successful = False # Save Running result state and Running result value as dict for __result in (__process_running_result or []): self._result_saving(successful=__process_run_successful, result=__result, exception=None) def imap_unordered(self, function: Callable, args_iter: IterableType = (), chunksize: int = 1) -> None: self.reset_result() __process_running_result = None try: imap_running_result = self._Thread_Pool.imap_unordered( func=function, iterable=args_iter, chunksize=chunksize) __process_running_result = [ result for result in imap_running_result ] __exception = None __process_run_successful = True except Exception as e: __exception = e __process_run_successful = False # Save Running result state and Running result value as dict for __result in (__process_running_result or []): self._result_saving(successful=__process_run_successful, result=__result, exception=None) def _result_saving(self, successful: bool, result: List, exception: Exception) -> None: _thread_result = { "successful": successful, "result": result, "exception": exception } # Saving value into list self._Thread_Running_Result.append(_thread_result) def close(self) -> None: self._Thread_Pool.close() self._Thread_Pool.join() def terminal(self) -> None: self._Thread_Pool.terminate() def get_result(self) -> List[_ConcurrentResult]: return self.result() def _saving_process(self) -> List[_ThreadPoolResult]: _pool_results = [] for __result in self._Thread_Running_Result: _pool_result = _ThreadPoolResult() _pool_result.is_successful = __result["successful"] _pool_result.data = __result["result"] _pool_results.append(_pool_result) return _pool_results
class AgentService(metaclass=ABCMeta): """Base class for agent_service""" def __init__(self, config_path: str = 'settings.conf') -> None: # Config-based parameters try: self._agent_config = ConfigParser() self._agent_config.read(config_path) # Meta-data extraction self.name = self._agent_config.get(META_SECTION_KEY, 'name') self.redis_name = self._agent_config.get( META_SECTION_KEY, 'redis_name') or f'{self.name}_agent' self.description = self._agent_config.get(META_SECTION_KEY, 'description') self.status = self._agent_config.getint(META_SECTION_KEY, 'status') self.pool = ThreadPool(NUM_THREADS_PER_SERVICE) # Subscription data extraction currency = self._agent_config.get(SUBSCRIPTION_SECTION_KEY, CURRENCY_KEY) self.subscriptions = { SUBSCRIPTION_PLANS_KEY: [ { SUBSCRIPTION_INTERVAL_KEY: DAY_KEY, SUBSCRIPTION_INTERVAL_COUNT_KEY: self._agent_config.getint(SUBSCRIPTION_SECTION_KEY, DAY_INTERVAL_KEY), CAPACITY_KEY: DAY_CAPACITY_KEY, PLAN_PRICINGS_KEY: [ # TODO: Somehow read it through loop { CURRENCY_KEY: currency, SALARY_KEY: self._agent_config.getfloat( SUBSCRIPTION_SECTION_KEY, DAY_SALARY_KEY) } ] }, { SUBSCRIPTION_INTERVAL_KEY: WEEK_KEY, SUBSCRIPTION_INTERVAL_COUNT_KEY: self._agent_config.getint(SUBSCRIPTION_SECTION_KEY, WEEK_INTERVAL_KEY), CAPACITY_KEY: self._agent_config.getint(SUBSCRIPTION_SECTION_KEY, WEEK_CAPACITY_KEY), PLAN_PRICINGS_KEY: [ # TODO: Somehow read it through loop { CURRENCY_KEY: currency, SALARY_KEY: self._agent_config.getfloat( SUBSCRIPTION_SECTION_KEY, WEEK_SALARY_KEY) } ] }, { SUBSCRIPTION_INTERVAL_KEY: MONTH_KEY, SUBSCRIPTION_INTERVAL_COUNT_KEY: self._agent_config.getint(SUBSCRIPTION_SECTION_KEY, MONTH_INTERVAL_KEY), CAPACITY_KEY: self._agent_config.getint(SUBSCRIPTION_SECTION_KEY, MONTH_CAPACITY_KEY), PLAN_PRICINGS_KEY: [ # TODO: Somehow read it through loop { CURRENCY_KEY: currency, SALARY_KEY: self._agent_config.getfloat( SUBSCRIPTION_SECTION_KEY, MONTH_SALARY_KEY) } ] }, ] } # Service data extraction self.port = str(self._agent_config.get(SERVICE_SECTION_KEY, 'port')) self.elements_checker_period = self._agent_config.getfloat( SERVICE_SECTION_KEY, 'elements_checker_period') self.updating_time_interval = self._agent_config.getfloat( SERVICE_SECTION_KEY, 'updating_period') self.new_checker_period = self._agent_config.getfloat( SERVICE_SECTION_KEY, 'new_checker_period') # Content data (messages) extraction self.hello_message = self._agent_config.get( CONTENT_SECTION_KEY, 'hello_message').format(name=self.name) self.thread_names = { 'service': f'{self.name}_service_thread', 'websocket': f'{self.name}_websocket_thread', 'updater': f'{self.name}_updater_thread' } self.platform_access = None self.agent_db_key = os.environ[f'{self.name.upper()}_AGENT_DB_KEY'] self.db = Firestore(self.agent_db_key) self.task_running_listeners = {} self.task_running_updaters = {} self.running = True self.updater_thread_lock = threading.Lock() self.listener_thread_lock = threading.Lock() atexit.register(self.stop_running_tasks_on_exit) for task_file_name in self.get_agent_tasks_names(): task_name = task_file_name.replace('.json', '') threading.Thread(target=lambda: self._start_periodical_updates( task_name, DEFAULT_PERIOD_TIME_FOR_UPDATES_CHECKER), daemon=True).start() except NoSectionError as no_section_err: error(ENGINE_NAME, f'Missing config section: {no_section_err}') except NoOptionError as no_option_err: error(ENGINE_NAME, f'Missing config option: {no_option_err}') except KeyError as key_error: error(ENGINE_NAME, f'Missing parameter in config. {key_error}') except Exception as excpt: error(ENGINE_NAME, f'Unexpectedly couldn\'t read config. {excpt}') else: debug(self.name, f'New agent-service [{self.name}] is registered') try: platform_access = self.__get_platform_access() self.agents_user_id = platform_access.get_user_id() except Exception as excpt: # TODO: Catch all the exceptions error(ENGINE_NAME, excpt) @abstractmethod def identify_and_pass_task(self, element: Element, agent_task: Dict, update: bool = False, doc_id: str = None) -> Optional[Element]: """ Identify a task and pass it to appropriate method :param element: own_adapter.element.Element on which Company-agent should run :param agent_task: an agent task to get details from :param update: an element meant to be updated :param doc_id: optional id of a document in Firestore to send messages from Agent :return: Target element or None if task wasn't complete correctly """ raise NotImplementedError def _run_on_element(self, element: Element, agent_task: Dict = None, update: bool = False, doc_id: str = None) -> Optional[Element]: """ Running on a target element... :param element: own_adapter.element.Element on which the agent should run... :param agent_task: an agent_task with answers for an agent to process :param update: if an element is being updated :param doc_id: optional id of a document in Firestore to send messages from Agent :return: Target element or None if board or board.element or task are not defined """ if not element: logger.exception(self.name, 'Element was not provided') return None board = element.get_board() if not board: logger.exception(self.name, f'Board is undefined.') return None if agent_task: return self.identify_and_pass_task(element, agent_task, update) return None def communication_handling(self, doc_ref: firestore.firestore.DocumentReference): """ Periodically check messages in Firestore and send them to OWN :param doc_ref: a reference to document containing messages :return: Nothing """ doc_id = None try: doc_id = self.db.get_doc_id(doc_ref) self.task_running_listeners[doc_id] = doc_ref platform_access = self.__get_platform_access() content = self.db.get_doc_content(doc_ref) if not content: return board = Board(platform_access, identifier=content[BOARD_IDENTIFIER_KEY]) self.retrieve_and_upload_data(doc_ref, board) finally: if doc_id: self.db.finish_monitoring_task(doc_ref, listener=True) self.task_running_listeners.pop(doc_id) def retrieve_and_upload_data( self, doc_ref: firestore.firestore.DocumentReference, board: Board): """ Implements communication functionality between agents and platform for constant monitoring tasks :param doc_ref: a reference to a document containing info about the task :param board: constant monitoring task's board :return: Nothing """ self.retrieve_and_upload_messages_to_board(doc_ref, board) def retrieve_and_upload_messages_to_board( self, doc_ref: firestore.firestore.DocumentReference, board: Board): """ Method for upload messages on a board :param doc_ref: a reference to a document containing info about the task :param board: constant monitoring task's board :return: Nothing """ content = self.db.get_doc_content(doc_ref) if not content: return last_message_read = content[LAST_MESSAGE_READ_KEY] messages = content[MESSAGES_KEY] last_index = max(last_message_read, messages[-1][INDEX_KEY] if messages else 0) new_messages = list( filter(lambda message: message[INDEX_KEY] > last_index, messages)) self.db.update_document(doc_ref, { LAST_MESSAGE_READ_KEY: last_index, MESSAGES_KEY: new_messages, }) for message in messages: if message[INDEX_KEY] <= last_index: board.put_message(message[MESSAGE_KEY]) def stop_running_tasks_on_exit(self): """ Update values in DB for monitoring tasks before exiting :return: Nothing """ self.running = False with self.listener_thread_lock: for doc in self.task_running_listeners.copy().values(): self.db.finish_monitoring_task(doc_ref=doc, listener=True) with self.updater_thread_lock: for doc in self.task_running_updaters.copy().values(): self.db.finish_monitoring_task(doc_ref=doc, update=True) def _run_on_element_default( self, element: Element, query_id: str = '', agent_task: Optional[Dict] = None, update: bool = False, constant_monitoring: bool = False) -> Optional[Element]: """ Run on element and save task to db :param query_id: an id of agent task query :param update: if an element is being updated :param element: own_adapter.element.Element on which the agent should run :param agent_task: an agent_task with answers for an agent to process :param constant_monitoring: whether this task should run in constant monitoring mode :return: Target element """ task_name = self.identify_task(agent_task) return self._run_on_element_and_save_task( element, query_id=query_id, task_name=task_name, agent_task=agent_task, update=update, constant_monitoring=constant_monitoring) def _run_on_element_and_save_task(self, element: Element, task_name: str, query_id: str, agent_task: Dict = None, start_listener: bool = False, update: bool = False, constant_monitoring: bool = False) \ -> Optional[Element]: """ Create document to communicate with agent, run on a target element :param query_id: an id of agent task query :param task_name: a name of task which is being executed :param update: if an element is being updated :param element: own_adapter.element.Element on which the agent should run :param agent_task: an agent_task with answers for an agent to process :param constant_monitoring: whether this task should run in constant monitoring mode :param start_listener: if a communication listener should be started for this task :return: Target element """ self.db.delete_old_agent_tasks(element.get_id()) doc_ref = self.db.create_new_doc_for_task( element.get_board().get_id(), element.get_id(), query_id, task_name, update_period=self.updating_time_interval, constant_monitoring=constant_monitoring, agent_task=agent_task) doc_id = self.db.get_doc_id(doc_ref) try: self.task_running_updaters[doc_id] = doc_ref if start_listener or constant_monitoring: threading.Thread( target=lambda: self.communication_handling(doc_ref), daemon=True).start() res = self._run_on_element(element, agent_task, update, doc_id) if not res: self.db.delete_doc(doc_ref) return res except Exception as e: logger.exception( self.name, f'Couldn\'t run updates on element for task: {task_name}. Error: {e}' ) finally: self.db.finish_monitoring_task(doc_ref, update=True) self.task_running_updaters.pop(doc_id) def _run_update_for_task(self, doc_ref: firestore.firestore.DocumentReference, task_name: str) \ -> Optional[Element]: """ Run a periodical update for agent task stored in Firestore :param doc_ref: a reference to Firestore document with info about the task :param task_name: a name of the task :return: Nothing """ doc_id = None try: task_content = self.db.get_doc_content(doc_ref) platform_access = self.__get_platform_access() board = Board(platform_access, identifier=task_content[BOARD_IDENTIFIER_KEY]) element = Element(platform_access, identifier=task_content[ELEMENT_ID_KEY], board=board) agent_task = task_content[AGENT_TASK_KEY] doc_id = self.db.get_doc_id(doc_ref) self.task_running_updaters[doc_id] = doc_ref except Exception as e: logger.exception( self.name, f'Could not perform periodical update for task {task_name}. Error: {e}' ) return None else: self._run_on_element(element=element, agent_task=agent_task, update=True, doc_id=doc_id) finally: self.db.finish_monitoring_task(doc_ref, update=True) if doc_id: self.task_running_updaters.pop(doc_id) # Web-sockets def on_websocket_message(self, ws: websocket.WebSocketApp, message: str) -> None: """Processes websocket messages""" message_dict = json.loads(message) content_type = message_dict['contentType'] message_type = content_type.replace('application/vnd.uberblik.', '') debug(self.name, message) if message_type == 'liveUpdateAgentTaskElementAnswersSaved+json': # Get the data from the LiveUpdate's message agent_data_id = int(message_dict['agentDataId']) agent_task_id = int(message_dict['agentTaskId']) element_id = int(message_dict['elementId']) board_id = int(message_dict['boardId']) query_id = str(message_dict['agentQueryId']) agent = self.get_agent() agent_task = get_agent_task_answers_by_id( agent.get_platform_access(), agent_data_id=agent_data_id, agent_task_id=agent_task_id, board_id=board_id, element_id=element_id) board = Board.get_board_by_id(board_id, agent.get_platform_access(), need_name=False) element = Element.get_element_by_id(element_id, agent.get_platform_access(), board) if element: # Run AgentTask on the element updated_element = self._run_on_element_default( element, query_id, agent_task) if updated_element: element = updated_element element.set_last_processing_time(datetime.datetime.now()) agent.cache_element_to_redis(element) elif message_type in [ 'liveUpdateElementPermanentlyDeleted+json', 'liveUpdateElementDeleted+json' ]: element_id = f'/{"/".join(message_dict["path"].split("/")[-4:])}' self.db.delete_old_agent_tasks(element_id) elif message_type == 'liveUpdateAgentTaskElementDeleted+json': # Get the data from the LiveUpdate's message element_id = int(message_dict['elementId']) board_id = int(message_dict['boardId']) full_element_id = f'/boards/{board_id}/elements/{element_id}' self.db.delete_old_agent_tasks(full_element_id) elif message_type == 'liveUpdateBoardDeleted+json': board_id = int(message_dict['path'].split('/')[-1]) self.db.delete_old_agent_tasks('', board_id) def identify_task(self, agent_task: Optional[Dict] = None, agent_task_id: Optional[int] = None) -> str: """ Identify a task :param agent_task_id: an id of the agent task :param agent_task: an agent task to get details from :return: a name of the task """ if agent_task_id: agent_task_config_id = agent_task_id else: if not agent_task: return '' agent_task_answers = agent_task['agentTaskElement']['agentTask'] agent_task_config_id = agent_task_answers['id'] # Get all the agent-tasks possible_files = self.get_agent_tasks_names() # Find the AgentTask to run for filename in possible_files: if self.get_agent_task_config_from_file(filename)['agentTask'].get( 'id', None) == agent_task_config_id: return filename.replace('.json', '') return '' def on_websocket_error(self, ws, err): """Logs websocket errors""" error(self.name, err) def on_websocket_open(self, ws): """Logs websocket openings""" info(self.name, f'{self.redis_name}\'s websocket is open') def on_websocket_close(self, ws): """Logs websocket closings""" info(self.name, f'{self.redis_name}\'s Websocket is closed') def open_websocket(self): """Opens a websocket to receive messages from the boards about events""" agent = self.get_agent() query = {} # getting the service url without protocol name platform_url_no_protocol = agent.get_platform_access( ).get_platform_url().split('://')[1] query['token'] = agent.get_platform_access().get_access_token() url = f'{PROTOCOL}://{platform_url_no_protocol}/opensocket?{urllib.parse.urlencode(query)}' ws = websocket.WebSocketApp( url, on_message=lambda *args: self.pool.starmap_async( self.on_websocket_message, [args]), on_error=self.on_websocket_error, on_open=self.on_websocket_open, on_close=self.on_websocket_close) ws.run_forever() def run(self) -> None: """ Runs the agent by starting websocket and updater threads :return: Nothing """ if self.status == STATUS['INACTIVE']: info(self.name, f'The {self.name}-agent is inactive.') return websocket_thread = None # TODO: Add multi-threaded lock/stop here while True: # opening a websocket for catching server messages if websocket_thread is None or not websocket_thread.is_alive(): try: websocket_thread = threading.Thread( target=self.open_websocket, name=self.thread_names['websocket'], args={}, daemon=True) websocket_thread.start() debug(self.name, f'{self.name} started websocket') except Exception as excpt: exception( self.name, f'Could not open a websocket. Exception message: {str(excpt)}' ) # wait until next check time.sleep(10) # Util def get_agent(self) -> [Agent, None]: """ Returns the current agent :return: Object: Instance of own_platform.agent.Agent, or None with incorrect authentication data """ platform_access = self.__get_platform_access() agent = Agent(platform_access, self.redis_name) return agent def __get_platform_access(self) -> [PlatformAccess, None]: """Returns PlatformAccess with valid agent's credentials (environ), None otherwise""" if not self.platform_access: try: login = os.environ[f'OWN_{self.name.upper()}_AGENT_LOGIN'] password = os.environ[ f'OWN_{self.name.upper()}_AGENT_PASSWORD'] self.platform_access = PlatformAccess(login, password) except KeyError as key_error: exception( self.name, f'Failed get credentials for {self.name}-agent. Error message: {str(key_error)}' ) except Exception as err: error( self.name, f'Some error occurred while establishing connection to the platform: {err}' ) return self.platform_access def get_service_id(self) -> int: """Returns ID of the agent-service""" response = get_agent_data_by_user_id(self.__get_platform_access(), self.agents_user_id) resp_content = json.loads(response.content.decode()) result = int( resp_content['agentData']['_links'][0]['href'].split('/')[-1]) return result def get_agent_task_config_from_file(self, file_name: str) -> Dict: """ Return an agent task config stored in a file :param file_name: a name of the file relative to agent tasks folder :return: the agent task config """ file_location = os.path.join(AGENTS_SERVICES_PATH, f'{self.name}_service', 'agent_tasks', file_name) try: with open(file_location, 'r') as file: agent_task_config = json.load(file) except Exception as e: error( self.name, f'Agent task config in {file_name} not found. Exception {e}.') raise return agent_task_config def update_element_caption(self, element: Element, update_with_data: str) -> None: """ Sets the new caption for the element "<agent_name>: <new_data>[, <old_query>]" :param element: Element to update the caption on :param update_with_data: the data (sentence, or a keyword) to update the element with. :return: Nothing """ # Save the old caption old_caption = element.get_name() # Form the new caption new_caption = f'{self.name}: {update_with_data}' # Do nothing if they are the same if old_caption == new_caption: return # Check if the old caption matches the pattern pattern = re.compile(f'{self.name}: [^\n\r]+') if pattern.match(old_caption): # If it is, extract the query itself, leaving the sender old_caption_queries = old_caption.replace(f'{self.name}: ', '') # Check if last query wasn't the same (i.e., it's not a periodical update last_old_query = old_caption_queries.split(', ')[0] if last_old_query != update_with_data: new_caption += f', {old_caption_queries}' element.set_name(new_caption) def get_agent_tasks_names(self) -> List[str]: """Returns AgentTasks' names in the agent-service agent_task directory""" # Form the agent_tasks absolute path this_service_tasks_path = os.path.join(AGENTS_SERVICES_PATH, f'{self.name}_service', 'agent_tasks') if not os.path.exists(this_service_tasks_path): return [] # Get all the filenames that are .json possible_files = [ file for file in os.listdir(this_service_tasks_path) if file.endswith('.json') ] # TODO: Check if they are really AgentTasks return possible_files def send_request_to_agent_handler(self, method: str, url: str, params: Dict = None, data: Dict = None) \ -> Optional[requests.Response]: """ Send a task to a free agent handler :param method: an http method to use :param url: An URL of endpoint Example: for http://localhost:5000/handler, 'url' will be 'handler' Example: for http://localhost:5000/, 'url' will be an empty string ('') :param params: query params to transfer :param data: data to transfer :return: response """ MAX_NUMBER_OF_ATTEMPTS = 5 number_of_attempts = 0 black_list_of_agent_handlers = set() response = None SLEEP_TIME = 2 while number_of_attempts <= MAX_NUMBER_OF_ATTEMPTS: handler = self.db.get_agent_handler_with_least_amount_of_tasks( list(black_list_of_agent_handlers)) if not handler: break handler_id, handler_info = handler handler_ip = handler_info.get(AGENT_HANDLER_ADDRESS_KEY, None) if not handler_ip: continue black_list_of_agent_handlers.add(handler_id) num_of_tasks_in_handler = handler_info.get( AGENT_HANDLER_NUM_TASKS_KEY, 0) if not params: params = {} params[NUMBER_OF_TASKS_KEY] = num_of_tasks_in_handler try: response = requests.request(method=method, url=f'{handler_ip}/{url}', params=params, data=data) if response.status_code == HTTPStatus.CONFLICT: black_list_of_agent_handlers.remove(handler_id) response.raise_for_status() return response except requests.ConnectionError as e: self.db.increase_handler_number_of_fails(handler_id) logger.exception( self.name, f'Agent handler is not reachable. Error {e}.', response) except requests.RequestException as e: logger.exception( self.name, f'Agent handler responded with an error {e}.', response) number_of_attempts += 1 time.sleep(SLEEP_TIME * (number_of_attempts**2)) logger.exception( self.name, f'Amount of requests to agents from handler exceeded limit.') return response def _start_listeners(self, task_name: str, period_time: float): """ Check constant tasks in Firestore and start listeners for them :param task_name: a name of task which is being executed :param period_time: amount of seconds to wait between checks :return: Nothing """ SLEEP_TIME = 1 MAX_NUMBER_OF_TASKS_FOR_ONE_PERIOD = 10 docs = {} while True: for i in range(MAX_NUMBER_OF_TASKS_FOR_ONE_PERIOD): with self.listener_thread_lock: if not self.running: return doc = self.db.get_agent_task_from_firestore( task_name, runners_dict=self.task_running_listeners, listener=True, constant_monitoring=True) if not doc: break docs[self.db.get_doc_id(doc)] = doc time.sleep(SLEEP_TIME) for doc_id, doc in docs.copy().items(): docs.pop(doc_id) threading.Thread( target=lambda: self.communication_handling(doc), daemon=True).start() time.sleep(period_time) def _start_periodical_updates(self, task_name: str, period_time: float): """ Check if there are tasks in Firestore that should be updated :param task_name: a name of a task to check :param period_time: amount of seconds to wait between checks :return: Nothing """ while True: with self.updater_thread_lock: if not self.running: return doc = self.db.get_agent_task_from_firestore( task_name, runners_dict=self.task_running_updaters, constant_monitoring=False, update=True) if doc: threading.Thread( target=lambda: self._run_update_for_task(doc, task_name), daemon=True).start() time.sleep(period_time) def get_file_from_agent_and_send_to_element(self, file_url: str, filename: str, element: Element) -> bool: """ Downloads a file from agent and uploads it to an element :param file_url: a url of the file :param filename: a name of the file to display in the element :param element: the element where file should be placed :return: True if files was uploaded, False otherwise """ if not (file_url and filename and element): return False try: file_bytes = requests.get(file_url).content except Exception as e: logger.error( self.name, f'Agent can\'t download file by url {file_url}. Error {e}') return False returned_code = element.put_file(filename, file_bytes) if returned_code not in [ HTTPStatus.CREATED, AdapterStatus.CONNECTION_ABORTED ]: logger.error( self.name, f'Agent can\'t upload file to element {element.get_id()}.') return False else: return True
class PiperBot(threading.Thread): def __init__(self): super(PiperBot, self).__init__(daemon=True) self.servers = MutableNameSpace({}) self.admins = defaultdict(list) self.command_char = "#" self.in_queue = PriorityQueue() self.apikeys = {} self.commands = {} self.aliases = {} self.plugins = {} self.pre_dispatch_exts = [] self.pre_command_exts = [] self.post_command_exts = [] self.pre_event_exts = [] self.post_event_exts = [] self.pre_trigger_exts = [] self.post_trigger_exts = [] self.pre_regex_exts = [] self.post_regex_exts = [] self.worker_pool = ThreadPool(processes=4) self.scheduler = Scheduler() self.scheduler.start() self.message_buffer = defaultdict( lambda: defaultdict(lambda: deque(maxlen=50))) self.buffer_pattern = re.compile(r"(?:(\w+)|(\s)|^)(?:\^(\d+)|(\^+))") self.buffer_pattern_escape = re.compile(r"\\\^") self.running = False def buffer_replace(self, text, servername, channel, offset=0): buffer = list(self.message_buffer[servername][channel])[offset:] def buffer_sub(match_object): if match_object.group(1): messgages = [ x for x in buffer if x.nick == match_object.group(1) ] if not messgages: raise Exception("user not in memory") else: messgages = buffer if match_object.group(3): count = int(match_object.group(3)) else: count = len(match_object.group(4)) if count <= len(messgages): return (" " if match_object.group(2) else "") + messgages[count - 1].text else: raise Exception("line not in memory") text = self.buffer_pattern.sub(buffer_sub, text) text = self.buffer_pattern_escape.sub("^", text) return text def connect_to(self, servername, network, port, nick, channels=None, admins=None, password=None, username=None, ircname=None, ssl=False): self.servers[servername] = ServerConnection(self.in_queue, servername, network, port, nick, password, username, ircname, channels, ssl) self.servers[servername].connect() if admins: self.admins[servername] += admins def run(self): self.running = True try: while self.running: try: message = self.in_queue.get(timeout=10) print("<< " + str(message)) self.handle_message(message) except Empty: pass finally: self.shutdown() def shutdown(self): self.running = False for plugin in self.plugins.values(): for func in plugin._onUnloads: func() for server in self.servers.values(): server.disconnect(message="shutting down") def load_plugin_from_module(self, plugin): if "." in plugin: module = "".join(plugin.split(".")[:-1]) plugin_name = plugin.split(".")[-1] temp = importlib.machinery.SourceFileLoader( module, os.path.dirname(os.path.abspath(__file__)) + "/plugins/" + module + ".py").load_module() found = False for name, Class in inspect.getmembers( temp, lambda x: inspect.isclass(x) and hasattr(x, "_plugin")): if name == plugin_name: self.load_plugin(Class) found = True if not found: raise Exception("no such plugin to load") else: temp = importlib.machinery.SourceFileLoader( plugin, os.path.dirname(os.path.abspath(__file__)) + "/plugins/" + plugin + ".py").load_module() found = False for name, Class in inspect.getmembers( temp, lambda x: inspect.isclass(x) and hasattr(x, "_plugin")): self.load_plugin(Class) found = True if not found: raise Exception("no such plugin to load") def unload_module(self, module_name): found = False plugins = [] for plugin_name, plugin in self.plugins.items(): if plugin_name.startswith(module_name + ".") or plugin_name == module_name: plugins.append(plugin_name) found = True for plugin in plugins: self.unload_plugin(plugin) if not found: raise Exception("no such plugin to unload") def load_plugin(self, plugin): if plugin.__module__ + "." + plugin.__name__ in self.plugins: raise Exception("already loaded!") plugin_instance = plugin() plugin_instance._plugin__init__(self) for func in plugin_instance._onLoads: func() self.plugins[plugin_instance.__module__ + "." + plugin_instance.__class__.__name__] = plugin_instance for (func, args) in plugin_instance._commands: if args["command"] not in self.commands: self.commands[args["command"]] = (func, args) else: print("command overlap! : " + args["command"]) for task in plugin_instance._scheduleds: self.scheduler.add_task(task) for (priority, func) in plugin_instance._command_extensions: if priority < 1: self.pre_command_exts.append( (priority, plugin_instance.__module__ + "." + plugin_instance.__class__.__name__, func)) else: self.post_command_exts.append( (priority, plugin_instance.__module__ + "." + plugin_instance.__class__.__name__, func)) for (priority, func) in plugin_instance._event_extensions: if priority < 1: self.pre_event_exts.append( (priority, plugin_instance.__module__ + "." + plugin_instance.__class__.__name__, func)) else: self.post_event_exts.append( (priority, plugin_instance.__module__ + "." + plugin_instance.__class__.__name__, func)) for (priority, func) in plugin_instance._trigger_extensions: if priority < 1: self.pre_trigger_exts.append( (priority, plugin_instance.__module__ + "." + plugin_instance.__class__.__name__, func)) else: self.post_trigger_exts.append( (priority, plugin_instance.__module__ + "." + plugin_instance.__class__.__name__, func)) for (priority, func) in plugin_instance._regex_extensions: if priority < 1: self.pre_regex_exts.append( (priority, plugin_instance.__module__ + "." + plugin_instance.__class__.__name__, func)) else: self.post_regex_exts.append( (priority, plugin_instance.__module__ + "." + plugin_instance.__class__.__name__, func)) self.post_command_exts.sort(key=itemgetter(0, 1)) self.pre_command_exts.sort(key=itemgetter(0, 1)) self.post_event_exts.sort(key=itemgetter(0, 1)) self.pre_event_exts.sort(key=itemgetter(0, 1)) self.post_regex_exts.sort(key=itemgetter(0, 1)) self.pre_regex_exts.sort(key=itemgetter(0, 1)) self.post_trigger_exts.sort(key=itemgetter(0, 1)) self.pre_trigger_exts.sort(key=itemgetter(0, 1)) if plugin._plugin_thread: plugin_instance.start() def unload_plugin(self, plugin_name): for func, args in self.plugins[plugin_name]._commands: if args["command"] in self.commands.keys(): del self.commands[args["command"]] for task in self.plugins[plugin_name]._scheduleds: self.scheduler.cancel_task(task) for lump in self.post_command_exts: if lump[1] == plugin_name: self.post_command_exts.remove(lump) for lump in self.pre_command_exts: if lump[1] == plugin_name: self.pre_command_exts.remove(lump) for lump in self.post_event_exts: if lump[1] == plugin_name: self.post_event_exts.remove(lump) for lump in self.pre_event_exts: if lump[1] == plugin_name: self.pre_event_exts.remove(lump) for lump in self.post_regex_exts: if lump[1] == plugin_name: self.post_regex_exts.remove(lump) for lump in self.pre_regex_exts: if lump[1] == plugin_name: self.pre_regex_exts.remove(lump) for lump in self.post_trigger_exts: if lump[1] == plugin_name: self.post_trigger_exts.remove(lump) for lump in self.pre_trigger_exts: if lump[1] == plugin_name: self.pre_trigger_exts.remove(lump) for func in self.plugins[plugin_name]._onUnloads: func() del self.plugins[plugin_name] def handle_message(self, message): if message.command == "PING": response = message.copy() response.command = "PONG" self.send(response) elif message.command == "PRIVMSG": self.message_buffer[message.server][message.params].appendleft( message) if message.text.startswith(self.command_char) and message.text[1:]: first = message.text[1:].split()[0] if first == "alias": self.handle_alias_assign(message) elif first in self.commands or first in self.aliases: self.worker_pool.apply_async(self.handle_command, args=(message, )) triggered = [] for plugin in self.plugins.values(): if message.command == "PRIVMSG": for regex, rfunc in plugin._regexes: matches = regex.findall(message.text) for groups in matches: temp = message.copy() temp.groups = groups triggered.append((rfunc, temp, self.pre_regex_exts, self.post_trigger_exts)) for trigger, tfunc in plugin._triggers: if trigger(message, bot): triggered.append((tfunc, message, self.pre_trigger_exts, self.post_trigger_exts)) for event, efunc in plugin._handlers: if message.command.lower() == event.lower(): triggered.append((efunc, message, self.pre_event_exts, self.post_event_exts)) self.worker_pool.starmap_async(self.call_triggered, triggered, error_callback=print) def call_triggered(self, func, message, pre=[], post=[]): try: pipe = self.resulter() for _, _, func_, in post: pipe = func_(message, pipe) pipe = func(pipe) next(pipe) for _, _, func_ in pre: pipe = func_(message, pipe) pipe.send(message) except StopIteration as e: raise e except Exception as e: print("Error calling trigger: %s : %s: %s" % (func.__name__, type(e), e)) exc_type, exc_value, exc_traceback = sys.exc_info() print("*** print_tb:") traceback.print_tb(exc_traceback, file=sys.stdout) def handle_alias_assign(self, message): try: name, *alias = message.text[len(self.command_char) + 5:].split("=") name = name.strip() alias = "=".join(alias).strip() if name in self.commands: raise Exception("cannot overwrite existing command") commands = map( lambda x: [(command, " ".join(args)) for command, *args in [x.split(" ")]][0], map(lambda x: x.strip(), alias.split(" || "))) funcs = [] args = [] for func, arg in commands: if func in self.commands: funcs.append(self.commands[func][1]["command"]) args.append(arg) elif func in self.aliases: if func == name: raise Exception("recursion detected") funcs_, args_ = self.aliases[func] funcs.extend(funcs_) args.extend(args_) else: raise Exception("unrecognised command: " + func) self.aliases[name] = list(zip(funcs, args)) self.send(message.reply("The alias %s has been saved" % name)) except Exception as e: self.send( message.reply( type(e).__name__ + (": " + str(e)) if str(e) else "")) exc_type, exc_value, exc_traceback = sys.exc_info() print("*** print_tb:") traceback.print_tb(exc_traceback, file=sys.stdout) def handle_command(self, message): try: def prepper(results): while 1: x = yield results.append(x) results = [] prepipe = prepper(results) next(prepipe) for _, _, f in self.pre_command_exts: prepipe = f(message, prepipe) prepipe.send(message) for result in results: while "$(" in result.text: result = self.handle_inners(result) funcs, args = self.funcs_n_args(result) pipe = self.resulter() for func, args in chain( [(f, message) for _, _, f in self.post_command_exts], list(zip(funcs[::-1], args[::-1]))): pipe = func(args, pipe) pipe.send(None) pipe.close() except Exception as e: self.send( message.reply( type(e).__name__ + (": " + str(e)) if str(e) else "")) exc_type, exc_value, exc_traceback = sys.exc_info() print("*** print_tb:") traceback.print_tb(exc_traceback, file=sys.stdout) def funcs_n_args(self, message): commands = map( lambda x: [(command, " ".join(args)) for command, *args in [x.split(" ")]][0], map(lambda x: x.strip(), message.text[1:].split(" || "))) funcs = [] args = [] for i, (func, arg) in enumerate(commands): if func in self.commands: if self.commands[func][1].get( "adminonly", False ) and message.nick not in self.admins[message.server]: raise Exception("admin only command: " + func) else: funcs.append(self.commands[func][0]) if self.commands[func][1].get("bufferreplace", True): arg = self.buffer_replace(arg, message.server, message.params, offset=1) temp = message.reply(arg) if self.commands[func][1].get("argparse", False): text, args_ = self.parse_args(temp, self.commands[func][0], func) temp = message.reply(text) temp.args = args_ args.append(temp) elif func in self.aliases: funcs.append(self.argpass) args.append(message.reply(arg)) for func_, arg_ in self.aliases[func]: if self.commands[func_][1].get("adminonly", False) \ and message.nick not in self.admins[message.server]: raise Exception("admin only command: " + func) else: funcs.append(self.commands[func_][0]) if self.commands[func_][1].get("bufferreplace", True): arg__ = self.buffer_replace(arg_, message.server, message.params, offset=1) else: arg__ = arg_ temp = message.reply(arg__) if self.commands[func_][1].get("argparse", True): text, args_ = self.parse_args( temp, self.commands[func_][0], func_) temp = message.reply(text) temp.args = args_ args.append(temp) else: raise Exception("unrecognised command: " + func) return funcs, args def parse_args(self, message, func, funcname): if hasattr(func, "_args"): args = dict(func._args) else: args = {} if message.text.startswith('"') and message.text.endswith('"'): return message.data[1:-1], args words = message.data.split() msg = [] for word in words: if word.startswith("--"): word = word[2:] if "=" in word: arg, _, value = word.partition("=") if arg not in args: raise Exception("invalid argument: %s for command %s" % (arg, funcname)) values = { "True": True, "False": False, } if value in values: value = values[value] args[arg] = value else: if word not in args: raise Exception("invalid argument: %s for command %s" % (word, funcname)) args[word] = True elif word.startswith("-"): word = word[1:] for flag in word: if flag not in args: raise Exception("invalid flag: %s for command %s" % (flag, funcname)) else: msg.append(word) return " ".join(msg), args def handle_inners(self, message): first = message.text.find("$(") left = message.text[:first] rest = message.text[first + 1:] openbr = 0 prevchar = "" for i, char in enumerate(rest): if prevchar + char == "$(": openbr += 1 if char == "$" and prevchar == ")": if openbr != 0: openbr -= 1 else: final = i break prevchar = char else: raise Exception("no closing bracket") middle = rest[:final - 1] right = rest[final + 1:] while "$(" in middle: middle = self.handle_inners(message.reply(middle)).text funcs, args = self.funcs_n_args(message.reply(middle)) text = [] cat = self.concater(text) pipe = cat for func, args in zip(funcs[::-1], args[::-1]): pipe = func(args, pipe) pipe.send(None) pipe.close() return message.reply(text=left + " ".join(text) + right) def send(self, message): if message.server in self.servers: print(">> " + str(message)) line = message.to_line() if not line.endswith("\n"): line += "\n" self.servers[message.server].socket.send(line.encode()) if message.command == "PRIVMSG": temp = message.copy() temp.nick = self.servers[message.server].nick self.message_buffer[message.server][message.params].appendleft( temp) else: raise Exception("no such server: " + message.server) @_coroutine def argpass(self, arg, target): formats = len(list(string.Formatter().parse(arg.text))) try: while 1: x = yield if x is None: if not arg.data: target.send(None) else: if formats: target.send( arg.reply(data=arg.data.format(*([""] * formats)), args=arg.args)) else: target.send(arg) else: if formats: if x.data is not None: target.send( x.reply(data=arg.data.format(*([x.data] * formats)), args=arg.args)) else: target.send( x.reply(data=arg.data.format(*([""] * formats)), args=arg.args)) else: target.send(x.reply(data=x.data, args=arg.args)) except GeneratorExit: target.close() @_coroutine def resulter(self): try: while 1: x = yield self.send(x) except GeneratorExit: pass # pipe closed @_coroutine def concater(self, results): try: while 1: x = yield results.append(x.data) except GeneratorExit: pass # pipe closed @staticmethod def from_config(filename): config = json.load(json_config) print(json.dumps(config, sort_keys=True, indent=4)) print(list(config.keys())) bot = PiperBot() if "apikeys" in config: for api, key in config["apikeys"]: bot.apikeys[api] = key for plugin, config_ in config["plugins"].items(): bot.load_plugin_from_module(plugin) for server in config["servers"]: server_name = server["IRCNet"] network = server["IRCHost"] name = server["IRCName"] user = server["IRCUser"] port = server["IRCPort"] nick = server["IRCNick"] password = server["IRCPass"] if "IRCPass" in server else None autojoin = server["AutoJoin"] admins = server["Admins"] usessl = server["UseSSL"] bot.connect_to(server_name, network, port, nick, autojoin, admins, password, user, name, usessl) return bot