class SubtaskWorker(LoadtestWorkerServicer): """ SubtaskWorker must be pickleable before Start is called. """ def __init__(self): self._executor: Executor = None self.metrics_tracker: MetricsTracker = None @abstractmethod def run_worker(self, request: StartRequest): """ run_worker should run the worker forever. :param request: the request to start the worker :return: never """ pass def Start(self, request: StartRequest, context): self._executor = ThreadPoolExecutor(max_workers=1) self.metrics_tracker = MetricsTracker(request.include_ids) self._executor.submit(self.run_worker, request) return StartResponse() def Check(self, request, context): return self.metrics_tracker.check()
def crawl_single_kind(self, kind='basketball'): print("GET: KIND = ", kind) # 爬取页面,找到页面中各个鞋的类别的页面 url = 'http://www.shihuo.cn/' + kind + '/list?page_size=60&page=1' response = requests.get(url, headers=self.header) html = response.text # 首先找到总页面数 start = html.find('var totalPage = parseInt(') + len('var totalPage = parseInt(') end = html.find(')', start) page_nums = math.ceil(float(html[start:end])) # print(page_nums) # 循环找到全部下级链接 links = [] for page_num in range(page_nums): url = 'http://www.shihuo.cn/' + kind + '/list?page_size=60&page=' + str(page_num) response = requests.get(url, headers=self.header) html = response.text soup = BeautifulSoup(html, "lxml") # print(soup.prettify()) link_class = soup.select('#js_hover li .imgs-area .link ') for item in link_class: links.append('http:' + item['href']) thread_pool = ThreadPoolExecutor(8) for link in links: thread_pool.submit(self.crawl_all_color_for_shoes, link) thread_pool.shutdown(wait=True)
def preprocess(data_type, img_dir, geojson_dir, rgb_tg, mask_tg, num_workers=8): pool = ThreadPoolExecutor(max_workers=num_workers) re_img_index = re.compile("img\d+") re_pat = re.compile("(.*?)img\d+") # building_pat = re_pat.search(os.listdir(Path(geojson_dir) / "buildings")[0]).group(1) pat = re_pat.search(os.listdir(Path(geojson_dir))[0]).group(1) for f in os.listdir(img_dir): img_index = re_img_index.search(f).group(0) geojson = Path(geojson_dir) / (pat + img_index + ".geojson") def pool_wrapper(p1, p2, p3, p4): if data_type == "building": thread = GeoLabelUtil.BuildingRenderThread(p1, p2, p3, p4) elif data_type == "road": thread = GeoLabelUtil.RoadRenderThread(p1, p2, p3, p4) else: raise NotImplementedError("Not Implemented Data Type: " + data_type) thread.run() # thread.start() pool.submit(pool_wrapper, Path(img_dir) / f, Path(rgb_tg) / (img_index + ".png"), geojson, Path(mask_tg) / (img_index + ".png")) pool.shutdown(wait=True)
class UrlGet: global lis, lst lis, lst = [], [] def __init__(self): self.thread_pool = ThreadPoolExecutor(10) def magic_func(self): for i in range(200): self.thread_pool.submit(self.get_url, ) def get_url(self): thread_name = threading.current_thread().name r = requests.get('https://www.baidu.com') s = r.status_code if s == 200: lst.append(1) t = r.elapsed.total_seconds() lis.append(t) print(f"response time is : {t}s") print(f"response code is : {s}") print(f"this is thread : {thread_name}") def calculiates(self): m = max(lis) n = min(lis) a = numpy.mean(lis) p = lst.count(1) print(f"max response time is : {m}s") print(f"min response time is : {n}s") print(f"avg response time is : {a}s") print(f"code200 percentage is : {p / 200 * 100}%")
def novel_chapter_detail_save(self): sql_tables = "SELECT `table_name` from router where sourceid = %s" tables = default_dbhelper.query(sql_tables, (SpiderTools.sourceid)) executor = ThreadPoolExecutor(max_workers=len(tables)) for table_name in tables: executor.submit(novel_chapter_detail_save_by_tablename, (table_name[0]))
class RPCConnector(ConsensusConnectorModel): def __init__(self, consensus: ConsensusModel, *args, **kwargs): super().__init__(consensus) self.rpc_server = RPCServer(self) self.rpc_server.serve() self.executors = ThreadPoolExecutor(max_workers=10) def handle_training_request(self, data): pass def broadcast_proposal(self, data): # print("begin to broadcast") for peer in self.peers: client = RPCClient(peer) client.send_proposal(data) def handle_upload_request(self, data: str): # print("handling upload request") self.executors.submit(self.consensus.make_consensus, data=data, connector=self) def handle_consensus_data(self, data): # print('received proposal') self.consensus.handle_block(data)
class Stub: def __init__(self, adapter, port): self._port = port self._adapter = adapter self.server = None self._executor = None def _setup(self): self.server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.server.bind(('192.168.0.106', self._port)) self._executor = ThreadPoolExecutor(max_workers=2) def run(self): self._setup() self.server.listen() try: while True: connection, client_address = self.server.accept() self._executor.submit(self.process_request, connection) except KeyboardInterrupt: connection.close() self.server.stop(0) def process_request(self, connection): try: newthread = FSStub(connection, self._adapter) except Exception as e: print('ERROR!!! ', e) return
def testWithStores(self): # store_opts2 = {"store_type": StoreTypes.ROLLPAIR_QUEUE, "capacity": 100} store_opts2 = {"store_type": StoreTypes.ROLLPAIR_QUEUE} rp = self.ctx.load("ns1", "n1") rp_q = self.ctx.load("ns1", "n2", store_opts2) rp.put_all([("k1", "v1"), ("k2", "v2")]) def func_asyn(partitions): import time part1 = partitions[0] serder1 = create_serdes(part1._store_locator._serdes) with create_adapter(part1) as db1: for i in range(100): db1.put(serder1.serialize("a" + str(i))) time.sleep(0.1) def func_syn(partitions): part1, part2 = partitions serder1 = create_serdes(part1._store_locator._serdes) serder2 = create_serdes(part2._store_locator._serdes) with create_adapter(part1) as db1, create_adapter(part2) as db2: for i in range(100): db1.put(serder1.serialize("q" + str(i)), db2.get()) pool = ThreadPoolExecutor() pool.submit(rp_q.with_stores, func_asyn) rp.with_stores(func_syn, [rp_q]) print(list(rp.get_all())) pool.shutdown()
class MessageForwardDaemon(object): MAX_WORKERS = 25 def __init__(self): self.logger = retrieve_logger("message_forward") self.devices_service = DevicesService(logger=self.logger) self.redis_connection = redis_connection = redis.StrictRedis( REDIS_HOSTNAME, REDIS_PORT) self.redis_pub_sub = redis_connection.pubsub() self.redis_pub_sub.subscribe(MESSAGE_FORWARD_REDIS_PUBSUB) self.thread_pool_executor = ThreadPoolExecutor( max_workers=self.MAX_WORKERS) self.message_forward_helper = { "MQTT": MQTTMessageForward(), "COAP": CoAPMessageForward(), } def _forward_message(self, message): protocol = message["device_info"]["protocol"] if protocol not in self.message_forward_helper: raise Exception("Unknown protocol {}".format(protocol)) try: self.message_forward_helper[protocol].forward( message["device_info"], message["value"]) except Exception as err: self.logger.error( "Failed to send the message to the device. Reason: {}".format( err)) self._notify_forward_failure(message) return def _notify_forward_failure(self, message): # TODO: Send message to GASS that failed to send the message to the device self.logger.critical(message) def start(self): self.logger.debug("Waiting for command messages") while True: message = self.redis_pub_sub.get_message(timeout=120) self.logger.debug("Received message: {}".format(message)) invalid_message = not message or message["type"] != "message" if invalid_message: continue try: message_body = json.loads(message['data'].decode("utf-8")) self.thread_pool_executor.submit(self._forward_message, message_body) except Exception as err: self.logger.error(err) continue self.logger.debug("Waiting for command messages")
class HttpService(object): def __init__(self): self.__async_executor = ThreadPoolExecutor(max_workers=10) self.logger = logging.getLogger(__name__) self.__http = Http() def get(self, request): return self.make_request(request, 'GET') def post(self, request): return self.make_request(request, 'POST') def put(self, request): return self.make_request(request, 'PUT') def delete(self, request): return self.make_request(request, 'DELETE') def make_request(self, request, method): future = HttpFuture() self.__async_executor.submit(self.__do_request, request, method, future) return future def __do_request(self, request, method, future): try: uri = request.url + urllib.parse.urlencode(request.parameters) headers, content = self.__http.request(uri, method, request.data, request.headers) future.fulfill(headers, content) except Exception as ex: self.logger.exception("Http __do_request attempt failed with exception")
def judge_hash_is_apk(self, hash_contain, cookies): headers = { "Cookie": self.crawl_login(), "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng," "*/*;q=0.8,application/signed-exchange;v=b3;q=0.9", "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) " "Chrome/87.0.4280.88 Safari/537.36", } max_workers = Config_Reader().get_max_worker() for md5 in hash_contain: try: params = {"search": md5} resp = requests.post(url=self.VIRUS_SHARE_SEARCH, data=params, headers=headers) is_apk = re.search( "<td class='lc'>Extension</td><td colspan=2>apk</td>", resp.text) is_login = re.search("login", resp.text) if is_apk: sha256 = re.search( "<td class='lc'>SHA256</td><td colspan=2>[0-9a-zA-Z]*", resp.text) executor = ThreadPoolExecutor(max_workers=max_workers) executor.submit(self.download, sha256.group(0)[40:], md5, cookies) if not is_apk and is_login: logging.info("======需要重新登录=====") time.sleep(5) except Exception as e: logging.error(e) pass
async def Main(): try: player = Player("Account Email/Username", "Account Password") except YggdrasilError as e: # Authentication Error print("Incorrect Login", e) return player.SetServer("Server to connect to.") # We do this to ensure it is non blocking as Connect() is a # forever loop used to maintain a connection to a server executor = ThreadPoolExecutor() executor.submit(player.Connect) # Forever do things unless the user wants us to logout while True: message = input("What should I do/say?\n") # Disconnect the client from the server before finishing everything up if message.lower() in ["logout", "disconnected", "exit"]: player.Disconnect() print("Disconnected") return # Send the message to the server via the player player.SendChat(message)
def executor(request): """ Receives http request and starts execution of Test """ if request.method == 'POST': try: test_id = generate_test_id() os.makedirs(f"{STORAGE_PATH}/{test_id}") logger.info('Directory created at: ' + f"{STORAGE_PATH}/{test_id}") received_json_data = json.loads(request.body.decode()) try: test_ymls, initial_variable_dictionary = __extract_test_data(test_id, received_json_data['test']) except BadArgumentsError as e: return HttpResponse(str(e), status=status.HTTP_400_BAD_REQUEST) pool_executor = ThreadPoolExecutor(max_workers=1) try: pool_executor.submit(thread_executor, test_ymls, initial_variable_dictionary, test_id, received_json_data['email']) logger.info("task submitted to thread pool executor") except Exception as e: logger.error("Exception occurred", e) response_message = "Your test ID is: " + test_id + ". We'll send you an email with report shortly" return HttpResponse(response_message, status=status.HTTP_200_OK) except Exception as e: logger.error("Exception caught", exc_info=True) return HttpResponse(e, status=status.HTTP_400_BAD_REQUEST) else: return HttpResponse(status=status.HTTP_405_METHOD_NOT_ALLOWED)
def multiplyKaratsubaParallel(args) -> Polynomial: depth = args[0] A = args[1] B = args[2] if depth > 4: return PolynomialOperations.multiplySequencially(A, B) if A.n < 2 or B.n < 2: return PolynomialOperations.multiplySequencially(A, B) m = int(max(A.n, B.n) / 2) lowA = Polynomial(len(A.coefficients[:m]), A.coefficients[:m]) highA = Polynomial(len(A.coefficients[m:]), A.coefficients[m:]) lowB = Polynomial(len(B.coefficients[:m]), B.coefficients[:m]) highB = Polynomial(len(B.coefficients[m:]), B.coefficients[m:]) karaPool = ThreadPoolExecutor(mp.cpu_count()) futureResult1 = karaPool.submit(PolynomialOperations.multiplyKaratsubaParallel, ([depth+1, lowA, lowB])) futureResult2 = karaPool.submit(PolynomialOperations.multiplyKaratsubaParallel, ([depth+1, PolynomialOperations.add(lowA, highA), PolynomialOperations.add(lowB, highB)])) futureResult3 = karaPool.submit(PolynomialOperations.multiplyKaratsubaParallel, ([depth+1, highA, highB])) karaPool.shutdown(wait=True) result1 = futureResult1.result() result2 = futureResult2.result() result3 = futureResult3.result() r1 = PolynomialOperations.shift(result3, 2 * m) r2 = PolynomialOperations.shift( PolynomialOperations.subtract(PolynomialOperations.subtract(result2, result3), result1), m) return PolynomialOperations.add(PolynomialOperations.add(r1, r2), result1)
def get_data(): executor = ThreadPoolExecutor(max_workers=WORKER) # get the res dir ready mkdir_res() # get url list( as_completed( executor.submit(get_chengjiao_house_url, ), executor.submit(get_ershoufang_house_url, ), )) # get ershoufang info list( as_completed( executor.submit(get_ershoufang_house_info, hs) for hs, name in HOUSE_DISTRICT_DICT.items() if not (DATA_DIR / "house_info" / "ershoufang" / f"{hs}.json").is_file())) # get chengjiao info list( as_completed( executor.submit(get_chengjiao_house_info, hs) for hs, name in HOUSE_DISTRICT_DICT.items() if not (DATA_DIR / "house_info" / "chengjiao" / f"{hs}.json").is_file())) # save to csv to_csv()
def run_generators(task_generators: List[BaseSpiderTaskGenerator], item_pool_workers: int = 32, retries: int = 5): g_pool = ThreadPoolExecutor(max_workers=len(task_generators)) i_pool = ThreadPoolExecutor(max_workers=item_pool_workers) def retry_and_handle_exception(func: Callable): def wrapper(): for i in range(retries): try: func() break except Exception as ex: log.error(f"Error while executing task (retries={i}).", exc_info=ex) return wrapper def submit_all_items(stg: BaseSpiderTaskGenerator): def wrapper(): try: for sub_task in stg.generate(): i_pool.submit(retry_and_handle_exception(sub_task)) except Exception as ex: log.error("Error while generating tasks.", exc_info=ex) return wrapper for g in task_generators: g_pool.submit(submit_all_items(g)) log.info("All generators started.") g_pool.shutdown(wait=True) log.info("All generators terminated.") i_pool.shutdown(wait=True) log.info("All tasks finished.")
class ProcessingTimesCollector: def __init__(self): self._executor = ThreadPoolExecutor(max_workers=1) self._times_map = {} def _add_times(self, times): for k, v in times.items(): try: agg = self._times_map[k] except KeyError: agg = TimerStatsAggregator() self._times_map[k] = agg agg.add_time(v) def add_times(self, times): self._executor.submit(self._add_times, times) def _get_aggregates(self, prefix): return { identifier: stats.finalize() for identifier, stats in self._times_map.items() if identifier.startswith(prefix) } def get_aggregates(self, identifier=None) -> Dict[str, TimerStats]: future = self._executor.submit(self._get_aggregates, identifier or '') return future.result() def close(self): self._executor.shutdown(wait=True)
def compare(request, name): name_amazon = name name_flikart = name name_tata = name tick = time.time() executor = ThreadPoolExecutor() try: task1 = executor.submit(util_amazon, name_amazon) amazon = task1.result() except: amazon = None try: task2 = executor.submit(util_flipkart, name_flikart) flipkart = task2.result() except: flipkart = None try: task3 = executor.submit(util_tatacliq, name_tata) tata = task3.result() except: tata = None compare_context = { 'amazon': amazon, 'flipkart': flipkart, 'tata': tata, 'prd_name': name, 'time_taken': time.time() - tick, } return render(request, 'compare.html', compare_context)
class TaskManager(): def __init__(self, max_count=5): self.pool = ThreadPoolExecutor(max_count) self.futures = [] def submit_func(self, func, *args): self.cleanup_pool() future = self.pool.submit(func, *args) self.futures.append(future) def submit(self, argv): self.cleanup_pool() future = self.pool.submit(check_output, ' '.join(argv), shell=True) self.futures.append(future) return future.result() def cleanup_pool(self): self.futures = list(filter(lambda f: f and not f.done(), self.futures)) def wait(self): for future in self.futures: future.result() # blocks and forward exceptions def idle(self): ''' True if no task is running ''' for future in self.futures: if not future.done(): return False return True
def run_case(request, *args, **kwargs): case_id = kwargs["pk"] run_env = request.data.get("runEnv") run_user_nickname = request.data.get("runUserNickname") case = Case.objects.get(id=case_id) project_id = case.project_id request_jwt = request.headers.get("Authorization").replace("Bearer ", "") request_jwt_decoded = jwt.decode(request_jwt, verify=False, algorithms=['HS512']) user_id = request_jwt_decoded["user_id"] p = ProjectPath(project_id, run_env, user_id) if not os.path.exists(p.project_temp_dir()): os.chdir(p.projects_root) startproject(p.project_temp_name) clean_fixtures_dir(os.path.join(p.project_temp_dir(), "fixtures")) clean_tests_dir(os.path.join(p.project_temp_dir(), "tests")) pull_tep_files(project_id, p.project_temp_dir(), run_env) thread_pool = ThreadPoolExecutor() tests_dir = os.path.join(p.project_temp_dir(), "tests") for newest_case in pull_case_files(tests_dir, [case]): case_id, filepath = newest_case delete_case_result(case_id, run_user_nickname) os.chdir(tests_dir) cmd = rf"pytest -s {filepath}" args = (pytest_subprocess, cmd, case_id, run_env, run_user_nickname) thread_pool.submit(*args).add_done_callback(save_case_result) return Response({"msg": "用例运行成功"}, status=status.HTTP_200_OK)
def run(self): self.turn_me() if not self.exit: res = self.get_host_video_data() m3u8_data = self.get_m3u8_data(res) m3u8_count = m3u8_data.count('EXTINF') host = sorted(res['host'], key=lambda i: i.get('weight'), reverse=True) executor = ThreadPoolExecutor(max_workers=self.anime.speed_value) for i in range(self.data['video_ts'], m3u8_count): executor.submit(self.video, i, res, host, m3u8_count) while True: if self.data['video_ts'] == m3u8_count or self.exit: break self.data.update({ 'schedule': int(self.data['video_ts'] / (m3u8_count - 1) * 100), 'status': '下載中', }) self.download_video.emit(self.data) time.sleep(1) self.download_video.emit(self.data) self.anime.now_download_value -= 1 try: if self.data['video_ts'] == m3u8_count: self.anime.download_queue.remove(self.data["total_name"]) except BaseException as e: print(f'抓刪除時 queue 有錯誤 {e}') json.dump({'queue': self.anime.download_queue}, open('./Log/DownloadQueue.json', 'w', encoding='utf-8'), indent=2) self.quit() self.wait()
def upload_files(files, s3_path): executor = ThreadPoolExecutor(max_workers=32) for i,file_path in enumerate(files): executor.submit(upload_s3_file, file_path, s3_path) if i%100==0: print(f"{i}\t{file_path}") executor.shutdown(wait=True)
class OutPutThreadPool(object): ''' 启用最大并发线程数为5的线程池对上面爬取解析线程池结果进行并发处理存储; ''' def __init__(self): self.thread_pool = ThreadPoolExecutor(max_workers=5) def _output_runnable(self, crawl_result): try: url = crawl_result['url'] title = crawl_result['title'] summary = crawl_result['summary'] save_dir = 'output' print('start save %s as %s.txt.' % (url, title)) if os.path.exists(save_dir) is False: os.makedirs(save_dir) save_file = save_dir + os.path.sep + title + '.txt' if os.path.exists(save_file): print('file %s is already exist!' % title) return with open(save_file, "w") as file_input: file_input.write(summary) except Exception as e: print('save file error.' + str(e)) def save(self, crawl_result): self.thread_pool.submit(self._output_runnable, crawl_result)
class DistanceCalculator(threading.Thread): def __init__(self, task_queue, result_queue, origin_address, *args, **kwargs): self.google_maps_helper = GoogleMapsHelper() self.executor = ThreadPoolExecutor(10) self.task_queue = task_queue self.result_queue = result_queue self.origin_address = origin_address super().__init__(*args, **kwargs) def run(self): while True: try: offering = self.task_queue.get(timeout=0.1) except queue.Empty: return restaurant = offering['restaurant'] destination_id = restaurant['id'] duration = dynamodb.get_distance(self.origin_address, destination_id) if duration is None: destination = F"{restaurant['address']}, {restaurant['city']['name']}, {restaurant['state']}" duration = self.google_maps_helper.get_walking_time( self.origin_address, destination) if duration is not None: self.executor.submit(dynamodb.store_distance, self.origin_address, destination_id, duration) offering['walkingTimeFromOrigin'] = duration self.result_queue.put_nowait(offering) self.task_queue.task_done()
def prepareServer(RequestHandlerClass, pipe, threads, timeout): ''' Prepare in a process the request handling. ''' def process(request, address): RequestHandlerClass(request, address, None) try: request.shutdown(socket.SHUT_WR) except socket.error: pass # some platforms may raise ENOTCONN here request.close() pool = ThreadPoolExecutor(threads) while True: if not pipe.poll(timeout): break else: data = pipe.recv() if data is None: break elif data is True: continue requestfd, address = data request = socket.fromfd(rebuild_handle(requestfd), socket.AF_INET, socket.SOCK_STREAM) pool.submit(process, request, address) pool.shutdown(False)
class ModelCaller: """ Class for asynchronous calling of model's API """ def __init__(self): self.query = TileInformation.objects.filter( tile_name__isnull=False, source_b04_location__isnull=False, source_b08_location__isnull=False, source_tci_location__isnull=False) self.data_dir = 'data' self.executor = ThreadPoolExecutor(max_workers=10) def start(self): for tile in self.query: self.executor.submit(self.process, tile) def process(self, tile): """ Converting jp2file to tiff, then sending its to model and saving results to db """ prepare_tiff(tile) results = raster_prediction(tile.model_tiff_location) results_path = os.path.join(self.data_dir, results[0].get('polygons')) save(results_path) # clean up rmtree(os.path.dirname(tile.model_tiff_location)) rmtree(os.path.dirname(results_path))
def main(dir_path: Path): start_time = time.time() executor = ThreadPoolExecutor(max_workers=10) for path in dir_path.rglob("*.java"): executor.submit(replace, path) end_time = time.time() print(end_time - start_time)
def run_plan_engine(project_id, plan_id, run_env, run_user_nickname, user_id): p = ProjectPath(project_id, run_env, user_id) if not os.path.exists(p.project_temp_dir()): os.chdir(p.projects_root) startproject(p.project_temp_name) clean_fixtures_dir(os.path.join(p.project_temp_dir(), "fixtures")) clean_tests_dir(os.path.join(p.project_temp_dir(), "tests")) pull_tep_files(project_id, p.project_temp_dir(), run_env) # 根据plan_id从PlanCase中找到关联用例 plan_case_ids = [ plan_case.case_id for plan_case in PlanCase.objects.filter(plan_id=plan_id) ] case_list = Case.objects.filter(Q(id__in=plan_case_ids)) thread_pool = ThreadPoolExecutor() tests_dir = os.path.join(p.project_temp_dir(), "tests") for newest_case in pull_case_files(tests_dir, case_list): case_id, filepath = newest_case delete_plan_result(plan_id, case_id) os.chdir(tests_dir) cmd = rf"pytest -s {filepath}" args = (pytest_subprocess, cmd, case_id, run_env, run_user_nickname, plan_id) thread_pool.submit(*args).add_done_callback(save_case_result)
class Server: def __init__(self, host='', port=5555): self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.host = host self.port = port self.bind() self.listen() self.executor = ThreadPoolExecutor(max_workers=10) def bind(self): try: self.socket.bind((self.host, self.port)) except socket.error as msg: print('Bind failed.', msg) sys.exit() print('Socket bind complete') def listen(self, backlog=10): # Start listening on socket self.socket.listen(backlog) print('Socket now listening') def handle_request(self, data): print("Received from client: ", data) timestamp = datetime.datetime.now().timestamp() data["timestamp"] = timestamp print("Updated data", data) return data def handle_connection(self, conn): while True: data = conn.recv(1024) if not data: break try: data = json.loads(data.decode()) except ValueError: data = json.dumps({"error": "Bad request"}) conn.sendall(json.dumps(data).encode()) else: response = self.handle_request(data) conn.sendall(json.dumps(response).encode()) conn.close() def start(self): try: while True: # wait to accept a connection - blocking call conn, addr = self.socket.accept() print('Connected with ', addr[0], ':', str(addr[1])) self.executor.submit(self.handle_connection, conn) finally: print("Stop") self.socket.close()
def multi_test(self): # using thread pool to improve testing speed t = ThreadPoolExecutor(10) print('testing proxy, it will take several minutes......') for proxy in self.proxy_lis: t.submit(self.tes_proxy, proxy).add_done_callback(self.save_valid_proxy_lis) t.shutdown() print(f'fetch {self.p_count} valid proxies!')
def echo_server(addr): pool = ThreadPoolExecutor(128) sock = socket(AF_INET, SOCK_STREAM) sock.bind(addr) sock.listen(5) while True: client_sock, client_addr = sock.accept() pool.submit(echo_client, client_sock, client_addr)
class ThreadedPoolExecutor(PoolExecutor): ''' Pooled executor implementation based on a wrapped ThreadPoolExecutor object. ''' def __init__(self, context, max_workers=1): super(ThreadedPoolExecutor, self).__init__(context) self._pool = ThreadPoolExecutor(max_workers) def execute(self, task): self._pool.submit(task.processor)
def test_log_traceback_threaded(caplog): @log_traceback def f(): raise Exception() e = ThreadPoolExecutor(max_workers=1) f = e.submit(f) while f.running(): time.sleep(0.1) assert caplog.records()[0].message.endswith(" is about to be started") assert caplog.records()[1].message.startswith("Traceback") assert caplog.records()[1].message.endswith("Exception\n")
class DispatcherHTTPServer(HTTPServer): def __init__(self, server_address, RequestHandlerClass, bind_and_activate=True, handlers=[], srv_path=".", configuration={}): HTTPServer.__init__(self, server_address, RequestHandlerClass, bind_and_activate) self.handlers = sorted(handlers, key=lambda k: k["weight"]) self.srv_path = srv_path self.configuration = configuration self.logger = self.setup_logger() self.executor = ThreadPoolExecutor(max_workers=20) self.initialize_server() def initialize_server(self): self.logger.info('Initializing server') def finish_request(self, request, client_address): def async_finish_request(server, request, client_address, logger): server.RequestHandlerClass(request, client_address, server, logger) self.executor.submit(async_finish_request(self, request, client_address, self.logger)) def setup_logger(self): logger = None if self.configuration.get('log_config_file') is not None: logger = self.get_logger(self.configuration.get('log_config_file')) else: logger = self.get_default_logger() return logger def get_logger(self, config_file): logging.config.fileConfig(config_file) return logging.getLogger('srv') def get_default_logger(self): logging.basicConfig(level=logging.INFO) return logging.getLogger('srv')
class MethodCallGroup(object): ''' This class groups similar shared method calls, fires only once and replies to all callers with the result. ''' def __init__(self, max_workers=5): ''' This is multi-threaded caller. max_workers - Maximum simultaneous threads. async - Set it to True then "submit" will not wait for the response. ''' self.__callId_callbackList = {} # 1-level dict, [callId] -- [callback tasks] self.__fs_callbackList = {} # 1-level dict, [fs] = [callback tasks] self.__fs_callId = {} self.__threadPool = ThreadPoolExecutor(max_workers=max_workers) self.__lock = Lock() self.__taskId = 0 def submit(self, callId, callFn, callKwargs, callbackFn=None, **callbackKwargs): ''' callId - A string. Same call Id will be called only once. callFn - A callable function where "**kwargs" will be pass as arguments later. callKwargs - A dictionary object which can be passed into callerFn. callbackFn - A callable which takes effect only when async = True. callbackKwargs - "callback" will accepts the result as the first argument, and **callbackKW as the rest of the arguments. -- result can be the return value, or an instance of Exception. ''' # Generate task id taskId = self.__taskId = (self.__taskId + 1) % 9999 fs = None with self.__lock: if callId not in self.__callId_callbackList: fs = self.__threadPool.submit(callFn, **callKwargs) callbackList = {} self.__fs_callbackList[fs] = callbackList self.__callId_callbackList[callId] = callbackList self.__fs_callId[fs] = callId else: callbackList = self.__callId_callbackList[callId] callbackList[taskId] = (callbackFn, callbackKwargs) if fs is not None: fs.add_done_callback(self.__done_callback) return taskId def __done_callback(self, fs): with self.__lock: callbackList = self.__fs_callbackList[fs] del(self.__fs_callbackList[fs]) del(self.__callId_callbackList[self.__fs_callId[fs]]) del(self.__fs_callId[fs]) try: result = fs.result() except Exception as e: result = e for taskId, (callbackFn, callbackKwargs) in callbackList.items(): callbackFn(taskId, result, **callbackKwargs)
class OpticalPathManager(object): """ The purpose of this module is setting the physical components contained in the optical path of a SPARC system to the right position/configuration with respect to the mode given. """ def __init__(self, microscope): """ microscope (Microscope): the whole microscope component, thus it can handle all the components needed """ self.microscope = microscope self._graph = affectsGraph(self.microscope) # Use subset for modes guessed if microscope.role == "sparc2": self._modes = copy.deepcopy(SPARC2_MODES) elif microscope.role in ("sparc-simplex", "sparc"): self._modes = copy.deepcopy(SPARC_MODES) else: raise NotImplementedError("Microscope role '%s' unsupported" % (microscope.role,)) # keep list of already accessed components, to avoid creating new proxys # every time the mode changes self._known_comps = dict() # str (role) -> component # All the actuators in the microscope, to cache proxy's to them self._actuators = [] for comp in model.getComponents(): if hasattr(comp, 'axes') and isinstance(comp.axes, dict): self._actuators.append(comp) # last known axes position self._stored = {} self._last_mode = None # previous mode that was set # Removes modes which are not supported by the current microscope for m, (det, conf) in self._modes.items(): try: comp = self._getComponent(det) except LookupError: logging.debug("Removing mode %s, which is not supported", m) del self._modes[m] # Create the guess information out of the mode # TODO: just make it a dict comprole -> mode self.guessed = self._modes.copy() # No stream should ever imply alignment mode for m in ALIGN_MODES: try: del self.guessed[m] except KeyError: pass # Mode to delete is just not there # Handle different focus for chamber-view (in SPARCv2) if "chamber-view" in self._modes: self._focus_in_chamber_view = None self._focus_out_chamber_view = None # Check whether the focus affects the chamber view self._chamber_view_own_focus = False try: chamb_det = self._getComponent(self._modes["chamber-view"][0]) focus = self._getComponent("focus") if self.affects(focus.name, chamb_det.name): self._chamber_view_own_focus = True except LookupError: pass if not self._chamber_view_own_focus: logging.debug("No focus component affecting chamber") try: spec = self._getComponent("spectrometer") except LookupError: spec = None if self.microscope.role == "sparc2" and spec: # Remove the moves that don't affects the detector # TODO: do this for _all_ modes for mode in ('spectral', 'monochromator'): if mode in self._modes: det_role = self._modes[mode][0] det = self._getComponent(det_role) modeconf = self._modes[mode][1] for act_role in modeconf.keys(): try: act = self._getComponent(act_role) except LookupError: # TODO: just remove that move too? logging.debug("Failed to find component %s, skipping it", act_role) continue if not self.affects(act.name, det.name): logging.debug("Actuator %s doesn't affect %s, so removing it from mode %s", act_role, det_role, mode) del modeconf[act_role] # will take care of executing setPath asynchronously self._executor = ThreadPoolExecutor(max_workers=1) def __del__(self): logging.debug("Ending path manager") # Restore the spectrometer focus, so that on next start, this value will # be used again as "out of chamber view". if self._chamber_view_own_focus and self._last_mode == "chamber-view": focus_comp = self._getComponent("focus") if self._focus_out_chamber_view is not None: logging.debug("Restoring focus from before coming to chamber view to %s", self._focus_out_chamber_view) try: focus_comp.moveAbsSync(self._focus_out_chamber_view) except IOError as e: logging.info("Actuator move failed giving the error %s", e) self._executor.shutdown(wait=False) def _getComponent(self, role): """ same as model.getComponent, but optimised by caching the result return Component raise LookupError: if no component found """ try: comp = self._known_comps[role] except LookupError: comp = model.getComponent(role=role) self._known_comps[role] = comp return comp @isasync def setPath(self, mode): """ Just a wrapper of _doSetPath """ f = self._executor.submit(self._doSetPath, mode) return f def _doSetPath(self, path): """ Given a particular mode it sets all the necessary components of the optical path (found through the microscope component) to the corresponding positions. path (stream.Stream or str): The stream or the optical path mode raises: ValueError if the given mode does not exist IOError if a detector is missing """ if isinstance(path, stream.Stream): mode = self.guessMode(path) if mode not in self._modes: raise ValueError("Mode '%s' does not exist" % (mode,)) target = self.getStreamDetector(path) # target detector else: mode = path if mode not in self._modes: raise ValueError("Mode '%s' does not exist" % (mode,)) comp_role = self._modes[mode][0] comp = self._getComponent(comp_role) target = comp.name logging.debug("Going to optical path '%s', with target detector %s.", mode, target) fmoves = [] # moves in progress # Restore the spectrometer focus before any other move, as (on the SR193), # the value is grating/output dependent if self._chamber_view_own_focus and self._last_mode == "chamber-view": focus_comp = self._getComponent("focus") self._focus_in_chamber_view = focus_comp.position.value.copy() if self._focus_out_chamber_view is not None: logging.debug("Restoring focus from before coming to chamber view to %s", self._focus_out_chamber_view) fmoves.append(focus_comp.moveAbs(self._focus_out_chamber_view)) modeconf = self._modes[mode][1] for comp_role, conf in modeconf.items(): # Try to access the component needed try: comp = self._getComponent(comp_role) except LookupError: logging.debug("Failed to find component %s, skipping it", comp_role) continue mv = {} for axis, pos in conf.items(): if axis == "power": if model.hasVA(comp, "power"): try: if pos == 'on': comp.power.value = comp.power.range[1] else: comp.power.value = comp.power.range[0] logging.debug("Updating power of comp %s to %f", comp.name, comp.power.value) except AttributeError: logging.debug("Could not retrieve power range of %s component", comp_role) continue if isinstance(pos, str) and pos.startswith("MD:"): pos = self.mdToValue(comp, pos[3:])[axis] if axis in comp.axes: if axis == "band": # Handle the filter wheel in a special way. Search # for the key that corresponds to the value, most probably # to the 'pass-through' choices = comp.axes[axis].choices for key, value in choices.items(): if value == pos: pos = key # Just to store current band in order to restore # it once we leave this mode if self._last_mode not in ALIGN_MODES: self._stored[axis] = comp.position.value[axis] break else: logging.debug("Choice %s is not present in %s axis", pos, axis) continue elif axis == "grating": # If mirror is to be used but not found in grating # choices, then we use zero order. In case of # GRATING_NOT_MIRROR we either use the last known # grating or the first grating that is not mirror. choices = comp.axes[axis].choices if pos == "mirror": # Store current grating (if we use one at the moment) # to restore it once we use a normal grating again if choices[comp.position.value[axis]] != "mirror": self._stored[axis] = comp.position.value[axis] self._stored['wavelength'] = comp.position.value['wavelength'] # Use the special "mirror" grating, if it exists for key, value in choices.items(): if value == "mirror": pos = key break else: # Fallback to zero order (aka "low-quality mirror") axis = 'wavelength' pos = 0 elif pos == GRATING_NOT_MIRROR: if choices[comp.position.value[axis]] == "mirror": # if there is a grating stored use this one # otherwise find the non-mirror grating if axis in self._stored: pos = self._stored[axis] else: pos = self.findNonMirror(choices) if 'wavelength' in self._stored: mv['wavelength'] = self._stored['wavelength'] else: pos = comp.position.value[axis] # no change try: del self._stored[axis] except KeyError: pass try: del self._stored['wavelength'] except KeyError: pass else: logging.debug("Using grating position as-is: '%s'", pos) pass # use pos as-is elif axis == "slit-in": if self._last_mode not in ALIGN_MODES: # TODO: save also the component self._stored[axis] = comp.position.value[axis] elif hasattr(comp.axes[axis], "choices") and isinstance(comp.axes[axis].choices, dict): choices = comp.axes[axis].choices for key, value in choices.items(): if value == pos: pos = key break mv[axis] = pos else: logging.debug("Not moving axis %s.%s as it is not present", comp_role, axis) try: fmoves.append(comp.moveAbs(mv)) except AttributeError: logging.debug("%s not an actuator", comp_role) # Now take care of the selectors based on the target detector fmoves.extend(self.selectorsToPath(target)) # If we are about to leave alignment modes, restore values if self._last_mode in ALIGN_MODES and mode not in ALIGN_MODES: if 'band' in self._stored: try: flter = self._getComponent("filter") fmoves.append(flter.moveAbs({"band": self._stored['band']})) except LookupError: logging.debug("No filter component available") if 'slit-in' in self._stored: try: spectrograph = self._getComponent("spectrograph") fmoves.append(spectrograph.moveAbs({"slit-in": self._stored['slit-in']})) except LookupError: logging.debug("No spectrograph component available") # Save last mode self._last_mode = mode # wait for all the moves to be completed for f in fmoves: try: f.result() except IOError as e: logging.warning("Actuator move failed giving the error %s", e) # When going to chamber view, store the current focus position, and # restore the special focus position for chamber, after _really_ all # the other moves have finished, because the grating/output selector # moves affects the current position of the focus. if self._chamber_view_own_focus and mode == "chamber-view": focus_comp = self._getComponent("focus") self._focus_out_chamber_view = focus_comp.position.value.copy() if self._focus_in_chamber_view is not None: logging.debug("Restoring focus from previous chamber view to %s", self._focus_in_chamber_view) try: focus_comp.moveAbsSync(self._focus_in_chamber_view) except IOError as e: logging.warning("Actuator move failed giving the error %s", e) def selectorsToPath(self, target): """ Sets the selectors so the optical path leads to the target component (usually a detector). target (str): component name return (list of futures) """ fmoves = [] for comp in self._actuators: # TODO: pre-cache this as comp/target -> axis/pos # TODO: extend the path computation to "for every actuator which _affects_ # the target, move if if position known, and update path to that actuator"? # Eg, this would improve path computation on SPARCv2 with fiber aligner mv = {} for an, ad in comp.axes.items(): if hasattr(ad, "choices") and isinstance(ad.choices, dict): for pos, value in ad.choices.items(): if target in value: # set the position so it points to the target mv[an] = pos comp_md = comp.getMetadata() if target in comp_md.get(model.MD_FAV_POS_ACTIVE_DEST, {}): mv.update(comp_md[model.MD_FAV_POS_ACTIVE]) elif target in comp_md.get(model.MD_FAV_POS_DEACTIVE_DEST, {}): mv.update(comp_md[model.MD_FAV_POS_DEACTIVE]) if mv: logging.debug("Move %s added so %s targets to %s", mv, comp.name, target) fmoves.append(comp.moveAbs(mv)) # make sure this component is also on the optical path fmoves.extend(self.selectorsToPath(comp.name)) return fmoves def guessMode(self, guess_stream): """ Given a stream and by checking its components (e.g. role of detector) guesses and returns the corresponding optical path mode. guess_stream (object): The given optical stream returns (str): Mode estimated raises: LookupError if no mode can be inferred for the given stream IOError if given object is not a stream """ if not isinstance(guess_stream, stream.Stream): raise IOError("Given object is not a stream") # Handle multiple detector streams if isinstance(guess_stream, stream.MultipleDetectorStream): for st in guess_stream.streams: try: return self.guessMode(st) except LookupError: pass else: for mode, conf in self.guessed.items(): if conf[0] == guess_stream.detector.role: return mode # In case no mode was found yet raise LookupError("No mode can be inferred for the given stream") def getStreamDetector(self, path_stream): """ Given a stream find the detector. path_stream (object): The given stream returns (str): detector name raises: IOError if given object is not a stream LookupError: if stream has no detector """ if not isinstance(path_stream, stream.Stream): raise IOError("Given object is not a stream") # Handle multiple detector streams if isinstance(path_stream, stream.MultipleDetectorStream): dets = [] for st in path_stream.streams: try: # Prefer the detectors which have a role in the mode, as it's much # more likely to be the optical detector # TODO: handle setting multiple optical paths? => return all the detectors role = st.detector.role name = st.detector.name for conf in self.guessed.values(): if conf[0] == role: return name dets.append(name) except AttributeError: pass if dets: logging.warning("No detector on stream %s has a known optical role", path_stream.name.value) return dets[0] else: try: return path_stream.detector.name except AttributeError: pass # will raise error just after raise LookupError("Failed to find a detector on stream %s" % (path_stream.name.value)) def findNonMirror(self, choices): """ Given a dict of choices finds the one with value different than "mirror" """ for key, value in choices.items(): if value != "mirror": return key else: raise ValueError("Cannot find grating value in given choices") def mdToValue(self, comp, md_name): """ Just retrieves the "md_name" metadata from component "comp" """ md = comp.getMetadata() try: value = md.get(md_name) return value except KeyError: raise KeyError("Metadata %s does not exist in component %s" % (md_name, comp.name)) def affects(self, affecting, affected): """ Returns True if "affecting" component affects -directly of indirectly- the "affected" component """ path = self.findPath(affecting, affected) if path is None: return False else: return True def findPath(self, node1, node2, path=[]): """ Find any path between node1 and node2 (may not be shortest) """ path = path + [node1] if node1 == node2: return path if node1 not in self._graph: return None for node in self._graph[node1]: if node not in path: new_path = self.findPath(node, node2, path) if new_path: return new_path return None
class MyComponent(model.Component): """ A component that does everything """ def __init__(self, name, daemon): model.Component.__init__(self, name=name, daemon=daemon) self.executor = ThreadPoolExecutor(max_workers=1) self.number_futures = 0 self.startAcquire = model.Event() # triggers when the acquisition of .data starts self.data = FakeDataFlow(sae=self.startAcquire) self.datas = SynchronizableDataFlow() self.data_count = 0 self._df = None # TODO automatically register the property when serializing the Component self.prop = model.IntVA(42) self.cont = model.FloatContinuous(2.0, [-1, 3.4], unit="C") self.enum = model.StringEnumerated("a", set(["a", "c", "bfds"])) self.cut = model.IntVA(0, setter=self._setCut) self.listval = model.ListVA([2, 65]) def _setCut(self, value): self.data.cut = value return self.data.cut @roattribute def my_value(self): return "ro" def ping(self): """ Returns (string): pong """ return "pong" def bad_call(self): """ always raise an exception """ raise MyError # oneway to ensure that it will be set in a different thread than the call @oneway def change_prop(self, value): """ set a new value for the VA prop """ self.prop.value = value @isasync def do_long(self, duration=5): """ return a futures.Future """ ft = self.executor.submit(self._long_task, duration) ft.add_done_callback(self._on_end_long) return ft def _long_task(self, duration): """ returns the time it took """ start = time.time() time.sleep(duration) return (time.time() - start) def get_number_futures(self): return self.number_futures def set_number_futures(self, value): self.number_futures = value def _on_end_long(self, future): self.number_futures += 1 def sub(self, df): self._df = df df.subscribe(self.data_receive) def unsub(self): self._df.unsubscribe(self.data_receive) def data_receive(self, df, data): logging.info("Received data of shape %r", data.shape) self.data_count += 1 def get_data_count(self): return self.data_count # it'll never be able to answer back if everything goes fine @oneway def stopServer(self): self._pyroDaemon.shutdown()
class OpticalPathManager(object): """ The purpose of this module is setting the physical components contained in the optical path of a SPARC system to the right position/configuration with respect to the mode given. """ def __init__(self, microscope): """ microscope (Microscope): the whole microscope component, thus it can handle all the components needed """ self.microscope = microscope self._graph = affectsGraph(self.microscope) self._chamber_view_own_focus = False # Use subset for modes guessed if microscope.role == "sparc2": self._modes = copy.deepcopy(SPARC2_MODES) elif microscope.role in ("sparc-simplex", "sparc"): self._modes = copy.deepcopy(SPARC_MODES) elif microscope.role in ("secom", "delphi"): self._modes = copy.deepcopy(SECOM_MODES) else: raise NotImplementedError("Microscope role '%s' unsupported" % (microscope.role,)) # Currently only used with the SECOM/DELPHI self.quality = ACQ_QUALITY_FAST # keep list of all components, to avoid creating new proxies # every time the mode changes self._cached_components = model.getComponents() # All the actuators in the microscope, to cache proxy's to them self._actuators = [] for comp in self._cached_components: if hasattr(comp, 'axes') and isinstance(comp.axes, dict): self._actuators.append(comp) # last known axes position (before going to an alignment mode) self._stored = {} # (str, str) -> pos: (comp role, axis name) -> position self._last_mode = None # previous mode that was set # Removes modes which are not supported by the current microscope for m, (det, conf) in self._modes.items(): try: comp = self._getComponent(det) except LookupError: logging.debug("Removing mode %s, which is not supported", m) del self._modes[m] # Create the guess information out of the mode # TODO: just make it a dict comprole -> mode self.guessed = self._modes.copy() # No stream should ever imply alignment mode for m in ALIGN_MODES: try: del self.guessed[m] except KeyError: pass # Mode to delete is just not there if self.microscope.role in ("secom", "delphi"): # To record the fan settings when in "fast" acq quality try: ccd = self._getComponent("ccd") except LookupError: ccd = None # Check that at least it's a confocal microscope try: lm = self._getComponent("laser-mirror") except LookupError: logging.warning("Couldn't find a CCD on a SECOM/DELPHI") self._has_fan_speed = model.hasVA(ccd, "fanSpeed") self._has_fan_temp = (model.hasVA(ccd, "targetTemperature") and not ccd.targetTemperature.readonly) # Consider that by default we are in "fast" acquisition, with the fan # active (if it ought to be active) self._fan_enabled = True # Settings of the fan when the fan is in "active cooling" mode self._enabled_fan_speed = None self._enabled_fan_temp = None # Handle different focus for chamber-view (in SPARCv2) if "chamber-view" in self._modes: self._focus_in_chamber_view = None self._focus_out_chamber_view = None # Check whether the focus affects the chamber view try: chamb_det = self._getComponent(self._modes["chamber-view"][0]) focus = self._getComponent("focus") if self.affects(focus.name, chamb_det.name): self._chamber_view_own_focus = True except LookupError: pass if not self._chamber_view_own_focus: logging.debug("No focus component affecting chamber") # will take care of executing setPath asynchronously self._executor = ThreadPoolExecutor(max_workers=1) def __del__(self): logging.debug("Ending path manager") # Restore the spectrometer focus, so that on next start, this value will # be used again as "out of chamber view". if self._chamber_view_own_focus and self._last_mode == "chamber-view": focus_comp = self._getComponent("focus") if self._focus_out_chamber_view is not None: logging.debug("Restoring focus from before coming to chamber view to %s", self._focus_out_chamber_view) try: focus_comp.moveAbsSync(self._focus_out_chamber_view) except IOError as e: logging.info("Actuator move failed giving the error %s", e) try: self._executor.shutdown(wait=False) except AttributeError: pass # Not created def _getComponent(self, role): """ same as model.getComponent, but optimised by caching the result. Uses regex to match the name to a list of cached components return Component raise LookupError: if matching component not found """ # if we have not returned raise an exception for comp in self._cached_components: if comp.role is not None and re.match(role + "$", comp.role): return comp # if not found... raise LookupError("No component with the role %s" % (role,)) def setAcqQuality(self, quality): """ Update the acquisition quality expected. Depending on the quality, some hardware settings will be adjusted. quality (ACQ_QUALITY): the acquisition quality """ assert quality in (ACQ_QUALITY_FAST, ACQ_QUALITY_BEST) if quality == self.quality: return self.quality = quality if self.microscope.role in ("secom", "delphi"): if quality == ACQ_QUALITY_FAST: # Restore the fan (if it was active before) self._setCCDFan(True) # Don't turn off the fan if BEST: first wait for setPath() def setPath(self, mode, detector=None): """ Given a particular mode it sets all the necessary components of the optical path (found through the microscope component) to the corresponding positions. path (stream.Stream or str): The stream or the optical path mode detector (Component or None): The detector which will be targeted on this path. This can only be set if the path is a str (optical mode). That is useful in case the mode can be used with multiple detectors (eg, fiber-align on a SPARC with multiple spectrometers). When path is a Stream, the Stream.detector is always used. return (Future): a Future allowing to follow the status of the path update. raises (via the future): ValueError if the given mode does not exist IOError if a detector is missing """ f = self._executor.submit(self._doSetPath, mode, detector) return f def _doSetPath(self, path, detector): """ Actual implementation of setPath() """ if isinstance(path, stream.Stream): if detector is not None: raise ValueError("Not possible to specify both a stream, and a detector") try: mode = self.guessMode(path) except LookupError: logging.debug("%s doesn't require optical path change", path) return target = self.getStreamDetector(path) # target detector else: mode = path if mode not in self._modes: raise ValueError("Mode '%s' does not exist" % (mode,)) comp_role = self._modes[mode][0] if detector is None: target = self._getComponent(comp_role) else: target = detector logging.debug("Going to optical path '%s', with target detector %s.", mode, target.name) # Special SECOM mode: just look at the fan and be done if self.microscope.role in ("secom", "delphi"): if self.quality == ACQ_QUALITY_FAST: self._setCCDFan(True) elif self.quality == ACQ_QUALITY_BEST: self._setCCDFan(target.role == "ccd") fmoves = [] # moves in progress, list of (future, Component, dict(axis->pos) tuples # Restore the spectrometer focus before any other move, as (on the SR193), # the value is grating/output dependent if self._chamber_view_own_focus and self._last_mode == "chamber-view": focus_comp = self._getComponent("focus") self._focus_in_chamber_view = focus_comp.position.value.copy() if self._focus_out_chamber_view is not None: logging.debug("Restoring focus from before coming to chamber view to %s", self._focus_out_chamber_view) fmoves.append((focus_comp.moveAbs(self._focus_out_chamber_view), focus_comp, self._focus_out_chamber_view)) modeconf = self._modes[mode][1] for comp_role, conf in modeconf.items(): # Try to access the component needed try: comp = self._getComponent(comp_role) except LookupError: logging.debug("Failed to find component %s, skipping it", comp_role) continue # Check whether that actuator affects the target targets = {target.name} | set(target.affects.value) if not any(self.affects(comp.name, n) for n in targets): logging.debug("Actuator %s doesn't affect %s, so not moving it", comp.name, target.name) continue mv = {} for axis, pos in conf.items(): if axis == "power": if model.hasVA(comp, "power"): try: if pos == 'on': comp.power.value = comp.power.range[1] else: comp.power.value = comp.power.range[0] logging.debug("Updating power of comp %s to %f", comp.name, comp.power.value) except AttributeError: logging.debug("Could not retrieve power range of %s component", comp_role) continue if not hasattr(comp, "axes") or not isinstance(comp.axes, dict): continue if isinstance(pos, str) and pos.startswith("MD:"): pos = self.mdToValue(comp, pos[3:])[axis] if axis in comp.axes: if axis == "band": # Handle the filter wheel in a special way. Search # for the key that corresponds to the value, most probably # to the 'pass-through' choices = comp.axes[axis].choices for key, value in choices.items(): if value == pos: pos = key # Just to store current band in order to restore # it once we leave this mode if self._last_mode not in ALIGN_MODES: self._stored[comp_role, axis] = comp.position.value[axis] break else: logging.debug("Choice %s is not present in %s axis", pos, axis) continue elif axis == "grating": # If mirror is to be used but not found in grating # choices, then we use zero order. In case of # GRATING_NOT_MIRROR we either use the last known # grating or the first grating that is not mirror. choices = comp.axes[axis].choices if pos == "mirror": # Store current grating (if we use one at the moment) # to restore it once we use a normal grating again if choices[comp.position.value[axis]] != "mirror": self._stored[comp_role, axis] = comp.position.value[axis] self._stored[comp_role, 'wavelength'] = comp.position.value['wavelength'] # Use the special "mirror" grating, if it exists for key, value in choices.items(): if value == "mirror": pos = key break else: # Fallback to zero order (aka "low-quality mirror") axis = 'wavelength' pos = 0 elif pos == GRATING_NOT_MIRROR: if choices[comp.position.value[axis]] == "mirror": # if there is a grating stored use this one # otherwise find the non-mirror grating if (comp_role, axis) in self._stored: pos = self._stored[comp_role, axis] else: pos = self.findNonMirror(choices) if (comp_role, 'wavelength') in self._stored: mv['wavelength'] = self._stored[comp_role, 'wavelength'] else: pos = comp.position.value[axis] # no change try: del self._stored[comp_role, axis] except KeyError: pass try: del self._stored[comp_role, 'wavelength'] except KeyError: pass else: logging.debug("Using grating position as-is: '%s'", pos) pass # use pos as-is elif axis == "slit-in": if mode in ALIGN_MODES and (comp_role, axis) not in self._stored: self._stored[comp_role, axis] = comp.position.value[axis] elif hasattr(comp.axes[axis], "choices") and isinstance(comp.axes[axis].choices, dict): choices = comp.axes[axis].choices for key, value in choices.items(): if value == pos: pos = key break # write actuator axis and position in dict mv[axis] = pos else: logging.debug("Not moving axis %s.%s as it is not present", comp_role, axis) try: # move actuator fmoves.append((comp.moveAbs(mv), comp, mv)) except AttributeError: logging.warning("%s not an actuator", comp_role) # Now take care of the selectors based on the target detector fmoves.extend(self.selectorsToPath(target.name)) # If we are about to leave alignment modes, restore values if self._last_mode in ALIGN_MODES and mode not in ALIGN_MODES: logging.debug("Leaving align mode %s for %s, will restore positions: %s", self._last_mode, mode, self._stored) for (cr, an), pos in self._stored.copy().items(): # copy for deleting entries if an == "grating": continue # handled separately via GRATING_NOT_MIRROR comp = self._getComponent(cr) fmoves.append((comp.moveAbs({an: pos}), comp, {an: pos})) del self._stored[cr, an] # Save last mode self._last_mode = mode # wait for all the moves to be completed for f, comp, mv in fmoves: try: # Can be large, eg within 5 min one (any) move should finish. f.result(timeout=180) # To do an absolute move, an axis should be referenced (if it # supports referencing). If not, that's an error (but for now we # still try, just in case it might work anyway). for a in mv: try: if (model.hasVA(comp, "referenced") and not comp.referenced.value.get(a, True)): logging.error("%s.%s is not referenced, it might be a sign of a hardware issue", comp.name, a) except Exception: logging.exception("Failed to check %s.%s is referenced", comp.name, a) except IOError as e: logging.warning("Actuator move failed giving the error %s", e) except: logging.exception("Actuator move failed!") raise # When going to chamber view, store the current focus position, and # restore the special focus position for chamber, after _really_ all # the other moves have finished, because the grating/output selector # moves affects the current position of the focus. if self._chamber_view_own_focus and mode == "chamber-view": focus_comp = self._getComponent("focus") self._focus_out_chamber_view = focus_comp.position.value.copy() if self._focus_in_chamber_view is not None: logging.debug("Restoring focus from previous chamber view to %s", self._focus_in_chamber_view) try: focus_comp.moveAbsSync(self._focus_in_chamber_view) except IOError as e: logging.warning("Actuator move failed giving the error %s", e) def selectorsToPath(self, target): """ Sets the selectors so the optical path leads to the target component (usually a detector). target (str): component name return (list of futures) """ fmoves = [] for comp in self._actuators: # TODO: pre-cache this as comp/target -> axis/pos # TODO: don't do moves already done # TODO: extend the path computation to "for every actuator which _affects_ # the target, move if position known, and update path to that actuator"? # Eg, this would improve path computation on SPARCv2 with fiber aligner mv = {} for an, ad in comp.axes.items(): if hasattr(ad, "choices") and isinstance(ad.choices, dict): for pos, value in ad.choices.items(): if target in value: # set the position so it points to the target mv[an] = pos comp_md = comp.getMetadata() if target in comp_md.get(model.MD_FAV_POS_ACTIVE_DEST, {}): mv.update(comp_md[model.MD_FAV_POS_ACTIVE]) elif target in comp_md.get(model.MD_FAV_POS_DEACTIVE_DEST, {}): mv.update(comp_md[model.MD_FAV_POS_DEACTIVE]) if mv: logging.debug("Move %s added so %s targets to %s", mv, comp.name, target) fmoves.append((comp.moveAbs(mv), comp, mv)) # make sure this component is also on the optical path fmoves.extend(self.selectorsToPath(comp.name)) return fmoves def guessMode(self, guess_stream): """ Given a stream and by checking its components (e.g. role of detector) guesses and returns the corresponding optical path mode. guess_stream (object): The given optical stream returns (str): Mode estimated raises: LookupError if no mode can be inferred for the given stream IOError if given object is not a stream """ if not isinstance(guess_stream, stream.Stream): raise IOError("Given object is not a stream") # Handle multiple detector streams if isinstance(guess_stream, stream.MultipleDetectorStream): for st in guess_stream.streams: try: return self.guessMode(st) except LookupError: pass elif isinstance(guess_stream, stream.OverlayStream): return "overlay" else: for mode, conf in self.guessed.items(): # match the name using regex if re.match(conf[0] + '$', guess_stream.detector.role): return mode # In case no mode was found yet raise LookupError("No mode can be inferred for the given stream") def getStreamDetector(self, path_stream): """ Given a stream find the optical detector. path_stream (Stream): The given stream returns (HwComponent): detector raises: ValueError if given object is not a stream LookupError: if stream has no detector """ if not isinstance(path_stream, stream.Stream): raise ValueError("Given object is not a stream") # Handle multiple detector streams if isinstance(path_stream, stream.MultipleDetectorStream): dets = [] for st in path_stream.streams: try: # Prefer the detectors which have a role in the mode, as it's much # more likely to be the optical detector # TODO: handle setting multiple optical paths? => return all the detectors role = st.detector.role for conf in self.guessed.values(): if re.match(conf[0] + '$', role): return st.detector dets.append(st.detector) except AttributeError: pass if dets: logging.warning("No detector on stream %s has a known optical role", path_stream.name.value) return dets[0] elif isinstance(path_stream, stream.OverlayStream): return path_stream._ccd else: try: return path_stream.detector except AttributeError: pass # will raise error just after raise LookupError("Failed to find a detector on stream %s" % (path_stream.name.value,)) def findNonMirror(self, choices): """ Given a dict of choices finds the one with value different than "mirror" """ for key, value in choices.items(): if value != "mirror": return key else: raise ValueError("Cannot find grating value in given choices") def mdToValue(self, comp, md_name): """ Just retrieves the "md_name" metadata from component "comp" """ md = comp.getMetadata() try: return md[md_name] except KeyError: raise KeyError("Metadata %s does not exist in component %s" % (md_name, comp.name)) def affects(self, affecting, affected): """ Returns True if "affecting" component affects -directly of indirectly- the "affected" component affecting (str): component name affected (str): component name return bool """ path = self.findPath(affecting, affected) if path is None: return False else: return True def findPath(self, node1, node2, path=None): """ Find any path between node1 and node2 (may not be shortest) """ if path is None: path = [] path = path + [node1] if node1 == node2: return path if node1 not in self._graph: return None for node in self._graph[node1]: if node not in path: new_path = self.findPath(node, node2, path) if new_path: return new_path return None def _setCCDFan(self, enable): """ Turn on/off the fan of the CCD enable (boolean): True to turn on/restore the fan, and False to turn if off """ if not self._has_fan_speed: return if self._fan_enabled == enable: return self._fan_enabled = enable comp = self._getComponent("ccd") if enable: if self._enabled_fan_speed is not None: logging.debug("Turning fan on of %s", comp.name) comp.fanSpeed.value = max(comp.fanSpeed.value, self._enabled_fan_speed) else: if comp.fanSpeed.value == 0: # Already off => don't touch it self._enabled_fan_speed = None self._enabled_fan_temp = None else: logging.debug("Turning fan off of %s", comp.name) self._enabled_fan_speed = comp.fanSpeed.value comp.fanSpeed.value = 0 # Raise targetTemperature to max/ambient to avoid the fan from # automatically starting again. (Some hardware have this built-in when # the current temperature is too high compared to the target) if self._has_fan_temp: temp = comp.targetTemperature if enable: if self._enabled_fan_temp is not None: temp.value = min(comp.targetTemperature.value, self._enabled_fan_temp) try: self._waitTemperatureReached(comp, timeout=60) except Exception as ex: logging.warning("Failed to reach target temperature of CCD: %s", ex) else: # Set ~25°C == ambient temperature self._enabled_fan_temp = temp.value try: try: temp.value = min(comp.targetTemperature.range[1], 25) except (AttributeError, NotApplicableError): temp.value = util.find_closest(25, comp.targetTemperature.choices) except Exception: logging.warning("Failed to change targetTemperature when disabling fan", exc_info=True) def _waitTemperatureReached(self, comp, timeout=None): """ Wait until the current temperature of the component has reached the target temperature (within some margin). comp (Component) timeout (0<float or None): maximum time to wait (in s) raises: TimeoutError: if time-out reached """ tstart = time.time() while timeout is None or time.time() < tstart + timeout: # TODO: adjust the timeout depending on whether the temperature # gets closer to the target over time or not. ttemp = comp.targetTemperature.value atemp = comp.temperature.value if atemp < ttemp + TEMP_EPSILON: return else: logging.debug(u"Waiting for temperature to reach %g °C (currently at %g °C)", ttemp, atemp) time.sleep(1) raise TimeoutError("Target temperature (%g C) not reached after %g s" % (comp.targetTemperature.value, timeout))
class SceneService: # Locks down edit/delete/execute SCENE_PROCESS_LOCK = threading.Lock() def __init__(self): # Scene processors. self.__sceneController = SceneController.instance() self.__methodController = MethodController.instance() self.__sceneUpdateThreadPool = ThreadPoolExecutor(max_workers=1) self.__sceneExecThreadPool = ThreadPoolExecutor(max_workers=AppConstants.MAX_SCENE_EXEC_THREAD_SIZE) self.__sceneExecutionResultThreadPool = ThreadPoolExecutor(max_workers=5) # Scene run workers. self.__sceneExecLocks = {} # Favorite edit lock self.__fav_lock = threading.Lock() # Listeners. GroupController.instance().listen_to_group_icon_change(self.__on_group_icon_changed) self.__methodController.listen_to_method_status_change(self.__on_method_status_changed) def __on_group_icon_changed(self, kbxGroupId): ''' Trigger Source: GroupController --> This Callback when kbxGroupIcon changed. ''' sceneIds = self.__sceneController.list_scene_ids_which_has_kbx_group_id_as_execution(kbxGroupId) # Broadcast scenes updated messages. for sceneId in sceneIds: self.__broadcast_message__scene_updated(sceneId) def __on_method_status_changed(self, kbxMethodId, oldKBXMethodStatus, newKBXMethodStatus): ''' Trigger Source: MethodController --> This Callback when kbxMethodStatus changed. ''' if oldKBXMethodStatus != newKBXMethodStatus: sceneIds = self.__sceneController.list_scene_ids_which_has_kbx_method_id_as_execution(kbxMethodId) # Broadcast scenes updated messages. for sceneId in sceneIds: self.__broadcast_message__scene_updated(sceneId) @SCENE_PROCESS_SYNC def set_scene(self, execution, sceneId=None, sceneName=None, sceneProtected=False, sceneIcon=None): ''' Create/Edit(with sceneId provided) an existing scene. execution:Dictionary sceneId:Integer <Optional> sceneName:String <Optional> sceneProtected:Boolean <Optional> sceneIcon:Boolean Returns "sceneId" ''' def process_method_list(methodList): #=================================================================== # Basic type validation #=================================================================== if not isinstance(methodList, list): Logger.log_error("SceneService.set_scene: 'execution' must be type of list.") Logger.log_debug("type:", type(methodList), "value:", methodList) raise AutomationException(11904, "List is required for both 'execution'") #=================================================================== # Check allowed size, raise error if exceeded. #=================================================================== if len(methodList) > AppConstants.MAX_METHOD_SIZE: Logger.log_error("SceneService.set_scene: 'execution' cannot have more than", AppConstants.MAX_METHOD_SIZE, "items.") raise AutomationException(11907, "Only a maximum of " + \ str(AppConstants.MAX_METHOD_SIZE) + " items is allowed for each 'execution'", lambda text: str(AppConstants.MAX_METHOD_SIZE).join(text.split(":max_item_size:"))) #=================================================================== # Check if all kbxMethodIds are valid and all kbxMethodParams are list #=================================================================== idValidator = NumberValidator(isRequired=True, decimalPoint=False) if not all([idValidator.is_valid(eachMethod["kbxMethodId"]) and isinstance(eachMethod["kbxMethodParams"], list) for eachMethod in methodList]): raise AutomationException(11904, "'execution' have incorrect data structure.") #=================================================================== # Check if all kbxParamName and kbxParamCurrentValue exists #=================================================================== paramNameValidator = StringValidator(isRequired=True) for eachMethod in methodList: methodArgs = eachMethod["kbxMethodParams"] for methodArg in methodArgs: if not paramNameValidator.is_valid(methodArg[AppConstants.ARG_NAME]): raise AutomationException(11904, "'execution' have invalid params structure") if not AppConstants.ARG_CURRENT_VALUE in methodArg: methodArg[AppConstants.ARG_CURRENT_VALUE] = None return methodList #======================================================================= # Data structure validations #======================================================================= sceneId = NumberValidator(isRequired=False, decimalPoint=False).get_value(sceneId) execution = process_method_list(execution) #======================================================================= # Add to database #======================================================================= if Util.is_empty(sceneId): # Validate_max_scene_size if self.__sceneController.count() >= AppConstants.MAX_SCENE_SIZE: raise AutomationException(11908, "Scene size cannot be more than " + str(AppConstants.MAX_SCENE_SIZE), lambda text: str(AppConstants.MAX_SCENE_SIZE).join(text.split(":max_scene_size:"))) sceneId = self.__sceneController.generate_id(sceneName) scene = {} elif self.__sceneController.has(sceneId): self.__verify_scene_updated(sceneId) self.__sceneController.change_to_updating(sceneId, sceneName) scene = self.__sceneController.get(sceneId) scene = dict(scene) else: raise AutomationException(11902, "Scene not found") #======================================================================= # Broadcast message: starts to update scene. #======================================================================= self.__broadcast_message__scene_update_started(sceneId, sceneName) #======================================================================= # Set basic information of the scene #======================================================================= scene["sceneId"] = sceneId scene["sceneName"] = sceneName scene["sceneProtected"] = sceneProtected scene["sceneIcon"] = sceneIcon scene["execution"] = execution #======================================================================= # Append if its new scene #======================================================================= def __update_scene(scene): try: # Fire scene update start event sceneId = scene["sceneId"] sceneName = scene["sceneName"] # Add methods to subscribe list methodIds = [kbxMethod["kbxMethodId"] for kbxMethod in scene["execution"]] self.__methodController.add(methodIds) # Update "scene" base table self.__sceneController.update(scene) self.__sceneController.commit() except Exception as e: self.__sceneController.rollback() self.__broadcast_message__scene_update_failed(sceneId, sceneName) Logger.log_error("SceneService __update_scene failed:", e, "-- rolledback") else: # Broadcast message: completed updating a scene self.__broadcast_message__scene_updated(sceneId) #======================================================================= # Submit to a thread to process other info, and return... performance... #======================================================================= # Only 1 worker in the threadPool, it works as threading.Lock self.__sceneUpdateThreadPool.submit(__update_scene, scene) return sceneId @SCENE_PROCESS_SYNC def delete_scene(self, sceneId): self.__verify_scene_updated(sceneId) try: favSort = self.__sceneController.get_favsort_of(sceneId) # To determine should favorited_Scene_deleted broadcasted self.__sceneController.delete(sceneId) self.__sceneController.commit() except Exception as e: self.__sceneController.rollback() traceback.print_exc() Logger.log_error("SceneService delete_scene ex:", e, "-- rolled back") raise AutomationException(11906, "Unable to delete scene, problem - " + str(e)) else: self.__broadcast_message__scene_deleted(sceneId) if favSort is not None: self.__broadcast_message__favorited_scene_deleted(sceneId) @SCENE_PROCESS_SYNC def execute_scene(self, sceneId, serUrl=None, language="en"): ''' Execute a scene. Scene execution will only be recorded if serUrl is specified. ''' self.__verify_scene_updated(sceneId) self.__sceneExecLocks.setdefault(sceneId, SceneExecLock()) sceneExecLock = self.__sceneExecLocks.get(sceneId) isAcquired = sceneExecLock.acquire(False) # Raise error if failed to acquire. if isAcquired == True: try: #=================================================================== # Should record execution? Depends on serUrl. #=================================================================== if not Util.is_empty(serUrl): # Get sceneName and sceneProtected scene = self.__sceneController.get(sceneId) sceneName, sceneProtected = scene["sceneName"], scene["sceneProtected"] serObj = {"serUrl":serUrl, "sceneName":sceneName, "sceneProtected":sceneProtected} else: serObj = None #======================================================================= # List execution methods #======================================================================= execution = self.__sceneController.list_executions(sceneId) self.__sceneExecThreadPool.submit(self.__execute_scene_implementation, sceneId=sceneId, execution=execution, serObj=serObj, language=language) except Exception as e: del(self.__sceneExecLocks[sceneId]) sceneExecLock.release() traceback.print_exc() # This is unusual error raise e else: raise AutomationException(11901, "the scene is already executing. " + \ "Use 'stop_scene' to cancel its execution.") def stop_scene(self, sceneId): ''' Stop a scene execution. ''' sceneExecLock = self.__sceneExecLocks.get(sceneId, None) if sceneExecLock is None: raise AutomationException(11905, "Call stop only after execute_scene is called") else: sceneExecLock.set_stop() def get_scene(self, sceneId, language=AppInfo.DEFAULT_API_LANGUAGE): try: scene = self.__sceneController.get_detail(sceneId) except: raise AutomationException(11902, "Scene ID provided: " + str(sceneId)) kbxMethods = scene["execution"] # -------------- Compile lists of kbxMethod and group IDs contains in this scene. kbxMethodIdsToList = {} kbxGroupIdsToList = set([]) for kbxMethod in kbxMethods: # Variables kbxMethodId = kbxMethod["kbxMethodId"] kbxMethodAppId = kbxMethod["kbxMethodAppId"] kbxMethodStatus = kbxMethod["kbxMethodStatus"] kbxGroupId = kbxMethod["kbxGroupId"] kbxGroupStatus = kbxMethod["kbxGroupStatus"] if kbxMethodStatus is not -1 and kbxMethodAppId is not None: kbxMethodIdsToList.setdefault(kbxMethodAppId, set([])) kbxMethodIdsToList[kbxMethodAppId].add(kbxMethodId) if kbxGroupId is not None and kbxGroupStatus is not -1: kbxGroupIdsToList.add(kbxGroupId) # -------------- Get methods and groups based on requested language. kbxMethodIdsListed = {} kbxGroupIdsListed = {} for kbxMethodAppId, kbxMethodIds in kbxMethodIdsToList.items(): kbxMethodIdsListed[kbxMethodAppId] = SharedMethodWrapper.list_shared_methods_by_app_id(kbxMethodAppId, kbxMethodIds, language=language) groupList = SharedMethodWrapper.list_shared_method_groups(kbxGroupId=kbxGroupIdsToList, language=language) for row in groupList: kbxGroupIdsListed[row["kbxGroupId"]] = row # -------------- Set method and group data into scene. for kbxMethod in kbxMethods: # Variables kbxMethodId = kbxMethod["kbxMethodId"] kbxMethodAppId = kbxMethod["kbxMethodAppId"] kbxMethodStatus = kbxMethod["kbxMethodStatus"] kbxGroupId = kbxMethod["kbxGroupId"] kbxGroupStatus = kbxMethod["kbxGroupStatus"] if kbxMethodStatus is not -1 and kbxMethodAppId is not None: kbxMethodParamsWithCurrentValue = { kbxMethodParam["kbxParamName"]: kbxMethodParam["kbxParamCurrentValue"] \ for kbxMethodParam in kbxMethod["kbxMethodParams"] } kbxMethodWithDetails = kbxMethodIdsListed[kbxMethodAppId][kbxMethodId] if kbxMethodWithDetails is not None: kbxMethodParamsWithDetails = kbxMethodWithDetails["kbxMethodParams"] kbxMethodParamsWithDetails = copy.deepcopy(kbxMethodParamsWithDetails) for kbxMethodParam in kbxMethodParamsWithDetails: kbxMethodParam["kbxParamCurrentValue"] = kbxMethodParamsWithCurrentValue.get( kbxMethodParam["kbxParamName"], None) kbxMethod["kbxMethodParams"] = kbxMethodParamsWithDetails kbxMethod["kbxMethodLabel"] = kbxMethodWithDetails.get("kbxMethodLabel") kbxMethod["kbxMethodDesc"] = kbxMethodWithDetails.get("kbxMethodDesc") else: kbxMethod["atDebug"] = "Unable to get shared method, caused by a method which never register itself on this bootup." if kbxGroupId is not None and kbxGroupStatus is not -1: try: kbxMethod["kbxGroupLabel"] = kbxGroupIdsListed[kbxGroupId]["kbxGroupLabel"] kbxMethod["kbxGroupDesc"] = kbxGroupIdsListed[kbxGroupId]["kbxGroupDesc"] except: kbxMethod["atDebugGroup"] = "Unable to get shared method group, caused by a group which never register itself on this bootup." return scene def list_scenes(self, offset=0, limit=20): return self.__sceneController.list(offset, limit), \ self.__sceneController.count() def set_favorited_scene(self, sceneId, prevSceneId=None): ''' sceneId - can be either favorited/non-favorited scene, but must be a valid scene id. prevSceneId - must be another favorited scene or error is raised. Both updating and executing doesn't blocks a scene from adding to/removing from favorited list. ''' with self.__fav_lock: # Validation for sceneId. if not self.__sceneController.has(sceneId): raise AutomationException(11092, "sceneId does not belongs to any scene - sceneId provided: " + str(sceneId)) # Because UI display in reversed order, hence their prevSceneId == our nextSceneId nextSceneId = prevSceneId or None # Get favSort before sceneId of nextSceneId if nextSceneId is None: maxFavSort = self.__sceneController.get_largest_favsort_num() # If len of favorited list is 0, maxFavSort = 0 # favSort stores number to be assigned to sceneId. favSort = maxFavSort + 1 else: try: # favSort stores number to be assigned to sceneId. # current favSort of prevSceneId will becomes favSort of sceneId. favSort = self.__sceneController.get_favsort_of(nextSceneId) except Exception as e: raise AutomationException(11092, "prevSceneId does not belongs to any scene - " + \ "prevSceneId provided: " + str(nextSceneId) + ", error: " + str(e)) else: if favSort is None: raise AutomationException(13000, "prevSceneId does not belongs to any favorited scene - " + \ "prevSceneId provided: " + str(nextSceneId) + ", error: " + str(e)) # Update favSort of the scene try: self.__sceneController.update_favorited_scene(sceneId, favSort) self.__sceneController.commit() except Exception as e: self.__sceneController.rollback() Logger.log_error("SceneService.set_favorited_scene failed to update favorited scene, ex:", str(e)) raise AutomationException(13001, "unexpected error: " + str(e)) # Broadcast events self.__broadcast_message__scene_updated(sceneId) self.__broadcast_message__favorited_scene_added(sceneId, prevSceneId) def delete_favorited_scene(self, sceneId): ''' sceneId - must be a favorited scene. ''' with self.__fav_lock: try: favSort = self.__sceneController.get_favsort_of(sceneId) except Exception as e: raise AutomationException(11092, "prevSceneId does not belongs to any scene - " + \ "prevSceneId provided: " + str(sceneId) + ", error: " + str(e)) else: if favSort is None: raise AutomationException(13000, "sceneId does not belongs to any favorited scene - " + \ "sceneId provided: " + str(sceneId)) # This method raise error and rollback automatically if failed, or commit once succeed. try: self.__sceneController.delete_favorited_scene(sceneId) self.__sceneController.commit() except Exception as e: self.__sceneController.rollback() Logger.log_error("SceneService.delete_favorited_scene failed to delete favorited scene, ex:", str(e)) raise AutomationException(13002, "unexpected error: " + str(e)) #Broadcast event self.__broadcast_message__scene_updated(sceneId) self.__broadcast_message__favorited_scene_deleted(sceneId) def list_favorited_scene(self, offset=0, limit=200): ''' Favorited scenes. ''' return self.__sceneController.list_favorited_scenes_reversed(offset, limit), \ self.__sceneController.count_favorited_scenes() def __verify_scene_updated(self, sceneId): try: statusProcessed = self.__sceneController.get_status_processed(sceneId) if statusProcessed == AppConstants.SCENE_STATUS_UPDATING: raise AutomationException(11903, "edit/execute/delete scene is not allowed when its updating. " + \ "Wait until the update process is completed.") except: raise AutomationException(11902, "Scene ID provided:" + str(sceneId)) def __broadcast_message__scene_update_started(self, sceneId, sceneName=None): eventTag = AppConstants.EVENT_SCENE_UPDATE_STARTED eventData = {"sceneId": sceneId, "newSceneName":sceneName} self.__broadcast_message(eventTag, eventData) Logger.log_info("Scene Start Update:", sceneName) def __broadcast_message__scene_updated(self, sceneId): try: scene = self.__sceneController.get_summary(sceneId) except Exception as e: Logger.log_error("SceneService.__broadcast_message__scene_updated get_summary ex:", e) scene = None eventTag = AppConstants.EVENT_SCENE_UPDATED eventData = {"sceneId":sceneId, "newSceneSummary":scene} self.__broadcast_message(eventTag, eventData) Logger.log_info("Scene Updated:", scene["sceneName"]) def __broadcast_message__scene_update_failed(self, sceneId, sceneName=None): ''' sceneName - For debugging purpose. ''' try: scene = self.__sceneController.get_summary(sceneId) except Exception: scene = None eventTag = AppConstants.EVENT_SCENE_UPDATE_FAILED eventData = {"sceneId": sceneId, "oldSceneSummary":scene} self.__broadcast_message(eventTag, eventData) Logger.log_info("Scene Update Failed:", sceneName) def __broadcast_message__scene_deleted(self, sceneId): eventTag = AppConstants.EVENT_SCENE_DELETED eventData = {"sceneId": sceneId} self.__broadcast_message(eventTag, eventData) Logger.log_info("Scene Deleted: Id -", sceneId) def __broadcast_message__favorited_scene_added(self, sceneId, prevSceneId): eventTag = AppConstants.EVENT_FAVORITED_SCENE_ADDED eventData = {"sceneId":sceneId, "prevSceneId":prevSceneId} self.__broadcast_message(eventTag, eventData) Logger.log_info("Favorited scene added/updated: Id", sceneId, "prevId:", prevSceneId) def __broadcast_message__favorited_scene_deleted(self, sceneId): eventTag = AppConstants.EVENT_FAVORITED_SCENE_DELETED eventData = {"sceneId":sceneId} self.__broadcast_message(eventTag, eventData) Logger.log_info("Favorited scene deleted: Id", sceneId) def __broadcast_message(self, eventTag, eventData): eventData = json.dumps(eventData, cls=AutomationJSONEncoder) Application.send_web_server_event(eventTag, eventData) def __execute_scene_implementation(self, sceneId, execution, serObj, language): ''' ** Call execute_scene; DO NOT call this function directly. ''' Logger.log_info("execute scene id:", sceneId) def execution_func(sceneThreadEvent, kbxMethodId, seri, **kwargs): try: if kbxMethodId == -291: # Delay Timer delayInSec = kwargs["delayInSec"] sceneThreadEvent.wait(delayInSec) seri["seriError"] = None else: # Execute method result = SharedMethod.call(**kwargs) seri["seriError"] = str(result) seri["seriStatus"] = "ok" except Exception as e: seri["seriStatus"] = "error" seri["seriError"] = str(e) Logger.log_debug("Execution failed, method:", kwargs["kbxMethodName"]) finally: sceneThreadEvent.set() try: # Record for debugging purpose serStartTime = time.time() #=================================================================== # Prepare to execute execution methods #=================================================================== sceneExecLock = self.__sceneExecLocks.get(sceneId) sceneThreadEvent = sceneExecLock.get_thread_event() seris = deque() methodExecTime = int(time.time()) isLoopCompleted = True for row in execution: kbxMethodId = row["kbxMethodId"] kbxMethodName = row["kbxMethodName"] kbxGroupId = row["kbxGroupId"] kbxMethodStatus = row["kbxMethodStatus"] methodParamsWithCurrentValues = row["kbxMethodParams"] seri = {"kbxMethodId":kbxMethodId, "kbxGroupId":kbxGroupId, "kbxMethodName":kbxMethodName, "kbxMethodParams":methodParamsWithCurrentValues} seris.append(seri) # Check is stop if sceneExecLock.is_stop(): if not isLoopCompleted: serEndTime = time.time() isLoopCompleted = False # === Execution interrupted === continue # Check is method not removed elif kbxMethodStatus not in (SharedMethod.METHOD_STATUS_ACTIVE, SharedMethod.METHOD_STATUS_INACTIVE): seri["seriStatus"] = "error" seri["seriError"] = "method is removed" continue kwargs = {methodParam[AppConstants.ARG_NAME]: methodParam[AppConstants.ARG_CURRENT_VALUE] for methodParam in methodParamsWithCurrentValues} if AppInfo.REQUEST_KEY_LANGUAGE not in kwargs: kwargs[AppInfo.REQUEST_KEY_LANGUAGE] = AppInfo.DEFAULT_API_LANGUAGE kwargs["kbxMethodName"] = kbxMethodName kwargs["kbxGroupId"] = kbxGroupId kwargs["kbxModuleName"] = row["kbxModuleName"] kwargs["kbxMethodAppId"] = row["kbxMethodAppId"] kwargs[AppConstants.KEY_ACTION_TIMESTAMP] = methodExecTime #=========================================================== # Execute method #=========================================================== sceneThreadEvent.clear() execThread = threading.Thread(target=execution_func, args=[sceneThreadEvent, row["kbxMethodId"], seri], kwargs=kwargs) execThread.daemon = False execThread.start() sceneThreadEvent.wait() # Event will be set by "stop_scene" or SharedMethod.call returns. # Check is stop if sceneExecLock.is_stop(): if not isLoopCompleted: serEndTime = time.time() isLoopCompleted = False # === Execution interrupted === continue if isLoopCompleted: # Record for debugging purpose serEndTime = time.time() # === Execution completed === except Exception as e: # This is unexpected error. Execution will not be recorded by Scene Execution Result. Logger.log_error("SceneService.__execute_scene_implementation ex:", e) # === Execution completed with errors === else: if serObj is not None: # Log to sceneExecutionResultService self.__sceneExecutionResultThreadPool.submit(self.__add_scene_execution_result, serUrl=serObj["serUrl"], serStartTime=serStartTime, serEndTime=serEndTime, sceneName=serObj["sceneName"], sceneProtected=serObj["sceneProtected"], sceneId=sceneId, execution=seris, language=language) finally: del(self.__sceneExecLocks[sceneId]) # Delete exec lock sceneExecLock.release() def __add_scene_execution_result(self, serUrl, serStartTime, serEndTime, sceneName, sceneProtected, sceneId, execution, language): try: SceneExecutionResultService.instance().add_scene_execution_result(serUrl=serUrl, serStartTime=serStartTime, serEndTime=serEndTime, sceneName=sceneName, sceneProtected=sceneProtected, sceneId=sceneId, execution=execution, language=language) except Exception as e: Logger.log_error("SceneService.__add_scene_execution_result ex:", e)
def do_test3(workers): param = {"max_workers": workers} loop = asyncio.new_event_loop() lock = threading.Lock() tresult = [] presult = [] cresult = [] pre_input1 = input_generator(workers, 0) pre_input2 = input_generator(workers, max(pre_input1)) pre_input3 = input_generator(workers, max(pre_input2)) def result_checker(list, lock, fut): with lock: try: list.append(fut.result()) except Exception as e: list.append(e) texec = ThreadPoolExecutor(**param) pexec = ProcessPoolExecutor(**param) cexec = CoroutinePoolExecutor(**param, loop=loop) tstart = round(time.time()+1) input1 = [tstart + i for i in pre_input1] input2 = [tstart + i for i in pre_input2] input3 = [tstart + i for i in pre_input3] for x in input1: future = texec.submit(wake_at, x) future.add_done_callback( functools.partial(result_checker, tresult, lock)) result_iter = texec.map(wake_at, input2) for x in input3: future = texec.submit(wake_at, x) future.add_done_callback( functools.partial(result_checker, tresult, lock)) for x in result_iter: with lock: tresult.append(x) texec.shutdown(True) pstart = round(time.time() + _start_warm_up) input1 = [pstart + i for i in pre_input1] input2 = [pstart + i for i in pre_input2] input3 = [pstart + i for i in pre_input3] for x in input1: future = pexec.submit(wake_at, x) future.add_done_callback( functools.partial(result_checker, presult, lock)) result_iter = pexec.map(wake_at, input2) for x in input3: future = pexec.submit(wake_at, x) future.add_done_callback( functools.partial(result_checker, presult, lock)) for x in result_iter: with lock: presult.append(x) pexec.shutdown(True) cstart = round(time.time() + _start_warm_up) input1 = [cstart + i for i in pre_input1] input2 = [cstart + i for i in pre_input2] input3 = [cstart + i for i in pre_input3] async def async_main(): for x in input1: future = cexec.submit(async_wake_at, x) future.add_done_callback( functools.partial(result_checker, cresult, lock)) result_iter = cexec.map(async_wake_at, input2) for x in input3: future = cexec.submit(async_wake_at, x) future.add_done_callback( functools.partial(result_checker, cresult, lock)) async for x in result_iter: with lock: cresult.append(x) await cexec.shutdown(False) loop.run_until_complete(async_main()) try: loop.run_until_complete(cexec.shutdown(True)) texec.shutdown(True) pexec.shutdown(True) finally: loop.close() tresult = [round((x - tstart) / _precision) for x in tresult] presult = [round((x - pstart) / _precision) for x in presult] cresult = [round((x - cstart) / _precision) for x in cresult] result = True for (t, p, c) in zip(tresult, presult, cresult): result = result and (t == p) if not result: print(tresult) print(presult) print(cresult) print(t,p,c) assert False result = result and (p == c) if not result: print(tresult) print(presult) print(cresult) print(t, p, c) assert False result = result and (c == t) if not result: print(tresult) print(presult) print(cresult) print(t, p, c) assert False return result
def do_test1(workers): param = {"max_workers": workers} start = round(time.time() + _start_warm_up) input = input_generator(workers, start) loop = asyncio.new_event_loop() lock = threading.Lock() tresult = [] presult = [] cresult = [] def result_checker(list, lock, fut): with lock: try: list.append(fut.result()) except Exception as e: list.append(e) texec = ThreadPoolExecutor(**param) pexec = ProcessPoolExecutor(**param) cexec = CoroutinePoolExecutor(**param, loop=loop) for x in input: future = texec.submit(wake_at, x) future.add_done_callback( functools.partial(result_checker, tresult, lock)) future = pexec.submit(wake_at, x) future.add_done_callback( functools.partial(result_checker, presult, lock)) future = cexec.submit(async_wake_at, x) future.add_done_callback( functools.partial(result_checker, cresult, lock)) texec.shutdown(False) pexec.shutdown(False) loop.run_until_complete(cexec.shutdown(False)) try: loop.run_until_complete(cexec.shutdown(True)) texec.shutdown(True) pexec.shutdown(True) finally: loop.close() tresult = [round((x - start) / _precision) for x in tresult] presult = [round((x - start) / _precision) for x in presult] cresult = [round((x - start) / _precision) for x in cresult] result = True for (t, p, c) in zip(tresult, presult, cresult): result = result and (t == p) if not result: print(tresult) print(presult) print(cresult) print(t, p, c) assert False result = result and (p == c) if not result: print(tresult) print(presult) print(cresult) print(t, p, c) assert False result = result and (c == t) if not result: print(tresult) print(presult) print(cresult) print(t, p, c) assert False return result
class AlignedSEMStream(SEMStream): """ This is a special SEM stream which automatically first aligns with the CCD (using spot alignment) every time the stage position changes. Alignment correction can either be done via beam shift (=shift), or by just updating the image position. """ def __init__(self, name, detector, dataflow, emitter, ccd, stage, focus, shiftebeam=MTD_MD_UPD, **kwargs): """ shiftebeam (MTD_*): if MTD_EBEAM_SHIFT, will correct the SEM position using beam shift (iow, using emitter.shift). If MTD_MD_UPD, it will just update the position correction metadata on the SEM images. ccd (Optical detector) stage (actuator): the sample stage, just to know when re-alignment is needed focus (actuator): the _optical_ focuser, just to know when re-alignment is needed focuser (actuator): the _e-beam_ focuser, to allow focusing the image """ super(AlignedSEMStream, self).__init__(name, detector, dataflow, emitter, **kwargs) self._ccd = ccd self._stage = stage self._focus = focus self._shiftebeam = shiftebeam self.calibrated = model.BooleanVA(False) # whether the calibration has been already done self._last_pos = stage.position.value.copy() self._last_pos.update(focus.position.value) # last known position of the stage self._shift = (0, 0) # (float, float): shift to apply in meters self._last_shift = (0, 0) # (float, float): last ebeam shift applied # In case initialization takes place in unload position the # calibration values are not obtained yet. Thus we avoid to initialize # cur_trans before spot alignment takes place. self._cur_trans = None stage.position.subscribe(self._onMove) focus.position.subscribe(self._onMove) self._executor = ThreadPoolExecutor(max_workers=1) self._beamshift = None def _onMove(self, pos): """ Called when the stage moves (changes position) pos (dict): new position """ # Check if the position has really changed, as some stage tend to # report "new" position even when no actual move has happened logging.debug("Stage location is %s m,m,m", pos) if self._last_pos == pos: return self._last_pos.update(pos) # if self.is_active.value: self.calibrated.value = False # just reset status self._setStatus(None) # need to override it to support beam shift def _applyROI(self): """ Update the scanning area of the SEM according to the roi """ res, shift = self._computeROISettings(self.roi.value) if (self._shiftebeam == MTD_EBEAM_SHIFT) and (self._beamshift is not None): shift = tuple(s + c for s, c in zip(shift, self._beamshift)) # always in this order self._emitter.resolution.value = res self._emitter.shift.value = shift def _compensateShift(self): """ Compensate the SEM shift, using either beam shift or metadata update """ # update the correction metadata logging.debug("Update metadata for SEM image shift") self._detector.updateMetadata({MD_POS_COR: self._shift}) def _prepare(self): """ Perform calibration if needed """ logging.debug("Preparing stream %s ...", self) # actually indicate that preparation has been triggered, don't wait for # it to be completed self._prepared = True f = self._executor.submit(self._DoPrepare) # Note that there is no need to call super(). This would only check # for an optical path manager which in this case has no effect. return f def __del__(self): self._executor.shutdown(wait=False) def _DoPrepare(self): # Need to calibrate ? if not self.calibrated.value: self._setStatus(logging.INFO, u"Automatic SEM alignment in progress…") # store current settings no_spot_settings = (self._emitter.dwellTime.value, self._emitter.resolution.value) # Don't mess up with un/subscribing while doing the calibration self._getEmitterVA("dwellTime").unsubscribe(self._onDwellTime) self._getEmitterVA("resolution").unsubscribe(self._onResolution) shift = (0, 0) self._beamshift = None try: logging.info("Determining the Ebeam center position") # TODO Handle cases where current beam shift is larger than # current limit. Happens when accel. voltage is changed self._emitter.shift.value = (0, 0) shift = FindEbeamCenter(self._ccd, self._detector, self._emitter) logging.debug("Spot shift is %s m,m", shift) self._beamshift = shift # Also update the last beam shift in order to be used for stage # offset correction in the next stage moves self._last_shift = (0.75 * self._last_shift[0] - 0.25 * shift[0], 0.75 * self._last_shift[1] - 0.25 * shift[1]) cur_trans = self._stage.getMetadata().get(model.MD_POS_COR, (0, 0)) self._cur_trans = (cur_trans[0] - self._last_shift[0], cur_trans[1] - self._last_shift[1]) self._stage.updateMetadata({ model.MD_POS_COR: self._cur_trans }) logging.debug("Compensated stage translation %s m,m", self._cur_trans) if self._shiftebeam == MTD_EBEAM_SHIFT: # First align using shift self._applyROI() # Then by updating the metadata shift = (0, 0) # just in case of failure shift = FindEbeamCenter(self._ccd, self._detector, self._emitter) elif self._shiftebeam == MTD_MD_UPD: pass else: raise NotImplementedError("Unknown shiftbeam method %s" % (self._shiftebeam,)) except LookupError: self._setStatus(logging.WARNING, (u"Automatic SEM alignment unsuccessful", u"Need to focus all streams")) # logging.warning("Failed to locate the ebeam center, SEM image will not be aligned") except Exception: self._setStatus(logging.WARNING, (u"Automatic SEM alignment unsuccessful", u"Need to focus all streams")) logging.exception("Failure while looking for the ebeam center") else: self._setStatus(None) logging.info("Aligning SEM image using shift of %s", shift) self.calibrated.value = True finally: # restore hw settings (self._emitter.dwellTime.value, self._emitter.resolution.value) = no_spot_settings self._getEmitterVA("dwellTime").subscribe(self._onDwellTime) self._getEmitterVA("resolution").subscribe(self._onResolution) self._shift = shift self._compensateShift()
class Bender(object): def __init__(self, backbone, brain=None): self._backbone = backbone self._brain = brain if brain is not None else Brain() self._brain_lock = threading.Lock() self._regex_to_response = OrderedDict() self._scripts = OrderedDict() self._pool = ThreadPoolExecutor(max_workers=4) self._futures = [] # list of futures submitted to the pool self._stop_loop = threading.Event() def register_script(self, name, script): self._scripts[name] = script def register_builtin_scripts(self): for name, script in scripts.get_builtin_scripts(): self.register_script(name, script) def register_setuptools_scripts(self): for p in pkg_resources.iter_entry_points('bender_script'): obj = p.load() if inspect.isclass(obj): obj = obj() self.register_script(p.name, obj) def get_script(self, name): return self._scripts[name] def iter_scripts(self): return iter(self._scripts.items()) def start(self): self._brain.load() self._backbone.on_message_received = self.on_message_received self.register_builtin_scripts() self.register_setuptools_scripts() for script in self._scripts.values(): hooks.call_unique_hook(script, 'script_initialize_hook', brain=self._brain) hooks.call_unique_hook(self._backbone, 'backbone_start_hook') def shutdown(self): self._pool.shutdown(wait=True) for name, script in list(self._scripts.items()): self._scripts.pop(name) hooks.call_unique_hook(script, 'script_shutdown_hook', brain=self._brain) hooks.call_unique_hook(self._backbone, 'backbone_shutdown_hook', brain=self._brain) self._brain.dump() self._stop_loop.set() def request_shutdown(self): self._stop_loop.set() def loop(self): self.start() self._stop_loop.wait() self.shutdown() def on_message_received(self, msg): def thread_exec(hook, brain, msg, match): try: hooks.call(hook, brain=self._brain, msg=msg, match=match, bender=self) except Exception as e: msg.reply('*BZZT* %s' % e) else: with self._brain_lock: brain.dump() handled = False for script in self._scripts.values(): for hook in hooks.find_hooks(script, 'respond_hook'): match = re.match(hook.inputs['regex'], msg.get_body(), re.IGNORECASE | re.DOTALL) if match: f = self._pool.submit(thread_exec, hook, self._brain, msg, match) self._futures.append(f) handled = True if not handled: msg.reply('Command not recognized') def wait_all_messages(self): while self._futures: f = self._futures.pop() f.result() # wait until future returns
def submit(self, fn, *args, **kwargs): if isinstance(fn, Task): self._task_map[id(fn)] = fn return ThreadPoolExecutor.submit(self, fn, *args, **kwargs)
class SceneExecutionResultService: __INSTANCE = None __LOCK = threading.Lock() @staticmethod def instance(): with SceneExecutionResultService.__LOCK: SceneExecutionResultService.__INSTANCE or SceneExecutionResultService() return SceneExecutionResultService.__INSTANCE def __init__(self): self.__lock = threading.Lock() self.__serLock = SERLock() self.__serController = SceneExecutionResultController() self.__serRetryThreadPool = ThreadPoolExecutor(max_workers=3) def after_ser_deleted(serId): self.__serLock.declare_as_deleted(serId) # Report serId is deleted to SERLock instance. self.__broadcast_message__ser_deleted(serId) # Broadcast event to mobile client. SceneExecutionResultDataHandler.AFTER_RECORD_DELETE = after_ser_deleted SceneExecutionResultService.__INSTANCE = self def add_scene_execution_result(self, serUrl, serStartTime, serEndTime, sceneName, sceneProtected, sceneId, execution, language="en"): ''' Get kbxGroupLabel for each execution item and pass to serController. ''' # Get kbxGroupLabel for item in execution: kbxGroupId = item.pop("kbxGroupId", None) if kbxGroupId is None: item["kbxGroupLabel"] = "-- Unknown --" else: try: kbxGroup = SharedMethodWrapper.get_shared_method_group_by_id(kbxGroupId, enableTagCount=False, language=language) item["kbxGroupLabel"] = kbxGroup.get("kbxGroupLabel", "-- Unnamed --") except: item["kbxGroupLabel"] = "-- Removed --" with self.__lock: # Add scene execution result happens sequentially. # Make serCreatedTime unique. # This is the smallest pause unit to make a difference on each time.time() call. time.sleep(0.001) serCreatedTime = time.time() serId = str(serCreatedTime) self.__serController.add(serId, serCreatedTime, serStartTime, serEndTime, sceneId, sceneName, sceneProtected, execution) # Build notification information. KBXLang.set_preferred_lang(language) notiLink = str(serUrl) + "?serId=" + str(serId) notiLinkLabel = str(KBXLang("ser_noti_link_label")) for seri in execution: if seri["seriStatus"] == "error": notiContent = str(KBXLang("ser_noti_exec_completed_with_error")) break else: notiContent = str(KBXLang("ser_noti_exec_completed")) notiContent = str(sceneName).join(notiContent.split(":scenename:")) self.__notify_ser_added(notiContent, notiLink, notiLinkLabel) def get_scene_execution_result(self, serId, language): ''' Get complete serObject from serController, and add in full details obtained from shared method manager. ''' try: serObject = self.__serController.get_ser_and_seris(serId) except: raise AutomationException(12000, "Scene Execution Result not found, ID provided:" + str(serId)) execution = serObject["execution"] # Add details into each execution item. for kbxMethod in execution: kbxMethodId = kbxMethod["kbxMethodId"] try: kbxMethodWithDetails = SharedMethodWrapper.get_shared_method_by_id(kbxMethodId, language) kbxMethod["kbxMethodLabel"] = kbxMethodWithDetails.get("kbxMethodLabel") # Rebuild parameters. paramsWithDetails = {} for paramWithDetails in kbxMethodWithDetails.get("kbxMethodParams", None): paramsWithDetails[paramWithDetails["kbxParamName"]] = paramWithDetails for kbxMethodParam in kbxMethod["kbxMethodParams"]: try: # Update kbxMethodParam with the one with details. kbxParamCurrentValue = kbxMethodParam["kbxParamCurrentValue"] kbxMethodParam.update(paramsWithDetails[kbxMethodParam["kbxParamName"]]) kbxMethodParam["kbxParamCurrentValue"] = kbxParamCurrentValue # But remains current value no matter what. except Exception as e: kbxMethodParam["atDebug"] = "Unable to match parameter: " + str(e) continue except Exception as e: kbxMethod["atDebug"] = "Unable to get shared method:" + str(e) continue return serObject def retry_scene_execution_result_item(self, serId, seriIndex=None): ''' Retry all scene execution result item with error according seriIndex Each ser can be in retry mode one at a time. If an item is deleted during execution, execution progress will be completed without event broadcasted. ''' # Lock scene execution result for retry. try: self.__serLock.lock(serId) except Exception: raise AutomationException(12004, "Scene execution result is retrying.") try: if Util.is_empty(seriIndex): # Retry all items with error. serisWithError = self.__serController.list_seris_with_error(serId) if len(serisWithError) == 0: raise AutomationException(12002, "We've found no scene execution result item with error to retry") else: serisWithError = [dict(seriWithError) for seriWithError in serisWithError] else: seriWithError = self.__serController.get_seri_by_index(serId, seriIndex) if seriWithError is None: raise AutomationException(12001, "scene execution result item not found, serId: " + str(serId) + " seriIndex: " + str(seriIndex)) elif seriWithError["seriStatus"] != "error": raise AutomationException(12003, "scene execution result item not error, current status: " + str(seriWithError["seriStatus"])) serisWithError = [seriWithError] except Exception as e: # Unlock scene execution after retry. self.__serLock.unlock(serId) raise e else: seriIndexes = [seriWithError["seriIndex"] for seriWithError in serisWithError] self.__serRetryThreadPool.submit(self.__retry_seri_implementation, serId, serisWithError, seriIndexes) return seriIndexes def __retry_seri_implementation(self, serId, serisWithError, seriIndexes): try: # Update statuses of all items to be retried to "busy". self.__serController.update_seri_status(serId=serId, seriIndexes=seriIndexes, seriStatus="busy", seriError=None) # Broadcast event. self.__broadcast_message__seri_retry_started(serId, seriIndexes=seriIndexes) # Retry process starts here. for seri in serisWithError: try: kbxMethodId = seri["kbxMethodId"] if kbxMethodId != -291: # Skips all -291 and declare as "ok" immediately. params = {kbxMethodParam["kbxParamName"]:kbxMethodParam["kbxParamCurrentValue"] for kbxMethodParam in seri["kbxMethodParams"]} result = SharedMethod.call_by_method_id(kbxMethodId, **params) seriError = str(result) else: seriError = None seriStatus = "ok" except Exception as e: seriStatus = "error" seriError = str(e) finally: seriIndex = seri["seriIndex"] if not self.__serLock.is_deleted(serId): # Update statuses and broadcast events only if ser is not deleted. self.__serController.update_seri_status(serId=serId, seriIndexes=[seriIndex], seriStatus=seriStatus, seriError=seriError) self.__broadcast_message__seri_retry_completed(serId, seriIndex, seriStatus, seriError) except Exception as e: ''' THIS PORTION SHOULD NEVER RUN. (it's bug if this portion is executed) ''' Logger.log_error("SceneExecutionResultService.retry_scene_execution_result_item ex:", e) finally: self.__serLock.unlock(serId) def list_scene_execution_results(self, offset, limit): ''' List all scene execution results without items. ''' return self.__serController.list(offset, limit) def delete_scene_execution_result(self, serId): ''' Delete a scene execution result. Relevant events will be broadcasted. NOTE: scene execution result records will be maintained automatically to ensure only 10 records in database at most. ''' self.__serController.delete(serId) def __notify_ser_added(self, notiContent, notiLink, notiLinkLabel): Logger.log_debug("Scene Execution Result logged at:", notiLink) NotificationManagerService.dispatch_notification(text=notiContent, link=notiLink, linkLabel=notiLinkLabel) def __broadcast_message__ser_deleted(self, serId): eventTag = AppConstants.EVENT_SER_DELETED eventData = {"serId":serId} self.__broadcast_message(eventTag, eventData) def __broadcast_message__seri_retry_started(self, serId, seriIndexes): eventTag = AppConstants.EVENT_SERI_RETRY_STARTED eventData = {"serId":serId, "seriIndexes":seriIndexes} self.__broadcast_message(eventTag, eventData) def __broadcast_message__seri_retry_completed(self, serId, seriIndex, seriStatus, seriError): eventTag = AppConstants.EVENT_SERI_RETRY_COMPLETED eventData = {"serId":serId, "seriIndex":seriIndex, "seriStatus":seriStatus, "seriError":seriError} self.__broadcast_message(eventTag, eventData) def __broadcast_message(self, eventTag, eventData): eventData = json.dumps(eventData, cls=AutomationJSONEncoder) Application.send_web_server_event(eventTag, eventData)
class RuleService: def __init__(self): # Rule processors. self.__ruleController = RuleController() self.__methodController = MethodController() self.__triggerController = TriggerController.instance() self.__ruleUpdateThreadPool = ThreadPoolExecutor(max_workers=1) self.__ruleExecThreadPool = ThreadPoolExecutor(max_workers=AppConstants.MAX_RULE_EXEC_THREAD_SIZE) # Rule run workers. self.__ruleExecInfos = {} self.__condCallGroup = MethodCallGroup() self.__execCallGroup = MethodCallGroup() # Listeners. self.__ruleController.listen_to_rule_status_change(self.__on_rule_status_changed) GroupController.instance().listen_to_group_icon_change(self.__on_group_icon_changed) self.__methodController.listen_to_method_status_change(self.__on_method_status_changed) EventController.instance().listen_to_event_callback(self.__on_method_event_callback) self.__triggerController.listen_to_trigger_callback(self.__on_trigger_callback) def __on_rule_status_changed(self, ruleId, oldEnabled, newEnabled, oldStatusProcessed, newStatusProcessed): ''' Trigger Source: RuleController --> This Callback when a rule is re-enabled OR statusProcessed changed to "updated". ''' if newEnabled == True and newStatusProcessed == AppConstants.RULE_STATUS_UPDATED: if oldEnabled != newEnabled or oldStatusProcessed != newStatusProcessed: self.__ruleExecThreadPool.submit(self.__trigger_rule_implementation, ruleId=ruleId, checkCondition=True) def __on_group_icon_changed(self, kbxGroupId): ''' Trigger Source: GroupController --> This Callback when kbxGroupIcon changed. ''' ruleIdsFromCond = self.__ruleController.list_rule_ids_which_has_kbx_group_id_as_condition(kbxGroupId) ruleIdsFromExec = self.__ruleController.list_rule_ids_which_has_kbx_group_id_as_execution(kbxGroupId) # Broadcast rules updated messages. for ruleId in set(ruleIdsFromCond + ruleIdsFromExec): self.__broadcast_message__rule_updated(ruleId) def __on_method_status_changed(self, kbxMethodId, oldKBXMethodStatus, newKBXMethodStatus): ''' Trigger Source: MethodController --> This Callback when kbxMethodStatus changed. ''' if oldKBXMethodStatus != newKBXMethodStatus: ruleIdsFromCond = self.__ruleController.list_rule_ids_which_has_kbx_method_id_as_condition(kbxMethodId) ruleIdsFromExec = self.__ruleController.list_rule_ids_which_has_kbx_method_id_as_execution(kbxMethodId) # # Executes rules with conditions affected. # if newKBXMethodStatus == SharedMethod.METHOD_STATUS_ACTIVE: # for ruleId in ruleIdsFromCond: # self.__ruleExecThreadPool.submit(self.__trigger_rule_implementation, ruleId=ruleId, checkCondition=True) # Broadcast rules updated messages. for ruleId in set(ruleIdsFromCond + ruleIdsFromExec): self.__broadcast_message__rule_updated(ruleId) def __on_method_event_callback(self, kbxMethodId, eventTag, eventData): ''' Trigger Source: EventController --> MethodController --> This Callback when a method with event broadcasted event. ''' ruleIds = self.__ruleController.list_rule_ids_which_has_kbx_method_id_as_condition(kbxMethodId) for ruleId in ruleIds: self.__ruleExecThreadPool.submit(self.__trigger_rule_implementation, ruleId=ruleId, checkCondition=True, eventTag=eventTag, eventData=eventData, eventMethodId=kbxMethodId) def __on_trigger_callback(self, ruleId): ''' Trigger Source: TriggerController --> This Callback when a rule is triggered. ''' self.__ruleExecThreadPool.submit(self.__trigger_rule_implementation, ruleId=ruleId, checkCondition=True) def set_rule(self, trigger, condition, execution, ruleId=None, ruleName=None, ruleProtected=False, enabled=True): ''' Create/Edit(with ruleId provided) an existing rule. trigger:Dictionary condition:List execution:List ruleId:Integer <Optional> ruleName:String <Optional> ruleProtected:Boolean <Optional> enabled:Boolean Returns "ruleId" ''' def process_method_list(methodList): #=================================================================== # Basic type validation #=================================================================== if not isinstance(methodList, list): Logger.log_error("RuleService.set_rule: 'condition' and 'execution' must be type of list.") Logger.log_debug("type:", type(methodList), "value:", methodList) raise AutomationException(11704, "List is required for both 'condition' and 'execution'") #=================================================================== # Check allowed size, raise error if exceeded. #=================================================================== methodListLen = len(methodList) if methodListLen > AppConstants.MAX_METHOD_SIZE: Logger.log_error("RuleService.set_rule: 'condition' and 'execution' cannot have more than", AppConstants.MAX_METHOD_SIZE, "items respectively.") raise AutomationException(11705, "Only a maximum of " + \ str(AppConstants.MAX_METHOD_SIZE) + \ " items is allowed for each 'condition' and 'execution' - given size " + \ str(methodListLen), lambda text: str(AppConstants.MAX_METHOD_SIZE).join(text.split(":max_item_size:"))) #=================================================================== # Check if all kbxMethodIds are valid and all kbxMethodParams are list #=================================================================== idValidator = NumberValidator(isRequired=True, decimalPoint=False) if not all([idValidator.is_valid(eachMethod["kbxMethodId"]) and isinstance(eachMethod["kbxMethodParams"], list) for eachMethod in methodList]): raise AutomationException(11704, "'condition' and 'execution' have incorrect data structure.") #=================================================================== # Check if all kbxParamName and kbxParamCurrentValue exists #=================================================================== paramNameValidator = StringValidator(isRequired=True) for eachMethod in methodList: methodArgs = eachMethod["kbxMethodParams"] for methodArg in methodArgs: if not paramNameValidator.is_valid(methodArg[AppConstants.ARG_NAME]): raise AutomationException(11704, "'condition' and 'execution' have invalid params structure") if not AppConstants.ARG_CURRENT_VALUE in methodArg: methodArg[AppConstants.ARG_CURRENT_VALUE] = None return methodList #======================================================================= # Data structure validations #======================================================================= ruleId = NumberValidator(isRequired=False, decimalPoint=False).get_value(ruleId) triggerDTO = self.__triggerController.parse_to_trigger_dto(trigger) condition = process_method_list(condition) execution = process_method_list(execution) #======================================================================= # Add to database #======================================================================= if Util.is_empty(ruleId): # Validate_max_rule_size if self.__ruleController.count() >= AppConstants.MAX_RULE_SIZE: raise AutomationException(11706, "Total amount of rules cannot be more than " + str(AppConstants.MAX_RULE_SIZE), lambda text: str(AppConstants.MAX_RULE_SIZE).join(text.split(":max_rule_size:"))) ruleId = self.__ruleController.generate_id(ruleName) rule = {} elif self.__ruleController.has(ruleId): ruleFromDB = self.__ruleController.get(ruleId) rule = dict(ruleFromDB) self.__check_rule_process_status(ruleId) self.__ruleController.change_to_updating(ruleId, ruleName) else: raise AutomationException(11704, "Rule ID provided not found - " + str(ruleId)) #======================================================================= # Broadcast message: starts to update rule. #======================================================================= self.__broadcast_message__rule_update_started(ruleId, ruleName) #======================================================================= # Set basic information of the rule #======================================================================= rule["ruleId"] = ruleId rule["ruleName"] = ruleName rule["ruleProtected"] = ruleProtected rule["trigger"] = triggerDTO rule["enabled"] = enabled rule["condition"] = condition rule["execution"] = execution #======================================================================= # Update rule #======================================================================= def __update_rule(rule): try: # Fire rule update start event ruleId = rule["ruleId"] # Add methods to subscribe list methodIds = [kbxMethod["kbxMethodId"] for kbxMethod in rule["condition"] + rule["execution"]] self.__methodController.add(methodIds) # Update "rule" base table self.__ruleController.update(rule) self.__ruleController.commit() except Exception as e: self.__ruleController.rollback() self.__broadcast_message__rule_update_failed(ruleId, ruleName) Logger.log_error("RuleService __update_rule failed:", e, "-- rolledback") else: self.__triggerController.register_listener(ruleId, rule["trigger"]) # Process for Timer Module TimerModule.delete_scheduler(ruleId) timerModuleHandlers = {TimerModule.METHOD_ID_DATE_TIME_RANGE:TimerModule.handle_date_time_range, TimerModule.METHOD_ID_DAY_OF_WEEK:TimerModule.handle_dow, TimerModule.METHOD_ID_TIME_RANGE:TimerModule.handle_time_range} for kbxMethod in rule["condition"]: kbxMethodId = kbxMethod["kbxMethodId"] timerModuleHandler = timerModuleHandlers.get(kbxMethodId, None) if timerModuleHandler is not None: timerModuleHandler(ruleId, kbxMethod["kbxMethodParams"]) # Broadcast message: completed updating a rule self.__broadcast_message__rule_updated(ruleId) #======================================================================= # Submit to a thread to process other info, and return... performance... #======================================================================= self.__ruleUpdateThreadPool.submit(__update_rule, rule) def delete_rule(self, ruleId): self.__check_rule_process_status(ruleId) try: self.__ruleController.delete(ruleId) self.__ruleController.commit() except Exception as e: self.__ruleController.rollback() Logger.log_error("RuleService delete_rule ex:", e, "-- rolled back") else: self.__broadcast_message__rule_deleted(ruleId) self.__triggerController.unregister_listener(ruleId) TimerModule.delete_scheduler(ruleId) def trigger_rule(self, ruleId, checkCondition=False): ''' self.__check_rule_process_status(ruleId) <-- Check again in self.__trigger_rule_implementation. ''' self.__trigger_rule_implementation(ruleId=ruleId, checkCondition=checkCondition) def enable_rule(self, ruleId, enabled): self.__check_rule_process_status(ruleId) try: self.__ruleController.enable(ruleId, enabled) self.__ruleController.commit() except Exception as e: self.__ruleController.rollback() Logger.log_error("RuleService enable_rule ex:", e, "-- rolled back") else: self.__broadcast_message__rule_updated(ruleId) def get_rule(self, ruleId, language=AppInfo.DEFAULT_API_LANGUAGE): try: rule = self.__ruleController.get_detail(ruleId) except: raise AutomationException(11702, "Rule ID provided not found - " + str(ruleId)) kbxMethods = list(rule["condition"]) + list(rule["execution"]) # -------------- Compile lists of kbxMethod and group IDs contains in this rule. kbxMethodIdsToList = {} kbxGroupIdsToList = set([]) for kbxMethod in kbxMethods: # Variables kbxMethodId = kbxMethod["kbxMethodId"] kbxMethodAppId = kbxMethod["kbxMethodAppId"] kbxMethodStatus = kbxMethod["kbxMethodStatus"] kbxGroupId = kbxMethod["kbxGroupId"] kbxGroupStatus = kbxMethod["kbxGroupStatus"] if kbxMethodStatus is not -1 and kbxMethodAppId is not None: kbxMethodIdsToList.setdefault(kbxMethodAppId, set([])) kbxMethodIdsToList[kbxMethodAppId].add(kbxMethodId) if kbxGroupId is not None and kbxGroupStatus is not -1: kbxGroupIdsToList.add(kbxGroupId) # -------------- Get methods and groups based on requested language. kbxMethodIdsListed = {} kbxGroupIdsListed = {} for kbxMethodAppId, kbxMethodIds in kbxMethodIdsToList.items(): kbxMethodIdsListed[kbxMethodAppId] = SharedMethodWrapper.list_shared_methods_by_app_id(kbxMethodAppId, list(kbxMethodIds), language=language) groupList = SharedMethodWrapper.list_shared_method_groups(kbxGroupId=kbxGroupIdsToList, language=language) for row in groupList: kbxGroupIdsListed[row["kbxGroupId"]] = row # -------------- Set method and group data into rule. for kbxMethod in kbxMethods: # Variables kbxMethodId = kbxMethod["kbxMethodId"] kbxMethodAppId = kbxMethod["kbxMethodAppId"] kbxMethodStatus = kbxMethod["kbxMethodStatus"] kbxGroupId = kbxMethod["kbxGroupId"] kbxGroupStatus = kbxMethod["kbxGroupStatus"] if kbxMethodStatus is not -1 and kbxMethodAppId is not None: kbxMethodParamsWithCurrentValue = {kbxMethodParam["kbxParamName"]:kbxMethodParam["kbxParamCurrentValue"] \ for kbxMethodParam in kbxMethod["kbxMethodParams"]} kbxMethodWithDetails = kbxMethodIdsListed[kbxMethodAppId][kbxMethodId] if kbxMethodWithDetails is not None: kbxMethodParamsWithDetails = kbxMethodWithDetails["kbxMethodParams"] kbxMethodParamsWithDetails = copy.deepcopy(kbxMethodParamsWithDetails) for kbxMethodParam in kbxMethodParamsWithDetails: kbxMethodParam["kbxParamCurrentValue"] = kbxMethodParamsWithCurrentValue.get(kbxMethodParam["kbxParamName"], None) kbxMethod["kbxMethodParams"] = kbxMethodParamsWithDetails kbxMethod["kbxMethodHasEvent"] = not Util.is_empty(kbxMethodWithDetails.get("kbxMethodEvent", None)) \ and not Util.is_empty(kbxMethodWithDetails.get("kbxMethodIdentifier", None)) kbxMethod["kbxMethodLabel"] = kbxMethodWithDetails.get("kbxMethodLabel") kbxMethod["kbxMethodDesc"] = kbxMethodWithDetails.get("kbxMethodDesc") else: kbxMethod["atDebugMethod"] = "Unable to get shared method, caused by a method which never register itself on this bootup." else: kbxMethod["kbxMethodHasEvent"] = False if kbxGroupId is not None and kbxGroupStatus is not -1: try: kbxMethod["kbxGroupLabel"] = kbxGroupIdsListed[kbxGroupId]["kbxGroupLabel"] kbxMethod["kbxGroupDesc"] = kbxGroupIdsListed[kbxGroupId]["kbxGroupDesc"] except: kbxMethod["atDebugGroup"] = "Unable to get shared method group, caused by a group which never register itself on this bootup." return rule def list_rules(self, offset=0, limit=20): return self.__ruleController.list(offset, limit), \ self.__ruleController.count() def run_all_enabled_rules(self): ''' Add the following 2 lines of code at AutomationModuleWrapper.py - start(), after last statement, to enable run all rules on bootup. # Logger.log_info("Attempts to execute all enabled rules ...") # self.__ruleService.run_all_enabled_rules() ''' ruleIds = self.__ruleController.list_rule_ids_which_are_enabled() for ruleId in ruleIds: self.__ruleExecThreadPool.submit(self.__trigger_rule_implementation, ruleId=ruleId, checkCondition=True) def __check_rule_process_status(self, ruleId): try: statusProcessed = self.__ruleController.get_status_processed(ruleId) if statusProcessed != AppConstants.RULE_STATUS_UPDATED: raise AutomationException(11703, "edit/delete/execute is not allowed on rule update in progress") except: raise AutomationException(11702, "Rule ID provided not found - " + str(ruleId)) def __broadcast_message__rule_update_started(self, ruleId, ruleName=None): eventTag = AppConstants.EVENT_RULE_UPDATE_STARTED eventData = {"ruleId":ruleId, "newRuleName":ruleName} self.__broadcast_message(eventTag, eventData) Logger.log_info("Rule Start Update:", ruleName) def __broadcast_message__rule_updated(self, ruleId): try: rule = self.__ruleController.get_summary(ruleId) except Exception as e: Logger.log_error("RuleService.__broadcast_message__rule_updated get_summary ex:", e) return eventTag = AppConstants.EVENT_RULE_UPDATED eventData = rule self.__broadcast_message(eventTag, eventData) Logger.log_info("Rule Updated:", rule["ruleName"]) def __broadcast_message__rule_update_failed(self, ruleId, ruleName=None): ''' ruleName - For debugging purpose. ''' try: rule = self.__ruleController.get_summary(ruleId) except Exception: rule = None eventTag = AppConstants.EVENT_RULE_UPDATE_FAILED eventData = {"ruleId": ruleId, "oldRuleSummary":rule} self.__broadcast_message(eventTag, eventData) Logger.log_info("Rule Update Failed:", ruleName) def __broadcast_message__rule_deleted(self, ruleId): eventTag = AppConstants.EVENT_RULE_DELETED eventData = {"ruleId": ruleId} self.__broadcast_message(eventTag, eventData) Logger.log_info("Rule Deleted: Id -", ruleId) def __broadcast_message(self, eventTag, eventData): eventData = json.dumps(eventData, cls=AutomationJSONEncoder) Application.send_web_server_event(eventTag, eventData) def __trigger_rule_implementation(self, ruleId, checkCondition=False, eventTag=None, eventData=None, eventMethodId=None): ''' Triggers a rule by given ruleId. ''' Logger.log_info("trigger rule id:", ruleId) # Check if rule is "updated" AND enabled. statusProcessed, enabled = self.__ruleController.get_status_processed_and_enabled(ruleId) if statusProcessed != AppConstants.RULE_STATUS_UPDATED or enabled != True: return self.__ruleExecInfos.setdefault(ruleId, RuleExecInfo()) ruleExecInfo = self.__ruleExecInfos.get(ruleId) ruleExecInfo.increase_trigger_count() triggerCountInThisSession = ruleExecInfo.get_trigger_count() with ruleExecInfo.get_rlock(): #======================================================================= # Check conditions #======================================================================= if checkCondition is True: # Check if we should proceed (stop if there is another pending request on the same ruleId). if triggerCountInThisSession != ruleExecInfo.get_trigger_count(): return methodListToCheck = deque() result = self.__ruleController.list_conditions(ruleId) methodCheckingTime = int(time.time()) for row in result: if row["kbxMethodStatus"] not in (SharedMethod.METHOD_STATUS_ACTIVE, SharedMethod.METHOD_STATUS_INACTIVE): return else: methodArgs = row["kbxMethodParams"] kwargs = {methodArg[AppConstants.ARG_NAME]:methodArg[AppConstants.ARG_CURRENT_VALUE] for methodArg in methodArgs} if eventTag is not None and eventMethodId == row["kbxMethodId"]: kwargs[AppConstants.KEY_CONDITION_EVENT_TAG] = eventTag kwargs[AppConstants.KEY_CONDITION_EVENT_DATA] = eventData if AppInfo.REQUEST_KEY_LANGUAGE not in kwargs: kwargs[AppInfo.REQUEST_KEY_LANGUAGE] = AppInfo.DEFAULT_API_LANGUAGE kwargs["kbxMethodName"] = row["kbxMethodName"] kwargs["kbxModuleName"] = row["kbxModuleName"] kwargs["kbxGroupId"] = row["kbxGroupId"] kwargs["kbxMethodAppId"] = row["kbxMethodAppId"] # Update ruleId if it is required by the method if "ruleId" in kwargs: kwargs["ruleId"] = str(ruleId) callId = hash(str(kwargs)) # Generate condition checking ID kwargs[AppConstants.KEY_CONDITION_TIMESTAMP] = methodCheckingTime # So that timestamp will not caused the generated id to be different methodListToCheck.append({"callId":callId, "callFn":SharedMethod.call, "callKwargs":kwargs}) #=============================================================== # Submit all conditions for checking #=============================================================== methodListToCheckLen = len(methodListToCheck) if methodListToCheckLen > 0: ruleExecResult = RuleExecResult(methodListToCheckLen) for methodItem in methodListToCheck: self.__condCallGroup.submit(callbackFn=self.__on_method_call_complete, ruleExecResult=ruleExecResult, **methodItem) result = ruleExecResult.wait(40.0) if result is False or ruleExecResult.get_result() is False: return # Failed at condition checking. # Clear cache del(methodListToCheck) del(methodCheckingTime) del(methodListToCheckLen) # Check if we should proceed (stop if there is another pending request on the same ruleId). if triggerCountInThisSession != ruleExecInfo.get_trigger_count(): return #======================================================================= # Execute executions #======================================================================= methodListToExec = deque() result = self.__ruleController.list_executions(ruleId) methodExecTime = int(time.time()) for row in result: if row["kbxMethodStatus"] not in (SharedMethod.METHOD_STATUS_ACTIVE, SharedMethod.METHOD_STATUS_INACTIVE): continue else: methodArgs = row["kbxMethodParams"] kwargs = {methodArg[AppConstants.ARG_NAME]:methodArg[AppConstants.ARG_CURRENT_VALUE] for methodArg in methodArgs} if AppInfo.REQUEST_KEY_LANGUAGE not in kwargs: kwargs[AppInfo.REQUEST_KEY_LANGUAGE] = AppInfo.DEFAULT_API_LANGUAGE kwargs["kbxMethodName"] = row["kbxMethodName"] kwargs["kbxModuleName"] = row["kbxModuleName"] kwargs["kbxGroupId"] = row["kbxGroupId"] kwargs["kbxMethodAppId"] = row["kbxMethodAppId"] # Update ruleId if it is required by the method if "ruleId" in kwargs: kwargs["ruleId"] = str(ruleId) callId = hash(str(kwargs)) # Generate execution id kwargs[AppConstants.KEY_ACTION_TIMESTAMP] = methodExecTime methodListToExec.append({"callId":callId, "callFn":SharedMethod.call, "callKwargs":kwargs}) #=============================================================== # Submit all methods for executions #=============================================================== methodListToExecLen = len(methodListToExec) if methodListToExecLen > 0: ruleExecResult = RuleExecResult(methodListToExecLen) for methodItem in methodListToExec: self.__execCallGroup.submit(callbackFn=self.__on_method_call_complete, ruleExecResult=ruleExecResult, **methodItem) result = ruleExecResult.wait(30.0) return def __on_method_call_complete(self, checkingId, result, ruleExecResult): ''' When rule/execution checking is completed. ''' ruleExecResult.set_result(result)
class TriggerController: __INSTANCE = None __LOCK = threading.Lock() @staticmethod def instance(): with TriggerController.__LOCK: TriggerController.__INSTANCE or TriggerController() return TriggerController.__INSTANCE def __init__(self): TriggerController.__INSTANCE = self self.__registeredRuleIds = set([]) self.__threadPool = ThreadPoolExecutor(max_workers=1) def listen_to_trigger_callback(self, aFunc): ''' aFunc:Function - Parameters (ruleId:Integer) ''' ControllerModule.ON_TRIGGER_CALLBACK = aFunc def register_listener(self, ruleId, triggerDict): self.unregister_listener(ruleId) triggerType = triggerDict["type"] triggerParsedValue = triggerDict.get("parsedValue") #======================================================================= # Register trigger to scheduler #======================================================================= if triggerType == AppConstants.TRIGGER_TYPE_INTERVAL: def __signal_scheduler_add_interval_job(ruleId, seconds, minutes, hours): kwargs = {k:v for k, v in {"seconds":seconds, "minutes":minutes, "hours":hours}.items() if v > 0} try: SchedulerService.add_interval_job(jobName=str(ruleId), kbxTargetAppId=AppInfo.get_app_id(), kbxTargetMethod="on_trigger_callback", kbxTargetModule="controller_module", kbxTargetParams={"ruleId":ruleId}, store=False, **kwargs) self.__registeredRuleIds.add(ruleId) except SystemException as e: Logger.log_debug(e) self.__threadPool.submit(__signal_scheduler_add_interval_job, ruleId, **triggerParsedValue) elif triggerType == AppConstants.TRIGGER_TYPE_TIME: def __signal_scheduler_add_cron_job(ruleId, hour, minute): try: SchedulerService.add_cron_job(jobName=str(ruleId), kbxTargetAppId=AppInfo.get_app_id(), kbxTargetMethod="on_trigger_callback", kbxTargetModule="controller_module", kbxTargetParams={"ruleId":ruleId}, store=False, hour=str(hour), minute=str(minute)) self.__registeredRuleIds.add(ruleId) except SystemException as e: Logger.log_debug(e) self.__threadPool.submit(__signal_scheduler_add_cron_job, ruleId, **triggerParsedValue) def unregister_listener(self, ruleId): def __signal_scheduler_remove_task(ruleId): try: SchedulerService.remove_job(str(ruleId)) self.__registeredRuleIds.remove(ruleId) except SystemException as e: Logger.log_debug(e) if ruleId in self.__registeredRuleIds: self.__threadPool.submit(__signal_scheduler_remove_task, ruleId) def parse_to_trigger_dto(self, trigger): #======================================================================= # Check if all required keys must exists #======================================================================= if "type" not in trigger: raise AutomationException(11703, "'type' must exists") #======================================================================= # Check 'type' against allowed values #======================================================================= triggerType = trigger["type"] if triggerType not in (AppConstants.TRIGGER_TYPE_EVENT, AppConstants.TRIGGER_TYPE_INTERVAL, AppConstants.TRIGGER_TYPE_TIME): raise AutomationException(11703, "'type' has invalid value") #======================================================================= # Compute 'parsedValue' #======================================================================= triggerValue = trigger.get("value") if triggerType == AppConstants.TRIGGER_TYPE_INTERVAL: triggerValue = ValueParser.get_number(triggerValue) # this must be an integer (seconds) if isinstance(triggerValue, int): if triggerValue > 0: seconds = triggerValue % 60 minutes = math.floor(triggerValue / 60) % 60 hours = math.floor((triggerValue / 3600)) % 24 trigger["parsedValue"] = {"seconds":seconds, "minutes":minutes, "hours":hours} else: raise AutomationException(11703, "'value' in 'trigger' must be larger than 0") else: raise AutomationException(11703, "'value' in 'trigger' must be a number in seconds") elif triggerType == AppConstants.TRIGGER_TYPE_TIME: triggerValue = ValueParser.get_string(triggerValue) # this must be in (HH, MM) if isinstance(triggerValue, str): triggerValue = triggerValue.split(":") if len(triggerValue) == 2: #======================================================= # Validate hour #======================================================= hour = ValueParser.get_number(triggerValue[0]) if hour is None or not 0 <= hour <= 23: raise AutomationException(11703, "'HH is ranged from 00 - 23'") #======================================================= # Validate minute #======================================================= minute = ValueParser.get_number(triggerValue[1]) if minute is None or not 0 <= minute <= 59: raise AutomationException(11703, "MM is ranged from 00 - 59") trigger["parsedValue"] = {"hour":hour, "minute":minute} else: raise AutomationException(11703, "'value' in 'trigger' must be in HH:MM format") else: raise AutomationException(11703, "'value' in 'trigger' must be string in HH:MM format") else: trigger["parsedValue"] = None return trigger
''' Created on 16-Nov-2015 @author: Virendra ''' from concurrent.futures.thread import ThreadPoolExecutor import threading import time def square(n): print ("Calculating square of %d by thread name %s " % (n, threading.current_thread())) time.sleep(3) print ("Square of number %d is %d calculated by thread %s " % (n, n*n, threading.current_thread())) executor = ThreadPoolExecutor(max_workers=5) numbers = range(1,10) for number in numbers: executor.submit(square, number)
class UI(urwid.MainLoop): def __init__(self): self.d = DockerBackend() # root widget self.mainframe = urwid.Frame(urwid.SolidFill()) self.buffers = [] self.footer = Footer(self) self.executor = ThreadPoolExecutor(max_workers=4) root_widget = urwid.AttrMap(self.mainframe, "root") self.main_list_buffer = None # singleton screen = urwid.raw_display.Screen() screen.set_terminal_properties(256) screen.register_palette(PALLETE) super().__init__(root_widget, screen=screen) self.handle_mouse = False self.current_buffer = None def run_in_background(self, task, *args, **kwargs): logger.info("running task %r(%s, %s) in background", task, args, kwargs) self.executor.submit(task, *args, **kwargs) def refresh(self): try: self.draw_screen() except AssertionError: logger.warning("application is not running") pass def _set_main_widget(self, widget, redraw): """ add provided widget to widget list and display it :param widget: :return: """ self.mainframe.set_body(widget) self.reload_footer() if redraw: logger.debug("redraw main widget") self.refresh() def display_buffer(self, buffer, redraw=True): """ display provided buffer :param buffer: Buffer :return: """ self.current_buffer = buffer self._set_main_widget(buffer.widget, redraw=redraw) def add_and_display_buffer(self, buffer, redraw=True): """ add provided buffer to buffer list and display it :param buffer: :return: """ if buffer not in self.buffers: logger.debug("adding new buffer {!r}".format(buffer)) self.buffers.append(buffer) self.display_buffer(buffer, redraw=redraw) def pick_and_display_buffer(self, i): """ pick i-th buffer from list and display it :param i: int :return: None """ if len(self.buffers) == 1: # we don't need to display anything # listing is already displayed return else: try: self.display_buffer(self.buffers[i]) except IndexError: # i > len self.display_buffer(self.buffers[0]) @property def current_buffer_index(self): return self.buffers.index(self.current_buffer) def remove_current_buffer(self): # don't allow removing main_list if isinstance(self.current_buffer, MainListBuffer): logger.warning("you can't remove main list widget") return self.buffers.remove(self.current_buffer) self.current_buffer.destroy() # FIXME: we should display last displayed widget here self.display_buffer(self.buffers[0], True) def unhandled_input(self, key): logger.debug("unhandled input: %r", key) try: if key in ("q", "Q"): self.executor.shutdown(wait=False) raise urwid.ExitMainLoop() elif key == "ctrl o": self.pick_and_display_buffer(self.current_buffer_index - 1) elif key == "ctrl i": self.pick_and_display_buffer(self.current_buffer_index + 1) elif key == "x": self.remove_current_buffer() elif key == "/": self.prompt("/", search) elif key == "f4": self.footer.prompt("filter ", filter) elif key == "n": self.current_buffer.find_next() elif key == "N": self.current_buffer.find_previous() elif key in ["h", "?"]: self.display_help() elif key == "f5": self.display_tree() except NotifyError as ex: self.notify_message(str(ex), level="error") logger.error(repr(ex)) def run(self): self.main_list_buffer = MainListBuffer(self.d, self) @log_traceback def chain_fcs(): self.main_list_buffer.refresh(focus_on_top=True) self.add_and_display_buffer(self.main_list_buffer, redraw=True) self.run_in_background(chain_fcs) super().run() def display_logs(self, docker_container): self.add_and_display_buffer(LogsBuffer(docker_container, self)) def display_and_follow_logs(self, docker_container): self.add_and_display_buffer(LogsBuffer(docker_container, self, follow=True)) def inspect(self, docker_object): self.add_and_display_buffer(InspectBuffer(docker_object)) def display_image_info(self, docker_image): try: self.add_and_display_buffer(ImageInfoBuffer(docker_image, self)) except NotifyError as ex: self.notify_message(str(ex), level="error") logger.error(repr(ex)) def refresh_main_buffer(self, refresh_buffer=True): assert self.main_list_buffer is not None if refresh_buffer: self.main_list_buffer.refresh() self.display_buffer(self.main_list_buffer) def display_help(self): self.add_and_display_buffer(HelpBuffer()) def display_tree(self): self.add_and_display_buffer(TreeBuffer(self.d, self)) # FOOTER def set_footer(self, widget): self.mainframe.set_footer(widget) def reload_footer(self): self.footer.reload_footer() def remove_notification_message(self, message): self.footer.remove_notification_message(message) def notify_widget(self, *args, **kwargs): self.footer.notify_widget(*args, **kwargs) def notify_message(self, *args, **kwargs): self.footer.notify_message(*args, **kwargs) def prompt(self, *args, **kwargs): self.footer.prompt(*args, **kwargs)
class RabbitManager(BaseManager): """Base for managers that connects to rabbit """ def __init__(self, rabbitmq_url=None, queue=None, routing_key=None, exchange="message", exchange_type="direct", log=None, max_tasks=5, logging=None): """ == Config dict structure (case adjusted to json configuration): { "rabbit": { "url": "apmq://rabbit", "queue": "test", "routingKey": "example.json" "exchange": "message", // optional, default: message "exchangeType:" "topic" // optional, default: topic } } :param str rabbitmq_url: optional url to rabbitmq :param str queue: name of the queue :param str routing_key: routing key for queue :param str exchange: name of the exchange :param str exchange_type: type of the exchange :param dict config: Manager configuration from parsed json config all the above options can be configured from it :param logging.Logger log: optional logger that will replace new one :raises exceptions.NotConfigured: :return: """ if queue is None: raise exceptions.NotConfigured("Misssing queue") self._connection = None self._channel = None self._closing = False self._consumer_tag = None self._max_tasks = max_tasks # 2 cores + 1 self._tasks_number = 0 self._executor = ThreadPoolExecutor(max_workers=self._max_tasks) self._max_tasks_warning_counter = 0 self._rabbitmq_url = rabbitmq_url self._queue = queue self._routing_key = routing_key self._exchange = exchange self._exchange_type = exchange_type if log is None: from toddler.logging import setup_logging if logging is not None: self.log = setup_logging(config=logging) else: self.log = setup_logging() else: self.log = log def reconnect(self): """Will be run by IOLoop.time if the connection is closed. See on_connection_closed method. """ self._connection.ioloop.stop() if not self._closing: self._connection = self.connect() self._connection.ioloop.start() @property def queue(self): return self._queue def on_connection_closed(self, connection, reply_code, reply_text): """ :param pika.connection.Connection connection: closed connection ob :param int reply_code: reply code if given :param str reply_text: reply text if given :return: """ self._channel = None if self._closing: self._connection.ioloop.stop() else: self.log.warning( "Connection closed, will reopen in 5 seconds: (%s) %s", reply_code, reply_text ) self._connection.add_timeout(5, self.reconnect) def on_channel_closed(self, channel, reply_code, reply_text): """Invoked when channel has been closed :param pika.channel.Channel channel: :param int reply_code: :param str reply_text: :return: """ self.log.info("Channel to rabbit closed.") self._connection.close() def on_channel_open(self, channel): """Invoked when channel has been opened :param pika.channel.Channel channel: """ self.log.info("Channel opened") self._channel = channel self._channel.add_on_close_callback(self.on_channel_closed) self.start_consuming() def close_channel(self): self.log.info("Closing channel") self._channel.close() def open_channel(self): self.log.info("Opening channel") self._connection.channel(on_open_callback=self.on_channel_open) def on_connection_open(self, connection): self.log.info("Connected") self._connection = connection self._connection.add_on_close_callback(self.on_connection_closed) self.open_channel() def connect(self): """Connects to rabbitmq server, according to config :return pika.SelectConnection: """ self.log.info("Connecting to RabbitMQ") return pika.BlockingConnection( pika.URLParameters(self._rabbitmq_url + "?heartbeat_interval=5"), # self.on_connection_open, # stop_ioloop_on_close=False ) def on_cancel_ok(self, frame): """Invoked when locale Basic.Cancel is acknowledged by RabbitMQ :param pika.frame.Method frame: :return: """ self.log.info("Rabbit acknowledged the cancel of the consumer") self.close_channel() def on_consumer_cancelled(self, method_frame): """Invoked by pika when RabbitMQ sends a Basic.Cancel for a consumer receiving messages. :param pika.frame.Method method_frame: The Basic.Cancel frame :return: """ self.log.info("Consumer was cancelled remotely, shutting down: %r", method_frame) if self._channel: self._channel.close() def acknowledge_message(self, delivery_tag): """ :param delivery_tag: :return: """ self.log.info("Acknowledging message %s", delivery_tag) self._channel.basic_ack(delivery_tag) def requeue_message(self, delivery_tag): """ :param delivery_tag: :return: """ self.log.info("Requeuing message %s", delivery_tag) self._channel.basic_nack(delivery_tag, requeue=True) def on_message(self, channel, basic_deliver, properties, body): """Invoked when message received from rabbit :param pika.channel.Channel channel: :param pika.spec.Basic.Deliver basic_deliver: :param pika.spec.BasicProperties properties: :param str body: :return: """ self.log.info("Received messages # %s from %s", basic_deliver.delivery_tag, properties.app_id) try: if self._tasks_number >= self._max_tasks: raise RuntimeError("Max tasks limit reached") self._tasks_number += 1 ftr = self._executor.submit(self.process_task, body) def process_done(future: Future): nonlocal self self._tasks_number -= 1 if future.cancelled(): # process_task ended by cancel self.requeue_message(self.requeue_message( basic_deliver.delivery_tag) ) else: if future.exception(): exception = future.exception() if not isinstance(exception, RequeueMessage): self.log.exception(exception) self.requeue_message( basic_deliver.delivery_tag ) else: self.acknowledge_message(basic_deliver.delivery_tag) ftr.add_done_callback(process_done) return ftr except RuntimeError: self.requeue_message(basic_deliver.delivery_tag) time.sleep(0.5) except Exception as e: self.log.exception(e) self.requeue_message(basic_deliver.delivery_tag) time.sleep(10) def stop_consuming(self): """Send Basic.Cancel to rabbit :return: """ if self._channel: self.log.info("Stop consuming") self._channel.basic_cancel(self.on_cancel_ok, self._consumer_tag) def start_consuming(self): """Begins to consume messages :return: """ self.log.info("Start consuming") self._channel.add_on_cancel_callback(self.on_consumer_cancelled) self._consumer_tag = self._channel.basic_consume(self.on_message, self.queue) self.run() def run(self): """Run consumer""" self.log.info("Running consumer") connection = self.connect() """:type: pika.SelectConnection""" channel = connection.channel() self._channel = channel self._connection = connection for method_frame, properties, body in channel.consume(self.queue): while self._tasks_number >= self._max_tasks: time.sleep(0.1) self.on_message(channel, method_frame, properties, body) def stop(self): """Stops consuming service :return: """ self.log.info("Stopping") self._closing = True self.stop_consuming() self._executor.shutdown(True) # if self._connection is not None: # self._connection.ioloop.start() self.log.info("Stopped") def __exit__(self, *args, **kwargs): self.stop() super(RabbitManager, self).__exit__(*args, **kwargs)