class SubtaskWorker(LoadtestWorkerServicer):
"""
SubtaskWorker must be pickleable before Start is called.
"""
def __init__(self):
self._executor: Executor = None
self.metrics_tracker: MetricsTracker = None
@abstractmethod
def run_worker(self, request: StartRequest):
"""
run_worker should run the worker forever.
:param request: the request to start the worker
:return: never
"""
pass
def Start(self, request: StartRequest, context):
self._executor = ThreadPoolExecutor(max_workers=1)
self.metrics_tracker = MetricsTracker(request.include_ids)
self._executor.submit(self.run_worker, request)
return StartResponse()
def Check(self, request, context):
return self.metrics_tracker.check()
def crawl_single_kind(self, kind='basketball'):
print("GET: KIND = ", kind)
# 爬取页面，找到页面中各个鞋的类别的页面
url = 'http://www.shihuo.cn/' + kind + '/list?page_size=60&page=1'
response = requests.get(url, headers=self.header)
html = response.text
# 首先找到总页面数
start = html.find('var totalPage = parseInt(') + len('var totalPage = parseInt(')
end = html.find(')', start)
page_nums = math.ceil(float(html[start:end]))
# print(page_nums)
# 循环找到全部下级链接
links = []
for page_num in range(page_nums):
url = 'http://www.shihuo.cn/' + kind + '/list?page_size=60&page=' + str(page_num)
response = requests.get(url, headers=self.header)
html = response.text
soup = BeautifulSoup(html, "lxml")
# print(soup.prettify())
link_class = soup.select('#js_hover li .imgs-area .link ')
for item in link_class:
links.append('http:' + item['href'])
thread_pool = ThreadPoolExecutor(8)
for link in links:
thread_pool.submit(self.crawl_all_color_for_shoes, link)
thread_pool.shutdown(wait=True)
def preprocess(data_type,
img_dir,
geojson_dir,
rgb_tg,
mask_tg,
num_workers=8):
pool = ThreadPoolExecutor(max_workers=num_workers)
re_img_index = re.compile("img\d+")
re_pat = re.compile("(.*?)img\d+")
# building_pat = re_pat.search(os.listdir(Path(geojson_dir) / "buildings")[0]).group(1)
pat = re_pat.search(os.listdir(Path(geojson_dir))[0]).group(1)
for f in os.listdir(img_dir):
img_index = re_img_index.search(f).group(0)
geojson = Path(geojson_dir) / (pat + img_index + ".geojson")
def pool_wrapper(p1, p2, p3, p4):
if data_type == "building":
thread = GeoLabelUtil.BuildingRenderThread(p1, p2, p3, p4)
elif data_type == "road":
thread = GeoLabelUtil.RoadRenderThread(p1, p2, p3, p4)
else:
raise NotImplementedError("Not Implemented Data Type: " +
data_type)
thread.run()
# thread.start()
pool.submit(pool_wrapper,
Path(img_dir) / f,
Path(rgb_tg) / (img_index + ".png"), geojson,
Path(mask_tg) / (img_index + ".png"))
pool.shutdown(wait=True)
class UrlGet:
global lis, lst
lis, lst = [], []
def __init__(self):
self.thread_pool = ThreadPoolExecutor(10)
def magic_func(self):
for i in range(200):
self.thread_pool.submit(self.get_url, )
def get_url(self):
thread_name = threading.current_thread().name
r = requests.get('https://www.baidu.com')
s = r.status_code
if s == 200:
lst.append(1)
t = r.elapsed.total_seconds()
lis.append(t)
print(f"response time is : {t}s")
print(f"response code is : {s}")
print(f"this is thread : {thread_name}")
def calculiates(self):
m = max(lis)
n = min(lis)
a = numpy.mean(lis)
p = lst.count(1)
print(f"max response time is : {m}s")
print(f"min response time is : {n}s")
print(f"avg response time is : {a}s")
print(f"code200 percentage is : {p / 200 * 100}%")
def novel_chapter_detail_save(self):
sql_tables = "SELECT `table_name` from router where sourceid = %s"
tables = default_dbhelper.query(sql_tables, (SpiderTools.sourceid))
executor = ThreadPoolExecutor(max_workers=len(tables))
for table_name in tables:
executor.submit(novel_chapter_detail_save_by_tablename,
(table_name[0]))
class RPCConnector(ConsensusConnectorModel):
def __init__(self, consensus: ConsensusModel, *args, **kwargs):
super().__init__(consensus)
self.rpc_server = RPCServer(self)
self.rpc_server.serve()
self.executors = ThreadPoolExecutor(max_workers=10)
def handle_training_request(self, data):
pass
def broadcast_proposal(self, data):
# print("begin to broadcast")
for peer in self.peers:
client = RPCClient(peer)
client.send_proposal(data)
def handle_upload_request(self, data: str):
# print("handling upload request")
self.executors.submit(self.consensus.make_consensus,
data=data,
connector=self)
def handle_consensus_data(self, data):
# print('received proposal')
self.consensus.handle_block(data)
class Stub:
def __init__(self, adapter, port):
self._port = port
self._adapter = adapter
self.server = None
self._executor = None
def _setup(self):
self.server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.server.bind(('192.168.0.106', self._port))
self._executor = ThreadPoolExecutor(max_workers=2)
def run(self):
self._setup()
self.server.listen()
try:
while True:
connection, client_address = self.server.accept()
self._executor.submit(self.process_request, connection)
except KeyboardInterrupt:
connection.close()
self.server.stop(0)
def process_request(self, connection):
try:
newthread = FSStub(connection, self._adapter)
except Exception as e:
print('ERROR!!! ', e)
return
def testWithStores(self):
# store_opts2 = {"store_type": StoreTypes.ROLLPAIR_QUEUE, "capacity": 100}
store_opts2 = {"store_type": StoreTypes.ROLLPAIR_QUEUE}
rp = self.ctx.load("ns1", "n1")
rp_q = self.ctx.load("ns1", "n2", store_opts2)
rp.put_all([("k1", "v1"), ("k2", "v2")])
def func_asyn(partitions):
import time
part1 = partitions[0]
serder1 = create_serdes(part1._store_locator._serdes)
with create_adapter(part1) as db1:
for i in range(100):
db1.put(serder1.serialize("a" + str(i)))
time.sleep(0.1)
def func_syn(partitions):
part1, part2 = partitions
serder1 = create_serdes(part1._store_locator._serdes)
serder2 = create_serdes(part2._store_locator._serdes)
with create_adapter(part1) as db1, create_adapter(part2) as db2:
for i in range(100):
db1.put(serder1.serialize("q" + str(i)), db2.get())
pool = ThreadPoolExecutor()
pool.submit(rp_q.with_stores, func_asyn)
rp.with_stores(func_syn, [rp_q])
print(list(rp.get_all()))
pool.shutdown()
class MessageForwardDaemon(object):
MAX_WORKERS = 25
def __init__(self):
self.logger = retrieve_logger("message_forward")
self.devices_service = DevicesService(logger=self.logger)
self.redis_connection = redis_connection = redis.StrictRedis(
REDIS_HOSTNAME, REDIS_PORT)
self.redis_pub_sub = redis_connection.pubsub()
self.redis_pub_sub.subscribe(MESSAGE_FORWARD_REDIS_PUBSUB)
self.thread_pool_executor = ThreadPoolExecutor(
max_workers=self.MAX_WORKERS)
self.message_forward_helper = {
"MQTT": MQTTMessageForward(),
"COAP": CoAPMessageForward(),
}
def _forward_message(self, message):
protocol = message["device_info"]["protocol"]
if protocol not in self.message_forward_helper:
raise Exception("Unknown protocol {}".format(protocol))
try:
self.message_forward_helper[protocol].forward(
message["device_info"], message["value"])
except Exception as err:
self.logger.error(
"Failed to send the message to the device. Reason: {}".format(
err))
self._notify_forward_failure(message)
return
def _notify_forward_failure(self, message):
# TODO: Send message to GASS that failed to send the message to the device
self.logger.critical(message)
def start(self):
self.logger.debug("Waiting for command messages")
while True:
message = self.redis_pub_sub.get_message(timeout=120)
self.logger.debug("Received message: {}".format(message))
invalid_message = not message or message["type"] != "message"
if invalid_message:
continue
try:
message_body = json.loads(message['data'].decode("utf-8"))
self.thread_pool_executor.submit(self._forward_message,
message_body)
except Exception as err:
self.logger.error(err)
continue
self.logger.debug("Waiting for command messages")
class HttpService(object):
def __init__(self):
self.__async_executor = ThreadPoolExecutor(max_workers=10)
self.logger = logging.getLogger(__name__)
self.__http = Http()
def get(self, request):
return self.make_request(request, 'GET')
def post(self, request):
return self.make_request(request, 'POST')
def put(self, request):
return self.make_request(request, 'PUT')
def delete(self, request):
return self.make_request(request, 'DELETE')
def make_request(self, request, method):
future = HttpFuture()
self.__async_executor.submit(self.__do_request, request, method, future)
return future
def __do_request(self, request, method, future):
try:
uri = request.url + urllib.parse.urlencode(request.parameters)
headers, content = self.__http.request(uri, method, request.data, request.headers)
future.fulfill(headers, content)
except Exception as ex:
self.logger.exception("Http __do_request attempt failed with exception")
def judge_hash_is_apk(self, hash_contain, cookies):
headers = {
"Cookie":
self.crawl_login(),
"Accept":
"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,"
"*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
"User-Agent":
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/87.0.4280.88 Safari/537.36",
}
max_workers = Config_Reader().get_max_worker()
for md5 in hash_contain:
try:
params = {"search": md5}
resp = requests.post(url=self.VIRUS_SHARE_SEARCH,
data=params,
headers=headers)
is_apk = re.search(
"<td class='lc'>Extension</td><td colspan=2>apk</td>",
resp.text)
is_login = re.search("login", resp.text)
if is_apk:
sha256 = re.search(
"<td class='lc'>SHA256</td><td colspan=2>[0-9a-zA-Z]*",
resp.text)
executor = ThreadPoolExecutor(max_workers=max_workers)
executor.submit(self.download,
sha256.group(0)[40:], md5, cookies)
if not is_apk and is_login:
logging.info("======需要重新登录=====")
time.sleep(5)
except Exception as e:
logging.error(e)
pass
async def Main():
try:
player = Player("Account Email/Username", "Account Password")
except YggdrasilError as e:
# Authentication Error
print("Incorrect Login", e)
return
player.SetServer("Server to connect to.")
# We do this to ensure it is non blocking as Connect() is a
# forever loop used to maintain a connection to a server
executor = ThreadPoolExecutor()
executor.submit(player.Connect)
# Forever do things unless the user wants us to logout
while True:
message = input("What should I do/say?\n")
# Disconnect the client from the server before finishing everything up
if message.lower() in ["logout", "disconnected", "exit"]:
player.Disconnect()
print("Disconnected")
return
# Send the message to the server via the player
player.SendChat(message)
def executor(request):
"""
Receives http request and starts execution of Test
"""
if request.method == 'POST':
try:
test_id = generate_test_id()
os.makedirs(f"{STORAGE_PATH}/{test_id}")
logger.info('Directory created at: ' + f"{STORAGE_PATH}/{test_id}")
received_json_data = json.loads(request.body.decode())
try:
test_ymls, initial_variable_dictionary = __extract_test_data(test_id, received_json_data['test'])
except BadArgumentsError as e:
return HttpResponse(str(e), status=status.HTTP_400_BAD_REQUEST)
pool_executor = ThreadPoolExecutor(max_workers=1)
try:
pool_executor.submit(thread_executor, test_ymls, initial_variable_dictionary, test_id,
received_json_data['email'])
logger.info("task submitted to thread pool executor")
except Exception as e:
logger.error("Exception occurred", e)
response_message = "Your test ID is: " + test_id + ". We'll send you an email with report shortly"
return HttpResponse(response_message, status=status.HTTP_200_OK)
except Exception as e:
logger.error("Exception caught", exc_info=True)
return HttpResponse(e, status=status.HTTP_400_BAD_REQUEST)
else:
return HttpResponse(status=status.HTTP_405_METHOD_NOT_ALLOWED)
def multiplyKaratsubaParallel(args) -> Polynomial:
depth = args[0]
A = args[1]
B = args[2]
if depth > 4:
return PolynomialOperations.multiplySequencially(A, B)
if A.n < 2 or B.n < 2:
return PolynomialOperations.multiplySequencially(A, B)
m = int(max(A.n, B.n) / 2)
lowA = Polynomial(len(A.coefficients[:m]), A.coefficients[:m])
highA = Polynomial(len(A.coefficients[m:]), A.coefficients[m:])
lowB = Polynomial(len(B.coefficients[:m]), B.coefficients[:m])
highB = Polynomial(len(B.coefficients[m:]), B.coefficients[m:])
karaPool = ThreadPoolExecutor(mp.cpu_count())
futureResult1 = karaPool.submit(PolynomialOperations.multiplyKaratsubaParallel, ([depth+1, lowA, lowB]))
futureResult2 = karaPool.submit(PolynomialOperations.multiplyKaratsubaParallel, ([depth+1, PolynomialOperations.add(lowA, highA), PolynomialOperations.add(lowB, highB)]))
futureResult3 = karaPool.submit(PolynomialOperations.multiplyKaratsubaParallel, ([depth+1, highA, highB]))
karaPool.shutdown(wait=True)
result1 = futureResult1.result()
result2 = futureResult2.result()
result3 = futureResult3.result()
r1 = PolynomialOperations.shift(result3, 2 * m)
r2 = PolynomialOperations.shift(
PolynomialOperations.subtract(PolynomialOperations.subtract(result2, result3), result1), m)
return PolynomialOperations.add(PolynomialOperations.add(r1, r2), result1)
def get_data():
executor = ThreadPoolExecutor(max_workers=WORKER)
# get the res dir ready
mkdir_res()
# get url
list(
as_completed(
executor.submit(get_chengjiao_house_url, ),
executor.submit(get_ershoufang_house_url, ),
))
# get ershoufang info
list(
as_completed(
executor.submit(get_ershoufang_house_info, hs)
for hs, name in HOUSE_DISTRICT_DICT.items()
if not (DATA_DIR / "house_info" / "ershoufang" /
f"{hs}.json").is_file()))
# get chengjiao info
list(
as_completed(
executor.submit(get_chengjiao_house_info, hs)
for hs, name in HOUSE_DISTRICT_DICT.items()
if not (DATA_DIR / "house_info" / "chengjiao" /
f"{hs}.json").is_file()))
# save to csv
to_csv()
def run_generators(task_generators: List[BaseSpiderTaskGenerator],
item_pool_workers: int = 32,
retries: int = 5):
g_pool = ThreadPoolExecutor(max_workers=len(task_generators))
i_pool = ThreadPoolExecutor(max_workers=item_pool_workers)
def retry_and_handle_exception(func: Callable):
def wrapper():
for i in range(retries):
try:
func()
break
except Exception as ex:
log.error(f"Error while executing task (retries={i}).",
exc_info=ex)
return wrapper
def submit_all_items(stg: BaseSpiderTaskGenerator):
def wrapper():
try:
for sub_task in stg.generate():
i_pool.submit(retry_and_handle_exception(sub_task))
except Exception as ex:
log.error("Error while generating tasks.", exc_info=ex)
return wrapper
for g in task_generators:
g_pool.submit(submit_all_items(g))
log.info("All generators started.")
g_pool.shutdown(wait=True)
log.info("All generators terminated.")
i_pool.shutdown(wait=True)
log.info("All tasks finished.")
class ProcessingTimesCollector:
def __init__(self):
self._executor = ThreadPoolExecutor(max_workers=1)
self._times_map = {}
def _add_times(self, times):
for k, v in times.items():
try:
agg = self._times_map[k]
except KeyError:
agg = TimerStatsAggregator()
self._times_map[k] = agg
agg.add_time(v)
def add_times(self, times):
self._executor.submit(self._add_times, times)
def _get_aggregates(self, prefix):
return {
identifier: stats.finalize()
for identifier, stats in self._times_map.items()
if identifier.startswith(prefix)
}
def get_aggregates(self, identifier=None) -> Dict[str, TimerStats]:
future = self._executor.submit(self._get_aggregates, identifier or '')
return future.result()
def close(self):
self._executor.shutdown(wait=True)
def compare(request, name):
name_amazon = name
name_flikart = name
name_tata = name
tick = time.time()
executor = ThreadPoolExecutor()
try:
task1 = executor.submit(util_amazon, name_amazon)
amazon = task1.result()
except:
amazon = None
try:
task2 = executor.submit(util_flipkart, name_flikart)
flipkart = task2.result()
except:
flipkart = None
try:
task3 = executor.submit(util_tatacliq, name_tata)
tata = task3.result()
except:
tata = None
compare_context = {
'amazon': amazon,
'flipkart': flipkart,
'tata': tata,
'prd_name': name,
'time_taken': time.time() - tick,
}
return render(request, 'compare.html', compare_context)
class TaskManager():
def __init__(self, max_count=5):
self.pool = ThreadPoolExecutor(max_count)
self.futures = []
def submit_func(self, func, *args):
self.cleanup_pool()
future = self.pool.submit(func, *args)
self.futures.append(future)
def submit(self, argv):
self.cleanup_pool()
future = self.pool.submit(check_output, ' '.join(argv), shell=True)
self.futures.append(future)
return future.result()
def cleanup_pool(self):
self.futures = list(filter(lambda f: f and not f.done(), self.futures))
def wait(self):
for future in self.futures:
future.result() # blocks and forward exceptions
def idle(self):
'''
True if no task is running
'''
for future in self.futures:
if not future.done():
return False
return True
def run_case(request, *args, **kwargs):
case_id = kwargs["pk"]
run_env = request.data.get("runEnv")
run_user_nickname = request.data.get("runUserNickname")
case = Case.objects.get(id=case_id)
project_id = case.project_id
request_jwt = request.headers.get("Authorization").replace("Bearer ", "")
request_jwt_decoded = jwt.decode(request_jwt, verify=False, algorithms=['HS512'])
user_id = request_jwt_decoded["user_id"]
p = ProjectPath(project_id, run_env, user_id)
if not os.path.exists(p.project_temp_dir()):
os.chdir(p.projects_root)
startproject(p.project_temp_name)
clean_fixtures_dir(os.path.join(p.project_temp_dir(), "fixtures"))
clean_tests_dir(os.path.join(p.project_temp_dir(), "tests"))
pull_tep_files(project_id, p.project_temp_dir(), run_env)
thread_pool = ThreadPoolExecutor()
tests_dir = os.path.join(p.project_temp_dir(), "tests")
for newest_case in pull_case_files(tests_dir, [case]):
case_id, filepath = newest_case
delete_case_result(case_id, run_user_nickname)
os.chdir(tests_dir)
cmd = rf"pytest -s {filepath}"
args = (pytest_subprocess, cmd, case_id, run_env, run_user_nickname)
thread_pool.submit(*args).add_done_callback(save_case_result)
return Response({"msg": "用例运行成功"}, status=status.HTTP_200_OK)
def run(self):
self.turn_me()
if not self.exit:
res = self.get_host_video_data()
m3u8_data = self.get_m3u8_data(res)
m3u8_count = m3u8_data.count('EXTINF')
host = sorted(res['host'],
key=lambda i: i.get('weight'),
reverse=True)
executor = ThreadPoolExecutor(max_workers=self.anime.speed_value)
for i in range(self.data['video_ts'], m3u8_count):
executor.submit(self.video, i, res, host, m3u8_count)
while True:
if self.data['video_ts'] == m3u8_count or self.exit:
break
self.data.update({
'schedule':
int(self.data['video_ts'] / (m3u8_count - 1) * 100),
'status':
'下載中',
})
self.download_video.emit(self.data)
time.sleep(1)
self.download_video.emit(self.data)
self.anime.now_download_value -= 1
try:
if self.data['video_ts'] == m3u8_count:
self.anime.download_queue.remove(self.data["total_name"])
except BaseException as e:
print(f'抓刪除時 queue 有錯誤 {e}')
json.dump({'queue': self.anime.download_queue},
open('./Log/DownloadQueue.json', 'w', encoding='utf-8'),
indent=2)
self.quit()
self.wait()
def upload_files(files, s3_path):
executor = ThreadPoolExecutor(max_workers=32)
for i,file_path in enumerate(files):
executor.submit(upload_s3_file, file_path, s3_path)
if i%100==0:
print(f"{i}\t{file_path}")
executor.shutdown(wait=True)
class OutPutThreadPool(object):
'''
启用最大并发线程数为5的线程池对上面爬取解析线程池结果进行并发处理存储；
'''
def __init__(self):
self.thread_pool = ThreadPoolExecutor(max_workers=5)
def _output_runnable(self, crawl_result):
try:
url = crawl_result['url']
title = crawl_result['title']
summary = crawl_result['summary']
save_dir = 'output'
print('start save %s as %s.txt.' % (url, title))
if os.path.exists(save_dir) is False:
os.makedirs(save_dir)
save_file = save_dir + os.path.sep + title + '.txt'
if os.path.exists(save_file):
print('file %s is already exist!' % title)
return
with open(save_file, "w") as file_input:
file_input.write(summary)
except Exception as e:
print('save file error.' + str(e))
def save(self, crawl_result):
self.thread_pool.submit(self._output_runnable, crawl_result)
class DistanceCalculator(threading.Thread):
def __init__(self, task_queue, result_queue, origin_address, *args,
**kwargs):
self.google_maps_helper = GoogleMapsHelper()
self.executor = ThreadPoolExecutor(10)
self.task_queue = task_queue
self.result_queue = result_queue
self.origin_address = origin_address
super().__init__(*args, **kwargs)
def run(self):
while True:
try:
offering = self.task_queue.get(timeout=0.1)
except queue.Empty:
return
restaurant = offering['restaurant']
destination_id = restaurant['id']
duration = dynamodb.get_distance(self.origin_address,
destination_id)
if duration is None:
destination = F"{restaurant['address']}, {restaurant['city']['name']}, {restaurant['state']}"
duration = self.google_maps_helper.get_walking_time(
self.origin_address, destination)
if duration is not None:
self.executor.submit(dynamodb.store_distance,
self.origin_address, destination_id,
duration)
offering['walkingTimeFromOrigin'] = duration
self.result_queue.put_nowait(offering)
self.task_queue.task_done()
def prepareServer(RequestHandlerClass, pipe, threads, timeout):
'''
Prepare in a process the request handling.
'''
def process(request, address):
RequestHandlerClass(request, address, None)
try:
request.shutdown(socket.SHUT_WR)
except socket.error:
pass # some platforms may raise ENOTCONN here
request.close()
pool = ThreadPoolExecutor(threads)
while True:
if not pipe.poll(timeout): break
else:
data = pipe.recv()
if data is None: break
elif data is True: continue
requestfd, address = data
request = socket.fromfd(rebuild_handle(requestfd), socket.AF_INET,
socket.SOCK_STREAM)
pool.submit(process, request, address)
pool.shutdown(False)
class ModelCaller:
"""
Class for asynchronous calling of model's API
"""
def __init__(self):
self.query = TileInformation.objects.filter(
tile_name__isnull=False,
source_b04_location__isnull=False,
source_b08_location__isnull=False,
source_tci_location__isnull=False)
self.data_dir = 'data'
self.executor = ThreadPoolExecutor(max_workers=10)
def start(self):
for tile in self.query:
self.executor.submit(self.process, tile)
def process(self, tile):
"""
Converting jp2file to tiff, then sending its to model and saving results to db
"""
prepare_tiff(tile)
results = raster_prediction(tile.model_tiff_location)
results_path = os.path.join(self.data_dir, results[0].get('polygons'))
save(results_path)
# clean up
rmtree(os.path.dirname(tile.model_tiff_location))
rmtree(os.path.dirname(results_path))
def main(dir_path: Path):
start_time = time.time()
executor = ThreadPoolExecutor(max_workers=10)
for path in dir_path.rglob("*.java"):
executor.submit(replace, path)
end_time = time.time()
print(end_time - start_time)
def run_plan_engine(project_id, plan_id, run_env, run_user_nickname, user_id):
p = ProjectPath(project_id, run_env, user_id)
if not os.path.exists(p.project_temp_dir()):
os.chdir(p.projects_root)
startproject(p.project_temp_name)
clean_fixtures_dir(os.path.join(p.project_temp_dir(), "fixtures"))
clean_tests_dir(os.path.join(p.project_temp_dir(), "tests"))
pull_tep_files(project_id, p.project_temp_dir(), run_env)
# 根据plan_id从PlanCase中找到关联用例
plan_case_ids = [
plan_case.case_id
for plan_case in PlanCase.objects.filter(plan_id=plan_id)
]
case_list = Case.objects.filter(Q(id__in=plan_case_ids))
thread_pool = ThreadPoolExecutor()
tests_dir = os.path.join(p.project_temp_dir(), "tests")
for newest_case in pull_case_files(tests_dir, case_list):
case_id, filepath = newest_case
delete_plan_result(plan_id, case_id)
os.chdir(tests_dir)
cmd = rf"pytest -s {filepath}"
args = (pytest_subprocess, cmd, case_id, run_env, run_user_nickname,
plan_id)
thread_pool.submit(*args).add_done_callback(save_case_result)
class Server:
def __init__(self, host='', port=5555):
self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.host = host
self.port = port
self.bind()
self.listen()
self.executor = ThreadPoolExecutor(max_workers=10)
def bind(self):
try:
self.socket.bind((self.host, self.port))
except socket.error as msg:
print('Bind failed.', msg)
sys.exit()
print('Socket bind complete')
def listen(self, backlog=10):
# Start listening on socket
self.socket.listen(backlog)
print('Socket now listening')
def handle_request(self, data):
print("Received from client: ", data)
timestamp = datetime.datetime.now().timestamp()
data["timestamp"] = timestamp
print("Updated data", data)
return data
def handle_connection(self, conn):
while True:
data = conn.recv(1024)
if not data:
break
try:
data = json.loads(data.decode())
except ValueError:
data = json.dumps({"error": "Bad request"})
conn.sendall(json.dumps(data).encode())
else:
response = self.handle_request(data)
conn.sendall(json.dumps(response).encode())
conn.close()
def start(self):
try:
while True:
# wait to accept a connection - blocking call
conn, addr = self.socket.accept()
print('Connected with ', addr[0], ':', str(addr[1]))
self.executor.submit(self.handle_connection, conn)
finally:
print("Stop")
self.socket.close()
def multi_test(self):
# using thread pool to improve testing speed
t = ThreadPoolExecutor(10)
print('testing proxy, it will take several minutes......')
for proxy in self.proxy_lis:
t.submit(self.tes_proxy, proxy).add_done_callback(self.save_valid_proxy_lis)
t.shutdown()
print(f'fetch {self.p_count} valid proxies!')
def echo_server(addr):
pool = ThreadPoolExecutor(128)
sock = socket(AF_INET, SOCK_STREAM)
sock.bind(addr)
sock.listen(5)
while True:
client_sock, client_addr = sock.accept()
pool.submit(echo_client, client_sock, client_addr)
class ThreadedPoolExecutor(PoolExecutor):
'''
Pooled executor implementation based on a wrapped
ThreadPoolExecutor object.
'''
def __init__(self, context, max_workers=1):
super(ThreadedPoolExecutor, self).__init__(context)
self._pool = ThreadPoolExecutor(max_workers)
def execute(self, task):
self._pool.submit(task.processor)
def test_log_traceback_threaded(caplog):
@log_traceback
def f():
raise Exception()
e = ThreadPoolExecutor(max_workers=1)
f = e.submit(f)
while f.running():
time.sleep(0.1)
assert caplog.records()[0].message.endswith(" is about to be started")
assert caplog.records()[1].message.startswith("Traceback")
assert caplog.records()[1].message.endswith("Exception\n")
class DispatcherHTTPServer(HTTPServer):
def __init__(self, server_address, RequestHandlerClass,
bind_and_activate=True, handlers=[],
srv_path=".",
configuration={}):
HTTPServer.__init__(self, server_address, RequestHandlerClass, bind_and_activate)
self.handlers = sorted(handlers, key=lambda k: k["weight"])
self.srv_path = srv_path
self.configuration = configuration
self.logger = self.setup_logger()
self.executor = ThreadPoolExecutor(max_workers=20)
self.initialize_server()
def initialize_server(self):
self.logger.info('Initializing server')
def finish_request(self, request, client_address):
def async_finish_request(server, request, client_address, logger):
server.RequestHandlerClass(request, client_address, server, logger)
self.executor.submit(async_finish_request(self, request, client_address, self.logger))
def setup_logger(self):
logger = None
if self.configuration.get('log_config_file') is not None:
logger = self.get_logger(self.configuration.get('log_config_file'))
else:
logger = self.get_default_logger()
return logger
def get_logger(self, config_file):
logging.config.fileConfig(config_file)
return logging.getLogger('srv')
def get_default_logger(self):
logging.basicConfig(level=logging.INFO)
return logging.getLogger('srv')
def prepareServer(RequestHandlerClass, pipe, threads, timeout):
'''
Prepare in a process the request handling.
'''
def process(request, address):
RequestHandlerClass(request, address, None)
try: request.shutdown(socket.SHUT_WR)
except socket.error: pass # some platforms may raise ENOTCONN here
request.close()
pool = ThreadPoolExecutor(threads)
while True:
if not pipe.poll(timeout): break
else:
data = pipe.recv()
if data is None: break
elif data is True: continue
requestfd, address = data
request = socket.fromfd(rebuild_handle(requestfd), socket.AF_INET, socket.SOCK_STREAM)
pool.submit(process, request, address)
pool.shutdown(False)
class MethodCallGroup(object):
'''
This class groups similar shared method calls, fires only once and replies to all callers with the result.
'''
def __init__(self, max_workers=5):
'''
This is multi-threaded caller.
max_workers - Maximum simultaneous threads.
async - Set it to True then "submit" will not wait for the response.
'''
self.__callId_callbackList = {} # 1-level dict, [callId] -- [callback tasks]
self.__fs_callbackList = {} # 1-level dict, [fs] = [callback tasks]
self.__fs_callId = {}
self.__threadPool = ThreadPoolExecutor(max_workers=max_workers)
self.__lock = Lock()
self.__taskId = 0
def submit(self, callId, callFn, callKwargs, callbackFn=None, **callbackKwargs):
'''
callId - A string. Same call Id will be called only once.
callFn - A callable function where "**kwargs" will be pass as arguments later.
callKwargs - A dictionary object which can be passed into callerFn.
callbackFn - A callable which takes effect only when async = True.
callbackKwargs - "callback" will accepts the result as the first argument, and **callbackKW as the rest of the arguments.
-- result can be the return value, or an instance of Exception.
'''
# Generate task id
taskId = self.__taskId = (self.__taskId + 1) % 9999
fs = None
with self.__lock:
if callId not in self.__callId_callbackList:
fs = self.__threadPool.submit(callFn, **callKwargs)
callbackList = {}
self.__fs_callbackList[fs] = callbackList
self.__callId_callbackList[callId] = callbackList
self.__fs_callId[fs] = callId
else:
callbackList = self.__callId_callbackList[callId]
callbackList[taskId] = (callbackFn, callbackKwargs)
if fs is not None:
fs.add_done_callback(self.__done_callback)
return taskId
def __done_callback(self, fs):
with self.__lock:
callbackList = self.__fs_callbackList[fs]
del(self.__fs_callbackList[fs])
del(self.__callId_callbackList[self.__fs_callId[fs]])
del(self.__fs_callId[fs])
try:
result = fs.result()
except Exception as e:
result = e
for taskId, (callbackFn, callbackKwargs) in callbackList.items():
callbackFn(taskId, result, **callbackKwargs)
class OpticalPathManager(object):
"""
The purpose of this module is setting the physical components contained in
the optical path of a SPARC system to the right position/configuration with
respect to the mode given.
"""
def __init__(self, microscope):
"""
microscope (Microscope): the whole microscope component, thus it can
handle all the components needed
"""
self.microscope = microscope
self._graph = affectsGraph(self.microscope)
# Use subset for modes guessed
if microscope.role == "sparc2":
self._modes = copy.deepcopy(SPARC2_MODES)
elif microscope.role in ("sparc-simplex", "sparc"):
self._modes = copy.deepcopy(SPARC_MODES)
else:
raise NotImplementedError("Microscope role '%s' unsupported" % (microscope.role,))
# keep list of already accessed components, to avoid creating new proxys
# every time the mode changes
self._known_comps = dict() # str (role) -> component
# All the actuators in the microscope, to cache proxy's to them
self._actuators = []
for comp in model.getComponents():
if hasattr(comp, 'axes') and isinstance(comp.axes, dict):
self._actuators.append(comp)
# last known axes position
self._stored = {}
self._last_mode = None # previous mode that was set
# Removes modes which are not supported by the current microscope
for m, (det, conf) in self._modes.items():
try:
comp = self._getComponent(det)
except LookupError:
logging.debug("Removing mode %s, which is not supported", m)
del self._modes[m]
# Create the guess information out of the mode
# TODO: just make it a dict comprole -> mode
self.guessed = self._modes.copy()
# No stream should ever imply alignment mode
for m in ALIGN_MODES:
try:
del self.guessed[m]
except KeyError:
pass # Mode to delete is just not there
# Handle different focus for chamber-view (in SPARCv2)
if "chamber-view" in self._modes:
self._focus_in_chamber_view = None
self._focus_out_chamber_view = None
# Check whether the focus affects the chamber view
self._chamber_view_own_focus = False
try:
chamb_det = self._getComponent(self._modes["chamber-view"][0])
focus = self._getComponent("focus")
if self.affects(focus.name, chamb_det.name):
self._chamber_view_own_focus = True
except LookupError:
pass
if not self._chamber_view_own_focus:
logging.debug("No focus component affecting chamber")
try:
spec = self._getComponent("spectrometer")
except LookupError:
spec = None
if self.microscope.role == "sparc2" and spec:
# Remove the moves that don't affects the detector
# TODO: do this for _all_ modes
for mode in ('spectral', 'monochromator'):
if mode in self._modes:
det_role = self._modes[mode][0]
det = self._getComponent(det_role)
modeconf = self._modes[mode][1]
for act_role in modeconf.keys():
try:
act = self._getComponent(act_role)
except LookupError:
# TODO: just remove that move too?
logging.debug("Failed to find component %s, skipping it", act_role)
continue
if not self.affects(act.name, det.name):
logging.debug("Actuator %s doesn't affect %s, so removing it from mode %s",
act_role, det_role, mode)
del modeconf[act_role]
# will take care of executing setPath asynchronously
self._executor = ThreadPoolExecutor(max_workers=1)
def __del__(self):
logging.debug("Ending path manager")
# Restore the spectrometer focus, so that on next start, this value will
# be used again as "out of chamber view".
if self._chamber_view_own_focus and self._last_mode == "chamber-view":
focus_comp = self._getComponent("focus")
if self._focus_out_chamber_view is not None:
logging.debug("Restoring focus from before coming to chamber view to %s",
self._focus_out_chamber_view)
try:
focus_comp.moveAbsSync(self._focus_out_chamber_view)
except IOError as e:
logging.info("Actuator move failed giving the error %s", e)
self._executor.shutdown(wait=False)
def _getComponent(self, role):
"""
same as model.getComponent, but optimised by caching the result
return Component
raise LookupError: if no component found
"""
try:
comp = self._known_comps[role]
except LookupError:
comp = model.getComponent(role=role)
self._known_comps[role] = comp
return comp
@isasync
def setPath(self, mode):
"""
Just a wrapper of _doSetPath
"""
f = self._executor.submit(self._doSetPath, mode)
return f
def _doSetPath(self, path):
"""
Given a particular mode it sets all the necessary components of the
optical path (found through the microscope component) to the
corresponding positions.
path (stream.Stream or str): The stream or the optical path mode
raises:
ValueError if the given mode does not exist
IOError if a detector is missing
"""
if isinstance(path, stream.Stream):
mode = self.guessMode(path)
if mode not in self._modes:
raise ValueError("Mode '%s' does not exist" % (mode,))
target = self.getStreamDetector(path) # target detector
else:
mode = path
if mode not in self._modes:
raise ValueError("Mode '%s' does not exist" % (mode,))
comp_role = self._modes[mode][0]
comp = self._getComponent(comp_role)
target = comp.name
logging.debug("Going to optical path '%s', with target detector %s.", mode, target)
fmoves = [] # moves in progress
# Restore the spectrometer focus before any other move, as (on the SR193),
# the value is grating/output dependent
if self._chamber_view_own_focus and self._last_mode == "chamber-view":
focus_comp = self._getComponent("focus")
self._focus_in_chamber_view = focus_comp.position.value.copy()
if self._focus_out_chamber_view is not None:
logging.debug("Restoring focus from before coming to chamber view to %s",
self._focus_out_chamber_view)
fmoves.append(focus_comp.moveAbs(self._focus_out_chamber_view))
modeconf = self._modes[mode][1]
for comp_role, conf in modeconf.items():
# Try to access the component needed
try:
comp = self._getComponent(comp_role)
except LookupError:
logging.debug("Failed to find component %s, skipping it", comp_role)
continue
mv = {}
for axis, pos in conf.items():
if axis == "power":
if model.hasVA(comp, "power"):
try:
if pos == 'on':
comp.power.value = comp.power.range[1]
else:
comp.power.value = comp.power.range[0]
logging.debug("Updating power of comp %s to %f", comp.name, comp.power.value)
except AttributeError:
logging.debug("Could not retrieve power range of %s component", comp_role)
continue
if isinstance(pos, str) and pos.startswith("MD:"):
pos = self.mdToValue(comp, pos[3:])[axis]
if axis in comp.axes:
if axis == "band":
# Handle the filter wheel in a special way. Search
# for the key that corresponds to the value, most probably
# to the 'pass-through'
choices = comp.axes[axis].choices
for key, value in choices.items():
if value == pos:
pos = key
# Just to store current band in order to restore
# it once we leave this mode
if self._last_mode not in ALIGN_MODES:
self._stored[axis] = comp.position.value[axis]
break
else:
logging.debug("Choice %s is not present in %s axis", pos, axis)
continue
elif axis == "grating":
# If mirror is to be used but not found in grating
# choices, then we use zero order. In case of
# GRATING_NOT_MIRROR we either use the last known
# grating or the first grating that is not mirror.
choices = comp.axes[axis].choices
if pos == "mirror":
# Store current grating (if we use one at the moment)
# to restore it once we use a normal grating again
if choices[comp.position.value[axis]] != "mirror":
self._stored[axis] = comp.position.value[axis]
self._stored['wavelength'] = comp.position.value['wavelength']
# Use the special "mirror" grating, if it exists
for key, value in choices.items():
if value == "mirror":
pos = key
break
else:
# Fallback to zero order (aka "low-quality mirror")
axis = 'wavelength'
pos = 0
elif pos == GRATING_NOT_MIRROR:
if choices[comp.position.value[axis]] == "mirror":
# if there is a grating stored use this one
# otherwise find the non-mirror grating
if axis in self._stored:
pos = self._stored[axis]
else:
pos = self.findNonMirror(choices)
if 'wavelength' in self._stored:
mv['wavelength'] = self._stored['wavelength']
else:
pos = comp.position.value[axis] # no change
try:
del self._stored[axis]
except KeyError:
pass
try:
del self._stored['wavelength']
except KeyError:
pass
else:
logging.debug("Using grating position as-is: '%s'", pos)
pass # use pos as-is
elif axis == "slit-in":
if self._last_mode not in ALIGN_MODES:
# TODO: save also the component
self._stored[axis] = comp.position.value[axis]
elif hasattr(comp.axes[axis], "choices") and isinstance(comp.axes[axis].choices, dict):
choices = comp.axes[axis].choices
for key, value in choices.items():
if value == pos:
pos = key
break
mv[axis] = pos
else:
logging.debug("Not moving axis %s.%s as it is not present", comp_role, axis)
try:
fmoves.append(comp.moveAbs(mv))
except AttributeError:
logging.debug("%s not an actuator", comp_role)
# Now take care of the selectors based on the target detector
fmoves.extend(self.selectorsToPath(target))
# If we are about to leave alignment modes, restore values
if self._last_mode in ALIGN_MODES and mode not in ALIGN_MODES:
if 'band' in self._stored:
try:
flter = self._getComponent("filter")
fmoves.append(flter.moveAbs({"band": self._stored['band']}))
except LookupError:
logging.debug("No filter component available")
if 'slit-in' in self._stored:
try:
spectrograph = self._getComponent("spectrograph")
fmoves.append(spectrograph.moveAbs({"slit-in": self._stored['slit-in']}))
except LookupError:
logging.debug("No spectrograph component available")
# Save last mode
self._last_mode = mode
# wait for all the moves to be completed
for f in fmoves:
try:
f.result()
except IOError as e:
logging.warning("Actuator move failed giving the error %s", e)
# When going to chamber view, store the current focus position, and
# restore the special focus position for chamber, after _really_ all
# the other moves have finished, because the grating/output selector
# moves affects the current position of the focus.
if self._chamber_view_own_focus and mode == "chamber-view":
focus_comp = self._getComponent("focus")
self._focus_out_chamber_view = focus_comp.position.value.copy()
if self._focus_in_chamber_view is not None:
logging.debug("Restoring focus from previous chamber view to %s",
self._focus_in_chamber_view)
try:
focus_comp.moveAbsSync(self._focus_in_chamber_view)
except IOError as e:
logging.warning("Actuator move failed giving the error %s", e)
def selectorsToPath(self, target):
"""
Sets the selectors so the optical path leads to the target component
(usually a detector).
target (str): component name
return (list of futures)
"""
fmoves = []
for comp in self._actuators:
# TODO: pre-cache this as comp/target -> axis/pos
# TODO: extend the path computation to "for every actuator which _affects_
# the target, move if if position known, and update path to that actuator"?
# Eg, this would improve path computation on SPARCv2 with fiber aligner
mv = {}
for an, ad in comp.axes.items():
if hasattr(ad, "choices") and isinstance(ad.choices, dict):
for pos, value in ad.choices.items():
if target in value:
# set the position so it points to the target
mv[an] = pos
comp_md = comp.getMetadata()
if target in comp_md.get(model.MD_FAV_POS_ACTIVE_DEST, {}):
mv.update(comp_md[model.MD_FAV_POS_ACTIVE])
elif target in comp_md.get(model.MD_FAV_POS_DEACTIVE_DEST, {}):
mv.update(comp_md[model.MD_FAV_POS_DEACTIVE])
if mv:
logging.debug("Move %s added so %s targets to %s", mv, comp.name, target)
fmoves.append(comp.moveAbs(mv))
# make sure this component is also on the optical path
fmoves.extend(self.selectorsToPath(comp.name))
return fmoves
def guessMode(self, guess_stream):
"""
Given a stream and by checking its components (e.g. role of detector)
guesses and returns the corresponding optical path mode.
guess_stream (object): The given optical stream
returns (str): Mode estimated
raises:
LookupError if no mode can be inferred for the given stream
IOError if given object is not a stream
"""
if not isinstance(guess_stream, stream.Stream):
raise IOError("Given object is not a stream")
# Handle multiple detector streams
if isinstance(guess_stream, stream.MultipleDetectorStream):
for st in guess_stream.streams:
try:
return self.guessMode(st)
except LookupError:
pass
else:
for mode, conf in self.guessed.items():
if conf[0] == guess_stream.detector.role:
return mode
# In case no mode was found yet
raise LookupError("No mode can be inferred for the given stream")
def getStreamDetector(self, path_stream):
"""
Given a stream find the detector.
path_stream (object): The given stream
returns (str): detector name
raises:
IOError if given object is not a stream
LookupError: if stream has no detector
"""
if not isinstance(path_stream, stream.Stream):
raise IOError("Given object is not a stream")
# Handle multiple detector streams
if isinstance(path_stream, stream.MultipleDetectorStream):
dets = []
for st in path_stream.streams:
try:
# Prefer the detectors which have a role in the mode, as it's much
# more likely to be the optical detector
# TODO: handle setting multiple optical paths? => return all the detectors
role = st.detector.role
name = st.detector.name
for conf in self.guessed.values():
if conf[0] == role:
return name
dets.append(name)
except AttributeError:
pass
if dets:
logging.warning("No detector on stream %s has a known optical role", path_stream.name.value)
return dets[0]
else:
try:
return path_stream.detector.name
except AttributeError:
pass # will raise error just after
raise LookupError("Failed to find a detector on stream %s" % (path_stream.name.value))
def findNonMirror(self, choices):
"""
Given a dict of choices finds the one with value different than "mirror"
"""
for key, value in choices.items():
if value != "mirror":
return key
else:
raise ValueError("Cannot find grating value in given choices")
def mdToValue(self, comp, md_name):
"""
Just retrieves the "md_name" metadata from component "comp"
"""
md = comp.getMetadata()
try:
value = md.get(md_name)
return value
except KeyError:
raise KeyError("Metadata %s does not exist in component %s" % (md_name, comp.name))
def affects(self, affecting, affected):
"""
Returns True if "affecting" component affects -directly of indirectly-
the "affected" component
"""
path = self.findPath(affecting, affected)
if path is None:
return False
else:
return True
def findPath(self, node1, node2, path=[]):
"""
Find any path between node1 and node2 (may not be shortest)
"""
path = path + [node1]
if node1 == node2:
return path
if node1 not in self._graph:
return None
for node in self._graph[node1]:
if node not in path:
new_path = self.findPath(node, node2, path)
if new_path:
return new_path
return None
class MyComponent(model.Component):
"""
A component that does everything
"""
def __init__(self, name, daemon):
model.Component.__init__(self, name=name, daemon=daemon)
self.executor = ThreadPoolExecutor(max_workers=1)
self.number_futures = 0
self.startAcquire = model.Event() # triggers when the acquisition of .data starts
self.data = FakeDataFlow(sae=self.startAcquire)
self.datas = SynchronizableDataFlow()
self.data_count = 0
self._df = None
# TODO automatically register the property when serializing the Component
self.prop = model.IntVA(42)
self.cont = model.FloatContinuous(2.0, [-1, 3.4], unit="C")
self.enum = model.StringEnumerated("a", set(["a", "c", "bfds"]))
self.cut = model.IntVA(0, setter=self._setCut)
self.listval = model.ListVA([2, 65])
def _setCut(self, value):
self.data.cut = value
return self.data.cut
@roattribute
def my_value(self):
return "ro"
def ping(self):
"""
Returns (string): pong
"""
return "pong"
def bad_call(self):
"""
always raise an exception
"""
raise MyError
# oneway to ensure that it will be set in a different thread than the call
@oneway
def change_prop(self, value):
"""
set a new value for the VA prop
"""
self.prop.value = value
@isasync
def do_long(self, duration=5):
"""
return a futures.Future
"""
ft = self.executor.submit(self._long_task, duration)
ft.add_done_callback(self._on_end_long)
return ft
def _long_task(self, duration):
"""
returns the time it took
"""
start = time.time()
time.sleep(duration)
return (time.time() - start)
def get_number_futures(self):
return self.number_futures
def set_number_futures(self, value):
self.number_futures = value
def _on_end_long(self, future):
self.number_futures += 1
def sub(self, df):
self._df = df
df.subscribe(self.data_receive)
def unsub(self):
self._df.unsubscribe(self.data_receive)
def data_receive(self, df, data):
logging.info("Received data of shape %r", data.shape)
self.data_count += 1
def get_data_count(self):
return self.data_count
# it'll never be able to answer back if everything goes fine
@oneway
def stopServer(self):
self._pyroDaemon.shutdown()
class OpticalPathManager(object):
"""
The purpose of this module is setting the physical components contained in
the optical path of a SPARC system to the right position/configuration with
respect to the mode given.
"""
def __init__(self, microscope):
"""
microscope (Microscope): the whole microscope component, thus it can
handle all the components needed
"""
self.microscope = microscope
self._graph = affectsGraph(self.microscope)
self._chamber_view_own_focus = False
# Use subset for modes guessed
if microscope.role == "sparc2":
self._modes = copy.deepcopy(SPARC2_MODES)
elif microscope.role in ("sparc-simplex", "sparc"):
self._modes = copy.deepcopy(SPARC_MODES)
elif microscope.role in ("secom", "delphi"):
self._modes = copy.deepcopy(SECOM_MODES)
else:
raise NotImplementedError("Microscope role '%s' unsupported" % (microscope.role,))
# Currently only used with the SECOM/DELPHI
self.quality = ACQ_QUALITY_FAST
# keep list of all components, to avoid creating new proxies
# every time the mode changes
self._cached_components = model.getComponents()
# All the actuators in the microscope, to cache proxy's to them
self._actuators = []
for comp in self._cached_components:
if hasattr(comp, 'axes') and isinstance(comp.axes, dict):
self._actuators.append(comp)
# last known axes position (before going to an alignment mode)
self._stored = {} # (str, str) -> pos: (comp role, axis name) -> position
self._last_mode = None # previous mode that was set
# Removes modes which are not supported by the current microscope
for m, (det, conf) in self._modes.items():
try:
comp = self._getComponent(det)
except LookupError:
logging.debug("Removing mode %s, which is not supported", m)
del self._modes[m]
# Create the guess information out of the mode
# TODO: just make it a dict comprole -> mode
self.guessed = self._modes.copy()
# No stream should ever imply alignment mode
for m in ALIGN_MODES:
try:
del self.guessed[m]
except KeyError:
pass # Mode to delete is just not there
if self.microscope.role in ("secom", "delphi"):
# To record the fan settings when in "fast" acq quality
try:
ccd = self._getComponent("ccd")
except LookupError:
ccd = None
# Check that at least it's a confocal microscope
try:
lm = self._getComponent("laser-mirror")
except LookupError:
logging.warning("Couldn't find a CCD on a SECOM/DELPHI")
self._has_fan_speed = model.hasVA(ccd, "fanSpeed")
self._has_fan_temp = (model.hasVA(ccd, "targetTemperature") and
not ccd.targetTemperature.readonly)
# Consider that by default we are in "fast" acquisition, with the fan
# active (if it ought to be active)
self._fan_enabled = True
# Settings of the fan when the fan is in "active cooling" mode
self._enabled_fan_speed = None
self._enabled_fan_temp = None
# Handle different focus for chamber-view (in SPARCv2)
if "chamber-view" in self._modes:
self._focus_in_chamber_view = None
self._focus_out_chamber_view = None
# Check whether the focus affects the chamber view
try:
chamb_det = self._getComponent(self._modes["chamber-view"][0])
focus = self._getComponent("focus")
if self.affects(focus.name, chamb_det.name):
self._chamber_view_own_focus = True
except LookupError:
pass
if not self._chamber_view_own_focus:
logging.debug("No focus component affecting chamber")
# will take care of executing setPath asynchronously
self._executor = ThreadPoolExecutor(max_workers=1)
def __del__(self):
logging.debug("Ending path manager")
# Restore the spectrometer focus, so that on next start, this value will
# be used again as "out of chamber view".
if self._chamber_view_own_focus and self._last_mode == "chamber-view":
focus_comp = self._getComponent("focus")
if self._focus_out_chamber_view is not None:
logging.debug("Restoring focus from before coming to chamber view to %s",
self._focus_out_chamber_view)
try:
focus_comp.moveAbsSync(self._focus_out_chamber_view)
except IOError as e:
logging.info("Actuator move failed giving the error %s", e)
try:
self._executor.shutdown(wait=False)
except AttributeError:
pass # Not created
def _getComponent(self, role):
"""
same as model.getComponent, but optimised by caching the result.
Uses regex to match the name to a list of cached components
return Component
raise LookupError: if matching component not found
"""
# if we have not returned raise an exception
for comp in self._cached_components:
if comp.role is not None and re.match(role + "$", comp.role):
return comp
# if not found...
raise LookupError("No component with the role %s" % (role,))
def setAcqQuality(self, quality):
"""
Update the acquisition quality expected. Depending on the quality,
some hardware settings will be adjusted.
quality (ACQ_QUALITY): the acquisition quality
"""
assert quality in (ACQ_QUALITY_FAST, ACQ_QUALITY_BEST)
if quality == self.quality:
return
self.quality = quality
if self.microscope.role in ("secom", "delphi"):
if quality == ACQ_QUALITY_FAST:
# Restore the fan (if it was active before)
self._setCCDFan(True)
# Don't turn off the fan if BEST: first wait for setPath()
def setPath(self, mode, detector=None):
"""
Given a particular mode it sets all the necessary components of the
optical path (found through the microscope component) to the
corresponding positions.
path (stream.Stream or str): The stream or the optical path mode
detector (Component or None): The detector which will be targeted on this
path. This can only be set if the path is a str (optical mode). That
is useful in case the mode can be used with multiple detectors (eg,
fiber-align on a SPARC with multiple spectrometers). When path is a
Stream, the Stream.detector is always used.
return (Future): a Future allowing to follow the status of the path
update.
raises (via the future):
ValueError if the given mode does not exist
IOError if a detector is missing
"""
f = self._executor.submit(self._doSetPath, mode, detector)
return f
def _doSetPath(self, path, detector):
"""
Actual implementation of setPath()
"""
if isinstance(path, stream.Stream):
if detector is not None:
raise ValueError("Not possible to specify both a stream, and a detector")
try:
mode = self.guessMode(path)
except LookupError:
logging.debug("%s doesn't require optical path change", path)
return
target = self.getStreamDetector(path) # target detector
else:
mode = path
if mode not in self._modes:
raise ValueError("Mode '%s' does not exist" % (mode,))
comp_role = self._modes[mode][0]
if detector is None:
target = self._getComponent(comp_role)
else:
target = detector
logging.debug("Going to optical path '%s', with target detector %s.", mode, target.name)
# Special SECOM mode: just look at the fan and be done
if self.microscope.role in ("secom", "delphi"):
if self.quality == ACQ_QUALITY_FAST:
self._setCCDFan(True)
elif self.quality == ACQ_QUALITY_BEST:
self._setCCDFan(target.role == "ccd")
fmoves = [] # moves in progress, list of (future, Component, dict(axis->pos) tuples
# Restore the spectrometer focus before any other move, as (on the SR193),
# the value is grating/output dependent
if self._chamber_view_own_focus and self._last_mode == "chamber-view":
focus_comp = self._getComponent("focus")
self._focus_in_chamber_view = focus_comp.position.value.copy()
if self._focus_out_chamber_view is not None:
logging.debug("Restoring focus from before coming to chamber view to %s",
self._focus_out_chamber_view)
fmoves.append((focus_comp.moveAbs(self._focus_out_chamber_view), focus_comp, self._focus_out_chamber_view))
modeconf = self._modes[mode][1]
for comp_role, conf in modeconf.items():
# Try to access the component needed
try:
comp = self._getComponent(comp_role)
except LookupError:
logging.debug("Failed to find component %s, skipping it", comp_role)
continue
# Check whether that actuator affects the target
targets = {target.name} | set(target.affects.value)
if not any(self.affects(comp.name, n) for n in targets):
logging.debug("Actuator %s doesn't affect %s, so not moving it",
comp.name, target.name)
continue
mv = {}
for axis, pos in conf.items():
if axis == "power":
if model.hasVA(comp, "power"):
try:
if pos == 'on':
comp.power.value = comp.power.range[1]
else:
comp.power.value = comp.power.range[0]
logging.debug("Updating power of comp %s to %f", comp.name, comp.power.value)
except AttributeError:
logging.debug("Could not retrieve power range of %s component", comp_role)
continue
if not hasattr(comp, "axes") or not isinstance(comp.axes, dict):
continue
if isinstance(pos, str) and pos.startswith("MD:"):
pos = self.mdToValue(comp, pos[3:])[axis]
if axis in comp.axes:
if axis == "band":
# Handle the filter wheel in a special way. Search
# for the key that corresponds to the value, most probably
# to the 'pass-through'
choices = comp.axes[axis].choices
for key, value in choices.items():
if value == pos:
pos = key
# Just to store current band in order to restore
# it once we leave this mode
if self._last_mode not in ALIGN_MODES:
self._stored[comp_role, axis] = comp.position.value[axis]
break
else:
logging.debug("Choice %s is not present in %s axis", pos, axis)
continue
elif axis == "grating":
# If mirror is to be used but not found in grating
# choices, then we use zero order. In case of
# GRATING_NOT_MIRROR we either use the last known
# grating or the first grating that is not mirror.
choices = comp.axes[axis].choices
if pos == "mirror":
# Store current grating (if we use one at the moment)
# to restore it once we use a normal grating again
if choices[comp.position.value[axis]] != "mirror":
self._stored[comp_role, axis] = comp.position.value[axis]
self._stored[comp_role, 'wavelength'] = comp.position.value['wavelength']
# Use the special "mirror" grating, if it exists
for key, value in choices.items():
if value == "mirror":
pos = key
break
else:
# Fallback to zero order (aka "low-quality mirror")
axis = 'wavelength'
pos = 0
elif pos == GRATING_NOT_MIRROR:
if choices[comp.position.value[axis]] == "mirror":
# if there is a grating stored use this one
# otherwise find the non-mirror grating
if (comp_role, axis) in self._stored:
pos = self._stored[comp_role, axis]
else:
pos = self.findNonMirror(choices)
if (comp_role, 'wavelength') in self._stored:
mv['wavelength'] = self._stored[comp_role, 'wavelength']
else:
pos = comp.position.value[axis] # no change
try:
del self._stored[comp_role, axis]
except KeyError:
pass
try:
del self._stored[comp_role, 'wavelength']
except KeyError:
pass
else:
logging.debug("Using grating position as-is: '%s'", pos)
pass # use pos as-is
elif axis == "slit-in":
if mode in ALIGN_MODES and (comp_role, axis) not in self._stored:
self._stored[comp_role, axis] = comp.position.value[axis]
elif hasattr(comp.axes[axis], "choices") and isinstance(comp.axes[axis].choices, dict):
choices = comp.axes[axis].choices
for key, value in choices.items():
if value == pos:
pos = key
break
# write actuator axis and position in dict
mv[axis] = pos
else:
logging.debug("Not moving axis %s.%s as it is not present", comp_role, axis)
try:
# move actuator
fmoves.append((comp.moveAbs(mv), comp, mv))
except AttributeError:
logging.warning("%s not an actuator", comp_role)
# Now take care of the selectors based on the target detector
fmoves.extend(self.selectorsToPath(target.name))
# If we are about to leave alignment modes, restore values
if self._last_mode in ALIGN_MODES and mode not in ALIGN_MODES:
logging.debug("Leaving align mode %s for %s, will restore positions: %s",
self._last_mode, mode, self._stored)
for (cr, an), pos in self._stored.copy().items(): # copy for deleting entries
if an == "grating":
continue # handled separately via GRATING_NOT_MIRROR
comp = self._getComponent(cr)
fmoves.append((comp.moveAbs({an: pos}), comp, {an: pos}))
del self._stored[cr, an]
# Save last mode
self._last_mode = mode
# wait for all the moves to be completed
for f, comp, mv in fmoves:
try:
# Can be large, eg within 5 min one (any) move should finish.
f.result(timeout=180)
# To do an absolute move, an axis should be referenced (if it
# supports referencing). If not, that's an error (but for now we
# still try, just in case it might work anyway).
for a in mv:
try:
if (model.hasVA(comp, "referenced") and
not comp.referenced.value.get(a, True)):
logging.error("%s.%s is not referenced, it might be a sign of a hardware issue",
comp.name, a)
except Exception:
logging.exception("Failed to check %s.%s is referenced", comp.name, a)
except IOError as e:
logging.warning("Actuator move failed giving the error %s", e)
except:
logging.exception("Actuator move failed!")
raise
# When going to chamber view, store the current focus position, and
# restore the special focus position for chamber, after _really_ all
# the other moves have finished, because the grating/output selector
# moves affects the current position of the focus.
if self._chamber_view_own_focus and mode == "chamber-view":
focus_comp = self._getComponent("focus")
self._focus_out_chamber_view = focus_comp.position.value.copy()
if self._focus_in_chamber_view is not None:
logging.debug("Restoring focus from previous chamber view to %s",
self._focus_in_chamber_view)
try:
focus_comp.moveAbsSync(self._focus_in_chamber_view)
except IOError as e:
logging.warning("Actuator move failed giving the error %s", e)
def selectorsToPath(self, target):
"""
Sets the selectors so the optical path leads to the target component
(usually a detector).
target (str): component name
return (list of futures)
"""
fmoves = []
for comp in self._actuators:
# TODO: pre-cache this as comp/target -> axis/pos
# TODO: don't do moves already done
# TODO: extend the path computation to "for every actuator which _affects_
# the target, move if position known, and update path to that actuator"?
# Eg, this would improve path computation on SPARCv2 with fiber aligner
mv = {}
for an, ad in comp.axes.items():
if hasattr(ad, "choices") and isinstance(ad.choices, dict):
for pos, value in ad.choices.items():
if target in value:
# set the position so it points to the target
mv[an] = pos
comp_md = comp.getMetadata()
if target in comp_md.get(model.MD_FAV_POS_ACTIVE_DEST, {}):
mv.update(comp_md[model.MD_FAV_POS_ACTIVE])
elif target in comp_md.get(model.MD_FAV_POS_DEACTIVE_DEST, {}):
mv.update(comp_md[model.MD_FAV_POS_DEACTIVE])
if mv:
logging.debug("Move %s added so %s targets to %s", mv, comp.name, target)
fmoves.append((comp.moveAbs(mv), comp, mv))
# make sure this component is also on the optical path
fmoves.extend(self.selectorsToPath(comp.name))
return fmoves
def guessMode(self, guess_stream):
"""
Given a stream and by checking its components (e.g. role of detector)
guesses and returns the corresponding optical path mode.
guess_stream (object): The given optical stream
returns (str): Mode estimated
raises:
LookupError if no mode can be inferred for the given stream
IOError if given object is not a stream
"""
if not isinstance(guess_stream, stream.Stream):
raise IOError("Given object is not a stream")
# Handle multiple detector streams
if isinstance(guess_stream, stream.MultipleDetectorStream):
for st in guess_stream.streams:
try:
return self.guessMode(st)
except LookupError:
pass
elif isinstance(guess_stream, stream.OverlayStream):
return "overlay"
else:
for mode, conf in self.guessed.items():
# match the name using regex
if re.match(conf[0] + '$', guess_stream.detector.role):
return mode
# In case no mode was found yet
raise LookupError("No mode can be inferred for the given stream")
def getStreamDetector(self, path_stream):
"""
Given a stream find the optical detector.
path_stream (Stream): The given stream
returns (HwComponent): detector
raises:
ValueError if given object is not a stream
LookupError: if stream has no detector
"""
if not isinstance(path_stream, stream.Stream):
raise ValueError("Given object is not a stream")
# Handle multiple detector streams
if isinstance(path_stream, stream.MultipleDetectorStream):
dets = []
for st in path_stream.streams:
try:
# Prefer the detectors which have a role in the mode, as it's much
# more likely to be the optical detector
# TODO: handle setting multiple optical paths? => return all the detectors
role = st.detector.role
for conf in self.guessed.values():
if re.match(conf[0] + '$', role):
return st.detector
dets.append(st.detector)
except AttributeError:
pass
if dets:
logging.warning("No detector on stream %s has a known optical role", path_stream.name.value)
return dets[0]
elif isinstance(path_stream, stream.OverlayStream):
return path_stream._ccd
else:
try:
return path_stream.detector
except AttributeError:
pass # will raise error just after
raise LookupError("Failed to find a detector on stream %s" % (path_stream.name.value,))
def findNonMirror(self, choices):
"""
Given a dict of choices finds the one with value different than "mirror"
"""
for key, value in choices.items():
if value != "mirror":
return key
else:
raise ValueError("Cannot find grating value in given choices")
def mdToValue(self, comp, md_name):
"""
Just retrieves the "md_name" metadata from component "comp"
"""
md = comp.getMetadata()
try:
return md[md_name]
except KeyError:
raise KeyError("Metadata %s does not exist in component %s" % (md_name, comp.name))
def affects(self, affecting, affected):
"""
Returns True if "affecting" component affects -directly of indirectly-
the "affected" component
affecting (str): component name
affected (str): component name
return bool
"""
path = self.findPath(affecting, affected)
if path is None:
return False
else:
return True
def findPath(self, node1, node2, path=None):
"""
Find any path between node1 and node2 (may not be shortest)
"""
if path is None:
path = []
path = path + [node1]
if node1 == node2:
return path
if node1 not in self._graph:
return None
for node in self._graph[node1]:
if node not in path:
new_path = self.findPath(node, node2, path)
if new_path:
return new_path
return None
def _setCCDFan(self, enable):
"""
Turn on/off the fan of the CCD
enable (boolean): True to turn on/restore the fan, and False to turn if off
"""
if not self._has_fan_speed:
return
if self._fan_enabled == enable:
return
self._fan_enabled = enable
comp = self._getComponent("ccd")
if enable:
if self._enabled_fan_speed is not None:
logging.debug("Turning fan on of %s", comp.name)
comp.fanSpeed.value = max(comp.fanSpeed.value, self._enabled_fan_speed)
else:
if comp.fanSpeed.value == 0:
# Already off => don't touch it
self._enabled_fan_speed = None
self._enabled_fan_temp = None
else:
logging.debug("Turning fan off of %s", comp.name)
self._enabled_fan_speed = comp.fanSpeed.value
comp.fanSpeed.value = 0
# Raise targetTemperature to max/ambient to avoid the fan from
# automatically starting again. (Some hardware have this built-in when
# the current temperature is too high compared to the target)
if self._has_fan_temp:
temp = comp.targetTemperature
if enable:
if self._enabled_fan_temp is not None:
temp.value = min(comp.targetTemperature.value, self._enabled_fan_temp)
try:
self._waitTemperatureReached(comp, timeout=60)
except Exception as ex:
logging.warning("Failed to reach target temperature of CCD: %s",
ex)
else:
# Set ~25°C == ambient temperature
self._enabled_fan_temp = temp.value
try:
try:
temp.value = min(comp.targetTemperature.range[1], 25)
except (AttributeError, NotApplicableError):
temp.value = util.find_closest(25, comp.targetTemperature.choices)
except Exception:
logging.warning("Failed to change targetTemperature when disabling fan",
exc_info=True)
def _waitTemperatureReached(self, comp, timeout=None):
"""
Wait until the current temperature of the component has reached the
target temperature (within some margin).
comp (Component)
timeout (0<float or None): maximum time to wait (in s)
raises:
TimeoutError: if time-out reached
"""
tstart = time.time()
while timeout is None or time.time() < tstart + timeout:
# TODO: adjust the timeout depending on whether the temperature
# gets closer to the target over time or not.
ttemp = comp.targetTemperature.value
atemp = comp.temperature.value
if atemp < ttemp + TEMP_EPSILON:
return
else:
logging.debug(u"Waiting for temperature to reach %g °C (currently at %g °C)",
ttemp, atemp)
time.sleep(1)
raise TimeoutError("Target temperature (%g C) not reached after %g s" %
(comp.targetTemperature.value, timeout))
class SceneService:
# Locks down edit/delete/execute
SCENE_PROCESS_LOCK = threading.Lock()
def __init__(self):
# Scene processors.
self.__sceneController = SceneController.instance()
self.__methodController = MethodController.instance()
self.__sceneUpdateThreadPool = ThreadPoolExecutor(max_workers=1)
self.__sceneExecThreadPool = ThreadPoolExecutor(max_workers=AppConstants.MAX_SCENE_EXEC_THREAD_SIZE)
self.__sceneExecutionResultThreadPool = ThreadPoolExecutor(max_workers=5)
# Scene run workers.
self.__sceneExecLocks = {}
# Favorite edit lock
self.__fav_lock = threading.Lock()
# Listeners.
GroupController.instance().listen_to_group_icon_change(self.__on_group_icon_changed)
self.__methodController.listen_to_method_status_change(self.__on_method_status_changed)
def __on_group_icon_changed(self, kbxGroupId):
'''
Trigger Source: GroupController --> This
Callback when kbxGroupIcon changed.
'''
sceneIds = self.__sceneController.list_scene_ids_which_has_kbx_group_id_as_execution(kbxGroupId)
# Broadcast scenes updated messages.
for sceneId in sceneIds:
self.__broadcast_message__scene_updated(sceneId)
def __on_method_status_changed(self, kbxMethodId, oldKBXMethodStatus, newKBXMethodStatus):
'''
Trigger Source: MethodController --> This
Callback when kbxMethodStatus changed.
'''
if oldKBXMethodStatus != newKBXMethodStatus:
sceneIds = self.__sceneController.list_scene_ids_which_has_kbx_method_id_as_execution(kbxMethodId)
# Broadcast scenes updated messages.
for sceneId in sceneIds:
self.__broadcast_message__scene_updated(sceneId)
@SCENE_PROCESS_SYNC
def set_scene(self, execution, sceneId=None, sceneName=None, sceneProtected=False, sceneIcon=None):
'''
Create/Edit(with sceneId provided) an existing scene.
execution:Dictionary
sceneId:Integer <Optional>
sceneName:String <Optional>
sceneProtected:Boolean <Optional>
sceneIcon:Boolean
Returns "sceneId"
'''
def process_method_list(methodList):
#===================================================================
# Basic type validation
#===================================================================
if not isinstance(methodList, list):
Logger.log_error("SceneService.set_scene: 'execution' must be type of list.")
Logger.log_debug("type:", type(methodList), "value:", methodList)
raise AutomationException(11904, "List is required for both 'execution'")
#===================================================================
# Check allowed size, raise error if exceeded.
#===================================================================
if len(methodList) > AppConstants.MAX_METHOD_SIZE:
Logger.log_error("SceneService.set_scene: 'execution' cannot have more than",
AppConstants.MAX_METHOD_SIZE, "items.")
raise AutomationException(11907, "Only a maximum of " + \
str(AppConstants.MAX_METHOD_SIZE) + " items is allowed for each 'execution'",
lambda text: str(AppConstants.MAX_METHOD_SIZE).join(text.split(":max_item_size:")))
#===================================================================
# Check if all kbxMethodIds are valid and all kbxMethodParams are list
#===================================================================
idValidator = NumberValidator(isRequired=True, decimalPoint=False)
if not all([idValidator.is_valid(eachMethod["kbxMethodId"])
and isinstance(eachMethod["kbxMethodParams"], list)
for eachMethod in methodList]):
raise AutomationException(11904, "'execution' have incorrect data structure.")
#===================================================================
# Check if all kbxParamName and kbxParamCurrentValue exists
#===================================================================
paramNameValidator = StringValidator(isRequired=True)
for eachMethod in methodList:
methodArgs = eachMethod["kbxMethodParams"]
for methodArg in methodArgs:
if not paramNameValidator.is_valid(methodArg[AppConstants.ARG_NAME]):
raise AutomationException(11904, "'execution' have invalid params structure")
if not AppConstants.ARG_CURRENT_VALUE in methodArg:
methodArg[AppConstants.ARG_CURRENT_VALUE] = None
return methodList
#=======================================================================
# Data structure validations
#=======================================================================
sceneId = NumberValidator(isRequired=False, decimalPoint=False).get_value(sceneId)
execution = process_method_list(execution)
#=======================================================================
# Add to database
#=======================================================================
if Util.is_empty(sceneId):
# Validate_max_scene_size
if self.__sceneController.count() >= AppConstants.MAX_SCENE_SIZE:
raise AutomationException(11908,
"Scene size cannot be more than " + str(AppConstants.MAX_SCENE_SIZE),
lambda text: str(AppConstants.MAX_SCENE_SIZE).join(text.split(":max_scene_size:")))
sceneId = self.__sceneController.generate_id(sceneName)
scene = {}
elif self.__sceneController.has(sceneId):
self.__verify_scene_updated(sceneId)
self.__sceneController.change_to_updating(sceneId, sceneName)
scene = self.__sceneController.get(sceneId)
scene = dict(scene)
else:
raise AutomationException(11902, "Scene not found")
#=======================================================================
# Broadcast message: starts to update scene.
#=======================================================================
self.__broadcast_message__scene_update_started(sceneId, sceneName)
#=======================================================================
# Set basic information of the scene
#=======================================================================
scene["sceneId"] = sceneId
scene["sceneName"] = sceneName
scene["sceneProtected"] = sceneProtected
scene["sceneIcon"] = sceneIcon
scene["execution"] = execution
#=======================================================================
# Append if its new scene
#=======================================================================
def __update_scene(scene):
try:
# Fire scene update start event
sceneId = scene["sceneId"]
sceneName = scene["sceneName"]
# Add methods to subscribe list
methodIds = [kbxMethod["kbxMethodId"] for kbxMethod in scene["execution"]]
self.__methodController.add(methodIds)
# Update "scene" base table
self.__sceneController.update(scene)
self.__sceneController.commit()
except Exception as e:
self.__sceneController.rollback()
self.__broadcast_message__scene_update_failed(sceneId, sceneName)
Logger.log_error("SceneService __update_scene failed:", e, "-- rolledback")
else:
# Broadcast message: completed updating a scene
self.__broadcast_message__scene_updated(sceneId)
#=======================================================================
# Submit to a thread to process other info, and return... performance...
#=======================================================================
# Only 1 worker in the threadPool, it works as threading.Lock
self.__sceneUpdateThreadPool.submit(__update_scene, scene)
return sceneId
@SCENE_PROCESS_SYNC
def delete_scene(self, sceneId):
self.__verify_scene_updated(sceneId)
try:
favSort = self.__sceneController.get_favsort_of(sceneId) # To determine should favorited_Scene_deleted broadcasted
self.__sceneController.delete(sceneId)
self.__sceneController.commit()
except Exception as e:
self.__sceneController.rollback()
traceback.print_exc()
Logger.log_error("SceneService delete_scene ex:", e, "-- rolled back")
raise AutomationException(11906, "Unable to delete scene, problem - " + str(e))
else:
self.__broadcast_message__scene_deleted(sceneId)
if favSort is not None:
self.__broadcast_message__favorited_scene_deleted(sceneId)
@SCENE_PROCESS_SYNC
def execute_scene(self, sceneId, serUrl=None, language="en"):
'''
Execute a scene.
Scene execution will only be recorded if serUrl is specified.
'''
self.__verify_scene_updated(sceneId)
self.__sceneExecLocks.setdefault(sceneId, SceneExecLock())
sceneExecLock = self.__sceneExecLocks.get(sceneId)
isAcquired = sceneExecLock.acquire(False) # Raise error if failed to acquire.
if isAcquired == True:
try:
#===================================================================
# Should record execution? Depends on serUrl.
#===================================================================
if not Util.is_empty(serUrl):
# Get sceneName and sceneProtected
scene = self.__sceneController.get(sceneId)
sceneName, sceneProtected = scene["sceneName"], scene["sceneProtected"]
serObj = {"serUrl":serUrl, "sceneName":sceneName, "sceneProtected":sceneProtected}
else:
serObj = None
#=======================================================================
# List execution methods
#=======================================================================
execution = self.__sceneController.list_executions(sceneId)
self.__sceneExecThreadPool.submit(self.__execute_scene_implementation,
sceneId=sceneId,
execution=execution,
serObj=serObj,
language=language)
except Exception as e:
del(self.__sceneExecLocks[sceneId])
sceneExecLock.release()
traceback.print_exc() # This is unusual error
raise e
else:
raise AutomationException(11901, "the scene is already executing. " + \
"Use 'stop_scene' to cancel its execution.")
def stop_scene(self, sceneId):
'''
Stop a scene execution.
'''
sceneExecLock = self.__sceneExecLocks.get(sceneId, None)
if sceneExecLock is None:
raise AutomationException(11905, "Call stop only after execute_scene is called")
else:
sceneExecLock.set_stop()
def get_scene(self, sceneId, language=AppInfo.DEFAULT_API_LANGUAGE):
try:
scene = self.__sceneController.get_detail(sceneId)
except:
raise AutomationException(11902, "Scene ID provided: " + str(sceneId))
kbxMethods = scene["execution"]
# -------------- Compile lists of kbxMethod and group IDs contains in this scene.
kbxMethodIdsToList = {}
kbxGroupIdsToList = set([])
for kbxMethod in kbxMethods:
# Variables
kbxMethodId = kbxMethod["kbxMethodId"]
kbxMethodAppId = kbxMethod["kbxMethodAppId"]
kbxMethodStatus = kbxMethod["kbxMethodStatus"]
kbxGroupId = kbxMethod["kbxGroupId"]
kbxGroupStatus = kbxMethod["kbxGroupStatus"]
if kbxMethodStatus is not -1 and kbxMethodAppId is not None:
kbxMethodIdsToList.setdefault(kbxMethodAppId, set([]))
kbxMethodIdsToList[kbxMethodAppId].add(kbxMethodId)
if kbxGroupId is not None and kbxGroupStatus is not -1:
kbxGroupIdsToList.add(kbxGroupId)
# -------------- Get methods and groups based on requested language.
kbxMethodIdsListed = {}
kbxGroupIdsListed = {}
for kbxMethodAppId, kbxMethodIds in kbxMethodIdsToList.items():
kbxMethodIdsListed[kbxMethodAppId] = SharedMethodWrapper.list_shared_methods_by_app_id(kbxMethodAppId,
kbxMethodIds,
language=language)
groupList = SharedMethodWrapper.list_shared_method_groups(kbxGroupId=kbxGroupIdsToList, language=language)
for row in groupList:
kbxGroupIdsListed[row["kbxGroupId"]] = row
# -------------- Set method and group data into scene.
for kbxMethod in kbxMethods:
# Variables
kbxMethodId = kbxMethod["kbxMethodId"]
kbxMethodAppId = kbxMethod["kbxMethodAppId"]
kbxMethodStatus = kbxMethod["kbxMethodStatus"]
kbxGroupId = kbxMethod["kbxGroupId"]
kbxGroupStatus = kbxMethod["kbxGroupStatus"]
if kbxMethodStatus is not -1 and kbxMethodAppId is not None:
kbxMethodParamsWithCurrentValue = {
kbxMethodParam["kbxParamName"]: kbxMethodParam["kbxParamCurrentValue"] \
for kbxMethodParam in kbxMethod["kbxMethodParams"]
}
kbxMethodWithDetails = kbxMethodIdsListed[kbxMethodAppId][kbxMethodId]
if kbxMethodWithDetails is not None:
kbxMethodParamsWithDetails = kbxMethodWithDetails["kbxMethodParams"]
kbxMethodParamsWithDetails = copy.deepcopy(kbxMethodParamsWithDetails)
for kbxMethodParam in kbxMethodParamsWithDetails:
kbxMethodParam["kbxParamCurrentValue"] = kbxMethodParamsWithCurrentValue.get(
kbxMethodParam["kbxParamName"], None)
kbxMethod["kbxMethodParams"] = kbxMethodParamsWithDetails
kbxMethod["kbxMethodLabel"] = kbxMethodWithDetails.get("kbxMethodLabel")
kbxMethod["kbxMethodDesc"] = kbxMethodWithDetails.get("kbxMethodDesc")
else:
kbxMethod["atDebug"] = "Unable to get shared method, caused by a method which never register itself on this bootup."
if kbxGroupId is not None and kbxGroupStatus is not -1:
try:
kbxMethod["kbxGroupLabel"] = kbxGroupIdsListed[kbxGroupId]["kbxGroupLabel"]
kbxMethod["kbxGroupDesc"] = kbxGroupIdsListed[kbxGroupId]["kbxGroupDesc"]
except:
kbxMethod["atDebugGroup"] = "Unable to get shared method group, caused by a group which never register itself on this bootup."
return scene
def list_scenes(self, offset=0, limit=20):
return self.__sceneController.list(offset, limit), \
self.__sceneController.count()
def set_favorited_scene(self, sceneId, prevSceneId=None):
'''
sceneId - can be either favorited/non-favorited scene, but must be a valid scene id.
prevSceneId - must be another favorited scene or error is raised.
Both updating and executing doesn't blocks a scene from adding to/removing from favorited list.
'''
with self.__fav_lock:
# Validation for sceneId.
if not self.__sceneController.has(sceneId):
raise AutomationException(11092, "sceneId does not belongs to any scene - sceneId provided: " + str(sceneId))
# Because UI display in reversed order, hence their prevSceneId == our nextSceneId
nextSceneId = prevSceneId or None
# Get favSort before sceneId of nextSceneId
if nextSceneId is None:
maxFavSort = self.__sceneController.get_largest_favsort_num() # If len of favorited list is 0, maxFavSort = 0
# favSort stores number to be assigned to sceneId.
favSort = maxFavSort + 1
else:
try:
# favSort stores number to be assigned to sceneId.
# current favSort of prevSceneId will becomes favSort of sceneId.
favSort = self.__sceneController.get_favsort_of(nextSceneId)
except Exception as e:
raise AutomationException(11092, "prevSceneId does not belongs to any scene - " + \
"prevSceneId provided: " + str(nextSceneId) + ", error: " + str(e))
else:
if favSort is None:
raise AutomationException(13000, "prevSceneId does not belongs to any favorited scene - " + \
"prevSceneId provided: " + str(nextSceneId) + ", error: " + str(e))
# Update favSort of the scene
try:
self.__sceneController.update_favorited_scene(sceneId, favSort)
self.__sceneController.commit()
except Exception as e:
self.__sceneController.rollback()
Logger.log_error("SceneService.set_favorited_scene failed to update favorited scene, ex:", str(e))
raise AutomationException(13001, "unexpected error: " + str(e))
# Broadcast events
self.__broadcast_message__scene_updated(sceneId)
self.__broadcast_message__favorited_scene_added(sceneId, prevSceneId)
def delete_favorited_scene(self, sceneId):
'''
sceneId - must be a favorited scene.
'''
with self.__fav_lock:
try:
favSort = self.__sceneController.get_favsort_of(sceneId)
except Exception as e:
raise AutomationException(11092, "prevSceneId does not belongs to any scene - " + \
"prevSceneId provided: " + str(sceneId) + ", error: " + str(e))
else:
if favSort is None:
raise AutomationException(13000, "sceneId does not belongs to any favorited scene - " + \
"sceneId provided: " + str(sceneId))
# This method raise error and rollback automatically if failed, or commit once succeed.
try:
self.__sceneController.delete_favorited_scene(sceneId)
self.__sceneController.commit()
except Exception as e:
self.__sceneController.rollback()
Logger.log_error("SceneService.delete_favorited_scene failed to delete favorited scene, ex:", str(e))
raise AutomationException(13002, "unexpected error: " + str(e))
#Broadcast event
self.__broadcast_message__scene_updated(sceneId)
self.__broadcast_message__favorited_scene_deleted(sceneId)
def list_favorited_scene(self, offset=0, limit=200):
'''
Favorited scenes.
'''
return self.__sceneController.list_favorited_scenes_reversed(offset, limit), \
self.__sceneController.count_favorited_scenes()
def __verify_scene_updated(self, sceneId):
try:
statusProcessed = self.__sceneController.get_status_processed(sceneId)
if statusProcessed == AppConstants.SCENE_STATUS_UPDATING:
raise AutomationException(11903, "edit/execute/delete scene is not allowed when its updating. " + \
"Wait until the update process is completed.")
except:
raise AutomationException(11902, "Scene ID provided:" + str(sceneId))
def __broadcast_message__scene_update_started(self, sceneId, sceneName=None):
eventTag = AppConstants.EVENT_SCENE_UPDATE_STARTED
eventData = {"sceneId": sceneId, "newSceneName":sceneName}
self.__broadcast_message(eventTag, eventData)
Logger.log_info("Scene Start Update:", sceneName)
def __broadcast_message__scene_updated(self, sceneId):
try:
scene = self.__sceneController.get_summary(sceneId)
except Exception as e:
Logger.log_error("SceneService.__broadcast_message__scene_updated get_summary ex:", e)
scene = None
eventTag = AppConstants.EVENT_SCENE_UPDATED
eventData = {"sceneId":sceneId, "newSceneSummary":scene}
self.__broadcast_message(eventTag, eventData)
Logger.log_info("Scene Updated:", scene["sceneName"])
def __broadcast_message__scene_update_failed(self, sceneId, sceneName=None):
'''
sceneName - For debugging purpose.
'''
try:
scene = self.__sceneController.get_summary(sceneId)
except Exception:
scene = None
eventTag = AppConstants.EVENT_SCENE_UPDATE_FAILED
eventData = {"sceneId": sceneId, "oldSceneSummary":scene}
self.__broadcast_message(eventTag, eventData)
Logger.log_info("Scene Update Failed:", sceneName)
def __broadcast_message__scene_deleted(self, sceneId):
eventTag = AppConstants.EVENT_SCENE_DELETED
eventData = {"sceneId": sceneId}
self.__broadcast_message(eventTag, eventData)
Logger.log_info("Scene Deleted: Id -", sceneId)
def __broadcast_message__favorited_scene_added(self, sceneId, prevSceneId):
eventTag = AppConstants.EVENT_FAVORITED_SCENE_ADDED
eventData = {"sceneId":sceneId, "prevSceneId":prevSceneId}
self.__broadcast_message(eventTag, eventData)
Logger.log_info("Favorited scene added/updated: Id", sceneId, "prevId:", prevSceneId)
def __broadcast_message__favorited_scene_deleted(self, sceneId):
eventTag = AppConstants.EVENT_FAVORITED_SCENE_DELETED
eventData = {"sceneId":sceneId}
self.__broadcast_message(eventTag, eventData)
Logger.log_info("Favorited scene deleted: Id", sceneId)
def __broadcast_message(self, eventTag, eventData):
eventData = json.dumps(eventData, cls=AutomationJSONEncoder)
Application.send_web_server_event(eventTag, eventData)
def __execute_scene_implementation(self, sceneId, execution, serObj, language):
'''
** Call execute_scene; DO NOT call this function directly.
'''
Logger.log_info("execute scene id:", sceneId)
def execution_func(sceneThreadEvent, kbxMethodId, seri, **kwargs):
try:
if kbxMethodId == -291:
# Delay Timer
delayInSec = kwargs["delayInSec"]
sceneThreadEvent.wait(delayInSec)
seri["seriError"] = None
else:
# Execute method
result = SharedMethod.call(**kwargs)
seri["seriError"] = str(result)
seri["seriStatus"] = "ok"
except Exception as e:
seri["seriStatus"] = "error"
seri["seriError"] = str(e)
Logger.log_debug("Execution failed, method:", kwargs["kbxMethodName"])
finally:
sceneThreadEvent.set()
try:
# Record for debugging purpose
serStartTime = time.time()
#===================================================================
# Prepare to execute execution methods
#===================================================================
sceneExecLock = self.__sceneExecLocks.get(sceneId)
sceneThreadEvent = sceneExecLock.get_thread_event()
seris = deque()
methodExecTime = int(time.time())
isLoopCompleted = True
for row in execution:
kbxMethodId = row["kbxMethodId"]
kbxMethodName = row["kbxMethodName"]
kbxGroupId = row["kbxGroupId"]
kbxMethodStatus = row["kbxMethodStatus"]
methodParamsWithCurrentValues = row["kbxMethodParams"]
seri = {"kbxMethodId":kbxMethodId,
"kbxGroupId":kbxGroupId,
"kbxMethodName":kbxMethodName,
"kbxMethodParams":methodParamsWithCurrentValues}
seris.append(seri)
# Check is stop
if sceneExecLock.is_stop():
if not isLoopCompleted:
serEndTime = time.time()
isLoopCompleted = False
# === Execution interrupted ===
continue
# Check is method not removed
elif kbxMethodStatus not in (SharedMethod.METHOD_STATUS_ACTIVE, SharedMethod.METHOD_STATUS_INACTIVE):
seri["seriStatus"] = "error"
seri["seriError"] = "method is removed"
continue
kwargs = {methodParam[AppConstants.ARG_NAME]: methodParam[AppConstants.ARG_CURRENT_VALUE]
for methodParam in methodParamsWithCurrentValues}
if AppInfo.REQUEST_KEY_LANGUAGE not in kwargs:
kwargs[AppInfo.REQUEST_KEY_LANGUAGE] = AppInfo.DEFAULT_API_LANGUAGE
kwargs["kbxMethodName"] = kbxMethodName
kwargs["kbxGroupId"] = kbxGroupId
kwargs["kbxModuleName"] = row["kbxModuleName"]
kwargs["kbxMethodAppId"] = row["kbxMethodAppId"]
kwargs[AppConstants.KEY_ACTION_TIMESTAMP] = methodExecTime
#===========================================================
# Execute method
#===========================================================
sceneThreadEvent.clear()
execThread = threading.Thread(target=execution_func, args=[sceneThreadEvent, row["kbxMethodId"], seri], kwargs=kwargs)
execThread.daemon = False
execThread.start()
sceneThreadEvent.wait() # Event will be set by "stop_scene" or SharedMethod.call returns.
# Check is stop
if sceneExecLock.is_stop():
if not isLoopCompleted:
serEndTime = time.time()
isLoopCompleted = False
# === Execution interrupted ===
continue
if isLoopCompleted:
# Record for debugging purpose
serEndTime = time.time()
# === Execution completed ===
except Exception as e:
# This is unexpected error. Execution will not be recorded by Scene Execution Result.
Logger.log_error("SceneService.__execute_scene_implementation ex:", e)
# === Execution completed with errors ===
else:
if serObj is not None:
# Log to sceneExecutionResultService
self.__sceneExecutionResultThreadPool.submit(self.__add_scene_execution_result,
serUrl=serObj["serUrl"],
serStartTime=serStartTime,
serEndTime=serEndTime,
sceneName=serObj["sceneName"],
sceneProtected=serObj["sceneProtected"],
sceneId=sceneId,
execution=seris,
language=language)
finally:
del(self.__sceneExecLocks[sceneId]) # Delete exec lock
sceneExecLock.release()
def __add_scene_execution_result(self, serUrl, serStartTime, serEndTime, sceneName, sceneProtected, sceneId, execution, language):
try:
SceneExecutionResultService.instance().add_scene_execution_result(serUrl=serUrl,
serStartTime=serStartTime,
serEndTime=serEndTime,
sceneName=sceneName,
sceneProtected=sceneProtected,
sceneId=sceneId,
execution=execution,
language=language)
except Exception as e:
Logger.log_error("SceneService.__add_scene_execution_result ex:", e)
def do_test3(workers):
param = {"max_workers": workers}
loop = asyncio.new_event_loop()
lock = threading.Lock()
tresult = []
presult = []
cresult = []
pre_input1 = input_generator(workers, 0)
pre_input2 = input_generator(workers, max(pre_input1))
pre_input3 = input_generator(workers, max(pre_input2))
def result_checker(list, lock, fut):
with lock:
try:
list.append(fut.result())
except Exception as e:
list.append(e)
texec = ThreadPoolExecutor(**param)
pexec = ProcessPoolExecutor(**param)
cexec = CoroutinePoolExecutor(**param, loop=loop)
tstart = round(time.time()+1)
input1 = [tstart + i for i in pre_input1]
input2 = [tstart + i for i in pre_input2]
input3 = [tstart + i for i in pre_input3]
for x in input1:
future = texec.submit(wake_at, x)
future.add_done_callback(
functools.partial(result_checker, tresult, lock))
result_iter = texec.map(wake_at, input2)
for x in input3:
future = texec.submit(wake_at, x)
future.add_done_callback(
functools.partial(result_checker, tresult, lock))
for x in result_iter:
with lock:
tresult.append(x)
texec.shutdown(True)
pstart = round(time.time() + _start_warm_up)
input1 = [pstart + i for i in pre_input1]
input2 = [pstart + i for i in pre_input2]
input3 = [pstart + i for i in pre_input3]
for x in input1:
future = pexec.submit(wake_at, x)
future.add_done_callback(
functools.partial(result_checker, presult, lock))
result_iter = pexec.map(wake_at, input2)
for x in input3:
future = pexec.submit(wake_at, x)
future.add_done_callback(
functools.partial(result_checker, presult, lock))
for x in result_iter:
with lock:
presult.append(x)
pexec.shutdown(True)
cstart = round(time.time() + _start_warm_up)
input1 = [cstart + i for i in pre_input1]
input2 = [cstart + i for i in pre_input2]
input3 = [cstart + i for i in pre_input3]
async def async_main():
for x in input1:
future = cexec.submit(async_wake_at, x)
future.add_done_callback(
functools.partial(result_checker, cresult, lock))
result_iter = cexec.map(async_wake_at, input2)
for x in input3:
future = cexec.submit(async_wake_at, x)
future.add_done_callback(
functools.partial(result_checker, cresult, lock))
async for x in result_iter:
with lock:
cresult.append(x)
await cexec.shutdown(False)
loop.run_until_complete(async_main())
try:
loop.run_until_complete(cexec.shutdown(True))
texec.shutdown(True)
pexec.shutdown(True)
finally:
loop.close()
tresult = [round((x - tstart) / _precision) for x in tresult]
presult = [round((x - pstart) / _precision) for x in presult]
cresult = [round((x - cstart) / _precision) for x in cresult]
result = True
for (t, p, c) in zip(tresult, presult, cresult):
result = result and (t == p)
if not result:
print(tresult)
print(presult)
print(cresult)
print(t,p,c)
assert False
result = result and (p == c)
if not result:
print(tresult)
print(presult)
print(cresult)
print(t, p, c)
assert False
result = result and (c == t)
if not result:
print(tresult)
print(presult)
print(cresult)
print(t, p, c)
assert False
return result
def do_test1(workers):
param = {"max_workers": workers}
start = round(time.time() + _start_warm_up)
input = input_generator(workers, start)
loop = asyncio.new_event_loop()
lock = threading.Lock()
tresult = []
presult = []
cresult = []
def result_checker(list, lock, fut):
with lock:
try:
list.append(fut.result())
except Exception as e:
list.append(e)
texec = ThreadPoolExecutor(**param)
pexec = ProcessPoolExecutor(**param)
cexec = CoroutinePoolExecutor(**param, loop=loop)
for x in input:
future = texec.submit(wake_at, x)
future.add_done_callback(
functools.partial(result_checker, tresult, lock))
future = pexec.submit(wake_at, x)
future.add_done_callback(
functools.partial(result_checker, presult, lock))
future = cexec.submit(async_wake_at, x)
future.add_done_callback(
functools.partial(result_checker, cresult, lock))
texec.shutdown(False)
pexec.shutdown(False)
loop.run_until_complete(cexec.shutdown(False))
try:
loop.run_until_complete(cexec.shutdown(True))
texec.shutdown(True)
pexec.shutdown(True)
finally:
loop.close()
tresult = [round((x - start) / _precision) for x in tresult]
presult = [round((x - start) / _precision) for x in presult]
cresult = [round((x - start) / _precision) for x in cresult]
result = True
for (t, p, c) in zip(tresult, presult, cresult):
result = result and (t == p)
if not result:
print(tresult)
print(presult)
print(cresult)
print(t, p, c)
assert False
result = result and (p == c)
if not result:
print(tresult)
print(presult)
print(cresult)
print(t, p, c)
assert False
result = result and (c == t)
if not result:
print(tresult)
print(presult)
print(cresult)
print(t, p, c)
assert False
return result
class AlignedSEMStream(SEMStream):
"""
This is a special SEM stream which automatically first aligns with the
CCD (using spot alignment) every time the stage position changes.
Alignment correction can either be done via beam shift (=shift), or
by just updating the image position.
"""
def __init__(self, name, detector, dataflow, emitter,
ccd, stage, focus, shiftebeam=MTD_MD_UPD, **kwargs):
"""
shiftebeam (MTD_*): if MTD_EBEAM_SHIFT, will correct the SEM position using beam shift
(iow, using emitter.shift). If MTD_MD_UPD, it will just update the
position correction metadata on the SEM images.
ccd (Optical detector)
stage (actuator): the sample stage, just to know when re-alignment is needed
focus (actuator): the _optical_ focuser, just to know when re-alignment is needed
focuser (actuator): the _e-beam_ focuser, to allow focusing the image
"""
super(AlignedSEMStream, self).__init__(name, detector, dataflow, emitter, **kwargs)
self._ccd = ccd
self._stage = stage
self._focus = focus
self._shiftebeam = shiftebeam
self.calibrated = model.BooleanVA(False) # whether the calibration has been already done
self._last_pos = stage.position.value.copy()
self._last_pos.update(focus.position.value) # last known position of the stage
self._shift = (0, 0) # (float, float): shift to apply in meters
self._last_shift = (0, 0) # (float, float): last ebeam shift applied
# In case initialization takes place in unload position the
# calibration values are not obtained yet. Thus we avoid to initialize
# cur_trans before spot alignment takes place.
self._cur_trans = None
stage.position.subscribe(self._onMove)
focus.position.subscribe(self._onMove)
self._executor = ThreadPoolExecutor(max_workers=1)
self._beamshift = None
def _onMove(self, pos):
"""
Called when the stage moves (changes position)
pos (dict): new position
"""
# Check if the position has really changed, as some stage tend to
# report "new" position even when no actual move has happened
logging.debug("Stage location is %s m,m,m", pos)
if self._last_pos == pos:
return
self._last_pos.update(pos)
# if self.is_active.value:
self.calibrated.value = False
# just reset status
self._setStatus(None)
# need to override it to support beam shift
def _applyROI(self):
"""
Update the scanning area of the SEM according to the roi
"""
res, shift = self._computeROISettings(self.roi.value)
if (self._shiftebeam == MTD_EBEAM_SHIFT) and (self._beamshift is not None):
shift = tuple(s + c for s, c in zip(shift, self._beamshift))
# always in this order
self._emitter.resolution.value = res
self._emitter.shift.value = shift
def _compensateShift(self):
"""
Compensate the SEM shift, using either beam shift or metadata update
"""
# update the correction metadata
logging.debug("Update metadata for SEM image shift")
self._detector.updateMetadata({MD_POS_COR: self._shift})
def _prepare(self):
"""
Perform calibration if needed
"""
logging.debug("Preparing stream %s ...", self)
# actually indicate that preparation has been triggered, don't wait for
# it to be completed
self._prepared = True
f = self._executor.submit(self._DoPrepare)
# Note that there is no need to call super(). This would only check
# for an optical path manager which in this case has no effect.
return f
def __del__(self):
self._executor.shutdown(wait=False)
def _DoPrepare(self):
# Need to calibrate ?
if not self.calibrated.value:
self._setStatus(logging.INFO, u"Automatic SEM alignment in progress…")
# store current settings
no_spot_settings = (self._emitter.dwellTime.value,
self._emitter.resolution.value)
# Don't mess up with un/subscribing while doing the calibration
self._getEmitterVA("dwellTime").unsubscribe(self._onDwellTime)
self._getEmitterVA("resolution").unsubscribe(self._onResolution)
shift = (0, 0)
self._beamshift = None
try:
logging.info("Determining the Ebeam center position")
# TODO Handle cases where current beam shift is larger than
# current limit. Happens when accel. voltage is changed
self._emitter.shift.value = (0, 0)
shift = FindEbeamCenter(self._ccd, self._detector, self._emitter)
logging.debug("Spot shift is %s m,m", shift)
self._beamshift = shift
# Also update the last beam shift in order to be used for stage
# offset correction in the next stage moves
self._last_shift = (0.75 * self._last_shift[0] - 0.25 * shift[0],
0.75 * self._last_shift[1] - 0.25 * shift[1])
cur_trans = self._stage.getMetadata().get(model.MD_POS_COR, (0, 0))
self._cur_trans = (cur_trans[0] - self._last_shift[0],
cur_trans[1] - self._last_shift[1])
self._stage.updateMetadata({
model.MD_POS_COR: self._cur_trans
})
logging.debug("Compensated stage translation %s m,m", self._cur_trans)
if self._shiftebeam == MTD_EBEAM_SHIFT:
# First align using shift
self._applyROI()
# Then by updating the metadata
shift = (0, 0) # just in case of failure
shift = FindEbeamCenter(self._ccd, self._detector, self._emitter)
elif self._shiftebeam == MTD_MD_UPD:
pass
else:
raise NotImplementedError("Unknown shiftbeam method %s" % (self._shiftebeam,))
except LookupError:
self._setStatus(logging.WARNING, (u"Automatic SEM alignment unsuccessful", u"Need to focus all streams"))
# logging.warning("Failed to locate the ebeam center, SEM image will not be aligned")
except Exception:
self._setStatus(logging.WARNING, (u"Automatic SEM alignment unsuccessful", u"Need to focus all streams"))
logging.exception("Failure while looking for the ebeam center")
else:
self._setStatus(None)
logging.info("Aligning SEM image using shift of %s", shift)
self.calibrated.value = True
finally:
# restore hw settings
(self._emitter.dwellTime.value,
self._emitter.resolution.value) = no_spot_settings
self._getEmitterVA("dwellTime").subscribe(self._onDwellTime)
self._getEmitterVA("resolution").subscribe(self._onResolution)
self._shift = shift
self._compensateShift()
class Bender(object):
def __init__(self, backbone, brain=None):
self._backbone = backbone
self._brain = brain if brain is not None else Brain()
self._brain_lock = threading.Lock()
self._regex_to_response = OrderedDict()
self._scripts = OrderedDict()
self._pool = ThreadPoolExecutor(max_workers=4)
self._futures = [] # list of futures submitted to the pool
self._stop_loop = threading.Event()
def register_script(self, name, script):
self._scripts[name] = script
def register_builtin_scripts(self):
for name, script in scripts.get_builtin_scripts():
self.register_script(name, script)
def register_setuptools_scripts(self):
for p in pkg_resources.iter_entry_points('bender_script'):
obj = p.load()
if inspect.isclass(obj):
obj = obj()
self.register_script(p.name, obj)
def get_script(self, name):
return self._scripts[name]
def iter_scripts(self):
return iter(self._scripts.items())
def start(self):
self._brain.load()
self._backbone.on_message_received = self.on_message_received
self.register_builtin_scripts()
self.register_setuptools_scripts()
for script in self._scripts.values():
hooks.call_unique_hook(script, 'script_initialize_hook',
brain=self._brain)
hooks.call_unique_hook(self._backbone, 'backbone_start_hook')
def shutdown(self):
self._pool.shutdown(wait=True)
for name, script in list(self._scripts.items()):
self._scripts.pop(name)
hooks.call_unique_hook(script, 'script_shutdown_hook',
brain=self._brain)
hooks.call_unique_hook(self._backbone, 'backbone_shutdown_hook',
brain=self._brain)
self._brain.dump()
self._stop_loop.set()
def request_shutdown(self):
self._stop_loop.set()
def loop(self):
self.start()
self._stop_loop.wait()
self.shutdown()
def on_message_received(self, msg):
def thread_exec(hook, brain, msg, match):
try:
hooks.call(hook, brain=self._brain, msg=msg, match=match,
bender=self)
except Exception as e:
msg.reply('*BZZT* %s' % e)
else:
with self._brain_lock:
brain.dump()
handled = False
for script in self._scripts.values():
for hook in hooks.find_hooks(script, 'respond_hook'):
match = re.match(hook.inputs['regex'], msg.get_body(),
re.IGNORECASE | re.DOTALL)
if match:
f = self._pool.submit(thread_exec, hook, self._brain, msg,
match)
self._futures.append(f)
handled = True
if not handled:
msg.reply('Command not recognized')
def wait_all_messages(self):
while self._futures:
f = self._futures.pop()
f.result() # wait until future returns
def submit(self, fn, *args, **kwargs):
if isinstance(fn, Task):
self._task_map[id(fn)] = fn
return ThreadPoolExecutor.submit(self, fn, *args, **kwargs)
class SceneExecutionResultService:
__INSTANCE = None
__LOCK = threading.Lock()
@staticmethod
def instance():
with SceneExecutionResultService.__LOCK:
SceneExecutionResultService.__INSTANCE or SceneExecutionResultService()
return SceneExecutionResultService.__INSTANCE
def __init__(self):
self.__lock = threading.Lock()
self.__serLock = SERLock()
self.__serController = SceneExecutionResultController()
self.__serRetryThreadPool = ThreadPoolExecutor(max_workers=3)
def after_ser_deleted(serId):
self.__serLock.declare_as_deleted(serId) # Report serId is deleted to SERLock instance.
self.__broadcast_message__ser_deleted(serId) # Broadcast event to mobile client.
SceneExecutionResultDataHandler.AFTER_RECORD_DELETE = after_ser_deleted
SceneExecutionResultService.__INSTANCE = self
def add_scene_execution_result(self, serUrl, serStartTime, serEndTime, sceneName, sceneProtected, sceneId, execution, language="en"):
'''
Get kbxGroupLabel for each execution item and pass to serController.
'''
# Get kbxGroupLabel
for item in execution:
kbxGroupId = item.pop("kbxGroupId", None)
if kbxGroupId is None:
item["kbxGroupLabel"] = "-- Unknown --"
else:
try:
kbxGroup = SharedMethodWrapper.get_shared_method_group_by_id(kbxGroupId, enableTagCount=False, language=language)
item["kbxGroupLabel"] = kbxGroup.get("kbxGroupLabel", "-- Unnamed --")
except:
item["kbxGroupLabel"] = "-- Removed --"
with self.__lock: # Add scene execution result happens sequentially.
# Make serCreatedTime unique.
# This is the smallest pause unit to make a difference on each time.time() call.
time.sleep(0.001)
serCreatedTime = time.time()
serId = str(serCreatedTime)
self.__serController.add(serId, serCreatedTime, serStartTime, serEndTime,
sceneId, sceneName, sceneProtected, execution)
# Build notification information.
KBXLang.set_preferred_lang(language)
notiLink = str(serUrl) + "?serId=" + str(serId)
notiLinkLabel = str(KBXLang("ser_noti_link_label"))
for seri in execution:
if seri["seriStatus"] == "error":
notiContent = str(KBXLang("ser_noti_exec_completed_with_error"))
break
else:
notiContent = str(KBXLang("ser_noti_exec_completed"))
notiContent = str(sceneName).join(notiContent.split(":scenename:"))
self.__notify_ser_added(notiContent, notiLink, notiLinkLabel)
def get_scene_execution_result(self, serId, language):
'''
Get complete serObject from serController, and add in full details obtained from shared method manager.
'''
try:
serObject = self.__serController.get_ser_and_seris(serId)
except:
raise AutomationException(12000, "Scene Execution Result not found, ID provided:" + str(serId))
execution = serObject["execution"]
# Add details into each execution item.
for kbxMethod in execution:
kbxMethodId = kbxMethod["kbxMethodId"]
try:
kbxMethodWithDetails = SharedMethodWrapper.get_shared_method_by_id(kbxMethodId, language)
kbxMethod["kbxMethodLabel"] = kbxMethodWithDetails.get("kbxMethodLabel")
# Rebuild parameters.
paramsWithDetails = {}
for paramWithDetails in kbxMethodWithDetails.get("kbxMethodParams", None):
paramsWithDetails[paramWithDetails["kbxParamName"]] = paramWithDetails
for kbxMethodParam in kbxMethod["kbxMethodParams"]:
try:
# Update kbxMethodParam with the one with details.
kbxParamCurrentValue = kbxMethodParam["kbxParamCurrentValue"]
kbxMethodParam.update(paramsWithDetails[kbxMethodParam["kbxParamName"]])
kbxMethodParam["kbxParamCurrentValue"] = kbxParamCurrentValue # But remains current value no matter what.
except Exception as e:
kbxMethodParam["atDebug"] = "Unable to match parameter: " + str(e)
continue
except Exception as e:
kbxMethod["atDebug"] = "Unable to get shared method:" + str(e)
continue
return serObject
def retry_scene_execution_result_item(self, serId, seriIndex=None):
'''
Retry all scene execution result item with error according seriIndex
Each ser can be in retry mode one at a time.
If an item is deleted during execution, execution progress will be completed without event broadcasted.
'''
# Lock scene execution result for retry.
try:
self.__serLock.lock(serId)
except Exception:
raise AutomationException(12004, "Scene execution result is retrying.")
try:
if Util.is_empty(seriIndex): # Retry all items with error.
serisWithError = self.__serController.list_seris_with_error(serId)
if len(serisWithError) == 0:
raise AutomationException(12002, "We've found no scene execution result item with error to retry")
else:
serisWithError = [dict(seriWithError) for seriWithError in serisWithError]
else:
seriWithError = self.__serController.get_seri_by_index(serId, seriIndex)
if seriWithError is None:
raise AutomationException(12001, "scene execution result item not found, serId: " + str(serId) + " seriIndex: " + str(seriIndex))
elif seriWithError["seriStatus"] != "error":
raise AutomationException(12003, "scene execution result item not error, current status: " + str(seriWithError["seriStatus"]))
serisWithError = [seriWithError]
except Exception as e:
# Unlock scene execution after retry.
self.__serLock.unlock(serId)
raise e
else:
seriIndexes = [seriWithError["seriIndex"] for seriWithError in serisWithError]
self.__serRetryThreadPool.submit(self.__retry_seri_implementation, serId, serisWithError, seriIndexes)
return seriIndexes
def __retry_seri_implementation(self, serId, serisWithError, seriIndexes):
try:
# Update statuses of all items to be retried to "busy".
self.__serController.update_seri_status(serId=serId, seriIndexes=seriIndexes, seriStatus="busy", seriError=None)
# Broadcast event.
self.__broadcast_message__seri_retry_started(serId, seriIndexes=seriIndexes)
# Retry process starts here.
for seri in serisWithError:
try:
kbxMethodId = seri["kbxMethodId"]
if kbxMethodId != -291: # Skips all -291 and declare as "ok" immediately.
params = {kbxMethodParam["kbxParamName"]:kbxMethodParam["kbxParamCurrentValue"]
for kbxMethodParam in seri["kbxMethodParams"]}
result = SharedMethod.call_by_method_id(kbxMethodId, **params)
seriError = str(result)
else:
seriError = None
seriStatus = "ok"
except Exception as e:
seriStatus = "error"
seriError = str(e)
finally:
seriIndex = seri["seriIndex"]
if not self.__serLock.is_deleted(serId):
# Update statuses and broadcast events only if ser is not deleted.
self.__serController.update_seri_status(serId=serId, seriIndexes=[seriIndex],
seriStatus=seriStatus, seriError=seriError)
self.__broadcast_message__seri_retry_completed(serId, seriIndex, seriStatus, seriError)
except Exception as e:
''' THIS PORTION SHOULD NEVER RUN. (it's bug if this portion is executed) '''
Logger.log_error("SceneExecutionResultService.retry_scene_execution_result_item ex:", e)
finally:
self.__serLock.unlock(serId)
def list_scene_execution_results(self, offset, limit):
'''
List all scene execution results without items.
'''
return self.__serController.list(offset, limit)
def delete_scene_execution_result(self, serId):
'''
Delete a scene execution result. Relevant events will be broadcasted.
NOTE: scene execution result records will be maintained automatically to ensure only 10 records in database at most.
'''
self.__serController.delete(serId)
def __notify_ser_added(self, notiContent, notiLink, notiLinkLabel):
Logger.log_debug("Scene Execution Result logged at:", notiLink)
NotificationManagerService.dispatch_notification(text=notiContent,
link=notiLink,
linkLabel=notiLinkLabel)
def __broadcast_message__ser_deleted(self, serId):
eventTag = AppConstants.EVENT_SER_DELETED
eventData = {"serId":serId}
self.__broadcast_message(eventTag, eventData)
def __broadcast_message__seri_retry_started(self, serId, seriIndexes):
eventTag = AppConstants.EVENT_SERI_RETRY_STARTED
eventData = {"serId":serId, "seriIndexes":seriIndexes}
self.__broadcast_message(eventTag, eventData)
def __broadcast_message__seri_retry_completed(self, serId, seriIndex, seriStatus, seriError):
eventTag = AppConstants.EVENT_SERI_RETRY_COMPLETED
eventData = {"serId":serId, "seriIndex":seriIndex,
"seriStatus":seriStatus, "seriError":seriError}
self.__broadcast_message(eventTag, eventData)
def __broadcast_message(self, eventTag, eventData):
eventData = json.dumps(eventData, cls=AutomationJSONEncoder)
Application.send_web_server_event(eventTag, eventData)
class RuleService:
def __init__(self):
# Rule processors.
self.__ruleController = RuleController()
self.__methodController = MethodController()
self.__triggerController = TriggerController.instance()
self.__ruleUpdateThreadPool = ThreadPoolExecutor(max_workers=1)
self.__ruleExecThreadPool = ThreadPoolExecutor(max_workers=AppConstants.MAX_RULE_EXEC_THREAD_SIZE)
# Rule run workers.
self.__ruleExecInfos = {}
self.__condCallGroup = MethodCallGroup()
self.__execCallGroup = MethodCallGroup()
# Listeners.
self.__ruleController.listen_to_rule_status_change(self.__on_rule_status_changed)
GroupController.instance().listen_to_group_icon_change(self.__on_group_icon_changed)
self.__methodController.listen_to_method_status_change(self.__on_method_status_changed)
EventController.instance().listen_to_event_callback(self.__on_method_event_callback)
self.__triggerController.listen_to_trigger_callback(self.__on_trigger_callback)
def __on_rule_status_changed(self, ruleId, oldEnabled, newEnabled, oldStatusProcessed, newStatusProcessed):
'''
Trigger Source: RuleController --> This
Callback when a rule is re-enabled OR statusProcessed changed to "updated".
'''
if newEnabled == True and newStatusProcessed == AppConstants.RULE_STATUS_UPDATED:
if oldEnabled != newEnabled or oldStatusProcessed != newStatusProcessed:
self.__ruleExecThreadPool.submit(self.__trigger_rule_implementation, ruleId=ruleId, checkCondition=True)
def __on_group_icon_changed(self, kbxGroupId):
'''
Trigger Source: GroupController --> This
Callback when kbxGroupIcon changed.
'''
ruleIdsFromCond = self.__ruleController.list_rule_ids_which_has_kbx_group_id_as_condition(kbxGroupId)
ruleIdsFromExec = self.__ruleController.list_rule_ids_which_has_kbx_group_id_as_execution(kbxGroupId)
# Broadcast rules updated messages.
for ruleId in set(ruleIdsFromCond + ruleIdsFromExec):
self.__broadcast_message__rule_updated(ruleId)
def __on_method_status_changed(self, kbxMethodId, oldKBXMethodStatus, newKBXMethodStatus):
'''
Trigger Source: MethodController --> This
Callback when kbxMethodStatus changed.
'''
if oldKBXMethodStatus != newKBXMethodStatus:
ruleIdsFromCond = self.__ruleController.list_rule_ids_which_has_kbx_method_id_as_condition(kbxMethodId)
ruleIdsFromExec = self.__ruleController.list_rule_ids_which_has_kbx_method_id_as_execution(kbxMethodId)
# # Executes rules with conditions affected.
# if newKBXMethodStatus == SharedMethod.METHOD_STATUS_ACTIVE:
# for ruleId in ruleIdsFromCond:
# self.__ruleExecThreadPool.submit(self.__trigger_rule_implementation, ruleId=ruleId, checkCondition=True)
# Broadcast rules updated messages.
for ruleId in set(ruleIdsFromCond + ruleIdsFromExec):
self.__broadcast_message__rule_updated(ruleId)
def __on_method_event_callback(self, kbxMethodId, eventTag, eventData):
'''
Trigger Source: EventController --> MethodController --> This
Callback when a method with event broadcasted event.
'''
ruleIds = self.__ruleController.list_rule_ids_which_has_kbx_method_id_as_condition(kbxMethodId)
for ruleId in ruleIds:
self.__ruleExecThreadPool.submit(self.__trigger_rule_implementation, ruleId=ruleId,
checkCondition=True, eventTag=eventTag,
eventData=eventData, eventMethodId=kbxMethodId)
def __on_trigger_callback(self, ruleId):
'''
Trigger Source: TriggerController --> This
Callback when a rule is triggered.
'''
self.__ruleExecThreadPool.submit(self.__trigger_rule_implementation, ruleId=ruleId, checkCondition=True)
def set_rule(self, trigger, condition, execution, ruleId=None, ruleName=None, ruleProtected=False, enabled=True):
'''
Create/Edit(with ruleId provided) an existing rule.
trigger:Dictionary
condition:List
execution:List
ruleId:Integer <Optional>
ruleName:String <Optional>
ruleProtected:Boolean <Optional>
enabled:Boolean
Returns "ruleId"
'''
def process_method_list(methodList):
#===================================================================
# Basic type validation
#===================================================================
if not isinstance(methodList, list):
Logger.log_error("RuleService.set_rule: 'condition' and 'execution' must be type of list.")
Logger.log_debug("type:", type(methodList), "value:", methodList)
raise AutomationException(11704, "List is required for both 'condition' and 'execution'")
#===================================================================
# Check allowed size, raise error if exceeded.
#===================================================================
methodListLen = len(methodList)
if methodListLen > AppConstants.MAX_METHOD_SIZE:
Logger.log_error("RuleService.set_rule: 'condition' and 'execution' cannot have more than", AppConstants.MAX_METHOD_SIZE, "items respectively.")
raise AutomationException(11705, "Only a maximum of " + \
str(AppConstants.MAX_METHOD_SIZE) + \
" items is allowed for each 'condition' and 'execution' - given size " + \
str(methodListLen),
lambda text: str(AppConstants.MAX_METHOD_SIZE).join(text.split(":max_item_size:")))
#===================================================================
# Check if all kbxMethodIds are valid and all kbxMethodParams are list
#===================================================================
idValidator = NumberValidator(isRequired=True, decimalPoint=False)
if not all([idValidator.is_valid(eachMethod["kbxMethodId"])
and isinstance(eachMethod["kbxMethodParams"], list)
for eachMethod in methodList]):
raise AutomationException(11704, "'condition' and 'execution' have incorrect data structure.")
#===================================================================
# Check if all kbxParamName and kbxParamCurrentValue exists
#===================================================================
paramNameValidator = StringValidator(isRequired=True)
for eachMethod in methodList:
methodArgs = eachMethod["kbxMethodParams"]
for methodArg in methodArgs:
if not paramNameValidator.is_valid(methodArg[AppConstants.ARG_NAME]):
raise AutomationException(11704, "'condition' and 'execution' have invalid params structure")
if not AppConstants.ARG_CURRENT_VALUE in methodArg:
methodArg[AppConstants.ARG_CURRENT_VALUE] = None
return methodList
#=======================================================================
# Data structure validations
#=======================================================================
ruleId = NumberValidator(isRequired=False, decimalPoint=False).get_value(ruleId)
triggerDTO = self.__triggerController.parse_to_trigger_dto(trigger)
condition = process_method_list(condition)
execution = process_method_list(execution)
#=======================================================================
# Add to database
#=======================================================================
if Util.is_empty(ruleId):
# Validate_max_rule_size
if self.__ruleController.count() >= AppConstants.MAX_RULE_SIZE:
raise AutomationException(11706,
"Total amount of rules cannot be more than " + str(AppConstants.MAX_RULE_SIZE),
lambda text: str(AppConstants.MAX_RULE_SIZE).join(text.split(":max_rule_size:")))
ruleId = self.__ruleController.generate_id(ruleName)
rule = {}
elif self.__ruleController.has(ruleId):
ruleFromDB = self.__ruleController.get(ruleId)
rule = dict(ruleFromDB)
self.__check_rule_process_status(ruleId)
self.__ruleController.change_to_updating(ruleId, ruleName)
else:
raise AutomationException(11704, "Rule ID provided not found - " + str(ruleId))
#=======================================================================
# Broadcast message: starts to update rule.
#=======================================================================
self.__broadcast_message__rule_update_started(ruleId, ruleName)
#=======================================================================
# Set basic information of the rule
#=======================================================================
rule["ruleId"] = ruleId
rule["ruleName"] = ruleName
rule["ruleProtected"] = ruleProtected
rule["trigger"] = triggerDTO
rule["enabled"] = enabled
rule["condition"] = condition
rule["execution"] = execution
#=======================================================================
# Update rule
#=======================================================================
def __update_rule(rule):
try:
# Fire rule update start event
ruleId = rule["ruleId"]
# Add methods to subscribe list
methodIds = [kbxMethod["kbxMethodId"] for kbxMethod in rule["condition"] + rule["execution"]]
self.__methodController.add(methodIds)
# Update "rule" base table
self.__ruleController.update(rule)
self.__ruleController.commit()
except Exception as e:
self.__ruleController.rollback()
self.__broadcast_message__rule_update_failed(ruleId, ruleName)
Logger.log_error("RuleService __update_rule failed:", e, "-- rolledback")
else:
self.__triggerController.register_listener(ruleId, rule["trigger"])
# Process for Timer Module
TimerModule.delete_scheduler(ruleId)
timerModuleHandlers = {TimerModule.METHOD_ID_DATE_TIME_RANGE:TimerModule.handle_date_time_range,
TimerModule.METHOD_ID_DAY_OF_WEEK:TimerModule.handle_dow,
TimerModule.METHOD_ID_TIME_RANGE:TimerModule.handle_time_range}
for kbxMethod in rule["condition"]:
kbxMethodId = kbxMethod["kbxMethodId"]
timerModuleHandler = timerModuleHandlers.get(kbxMethodId, None)
if timerModuleHandler is not None:
timerModuleHandler(ruleId, kbxMethod["kbxMethodParams"])
# Broadcast message: completed updating a rule
self.__broadcast_message__rule_updated(ruleId)
#=======================================================================
# Submit to a thread to process other info, and return... performance...
#=======================================================================
self.__ruleUpdateThreadPool.submit(__update_rule, rule)
def delete_rule(self, ruleId):
self.__check_rule_process_status(ruleId)
try:
self.__ruleController.delete(ruleId)
self.__ruleController.commit()
except Exception as e:
self.__ruleController.rollback()
Logger.log_error("RuleService delete_rule ex:", e, "-- rolled back")
else:
self.__broadcast_message__rule_deleted(ruleId)
self.__triggerController.unregister_listener(ruleId)
TimerModule.delete_scheduler(ruleId)
def trigger_rule(self, ruleId, checkCondition=False):
'''
self.__check_rule_process_status(ruleId) <-- Check again in self.__trigger_rule_implementation.
'''
self.__trigger_rule_implementation(ruleId=ruleId, checkCondition=checkCondition)
def enable_rule(self, ruleId, enabled):
self.__check_rule_process_status(ruleId)
try:
self.__ruleController.enable(ruleId, enabled)
self.__ruleController.commit()
except Exception as e:
self.__ruleController.rollback()
Logger.log_error("RuleService enable_rule ex:", e, "-- rolled back")
else:
self.__broadcast_message__rule_updated(ruleId)
def get_rule(self, ruleId, language=AppInfo.DEFAULT_API_LANGUAGE):
try:
rule = self.__ruleController.get_detail(ruleId)
except:
raise AutomationException(11702, "Rule ID provided not found - " + str(ruleId))
kbxMethods = list(rule["condition"]) + list(rule["execution"])
# -------------- Compile lists of kbxMethod and group IDs contains in this rule.
kbxMethodIdsToList = {}
kbxGroupIdsToList = set([])
for kbxMethod in kbxMethods:
# Variables
kbxMethodId = kbxMethod["kbxMethodId"]
kbxMethodAppId = kbxMethod["kbxMethodAppId"]
kbxMethodStatus = kbxMethod["kbxMethodStatus"]
kbxGroupId = kbxMethod["kbxGroupId"]
kbxGroupStatus = kbxMethod["kbxGroupStatus"]
if kbxMethodStatus is not -1 and kbxMethodAppId is not None:
kbxMethodIdsToList.setdefault(kbxMethodAppId, set([]))
kbxMethodIdsToList[kbxMethodAppId].add(kbxMethodId)
if kbxGroupId is not None and kbxGroupStatus is not -1:
kbxGroupIdsToList.add(kbxGroupId)
# -------------- Get methods and groups based on requested language.
kbxMethodIdsListed = {}
kbxGroupIdsListed = {}
for kbxMethodAppId, kbxMethodIds in kbxMethodIdsToList.items():
kbxMethodIdsListed[kbxMethodAppId] = SharedMethodWrapper.list_shared_methods_by_app_id(kbxMethodAppId,
list(kbxMethodIds),
language=language)
groupList = SharedMethodWrapper.list_shared_method_groups(kbxGroupId=kbxGroupIdsToList, language=language)
for row in groupList:
kbxGroupIdsListed[row["kbxGroupId"]] = row
# -------------- Set method and group data into rule.
for kbxMethod in kbxMethods:
# Variables
kbxMethodId = kbxMethod["kbxMethodId"]
kbxMethodAppId = kbxMethod["kbxMethodAppId"]
kbxMethodStatus = kbxMethod["kbxMethodStatus"]
kbxGroupId = kbxMethod["kbxGroupId"]
kbxGroupStatus = kbxMethod["kbxGroupStatus"]
if kbxMethodStatus is not -1 and kbxMethodAppId is not None:
kbxMethodParamsWithCurrentValue = {kbxMethodParam["kbxParamName"]:kbxMethodParam["kbxParamCurrentValue"] \
for kbxMethodParam in kbxMethod["kbxMethodParams"]}
kbxMethodWithDetails = kbxMethodIdsListed[kbxMethodAppId][kbxMethodId]
if kbxMethodWithDetails is not None:
kbxMethodParamsWithDetails = kbxMethodWithDetails["kbxMethodParams"]
kbxMethodParamsWithDetails = copy.deepcopy(kbxMethodParamsWithDetails)
for kbxMethodParam in kbxMethodParamsWithDetails:
kbxMethodParam["kbxParamCurrentValue"] = kbxMethodParamsWithCurrentValue.get(kbxMethodParam["kbxParamName"], None)
kbxMethod["kbxMethodParams"] = kbxMethodParamsWithDetails
kbxMethod["kbxMethodHasEvent"] = not Util.is_empty(kbxMethodWithDetails.get("kbxMethodEvent", None)) \
and not Util.is_empty(kbxMethodWithDetails.get("kbxMethodIdentifier", None))
kbxMethod["kbxMethodLabel"] = kbxMethodWithDetails.get("kbxMethodLabel")
kbxMethod["kbxMethodDesc"] = kbxMethodWithDetails.get("kbxMethodDesc")
else:
kbxMethod["atDebugMethod"] = "Unable to get shared method, caused by a method which never register itself on this bootup."
else:
kbxMethod["kbxMethodHasEvent"] = False
if kbxGroupId is not None and kbxGroupStatus is not -1:
try:
kbxMethod["kbxGroupLabel"] = kbxGroupIdsListed[kbxGroupId]["kbxGroupLabel"]
kbxMethod["kbxGroupDesc"] = kbxGroupIdsListed[kbxGroupId]["kbxGroupDesc"]
except:
kbxMethod["atDebugGroup"] = "Unable to get shared method group, caused by a group which never register itself on this bootup."
return rule
def list_rules(self, offset=0, limit=20):
return self.__ruleController.list(offset, limit), \
self.__ruleController.count()
def run_all_enabled_rules(self):
'''
Add the following 2 lines of code at AutomationModuleWrapper.py - start(),
after last statement, to enable run all rules on bootup.
# Logger.log_info("Attempts to execute all enabled rules ...")
# self.__ruleService.run_all_enabled_rules()
'''
ruleIds = self.__ruleController.list_rule_ids_which_are_enabled()
for ruleId in ruleIds:
self.__ruleExecThreadPool.submit(self.__trigger_rule_implementation, ruleId=ruleId, checkCondition=True)
def __check_rule_process_status(self, ruleId):
try:
statusProcessed = self.__ruleController.get_status_processed(ruleId)
if statusProcessed != AppConstants.RULE_STATUS_UPDATED:
raise AutomationException(11703, "edit/delete/execute is not allowed on rule update in progress")
except:
raise AutomationException(11702, "Rule ID provided not found - " + str(ruleId))
def __broadcast_message__rule_update_started(self, ruleId, ruleName=None):
eventTag = AppConstants.EVENT_RULE_UPDATE_STARTED
eventData = {"ruleId":ruleId, "newRuleName":ruleName}
self.__broadcast_message(eventTag, eventData)
Logger.log_info("Rule Start Update:", ruleName)
def __broadcast_message__rule_updated(self, ruleId):
try:
rule = self.__ruleController.get_summary(ruleId)
except Exception as e:
Logger.log_error("RuleService.__broadcast_message__rule_updated get_summary ex:", e)
return
eventTag = AppConstants.EVENT_RULE_UPDATED
eventData = rule
self.__broadcast_message(eventTag, eventData)
Logger.log_info("Rule Updated:", rule["ruleName"])
def __broadcast_message__rule_update_failed(self, ruleId, ruleName=None):
'''
ruleName - For debugging purpose.
'''
try:
rule = self.__ruleController.get_summary(ruleId)
except Exception:
rule = None
eventTag = AppConstants.EVENT_RULE_UPDATE_FAILED
eventData = {"ruleId": ruleId, "oldRuleSummary":rule}
self.__broadcast_message(eventTag, eventData)
Logger.log_info("Rule Update Failed:", ruleName)
def __broadcast_message__rule_deleted(self, ruleId):
eventTag = AppConstants.EVENT_RULE_DELETED
eventData = {"ruleId": ruleId}
self.__broadcast_message(eventTag, eventData)
Logger.log_info("Rule Deleted: Id -", ruleId)
def __broadcast_message(self, eventTag, eventData):
eventData = json.dumps(eventData, cls=AutomationJSONEncoder)
Application.send_web_server_event(eventTag, eventData)
def __trigger_rule_implementation(self, ruleId, checkCondition=False, eventTag=None, eventData=None, eventMethodId=None):
'''
Triggers a rule by given ruleId.
'''
Logger.log_info("trigger rule id:", ruleId)
# Check if rule is "updated" AND enabled.
statusProcessed, enabled = self.__ruleController.get_status_processed_and_enabled(ruleId)
if statusProcessed != AppConstants.RULE_STATUS_UPDATED or enabled != True:
return
self.__ruleExecInfos.setdefault(ruleId, RuleExecInfo())
ruleExecInfo = self.__ruleExecInfos.get(ruleId)
ruleExecInfo.increase_trigger_count()
triggerCountInThisSession = ruleExecInfo.get_trigger_count()
with ruleExecInfo.get_rlock():
#=======================================================================
# Check conditions
#=======================================================================
if checkCondition is True:
# Check if we should proceed (stop if there is another pending request on the same ruleId).
if triggerCountInThisSession != ruleExecInfo.get_trigger_count():
return
methodListToCheck = deque()
result = self.__ruleController.list_conditions(ruleId)
methodCheckingTime = int(time.time())
for row in result:
if row["kbxMethodStatus"] not in (SharedMethod.METHOD_STATUS_ACTIVE, SharedMethod.METHOD_STATUS_INACTIVE):
return
else:
methodArgs = row["kbxMethodParams"]
kwargs = {methodArg[AppConstants.ARG_NAME]:methodArg[AppConstants.ARG_CURRENT_VALUE] for methodArg in methodArgs}
if eventTag is not None and eventMethodId == row["kbxMethodId"]:
kwargs[AppConstants.KEY_CONDITION_EVENT_TAG] = eventTag
kwargs[AppConstants.KEY_CONDITION_EVENT_DATA] = eventData
if AppInfo.REQUEST_KEY_LANGUAGE not in kwargs:
kwargs[AppInfo.REQUEST_KEY_LANGUAGE] = AppInfo.DEFAULT_API_LANGUAGE
kwargs["kbxMethodName"] = row["kbxMethodName"]
kwargs["kbxModuleName"] = row["kbxModuleName"]
kwargs["kbxGroupId"] = row["kbxGroupId"]
kwargs["kbxMethodAppId"] = row["kbxMethodAppId"]
# Update ruleId if it is required by the method
if "ruleId" in kwargs:
kwargs["ruleId"] = str(ruleId)
callId = hash(str(kwargs)) # Generate condition checking ID
kwargs[AppConstants.KEY_CONDITION_TIMESTAMP] = methodCheckingTime # So that timestamp will not caused the generated id to be different
methodListToCheck.append({"callId":callId,
"callFn":SharedMethod.call,
"callKwargs":kwargs})
#===============================================================
# Submit all conditions for checking
#===============================================================
methodListToCheckLen = len(methodListToCheck)
if methodListToCheckLen > 0:
ruleExecResult = RuleExecResult(methodListToCheckLen)
for methodItem in methodListToCheck:
self.__condCallGroup.submit(callbackFn=self.__on_method_call_complete, ruleExecResult=ruleExecResult, **methodItem)
result = ruleExecResult.wait(40.0)
if result is False or ruleExecResult.get_result() is False:
return # Failed at condition checking.
# Clear cache
del(methodListToCheck)
del(methodCheckingTime)
del(methodListToCheckLen)
# Check if we should proceed (stop if there is another pending request on the same ruleId).
if triggerCountInThisSession != ruleExecInfo.get_trigger_count():
return
#=======================================================================
# Execute executions
#=======================================================================
methodListToExec = deque()
result = self.__ruleController.list_executions(ruleId)
methodExecTime = int(time.time())
for row in result:
if row["kbxMethodStatus"] not in (SharedMethod.METHOD_STATUS_ACTIVE, SharedMethod.METHOD_STATUS_INACTIVE):
continue
else:
methodArgs = row["kbxMethodParams"]
kwargs = {methodArg[AppConstants.ARG_NAME]:methodArg[AppConstants.ARG_CURRENT_VALUE] for methodArg in methodArgs}
if AppInfo.REQUEST_KEY_LANGUAGE not in kwargs:
kwargs[AppInfo.REQUEST_KEY_LANGUAGE] = AppInfo.DEFAULT_API_LANGUAGE
kwargs["kbxMethodName"] = row["kbxMethodName"]
kwargs["kbxModuleName"] = row["kbxModuleName"]
kwargs["kbxGroupId"] = row["kbxGroupId"]
kwargs["kbxMethodAppId"] = row["kbxMethodAppId"]
# Update ruleId if it is required by the method
if "ruleId" in kwargs:
kwargs["ruleId"] = str(ruleId)
callId = hash(str(kwargs)) # Generate execution id
kwargs[AppConstants.KEY_ACTION_TIMESTAMP] = methodExecTime
methodListToExec.append({"callId":callId,
"callFn":SharedMethod.call,
"callKwargs":kwargs})
#===============================================================
# Submit all methods for executions
#===============================================================
methodListToExecLen = len(methodListToExec)
if methodListToExecLen > 0:
ruleExecResult = RuleExecResult(methodListToExecLen)
for methodItem in methodListToExec:
self.__execCallGroup.submit(callbackFn=self.__on_method_call_complete, ruleExecResult=ruleExecResult, **methodItem)
result = ruleExecResult.wait(30.0)
return
def __on_method_call_complete(self, checkingId, result, ruleExecResult):
'''
When rule/execution checking is completed.
'''
ruleExecResult.set_result(result)
class TriggerController:
__INSTANCE = None
__LOCK = threading.Lock()
@staticmethod
def instance():
with TriggerController.__LOCK:
TriggerController.__INSTANCE or TriggerController()
return TriggerController.__INSTANCE
def __init__(self):
TriggerController.__INSTANCE = self
self.__registeredRuleIds = set([])
self.__threadPool = ThreadPoolExecutor(max_workers=1)
def listen_to_trigger_callback(self, aFunc):
'''
aFunc:Function - Parameters (ruleId:Integer)
'''
ControllerModule.ON_TRIGGER_CALLBACK = aFunc
def register_listener(self, ruleId, triggerDict):
self.unregister_listener(ruleId)
triggerType = triggerDict["type"]
triggerParsedValue = triggerDict.get("parsedValue")
#=======================================================================
# Register trigger to scheduler
#=======================================================================
if triggerType == AppConstants.TRIGGER_TYPE_INTERVAL:
def __signal_scheduler_add_interval_job(ruleId, seconds, minutes, hours):
kwargs = {k:v for k, v in {"seconds":seconds, "minutes":minutes, "hours":hours}.items() if v > 0}
try:
SchedulerService.add_interval_job(jobName=str(ruleId),
kbxTargetAppId=AppInfo.get_app_id(),
kbxTargetMethod="on_trigger_callback",
kbxTargetModule="controller_module",
kbxTargetParams={"ruleId":ruleId},
store=False,
**kwargs)
self.__registeredRuleIds.add(ruleId)
except SystemException as e:
Logger.log_debug(e)
self.__threadPool.submit(__signal_scheduler_add_interval_job, ruleId, **triggerParsedValue)
elif triggerType == AppConstants.TRIGGER_TYPE_TIME:
def __signal_scheduler_add_cron_job(ruleId, hour, minute):
try:
SchedulerService.add_cron_job(jobName=str(ruleId),
kbxTargetAppId=AppInfo.get_app_id(),
kbxTargetMethod="on_trigger_callback",
kbxTargetModule="controller_module",
kbxTargetParams={"ruleId":ruleId},
store=False,
hour=str(hour),
minute=str(minute))
self.__registeredRuleIds.add(ruleId)
except SystemException as e:
Logger.log_debug(e)
self.__threadPool.submit(__signal_scheduler_add_cron_job, ruleId, **triggerParsedValue)
def unregister_listener(self, ruleId):
def __signal_scheduler_remove_task(ruleId):
try:
SchedulerService.remove_job(str(ruleId))
self.__registeredRuleIds.remove(ruleId)
except SystemException as e:
Logger.log_debug(e)
if ruleId in self.__registeredRuleIds:
self.__threadPool.submit(__signal_scheduler_remove_task, ruleId)
def parse_to_trigger_dto(self, trigger):
#=======================================================================
# Check if all required keys must exists
#=======================================================================
if "type" not in trigger:
raise AutomationException(11703, "'type' must exists")
#=======================================================================
# Check 'type' against allowed values
#=======================================================================
triggerType = trigger["type"]
if triggerType not in (AppConstants.TRIGGER_TYPE_EVENT, AppConstants.TRIGGER_TYPE_INTERVAL, AppConstants.TRIGGER_TYPE_TIME):
raise AutomationException(11703, "'type' has invalid value")
#=======================================================================
# Compute 'parsedValue'
#=======================================================================
triggerValue = trigger.get("value")
if triggerType == AppConstants.TRIGGER_TYPE_INTERVAL:
triggerValue = ValueParser.get_number(triggerValue) # this must be an integer (seconds)
if isinstance(triggerValue, int):
if triggerValue > 0:
seconds = triggerValue % 60
minutes = math.floor(triggerValue / 60) % 60
hours = math.floor((triggerValue / 3600)) % 24
trigger["parsedValue"] = {"seconds":seconds, "minutes":minutes, "hours":hours}
else:
raise AutomationException(11703, "'value' in 'trigger' must be larger than 0")
else:
raise AutomationException(11703, "'value' in 'trigger' must be a number in seconds")
elif triggerType == AppConstants.TRIGGER_TYPE_TIME:
triggerValue = ValueParser.get_string(triggerValue) # this must be in (HH, MM)
if isinstance(triggerValue, str):
triggerValue = triggerValue.split(":")
if len(triggerValue) == 2:
#=======================================================
# Validate hour
#=======================================================
hour = ValueParser.get_number(triggerValue[0])
if hour is None or not 0 <= hour <= 23:
raise AutomationException(11703, "'HH is ranged from 00 - 23'")
#=======================================================
# Validate minute
#=======================================================
minute = ValueParser.get_number(triggerValue[1])
if minute is None or not 0 <= minute <= 59:
raise AutomationException(11703, "MM is ranged from 00 - 59")
trigger["parsedValue"] = {"hour":hour, "minute":minute}
else:
raise AutomationException(11703, "'value' in 'trigger' must be in HH:MM format")
else:
raise AutomationException(11703, "'value' in 'trigger' must be string in HH:MM format")
else:
trigger["parsedValue"] = None
return trigger
'''
Created on 16-Nov-2015
@author: Virendra
'''
from concurrent.futures.thread import ThreadPoolExecutor
import threading
import time
def square(n):
print ("Calculating square of %d by thread name %s " % (n, threading.current_thread()))
time.sleep(3)
print ("Square of number %d is %d calculated by thread %s " % (n, n*n, threading.current_thread()))
executor = ThreadPoolExecutor(max_workers=5)
numbers = range(1,10)
for number in numbers:
executor.submit(square, number)
class UI(urwid.MainLoop):
def __init__(self):
self.d = DockerBackend()
# root widget
self.mainframe = urwid.Frame(urwid.SolidFill())
self.buffers = []
self.footer = Footer(self)
self.executor = ThreadPoolExecutor(max_workers=4)
root_widget = urwid.AttrMap(self.mainframe, "root")
self.main_list_buffer = None # singleton
screen = urwid.raw_display.Screen()
screen.set_terminal_properties(256)
screen.register_palette(PALLETE)
super().__init__(root_widget, screen=screen)
self.handle_mouse = False
self.current_buffer = None
def run_in_background(self, task, *args, **kwargs):
logger.info("running task %r(%s, %s) in background", task, args, kwargs)
self.executor.submit(task, *args, **kwargs)
def refresh(self):
try:
self.draw_screen()
except AssertionError:
logger.warning("application is not running")
pass
def _set_main_widget(self, widget, redraw):
"""
add provided widget to widget list and display it
:param widget:
:return:
"""
self.mainframe.set_body(widget)
self.reload_footer()
if redraw:
logger.debug("redraw main widget")
self.refresh()
def display_buffer(self, buffer, redraw=True):
"""
display provided buffer
:param buffer: Buffer
:return:
"""
self.current_buffer = buffer
self._set_main_widget(buffer.widget, redraw=redraw)
def add_and_display_buffer(self, buffer, redraw=True):
"""
add provided buffer to buffer list and display it
:param buffer:
:return:
"""
if buffer not in self.buffers:
logger.debug("adding new buffer {!r}".format(buffer))
self.buffers.append(buffer)
self.display_buffer(buffer, redraw=redraw)
def pick_and_display_buffer(self, i):
"""
pick i-th buffer from list and display it
:param i: int
:return: None
"""
if len(self.buffers) == 1:
# we don't need to display anything
# listing is already displayed
return
else:
try:
self.display_buffer(self.buffers[i])
except IndexError:
# i > len
self.display_buffer(self.buffers[0])
@property
def current_buffer_index(self):
return self.buffers.index(self.current_buffer)
def remove_current_buffer(self):
# don't allow removing main_list
if isinstance(self.current_buffer, MainListBuffer):
logger.warning("you can't remove main list widget")
return
self.buffers.remove(self.current_buffer)
self.current_buffer.destroy()
# FIXME: we should display last displayed widget here
self.display_buffer(self.buffers[0], True)
def unhandled_input(self, key):
logger.debug("unhandled input: %r", key)
try:
if key in ("q", "Q"):
self.executor.shutdown(wait=False)
raise urwid.ExitMainLoop()
elif key == "ctrl o":
self.pick_and_display_buffer(self.current_buffer_index - 1)
elif key == "ctrl i":
self.pick_and_display_buffer(self.current_buffer_index + 1)
elif key == "x":
self.remove_current_buffer()
elif key == "/":
self.prompt("/", search)
elif key == "f4":
self.footer.prompt("filter ", filter)
elif key == "n":
self.current_buffer.find_next()
elif key == "N":
self.current_buffer.find_previous()
elif key in ["h", "?"]:
self.display_help()
elif key == "f5":
self.display_tree()
except NotifyError as ex:
self.notify_message(str(ex), level="error")
logger.error(repr(ex))
def run(self):
self.main_list_buffer = MainListBuffer(self.d, self)
@log_traceback
def chain_fcs():
self.main_list_buffer.refresh(focus_on_top=True)
self.add_and_display_buffer(self.main_list_buffer, redraw=True)
self.run_in_background(chain_fcs)
super().run()
def display_logs(self, docker_container):
self.add_and_display_buffer(LogsBuffer(docker_container, self))
def display_and_follow_logs(self, docker_container):
self.add_and_display_buffer(LogsBuffer(docker_container, self, follow=True))
def inspect(self, docker_object):
self.add_and_display_buffer(InspectBuffer(docker_object))
def display_image_info(self, docker_image):
try:
self.add_and_display_buffer(ImageInfoBuffer(docker_image, self))
except NotifyError as ex:
self.notify_message(str(ex), level="error")
logger.error(repr(ex))
def refresh_main_buffer(self, refresh_buffer=True):
assert self.main_list_buffer is not None
if refresh_buffer:
self.main_list_buffer.refresh()
self.display_buffer(self.main_list_buffer)
def display_help(self):
self.add_and_display_buffer(HelpBuffer())
def display_tree(self):
self.add_and_display_buffer(TreeBuffer(self.d, self))
# FOOTER
def set_footer(self, widget):
self.mainframe.set_footer(widget)
def reload_footer(self):
self.footer.reload_footer()
def remove_notification_message(self, message):
self.footer.remove_notification_message(message)
def notify_widget(self, *args, **kwargs):
self.footer.notify_widget(*args, **kwargs)
def notify_message(self, *args, **kwargs):
self.footer.notify_message(*args, **kwargs)
def prompt(self, *args, **kwargs):
self.footer.prompt(*args, **kwargs)
class RabbitManager(BaseManager):
"""Base for managers that connects to rabbit
"""
def __init__(self, rabbitmq_url=None, queue=None, routing_key=None,
exchange="message", exchange_type="direct", log=None,
max_tasks=5, logging=None):
"""
== Config dict structure (case adjusted to json configuration):
{
"rabbit": {
"url": "apmq://rabbit",
"queue": "test",
"routingKey": "example.json"
"exchange": "message", // optional, default: message
"exchangeType:" "topic" // optional, default: topic
}
}
:param str rabbitmq_url: optional url to rabbitmq
:param str queue: name of the queue
:param str routing_key: routing key for queue
:param str exchange: name of the exchange
:param str exchange_type: type of the exchange
:param dict config: Manager configuration from parsed json config all
the above options can be configured from it
:param logging.Logger log: optional logger that will replace new one
:raises exceptions.NotConfigured:
:return:
"""
if queue is None:
raise exceptions.NotConfigured("Misssing queue")
self._connection = None
self._channel = None
self._closing = False
self._consumer_tag = None
self._max_tasks = max_tasks # 2 cores + 1
self._tasks_number = 0
self._executor = ThreadPoolExecutor(max_workers=self._max_tasks)
self._max_tasks_warning_counter = 0
self._rabbitmq_url = rabbitmq_url
self._queue = queue
self._routing_key = routing_key
self._exchange = exchange
self._exchange_type = exchange_type
if log is None:
from toddler.logging import setup_logging
if logging is not None:
self.log = setup_logging(config=logging)
else:
self.log = setup_logging()
else:
self.log = log
def reconnect(self):
"""Will be run by IOLoop.time if the connection is closed.
See on_connection_closed method.
"""
self._connection.ioloop.stop()
if not self._closing:
self._connection = self.connect()
self._connection.ioloop.start()
@property
def queue(self):
return self._queue
def on_connection_closed(self, connection, reply_code, reply_text):
"""
:param pika.connection.Connection connection: closed connection ob
:param int reply_code: reply code if given
:param str reply_text: reply text if given
:return:
"""
self._channel = None
if self._closing:
self._connection.ioloop.stop()
else:
self.log.warning(
"Connection closed, will reopen in 5 seconds: (%s) %s",
reply_code,
reply_text
)
self._connection.add_timeout(5, self.reconnect)
def on_channel_closed(self, channel, reply_code, reply_text):
"""Invoked when channel has been closed
:param pika.channel.Channel channel:
:param int reply_code:
:param str reply_text:
:return:
"""
self.log.info("Channel to rabbit closed.")
self._connection.close()
def on_channel_open(self, channel):
"""Invoked when channel has been opened
:param pika.channel.Channel channel:
"""
self.log.info("Channel opened")
self._channel = channel
self._channel.add_on_close_callback(self.on_channel_closed)
self.start_consuming()
def close_channel(self):
self.log.info("Closing channel")
self._channel.close()
def open_channel(self):
self.log.info("Opening channel")
self._connection.channel(on_open_callback=self.on_channel_open)
def on_connection_open(self, connection):
self.log.info("Connected")
self._connection = connection
self._connection.add_on_close_callback(self.on_connection_closed)
self.open_channel()
def connect(self):
"""Connects to rabbitmq server, according to config
:return pika.SelectConnection:
"""
self.log.info("Connecting to RabbitMQ")
return pika.BlockingConnection(
pika.URLParameters(self._rabbitmq_url + "?heartbeat_interval=5"),
# self.on_connection_open,
# stop_ioloop_on_close=False
)
def on_cancel_ok(self, frame):
"""Invoked when locale Basic.Cancel is acknowledged by RabbitMQ
:param pika.frame.Method frame:
:return:
"""
self.log.info("Rabbit acknowledged the cancel of the consumer")
self.close_channel()
def on_consumer_cancelled(self, method_frame):
"""Invoked by pika when RabbitMQ sends a Basic.Cancel for a consumer
receiving messages.
:param pika.frame.Method method_frame: The Basic.Cancel frame
:return:
"""
self.log.info("Consumer was cancelled remotely, shutting down: %r",
method_frame)
if self._channel:
self._channel.close()
def acknowledge_message(self, delivery_tag):
"""
:param delivery_tag:
:return:
"""
self.log.info("Acknowledging message %s", delivery_tag)
self._channel.basic_ack(delivery_tag)
def requeue_message(self, delivery_tag):
"""
:param delivery_tag:
:return:
"""
self.log.info("Requeuing message %s", delivery_tag)
self._channel.basic_nack(delivery_tag, requeue=True)
def on_message(self, channel, basic_deliver, properties, body):
"""Invoked when message received from rabbit
:param pika.channel.Channel channel:
:param pika.spec.Basic.Deliver basic_deliver:
:param pika.spec.BasicProperties properties:
:param str body:
:return:
"""
self.log.info("Received messages # %s from %s",
basic_deliver.delivery_tag,
properties.app_id)
try:
if self._tasks_number >= self._max_tasks:
raise RuntimeError("Max tasks limit reached")
self._tasks_number += 1
ftr = self._executor.submit(self.process_task, body)
def process_done(future: Future):
nonlocal self
self._tasks_number -= 1
if future.cancelled():
# process_task ended by cancel
self.requeue_message(self.requeue_message(
basic_deliver.delivery_tag)
)
else:
if future.exception():
exception = future.exception()
if not isinstance(exception, RequeueMessage):
self.log.exception(exception)
self.requeue_message(
basic_deliver.delivery_tag
)
else:
self.acknowledge_message(basic_deliver.delivery_tag)
ftr.add_done_callback(process_done)
return ftr
except RuntimeError:
self.requeue_message(basic_deliver.delivery_tag)
time.sleep(0.5)
except Exception as e:
self.log.exception(e)
self.requeue_message(basic_deliver.delivery_tag)
time.sleep(10)
def stop_consuming(self):
"""Send Basic.Cancel to rabbit
:return:
"""
if self._channel:
self.log.info("Stop consuming")
self._channel.basic_cancel(self.on_cancel_ok, self._consumer_tag)
def start_consuming(self):
"""Begins to consume messages
:return:
"""
self.log.info("Start consuming")
self._channel.add_on_cancel_callback(self.on_consumer_cancelled)
self._consumer_tag = self._channel.basic_consume(self.on_message,
self.queue)
self.run()
def run(self):
"""Run consumer"""
self.log.info("Running consumer")
connection = self.connect()
""":type: pika.SelectConnection"""
channel = connection.channel()
self._channel = channel
self._connection = connection
for method_frame, properties, body in channel.consume(self.queue):
while self._tasks_number >= self._max_tasks:
time.sleep(0.1)
self.on_message(channel, method_frame, properties, body)
def stop(self):
"""Stops consuming service
:return:
"""
self.log.info("Stopping")
self._closing = True
self.stop_consuming()
self._executor.shutdown(True)
# if self._connection is not None:
# self._connection.ioloop.start()
self.log.info("Stopped")
def __exit__(self, *args, **kwargs):
self.stop()
super(RabbitManager, self).__exit__(*args, **kwargs)
