def get_messages(self): """ Get messages """ messages = core_messages.retrieve_messages( status=MessageStatus.New, bulk_size=self.retrieve_bulk_size) self.logger.info("Main thread get %s new messages" % len(messages)) return messages
def get_messages(self): """ Get messages """ messages = core_messages.retrieve_messages(status=MessageStatus.New, bulk_size=self.retrieve_bulk_size, destination=MessageDestination.Outside) self.logger.debug("Main thread get %s new messages" % len(messages)) if messages: self.logger.info("Main thread get %s new messages" % len(messages)) return messages
def get(self, request_id, workload_id): """ Get messages with given id. HTTP Success: 200 OK HTTP Error: 404 Not Found 500 InternalError :returns: dictionary of an request. """ try: if request_id == 'null': request_id = None if workload_id == 'null': workload_id = None msgs = retrieve_messages(request_id=request_id, workload_id=workload_id) rets = [] for msg in msgs: rets.append(msg['msg_content']) except exceptions.NoObject as error: return self.generate_http_response( HTTP_STATUS_CODE.NotFound, exc_cls=error.__class__.__name__, exc_msg=error) except exceptions.IDDSException as error: return self.generate_http_response( HTTP_STATUS_CODE.InternalError, exc_cls=error.__class__.__name__, exc_msg=error) except Exception as error: print(error) print(format_exc()) return self.generate_http_response( HTTP_STATUS_CODE.InternalError, exc_cls=exceptions.CoreException.__name__, exc_msg=error) return self.generate_http_response(HTTP_STATUS_CODE.OK, data=rets)
import json from idds.core.messages import retrieve_messages msgs = retrieve_messages() for msg in msgs: if msg['msg_id'] in [323720]: # print(msg) msg_sent = json.dumps(msg['msg_content']) print(msg_sent) recover_msg = json.loads(msg_sent) if recover_msg['relation_type'] == 'output': # only parse messages from output contents points = recover_msg['files'] for point in points: hp_loss = point['path'] hp_loss = json.loads(hp_loss) hp, loss = hp_loss print(hp) print(loss) model_id, hp_point = hp print(model_id) print(hp_point)
for req in reqs: if req['request_id'] == 113: # print(req) # print(req['request_metadata']['workflow'].to_dict()) # print(json_dumps(req, sort_keys=True, indent=4)) pass """ tfs = get_transforms(transform_id=53550) for tf in tfs: # print(tf) # print(tf['transform_metadata']['work'].to_dict()) # print(json_dumps(tf, sort_keys=True, indent=4)) pass msgs = retrieve_messages(workload_id=25972557) number_contents = 0 for msg in msgs: # if msg['msg_id'] in [323720]: # if True: # if msg['request_id'] in [208]: print(json_dumps(msg['msg_content'], sort_keys=True, indent=4)) if msg['msg_content']['msg_type'] == 'file_stagein' and msg['msg_content'][ 'relation_type'] == 'output': # number_contents += len(msg['msg_content']['files']) for i_file in msg['msg_content']['files']: if i_file['status'] == 'Available': number_contents += 1 pass print(number_contents)
from idds.common.constants import MessageStatus # noqa F401 from idds.core.messages import retrieve_messages, add_messages # noqa F401 def clean_msg(msg): new_msg = { 'msg_type': msg['msg_type'], 'status': MessageStatus.New, 'source': msg['source'], 'destination': msg['destination'], 'request_id': msg['request_id'], 'workload_id': msg['workload_id'], 'transform_id': msg['transform_id'], 'num_contents': msg['num_contents'], 'msg_content': msg['msg_content'] } return new_msg for req_id in range(1073, 1084): msgs = retrieve_messages(request_id=req_id) number_contents = 0 for msg in msgs: # print(json_dumps(msg['msg_content'], sort_keys=True, indent=4)) # if msg['num_contents'] > 10000 and msg['status'] == MessageStatus.New: if msg['num_contents'] >= 10000: print(msg['request_id'], msg['msg_id'], msg['num_contents']) msg = clean_msg(msg) add_messages([msg], bulk_size=10000) # break