def get_messages(self):
"""
Get messages
"""
messages = core_messages.retrieve_messages(
status=MessageStatus.New, bulk_size=self.retrieve_bulk_size)
self.logger.info("Main thread get %s new messages" % len(messages))
return messages
def get_messages(self):
"""
Get messages
"""
messages = core_messages.retrieve_messages(status=MessageStatus.New,
bulk_size=self.retrieve_bulk_size,
destination=MessageDestination.Outside)
self.logger.debug("Main thread get %s new messages" % len(messages))
if messages:
self.logger.info("Main thread get %s new messages" % len(messages))
return messages
def get(self, request_id, workload_id):
""" Get messages with given id.
HTTP Success:
200 OK
HTTP Error:
404 Not Found
500 InternalError
:returns: dictionary of an request.
"""
try:
if request_id == 'null':
request_id = None
if workload_id == 'null':
workload_id = None
msgs = retrieve_messages(request_id=request_id,
workload_id=workload_id)
rets = []
for msg in msgs:
rets.append(msg['msg_content'])
except exceptions.NoObject as error:
return self.generate_http_response(
HTTP_STATUS_CODE.NotFound,
exc_cls=error.__class__.__name__,
exc_msg=error)
except exceptions.IDDSException as error:
return self.generate_http_response(
HTTP_STATUS_CODE.InternalError,
exc_cls=error.__class__.__name__,
exc_msg=error)
except Exception as error:
print(error)
print(format_exc())
return self.generate_http_response(
HTTP_STATUS_CODE.InternalError,
exc_cls=exceptions.CoreException.__name__,
exc_msg=error)
return self.generate_http_response(HTTP_STATUS_CODE.OK, data=rets)
import json
from idds.core.messages import retrieve_messages
msgs = retrieve_messages()
for msg in msgs:
if msg['msg_id'] in [323720]:
# print(msg)
msg_sent = json.dumps(msg['msg_content'])
print(msg_sent)
recover_msg = json.loads(msg_sent)
if recover_msg['relation_type'] == 'output':
# only parse messages from output contents
points = recover_msg['files']
for point in points:
hp_loss = point['path']
hp_loss = json.loads(hp_loss)
hp, loss = hp_loss
print(hp)
print(loss)
model_id, hp_point = hp
print(model_id)
print(hp_point)
for req in reqs:
if req['request_id'] == 113:
# print(req)
# print(req['request_metadata']['workflow'].to_dict())
# print(json_dumps(req, sort_keys=True, indent=4))
pass
"""
tfs = get_transforms(transform_id=53550)
for tf in tfs:
# print(tf)
# print(tf['transform_metadata']['work'].to_dict())
# print(json_dumps(tf, sort_keys=True, indent=4))
pass
msgs = retrieve_messages(workload_id=25972557)
number_contents = 0
for msg in msgs:
# if msg['msg_id'] in [323720]:
# if True:
# if msg['request_id'] in [208]:
print(json_dumps(msg['msg_content'], sort_keys=True, indent=4))
if msg['msg_content']['msg_type'] == 'file_stagein' and msg['msg_content'][
'relation_type'] == 'output':
# number_contents += len(msg['msg_content']['files'])
for i_file in msg['msg_content']['files']:
if i_file['status'] == 'Available':
number_contents += 1
pass
print(number_contents)
from idds.common.constants import MessageStatus # noqa F401
from idds.core.messages import retrieve_messages, add_messages # noqa F401
def clean_msg(msg):
new_msg = {
'msg_type': msg['msg_type'],
'status': MessageStatus.New,
'source': msg['source'],
'destination': msg['destination'],
'request_id': msg['request_id'],
'workload_id': msg['workload_id'],
'transform_id': msg['transform_id'],
'num_contents': msg['num_contents'],
'msg_content': msg['msg_content']
}
return new_msg
for req_id in range(1073, 1084):
msgs = retrieve_messages(request_id=req_id)
number_contents = 0
for msg in msgs:
# print(json_dumps(msg['msg_content'], sort_keys=True, indent=4))
# if msg['num_contents'] > 10000 and msg['status'] == MessageStatus.New:
if msg['num_contents'] >= 10000:
print(msg['request_id'], msg['msg_id'], msg['num_contents'])
msg = clean_msg(msg)
add_messages([msg], bulk_size=10000)
# break