def __init__(self, num_threads=1, **kwargs):
super(Assigner, self).__init__(num_threads, **kwargs)
self.config_section = Sections.Assigner
self.setup_logger()
self.resource_name = self.get_resouce_name()
head_service = self.get_head_service()
if head_service:
self.head_client = Client(head_service)
else:
self.head_client = None
def __init__(self, num_threads=1, **kwargs):
super(Finisher, self).__init__(num_threads, **kwargs)
self.config_section = Sections.Finisher
self.setup_logger()
self.resource_name = self.get_resouce_name()
head_service = self.get_head_service()
if head_service:
self.head_client = Client(head_service)
else:
self.head_client = None
if hasattr(self, 'send_messaging') and self.send_messaging:
self.send_messaging = True
else:
self.send_messaging = False
#!/usr/bin/env python
#
# Licensed under the Apache License, Version 2.0 (the "License");
# You may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0OA
#
# Authors:
# - Wen Guan, <*****@*****.**>, 2019
from ess.client.client import Client
scope = 'mc16_13TeV'
name = 'EVNT.16986378._000012.pool.root.1'
client = Client(host='https://aipanda182.cern.ch:8443')
req = client.get_content(scope=scope, name=name, min_id=None, max_id=None)
print req
req = client.get_content(scope=scope, name=name, min_id=1, max_id=10)
print req
req = client.get_content(scope=scope, name=name, min_id=7001, max_id=7010)
print req
req = client.get_content(scope=scope,
name=name,
min_id=7001,
max_id=7010,
status='AVAILABLE')
'granularity_level': 50,
'priority': 99,
'edge_id': None,
'status': 'NEW',
'request_meta': {'jeditaskid': 16986388, 'site': 'CERN'},
}
"""
properties = {
'scope': 'mc15_13TeV',
'name': 'EVNT.06402143._000021.pool.root.1',
'data_type': 'FILE',
'granularity_type': 'PARTIAL',
'granularity_level': 50,
'priority': 99,
'edge_id': None,
'status': 'NEW',
'request_meta': {'jeditaskid': 16986388, 'site': 'CERN'},
}
"""
client = Client(host='https://aipanda182.cern.ch:8443')
try:
request_id = client.add_request(**properties)
except exceptions.DuplicatedObject as ex:
print(ex)
req = client.get_request(request_id=request_id)
print req
#!/usr/bin/env python
#
# Licensed under the Apache License, Version 2.0 (the "License");
# You may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0OA
#
# Authors:
# - Wen Guan, <*****@*****.**>, 2019
from ess.client.client import Client
scope = 'mc16_13TeV'
name = 'EVNT.16986378._000012.pool.root.1'
client = Client(host='https://aipanda182.cern.ch:8443')
ret = client.download(scope=scope, name=name, min_id=1, max_id=10)
print ret
ret = client.download(scope=scope,
name=name,
min_id=1,
max_id=10,
dest_dir='..')
print ret
'edge_type': 'EDGE',
'status': 'ACTIVE',
'is_independent': True,
'continent': 'US',
'country_name': 'US',
'region_code': 'US',
'city': 'Madison',
'time_zone': 'timeZone',
'longitude': '111111',
'latitude': '2222222',
'total_space': 0,
'used_space': 0,
'num_files': 0
}
client = Client(host='https://aipanda182.cern.ch:8443')
try:
client.register_edge('my_edge_name2', **edge_properties)
except exceptions.DuplicatedObject as ex:
print(ex)
edge = client.get_edge('my_edge_name2')
print('get edge:')
print(edge)
edges = client.list_edges()
print('list edges:')
print(edges)
status = client.update_edge('my_edge_name2', city='MyCity')
def test_create_and_check_for_request_rest(self):
""" Request (REST): Test the creation, query, and deletion of a Request """
host = get_rest_host()
properties = {
'scope': 'test_scope',
'name': 'test_name_%s' % str(uuid()),
'data_type': 'DATASET',
'granularity_type': 'FILE',
'granularity_level': 1,
'priority': 99,
'edge_id': None,
'status': 'NEW',
'request_meta': {
'taskid': 975,
'job_id': 864
},
'processing_meta': None,
'errors': None,
}
client = Client(host=host)
request_id = client.add_request(**properties)
request = client.get_request(request_id=request_id)
assert_equal(request_id, request['request_id'])
assert_equal(request['scope'], properties['scope'])
assert_equal(request['name'], properties['name'])
assert_equal(str(request['data_type']), properties['data_type'])
assert_equal(str(request['granularity_type']),
properties['granularity_type'])
assert_equal(request['granularity_level'],
properties['granularity_level'])
assert_equal(request['priority'], properties['priority'])
assert_equal(request['edge_id'], properties['edge_id'])
assert_equal(str(request['status']), properties['status'])
# assert_equal(json.dumps(request['request_meta']), json.dumps(properties['request_meta']))
assert_equal(str(request['processing_meta']),
str(properties['processing_meta']))
assert_equal(request['errors'], properties['errors'])
client.update_request(request_id, status='ERROR')
request = client.get_request(request_id=request_id)
assert_equal(str(request['status']), 'ERROR')
with assert_raises(exceptions.NoObject):
client.get_request(request_id=GUID().generate_uuid())
client.delete_request(request_id)
with assert_raises(exceptions.NoObject):
client.get_request(request_id=request_id)
def test_create_and_check_for_edge_rest(self):
""" Edge (REST): Test the creation, query, and deletion of a Edge """
host = get_rest_host()
edge_name = 'test_edge_%s' % str(uuid())
edge_name = edge_name[:29]
properties = {
'edge_type': 'EDGE',
'status': 'ACTIVE',
'is_independent': True,
'continent': 'US',
'country_name': 'US',
'region_code': 'US',
'city': 'Madison',
'longitude': '111111',
'latitude': '2222222',
'total_space': 0,
'used_space': 0,
'num_files': 0
}
client = Client(host=host)
client.register_edge(edge_name, **properties)
edge = client.get_edge(edge_name)
assert_equal(edge['edge_name'], edge_name)
assert_equal(str(edge['edge_type']), properties['edge_type'])
assert_equal(str(edge['status']), properties['status'])
assert_equal(edge['is_independent'], properties['is_independent'])
assert_equal(edge['continent'], properties['continent'])
assert_equal(edge['country_name'], properties['country_name'])
assert_equal(edge['region_code'], properties['region_code'])
assert_equal(edge['city'], properties['city'])
assert_equal(edge['longitude'], properties['longitude'])
assert_equal(edge['latitude'], properties['latitude'])
assert_equal(edge['total_space'], properties['total_space'])
assert_equal(edge['used_space'], properties['used_space'])
assert_equal(edge['num_files'], properties['num_files'])
with assert_raises(exceptions.NoObject):
client.get_edge('not_exist_edge')
with assert_raises(exceptions.DuplicatedObject):
client.register_edge(edge_name, **properties)
client.update_edge(edge_name, status='LOSTHEARTBEAT')
edge = client.get_edge(edge_name)
assert_equal(str(edge['status']), 'LOSTHEARTBEAT')
edges = client.list_edges()
assert_true(len(edges) >= 1)
client.delete_edge(edge_name)
with assert_raises(exceptions.NoObject):
client.get_edge(edge_name)
class Finisher(BaseDaemon):
"""
The Finisher daemon class
"""
def __init__(self, num_threads=1, **kwargs):
super(Finisher, self).__init__(num_threads, **kwargs)
self.config_section = Sections.Finisher
self.setup_logger()
self.resource_name = self.get_resouce_name()
head_service = self.get_head_service()
if head_service:
self.head_client = Client(head_service)
else:
self.head_client = None
if hasattr(self, 'send_messaging') and self.send_messaging:
self.send_messaging = True
else:
self.send_messaging = False
def sync_contents(self, collection_scope, collection_name, edge_name,
edge_id, coll_id):
"""
Synchronize the content to the head service.
"""
if not self.head_client:
return
contents = get_contents_by_edge(edge_name=edge_name,
edge_id=edge_id,
coll_id=coll_id)
contents_list = []
for content in contents:
cont = {
'scope': content.scope,
'name': content.name,
'min_id': content.min_id,
'max_id': content.max_id,
'content_type': str(content.content_type),
'status': str(content.status),
'priority': content.priority,
'num_success': content.num_success,
'num_failure': content.num_failure,
'last_failed_at': content.last_failed_at,
'pfn_size': content.pfn_size,
'pfn': content.pfn,
'object_metadata': content.object_metadata
}
contents_list.append(cont)
self.head_client.add_contents(collection_scope, collection_name,
edge_name, contents_list)
def finish_local_requests(self):
reqs = get_requests(edge_name=self.resource_name,
status=RequestStatus.SPLITTING)
for req in reqs:
if req.granularity_type == GranularityType.FILE:
statistics = get_contents_statistics(
edge_name=self.resource_name,
edge_id=req.edge_id,
coll_id=req.processing_meta['coll_id'],
content_type=ContentType.FILE)
items = {}
for item in statistics:
items[item.status] = items.counter
if len(items.keys()) == 1 and items.keys(
)[0] == ContentStatus.AVAILABLE and items.values()[0] > 0:
self.logger.info(
'All files are available for request(%s): %s' %
(req.request_id, items))
# To sync content info to the head service
self.sync_contents(collection_scope=req.scope,
collection_name=req.name,
edge_name=self.resource_name,
edge_id=req.edge_id,
coll_id=req.processing_meta['coll_id'])
req.status = RequestStatus.AVAILABLE
self.logger.info("Updating request %s to status %s" %
(req.request_id, req.status))
update_request(req.request_id, {'status': req.status})
if self.send_messaging:
msg = {
'event_type': 'REQUEST_DONE',
'payload': {
'scope': req.scope,
'name': req.name,
'metadata': req.request_metadata
},
'created_at':
date_to_str(datetime.datetime.utcnow())
}
self.messaging_queue.put(msg)
else:
self.logger.info(
'Not all files are available for request(%s): %s' %
(req.request_id, items))
if req.granularity_type == GranularityType.PARTIAL:
statistics = get_contents_statistics(
edge_name=self.resource_name,
edge_id=req.edge_id,
coll_id=req.processing_meta['coll_id'],
content_type=ContentType.PARTIAL)
items = {}
for item in statistics:
items[item.status] = item.counter
if len(items.keys()) == 1 and items.keys(
)[0] == ContentStatus.AVAILABLE and items.values()[0] > 0:
self.logger.info(
'All partial files are available for request(%s): %s' %
(req.request_id, items))
# To sync content info to the head service
self.sync_contents(collection_scope=req.scope,
collection_name=req.name,
edge_name=self.resource_name,
edge_id=req.edge_id,
coll_id=req.processing_meta['coll_id'])
req.status = RequestStatus.AVAILABLE
self.logger.info("Updating request %s to status %s" %
(req.request_id, req.status))
update_request(req.request_id, {'status': req.status})
if self.send_messaging:
msg = {
'event_type': 'REQUEST_DONE',
'payload': {
'scope': req.scope,
'name': req.name,
'metadata': req.request_meta
},
'created_at':
date_to_str(datetime.datetime.utcnow())
}
self.messaging_queue.put(msg)
else:
self.logger.info(
'Not all partial files are available for request(%s): %s'
% (req.request_id, items))
def run(self):
"""
Main run function.
"""
try:
self.logger.info("Starting main thread")
self.load_plugins()
if self.send_messaging:
self.start_messaging_broker()
while not self.graceful_stop.is_set():
try:
self.finish_local_requests()
for i in range(5):
time.sleep(1)
except ESSException as error:
self.logger.error("Main thread ESSException: %s" %
str(error))
except Exception as error:
self.logger.critical("Main thread exception: %s\n%s" %
(str(error), traceback.format_exc()))
except KeyboardInterrupt:
self.stop()
except Exception as error:
self.logger.error("Main thread ESSException: %s, %s" %
(str(error), traceback.format_exc()))
if self.send_messaging:
self.stop_messaging_broker()
class Assigner(BaseDaemon):
"""
The Assigner daemon class
"""
def __init__(self, num_threads=1, **kwargs):
super(Assigner, self).__init__(num_threads, **kwargs)
self.config_section = Sections.Assigner
self.setup_logger()
self.resource_name = self.get_resouce_name()
head_service = self.get_head_service()
if head_service:
self.head_client = Client(head_service)
else:
self.head_client = None
def assign_local_requests(self):
reqs = get_requests(edge_name=self.resource_name,
status=RequestStatus.ASSIGNING)
for req in reqs:
req.status = RequestStatus.ASSIGNED
update_request(req.request_id, {'status': req.status})
def assign_remote_requests(self):
if not self.head_client:
return
reqs = []
try:
reqs = self.head_client.get_requests(edge_name=self.resource_name,
status=str(
RequestStatus.ASSIGNING))
except NoObject:
pass
except ESSException as error:
self.logger.info(
"Caught exception when get requests from the head service: %s"
% str(error))
for req in reqs:
coll = self.head_client.get_collection(req.scope, req.name)
collection = {
'scope': coll.scope,
'name': coll.name,
'collection_type': coll.collection_type,
'coll_size': coll.coll_size,
'global_status': coll.global_status,
'total_files': coll.total_files,
'num_replicas': coll.num_replicas,
'coll_metadata': coll.coll_metadata
}
try:
add_collection(**collection)
except DuplicatedObject as error:
self.logger.info("Collection is already registered: %s, %s" %
(collection, error))
req.status = RequestStatus.ASSIGNED
processing_meta = req.processing_meta
processing_meta['original_request_id'] = req.request_id
request = {
'scope': req.scope,
'name': req.name,
'data_type': req.data_type,
'granularity_type': req.granularity_type,
'granularity_level': req.granularity_level,
'priority': req.priority,
'edge_id': get_edge_id(self.resource_name),
'status': req.status,
'request_meta': req.request_meta,
'processing_meta': processing_meta,
'errors': req.errors
}
try:
add_request(**request)
except DuplicatedObject as error:
self.logger.info("Request is already registered: %s, %s" %
(request, error))
self.head_client.update_request(req.request_id,
status=str(req.status))
def get_tasks(self):
"""
Get tasks to process
"""
tasks = [{
'name': 'assign_local_requests'
}, {
'name': 'assign_remote_requests'
}]
self.logger.info("Main thread get %s tasks" % len(tasks))
for task in tasks:
self.tasks.put(task)
def process_task(self, req):
"""
Process task
"""
if req['name'] == 'assign_local_requests':
self.assign_local_requests()
if req['name'] == 'assign_remote_requests':
self.assign_remote_requests()
def finish_tasks(self):
"""
Finish processing the finished tasks, for example, update db status.
"""
self.graceful_stop.wait(30)