def POST(self, edge_name):
""" Create Edge with given name.
HTTP Success:
200 OK
HTTP Error:
400 Bad request
500 Internal Error
"""
json_data = data()
args = ['edge_type', 'status', 'is_independent', 'continent', 'country_name', 'region_code', 'city',
'longitude', 'latitude', 'total_space', 'used_space', 'num_files']
try:
parameters = {}
json_data = json.loads(json_data)
for key, value in json_data.items():
if key in args:
parameters[key] = value
except ValueError:
raise self.generate_http_response(HTTP_STATUS_CODE.BadRequest, exc_cls=exceptions.BadRequest.__name__, exc_msg='Cannot decode json parameter dictionary')
try:
edge_id = register_edge(edge_name, **parameters)
except exceptions.DuplicatedObject as error:
raise self.generate_http_response(HTTP_STATUS_CODE.Conflict, exc_cls=error.__class__.__name__, exc_msg=error)
except exceptions.ESSException as error:
raise self.generate_http_response(HTTP_STATUS_CODE.InternalError, exc_cls=error.__class__.__name__, exc_msg=error)
except Exception as error:
print(error)
print(format_exc())
raise self.generate_http_response(HTTP_STATUS_CODE.InternalError, exc_cls=exceptions.CoreException.__name__, exc_msg=error)
raise self.generate_http_response(HTTP_STATUS_CODE.OK, data={'edge_id': edge_id})
def finish_tasks(self):
"""
Finish processing the finished tasks, for example, update db status.
"""
while not self.finished_tasks.empty():
task = self.finished_tasks.get()
self.logger.info("Main thread finishing task: %s" % task)
try:
parameters = {
'edge_type': self.edge_type,
'status': EdgeStatus.ACTIVE,
'is_independent': self.is_independent,
'continent': self.continent,
'country_name': self.country_name,
'region_code': self.region_code,
'city': self.city,
'longitude': self.longitude,
'latitude': self.latitude,
'total_space': self.total_space,
'used_space': self.used_space,
'num_files': self.num_files
}
self.logger.info("Updating edge %s with parameters: %s" %
(self.get_resouce_name(), parameters))
update_edge(edge_name=self.get_resouce_name(),
parameters=parameters)
except NoObject as error:
self.logger.info(
"Edge %s doesn't exist(%s), will register it" %
(self.get_resouce_name(), error))
register_edge(self.get_resouce_name(),
edge_type=self.edge_type,
status=EdgeStatus.ACTIVE,
is_independent=self.is_independent,
continent=self.continent,
country_name=self.country_name,
region_code=self.region_code,
city=self.city,
longitude=self.longitude,
latitude=self.latitude,
total_space=self.total_space,
used_space=self.used_space,
num_files=self.num_files)
def test_create_and_check_for_collection(self):
""" Catalog (CORE): Test the creation, query, and deletion of a Collection """
edge_name = 'test_rse_%s' % str(uuid())
edge_name = edge_name[:29]
properties_edge = {
'edge_type': 'EDGE',
'status': 'ACTIVE',
'is_independent': True,
'continent': 'US',
'country_name': 'US',
'region_code': 'US',
'city': 'Madison',
'longitude': '111111',
'latitude': '2222222',
'total_space': 0,
'used_space': 0,
'num_files': 0
}
register_edge(edge_name, **properties_edge)
properties = {
'scope': 'test_scope',
'name': 'test_name_%s' % str(uuid()),
'collection_type': 'DATASET',
'coll_size': 100,
'global_status': 'NEW',
'total_files': 100,
'num_replicas': 10,
'coll_metadata': {
'attr1': 'abd',
'attr2': 1
}
}
collection_id = add_collection(**properties)
collection = get_collection(properties['scope'], properties['name'])
assert_equal(collection_id, collection.coll_id)
json.dumps(collection.to_dict())
assert_equal(collection.scope, properties['scope'])
assert_equal(collection.name, properties['name'])
assert_equal(str(collection.collection_type),
properties['collection_type'])
assert_equal(str(collection.global_status),
properties['global_status'])
assert_equal(collection.coll_size, properties['coll_size'])
assert_equal(collection.total_files, properties['total_files'])
assert_equal(collection.num_replicas, properties['num_replicas'])
# assert_equal(collection.coll_metadata, properties['coll_metadata'])
with assert_raises(exceptions.DuplicatedObject):
add_collection(**properties)
with assert_raises(exceptions.NoObject):
get_collection(properties['scope'], 'not_exist_name')
update_collection(scope=properties['scope'],
name=properties['name'],
parameters={'global_status': 'UNAVAILABLE'})
collection = get_collection(properties['scope'],
properties['name'],
coll_id=collection.coll_id)
assert_equal(str(collection.global_status), 'UNAVAILABLE')
delete_collection(scope=properties['scope'],
name=properties['name'],
coll_id=collection.coll_id)
with assert_raises(exceptions.NoObject):
get_collection(scope=properties['scope'],
name=properties['name'],
coll_id=collection.coll_id)
delete_edge(edge_name)
def test_create_and_check_for_content(self):
""" Catalog (CORE): Test the creation, query, and deletion of a Content """
edge_name = 'test_rse_%s' % str(uuid())
edge_name = edge_name[:29]
properties_edge = {
'edge_type': 'EDGE',
'status': 'ACTIVE',
'is_independent': True,
'continent': 'US',
'country_name': 'US',
'region_code': 'US',
'city': 'Madison',
'longitude': '111111',
'latitude': '2222222',
'total_space': 0,
'used_space': 0,
'num_files': 0
}
edge_id = register_edge(edge_name, **properties_edge)
properties_collection = {
'scope': 'test_scope',
'name': 'test_name_%s' % str(uuid()),
'collection_type': 'DATASET',
'coll_size': 100,
'global_status': 'NEW',
'total_files': 100,
'num_replicas': 10,
'coll_metadata': {
'attr1': 'abd',
'attr2': 1
},
}
collection_id = add_collection(**properties_collection)
properties = {
'scope': 'test_scope',
'name': 'test_name_%s' % str(uuid()),
'min_id': 0,
'max_id': 7,
'coll_id': collection_id,
'content_type': 'PARTIAL',
'edge_id': edge_id,
'status': 'NEW',
'priority': 100,
'num_success': 1,
'num_failure': 0,
'last_failed_at': None,
'pfn_size': 1,
'pfn': 'adss',
'object_metadata': {
'size': 1,
'md5': '1322232',
'adler32': '12345678'
}
}
content_id = add_content(**properties)
content = get_content(properties['scope'],
properties['name'],
properties['min_id'],
properties['max_id'],
edge_name=edge_name)
assert_equal(content_id, content.content_id)
json.dumps(content.to_dict())
content = get_content(properties['scope'],
properties['name'],
properties['min_id'],
properties['max_id'],
edge_id=edge_id)
assert_equal(content_id, content.content_id)
assert_equal(content.scope, properties['scope'])
assert_equal(content.name, properties['name'])
assert_equal(content.min_id, properties['min_id'])
assert_equal(content.max_id, properties['max_id'])
assert_equal(content.coll_id, properties['coll_id'])
assert_equal(str(content.content_type), properties['content_type'])
assert_equal(content.edge_id, properties['edge_id'])
assert_equal(str(content.status), properties['status'])
assert_equal(content.priority, properties['priority'])
assert_equal(content.num_success, properties['num_success'])
assert_equal(content.num_failure, properties['num_failure'])
assert_equal(content.last_failed_at, properties['last_failed_at'])
assert_equal(content.pfn_size, properties['pfn_size'])
assert_equal(content.pfn, properties['pfn'])
with assert_raises(exceptions.DuplicatedObject):
add_content(**properties)
with assert_raises(exceptions.NoObject):
get_content(properties['scope'],
properties['name'],
properties['min_id'],
properties['max_id'],
edge_name='Not_exist_edge')
with assert_raises(exceptions.NoObject):
get_content(properties['scope'],
properties['name'],
properties['min_id'] + 1,
properties['max_id'] + 1,
edge_name='Not_exist_edge')
with assert_raises(exceptions.NoObject):
get_content(properties['scope'],
properties['name'],
edge_name=edge_name)
update_content(scope=properties['scope'],
name=properties['name'],
min_id=properties['min_id'],
max_id=properties['max_id'],
edge_id=edge_id,
parameters={'status': 'UNAVAILABLE'})
content = get_content(properties['scope'],
properties['name'],
content_id=content_id)
assert_equal(str(content.status), 'UNAVAILABLE')
content = get_content_best_match(properties['scope'],
properties['name'],
properties['min_id'] + 1,
properties['max_id'] - 1,
edge_id=edge_id)
assert_equal(content_id, content.content_id)
delete_content(scope=properties['scope'],
name=properties['name'],
content_id=content_id)
with assert_raises(exceptions.NoObject):
get_content(scope=properties['scope'],
name=properties['name'],
content_id=content_id)
delete_collection(scope=properties_collection['scope'],
name=properties_collection['name'],
coll_id=collection_id)
delete_edge(edge_name)
def test_create_and_check_for_collection_replicas(self):
""" Catalog (CORE): Test the creation, query, and deletion of a Collection replicas"""
edge_name = 'test_rse_%s' % str(uuid())
edge_name = edge_name[:29]
properties_edge = {
'edge_type': 'EDGE',
'status': 'ACTIVE',
'is_independent': True,
'continent': 'US',
'country_name': 'US',
'region_code': 'US',
'city': 'Madison',
'longitude': '111111',
'latitude': '2222222',
'total_space': 0,
'used_space': 0,
'num_files': 0
}
edge_id = register_edge(edge_name, **properties_edge)
properties_coll = {
'scope': 'test_scope',
'name': 'test_name_%s' % str(uuid()),
'collection_type': 'DATASET',
'coll_size': 100,
'global_status': 'NEW',
'total_files': 100,
'num_replicas': 10,
'coll_metadata': {
'attr1': 'abd',
'attr2': 1
},
}
collection_id = add_collection(**properties_coll)
properties = {
'status': 'NEW',
'transferring_files': 0,
'replicated_files': 0,
'num_active_requests': 1,
'retries': 0
}
add_collection_replicas(properties_coll['scope'],
properties_coll['name'], edge_name,
**properties)
coll_replicas = get_collection_replicas(properties_coll['scope'],
properties_coll['name'],
edge_name)
assert_equal(collection_id, coll_replicas.coll_id)
assert_equal(edge_id, coll_replicas.edge_id)
assert_equal(str(coll_replicas.status), properties['status'])
assert_equal(coll_replicas.transferring_files,
properties['transferring_files'])
assert_equal(coll_replicas.replicated_files,
properties['replicated_files'])
assert_equal(coll_replicas.num_active_requests,
properties['num_active_requests'])
assert_equal(coll_replicas.retries, properties['retries'])
with assert_raises(exceptions.DuplicatedObject):
add_collection_replicas(properties_coll['scope'],
properties_coll['name'], edge_name,
**properties)
with assert_raises(exceptions.NoObject):
get_collection_replicas(properties_coll['scope'], 'not_exist_name',
edge_name)
coll_replicas1 = get_collection_replicas(properties_coll['scope'],
properties_coll['name'],
edge_name,
edge_id=edge_id)
assert_equal(coll_replicas1.coll_id, coll_replicas.coll_id)
assert_equal(coll_replicas1.edge_id, coll_replicas.edge_id)
coll_replicas1 = get_collection_replicas(properties_coll['scope'],
properties_coll['name'],
edge_name,
coll_id=coll_replicas.coll_id,
edge_id=edge_id)
assert_equal(coll_replicas1.coll_id, coll_replicas.coll_id)
assert_equal(coll_replicas1.edge_id, coll_replicas.edge_id)
update_collection_replicas(scope=properties_coll['scope'],
name=properties_coll['name'],
edge_name=edge_name,
parameters={'status': 'UNAVAILABLE'})
coll_replicas = get_collection_replicas(properties_coll['scope'],
properties_coll['name'],
edge_name)
assert_equal(str(coll_replicas.status), 'UNAVAILABLE')
delete_collection_replicas(properties_coll['scope'],
properties_coll['name'], edge_name)
with assert_raises(exceptions.NoObject):
get_collection_replicas(properties_coll['scope'],
properties_coll['name'], edge_name)
delete_edge(edge_name)
def test_create_and_check_for_edge_core(self):
""" Edge (CORE): Test the creation, query, and deletion of a Edge """
edge_name = 'test_edge_%s' % str(uuid())
edge_name = edge_name[:29]
properties = {
'edge_type': 'EDGE',
'status': 'ACTIVE',
'is_independent': True,
'continent': 'US',
'country_name': 'US',
'region_code': 'US',
'city': 'Madison',
'longitude': '111111',
'latitude': '2222222',
'total_space': 0,
'used_space': 0,
'num_files': 0
}
register_edge(edge_name, **properties)
edge = get_edge(edge_name)
assert_equal(edge.edge_name, edge_name)
assert_equal(str(edge.edge_type), properties['edge_type'])
assert_equal(str(edge.status), properties['status'])
assert_equal(edge.is_independent, properties['is_independent'])
assert_equal(edge.continent, properties['continent'])
assert_equal(edge.country_name, properties['country_name'])
assert_equal(edge.region_code, properties['region_code'])
assert_equal(edge.city, properties['city'])
assert_equal(edge.longitude, properties['longitude'])
assert_equal(edge.latitude, properties['latitude'])
assert_equal(edge.total_space, properties['total_space'])
assert_equal(edge.used_space, properties['used_space'])
assert_equal(edge.num_files, properties['num_files'])
assert_equal(edge.edge_id, get_edge_id(edge_name))
with assert_raises(exceptions.NoObject):
get_edge_id('not_exist_edge')
with assert_raises(exceptions.NoObject):
get_edge('not_exist_edge')
with assert_raises(exceptions.DuplicatedObject):
register_edge(edge_name)
update_edge(edge_name, parameters={'status': 'LOSTHEARTBEAT'})
edge = get_edge(edge_name)
assert_equal(str(edge.status), 'LOSTHEARTBEAT')
edges = get_edges()
assert_true(len(edges) >= 1)
render_json(**edges[0])
edges[0].to_dict()
edges = get_edges(status='LOSTHEARTBEAT')
assert_true(len(edges) >= 1)
delete_edge(edge_name)
with assert_raises(exceptions.NoObject):
get_edge(edge_name)