def add_content(scope, name, min_id=None, max_id=None, coll_id=None, content_type=ContentType.FILE,
status=ContentStatus.NEW, priority=0, edge_name=None, edge_id=None, num_success=0,
num_failure=0, last_failed_at=None, pfn_size=0, pfn=None, object_metadata=None, session=None):
"""
Add a collection content.
:param scope: The scope of the collection data.
:param name: The name of the collection data.
:param min_id: The minimum id of the partial file, related to the whole file.
:param max_id: The maximum id of the partial file, related to the whole file.
:param coll_id: The collection it belongs to.
:param content_type: The type of the cotent data.
:param status: The status of the content.
:param priority: The priority as an integer.
:param edge_name: The name of the replicating edge.
:param edge_id: The id of the replicating edge.
:param num_success: The number of successful access.
:param num_failure; The number of failed access
:param last_failed_at: The time of last failure.
:param pfn_size: The size of physical file name.
:param pfn: Physical file name.
:raises DuplicatedObject: If an edge with the same name exists.
:raises DatabaseException: If there is a database error.
:returns: collection content id.
"""
if isinstance(content_type, str) or isinstance(content_type, unicode):
content_type = ContentType.from_sym(str(content_type))
if isinstance(status, str) or isinstance(status, unicode):
status = ContentStatus.from_sym(str(status))
if not edge_id:
edge_id = get_edge_id(edge_name=edge_name, session=session)
new_content = models.CollectionContent(scope=scope, name=name, min_id=min_id, max_id=max_id, coll_id=coll_id,
content_type=content_type, status=status, priority=priority,
edge_id=edge_id, num_success=num_success, num_failure=num_failure,
last_failed_at=last_failed_at, pfn_size=pfn_size, pfn=pfn,
object_metadata=object_metadata)
try:
new_content.save(session=session)
except IntegrityError:
raise exceptions.DuplicatedObject('Content %s:%s[%s:%s](at edge %s) already exists!' % (scope, name, min_id, max_id, edge_name))
except DatabaseError as error:
raise exceptions.DatabaseException(error.args)
return new_content.content_id
class CollectionContent(BASE, ModelBase):
"""Represents files"""
__tablename__ = 'ess_coll_content'
content_id = Column(BigInteger().with_variant(Integer, "sqlite"),
Sequence('ESS_CONTENT_ID_SEQ'))
coll_id = Column(BigInteger)
scope = Column(String(SCOPE_LENGTH))
name = Column(String(NAME_LENGTH))
min_id = Column(BigInteger)
max_id = Column(BigInteger)
content_type = Column(ContentType.db_type(name='ESS_CONTENT_TYPE'),
default=ContentType.FILE)
# size = Column(BigInteger)
# md5 = Column(String(32))
# adler32 = Column(String(8))
edge_id = Column(Integer)
status = Column(ContentStatus.db_type(name='ESS_CONTENT_STATUS'),
default=ContentStatus.NEW)
priority = Column(Integer())
num_success = Column(Integer())
num_failure = Column(Integer())
last_failed_at = Column(DateTime)
pfn_size = Column(BigInteger)
pfn = Column(String(1024))
object_metadata = Column(JSON())
_table_args = (
PrimaryKeyConstraint('content_id', name='ESS_COLL_CONTENT_PK'),
# PrimaryKeyConstraint('scope', 'name', 'coll_id', 'content_type', 'min_id', 'max_id', 'edge_id', 'content_id', name='ESS_COLL_CONTENT_PK'),
ForeignKeyConstraint(['edge_id'], ['ess_edges.edge_id'],
name='ESS_CONTENT_EDGE_ID_FK'),
ForeignKeyConstraint(['coll_id'], ['ess_coll.coll_id'],
name='ESS_CONTENT_COLL_ID_FK'),
CheckConstraint('status IS NOT NULL', name='ESS_CONTENT_STATUS_NN'),
UniqueConstraint('scope',
'name',
'coll_id',
'content_type',
'min_id',
'max_id',
'edge_id',
name='ESS_CONTENT_UQ'),
Index('ESS_CONTENT_SCOPE_NAME_IDX', 'scope', 'name', 'edge_id',
'status'),
Index('ESS_CONTENT_SCOPE_NAME_MM_IDX', 'scope', 'name', 'content_type',
'min_id', 'max_id', 'edge_id', 'status'),
Index('ESS_CONTENT_COLLECTION_ID_IDX', 'coll_id', 'status'),
Index('ESS_CONTENT_EDGE_ID_IDX', 'edge_id', 'status'),
Index('ESS_CONTENT_STATUS_PRIO_IDX', 'status', 'priority'))
def get_content_best_match(scope, name, min_id=None, max_id=None, edge_name=None, edge_id=None, status=None, session=None):
"""
Get a collection content or raise a NoObject exception.
:param scope: The scope of the collection data.
:param name: The name of the collection data.
:param min_id: The minimum id of the partial file, related to the whole file.
:param max_id: The maximum id of the partial file, related to the whole file.
:param edge_name: The name of the edge.
:param edge_id: The id of the Edge
:param session: The database session in use.
:raises NoObject: If no edge is founded.
:returns: Content model.
"""
try:
if not edge_id and edge_name:
edge_id = get_edge_id(edge_name=edge_name, session=session)
if status and (isinstance(status, str) or isinstance(status, unicode)):
status = ContentStatus.from_sym(str(status))
query = session.query(models.CollectionContent).filter_by(scope=scope, name=name)
if status:
query = query.filter_by(status=status)
if edge_id:
query = query.filter_by(edge_id=edge_id)
if min_id is not None and max_id is not None:
contents = query.filter(and_(CollectionContent.min_id <= min_id, CollectionContent.max_id >= max_id)).all()
content = None
for row in contents:
if (not content) or (content['max_id'] - content['min_id'] > row['max_id'] - row['min_id']):
content = row
if content is None:
raise sqlalchemy.orm.exc.NoResultFound()
else:
content = query.filter_by(content_type=ContentType.FILE).one()
content['content_type'] = content.content_type
content['status'] = content.status
return content
except sqlalchemy.orm.exc.NoResultFound:
raise exceptions.NoObject('Content %s:%s[%s:%s](at edge %s) cannot be found' % (scope, name, min_id, max_id, edge_id))
def get_contents_by_edge(edge_name, edge_id=None, status=None, coll_id=None, content_type=None,
collection_scope=None, collection_name=None, limit=None, session=None):
"""
Get a collection content or raise a NoObject exception.
:param edge_name: The name of the edge.
:param edge_id: The id of the Edge
:param status: The status of the content.
:param coll_id: The collection id.
:param content_type: The tyep of the content.
:param limit: Number to return limited items.
:param session: The database session in use.
:raises NoObject: If no edge is founded.
:returns: Content model.
"""
try:
if not edge_id:
edge_id = get_edge_id(edge_name=edge_name, session=session)
if not coll_id and (collection_scope and collection_name):
coll_id = get_collection_id(collection_scope, collection_name)
query = session.query(models.CollectionContent).filter_by(edge_id=edge_id)
if status:
if isinstance(status, str) or isinstance(status, unicode):
status = ContentStatus.from_sym(str(status))
query = query.filter_by(status=status)
if coll_id:
query = query.filter_by(coll_id=coll_id)
if content_type:
query = query.filter_by(content_type=content_type)
if limit:
query = query.limit(limit)
contents = query.all()
for content in contents:
content['content_type'] = content.content_type
content['status'] = content.status
return contents
except sqlalchemy.orm.exc.NoResultFound:
raise exceptions.NoObject('No contents at edge %s with status %s' % (edge_name, status))
def get_contents_statistics(edge_name, edge_id=None, coll_id=None, status=None, content_type=None, session=None):
"""
Get content statistics.
:param edge_name: The name of the edge.
:param edge_id: The id of the Edge
:param coll_id: The collection id.
:param status: The status of the content.
:param content_type: The tyep of the content.
:param session: The database session in use.
:returns: dict.
"""
try:
if not edge_id and edge_name:
edge_id = get_edge_id(edge_name=edge_name, session=session)
query = session.query(models.CollectionContent.edge_id, models.CollectionContent.coll_id,
models.CollectionContent.content_type, models.CollectionContent.status,
func.count(1).label('counter'))
if edge_id:
query = query.filter_by(edge_id=edge_id)
if coll_id:
query = query.filter_by(coll_id=coll_id)
if status:
if isinstance(status, str) or isinstance(status, unicode):
status = ContentStatus.from_sym(str(status))
query = query.filter_by(status=status)
if content_type:
if isinstance(content_type, str) or isinstance(content_type, unicode):
content_type = ContentType.from_sym(str(content_type))
query = query.filter_by(content_type=content_type)
query = query.group_by(models.CollectionContent.edge_id, models.CollectionContent.coll_id,
models.CollectionContent.content_type, models.CollectionContent.status)
statistics = query.all()
return statistics
except sqlalchemy.orm.exc.NoResultFound:
raise exceptions.NoObject('Failed to get statistics for edge %s and collection %s' % (edge_id, coll_id))
def get_contents_by_edge(edge_name, edge_id=None, status=None, coll_id=None, content_type=ContentType.FILE, session=None):
"""
Get a collection content or raise a NoObject exception.
:param edge_name: The name of the edge.
:param edge_id: The id of the Edge
:param status: The status of the content.
:param coll_id: The collection id.
:param session: The database session in use.
:raises NoObject: If no edge is founded.
:returns: Content model.
"""
try:
if not edge_id:
edge_id = get_edge_id(edge_name=edge_name, session=session)
if status:
if isinstance(status, str) or isinstance(status, unicode):
status = ContentStatus.from_sym(str(status))
if coll_id:
contents = session.query(models.CollectionContent).filter_by(edge_id=edge_id, status=status, coll_id=coll_id, content_type=content_type).all()
else:
contents = session.query(models.CollectionContent).filter_by(edge_id=edge_id, status=status, content_type=content_type).all()
else:
if coll_id:
contents = session.query(models.CollectionContent).filter_by(edge_id=edge_id, coll_id=coll_id, content_type=content_type).all()
else:
contents = session.query(models.CollectionContent).filter_by(edge_id=edge_id, content_type=content_type).all()
for content in contents:
content['content_type'] = content.content_type
content['status'] = content.status
return contents
except sqlalchemy.orm.exc.NoResultFound:
raise exceptions.NoObject('No contents at edge %s with status %s' % (edge_name, status))
def update_contents_by_id(files, session=None):
"""
update a collection content.
:param files: The list of content files.
:param session: The database session in use.
:raises NoObject: If no content is founded.
:raises DatabaseException: If there is a database error.
"""
try:
for id in files:
parameters = files[id]
if 'status' in parameters and \
(isinstance(parameters['status'], str) or isinstance(parameters['status'], unicode)):
parameters['status'] = ContentStatus.from_sym(str(parameters['status']))
session.query(models.CollectionContent).filter_by(content_id=id).update(parameters)
except DatabaseError as error:
raise exceptions.DatabaseException(error.args)
def update_content(scope, name, min_id=None, max_id=None, edge_name=None, edge_id=None, content_id=None, parameters=None, session=None):
"""
update a collection content.
:param scope: The scope of the collection data.
:param name: The name of the collection data.
:param min_id: The minimum id of the partial file, related to the whole file.
:param max_id: The maximum id of the partial file, related to the whole file.
:param edge_name: The name of the edge.
:param edge_id: The id of the Edge
:param session: The database session in use.
:raises NoObject: If no content is founded.
:raises DatabaseException: If there is a database error.
:returns: content id.
"""
content = get_content(scope, name, min_id=min_id, max_id=max_id,
edge_name=edge_name, edge_id=edge_id, session=session)
try:
content = get_content(scope, name, min_id=min_id, max_id=max_id, content_id=content_id,
edge_name=edge_name, edge_id=edge_id, session=session)
if 'content_type' in parameters and \
(isinstance(parameters['content_type'], str) or isinstance(parameters['content_type'], unicode)):
parameters['content_type'] = ContentType.from_sym(str(parameters['content_type']))
if 'status' in parameters and \
(isinstance(parameters['status'], str) or isinstance(parameters['status'], unicode)):
parameters['status'] = ContentStatus.from_sym(str(parameters['status']))
content.update(parameters)
except DatabaseError as error:
raise exceptions.DatabaseException(error.args)
return content.content_id