def delete(self, paths):
"""Delete L{Tag}s matching C{paths}.
L{TagValue}s and permissions associated with the deleted L{Tag}s are
removed by cascading deletes in the database schema.
@param paths: A sequence of L{Tag.path}s.
@return: A C{list} of C{(objectID, Tag.path)} 2-tuples representing the
L{Tag}s that were removed.
"""
if isgenerator(paths):
paths = list(paths)
result = getTags(paths=paths)
deletedTagPaths = list(result.values(Tag.objectID, Tag.path))
# Delete the fluiddb/tags/description tag values stored for removed
# tags. Associated TagValue's are removed by an ON DELETE CASCADE
# trigger.
self._factory.tagValues(self._user).delete(
[(objectID, path) for objectID, _ in deletedTagPaths
for path in [u'fluiddb/tags/description', u'fluiddb/tags/path']])
# Touch all the objects for the given tag paths.
objectIDs = list(getObjectIDs(paths))
touchObjects(objectIDs)
result.remove()
return deletedTagPaths
def deleteTagValues(store, tagIDs, objectIDs):
"""Delete those L{TagValue}s whose tagID is in tagIDs and whose
objectID is in objectIDs.
@param store: The store to use when fetching data.
@param tagIDs: A sequence of L{TagValue.tagID}s to match values against.
@param objectIDs: A sequence of L{TagValue.objectID}s to match values
against.
"""
result = store.find(And(TagValue.tagID.is_in(tagIDs),
TagValue.objectID.is_in(objectIDs))).remove()
if result:
touchObjects(objectIDs)
def deleteTagValues(store, tagIDs, objectIDs):
"""Delete those L{TagValue}s whose tagID is in tagIDs and whose
objectID is in objectIDs.
@param store: The store to use when fetching data.
@param tagIDs: A sequence of L{TagValue.tagID}s to match values against.
@param objectIDs: A sequence of L{TagValue.objectID}s to match values
against.
"""
result = store.find(
And(TagValue.tagID.is_in(tagIDs),
TagValue.objectID.is_in(objectIDs))).remove()
if result:
touchObjects(objectIDs)
def batchIndex(objectsFilename, interval, maxObjects, sleepFunction=None):
"""
Touches all the objects in a given file in batches every a given interval.
@param objectsFilename: The path of the file with the object IDS to touch.
@param interval: The interval in minutes to touch a batch of objects.
@param maxObjects: The number of objects to process in each batch.
@param sleepFunction: a C{time.sleep} like function used for testing
purposes.
"""
if sleepFunction is None:
sleepFunction = time.sleep
objectIDs = []
batch = 0
interval = interval * 60
with open(objectsFilename) as objectsFile:
for line in objectsFile:
if len(objectIDs) == 0:
logging.info('Processing batch %d (%d objects processed).' %
(batch, batch * maxObjects))
try:
objectID = UUID(line.strip())
objectIDs.append(objectID)
except ValueError:
logging.error('Invalid objectID: %r', line)
continue
if len(objectIDs) >= maxObjects:
touchObjects(objectIDs)
try:
transaction.commit()
except:
transaction.abort()
raise
logging.info('Batch done. Sleeping until next batch.')
objectIDs = []
batch += 1
sleepFunction(interval)
touchObjects(objectIDs)
try:
transaction.commit()
except:
transaction.abort()
raise
logging.info('All objects processed.')
def batchIndex(objectsFilename, interval, maxObjects, sleepFunction=None):
"""
Touches all the objects in a given file in batches every a given interval.
@param objectsFilename: The path of the file with the object IDS to touch.
@param interval: The interval in minutes to touch a batch of objects.
@param maxObjects: The number of objects to process in each batch.
@param sleepFunction: a C{time.sleep} like function used for testing
purposes.
"""
if sleepFunction is None:
sleepFunction = time.sleep
objectIDs = []
batch = 0
interval = interval * 60
with open(objectsFilename) as objectsFile:
for line in objectsFile:
if len(objectIDs) == 0:
logging.info('Processing batch %d (%d objects processed).'
% (batch, batch * maxObjects))
try:
objectID = UUID(line.strip())
objectIDs.append(objectID)
except ValueError:
logging.error('Invalid objectID: %r', line)
continue
if len(objectIDs) >= maxObjects:
touchObjects(objectIDs)
try:
transaction.commit()
except:
transaction.abort()
raise
logging.info('Batch done. Sleeping until next batch.')
objectIDs = []
batch += 1
sleepFunction(interval)
touchObjects(objectIDs)
try:
transaction.commit()
except:
transaction.abort()
raise
logging.info('All objects processed.')
def delete(self, values):
"""Delete L{TagValue}s.
@param values: A sequence of C{(objectID, Tag.path)} 2-tuples to
delete values for.
@raise FeatureError: Raised if the given list of values is empty.
@return: The number of values deleted.
"""
if isgenerator(values):
values = list(values)
if not values:
raise FeatureError("Can't delete an empty list of tag values.")
paths = set([path for objectID, path in values])
objectIDs = set([objectID for objectID, path in values])
tagIDs = dict(getTags(paths).values(Tag.path, Tag.id))
values = [(objectID, tagIDs[path]) for objectID, path in values]
result = getTagValues(values).remove()
if result:
touchObjects(objectIDs)
return result
def delete(self, values):
"""Delete L{TagValue}s.
@param values: A sequence of C{(objectID, Tag.path)} 2-tuples to
delete values for.
@raise FeatureError: Raised if the given list of values is empty.
@return: The number of values deleted.
"""
if isgenerator(values):
values = list(values)
if not values:
raise FeatureError("Can't delete an empty list of tag values.")
paths = set([path for objectID, path in values])
objectIDs = set([objectID for objectID, path in values])
tagIDs = dict(getTags(paths).values(Tag.path, Tag.id))
values = [(objectID, tagIDs[path]) for objectID, path in values]
result = getTagValues(values).remove()
if result:
touchObjects(objectIDs)
return result
def testTouchObjects(self):
"""L{touchObjects} adds the objects to the C{dirty_objects} table."""
objectID = uuid4()
touchObjects([objectID])
self.assertNotIdentical(None, getDirtyObjects([objectID]).one())
def set(self, values):
"""Set or update L{TagValue}s.
L{Tag}s that don't exist are created automatically before L{TagValue}s
are stored. Associated L{TagPermission}s are created automatically
with the system-wide default permissions.
@param values: A C{dict} mapping object IDs to tags and values,
matching the following format::
{<object-id>: {<path>: <value>,
<path>: {'mime-type': <mime-type>,
'contents': <contents>}}}
A binary L{TagValue} is represented using a different layout than
other values types, as shown for the second value.
@raise FeatureError: Raised if the given list of values is empty.
@raise MalformedPathError: Raised if one of the given paths for a
nonexistent tag is empty or has unacceptable characters.
"""
if not values:
raise FeatureError("Can't set an empty list of tag values.")
objectIDs = set(values.keys())
# Implicitly create missing tags, if there are any.
paths = set()
for tagValues in values.itervalues():
paths.update(tagValues.iterkeys())
tagIDs = dict(getTags(paths=paths).values(Tag.path, Tag.id))
existingPaths = set(tagIDs.iterkeys())
unknownPaths = paths - existingPaths
if unknownPaths:
tags = [(path, u'Object for the attribute %s' % path)
for path in unknownPaths]
self._factory.tags(self._user).create(tags)
tagIDs = dict(getTags(paths=paths).values(Tag.path, Tag.id))
# Delete all existing tag values for the specified object IDs and
# paths.
deleteValues = []
for objectID in values:
for path in values[objectID].iterkeys():
deleteValues.append((objectID, tagIDs[path]))
getTagValues(deleteValues).remove()
# Set new tag values for the specified object IDs and paths.
for objectID in values:
tagValues = values[objectID]
for path, value in tagValues.iteritems():
tagID = tagIDs[path]
if isinstance(value, dict):
content = value['contents']
value = createTagValue(self._user.id, tagID, objectID,
{'mime-type': value['mime-type'],
'size': len(content)})
# This is necessary to tell PostgreSQL that generates a
# `value.id` immediately.
value.id = AutoReload
createOpaqueValue(value.id, content)
else:
createTagValue(self._user.id, tagID, objectID, value)
touchObjects(objectIDs)
result2.remove()
result.remove()
# Update the about value.
value = getAboutTagValues(objectIDs=[tag.objectID]).one()
if value is not None:
value.value = newAbout
else:
createAboutTagValue(tag.objectID, newAbout)
print 'Fixing system values'
tagValues.set(systemValues)
print 'Touching objects.'
result = store.find(TagValue.objectID,
TagValue.tagID.is_in(currentIDs))
touchObjects(list(result.config(distinct=True)))
touchObjects(systemValues.keys())
i += BATCH_SIZE
currentIDs = tagsToFix[i:i + BATCH_SIZE]
print 'Fixing namespaces.'
i = 0
currentIDs = namespacesToFix[i:i + BATCH_SIZE]
while currentIDs:
systemValues = {}
for namespace in store.find(Namespace, Namespace.id.is_in(currentIDs)):
if u'/' in namespace.path:
root, rest = namespace.path.split('/', 1)
newPath = u'/'.join([root.lower(), rest])
else:
def set(self, values):
"""Set or update L{TagValue}s.
L{Tag}s that don't exist are created automatically before L{TagValue}s
are stored. Associated L{TagPermission}s are created automatically
with the system-wide default permissions.
@param values: A C{dict} mapping object IDs to tags and values,
matching the following format::
{<object-id>: {<path>: <value>,
<path>: {'mime-type': <mime-type>,
'contents': <contents>}}}
A binary L{TagValue} is represented using a different layout than
other values types, as shown for the second value.
@raise FeatureError: Raised if the given list of values is empty.
@raise MalformedPathError: Raised if one of the given paths for a
nonexistent tag is empty or has unacceptable characters.
"""
if not values:
raise FeatureError("Can't set an empty list of tag values.")
objectIDs = set(values.keys())
# Implicitly create missing tags, if there are any.
paths = set()
for tagValues in values.itervalues():
paths.update(tagValues.iterkeys())
tagIDs = dict(getTags(paths=paths).values(Tag.path, Tag.id))
existingPaths = set(tagIDs.iterkeys())
unknownPaths = paths - existingPaths
if unknownPaths:
tags = [(path, u'Object for the attribute %s' % path)
for path in unknownPaths]
self._factory.tags(self._user).create(tags)
tagIDs = dict(getTags(paths=paths).values(Tag.path, Tag.id))
# Delete all existing tag values for the specified object IDs and
# paths.
deleteValues = []
for objectID in values:
for path in values[objectID].iterkeys():
deleteValues.append((objectID, tagIDs[path]))
getTagValues(deleteValues).remove()
# Set new tag values for the specified object IDs and paths.
for objectID in values:
tagValues = values[objectID]
for path, value in tagValues.iteritems():
tagID = tagIDs[path]
if isinstance(value, dict):
content = value['contents']
value = createTagValue(self._user.id, tagID, objectID, {
'mime-type': value['mime-type'],
'size': len(content)
})
# This is necessary to tell PostgreSQL that generates a
# `value.id` immediately.
value.id = AutoReload
createOpaqueValue(value.id, content)
else:
createTagValue(self._user.id, tagID, objectID, value)
touchObjects(objectIDs)
def testTouchObjects(self):
"""L{touchObjects} adds the objects to the C{dirty_objects} table."""
objectID = uuid4()
touchObjects([objectID])
self.assertNotIdentical(None, getDirtyObjects([objectID]).one())