def findInvalidFiles(self, progress=progress.noProgress, filters=None,
checkSize=True, **kwargs):
"""
Goes through every file in this assetstore and finds those whose
underlying data is missing or invalid. This is a generator function --
for each invalid file found, a dictionary is yielded to the caller that
contains the file, its absolute path on disk, and a reason for invalid,
e.g. "missing" or "size".
:param progress: Pass a progress context to record progress.
:type progress: :py:class:`girder.utility.progress.ProgressContext`
:param filters: Additional query dictionary to restrict the search for
files. There is no need to set the ``assetstoreId`` in the filters,
since that is done automatically.
:type filters: dict or None
:param checkSize: Whether to make sure the size of the underlying
data matches the size of the file.
:type checkSize: bool
"""
filters = filters or {}
q = dict({
'assetstoreId': self.assetstore['_id']
}, **filters)
cursor = File().find(q)
progress.update(total=cursor.count(), current=0)
for file in cursor:
progress.update(increment=1, message=file['name'])
path = self.fullPath(file)
if not os.path.isfile(path):
yield {
'reason': 'missing',
'file': file,
'path': path
}
elif checkSize and os.path.getsize(path) != file['size']:
yield {
'reason': 'size',
'file': file,
'path': path
}
def deleteFile(self, file):
"""
Delete all of the chunks in the collection that correspond to the
given file.
"""
q = {
'chunkUuid': file['chunkUuid'],
'assetstoreId': self.assetstore['_id']
}
matching = File().find(q, limit=2, projection=[])
if matching.count(True) == 1:
# If we can't reach the database, we return anyway. A system check
# will be necessary to remove the abandoned file. Since we already
# can handle that case, tell Mongo to use a 0 write concern -- we
# don't need to know that the chunks have been deleted, and this
# can be faster.
try:
self.chunkColl.with_options(write_concern=pymongo.WriteConcern(
w=0)).delete_many({'uuid': file['chunkUuid']})
except pymongo.errors.AutoReconnect:
pass
def deleteFile(self, file):
"""
Deletes the file from disk if it is the only File in this assetstore
with the given sha512. Imported files are not actually deleted.
"""
from girder.models.file import File
if file.get('imported') or 'path' not in file:
return
q = {'sha512': file['sha512'], 'assetstoreId': self.assetstore['_id']}
path = os.path.join(self.assetstore['root'], file['path'])
if os.path.isfile(path):
with filelock.FileLock(path + '.deleteLock'):
matching = File().find(q, limit=2, fields=[])
matchingUpload = Upload().findOne(q)
if matching.count(True) == 1 and matchingUpload is None:
try:
os.unlink(path)
except Exception:
logger.exception('Failed to delete file %s' % path)
def deleteFile(self, file):
"""
We want to queue up files to be deleted asynchronously since it requires
an external HTTP request per file in order to delete them, and we don't
want to wait on that.
Files that were imported as pre-existing data will not actually be
deleted from S3, only their references in Girder will be deleted.
"""
if file['size'] > 0 and 'relpath' in file:
q = {
'relpath': file['relpath'],
'assetstoreId': self.assetstore['_id']
}
matching = File().find(q, limit=2, fields=[])
if matching.count(True) == 1:
events.daemon.trigger(info={
'client': self.client,
'bucket': self.assetstore['bucket'],
'key': file['s3Key']
}, callback=_deleteFileImpl)
def deleteFile(self, file):
"""
We want to queue up files to be deleted asynchronously since it requires
an external HTTP request per file in order to delete them, and we don't
want to wait on that.
Files that were imported as pre-existing data will not actually be
deleted from S3, only their references in Girder will be deleted.
"""
if file['size'] > 0 and 'relpath' in file:
q = {
'relpath': file['relpath'],
'assetstoreId': self.assetstore['_id']
}
matching = File().find(q, limit=2, fields=[])
if matching.count(True) == 1:
events.daemon.trigger(info={
'client': self.client,
'bucket': self.assetstore['bucket'],
'key': file['s3Key']
}, callback=_deleteFileImpl)
def deleteFile(self, file):
"""
Delete all of the chunks in the collection that correspond to the
given file.
"""
q = {
'chunkUuid': file['chunkUuid'],
'assetstoreId': self.assetstore['_id']
}
matching = File().find(q, limit=2, projection=[])
if matching.count(True) == 1:
# If we can't reach the database, we return anyway. A system check
# will be necessary to remove the abandoned file. Since we already
# can handle that case, tell Mongo to use a 0 write concern -- we
# don't need to know that the chunks have been deleted, and this
# can be faster.
try:
self.chunkColl.with_options(
write_concern=pymongo.WriteConcern(w=0)).delete_many(
{'uuid': file['chunkUuid']})
except pymongo.errors.AutoReconnect:
pass
def findInvalidFiles(self,
progress=progress.noProgress,
filters=None,
checkSize=True,
**kwargs):
"""
Goes through every file in this assetstore and finds those whose
underlying data is missing or invalid. This is a generator function --
for each invalid file found, a dictionary is yielded to the caller that
contains the file, its absolute path on disk, and a reason for invalid,
e.g. "missing" or "size".
:param progress: Pass a progress context to record progress.
:type progress: :py:class:`girder.utility.progress.ProgressContext`
:param filters: Additional query dictionary to restrict the search for
files. There is no need to set the ``assetstoreId`` in the filters,
since that is done automatically.
:type filters: dict or None
:param checkSize: Whether to make sure the size of the underlying
data matches the size of the file.
:type checkSize: bool
"""
filters = filters or {}
q = dict({'assetstoreId': self.assetstore['_id']}, **filters)
cursor = File().find(q)
progress.update(total=cursor.count(), current=0)
for file in cursor:
progress.update(increment=1, message=file['name'])
path = self.fullPath(file)
if not os.path.isfile(path):
yield {'reason': 'missing', 'file': file, 'path': path}
elif checkSize and os.path.getsize(path) != file['size']:
yield {'reason': 'size', 'file': file, 'path': path}
def deleteFile(self, file):
"""
Deletes the file from disk if it is the only File in this assetstore
with the given sha512. Imported files are not actually deleted.
"""
from girder.models.file import File
if file.get('imported') or 'path' not in file:
return
q = {
'sha512': file['sha512'],
'assetstoreId': self.assetstore['_id']
}
path = os.path.join(self.assetstore['root'], file['path'])
if os.path.isfile(path):
with filelock.FileLock(path + '.deleteLock'):
matching = File().find(q, limit=2, fields=[])
matchingUpload = Upload().findOne(q)
if matching.count(True) == 1 and matchingUpload is None:
try:
os.unlink(path)
except Exception:
logger.exception('Failed to delete file %s' % path)
