def addon_delete_file_node(self, node, user, event_type, payload):
""" Get addon StoredFileNode(s), move it into the TrashedFileNode collection
and remove it from StoredFileNode.
Required so that the guids of deleted addon files are not re-pointed when an
addon file or folder is moved or renamed.
"""
if event_type == 'file_removed' and payload.get('provider', None) != 'osfstorage':
provider = payload['provider']
path = payload['metadata']['path']
materialized_path = payload['metadata']['materialized']
if path.endswith('/'):
folder_children = FileNode.resolve_class(provider, FileNode.ANY).find(
Q('provider', 'eq', provider) &
Q('node', 'eq', node) &
Q('materialized_path', 'startswith', materialized_path)
)
for item in folder_children:
if item.kind == 'file' and not TrashedFileNode.load(item._id):
item.delete(user=user)
elif item.kind == 'folder':
StoredFileNode.remove_one(item.stored_object)
else:
try:
file_node = FileNode.resolve_class(provider, FileNode.FILE).find_one(
Q('node', 'eq', node) &
Q('materialized_path', 'eq', materialized_path)
)
except NoResultsFound:
file_node = None
if file_node and not TrashedFileNode.load(file_node._id):
file_node.delete(user=user)
def addon_delete_file_node(self, node, user, event_type, payload):
""" Get addon StoredFileNode(s), move it into the TrashedFileNode collection
and remove it from StoredFileNode.
Required so that the guids of deleted addon files are not re-pointed when an
addon file or folder is moved or renamed.
"""
if event_type == 'file_removed' and payload.get('provider',
None) != 'osfstorage':
provider = payload['provider']
path = payload['metadata']['path']
materialized_path = payload['metadata']['materialized']
if path.endswith('/'):
folder_children = FileNode.resolve_class(
provider, FileNode.ANY).find(
Q('provider', 'eq', provider) & Q('node', 'eq', node)
& Q('materialized_path', 'startswith', materialized_path))
for item in folder_children:
if item.kind == 'file' and not TrashedFileNode.load(item._id):
item.delete(user=user)
elif item.kind == 'folder':
StoredFileNode.remove_one(item.stored_object)
else:
try:
file_node = FileNode.resolve_class(
provider, FileNode.FILE).find_one(
Q('node', 'eq', node)
& Q('materialized_path', 'eq', materialized_path))
except NoResultsFound:
file_node = None
if file_node and not TrashedFileNode.load(file_node._id):
file_node.delete(user=user)
def get_number_downloads_unique_and_total():
number_downloads_unique = 0
number_downloads_total = 0
projects = get_projects()
for project in projects:
for filenode in OsfStorageFile.find(Q('node', 'eq', project)):
for idx, version in enumerate(filenode.versions):
page = ':'.join(['download', project._id, filenode._id, str(idx)])
unique, total = get_basic_counters(page)
number_downloads_unique += unique or 0
number_downloads_total += total or 0
for filenode in TrashedFileNode.find(Q('provider', 'eq', 'osfstorage') & Q('node', 'eq', project) & Q('is_file', 'eq', True)):
for idx, version in enumerate(filenode.versions):
page = ':'.join(['download', project._id, filenode._id, str(idx)])
unique, total = get_basic_counters(page)
number_downloads_total += total or 0
number_downloads_unique += unique or 0
clear_modm_cache()
return number_downloads_unique, number_downloads_total
def addon_deleted_file(auth, node, **kwargs):
"""Shows a nice error message to users when they try to view
a deleted file
"""
# Allow file_node to be passed in so other views can delegate to this one
trashed = kwargs.get('file_node') or TrashedFileNode.load(kwargs.get('trashed_id'))
if not trashed:
raise HTTPError(httplib.NOT_FOUND, {
'message_short': 'Not Found',
'message_long': 'This file does not exist'
})
ret = serialize_node(node, auth, primary=True)
ret.update(rubeus.collect_addon_assets(node))
ret.update({
'urls': {
'render': None,
'sharejs': None,
'mfr': settings.MFR_SERVER_URL,
'gravatar': get_gravatar(auth.user, 25),
'files': node.web_url_for('collect_file_trees'),
},
'extra': {},
'size': 9966699, # Prevent file from being editted, just in case
'sharejs_uuid': None,
'file_name': trashed.name,
'file_path': trashed.path,
'provider': trashed.provider,
'materialized_path': trashed.materialized_path,
'error': FILE_GONE_ERROR_MESSAGE.format(file_name=trashed.name),
'private': getattr(node.get_addon(trashed.provider), 'is_private', False),
})
return ret, httplib.GONE
def addon_deleted_file(auth, node, **kwargs):
"""Shows a nice error message to users when they try to view
a deleted file
"""
# Allow file_node to be passed in so other views can delegate to this one
trashed = kwargs.get("file_node") or TrashedFileNode.load(kwargs.get("trashed_id"))
if not trashed:
raise HTTPError(httplib.NOT_FOUND, {"message_short": "Not Found", "message_long": "This file does not exist"})
ret = serialize_node(node, auth, primary=True)
ret.update(rubeus.collect_addon_assets(node))
ret.update(
{
"urls": {
"render": None,
"sharejs": None,
"mfr": settings.MFR_SERVER_URL,
"gravatar": get_gravatar(auth.user, 25),
"files": node.web_url_for("collect_file_trees"),
},
"extra": {},
"size": 9966699, # Prevent file from being editted, just in case
"sharejs_uuid": None,
"file_name": trashed.name,
"file_path": trashed.path,
"provider": trashed.provider,
"materialized_path": trashed.materialized_path,
"error": FILE_GONE_ERROR_MESSAGE.format(file_name=trashed.name),
"private": getattr(node.get_addon(trashed.provider), "is_private", False),
}
)
return ret, httplib.GONE
def addon_deleted_file(auth, node, **kwargs):
"""Shows a nice error message to users when they try to view
a deleted file
"""
# Allow file_node to be passed in so other views can delegate to this one
trashed = kwargs.get('file_node') or TrashedFileNode.load(kwargs.get('trashed_id'))
if not trashed:
raise HTTPError(httplib.NOT_FOUND, {
'message_short': 'Not Found',
'message_long': 'This file does not exist'
})
ret = serialize_node(node, auth, primary=True)
ret.update(rubeus.collect_addon_assets(node))
ret.update({
'urls': {
'render': None,
'sharejs': None,
'mfr': settings.MFR_SERVER_URL,
'gravatar': get_gravatar(auth.user, 25),
'files': node.web_url_for('collect_file_trees'),
},
'extra': {},
'size': 9966699, # Prevent file from being editted, just in case
'sharejs_uuid': None,
'file_name': trashed.name,
'file_path': trashed.path,
'provider': trashed.provider,
'materialized_path': trashed.materialized_path,
'error': FILE_GONE_ERROR_MESSAGE.format(file_name=trashed.name),
'private': getattr(node.get_addon(trashed.provider), 'is_private', False),
})
return ret, httplib.GONE
def update_file_guid_referent(self, node, event_type, payload, user=None):
if event_type == 'addon_file_moved' or event_type == 'addon_file_renamed':
source = payload['source']
destination = payload['destination']
source_node = Node.load(source['node']['_id'])
destination_node = node
file_guids = FileNode.resolve_class(source['provider'], FileNode.ANY).get_file_guids(
materialized_path=source['materialized'] if source['provider'] != 'osfstorage' else source['path'],
provider=source['provider'],
node=source_node)
if event_type == 'addon_file_renamed' and source['provider'] in settings.ADDONS_BASED_ON_IDS:
return
if event_type == 'addon_file_moved' and (source['provider'] == destination['provider'] and
source['provider'] in settings.ADDONS_BASED_ON_IDS) and source_node == destination_node:
return
for guid in file_guids:
obj = Guid.load(guid)
if source_node != destination_node and Comment.find(Q('root_target', 'eq', guid)).count() != 0:
update_comment_node(guid, source_node, destination_node)
if source['provider'] != destination['provider'] or source['provider'] != 'osfstorage':
old_file = FileNode.load(obj.referent._id)
obj.referent = create_new_file(obj, source, destination, destination_node)
obj.save()
if old_file and not TrashedFileNode.load(old_file._id):
old_file.delete()
def get_number_downloads_unique_and_total():
number_downloads_unique = 0
number_downloads_total = 0
projects = get_projects()
for project in projects:
for filenode in OsfStorageFile.find(Q('node', 'eq', project)):
for idx, version in enumerate(filenode.versions):
page = ':'.join(
['download', project._id, filenode._id,
str(idx)])
unique, total = get_basic_counters(page)
number_downloads_unique += unique or 0
number_downloads_total += total or 0
for filenode in TrashedFileNode.find(
Q('provider', 'eq', 'osfstorage') & Q('node', 'eq', project)
& Q('is_file', 'eq', True)):
for idx, version in enumerate(filenode.versions):
page = ':'.join(
['download', project._id, filenode._id,
str(idx)])
unique, total = get_basic_counters(page)
number_downloads_total += total or 0
number_downloads_unique += unique or 0
return number_downloads_unique, number_downloads_total
def get_archived_from_url(node, file_node):
if file_node.copied_from:
trashed = TrashedFileNode.load(file_node.copied_from._id)
if not trashed:
return node.registered_from.web_url_for(
'addon_view_or_download_file',
provider=file_node.provider,
path=file_node.copied_from._id)
return None
def addon_deleted_file(auth, node, **kwargs):
"""Shows a nice error message to users when they try to view
a deleted file
"""
# Allow file_node to be passed in so other views can delegate to this one
trashed = kwargs.get('file_node') or TrashedFileNode.load(kwargs.get('trashed_id'))
if not trashed:
raise HTTPError(httplib.NOT_FOUND, {
'message_short': 'Not Found',
'message_long': 'This file does not exist'
})
ret = serialize_node(node, auth, primary=True)
ret.update(rubeus.collect_addon_assets(node))
error_template = FILE_SUSPENDED_ERROR_MESSAGE if getattr(trashed, 'suspended', False) else FILE_GONE_ERROR_MESSAGE
error = error_template.format(file_name=trashed.name)
ret.update({
'urls': {
'render': None,
'sharejs': None,
'mfr': settings.MFR_SERVER_URL,
'gravatar': get_gravatar(auth.user, 25),
'files': node.web_url_for('collect_file_trees'),
},
'extra': {},
'size': 9966699, # Prevent file from being editted, just in case
'sharejs_uuid': None,
'file_name': trashed.name,
'file_path': trashed.path,
'provider': trashed.provider,
'materialized_path': trashed.materialized_path,
'error': error,
'private': getattr(node.get_addon(trashed.provider), 'is_private', False),
'file_id': trashed._id,
# For the off chance that there is no GUID
'file_guid': getattr(trashed.get_guid(create=False), '_id', None),
'file_tags': [tag._id for tag in trashed.tags],
'file_name_ext': os.path.splitext(trashed.name)[1],
'file_name_title': os.path.splitext(trashed.name)[0],
'allow_comments': trashed.provider in settings.ADDONS_COMMENTABLE,
})
return ret, httplib.GONE
def migrate_file_representation(bad_file):
logger.info('Migrating file representation of File: {0}'.format(bad_file))
view_url = bad_file['viewUrl'].rstrip('/')
fid = view_url.split('/')[-1]
fnode = FileNode.load(fid)
if fnode is None:
fnode = TrashedFileNode.load(fid)
assert fnode is not None, 'Could not load FileNode or TrashedFileNode with id {}'.format(fid)
data = {
'data': {
'kind': 'file',
'name': bad_file['selectedFileName'],
'path': fnode.path,
'extra': {},
'sha256': fnode.versions[-1].metadata['sha256']
}
}
bad_file.update(data)
def get_usage(node):
vids = sum([
file_node.to_storage().get('versions', [])
for file_node in
OsfStorageFile.find(Q('node', 'eq', node))
], [])
t_vids = sum([
file_node.to_storage().get('versions', [])
for file_node in
TrashedFileNode.find(
Q('node', 'eq', node) &
Q('is_file', 'eq', True) &
Q('provider', 'eq', 'osfstorage')
)
], [])
usage = sum([v.size or 0 for v in FileVersion.find(Q('_id', 'in', vids))])
trashed_usage = sum([v.size or 0 for v in FileVersion.find(Q('_id', 'in', t_vids))])
return map(sum, zip(*([(usage, trashed_usage)] + [get_usage(child) for child in node.nodes_primary]))) # Adds tuples together, map(sum, zip((a, b), (c, d))) -> (a+c, b+d)
def update_file_guid_referent(self, node, event_type, payload, user=None):
if event_type == 'addon_file_moved' or event_type == 'addon_file_renamed':
source = payload['source']
destination = payload['destination']
source_node = Node.load(source['node']['_id'])
destination_node = node
file_guids = FileNode.resolve_class(
source['provider'], FileNode.ANY).get_file_guids(
materialized_path=source['materialized']
if source['provider'] != 'osfstorage' else source['path'],
provider=source['provider'],
node=source_node)
if event_type == 'addon_file_renamed' and source[
'provider'] in settings.ADDONS_BASED_ON_IDS:
return
if event_type == 'addon_file_moved' and (
source['provider'] == destination['provider']
and source['provider'] in settings.ADDONS_BASED_ON_IDS
) and source_node == destination_node:
return
for guid in file_guids:
obj = Guid.load(guid)
if source_node != destination_node and Comment.find(
Q('root_target', 'eq', guid)).count() != 0:
update_comment_node(guid, source_node, destination_node)
if source['provider'] != destination['provider'] or source[
'provider'] != 'osfstorage':
old_file = FileNode.load(obj.referent._id)
obj.referent = create_new_file(obj, source, destination,
destination_node)
obj.save()
if old_file and not TrashedFileNode.load(old_file._id):
old_file.delete()
def get_archived_from_url(node, file_node):
if file_node.copied_from:
trashed = TrashedFileNode.load(file_node.copied_from._id)
if not trashed:
return node.registered_from.web_url_for('addon_view_or_download_file', provider=file_node.provider, path=file_node.copied_from._id)
return None
def addon_deleted_file(auth, node, error_type='BLAME_PROVIDER', **kwargs):
"""Shows a nice error message to users when they try to view a deleted file
"""
# Allow file_node to be passed in so other views can delegate to this one
file_node = kwargs.get('file_node') or TrashedFileNode.load(kwargs.get('trashed_id'))
deleted_by, deleted_on = None, None
if isinstance(file_node, TrashedFileNode):
deleted_by = file_node.deleted_by
deleted_by_guid = file_node.deleted_by._id if deleted_by else None
deleted_on = file_node.deleted_on.strftime('%c') + ' UTC'
if file_node.suspended:
error_type = 'FILE_SUSPENDED'
elif file_node.deleted_by is None:
if file_node.provider == 'osfstorage':
error_type = 'FILE_GONE_ACTOR_UNKNOWN'
else:
error_type = 'BLAME_PROVIDER'
else:
error_type = 'FILE_GONE'
else:
error_type = 'DONT_KNOW'
file_path = kwargs.get('path', file_node.path)
file_name = file_node.name or os.path.basename(file_path)
file_name_title, file_name_ext = os.path.splitext(file_name)
provider_full = settings.ADDONS_AVAILABLE_DICT[file_node.provider].full_name
try:
file_guid = file_node.get_guid()._id
except AttributeError:
file_guid = None
format_params = dict(
file_name=markupsafe.escape(file_name),
deleted_by=markupsafe.escape(deleted_by),
deleted_on=markupsafe.escape(deleted_on),
provider=markupsafe.escape(provider_full)
)
if deleted_by:
format_params['deleted_by_guid'] = markupsafe.escape(deleted_by_guid)
ret = serialize_node(node, auth, primary=True)
ret.update(rubeus.collect_addon_assets(node))
ret.update({
'error': ERROR_MESSAGES[error_type].format(**format_params),
'urls': {
'render': None,
'sharejs': None,
'mfr': settings.MFR_SERVER_URL,
'gravatar': get_gravatar(auth.user, 25),
'files': node.web_url_for('collect_file_trees'),
},
'extra': {},
'size': 9966699, # Prevent file from being edited, just in case
'sharejs_uuid': None,
'file_name': file_name,
'file_path': file_path,
'file_name_title': file_name_title,
'file_name_ext': file_name_ext,
'version_id': None,
'file_guid': file_guid,
'file_id': file_node._id,
'provider': file_node.provider,
'materialized_path': file_node.materialized_path or file_path,
'private': getattr(node.get_addon(file_node.provider), 'is_private', False),
'file_tags': [tag._id for tag in file_node.tags],
'allow_comments': file_node.provider in settings.ADDONS_COMMENTABLE,
})
return ret, httplib.GONE
def addon_view_or_download_file(auth, path, provider, **kwargs):
extras = request.args.to_dict()
extras.pop('_', None) # Clean up our url params a bit
action = extras.get('action', 'view')
node = kwargs.get('node') or kwargs['project']
node_addon = node.get_addon(provider)
if not path:
raise HTTPError(httplib.BAD_REQUEST)
if not isinstance(node_addon, StorageAddonBase):
raise HTTPError(
httplib.BAD_REQUEST, {
'message_short':
'Bad Request',
'message_long':
'The add-on containing this file is no longer connected to the {}.'
.format(node.project_or_component)
})
if not node_addon.has_auth:
raise HTTPError(
httplib.UNAUTHORIZED, {
'message_short':
'Unauthorized',
'message_long':
'The add-on containing this file is no longer authorized.'
})
if not node_addon.complete:
raise HTTPError(
httplib.BAD_REQUEST, {
'message_short':
'Bad Request',
'message_long':
'The add-on containing this file is no longer configured.'
})
file_node = FileNode.resolve_class(provider, FileNode.FILE).get_or_create(
node, path)
# Note: Cookie is provided for authentication to waterbutler
# it is overriden to force authentication as the current user
# the auth header is also pass to support basic auth
version = file_node.touch(
request.headers.get('Authorization'),
**dict(extras, cookie=request.cookies.get(settings.COOKIE_NAME)))
if version is None:
if file_node.get_guid():
# If this file has been successfully view before but no longer exists
# Move file to trashed file node
if not TrashedFileNode.load(file_node._id):
file_node.delete()
# Show a nice error message
return addon_deleted_file(file_node=file_node, **kwargs)
raise HTTPError(
httplib.NOT_FOUND, {
'message_short': 'Not Found',
'message_long': 'This file does not exist'
})
# TODO clean up these urls and unify what is used as a version identifier
if request.method == 'HEAD':
return make_response(('', 200, {
'Location':
file_node.generate_waterbutler_url(
**dict(extras, direct=None, version=version.identifier))
}))
if action == 'download':
return redirect(
file_node.generate_waterbutler_url(
**dict(extras, direct=None, version=version.identifier)))
if len(request.path.strip('/').split('/')) > 1:
guid = file_node.get_guid(create=True)
return redirect(
furl.furl('/{}/'.format(guid._id)).set(args=extras).url)
return addon_view_file(auth, node, file_node, version)
def addon_deleted_file(auth, node, error_type='BLAME_PROVIDER', **kwargs):
"""Shows a nice error message to users when they try to view a deleted file
"""
# Allow file_node to be passed in so other views can delegate to this one
file_node = kwargs.get('file_node') or TrashedFileNode.load(
kwargs.get('trashed_id'))
deleted_by, deleted_on = None, None
if isinstance(file_node, TrashedFileNode):
deleted_by = file_node.deleted_by
deleted_by_guid = file_node.deleted_by._id if deleted_by else None
deleted_on = file_node.deleted_on.strftime("%c") + ' UTC'
if file_node.suspended:
error_type = 'FILE_SUSPENDED'
elif file_node.deleted_by is None:
if file_node.provider == 'osfstorage':
error_type = 'FILE_GONE_ACTOR_UNKNOWN'
else:
error_type = 'BLAME_PROVIDER'
else:
error_type = 'FILE_GONE'
else:
error_type = 'DONT_KNOW'
file_path = kwargs.get('path', file_node.path)
file_name = file_node.name or os.path.basename(file_path)
file_name_title, file_name_ext = os.path.splitext(file_name)
provider_full = settings.ADDONS_AVAILABLE_DICT[
file_node.provider].full_name
try:
file_guid = file_node.get_guid()._id
except AttributeError:
file_guid = None
format_params = dict(file_name=markupsafe.escape(file_name),
deleted_by=markupsafe.escape(deleted_by),
deleted_on=markupsafe.escape(deleted_on),
provider=markupsafe.escape(provider_full))
if deleted_by:
format_params['deleted_by_guid'] = markupsafe.escape(deleted_by_guid)
ret = serialize_node(node, auth, primary=True)
ret.update(rubeus.collect_addon_assets(node))
ret.update({
'error':
ERROR_MESSAGES[error_type].format(**format_params),
'urls': {
'render': None,
'sharejs': None,
'mfr': settings.MFR_SERVER_URL,
'gravatar': get_gravatar(auth.user, 25),
'files': node.web_url_for('collect_file_trees'),
},
'extra': {},
'size':
9966699, # Prevent file from being edited, just in case
'sharejs_uuid':
None,
'file_name':
file_name,
'file_path':
file_path,
'file_name_title':
file_name_title,
'file_name_ext':
file_name_ext,
'file_guid':
file_guid,
'file_id':
file_node._id,
'provider':
file_node.provider,
'materialized_path':
file_node.materialized_path or file_path,
'private':
getattr(node.get_addon(file_node.provider), 'is_private', False),
'file_tags': [tag._id for tag in file_node.tags],
'allow_comments':
file_node.provider in settings.ADDONS_COMMENTABLE,
})
return ret, httplib.GONE
def addon_view_or_download_file(auth, path, provider, **kwargs):
extras = request.args.to_dict()
extras.pop('_', None) # Clean up our url params a bit
action = extras.get('action', 'view')
node = kwargs.get('node') or kwargs['project']
node_addon = node.get_addon(provider)
if not path:
raise HTTPError(httplib.BAD_REQUEST)
if not isinstance(node_addon, StorageAddonBase):
raise HTTPError(httplib.BAD_REQUEST, {
'message_short': 'Bad Request',
'message_long': 'The add-on containing this file is no longer connected to the {}.'.format(node.project_or_component)
})
if not node_addon.has_auth:
raise HTTPError(httplib.UNAUTHORIZED, {
'message_short': 'Unauthorized',
'message_long': 'The add-on containing this file is no longer authorized.'
})
if not node_addon.complete:
raise HTTPError(httplib.BAD_REQUEST, {
'message_short': 'Bad Request',
'message_long': 'The add-on containing this file is no longer configured.'
})
file_node = FileNode.resolve_class(provider, FileNode.FILE).get_or_create(node, path)
# Note: Cookie is provided for authentication to waterbutler
# it is overriden to force authentication as the current user
# the auth header is also pass to support basic auth
version = file_node.touch(
request.headers.get('Authorization'),
**dict(
extras,
cookie=request.cookies.get(settings.COOKIE_NAME)
)
)
if version is None:
if file_node.get_guid():
# If this file has been successfully view before but no longer exists
# Move file to trashed file node
if not TrashedFileNode.load(file_node._id):
file_node.delete()
# Show a nice error message
return addon_deleted_file(file_node=file_node, **kwargs)
raise HTTPError(httplib.NOT_FOUND, {
'message_short': 'Not Found',
'message_long': 'This file does not exist'
})
# TODO clean up these urls and unify what is used as a version identifier
if request.method == 'HEAD':
return make_response(('', 200, {
'Location': file_node.generate_waterbutler_url(**dict(extras, direct=None, version=version.identifier))
}))
if action == 'download':
return redirect(file_node.generate_waterbutler_url(**dict(extras, direct=None, version=version.identifier)))
if len(request.path.strip('/').split('/')) > 1:
guid = file_node.get_guid(create=True)
return redirect(furl.furl('/{}/'.format(guid._id)).set(args=extras).url)
return addon_view_file(auth, node, file_node, version)
def get_usage(node):
usage = (
sum([v.size or 0 for file_node in OsfStorageFile.find(Q('node', 'eq', node)) for v in file_node.versions]), # Sum all versions of all files of this node
sum([v.size or 0 for file_node in TrashedFileNode.find(Q('node', 'eq', node) & Q('is_file', 'eq', True) & Q('provider', 'eq', 'osfstorage')) for v in file_node.versions]), # Sum all versions of all deleted files of this node
)
return map(sum, zip(*([usage] + [get_usage(child) for child in node.nodes_primary]))) # Adds tuples together, map(sum, zip((a, b), (c, d))) -> (a+c, b+d)
