def update_file_guid_referent(self, node, event_type, payload, user=None):
if event_type not in ('addon_file_moved', 'addon_file_renamed'):
return # Nothing to do
source, destination = payload['source'], payload['destination']
source_node, destination_node = Node.load(source['node']['_id']), Node.load(destination['node']['_id'])
if source['provider'] in settings.ADDONS_BASED_ON_IDS:
if event_type == 'addon_file_renamed':
return # Node has not changed and provider has not changed
# Must be a move
if source['provider'] == destination['provider'] and source_node == destination_node:
return # Node has not changed and provider has not changed
file_guids = BaseFileNode.resolve_class(source['provider'], BaseFileNode.ANY).get_file_guids(
materialized_path=source['materialized'] if source['provider'] != 'osfstorage' else source['path'],
provider=source['provider'],
node=source_node
)
for guid in file_guids:
obj = Guid.load(guid)
if source_node != destination_node and Comment.find(Q('root_target._id', 'eq', guid)).count() != 0:
update_comment_node(guid, source_node, destination_node)
if source['provider'] != destination['provider'] or source['provider'] != 'osfstorage':
old_file = BaseFileNode.load(obj.referent._id)
obj.referent = create_new_file(obj, source, destination, destination_node)
obj.save()
if old_file and not TrashedFileNode.load(old_file._id):
old_file.delete()
def create_new_file(obj, source, destination, destination_node):
# TODO: Remove when materialized paths are fixed in the payload returned from waterbutler
if not source['materialized'].startswith('/'):
source['materialized'] = '/' + source['materialized']
if not destination['materialized'].startswith('/'):
destination['materialized'] = '/' + destination['materialized']
if not source['path'].endswith('/'):
data = dict(destination)
new_file = BaseFileNode.resolve_class(destination['provider'],
BaseFileNode.FILE).get_or_create(
destination_node,
destination['path'])
if destination['provider'] != 'osfstorage':
new_file.update(revision=None, data=data)
else:
new_file = find_and_create_file_from_metadata(
destination.get('children', []), source, destination,
destination_node, obj)
if not new_file:
if source['provider'] == 'box':
new_path = obj.referent.path
else:
new_path = obj.referent.materialized_path.replace(
source['materialized'], destination['materialized'])
new_file = BaseFileNode.resolve_class(
destination['provider'],
BaseFileNode.FILE).get_or_create(destination_node, new_path)
new_file.name = new_path.split('/')[-1]
new_file.materialized_path = new_path
new_file.save()
return new_file
def addon_delete_file_node(self, node, user, event_type, payload):
""" Get addon BaseFileNode(s), move it into the TrashedFileNode collection
and remove it from StoredFileNode.
Required so that the guids of deleted addon files are not re-pointed when an
addon file or folder is moved or renamed.
"""
if event_type == 'file_removed' and payload.get('provider', None) != 'osfstorage':
provider = payload['provider']
path = payload['metadata']['path']
materialized_path = payload['metadata']['materialized']
if path.endswith('/'):
folder_children = BaseFileNode.resolve_class(provider, BaseFileNode.ANY).objects.filter(
provider=provider,
node=node,
_materialized_path__startswith=materialized_path
)
for item in folder_children:
if item.kind == 'file' and not TrashedFileNode.load(item._id):
item.delete(user=user)
elif item.kind == 'folder':
BaseFileNode.remove_one(item)
else:
try:
file_node = BaseFileNode.resolve_class(provider, BaseFileNode.FILE).objects.get(
node=node,
_materialized_path=materialized_path
)
except BaseFileNode.DoesNotExist:
file_node = None
if file_node and not TrashedFileNode.load(file_node._id):
file_node.delete(user=user)
def addon_delete_file_node(self, node, user, event_type, payload):
""" Get addon BaseFileNode(s), move it into the TrashedFileNode collection
and remove it from StoredFileNode.
Required so that the guids of deleted addon files are not re-pointed when an
addon file or folder is moved or renamed.
"""
if event_type == 'file_removed' and payload.get('provider',
None) != 'osfstorage':
provider = payload['provider']
path = payload['metadata']['path']
materialized_path = payload['metadata']['materialized']
if path.endswith('/'):
folder_children = BaseFileNode.resolve_class(
provider, BaseFileNode.ANY).objects.filter(
provider=provider,
node=node,
_materialized_path__startswith=materialized_path)
for item in folder_children:
if item.kind == 'file' and not TrashedFileNode.load(item._id):
item.delete(user=user)
elif item.kind == 'folder':
BaseFileNode.delete(item)
else:
try:
file_node = BaseFileNode.resolve_class(
provider, BaseFileNode.FILE).objects.get(
node=node, _materialized_path=materialized_path)
except BaseFileNode.DoesNotExist:
file_node = None
if file_node and not TrashedFileNode.load(file_node._id):
file_node.delete(user=user)
def update_file_guid_referent(self, node, event_type, payload, user=None):
if event_type not in ('addon_file_moved', 'addon_file_renamed'):
return # Nothing to do
source, destination = payload['source'], payload['destination']
source_node, destination_node = Node.load(source['node']['_id']), Node.load(destination['node']['_id'])
if source['provider'] in settings.ADDONS_BASED_ON_IDS:
if event_type == 'addon_file_renamed':
return # Node has not changed and provider has not changed
# Must be a move
if source['provider'] == destination['provider'] and source_node == destination_node:
return # Node has not changed and provider has not changed
file_guids = BaseFileNode.resolve_class(source['provider'], BaseFileNode.ANY).get_file_guids(
materialized_path=source['materialized'] if source['provider'] != 'osfstorage' else source['path'],
provider=source['provider'],
node=source_node
)
for guid in file_guids:
obj = Guid.load(guid)
if source_node != destination_node and Comment.objects.filter(root_target___id=guid).count() != 0:
update_comment_node(guid, source_node, destination_node)
if source['provider'] != destination['provider'] or source['provider'] != 'osfstorage':
old_file = BaseFileNode.load(obj.referent._id)
obj.referent = create_new_file(obj, source, destination, destination_node)
obj.save()
if old_file and not TrashedFileNode.load(old_file._id):
old_file.delete()
def get_metadata_files(draft):
data = draft.registration_metadata
for q, question in get_file_questions('prereg-prize.json'):
if not isinstance(data[q]['value'], dict):
for i, file_info in enumerate(data[q]['extra']):
provider = file_info['data']['provider']
if provider != 'osfstorage':
raise Http404(
'File does not exist in OSFStorage ({}: {})'.format(
q, question
))
file_guid = file_info.get('fileId')
if not file_guid:
node = Node.load(file_info.get('nodeId'))
path = file_info['data'].get('path')
item = BaseFileNode.resolve_class(
provider,
BaseFileNode.FILE
).get_or_create(node, path)
file_guid = item.get_guid(create=True)._id
data[q]['extra'][i]['fileId'] = file_guid
draft.update_metadata(data)
draft.save()
else:
item = BaseFileNode.load(file_info['data']['path'].replace('/', ''))
if item is None:
raise Http404(
'File with guid "{}" in "{}" does not exist'.format(
file_guid, question
))
yield item
continue
for i, file_info in enumerate(data[q]['value']['uploader']['extra']):
provider = file_info['data']['provider']
if provider != 'osfstorage':
raise Http404(
'File does not exist in OSFStorage ({}: {})'.format(
q, question
))
file_guid = file_info.get('fileId')
if not file_guid:
node = Node.load(file_info.get('nodeId'))
path = file_info['data'].get('path')
item = BaseFileNode.resolve_class(
provider,
BaseFileNode.FILE
).get_or_create(node, path)
file_guid = item.get_guid(create=True)._id
data[q]['value']['uploader']['extra'][i]['fileId'] = file_guid
draft.update_metadata(data)
draft.save()
else:
item = BaseFileNode.load(file_info['data']['path'].replace('/', ''))
if item is None:
raise Http404(
'File with guid "{}" in "{}" does not exist'.format(
file_guid, question
))
yield item
def get_metadata_files(draft):
data = draft.registration_metadata
for q, question in get_file_questions('prereg-prize.json'):
if not isinstance(data[q]['value'], dict):
for i, file_info in enumerate(data[q]['extra']):
provider = file_info['data']['provider']
if provider != 'osfstorage':
raise Http404(
'File does not exist in OSFStorage ({}: {})'.format(
q, question
))
file_guid = file_info.get('fileId')
if not file_guid:
node = Node.load(file_info.get('nodeId'))
path = file_info['data'].get('path')
item = BaseFileNode.resolve_class(
provider,
BaseFileNode.FILE
).get_or_create(node, path)
file_guid = item.get_guid(create=True)._id
data[q]['extra'][i]['fileId'] = file_guid
draft.update_metadata(data)
draft.save()
else:
item = BaseFileNode.load(file_info['data']['path'].replace('/', ''))
if item is None:
raise Http404(
'File with guid "{}" in "{}" does not exist'.format(
file_guid, question
))
yield item
continue
for i, file_info in enumerate(data[q]['value']['uploader']['extra']):
provider = file_info['data']['provider']
if provider != 'osfstorage':
raise Http404(
'File does not exist in OSFStorage ({}: {})'.format(
q, question
))
file_guid = file_info.get('fileId')
if not file_guid:
node = Node.load(file_info.get('nodeId'))
path = file_info['data'].get('path')
item = BaseFileNode.resolve_class(
provider,
BaseFileNode.FILE
).get_or_create(node, path)
file_guid = item.get_guid(create=True)._id
data[q]['value']['uploader']['extra'][i]['fileId'] = file_guid
draft.update_metadata(data)
draft.save()
else:
item = BaseFileNode.load(file_info['data']['path'].replace('/', ''))
if item is None:
raise Http404(
'File with guid "{}" in "{}" does not exist'.format(
file_guid, question
))
yield item
def test_comments_move_when_file_moved_to_osfstorage(self, project, user):
osfstorage = project.get_addon('osfstorage')
root_node = osfstorage.get_root()
osf_file = root_node.append_file('file.txt')
osf_file.create_version(user, {
'object': '06d80e',
'service': 'cloud',
osfstorage_settings.WATERBUTLER_RESOURCE: 'osf',
}, {
'size': 1337,
'contentType': 'img/png',
'etag': 'abcdefghijklmnop'
}).save()
source = {
'path': '/file.txt',
'node': project,
'provider': self.provider
}
destination = {
'path': osf_file.path,
'node': project,
'provider': 'osfstorage'
}
self._create_file_with_comment(node=source['node'], path=source['path'], user=user)
payload = self._create_payload('move', user, source, destination, self.file._id, destination_file_id=destination['path'].strip('/'))
update_file_guid_referent(self=None, target=destination['node'], event_type='addon_file_moved', payload=payload)
self.guid.reload()
file_node = BaseFileNode.resolve_class('osfstorage', BaseFileNode.FILE).get_or_create(destination['node'], destination['path'])
assert self.guid._id == file_node.get_guid()._id
file_comments = Comment.objects.filter(root_target=self.guid.pk)
assert file_comments.count() == 1
def test_comments_move_when_folder_moved_to_different_provider(self, destination_provider, destination_path, project, user):
if self.provider == destination_provider:
return True
project.add_addon(destination_provider, auth=Auth(user))
project.save()
self.addon_settings = project.get_addon(destination_provider)
self.addon_settings.folder = '/AddonFolder'
self.addon_settings.save()
source = {
'path': '/',
'node': project,
'provider': self.provider
}
destination = {
'path': '/subfolder/',
'node': project,
'provider': destination_provider,
'children': [{
'path': '/subfolder/file.txt',
'node': project,
'provider': destination_provider
}]
}
file_name = 'file.txt'
self._create_file_with_comment(node=source['node'], path='{}{}'.format(source['path'], file_name), user=user)
payload = self._create_payload('move', user, source, destination, self.file._id)
update_file_guid_referent(self=None, target=destination['node'], event_type='addon_file_moved', payload=payload)
self.guid.reload()
file_node = BaseFileNode.resolve_class(destination_provider, BaseFileNode.FILE).get_or_create(destination['node'], destination_path)
assert self.guid._id == file_node.get_guid()._id
file_comments = Comment.objects.filter(root_target=self.guid.pk)
assert file_comments.count() == 1
def find_and_create_file_from_metadata(children, source, destination,
destination_node, obj):
""" Given a Guid obj, recursively search for the metadata of its referent (a file obj)
in the waterbutler response. If found, create a new addon BaseFileNode with that metadata
and return the new file.
"""
for item in children:
# TODO: Remove when materialized paths are fixed in the payload returned from waterbutler
if not item['materialized'].startswith('/'):
item['materialized'] = '/' + item['materialized']
if item['kind'] == 'folder':
return find_and_create_file_from_metadata(item.get('children', []),
source, destination,
destination_node, obj)
elif item['kind'] == 'file' and item['materialized'].replace(
destination['materialized'],
source['materialized']) == obj.referent.materialized_path:
data = dict(item)
new_file = BaseFileNode.resolve_class(
destination['provider'],
BaseFileNode.FILE).get_or_create(destination_node,
item['path'])
if destination['provider'] != 'osfstorage':
new_file.update(revision=None, data=data)
return new_file
def test_comments_move_when_folder_moved_to_different_provider(self, destination_provider, destination_path, project, user):
if self.provider == destination_provider:
return True
project.add_addon(destination_provider, auth=Auth(user))
project.save()
self.addon_settings = project.get_addon(destination_provider)
self.addon_settings.folder = '/AddonFolder'
self.addon_settings.save()
source = {
'path': '/',
'node': project,
'provider': self.provider
}
destination = {
'path': '/subfolder/',
'node': project,
'provider': destination_provider,
'children': [{
'path': '/subfolder/file.txt',
'node': project,
'provider': destination_provider
}]
}
file_name = 'file.txt'
self._create_file_with_comment(node=source['node'], path='{}{}'.format(source['path'], file_name), user=user)
payload = self._create_payload('move', user, source, destination, self.file._id)
update_file_guid_referent(self=None, node=destination['node'], event_type='addon_file_moved', payload=payload)
self.guid.reload()
file_node = BaseFileNode.resolve_class(destination_provider, BaseFileNode.FILE).get_or_create(destination['node'], destination_path)
assert self.guid._id == file_node.get_guid()._id
file_comments = Comment.objects.filter(root_target=self.guid.pk)
assert file_comments.count() == 1
def test_comments_move_when_file_moved_to_osfstorage(self, project, user):
osfstorage = project.get_addon('osfstorage')
root_node = osfstorage.get_root()
osf_file = root_node.append_file('file.txt')
osf_file.create_version(user, {
'object': '06d80e',
'service': 'cloud',
osfstorage_settings.WATERBUTLER_RESOURCE: 'osf',
}, {
'size': 1337,
'contentType': 'img/png',
'etag': 'abcdefghijklmnop'
}).save()
source = {
'path': '/file.txt',
'node': project,
'provider': self.provider
}
destination = {
'path': osf_file.path,
'node': project,
'provider': 'osfstorage'
}
self._create_file_with_comment(node=source['node'], path=source['path'], user=user)
payload = self._create_payload('move', user, source, destination, self.file._id, destination_file_id=destination['path'].strip('/'))
update_file_guid_referent(self=None, node=destination['node'], event_type='addon_file_moved', payload=payload)
self.guid.reload()
file_node = BaseFileNode.resolve_class('osfstorage', BaseFileNode.FILE).get_or_create(destination['node'], destination['path'])
assert self.guid._id == file_node.get_guid()._id
file_comments = Comment.objects.filter(root_target=self.guid.pk)
assert file_comments.count() == 1
def upload_file_add_timestamptoken(payload, node):
from osf.models import Guid
import requests
from api.timestamp.add_timestamp import AddTimestamp
import shutil
verify_result = 0
tmp_dir = None
try:
metadata = payload['metadata']
file_node = BaseFileNode.resolve_class(
metadata['provider'],
BaseFileNode.FILE).get_or_create(node, metadata['path'])
file_node.save()
auth_id = payload['auth']['id']
guid = Guid.objects.get(_id=auth_id)
user_info = OSFUser.objects.get(id=guid.object_id)
cookie = user_info.get_or_create_cookie()
cookies = {settings.COOKIE_NAME: cookie}
headers = {'content-type': 'application/json'}
res_content = None
if metadata['provider'] == 'osfstorage':
res = requests.get(file_node.generate_waterbutler_url(
**dict(action='download',
version=metadata['extra']['version'],
direct=None,
_internal=False)),
headers=headers,
cookies=cookies)
else:
res = requests.get(file_node.generate_waterbutler_url(
**dict(action='download', mode=None, _internal=False)),
headers=headers,
cookies=cookies)
res_content = res.content
res.close()
current_datetime = timezone.now()
current_datetime_str = current_datetime.strftime('%Y%m%d%H%M%S%f')
tmp_dir = '/tmp/tmp_{}_{}_{}'.format(auth_id, file_node._id,
current_datetime_str)
os.mkdir(tmp_dir)
download_file_path = os.path.join(tmp_dir, metadata['name'])
with open(download_file_path, 'wb') as fout:
fout.write(res_content)
addTimestamp = AddTimestamp()
verify_result, verify_result_title, operator_user, operator_date, filepath = addTimestamp.add_timestamp(
auth_id, file_node._id, node._id, metadata['provider'],
metadata['materialized'], download_file_path, tmp_dir)
shutil.rmtree(tmp_dir)
except Exception as err:
if tmp_dir:
if os.path.exists(tmp_dir):
shutil.rmtree(tmp_dir)
logger.exception(err)
return verify_result
def waterbutler_folder_file_info(pid, provider, path, node, cookies, headers):
# get waterbutler folder file
if provider == 'osfstorage':
waterbutler_meta_url = util.waterbutler_api_url_for(
pid, provider,
'/' + path,
**dict(waterbutler_meta_parameter())
)
else:
waterbutler_meta_url = util.waterbutler_api_url_for(
pid, provider,
path,
**dict(waterbutler_meta_parameter())
)
waterbutler_res = requests.get(waterbutler_meta_url, headers=headers, cookies=cookies)
waterbutler_json_res = waterbutler_res.json()
waterbutler_res.close()
file_list = []
child_file_list = []
for file_data in waterbutler_json_res['data']:
if file_data['attributes']['kind'] == 'folder':
child_file_list.extend(waterbutler_folder_file_info(
pid, provider, file_data['attributes']['path'],
node, cookies, headers))
else:
basefile_node = BaseFileNode.resolve_class(
provider,
BaseFileNode.FILE
).get_or_create(
node,
file_data['attributes']['path']
)
basefile_node.save()
if provider == 'osfstorage':
file_info = {
'file_name': file_data['attributes']['name'],
'file_path': file_data['attributes']['materialized'],
'file_kind': file_data['attributes']['kind'],
'file_id': basefile_node._id,
'version': file_data['attributes']['extra']['version']
}
else:
file_info = {
'file_name': file_data['attributes']['name'],
'file_path': file_data['attributes']['materialized'],
'file_kind': file_data['attributes']['kind'],
'file_id': basefile_node._id,
'version': ''
}
file_list.append(file_info)
file_list.extend(child_file_list)
return file_list
def get_file_node_from_wb_resp(self, item):
"""Takes file data from wb response, touches/updates metadata for it, and returns file object"""
attrs = item['attributes']
file_node = BaseFileNode.resolve_class(
attrs['provider'], BaseFileNode.FOLDER
if attrs['kind'] == 'folder' else BaseFileNode.FILE).get_or_create(
self.get_node(check_object_permissions=False), attrs['path'])
file_node.update(None, attrs, user=self.request.user)
return file_node
def get_file_node_from_wb_resp(self, item):
"""Takes file data from wb response, touches/updates metadata for it, and returns file object"""
attrs = item['attributes']
file_node = BaseFileNode.resolve_class(
attrs['provider'],
BaseFileNode.FOLDER if attrs['kind'] == 'folder'
else BaseFileNode.FILE,
).get_or_create(self.get_node(check_object_permissions=False), attrs['path'])
file_node.update(None, attrs, user=self.request.user)
return file_node
def create_new_file(obj, source, destination, destination_node):
# TODO: Remove when materialized paths are fixed in the payload returned from waterbutler
if not source['materialized'].startswith('/'):
source['materialized'] = '/' + source['materialized']
if not destination['materialized'].startswith('/'):
destination['materialized'] = '/' + destination['materialized']
if not source['path'].endswith('/'):
data = dict(destination)
new_file = BaseFileNode.resolve_class(destination['provider'], BaseFileNode.FILE).get_or_create(destination_node, destination['path'])
if destination['provider'] != 'osfstorage':
new_file.update(revision=None, data=data)
else:
new_file = find_and_create_file_from_metadata(destination.get('children', []), source, destination, destination_node, obj)
if not new_file:
if source['provider'] == 'box':
new_path = obj.referent.path
else:
new_path = obj.referent.materialized_path.replace(source['materialized'], destination['materialized'])
new_file = BaseFileNode.resolve_class(destination['provider'], BaseFileNode.FILE).get_or_create(destination_node, new_path)
new_file.name = new_path.split('/')[-1]
new_file.materialized_path = new_path
new_file.save()
return new_file
def find_and_create_file_from_metadata(children, source, destination, destination_node, obj):
""" Given a Guid obj, recursively search for the metadata of its referent (a file obj)
in the waterbutler response. If found, create a new addon BaseFileNode with that metadata
and return the new file.
"""
for item in children:
# TODO: Remove when materialized paths are fixed in the payload returned from waterbutler
if not item['materialized'].startswith('/'):
item['materialized'] = '/' + item['materialized']
if item['kind'] == 'folder':
return find_and_create_file_from_metadata(item.get('children', []), source, destination, destination_node, obj)
elif item['kind'] == 'file' and item['materialized'].replace(destination['materialized'], source['materialized']) == obj.referent.materialized_path:
data = dict(item)
new_file = BaseFileNode.resolve_class(destination['provider'], BaseFileNode.FILE).get_or_create(destination_node, item['path'])
if destination['provider'] != 'osfstorage':
new_file.update(revision=None, data=data)
return new_file
def bulk_get_file_nodes_from_wb_resp(self, files_list):
"""Takes a list of file data from wb response, touches/updates metadata for each, and returns list of file objects.
This function mirrors all the actions of get_file_node_from_wb_resp except the create and updates are done in bulk.
The bulk_update and bulk_create do not call the base class update and create so the actions of those functions are
done here where needed
"""
node = self.get_node(check_object_permissions=False)
content_type = ContentType.objects.get_for_model(node)
objs_to_create = defaultdict(lambda: [])
file_objs = []
for item in files_list:
attrs = item['attributes']
base_class = BaseFileNode.resolve_class(
attrs['provider'],
BaseFileNode.FOLDER
if attrs['kind'] == 'folder' else BaseFileNode.FILE,
)
# mirrors BaseFileNode get_or_create
try:
file_obj = base_class.objects.get(
target_object_id=node.id,
target_content_type=content_type,
_path='/' + attrs['path'].lstrip('/'))
except base_class.DoesNotExist:
# create method on BaseFileNode appends provider, bulk_create bypasses this step so it is added here
file_obj = base_class(target=node,
_path='/' + attrs['path'].lstrip('/'),
provider=base_class._provider)
objs_to_create[base_class].append(file_obj)
else:
file_objs.append(file_obj)
file_obj.update(None, attrs, user=self.request.user, save=False)
bulk_update(file_objs)
for base_class in objs_to_create:
base_class.objects.bulk_create(objs_to_create[base_class])
file_objs += objs_to_create[base_class]
return file_objs
def test_comments_move_on_file_rename(self, project, user):
source = {
'path': '/file.txt',
'node': project,
'provider': self.provider
}
destination = {
'path': '/file_renamed.txt',
'node': project,
'provider': self.provider
}
self._create_file_with_comment(node=source['node'], path=source['path'], user=user)
payload = self._create_payload('move', user, source, destination, self.file._id)
update_file_guid_referent(self=None, node=destination['node'], event_type='addon_file_renamed', payload=payload)
self.guid.reload()
file_node = BaseFileNode.resolve_class(self.provider, BaseFileNode.FILE).get_or_create(destination['node'], self._format_path(destination['path'], file_id=self.file._id))
assert self.guid._id == file_node.get_guid()._id
file_comments = Comment.objects.filter(root_target=self.guid.pk)
assert file_comments.count() == 1
def test_comments_move_on_file_rename(self, project, user):
source = {
'path': '/file.txt',
'node': project,
'provider': self.provider
}
destination = {
'path': '/file_renamed.txt',
'node': project,
'provider': self.provider
}
self._create_file_with_comment(node=source['node'], path=source['path'], user=user)
payload = self._create_payload('move', user, source, destination, self.file._id)
update_file_guid_referent(self=None, target=destination['node'], event_type='addon_file_renamed', payload=payload)
self.guid.reload()
file_node = BaseFileNode.resolve_class(self.provider, BaseFileNode.FILE).get_or_create(destination['node'], self._format_path(destination['path'], file_id=self.file._id))
assert self.guid._id == file_node.get_guid()._id
file_comments = Comment.objects.filter(root_target=self.guid.pk)
assert file_comments.count() == 1
def test_comments_move_on_subfolder_file_when_parent_folder_is_renamed(self, project, user):
source = {
'path': '/subfolder1/',
'node': project,
'provider': self.provider
}
destination = {
'path': '/subfolder2/',
'node': project,
'provider': self.provider
}
file_path = 'sub-subfolder/file.txt'
self._create_file_with_comment(node=source['node'], path='{}{}'.format(source['path'], file_path), user=user)
payload = self._create_payload('move', user, source, destination, self.file._id)
update_file_guid_referent(self=None, node=destination['node'], event_type='addon_file_renamed', payload=payload)
self.guid.reload()
file_node = BaseFileNode.resolve_class(self.provider, BaseFileNode.FILE).get_or_create(destination['node'], self._format_path('{}{}'.format(destination['path'], file_path), file_id=self.file._id))
assert self.guid._id == file_node.get_guid()._id
file_comments = Comment.find(Q('root_target', 'eq', self.guid.pk))
assert file_comments.count() == 1
def test_comments_move_when_folder_moved_from_component_to_project(self, project, component, user):
source = {
'path': '/subfolder/',
'node': component,
'provider': self.provider
}
destination = {
'path': '/subfolder/',
'node': project,
'provider': self.provider
}
file_name = 'file.txt'
self._create_file_with_comment(node=source['node'], path='{}{}'.format(source['path'], file_name), user=user)
payload = self._create_payload('move', user, source, destination, self.file._id)
update_file_guid_referent(self=None, node=destination['node'], event_type='addon_file_moved', payload=payload)
self.guid.reload()
file_node = BaseFileNode.resolve_class(self.provider, BaseFileNode.FILE).get_or_create(destination['node'], self._format_path('{}{}'.format(destination['path'], file_name), file_id=self.file._id))
assert self.guid._id == file_node.get_guid()._id
file_comments = Comment.objects.filter(root_target=self.guid.pk)
assert file_comments.count() == 1
def test_comments_move_when_folder_moved_from_component_to_project(self, project, component, user):
source = {
'path': '/subfolder/',
'node': component,
'provider': self.provider
}
destination = {
'path': '/subfolder/',
'node': project,
'provider': self.provider
}
file_name = 'file.txt'
self._create_file_with_comment(node=source['node'], path='{}{}'.format(source['path'], file_name), user=user)
payload = self._create_payload('move', user, source, destination, self.file._id)
update_file_guid_referent(self=None, target=destination['node'], event_type='addon_file_moved', payload=payload)
self.guid.reload()
file_node = BaseFileNode.resolve_class(self.provider, BaseFileNode.FILE).get_or_create(destination['node'], self._format_path('{}{}'.format(destination['path'], file_name), file_id=self.file._id))
assert self.guid._id == file_node.get_guid()._id
file_comments = Comment.objects.filter(root_target=self.guid.pk)
assert file_comments.count() == 1
def test_comments_move_when_folder_moved_to_osfstorage(self, project, user):
osfstorage = project.get_addon('osfstorage')
root_node = osfstorage.get_root()
osf_folder = root_node.append_folder('subfolder')
osf_file = osf_folder.append_file('file.txt')
osf_file.create_version(user, {
'object': '06d80e',
'service': 'cloud',
osfstorage_settings.WATERBUTLER_RESOURCE: 'osf',
}, {
'size': 1337,
'contentType': 'img/png',
'etag': '1234567890abcde'
}).save()
source = {
'path': '/subfolder/',
'node': project,
'provider': self.provider
}
destination = {
'path': '/subfolder/',
'node': project,
'provider': 'osfstorage',
'children': [{
'path': '/subfolder/file.txt',
'node': project,
'provider': 'osfstorage'
}]
}
file_name = 'file.txt'
self._create_file_with_comment(node=source['node'], path='{}{}'.format(source['path'], file_name), user=user)
payload = self._create_payload('move', user, source, destination, self.file._id, destination_file_id=osf_file._id)
update_file_guid_referent(self=None, node=destination['node'], event_type='addon_file_moved', payload=payload)
self.guid.reload()
file_node = BaseFileNode.resolve_class('osfstorage', BaseFileNode.FILE).get_or_create(destination['node'], osf_file._id)
assert self.guid._id == file_node.get_guid()._id
file_comments = Comment.find(Q('root_target', 'eq', self.guid.pk))
assert file_comments.count() == 1
def bulk_get_file_nodes_from_wb_resp(self, files_list):
"""Takes a list of file data from wb response, touches/updates metadata for each, and returns list of file objects.
This function mirrors all the actions of get_file_node_from_wb_resp except the create and updates are done in bulk.
The bulk_update and bulk_create do not call the base class update and create so the actions of those functions are
done here where needed
"""
node = self.get_node(check_object_permissions=False)
content_type = ContentType.objects.get_for_model(node)
objs_to_create = defaultdict(lambda: [])
file_objs = []
for item in files_list:
attrs = item['attributes']
base_class = BaseFileNode.resolve_class(
attrs['provider'],
BaseFileNode.FOLDER if attrs['kind'] == 'folder'
else BaseFileNode.FILE,
)
# mirrors BaseFileNode get_or_create
try:
file_obj = base_class.objects.get(target_object_id=node.id, target_content_type=content_type, _path='/' + attrs['path'].lstrip('/'))
except base_class.DoesNotExist:
# create method on BaseFileNode appends provider, bulk_create bypasses this step so it is added here
file_obj = base_class(target=node, _path='/' + attrs['path'].lstrip('/'), provider=base_class._provider)
objs_to_create[base_class].append(file_obj)
else:
file_objs.append(file_obj)
file_obj.update(None, attrs, user=self.request.user, save=False)
bulk_update(file_objs)
for base_class in objs_to_create:
base_class.objects.bulk_create(objs_to_create[base_class])
file_objs += objs_to_create[base_class]
return file_objs
def test_comments_move_when_file_moved_from_component_to_project(self, project, component, user):
source = {
'path': '/file.txt',
'node': component,
'provider': self.provider
}
destination = {
'path': '/file.txt',
'node': project,
'provider': self.provider
}
self._create_file_with_comment(node=source['node'], path=source['path'], user=user)
self.file.move_under(destination['node'].get_addon(self.provider).get_root())
payload = self._create_payload('move', user, source, destination, self.file._id)
update_file_guid_referent(self=None, node=destination['node'], event_type='addon_file_moved', payload=payload)
self.guid.reload()
file_node = BaseFileNode.resolve_class(self.provider, BaseFileNode.FILE).get_or_create(destination['node'], self._format_path(destination['path'], file_id=self.file._id))
assert self.guid._id == file_node.get_guid()._id
assert self.guid.referent.node._id == destination['node']._id
file_comments = Comment.find(Q('root_target', 'eq', self.guid.pk))
assert file_comments.count() == 1
def file_created_or_updated(node, payload, user_id, created_flag):
file_node = BaseFileNode.resolve_class(
payload['metadata']['provider'], BaseFileNode.FILE).get_or_create(
node, payload['metadata'].get('materialized'))
file_node.save()
created_at = payload['metadata'].get('created_utc')
modified_at = payload['metadata'].get('modified_utc')
version = ''
if not created_at:
created_at = None
if not modified_at:
modified_at = None
if payload['metadata']['provider'] == 'osf_storage':
version = payload['metadata']['extra'].get('version')
file_info = {
'file_id': file_node._id,
'file_name': payload['metadata'].get('name'),
'file_path': payload['metadata'].get('materialized'),
'size': payload['metadata'].get('size'),
'created': created_at,
'modified': modified_at,
'version': version,
'provider': payload['metadata'].get('provider')
}
add_token(user_id, node, file_info)
# Update created/modified user in timestamp result
verify_data = RdmFileTimestamptokenVerifyResult.objects.filter(
file_id=file_info['file_id']).first()
if verify_data:
if created_flag:
verify_data.upload_file_created_user = user_id
else: # Updated
verify_data.upload_file_modified_user = user_id
verify_data.upload_file_created_at = file_info['created']
verify_data.upload_file_modified_at = file_info['modified']
verify_data.upload_file_size = file_info['size']
verify_data.save()
def add_token(uid, node, data):
user = OSFUser.objects.get(id=uid)
cookie = user.get_or_create_cookie()
tmp_dir = None
try:
file_node = BaseFileNode.resolve_class(
data['provider'],
BaseFileNode.FILE).get_or_create(node, data['file_path'])
file_node.save()
data['file_id'] = file_node._id
# Request To Download File
tmp_dir = 'tmp_{}'.format(user._id)
count = 1
while os.path.exists(tmp_dir):
count += 1
tmp_dir = 'tmp_{}_{}'.format(user._id, count)
os.mkdir(tmp_dir)
download_file_path = waterbutler.download_file(cookie, file_node,
tmp_dir)
if not userkey_generation_check(user._id):
userkey_generation(user._id)
addTimestamp = AddTimestamp()
result = addTimestamp.add_timestamp(user._id, data, node._id,
download_file_path, tmp_dir)
shutil.rmtree(tmp_dir)
return result
except Exception as err:
if tmp_dir and os.path.exists(tmp_dir):
shutil.rmtree(tmp_dir)
logger.exception(err)
raise
def view_file(request, node_id, provider, file_id):
fp = BaseFileNode.load(file_id)
wb_url = fp.generate_waterbutler_url()
return redirect(wb_url)
def addon_view_or_download_file(auth, path, provider, **kwargs):
extras = request.args.to_dict()
extras.pop('_', None) # Clean up our url params a bit
action = extras.get('action', 'view')
guid = kwargs.get('guid')
guid_target = getattr(Guid.load(guid), 'referent', None)
target = guid_target or kwargs.get('node') or kwargs['project']
provider_safe = markupsafe.escape(provider)
path_safe = markupsafe.escape(path)
if not path:
raise HTTPError(http_status.HTTP_400_BAD_REQUEST)
if hasattr(target, 'get_addon'):
node_addon = target.get_addon(provider)
if not isinstance(node_addon, BaseStorageAddon):
object_text = markupsafe.escape(
getattr(target, 'project_or_component', 'this object'))
raise HTTPError(
http_status.HTTP_400_BAD_REQUEST,
data={
'message_short':
'Bad Request',
'message_long':
'The {} add-on containing {} is no longer connected to {}.'
.format(provider_safe, path_safe, object_text)
})
if not node_addon.has_auth:
raise HTTPError(
http_status.HTTP_401_UNAUTHORIZED,
data={
'message_short':
'Unauthorized',
'message_long':
'The {} add-on containing {} is no longer authorized.'.
format(provider_safe, path_safe)
})
if not node_addon.complete:
raise HTTPError(
http_status.HTTP_400_BAD_REQUEST,
data={
'message_short':
'Bad Request',
'message_long':
'The {} add-on containing {} is no longer configured.'.
format(provider_safe, path_safe)
})
savepoint_id = transaction.savepoint()
file_node = BaseFileNode.resolve_class(provider,
BaseFileNode.FILE).get_or_create(
target, path)
# Note: Cookie is provided for authentication to waterbutler
# it is overriden to force authentication as the current user
# the auth header is also pass to support basic auth
version = file_node.touch(
request.headers.get('Authorization'),
**dict(extras, cookie=request.cookies.get(settings.COOKIE_NAME)))
if version is None:
# File is either deleted or unable to be found in the provider location
# Rollback the insertion of the file_node
transaction.savepoint_rollback(savepoint_id)
if not file_node.pk:
file_node = BaseFileNode.load(path)
if not file_node:
raise HTTPError(http_status.HTTP_404_NOT_FOUND,
data={
'message_short':
'File Not Found',
'message_long':
'The requested file could not be found.'
})
if file_node.kind == 'folder':
raise HTTPError(
http_status.HTTP_400_BAD_REQUEST,
data={
'message_short':
'Bad Request',
'message_long':
'You cannot request a folder from this endpoint.'
})
# Allow osfstorage to redirect if the deep url can be used to find a valid file_node
if file_node.provider == 'osfstorage' and not file_node.is_deleted:
return redirect(
file_node.target.web_url_for('addon_view_or_download_file',
path=file_node._id,
provider=file_node.provider))
return addon_deleted_file(target=target,
file_node=file_node,
path=path,
**kwargs)
else:
transaction.savepoint_commit(savepoint_id)
# TODO clean up these urls and unify what is used as a version identifier
if request.method == 'HEAD':
return make_response(('', http_status.HTTP_302_FOUND, {
'Location':
file_node.generate_waterbutler_url(
**dict(extras,
direct=None,
version=version.identifier,
_internal=extras.get('mode') == 'render'))
}))
if action == 'download':
format = extras.get('format')
_, extension = os.path.splitext(file_node.name)
# avoid rendering files with the same format type.
if format and '.{}'.format(format.lower()) != extension.lower():
return redirect('{}/export?format={}&url={}'.format(
get_mfr_url(target, provider), format,
quote(
file_node.generate_waterbutler_url(
**dict(extras,
direct=None,
version=version.identifier,
_internal=extras.get('mode') == 'render')))))
return redirect(
file_node.generate_waterbutler_url(
**dict(extras,
direct=None,
version=version.identifier,
_internal=extras.get('mode') == 'render')))
if action == 'get_guid':
draft_id = extras.get('draft')
draft = DraftRegistration.load(draft_id)
if draft is None or draft.is_approved:
raise HTTPError(http_status.HTTP_400_BAD_REQUEST,
data={
'message_short':
'Bad Request',
'message_long':
'File not associated with required object.'
})
guid = file_node.get_guid(create=True)
guid.referent.save()
return dict(guid=guid._id)
if len(request.path.strip('/').split('/')) > 1:
guid = file_node.get_guid(create=True)
return redirect(
furl.furl('/{}/'.format(guid._id)).set(args=extras).url)
if isinstance(target, Preprint):
# Redirecting preprint file guids to the preprint detail page
return redirect('/{}/'.format(target._id))
return addon_view_file(auth, target, file_node, version)
def collect_timestamp_trees_to_json(auth, node, **kwargs):
# admin call project to provider file list
serialized = _view_project(node, auth, primary=True)
serialized.update(rubeus.collect_addon_assets(node))
user_info = OSFUser.objects.get(id=Guid.objects.get(_id=serialized['user']['id']).object_id)
api_url = util.api_v2_url(api_url_path(kwargs.get('pid')))
cookie = user_info.get_or_create_cookie()
cookies = {settings.COOKIE_NAME: cookie}
headers = {'content-type': 'application/json'}
provider_json_res = None
file_res = requests.get(api_url, headers=headers, cookies=cookies)
provider_json_res = file_res.json()
file_res.close()
provider_list = []
for provider_data in provider_json_res['data']:
waterbutler_meta_url = util.waterbutler_api_url_for(
kwargs.get('pid'),
provider_data['attributes']['provider'],
'/',
**dict(waterbutler_meta_parameter())
)
waterbutler_json_res = None
waterbutler_res = requests.get(waterbutler_meta_url, headers=headers, cookies=cookies)
waterbutler_json_res = waterbutler_res.json()
waterbutler_res.close()
file_list = []
child_file_list = []
for file_data in waterbutler_json_res['data']:
if file_data['attributes']['kind'] == 'folder':
child_file_list.extend(
waterbutler_folder_file_info(
kwargs.get('pid'),
provider_data['attributes']['provider'],
file_data['attributes']['path'],
node, cookies, headers
)
)
else:
file_info = None
basefile_node = BaseFileNode.resolve_class(
provider_data['attributes']['provider'],
BaseFileNode.FILE
).get_or_create(
node,
file_data['attributes']['path']
)
basefile_node.save()
if provider_data['attributes']['provider'] == 'osfstorage':
file_info = {
'file_name': file_data['attributes']['name'],
'file_path': file_data['attributes']['materialized'],
'file_kind': file_data['attributes']['kind'],
'file_id': basefile_node._id,
'version': file_data['attributes']['extra']['version']
}
else:
file_info = {
'file_name': file_data['attributes']['name'],
'file_path': file_data['attributes']['materialized'],
'file_kind': file_data['attributes']['kind'],
'file_id': basefile_node._id,
'version': ''
}
if file_info:
file_list.append(file_info)
file_list.extend(child_file_list)
if file_list:
provider_files = {
'provider': provider_data['attributes']['provider'],
'provider_file_list': file_list
}
provider_list.append(provider_files)
return {'provider_list': provider_list}
def get_full_list(uid, pid, node):
'''
Get a full list of timestamps from all files uploaded to a storage.
'''
user_info = OSFUser.objects.get(id=uid)
cookie = user_info.get_or_create_cookie()
api_url = util.api_v2_url('nodes/{}/files'.format(pid))
headers = {'content-type': 'application/json'}
cookies = {settings.COOKIE_NAME: cookie}
file_res = requests.get(api_url, headers=headers, cookies=cookies)
provider_json_res = file_res.json()
file_res.close()
provider_list = []
for provider_data in provider_json_res['data']:
waterbutler_meta_url = waterbutler_api_url_for(
pid,
provider_data['attributes']['provider'],
'/',
meta=int(time.mktime(datetime.datetime.now().timetuple())))
waterbutler_json_res = None
waterbutler_res = requests.get(waterbutler_meta_url,
headers=headers,
cookies=cookies)
waterbutler_json_res = waterbutler_res.json()
waterbutler_res.close()
file_list = []
child_file_list = []
for file_data in waterbutler_json_res['data']:
if file_data['attributes']['kind'] == 'folder':
child_file_list.extend(
waterbutler_folder_file_info(
pid, provider_data['attributes']['provider'],
file_data['attributes']['path'], node, cookies,
headers))
else:
file_info = None
basefile_node = BaseFileNode.resolve_class(
provider_data['attributes']['provider'],
BaseFileNode.FILE).get_or_create(
node, file_data['attributes']['path'])
basefile_node.save()
file_info = {
'file_id': basefile_node._id,
'file_name': file_data['attributes'].get('name'),
'file_path': file_data['attributes'].get('materialized'),
'size': file_data['attributes'].get('size'),
'created': file_data['attributes'].get('created_utc'),
'modified': file_data['attributes'].get('modified_utc'),
'file_version': ''
}
if provider_data['attributes']['provider'] == 'osfstorage':
file_info['file_version'] = file_data['attributes'][
'extra'].get('version')
if file_info:
file_list.append(file_info)
file_list.extend(child_file_list)
if file_list:
provider_files = {
'provider': provider_data['attributes']['provider'],
'provider_file_list': file_list
}
provider_list.append(provider_files)
return provider_list
def get_object(self, file_id):
return BaseFileNode.load(file_id)
def addon_view_or_download_file(auth, path, provider, **kwargs):
extras = request.args.to_dict()
extras.pop('_', None) # Clean up our url params a bit
action = extras.get('action', 'view')
guid = kwargs.get('guid')
guid_target = getattr(Guid.load(guid), 'referent', None)
target = guid_target or kwargs.get('node') or kwargs['project']
provider_safe = markupsafe.escape(provider)
path_safe = markupsafe.escape(path)
if not path:
raise HTTPError(httplib.BAD_REQUEST)
if hasattr(target, 'get_addon'):
node_addon = target.get_addon(provider)
if not isinstance(node_addon, BaseStorageAddon):
object_text = markupsafe.escape(getattr(target, 'project_or_component', 'this object'))
raise HTTPError(httplib.BAD_REQUEST, data={
'message_short': 'Bad Request',
'message_long': 'The {} add-on containing {} is no longer connected to {}.'.format(provider_safe, path_safe, object_text)
})
if not node_addon.has_auth:
raise HTTPError(httplib.UNAUTHORIZED, data={
'message_short': 'Unauthorized',
'message_long': 'The {} add-on containing {} is no longer authorized.'.format(provider_safe, path_safe)
})
if not node_addon.complete:
raise HTTPError(httplib.BAD_REQUEST, data={
'message_short': 'Bad Request',
'message_long': 'The {} add-on containing {} is no longer configured.'.format(provider_safe, path_safe)
})
savepoint_id = transaction.savepoint()
file_node = BaseFileNode.resolve_class(provider, BaseFileNode.FILE).get_or_create(target, path)
# Note: Cookie is provided for authentication to waterbutler
# it is overriden to force authentication as the current user
# the auth header is also pass to support basic auth
version = file_node.touch(
request.headers.get('Authorization'),
**dict(
extras,
cookie=request.cookies.get(settings.COOKIE_NAME)
)
)
if version is None:
# File is either deleted or unable to be found in the provider location
# Rollback the insertion of the file_node
transaction.savepoint_rollback(savepoint_id)
if not file_node.pk:
file_node = BaseFileNode.load(path)
if file_node.kind == 'folder':
raise HTTPError(httplib.BAD_REQUEST, data={
'message_short': 'Bad Request',
'message_long': 'You cannot request a folder from this endpoint.'
})
# Allow osfstorage to redirect if the deep url can be used to find a valid file_node
if file_node and file_node.provider == 'osfstorage' and not file_node.is_deleted:
return redirect(
file_node.target.web_url_for('addon_view_or_download_file', path=file_node._id, provider=file_node.provider)
)
return addon_deleted_file(target=target, file_node=file_node, path=path, **kwargs)
else:
transaction.savepoint_commit(savepoint_id)
# TODO clean up these urls and unify what is used as a version identifier
if request.method == 'HEAD':
return make_response(('', httplib.FOUND, {
'Location': file_node.generate_waterbutler_url(**dict(extras, direct=None, version=version.identifier, _internal=extras.get('mode') == 'render'))
}))
if action == 'download':
format = extras.get('format')
_, extension = os.path.splitext(file_node.name)
# avoid rendering files with the same format type.
if format and '.{}'.format(format.lower()) != extension.lower():
return redirect('{}/export?format={}&url={}'.format(get_mfr_url(target, provider), format, urllib.quote(file_node.generate_waterbutler_url(
**dict(extras, direct=None, version=version.identifier, _internal=extras.get('mode') == 'render')
))))
return redirect(file_node.generate_waterbutler_url(**dict(extras, direct=None, version=version.identifier, _internal=extras.get('mode') == 'render')))
if action == 'get_guid':
draft_id = extras.get('draft')
draft = DraftRegistration.load(draft_id)
if draft is None or draft.is_approved:
raise HTTPError(httplib.BAD_REQUEST, data={
'message_short': 'Bad Request',
'message_long': 'File not associated with required object.'
})
guid = file_node.get_guid(create=True)
guid.referent.save()
return dict(guid=guid._id)
if len(request.path.strip('/').split('/')) > 1:
guid = file_node.get_guid(create=True)
return redirect(furl.furl('/{}/'.format(guid._id)).set(args=extras).url)
if isinstance(target, Preprint):
# Redirecting preprint file guids to the preprint detail page
return redirect('/{}/'.format(target._id))
return addon_view_file(auth, target, file_node, version)
def get_object(self, file_id):
return BaseFileNode.load(file_id)
def view_file(request, node_id, provider, file_id):
fp = BaseFileNode.load(file_id)
wb_url = fp.generate_waterbutler_url()
return redirect(wb_url)
