def update_file_guid_referent(self, node, event_type, payload, user=None): if event_type not in ('addon_file_moved', 'addon_file_renamed'): return # Nothing to do source, destination = payload['source'], payload['destination'] source_node, destination_node = Node.load(source['node']['_id']), Node.load(destination['node']['_id']) if source['provider'] in settings.ADDONS_BASED_ON_IDS: if event_type == 'addon_file_renamed': return # Node has not changed and provider has not changed # Must be a move if source['provider'] == destination['provider'] and source_node == destination_node: return # Node has not changed and provider has not changed file_guids = BaseFileNode.resolve_class(source['provider'], BaseFileNode.ANY).get_file_guids( materialized_path=source['materialized'] if source['provider'] != 'osfstorage' else source['path'], provider=source['provider'], node=source_node ) for guid in file_guids: obj = Guid.load(guid) if source_node != destination_node and Comment.find(Q('root_target._id', 'eq', guid)).count() != 0: update_comment_node(guid, source_node, destination_node) if source['provider'] != destination['provider'] or source['provider'] != 'osfstorage': old_file = BaseFileNode.load(obj.referent._id) obj.referent = create_new_file(obj, source, destination, destination_node) obj.save() if old_file and not TrashedFileNode.load(old_file._id): old_file.delete()
def create_new_file(obj, source, destination, destination_node): # TODO: Remove when materialized paths are fixed in the payload returned from waterbutler if not source['materialized'].startswith('/'): source['materialized'] = '/' + source['materialized'] if not destination['materialized'].startswith('/'): destination['materialized'] = '/' + destination['materialized'] if not source['path'].endswith('/'): data = dict(destination) new_file = BaseFileNode.resolve_class(destination['provider'], BaseFileNode.FILE).get_or_create( destination_node, destination['path']) if destination['provider'] != 'osfstorage': new_file.update(revision=None, data=data) else: new_file = find_and_create_file_from_metadata( destination.get('children', []), source, destination, destination_node, obj) if not new_file: if source['provider'] == 'box': new_path = obj.referent.path else: new_path = obj.referent.materialized_path.replace( source['materialized'], destination['materialized']) new_file = BaseFileNode.resolve_class( destination['provider'], BaseFileNode.FILE).get_or_create(destination_node, new_path) new_file.name = new_path.split('/')[-1] new_file.materialized_path = new_path new_file.save() return new_file
def addon_delete_file_node(self, node, user, event_type, payload): """ Get addon BaseFileNode(s), move it into the TrashedFileNode collection and remove it from StoredFileNode. Required so that the guids of deleted addon files are not re-pointed when an addon file or folder is moved or renamed. """ if event_type == 'file_removed' and payload.get('provider', None) != 'osfstorage': provider = payload['provider'] path = payload['metadata']['path'] materialized_path = payload['metadata']['materialized'] if path.endswith('/'): folder_children = BaseFileNode.resolve_class(provider, BaseFileNode.ANY).objects.filter( provider=provider, node=node, _materialized_path__startswith=materialized_path ) for item in folder_children: if item.kind == 'file' and not TrashedFileNode.load(item._id): item.delete(user=user) elif item.kind == 'folder': BaseFileNode.remove_one(item) else: try: file_node = BaseFileNode.resolve_class(provider, BaseFileNode.FILE).objects.get( node=node, _materialized_path=materialized_path ) except BaseFileNode.DoesNotExist: file_node = None if file_node and not TrashedFileNode.load(file_node._id): file_node.delete(user=user)
def addon_delete_file_node(self, node, user, event_type, payload): """ Get addon BaseFileNode(s), move it into the TrashedFileNode collection and remove it from StoredFileNode. Required so that the guids of deleted addon files are not re-pointed when an addon file or folder is moved or renamed. """ if event_type == 'file_removed' and payload.get('provider', None) != 'osfstorage': provider = payload['provider'] path = payload['metadata']['path'] materialized_path = payload['metadata']['materialized'] if path.endswith('/'): folder_children = BaseFileNode.resolve_class( provider, BaseFileNode.ANY).objects.filter( provider=provider, node=node, _materialized_path__startswith=materialized_path) for item in folder_children: if item.kind == 'file' and not TrashedFileNode.load(item._id): item.delete(user=user) elif item.kind == 'folder': BaseFileNode.delete(item) else: try: file_node = BaseFileNode.resolve_class( provider, BaseFileNode.FILE).objects.get( node=node, _materialized_path=materialized_path) except BaseFileNode.DoesNotExist: file_node = None if file_node and not TrashedFileNode.load(file_node._id): file_node.delete(user=user)
def update_file_guid_referent(self, node, event_type, payload, user=None): if event_type not in ('addon_file_moved', 'addon_file_renamed'): return # Nothing to do source, destination = payload['source'], payload['destination'] source_node, destination_node = Node.load(source['node']['_id']), Node.load(destination['node']['_id']) if source['provider'] in settings.ADDONS_BASED_ON_IDS: if event_type == 'addon_file_renamed': return # Node has not changed and provider has not changed # Must be a move if source['provider'] == destination['provider'] and source_node == destination_node: return # Node has not changed and provider has not changed file_guids = BaseFileNode.resolve_class(source['provider'], BaseFileNode.ANY).get_file_guids( materialized_path=source['materialized'] if source['provider'] != 'osfstorage' else source['path'], provider=source['provider'], node=source_node ) for guid in file_guids: obj = Guid.load(guid) if source_node != destination_node and Comment.objects.filter(root_target___id=guid).count() != 0: update_comment_node(guid, source_node, destination_node) if source['provider'] != destination['provider'] or source['provider'] != 'osfstorage': old_file = BaseFileNode.load(obj.referent._id) obj.referent = create_new_file(obj, source, destination, destination_node) obj.save() if old_file and not TrashedFileNode.load(old_file._id): old_file.delete()
def get_metadata_files(draft): data = draft.registration_metadata for q, question in get_file_questions('prereg-prize.json'): if not isinstance(data[q]['value'], dict): for i, file_info in enumerate(data[q]['extra']): provider = file_info['data']['provider'] if provider != 'osfstorage': raise Http404( 'File does not exist in OSFStorage ({}: {})'.format( q, question )) file_guid = file_info.get('fileId') if not file_guid: node = Node.load(file_info.get('nodeId')) path = file_info['data'].get('path') item = BaseFileNode.resolve_class( provider, BaseFileNode.FILE ).get_or_create(node, path) file_guid = item.get_guid(create=True)._id data[q]['extra'][i]['fileId'] = file_guid draft.update_metadata(data) draft.save() else: item = BaseFileNode.load(file_info['data']['path'].replace('/', '')) if item is None: raise Http404( 'File with guid "{}" in "{}" does not exist'.format( file_guid, question )) yield item continue for i, file_info in enumerate(data[q]['value']['uploader']['extra']): provider = file_info['data']['provider'] if provider != 'osfstorage': raise Http404( 'File does not exist in OSFStorage ({}: {})'.format( q, question )) file_guid = file_info.get('fileId') if not file_guid: node = Node.load(file_info.get('nodeId')) path = file_info['data'].get('path') item = BaseFileNode.resolve_class( provider, BaseFileNode.FILE ).get_or_create(node, path) file_guid = item.get_guid(create=True)._id data[q]['value']['uploader']['extra'][i]['fileId'] = file_guid draft.update_metadata(data) draft.save() else: item = BaseFileNode.load(file_info['data']['path'].replace('/', '')) if item is None: raise Http404( 'File with guid "{}" in "{}" does not exist'.format( file_guid, question )) yield item
def get_metadata_files(draft): data = draft.registration_metadata for q, question in get_file_questions('prereg-prize.json'): if not isinstance(data[q]['value'], dict): for i, file_info in enumerate(data[q]['extra']): provider = file_info['data']['provider'] if provider != 'osfstorage': raise Http404( 'File does not exist in OSFStorage ({}: {})'.format( q, question )) file_guid = file_info.get('fileId') if not file_guid: node = Node.load(file_info.get('nodeId')) path = file_info['data'].get('path') item = BaseFileNode.resolve_class( provider, BaseFileNode.FILE ).get_or_create(node, path) file_guid = item.get_guid(create=True)._id data[q]['extra'][i]['fileId'] = file_guid draft.update_metadata(data) draft.save() else: item = BaseFileNode.load(file_info['data']['path'].replace('/', '')) if item is None: raise Http404( 'File with guid "{}" in "{}" does not exist'.format( file_guid, question )) yield item continue for i, file_info in enumerate(data[q]['value']['uploader']['extra']): provider = file_info['data']['provider'] if provider != 'osfstorage': raise Http404( 'File does not exist in OSFStorage ({}: {})'.format( q, question )) file_guid = file_info.get('fileId') if not file_guid: node = Node.load(file_info.get('nodeId')) path = file_info['data'].get('path') item = BaseFileNode.resolve_class( provider, BaseFileNode.FILE ).get_or_create(node, path) file_guid = item.get_guid(create=True)._id data[q]['value']['uploader']['extra'][i]['fileId'] = file_guid draft.update_metadata(data) draft.save() else: item = BaseFileNode.load(file_info['data']['path'].replace('/', '')) if item is None: raise Http404( 'File with guid "{}" in "{}" does not exist'.format( file_guid, question )) yield item
def test_comments_move_when_file_moved_to_osfstorage(self, project, user): osfstorage = project.get_addon('osfstorage') root_node = osfstorage.get_root() osf_file = root_node.append_file('file.txt') osf_file.create_version(user, { 'object': '06d80e', 'service': 'cloud', osfstorage_settings.WATERBUTLER_RESOURCE: 'osf', }, { 'size': 1337, 'contentType': 'img/png', 'etag': 'abcdefghijklmnop' }).save() source = { 'path': '/file.txt', 'node': project, 'provider': self.provider } destination = { 'path': osf_file.path, 'node': project, 'provider': 'osfstorage' } self._create_file_with_comment(node=source['node'], path=source['path'], user=user) payload = self._create_payload('move', user, source, destination, self.file._id, destination_file_id=destination['path'].strip('/')) update_file_guid_referent(self=None, target=destination['node'], event_type='addon_file_moved', payload=payload) self.guid.reload() file_node = BaseFileNode.resolve_class('osfstorage', BaseFileNode.FILE).get_or_create(destination['node'], destination['path']) assert self.guid._id == file_node.get_guid()._id file_comments = Comment.objects.filter(root_target=self.guid.pk) assert file_comments.count() == 1
def test_comments_move_when_folder_moved_to_different_provider(self, destination_provider, destination_path, project, user): if self.provider == destination_provider: return True project.add_addon(destination_provider, auth=Auth(user)) project.save() self.addon_settings = project.get_addon(destination_provider) self.addon_settings.folder = '/AddonFolder' self.addon_settings.save() source = { 'path': '/', 'node': project, 'provider': self.provider } destination = { 'path': '/subfolder/', 'node': project, 'provider': destination_provider, 'children': [{ 'path': '/subfolder/file.txt', 'node': project, 'provider': destination_provider }] } file_name = 'file.txt' self._create_file_with_comment(node=source['node'], path='{}{}'.format(source['path'], file_name), user=user) payload = self._create_payload('move', user, source, destination, self.file._id) update_file_guid_referent(self=None, target=destination['node'], event_type='addon_file_moved', payload=payload) self.guid.reload() file_node = BaseFileNode.resolve_class(destination_provider, BaseFileNode.FILE).get_or_create(destination['node'], destination_path) assert self.guid._id == file_node.get_guid()._id file_comments = Comment.objects.filter(root_target=self.guid.pk) assert file_comments.count() == 1
def find_and_create_file_from_metadata(children, source, destination, destination_node, obj): """ Given a Guid obj, recursively search for the metadata of its referent (a file obj) in the waterbutler response. If found, create a new addon BaseFileNode with that metadata and return the new file. """ for item in children: # TODO: Remove when materialized paths are fixed in the payload returned from waterbutler if not item['materialized'].startswith('/'): item['materialized'] = '/' + item['materialized'] if item['kind'] == 'folder': return find_and_create_file_from_metadata(item.get('children', []), source, destination, destination_node, obj) elif item['kind'] == 'file' and item['materialized'].replace( destination['materialized'], source['materialized']) == obj.referent.materialized_path: data = dict(item) new_file = BaseFileNode.resolve_class( destination['provider'], BaseFileNode.FILE).get_or_create(destination_node, item['path']) if destination['provider'] != 'osfstorage': new_file.update(revision=None, data=data) return new_file
def test_comments_move_when_folder_moved_to_different_provider(self, destination_provider, destination_path, project, user): if self.provider == destination_provider: return True project.add_addon(destination_provider, auth=Auth(user)) project.save() self.addon_settings = project.get_addon(destination_provider) self.addon_settings.folder = '/AddonFolder' self.addon_settings.save() source = { 'path': '/', 'node': project, 'provider': self.provider } destination = { 'path': '/subfolder/', 'node': project, 'provider': destination_provider, 'children': [{ 'path': '/subfolder/file.txt', 'node': project, 'provider': destination_provider }] } file_name = 'file.txt' self._create_file_with_comment(node=source['node'], path='{}{}'.format(source['path'], file_name), user=user) payload = self._create_payload('move', user, source, destination, self.file._id) update_file_guid_referent(self=None, node=destination['node'], event_type='addon_file_moved', payload=payload) self.guid.reload() file_node = BaseFileNode.resolve_class(destination_provider, BaseFileNode.FILE).get_or_create(destination['node'], destination_path) assert self.guid._id == file_node.get_guid()._id file_comments = Comment.objects.filter(root_target=self.guid.pk) assert file_comments.count() == 1
def test_comments_move_when_file_moved_to_osfstorage(self, project, user): osfstorage = project.get_addon('osfstorage') root_node = osfstorage.get_root() osf_file = root_node.append_file('file.txt') osf_file.create_version(user, { 'object': '06d80e', 'service': 'cloud', osfstorage_settings.WATERBUTLER_RESOURCE: 'osf', }, { 'size': 1337, 'contentType': 'img/png', 'etag': 'abcdefghijklmnop' }).save() source = { 'path': '/file.txt', 'node': project, 'provider': self.provider } destination = { 'path': osf_file.path, 'node': project, 'provider': 'osfstorage' } self._create_file_with_comment(node=source['node'], path=source['path'], user=user) payload = self._create_payload('move', user, source, destination, self.file._id, destination_file_id=destination['path'].strip('/')) update_file_guid_referent(self=None, node=destination['node'], event_type='addon_file_moved', payload=payload) self.guid.reload() file_node = BaseFileNode.resolve_class('osfstorage', BaseFileNode.FILE).get_or_create(destination['node'], destination['path']) assert self.guid._id == file_node.get_guid()._id file_comments = Comment.objects.filter(root_target=self.guid.pk) assert file_comments.count() == 1
def upload_file_add_timestamptoken(payload, node): from osf.models import Guid import requests from api.timestamp.add_timestamp import AddTimestamp import shutil verify_result = 0 tmp_dir = None try: metadata = payload['metadata'] file_node = BaseFileNode.resolve_class( metadata['provider'], BaseFileNode.FILE).get_or_create(node, metadata['path']) file_node.save() auth_id = payload['auth']['id'] guid = Guid.objects.get(_id=auth_id) user_info = OSFUser.objects.get(id=guid.object_id) cookie = user_info.get_or_create_cookie() cookies = {settings.COOKIE_NAME: cookie} headers = {'content-type': 'application/json'} res_content = None if metadata['provider'] == 'osfstorage': res = requests.get(file_node.generate_waterbutler_url( **dict(action='download', version=metadata['extra']['version'], direct=None, _internal=False)), headers=headers, cookies=cookies) else: res = requests.get(file_node.generate_waterbutler_url( **dict(action='download', mode=None, _internal=False)), headers=headers, cookies=cookies) res_content = res.content res.close() current_datetime = timezone.now() current_datetime_str = current_datetime.strftime('%Y%m%d%H%M%S%f') tmp_dir = '/tmp/tmp_{}_{}_{}'.format(auth_id, file_node._id, current_datetime_str) os.mkdir(tmp_dir) download_file_path = os.path.join(tmp_dir, metadata['name']) with open(download_file_path, 'wb') as fout: fout.write(res_content) addTimestamp = AddTimestamp() verify_result, verify_result_title, operator_user, operator_date, filepath = addTimestamp.add_timestamp( auth_id, file_node._id, node._id, metadata['provider'], metadata['materialized'], download_file_path, tmp_dir) shutil.rmtree(tmp_dir) except Exception as err: if tmp_dir: if os.path.exists(tmp_dir): shutil.rmtree(tmp_dir) logger.exception(err) return verify_result
def waterbutler_folder_file_info(pid, provider, path, node, cookies, headers): # get waterbutler folder file if provider == 'osfstorage': waterbutler_meta_url = util.waterbutler_api_url_for( pid, provider, '/' + path, **dict(waterbutler_meta_parameter()) ) else: waterbutler_meta_url = util.waterbutler_api_url_for( pid, provider, path, **dict(waterbutler_meta_parameter()) ) waterbutler_res = requests.get(waterbutler_meta_url, headers=headers, cookies=cookies) waterbutler_json_res = waterbutler_res.json() waterbutler_res.close() file_list = [] child_file_list = [] for file_data in waterbutler_json_res['data']: if file_data['attributes']['kind'] == 'folder': child_file_list.extend(waterbutler_folder_file_info( pid, provider, file_data['attributes']['path'], node, cookies, headers)) else: basefile_node = BaseFileNode.resolve_class( provider, BaseFileNode.FILE ).get_or_create( node, file_data['attributes']['path'] ) basefile_node.save() if provider == 'osfstorage': file_info = { 'file_name': file_data['attributes']['name'], 'file_path': file_data['attributes']['materialized'], 'file_kind': file_data['attributes']['kind'], 'file_id': basefile_node._id, 'version': file_data['attributes']['extra']['version'] } else: file_info = { 'file_name': file_data['attributes']['name'], 'file_path': file_data['attributes']['materialized'], 'file_kind': file_data['attributes']['kind'], 'file_id': basefile_node._id, 'version': '' } file_list.append(file_info) file_list.extend(child_file_list) return file_list
def get_file_node_from_wb_resp(self, item): """Takes file data from wb response, touches/updates metadata for it, and returns file object""" attrs = item['attributes'] file_node = BaseFileNode.resolve_class( attrs['provider'], BaseFileNode.FOLDER if attrs['kind'] == 'folder' else BaseFileNode.FILE).get_or_create( self.get_node(check_object_permissions=False), attrs['path']) file_node.update(None, attrs, user=self.request.user) return file_node
def get_file_node_from_wb_resp(self, item): """Takes file data from wb response, touches/updates metadata for it, and returns file object""" attrs = item['attributes'] file_node = BaseFileNode.resolve_class( attrs['provider'], BaseFileNode.FOLDER if attrs['kind'] == 'folder' else BaseFileNode.FILE, ).get_or_create(self.get_node(check_object_permissions=False), attrs['path']) file_node.update(None, attrs, user=self.request.user) return file_node
def create_new_file(obj, source, destination, destination_node): # TODO: Remove when materialized paths are fixed in the payload returned from waterbutler if not source['materialized'].startswith('/'): source['materialized'] = '/' + source['materialized'] if not destination['materialized'].startswith('/'): destination['materialized'] = '/' + destination['materialized'] if not source['path'].endswith('/'): data = dict(destination) new_file = BaseFileNode.resolve_class(destination['provider'], BaseFileNode.FILE).get_or_create(destination_node, destination['path']) if destination['provider'] != 'osfstorage': new_file.update(revision=None, data=data) else: new_file = find_and_create_file_from_metadata(destination.get('children', []), source, destination, destination_node, obj) if not new_file: if source['provider'] == 'box': new_path = obj.referent.path else: new_path = obj.referent.materialized_path.replace(source['materialized'], destination['materialized']) new_file = BaseFileNode.resolve_class(destination['provider'], BaseFileNode.FILE).get_or_create(destination_node, new_path) new_file.name = new_path.split('/')[-1] new_file.materialized_path = new_path new_file.save() return new_file
def find_and_create_file_from_metadata(children, source, destination, destination_node, obj): """ Given a Guid obj, recursively search for the metadata of its referent (a file obj) in the waterbutler response. If found, create a new addon BaseFileNode with that metadata and return the new file. """ for item in children: # TODO: Remove when materialized paths are fixed in the payload returned from waterbutler if not item['materialized'].startswith('/'): item['materialized'] = '/' + item['materialized'] if item['kind'] == 'folder': return find_and_create_file_from_metadata(item.get('children', []), source, destination, destination_node, obj) elif item['kind'] == 'file' and item['materialized'].replace(destination['materialized'], source['materialized']) == obj.referent.materialized_path: data = dict(item) new_file = BaseFileNode.resolve_class(destination['provider'], BaseFileNode.FILE).get_or_create(destination_node, item['path']) if destination['provider'] != 'osfstorage': new_file.update(revision=None, data=data) return new_file
def bulk_get_file_nodes_from_wb_resp(self, files_list): """Takes a list of file data from wb response, touches/updates metadata for each, and returns list of file objects. This function mirrors all the actions of get_file_node_from_wb_resp except the create and updates are done in bulk. The bulk_update and bulk_create do not call the base class update and create so the actions of those functions are done here where needed """ node = self.get_node(check_object_permissions=False) content_type = ContentType.objects.get_for_model(node) objs_to_create = defaultdict(lambda: []) file_objs = [] for item in files_list: attrs = item['attributes'] base_class = BaseFileNode.resolve_class( attrs['provider'], BaseFileNode.FOLDER if attrs['kind'] == 'folder' else BaseFileNode.FILE, ) # mirrors BaseFileNode get_or_create try: file_obj = base_class.objects.get( target_object_id=node.id, target_content_type=content_type, _path='/' + attrs['path'].lstrip('/')) except base_class.DoesNotExist: # create method on BaseFileNode appends provider, bulk_create bypasses this step so it is added here file_obj = base_class(target=node, _path='/' + attrs['path'].lstrip('/'), provider=base_class._provider) objs_to_create[base_class].append(file_obj) else: file_objs.append(file_obj) file_obj.update(None, attrs, user=self.request.user, save=False) bulk_update(file_objs) for base_class in objs_to_create: base_class.objects.bulk_create(objs_to_create[base_class]) file_objs += objs_to_create[base_class] return file_objs
def test_comments_move_on_file_rename(self, project, user): source = { 'path': '/file.txt', 'node': project, 'provider': self.provider } destination = { 'path': '/file_renamed.txt', 'node': project, 'provider': self.provider } self._create_file_with_comment(node=source['node'], path=source['path'], user=user) payload = self._create_payload('move', user, source, destination, self.file._id) update_file_guid_referent(self=None, node=destination['node'], event_type='addon_file_renamed', payload=payload) self.guid.reload() file_node = BaseFileNode.resolve_class(self.provider, BaseFileNode.FILE).get_or_create(destination['node'], self._format_path(destination['path'], file_id=self.file._id)) assert self.guid._id == file_node.get_guid()._id file_comments = Comment.objects.filter(root_target=self.guid.pk) assert file_comments.count() == 1
def test_comments_move_on_file_rename(self, project, user): source = { 'path': '/file.txt', 'node': project, 'provider': self.provider } destination = { 'path': '/file_renamed.txt', 'node': project, 'provider': self.provider } self._create_file_with_comment(node=source['node'], path=source['path'], user=user) payload = self._create_payload('move', user, source, destination, self.file._id) update_file_guid_referent(self=None, target=destination['node'], event_type='addon_file_renamed', payload=payload) self.guid.reload() file_node = BaseFileNode.resolve_class(self.provider, BaseFileNode.FILE).get_or_create(destination['node'], self._format_path(destination['path'], file_id=self.file._id)) assert self.guid._id == file_node.get_guid()._id file_comments = Comment.objects.filter(root_target=self.guid.pk) assert file_comments.count() == 1
def test_comments_move_on_subfolder_file_when_parent_folder_is_renamed(self, project, user): source = { 'path': '/subfolder1/', 'node': project, 'provider': self.provider } destination = { 'path': '/subfolder2/', 'node': project, 'provider': self.provider } file_path = 'sub-subfolder/file.txt' self._create_file_with_comment(node=source['node'], path='{}{}'.format(source['path'], file_path), user=user) payload = self._create_payload('move', user, source, destination, self.file._id) update_file_guid_referent(self=None, node=destination['node'], event_type='addon_file_renamed', payload=payload) self.guid.reload() file_node = BaseFileNode.resolve_class(self.provider, BaseFileNode.FILE).get_or_create(destination['node'], self._format_path('{}{}'.format(destination['path'], file_path), file_id=self.file._id)) assert self.guid._id == file_node.get_guid()._id file_comments = Comment.find(Q('root_target', 'eq', self.guid.pk)) assert file_comments.count() == 1
def test_comments_move_when_folder_moved_from_component_to_project(self, project, component, user): source = { 'path': '/subfolder/', 'node': component, 'provider': self.provider } destination = { 'path': '/subfolder/', 'node': project, 'provider': self.provider } file_name = 'file.txt' self._create_file_with_comment(node=source['node'], path='{}{}'.format(source['path'], file_name), user=user) payload = self._create_payload('move', user, source, destination, self.file._id) update_file_guid_referent(self=None, node=destination['node'], event_type='addon_file_moved', payload=payload) self.guid.reload() file_node = BaseFileNode.resolve_class(self.provider, BaseFileNode.FILE).get_or_create(destination['node'], self._format_path('{}{}'.format(destination['path'], file_name), file_id=self.file._id)) assert self.guid._id == file_node.get_guid()._id file_comments = Comment.objects.filter(root_target=self.guid.pk) assert file_comments.count() == 1
def test_comments_move_when_folder_moved_from_component_to_project(self, project, component, user): source = { 'path': '/subfolder/', 'node': component, 'provider': self.provider } destination = { 'path': '/subfolder/', 'node': project, 'provider': self.provider } file_name = 'file.txt' self._create_file_with_comment(node=source['node'], path='{}{}'.format(source['path'], file_name), user=user) payload = self._create_payload('move', user, source, destination, self.file._id) update_file_guid_referent(self=None, target=destination['node'], event_type='addon_file_moved', payload=payload) self.guid.reload() file_node = BaseFileNode.resolve_class(self.provider, BaseFileNode.FILE).get_or_create(destination['node'], self._format_path('{}{}'.format(destination['path'], file_name), file_id=self.file._id)) assert self.guid._id == file_node.get_guid()._id file_comments = Comment.objects.filter(root_target=self.guid.pk) assert file_comments.count() == 1
def test_comments_move_when_folder_moved_to_osfstorage(self, project, user): osfstorage = project.get_addon('osfstorage') root_node = osfstorage.get_root() osf_folder = root_node.append_folder('subfolder') osf_file = osf_folder.append_file('file.txt') osf_file.create_version(user, { 'object': '06d80e', 'service': 'cloud', osfstorage_settings.WATERBUTLER_RESOURCE: 'osf', }, { 'size': 1337, 'contentType': 'img/png', 'etag': '1234567890abcde' }).save() source = { 'path': '/subfolder/', 'node': project, 'provider': self.provider } destination = { 'path': '/subfolder/', 'node': project, 'provider': 'osfstorage', 'children': [{ 'path': '/subfolder/file.txt', 'node': project, 'provider': 'osfstorage' }] } file_name = 'file.txt' self._create_file_with_comment(node=source['node'], path='{}{}'.format(source['path'], file_name), user=user) payload = self._create_payload('move', user, source, destination, self.file._id, destination_file_id=osf_file._id) update_file_guid_referent(self=None, node=destination['node'], event_type='addon_file_moved', payload=payload) self.guid.reload() file_node = BaseFileNode.resolve_class('osfstorage', BaseFileNode.FILE).get_or_create(destination['node'], osf_file._id) assert self.guid._id == file_node.get_guid()._id file_comments = Comment.find(Q('root_target', 'eq', self.guid.pk)) assert file_comments.count() == 1
def bulk_get_file_nodes_from_wb_resp(self, files_list): """Takes a list of file data from wb response, touches/updates metadata for each, and returns list of file objects. This function mirrors all the actions of get_file_node_from_wb_resp except the create and updates are done in bulk. The bulk_update and bulk_create do not call the base class update and create so the actions of those functions are done here where needed """ node = self.get_node(check_object_permissions=False) content_type = ContentType.objects.get_for_model(node) objs_to_create = defaultdict(lambda: []) file_objs = [] for item in files_list: attrs = item['attributes'] base_class = BaseFileNode.resolve_class( attrs['provider'], BaseFileNode.FOLDER if attrs['kind'] == 'folder' else BaseFileNode.FILE, ) # mirrors BaseFileNode get_or_create try: file_obj = base_class.objects.get(target_object_id=node.id, target_content_type=content_type, _path='/' + attrs['path'].lstrip('/')) except base_class.DoesNotExist: # create method on BaseFileNode appends provider, bulk_create bypasses this step so it is added here file_obj = base_class(target=node, _path='/' + attrs['path'].lstrip('/'), provider=base_class._provider) objs_to_create[base_class].append(file_obj) else: file_objs.append(file_obj) file_obj.update(None, attrs, user=self.request.user, save=False) bulk_update(file_objs) for base_class in objs_to_create: base_class.objects.bulk_create(objs_to_create[base_class]) file_objs += objs_to_create[base_class] return file_objs
def test_comments_move_when_file_moved_from_component_to_project(self, project, component, user): source = { 'path': '/file.txt', 'node': component, 'provider': self.provider } destination = { 'path': '/file.txt', 'node': project, 'provider': self.provider } self._create_file_with_comment(node=source['node'], path=source['path'], user=user) self.file.move_under(destination['node'].get_addon(self.provider).get_root()) payload = self._create_payload('move', user, source, destination, self.file._id) update_file_guid_referent(self=None, node=destination['node'], event_type='addon_file_moved', payload=payload) self.guid.reload() file_node = BaseFileNode.resolve_class(self.provider, BaseFileNode.FILE).get_or_create(destination['node'], self._format_path(destination['path'], file_id=self.file._id)) assert self.guid._id == file_node.get_guid()._id assert self.guid.referent.node._id == destination['node']._id file_comments = Comment.find(Q('root_target', 'eq', self.guid.pk)) assert file_comments.count() == 1
def file_created_or_updated(node, payload, user_id, created_flag): file_node = BaseFileNode.resolve_class( payload['metadata']['provider'], BaseFileNode.FILE).get_or_create( node, payload['metadata'].get('materialized')) file_node.save() created_at = payload['metadata'].get('created_utc') modified_at = payload['metadata'].get('modified_utc') version = '' if not created_at: created_at = None if not modified_at: modified_at = None if payload['metadata']['provider'] == 'osf_storage': version = payload['metadata']['extra'].get('version') file_info = { 'file_id': file_node._id, 'file_name': payload['metadata'].get('name'), 'file_path': payload['metadata'].get('materialized'), 'size': payload['metadata'].get('size'), 'created': created_at, 'modified': modified_at, 'version': version, 'provider': payload['metadata'].get('provider') } add_token(user_id, node, file_info) # Update created/modified user in timestamp result verify_data = RdmFileTimestamptokenVerifyResult.objects.filter( file_id=file_info['file_id']).first() if verify_data: if created_flag: verify_data.upload_file_created_user = user_id else: # Updated verify_data.upload_file_modified_user = user_id verify_data.upload_file_created_at = file_info['created'] verify_data.upload_file_modified_at = file_info['modified'] verify_data.upload_file_size = file_info['size'] verify_data.save()
def add_token(uid, node, data): user = OSFUser.objects.get(id=uid) cookie = user.get_or_create_cookie() tmp_dir = None try: file_node = BaseFileNode.resolve_class( data['provider'], BaseFileNode.FILE).get_or_create(node, data['file_path']) file_node.save() data['file_id'] = file_node._id # Request To Download File tmp_dir = 'tmp_{}'.format(user._id) count = 1 while os.path.exists(tmp_dir): count += 1 tmp_dir = 'tmp_{}_{}'.format(user._id, count) os.mkdir(tmp_dir) download_file_path = waterbutler.download_file(cookie, file_node, tmp_dir) if not userkey_generation_check(user._id): userkey_generation(user._id) addTimestamp = AddTimestamp() result = addTimestamp.add_timestamp(user._id, data, node._id, download_file_path, tmp_dir) shutil.rmtree(tmp_dir) return result except Exception as err: if tmp_dir and os.path.exists(tmp_dir): shutil.rmtree(tmp_dir) logger.exception(err) raise
def view_file(request, node_id, provider, file_id): fp = BaseFileNode.load(file_id) wb_url = fp.generate_waterbutler_url() return redirect(wb_url)
def addon_view_or_download_file(auth, path, provider, **kwargs): extras = request.args.to_dict() extras.pop('_', None) # Clean up our url params a bit action = extras.get('action', 'view') guid = kwargs.get('guid') guid_target = getattr(Guid.load(guid), 'referent', None) target = guid_target or kwargs.get('node') or kwargs['project'] provider_safe = markupsafe.escape(provider) path_safe = markupsafe.escape(path) if not path: raise HTTPError(http_status.HTTP_400_BAD_REQUEST) if hasattr(target, 'get_addon'): node_addon = target.get_addon(provider) if not isinstance(node_addon, BaseStorageAddon): object_text = markupsafe.escape( getattr(target, 'project_or_component', 'this object')) raise HTTPError( http_status.HTTP_400_BAD_REQUEST, data={ 'message_short': 'Bad Request', 'message_long': 'The {} add-on containing {} is no longer connected to {}.' .format(provider_safe, path_safe, object_text) }) if not node_addon.has_auth: raise HTTPError( http_status.HTTP_401_UNAUTHORIZED, data={ 'message_short': 'Unauthorized', 'message_long': 'The {} add-on containing {} is no longer authorized.'. format(provider_safe, path_safe) }) if not node_addon.complete: raise HTTPError( http_status.HTTP_400_BAD_REQUEST, data={ 'message_short': 'Bad Request', 'message_long': 'The {} add-on containing {} is no longer configured.'. format(provider_safe, path_safe) }) savepoint_id = transaction.savepoint() file_node = BaseFileNode.resolve_class(provider, BaseFileNode.FILE).get_or_create( target, path) # Note: Cookie is provided for authentication to waterbutler # it is overriden to force authentication as the current user # the auth header is also pass to support basic auth version = file_node.touch( request.headers.get('Authorization'), **dict(extras, cookie=request.cookies.get(settings.COOKIE_NAME))) if version is None: # File is either deleted or unable to be found in the provider location # Rollback the insertion of the file_node transaction.savepoint_rollback(savepoint_id) if not file_node.pk: file_node = BaseFileNode.load(path) if not file_node: raise HTTPError(http_status.HTTP_404_NOT_FOUND, data={ 'message_short': 'File Not Found', 'message_long': 'The requested file could not be found.' }) if file_node.kind == 'folder': raise HTTPError( http_status.HTTP_400_BAD_REQUEST, data={ 'message_short': 'Bad Request', 'message_long': 'You cannot request a folder from this endpoint.' }) # Allow osfstorage to redirect if the deep url can be used to find a valid file_node if file_node.provider == 'osfstorage' and not file_node.is_deleted: return redirect( file_node.target.web_url_for('addon_view_or_download_file', path=file_node._id, provider=file_node.provider)) return addon_deleted_file(target=target, file_node=file_node, path=path, **kwargs) else: transaction.savepoint_commit(savepoint_id) # TODO clean up these urls and unify what is used as a version identifier if request.method == 'HEAD': return make_response(('', http_status.HTTP_302_FOUND, { 'Location': file_node.generate_waterbutler_url( **dict(extras, direct=None, version=version.identifier, _internal=extras.get('mode') == 'render')) })) if action == 'download': format = extras.get('format') _, extension = os.path.splitext(file_node.name) # avoid rendering files with the same format type. if format and '.{}'.format(format.lower()) != extension.lower(): return redirect('{}/export?format={}&url={}'.format( get_mfr_url(target, provider), format, quote( file_node.generate_waterbutler_url( **dict(extras, direct=None, version=version.identifier, _internal=extras.get('mode') == 'render'))))) return redirect( file_node.generate_waterbutler_url( **dict(extras, direct=None, version=version.identifier, _internal=extras.get('mode') == 'render'))) if action == 'get_guid': draft_id = extras.get('draft') draft = DraftRegistration.load(draft_id) if draft is None or draft.is_approved: raise HTTPError(http_status.HTTP_400_BAD_REQUEST, data={ 'message_short': 'Bad Request', 'message_long': 'File not associated with required object.' }) guid = file_node.get_guid(create=True) guid.referent.save() return dict(guid=guid._id) if len(request.path.strip('/').split('/')) > 1: guid = file_node.get_guid(create=True) return redirect( furl.furl('/{}/'.format(guid._id)).set(args=extras).url) if isinstance(target, Preprint): # Redirecting preprint file guids to the preprint detail page return redirect('/{}/'.format(target._id)) return addon_view_file(auth, target, file_node, version)
def collect_timestamp_trees_to_json(auth, node, **kwargs): # admin call project to provider file list serialized = _view_project(node, auth, primary=True) serialized.update(rubeus.collect_addon_assets(node)) user_info = OSFUser.objects.get(id=Guid.objects.get(_id=serialized['user']['id']).object_id) api_url = util.api_v2_url(api_url_path(kwargs.get('pid'))) cookie = user_info.get_or_create_cookie() cookies = {settings.COOKIE_NAME: cookie} headers = {'content-type': 'application/json'} provider_json_res = None file_res = requests.get(api_url, headers=headers, cookies=cookies) provider_json_res = file_res.json() file_res.close() provider_list = [] for provider_data in provider_json_res['data']: waterbutler_meta_url = util.waterbutler_api_url_for( kwargs.get('pid'), provider_data['attributes']['provider'], '/', **dict(waterbutler_meta_parameter()) ) waterbutler_json_res = None waterbutler_res = requests.get(waterbutler_meta_url, headers=headers, cookies=cookies) waterbutler_json_res = waterbutler_res.json() waterbutler_res.close() file_list = [] child_file_list = [] for file_data in waterbutler_json_res['data']: if file_data['attributes']['kind'] == 'folder': child_file_list.extend( waterbutler_folder_file_info( kwargs.get('pid'), provider_data['attributes']['provider'], file_data['attributes']['path'], node, cookies, headers ) ) else: file_info = None basefile_node = BaseFileNode.resolve_class( provider_data['attributes']['provider'], BaseFileNode.FILE ).get_or_create( node, file_data['attributes']['path'] ) basefile_node.save() if provider_data['attributes']['provider'] == 'osfstorage': file_info = { 'file_name': file_data['attributes']['name'], 'file_path': file_data['attributes']['materialized'], 'file_kind': file_data['attributes']['kind'], 'file_id': basefile_node._id, 'version': file_data['attributes']['extra']['version'] } else: file_info = { 'file_name': file_data['attributes']['name'], 'file_path': file_data['attributes']['materialized'], 'file_kind': file_data['attributes']['kind'], 'file_id': basefile_node._id, 'version': '' } if file_info: file_list.append(file_info) file_list.extend(child_file_list) if file_list: provider_files = { 'provider': provider_data['attributes']['provider'], 'provider_file_list': file_list } provider_list.append(provider_files) return {'provider_list': provider_list}
def get_full_list(uid, pid, node): ''' Get a full list of timestamps from all files uploaded to a storage. ''' user_info = OSFUser.objects.get(id=uid) cookie = user_info.get_or_create_cookie() api_url = util.api_v2_url('nodes/{}/files'.format(pid)) headers = {'content-type': 'application/json'} cookies = {settings.COOKIE_NAME: cookie} file_res = requests.get(api_url, headers=headers, cookies=cookies) provider_json_res = file_res.json() file_res.close() provider_list = [] for provider_data in provider_json_res['data']: waterbutler_meta_url = waterbutler_api_url_for( pid, provider_data['attributes']['provider'], '/', meta=int(time.mktime(datetime.datetime.now().timetuple()))) waterbutler_json_res = None waterbutler_res = requests.get(waterbutler_meta_url, headers=headers, cookies=cookies) waterbutler_json_res = waterbutler_res.json() waterbutler_res.close() file_list = [] child_file_list = [] for file_data in waterbutler_json_res['data']: if file_data['attributes']['kind'] == 'folder': child_file_list.extend( waterbutler_folder_file_info( pid, provider_data['attributes']['provider'], file_data['attributes']['path'], node, cookies, headers)) else: file_info = None basefile_node = BaseFileNode.resolve_class( provider_data['attributes']['provider'], BaseFileNode.FILE).get_or_create( node, file_data['attributes']['path']) basefile_node.save() file_info = { 'file_id': basefile_node._id, 'file_name': file_data['attributes'].get('name'), 'file_path': file_data['attributes'].get('materialized'), 'size': file_data['attributes'].get('size'), 'created': file_data['attributes'].get('created_utc'), 'modified': file_data['attributes'].get('modified_utc'), 'file_version': '' } if provider_data['attributes']['provider'] == 'osfstorage': file_info['file_version'] = file_data['attributes'][ 'extra'].get('version') if file_info: file_list.append(file_info) file_list.extend(child_file_list) if file_list: provider_files = { 'provider': provider_data['attributes']['provider'], 'provider_file_list': file_list } provider_list.append(provider_files) return provider_list
def get_object(self, file_id): return BaseFileNode.load(file_id)
def addon_view_or_download_file(auth, path, provider, **kwargs): extras = request.args.to_dict() extras.pop('_', None) # Clean up our url params a bit action = extras.get('action', 'view') guid = kwargs.get('guid') guid_target = getattr(Guid.load(guid), 'referent', None) target = guid_target or kwargs.get('node') or kwargs['project'] provider_safe = markupsafe.escape(provider) path_safe = markupsafe.escape(path) if not path: raise HTTPError(httplib.BAD_REQUEST) if hasattr(target, 'get_addon'): node_addon = target.get_addon(provider) if not isinstance(node_addon, BaseStorageAddon): object_text = markupsafe.escape(getattr(target, 'project_or_component', 'this object')) raise HTTPError(httplib.BAD_REQUEST, data={ 'message_short': 'Bad Request', 'message_long': 'The {} add-on containing {} is no longer connected to {}.'.format(provider_safe, path_safe, object_text) }) if not node_addon.has_auth: raise HTTPError(httplib.UNAUTHORIZED, data={ 'message_short': 'Unauthorized', 'message_long': 'The {} add-on containing {} is no longer authorized.'.format(provider_safe, path_safe) }) if not node_addon.complete: raise HTTPError(httplib.BAD_REQUEST, data={ 'message_short': 'Bad Request', 'message_long': 'The {} add-on containing {} is no longer configured.'.format(provider_safe, path_safe) }) savepoint_id = transaction.savepoint() file_node = BaseFileNode.resolve_class(provider, BaseFileNode.FILE).get_or_create(target, path) # Note: Cookie is provided for authentication to waterbutler # it is overriden to force authentication as the current user # the auth header is also pass to support basic auth version = file_node.touch( request.headers.get('Authorization'), **dict( extras, cookie=request.cookies.get(settings.COOKIE_NAME) ) ) if version is None: # File is either deleted or unable to be found in the provider location # Rollback the insertion of the file_node transaction.savepoint_rollback(savepoint_id) if not file_node.pk: file_node = BaseFileNode.load(path) if file_node.kind == 'folder': raise HTTPError(httplib.BAD_REQUEST, data={ 'message_short': 'Bad Request', 'message_long': 'You cannot request a folder from this endpoint.' }) # Allow osfstorage to redirect if the deep url can be used to find a valid file_node if file_node and file_node.provider == 'osfstorage' and not file_node.is_deleted: return redirect( file_node.target.web_url_for('addon_view_or_download_file', path=file_node._id, provider=file_node.provider) ) return addon_deleted_file(target=target, file_node=file_node, path=path, **kwargs) else: transaction.savepoint_commit(savepoint_id) # TODO clean up these urls and unify what is used as a version identifier if request.method == 'HEAD': return make_response(('', httplib.FOUND, { 'Location': file_node.generate_waterbutler_url(**dict(extras, direct=None, version=version.identifier, _internal=extras.get('mode') == 'render')) })) if action == 'download': format = extras.get('format') _, extension = os.path.splitext(file_node.name) # avoid rendering files with the same format type. if format and '.{}'.format(format.lower()) != extension.lower(): return redirect('{}/export?format={}&url={}'.format(get_mfr_url(target, provider), format, urllib.quote(file_node.generate_waterbutler_url( **dict(extras, direct=None, version=version.identifier, _internal=extras.get('mode') == 'render') )))) return redirect(file_node.generate_waterbutler_url(**dict(extras, direct=None, version=version.identifier, _internal=extras.get('mode') == 'render'))) if action == 'get_guid': draft_id = extras.get('draft') draft = DraftRegistration.load(draft_id) if draft is None or draft.is_approved: raise HTTPError(httplib.BAD_REQUEST, data={ 'message_short': 'Bad Request', 'message_long': 'File not associated with required object.' }) guid = file_node.get_guid(create=True) guid.referent.save() return dict(guid=guid._id) if len(request.path.strip('/').split('/')) > 1: guid = file_node.get_guid(create=True) return redirect(furl.furl('/{}/'.format(guid._id)).set(args=extras).url) if isinstance(target, Preprint): # Redirecting preprint file guids to the preprint detail page return redirect('/{}/'.format(target._id)) return addon_view_file(auth, target, file_node, version)
def get_object(self, file_id): return BaseFileNode.load(file_id)
def view_file(request, node_id, provider, file_id): fp = BaseFileNode.load(file_id) wb_url = fp.generate_waterbutler_url() return redirect(wb_url)