Esempio n. 1
0
def waterbutler_folder_file_info(pid, provider, path, node, cookies, headers):
    # get waterbutler folder file
    if provider == 'osfstorage':
        waterbutler_meta_url = util.waterbutler_api_url_for(
            pid, provider,
            '/' + path,
            **dict(waterbutler_meta_parameter())
        )
    else:
        waterbutler_meta_url = util.waterbutler_api_url_for(
            pid, provider,
            path,
            **dict(waterbutler_meta_parameter())
        )

    waterbutler_res = requests.get(waterbutler_meta_url, headers=headers, cookies=cookies)
    waterbutler_json_res = waterbutler_res.json()
    waterbutler_res.close()
    file_list = []
    child_file_list = []
    for file_data in waterbutler_json_res['data']:
        if file_data['attributes']['kind'] == 'folder':
            child_file_list.extend(waterbutler_folder_file_info(
                pid, provider, file_data['attributes']['path'],
                node, cookies, headers))
        else:
            basefile_node = BaseFileNode.resolve_class(
                provider,
                BaseFileNode.FILE
            ).get_or_create(
                node,
                file_data['attributes']['path']
            )
            basefile_node.save()
            if provider == 'osfstorage':
                file_info = {
                    'file_name': file_data['attributes']['name'],
                    'file_path': file_data['attributes']['materialized'],
                    'file_kind': file_data['attributes']['kind'],
                    'file_id': basefile_node._id,
                    'version': file_data['attributes']['extra']['version']
                }
            else:
                file_info = {
                    'file_name': file_data['attributes']['name'],
                    'file_path': file_data['attributes']['materialized'],
                    'file_kind': file_data['attributes']['kind'],
                    'file_id': basefile_node._id,
                    'version': ''
                }

            file_list.append(file_info)

    file_list.extend(child_file_list)

    return file_list
Esempio n. 2
0
def send_file(name, content_type, stream, node, user, create=True, path='/'):
    """Upload file to OSF using waterbutler v1 api
    :param str name: The name of the requested file
    :param str content_type: Content-Type
    :param StringIO stream: file-like stream to be uploaded
    :param Node node: Project Node
    :param User user: User whose cookie will be used
    :param Bool create: Create or update file
    :param str path: Waterbutler V1 path of the requested file
    """

    if not node:
        return
    node_id = node._id

    if not user:
        return
    cookies = {website_settings.COOKIE_NAME:user.get_or_create_cookie()}

    # create a new folder
    if stream is None:
        upload_url = util.waterbutler_api_url_for(node_id, 'osfstorage', path, kind='folder', name=name)
        print('create folder: url={}'.format(upload_url))
        resp = requests.put(
            upload_url,
            headers={'Content-Type': content_type},
            cookies=cookies,
        )
        if resp.status_code != 201:
            resp.raise_for_status()
        return resp

    # create or update a file
    stream.seek(0)
    if create:
        upload_url = util.waterbutler_api_url_for(node_id, 'osfstorage', path, kind='file', name=name)
        print('create file: url={}'.format(upload_url))
    else:
        path = '/{}'.format(name)
        upload_url = util.waterbutler_api_url_for(node_id, 'osfstorage', path, kind='file')
        print('update file: url={}'.format(upload_url))
    resp = requests.put(
        upload_url,
        data=stream,
        headers={'Content-Type': content_type},
        cookies=cookies,
    )

    if resp.status_code not in [200, 201, 503, 409]:
        resp.raise_for_status()
    if resp.status_code == 503:
        pass  # forward 503 error back to the caller
    elif resp.status_code == 409:
        print('I/O Warning: cannot create new file/folder that already exists.') # this should never appear
    return resp
Esempio n. 3
0
    def _get_fileobj_child_metadata(self,
                                    filenode,
                                    user,
                                    cookie=None,
                                    version=None):

        kwargs = {}
        if version:
            kwargs['version'] = version
        if cookie:
            kwargs['cookie'] = cookie
        elif user:
            kwargs['cookie'] = user.get_or_create_cookie()

        metadata_url = waterbutler_api_url_for(self.owner._id,
                                               self.config.short_name,
                                               path=filenode.get('path', '/'),
                                               user=user,
                                               view_only=True,
                                               _internal=True,
                                               **kwargs)

        res = requests.get(metadata_url)

        if res.status_code != 200:
            raise HTTPError(res.status_code, data={'error': res.json()})

        # TODO: better throttling?
        time.sleep(1.0 / 5.0)

        data = res.json().get('data', None)
        if data:
            return [child['attributes'] for child in data]
        return []
Esempio n. 4
0
 def test_waterbutler_api_url_for(self):
     with self.app.test_request_context():
         url = waterbutler_api_url_for('fakeid', 'provider', '/path')
     assert_in('/fakeid/', url)
     assert_in('/path', url)
     assert_in('/providers/provider/', url)
     assert_in(settings.WATERBUTLER_URL, url)
Esempio n. 5
0
def recursive_upload(dst, fs_path, parent, name=None):
    is_folder = os.path.isdir(fs_path)
    name = name or fs_path.rstrip('/')[-1]
    params = {
        'cookie': dst.creator.get_or_create_cookie(),
        'kind': 'folder' if is_folder else 'file',
        'name': name
    }
    url = waterbutler_api_url_for(dst._id,
                                  'osfstorage',
                                  parent.path,
                                  _internal=True,
                                  **params)
    logger.info('Preparing to upload {} {}'.format(params['kind'],
                                                   params['name']))
    if not DRY_RUN:
        if is_folder:
            resp = requests.put(url)
            assert resp.status_code == 201
            logger.info('Folder upload complete')
            new_filenode = parent.children.get(name=name)
            for child in os.listdir(fs_path):
                recursive_upload(dst, child, new_filenode)
        else:
            with open(fs_path, 'r+') as fp:
                resp = requests.put(url, data=fp)
                assert resp.status_code == 201
            logger.info('File upload complete')
Esempio n. 6
0
 def test_waterbutler_api_url_for(self):
     with self.app.test_request_context():
         url = waterbutler_api_url_for('fakeid', 'provider', '/path')
     assert_in('/fakeid/', url)
     assert_in('/path', url)
     assert_in('/providers/provider/', url)
     assert_in(settings.WATERBUTLER_URL, url)
Esempio n. 7
0
    def test_get_files_has_links(self, app, user, url):
        res = app.get(url, auth=user.auth)
        file_detail_json = res.json['data'][0]
        quickfiles_node = quickfiles(user)
        waterbutler_url = website_utils.waterbutler_api_url_for(
            quickfiles_node._id,
            'osfstorage',
            file_detail_json['attributes']['path']
        )

        assert 'delete' in file_detail_json['links']
        assert file_detail_json['links']['delete'] == waterbutler_url

        assert 'download' in file_detail_json['links']
        assert file_detail_json['links']['download'] == waterbutler_url

        assert 'info' in file_detail_json['links']

        assert 'move' in file_detail_json['links']
        assert file_detail_json['links']['move'] == waterbutler_url

        assert 'self' in file_detail_json['links']

        assert 'upload' in file_detail_json['links']
        assert file_detail_json['links']['upload'] == waterbutler_url
Esempio n. 8
0
    def _get_fileobj_child_metadata(self, filenode, user, cookie=None, version=None):

        kwargs = {}
        if version:
            kwargs['version'] = version
        if cookie:
            kwargs['cookie'] = cookie
        elif user:
            kwargs['cookie'] = user.get_or_create_cookie()

        metadata_url = waterbutler_api_url_for(
            self.owner._id,
            self.config.short_name,
            path=filenode.get('path', '/'),
            user=user,
            view_only=True,
            _internal=True,
            **kwargs
        )

        res = requests.get(metadata_url)

        if res.status_code != 200:
            raise HTTPError(res.status_code, data={'error': res.json()})

        # TODO: better throttling?
        time.sleep(1.0 / 5.0)

        data = res.json().get('data', None)
        if data:
            return [child['attributes'] for child in data]
        return []
Esempio n. 9
0
def get_file_object(node, path, provider, request):
    if provider == 'osfstorage':
        # Kinda like /me for a user
        # The one odd case where path is not really path
        if path == '/':
            obj = node.get_addon('osfstorage').get_root()
        else:
            obj = get_object_or_error(
                OsfStorageFileNode,
                Q('node', 'eq', node._id) &
                Q('_id', 'eq', path.strip('/')) &
                Q('is_file', 'eq', not path.endswith('/'))
            )
        return obj

    url = waterbutler_api_url_for(node._id, provider, path, meta=True)
    waterbutler_request = requests.get(
        url,
        cookies=request.COOKIES,
        headers={'Authorization': request.META.get('HTTP_AUTHORIZATION')},
    )

    if waterbutler_request.status_code == 401:
        raise PermissionDenied

    if waterbutler_request.status_code == 404:
        raise NotFound

    if is_server_error(waterbutler_request.status_code):
        raise ServiceUnavailableError(detail='Could not retrieve files information at this time.')

    try:
        return waterbutler_request.json()['data']
    except KeyError:
        raise ServiceUnavailableError(detail='Could not retrieve files information at this time.')
Esempio n. 10
0
 def generate_waterbutler_url(self, **kwargs):
     return util.waterbutler_api_url_for(
         self.node._id,
         self.provider,
         self.path,
         **kwargs
     )
Esempio n. 11
0
 def generate_waterbutler_url(self, **kwargs):
     return waterbutler_api_url_for(
         self.node._id,
         self.provider,
         self.path,
         **kwargs
     )
Esempio n. 12
0
def perform_wb_copy(reg, node_settings):
    src, dst, user = reg.archive_job.info()
    if dst.files.filter(
            name=node_settings.archive_folder_name.replace('/', '-')).exists():
        if not DELETE_COLLISIONS and not SKIP_COLLISIONS:
            raise Exception(
                'Archive folder for {} already exists. Investigate manually and rerun with either --delete-collisions or --skip-collisions'
            )
        if DELETE_COLLISIONS:
            archive_folder = dst.files.exclude(type='osf.trashedfolder').get(
                name=node_settings.archive_folder_name.replace('/', '-'))
            logger.info('Removing {}'.format(archive_folder))
            archive_folder.delete()
        if SKIP_COLLISIONS:
            complete_archive_target(reg, node_settings.short_name)
            return
    params = {'cookie': user.get_or_create_cookie()}
    data = {
        'action': 'copy',
        'path': '/',
        'rename': node_settings.archive_folder_name.replace('/', '-'),
        'resource': dst._id,
        'provider': ARCHIVE_PROVIDER,
    }
    url = waterbutler_api_url_for(src._id,
                                  node_settings.short_name,
                                  _internal=True,
                                  **params)
    res = requests.post(url, data=json.dumps(data))
    if res.status_code not in (http.OK, http.CREATED, http.ACCEPTED):
        raise HTTPError(res.status_code)
def export_files(node, user, current_dir):
    """
    Creates a "files" directory within the current directory.
    Exports all of the OSFStorage files for a given node.
    Uses WB's download zip functionality to download osfstorage-archive.zip in a single request.

    """
    files_dir = os.path.join(current_dir, 'files')
    os.mkdir(files_dir)
    response = requests.get(
        url=waterbutler_api_url_for(
            node_id=node._id,
            _internal=True,
            provider='osfstorage',
            zip='',
            cookie=user.get_or_create_cookie()
        )
    )
    if response.status_code == 200:
        with open(os.path.join(files_dir, 'osfstorage-archive.zip'), 'wb') as f:
            f.write(response.content)
    else:
        ERRORS.append(
            'Error exporting files for node {}. Waterbutler responded with a {} status code. Response: {}'
            .format(node._id, response.status_code, response.json())
        )
Esempio n. 14
0
def archive_addon(addon_short_name, job_pk, stat_result):
    """Archive the contents of an addon by making a copy request to the
    WaterBulter API

    :param addon_short_name: AddonConfig.short_name of the addon to be archived
    :param job_pk: primary key of ArchiveJob
    :return: None
    """
    create_app_context()
    job = ArchiveJob.load(job_pk)
    src, dst, user = job.info()
    logger.info('Archiving addon: {0} on node: {1}'.format(addon_short_name, src._id))

    cookie = user.get_or_create_cookie()
    params = {'cookie': cookie}
    rename_suffix = ''
    # The dataverse API will not differentiate between published and draft files
    # unless expcicitly asked. We need to create seperate folders for published and
    # draft in the resulting archive.
    #
    # Additionally trying to run the archive without this distinction creates a race
    # condition that non-deterministically caused archive jobs to fail.
    if 'dataverse' in addon_short_name:
        params['revision'] = 'latest' if addon_short_name.split('-')[-1] == 'draft' else 'latest-published'
        rename_suffix = ' (draft)' if addon_short_name.split('-')[-1] == 'draft' else ' (published)'
        addon_short_name = 'dataverse'
    src_provider = src.get_addon(addon_short_name)
    folder_name = src_provider.archive_folder_name
    rename = '{}{}'.format(folder_name, rename_suffix)
    url = waterbutler_api_url_for(src._id, addon_short_name, _internal=True, **params)
    data = make_waterbutler_payload(dst._id, rename)
    make_copy_request.delay(job_pk=job_pk, url=url, data=data)
Esempio n. 15
0
    def get_comment(self, check_permissions=True):
        pk = self.kwargs[self.comment_lookup_url_kwarg]
        try:
            comment = Comment.find_one(Q('_id', 'eq', pk) & Q('root_target', 'ne', None))
        except NoResultsFound:
            raise NotFound

        # Deleted root targets still appear as tuples in the database and are included in
        # the above query, requiring an additional check
        if isinstance(comment.root_target.referent, TrashedFileNode):
            comment.root_target = None
            comment.save()

        if comment.root_target is None:
            raise NotFound

        if isinstance(comment.root_target.referent, StoredFileNode):
            root_target = comment.root_target
            referent = root_target.referent

            if referent.provider == 'osfstorage':
                try:
                    StoredFileNode.find(
                        Q('node', 'eq', comment.node._id) &
                        Q('_id', 'eq', referent._id) &
                        Q('is_file', 'eq', True)
                    )
                except NoResultsFound:
                    Comment.update(Q('root_target', 'eq', root_target), data={'root_target': None})
                    raise NotFound
            else:
                if referent.provider == 'dropbox':
                    # referent.path is the absolute path for the db file, but wb requires the relative path
                    referent = DropboxFile.load(referent._id)
                url = waterbutler_api_url_for(comment.node._id, referent.provider, referent.path, meta=True)
                waterbutler_request = requests.get(
                    url,
                    cookies=self.request.COOKIES,
                    headers={'Authorization': self.request.META.get('HTTP_AUTHORIZATION')},
                )

                if waterbutler_request.status_code == 401:
                    raise PermissionDenied

                if waterbutler_request.status_code == 404:
                    Comment.update(Q('root_target', 'eq', root_target), data={'root_target': None})
                    raise NotFound

                if is_server_error(waterbutler_request.status_code):
                    raise ServiceUnavailableError(detail='Could not retrieve files information at this time.')

                try:
                    waterbutler_request.json()['data']
                except KeyError:
                    raise ServiceUnavailableError(detail='Could not retrieve files information at this time.')

        if check_permissions:
            # May raise a permission denied
            self.check_object_permissions(self.request, comment)
        return comment
Esempio n. 16
0
def create_object(name, content_type, node, user, stream=None, kind=None, path='/'):
    """Create an object (file/folder) OSF using WaterButler v1 API
    :param str name: The name of the requested file
    :param str content_type: Content-Type
    :param StringIO stream: file-like stream to be uploaded
    :param Node node: Project Node
    :param User user: User whose cookie will be used
    :param str path: Waterbutler V1 path of the requested file
    """
    assert(kind != 'file' or stream != None)

    node_id = node._id
    cookies = {website_settings.COOKIE_NAME: user.get_or_create_cookie()}

    # create or update a file
    url = util.waterbutler_api_url_for(node_id, 'osfstorage', path)
    resp = requests.get(url, cookies=cookies)
    data = resp.json()['data']

    existing = None
    for item in data:
        if item['attributes']['name'] == name:
            existing = item

    if stream:
        stream.seek(0)

    # create a new file/folder?
    if not existing:
        url = util.waterbutler_api_url_for(node_id, 'osfstorage', path, kind=kind, name=name)
        resp = requests.put(
            url,
            data=stream,
            headers={'Content-Type': content_type},
            cookies=cookies,
        )
    elif kind == 'file':
        url = util.waterbutler_api_url_for(node_id, 'osfstorage', existing['attributes']['path'], kind=kind)
        resp = requests.put(
            url,
            data=stream,
            headers={'Content-Type': content_type},
            cookies=cookies,
        )
    else:
        return existing
    return resp.json()['data']
Esempio n. 17
0
def prepare_mock_wb_response(
    node=None,
    provider='github',
    files=None,
    folder=True,
    path='/',
    method=httpretty.GET,
    status_code=200
):
    """Prepare a mock Waterbutler response with httpretty.

    :param Node node: Target node.
    :param str provider: Addon provider
    :param list files: Optional list of files. You can specify partial data; missing values
        will have defaults.
    :param folder: True if mocking out a folder response, False if a file response.
    :param path: Waterbutler path, passed to waterbutler_api_url_for.
    :param str method: HTTP method.
    :param int status_code: HTTP status.
    """
    node = node
    files = files or []
    wb_url = waterbutler_api_url_for(
        node._id, provider=provider, path=path, meta=True)

    default_file = {
        u'contentType': None,
        u'extra': {u'downloads': 0, u'version': 1},
        u'kind': u'file',
        u'modified': None,
        u'name': u'NewFile',
        u'path': u'/NewFile',
        u'provider': provider,
        u'size': None,
        u'materialized': '/',
    }

    if len(files):
        data = [dict(default_file, **each) for each in files]
    else:
        data = [default_file]

    jsonapi_data = []
    for datum in data:
        jsonapi_data.append({'attributes': datum})

    if not folder:
        jsonapi_data = jsonapi_data[0]

    body = json.dumps({
        u'data': jsonapi_data
    })
    httpretty.register_uri(
        method,
        wb_url,
        body=body,
        status=status_code,
        content_type='application/json'
    )
Esempio n. 18
0
 def resolve_url(self, obj):
     """Reverse URL lookup for WaterButler routes
     """
     if self.must_be_folder is True and not obj.path.endswith('/'):
         return None
     if self.must_be_file is True and obj.path.endswith('/'):
         return None
     return waterbutler_api_url_for(obj.node._id, obj.provider, obj.path, **self.kwargs)
Esempio n. 19
0
 def resolve_url(self, obj):
     """Reverse URL lookup for WaterButler routes
     """
     if self.must_be_folder is True and not obj.path.endswith('/'):
         return None
     if self.must_be_file is True and obj.path.endswith('/'):
         return None
     return waterbutler_api_url_for(obj.node._id, obj.provider, obj.path, **self.kwargs)
Esempio n. 20
0
def prepare_mock_wb_response(node=None,
                             provider='github',
                             files=None,
                             folder=True,
                             path='/',
                             method=httpretty.GET,
                             status_code=200):
    """Prepare a mock Waterbutler response with httpretty.

    :param Node node: Target node.
    :param str provider: Addon provider
    :param list files: Optional list of files. You can specify partial data; missing values
        will have defaults.
    :param folder: True if mocking out a folder response, False if a file response.
    :param path: Waterbutler path, passed to waterbutler_api_url_for.
    :param str method: HTTP method.
    :param int status_code: HTTP status.
    """
    node = node
    files = files or []
    wb_url = waterbutler_api_url_for(node._id,
                                     provider=provider,
                                     path=path,
                                     meta=True)

    default_file = {
        u'contentType': None,
        u'extra': {
            u'downloads': 0,
            u'version': 1
        },
        u'kind': u'file',
        u'modified': None,
        u'name': u'NewFile',
        u'path': u'/NewFile',
        u'provider': provider,
        u'size': None,
        u'materialized': '/',
    }

    if len(files):
        data = [dict(default_file, **each) for each in files]
    else:
        data = [default_file]

    jsonapi_data = []
    for datum in data:
        jsonapi_data.append({'attributes': datum})

    if not folder:
        jsonapi_data = jsonapi_data[0]

    body = json.dumps({u'data': jsonapi_data})
    httpretty.register_uri(method,
                           wb_url,
                           body=body,
                           status=status_code,
                           content_type='application/json')
Esempio n. 21
0
    def test_waterbutler_api_url_for_internal(self):
        settings.WATERBUTLER_INTERNAL_URL = 'http://1.2.3.4:7777'
        with self.app.test_request_context():
            url = waterbutler_api_url_for('fakeid', 'provider', '/path', _internal=True)

        assert_not_in(settings.WATERBUTLER_URL, url)
        assert_in(settings.WATERBUTLER_INTERNAL_URL, url)
        assert_in('/fakeid/', url)
        assert_in('/path', url)
        assert_in('/providers/provider', url)
Esempio n. 22
0
    def test_files_relationship_upload(self, app, user_one):
        url = '/{}users/{}/'.format(API_BASE, user_one._id)
        res = app.get(url, auth=user_one)
        quickfiles = QuickFilesNode.objects.get(creator=user_one)
        user_json = res.json['data']
        upload_url = user_json['relationships']['quickfiles']['links']['upload']['href']
        waterbutler_upload = website_utils.waterbutler_api_url_for(
            quickfiles._id, 'osfstorage')

        assert upload_url == waterbutler_upload
Esempio n. 23
0
 def test_waterbutler_invalid_data_returns_503(self):
     wb_url = waterbutler_api_url_for(self.project._id, provider='github', path='/', meta=True)
     httpretty.register_uri(
         httpretty.GET,
         wb_url,
         body=json.dumps({}),
         status=400
     )
     url = '/{}nodes/{}/files/github/'.format(API_BASE, self.project._id)
     res = self.app.get(url, auth=self.user.auth, expect_errors=True)
     assert_equal(res.status_code, 503)
Esempio n. 24
0
    def test_files_relationship_upload(self, app, user_one):
        url = "/{}users/{}/".format(API_BASE, user_one._id)
        res = app.get(url, auth=user_one)
        quickfiles = QuickFilesNode.objects.get(creator=user_one)
        user_json = res.json['data']
        upload_url = user_json['relationships']['quickfiles']['links'][
            'upload']['href']
        waterbutler_upload = website_utils.waterbutler_api_url_for(
            quickfiles._id, 'osfstorage')

        assert upload_url == waterbutler_upload
Esempio n. 25
0
    def get_comment(self, check_permissions=True):
        pk = self.kwargs[self.comment_lookup_url_kwarg]
        try:
            comment = Comment.find_one(Q('_id', 'eq', pk) & Q('root_target', 'ne', None))
        except NoResultsFound:
            raise NotFound

        # Deleted root targets still appear as tuples in the database and are included in
        # the above query, requiring an additional check
        if comment.root_target is None:
            raise NotFound

        if isinstance(comment.root_target, StoredFileNode):
            root_target = comment.root_target
            if root_target.provider == 'osfstorage':
                try:
                    StoredFileNode.find(
                        Q('node', 'eq', comment.node._id) &
                        Q('_id', 'eq', root_target._id) &
                        Q('is_file', 'eq', True)
                    )
                except NoResultsFound:
                    Comment.update(Q('root_target', 'eq', root_target), data={'root_target': None})
                    del comment.node.commented_files[root_target._id]
                    raise NotFound
            else:
                if root_target.provider == 'dropbox':
                    root_target = DropboxFile.load(comment.root_target._id)
                url = waterbutler_api_url_for(comment.node._id, root_target.provider, root_target.path, meta=True)
                waterbutler_request = requests.get(
                    url,
                    cookies=self.request.COOKIES,
                    headers={'Authorization': self.request.META.get('HTTP_AUTHORIZATION')},
                )

                if waterbutler_request.status_code == 401:
                    raise PermissionDenied

                if waterbutler_request.status_code == 404:
                    Comment.update(Q('root_target', 'eq', root_target), data={'root_target': None})
                    raise NotFound

                if is_server_error(waterbutler_request.status_code):
                    raise ServiceUnavailableError(detail='Could not retrieve files information at this time.')

                try:
                    waterbutler_request.json()['data']
                except KeyError:
                    raise ServiceUnavailableError(detail='Could not retrieve files information at this time.')

        if check_permissions:
            # May raise a permission denied
            self.check_object_permissions(self.request, comment)
        return comment
Esempio n. 26
0
 def test_handles_bad_waterbutler_request(self):
     wb_url = waterbutler_api_url_for(self.project._id, provider='github', path='/', meta=True)
     httpretty.register_uri(
         httpretty.GET,
         wb_url,
         body=json.dumps({}),
         status=418
     )
     url = '/{}nodes/{}/files/github/'.format(API_BASE, self.project._id)
     res = self.app.get(url, auth=self.user.auth, expect_errors=True)
     assert_equal(res.status_code, 503)
     assert_in('detail', res.json['errors'][0])
Esempio n. 27
0
 def resolve_url(self, obj):
     """Reverse URL lookup for WaterButler routes
     """
     if self.must_be_folder is True and not obj.path.endswith('/'):
         raise SkipField
     if self.must_be_file is True and obj.path.endswith('/'):
         raise SkipField
     url = website_utils.waterbutler_api_url_for(obj.node._id, obj.provider, obj.path, **self.kwargs)
     if not url:
         raise SkipField
     else:
         return url
Esempio n. 28
0
 def resolve_url(self, obj):
     """Reverse URL lookup for WaterButler routes
     """
     if self.must_be_folder is True and not obj.path.endswith('/'):
         raise SkipField
     if self.must_be_file is True and obj.path.endswith('/'):
         raise SkipField
     url = website_utils.waterbutler_api_url_for(obj.node._id, obj.provider, obj.path, **self.kwargs)
     if not url:
         raise SkipField
     else:
         return url
Esempio n. 29
0
    def test_waterbutler_api_url_for_internal(self):
        settings.WATERBUTLER_INTERNAL_URL = 'http://1.2.3.4:7777'
        with self.app.test_request_context():
            url = waterbutler_api_url_for('fakeid',
                                          'provider',
                                          '/path',
                                          _internal=True)

        assert_not_in(settings.WATERBUTLER_URL, url)
        assert_in(settings.WATERBUTLER_INTERNAL_URL, url)
        assert_in('/fakeid/', url)
        assert_in('/path', url)
        assert_in('/providers/provider', url)
Esempio n. 30
0
 def to_representation(self, value):
     relationship_links = super(QuickFilesRelationshipField, self).to_representation(value)
     quickfiles_guid = value.nodes_created.filter(type=QuickFilesNode._typedmodels_type).values_list('guids___id', flat=True).get()
     upload_url = website_utils.waterbutler_api_url_for(quickfiles_guid, 'osfstorage')
     relationship_links['links']['upload'] = {
         'href': upload_url,
         'meta': {}
     }
     relationship_links['links']['download'] = {
         'href': '{}?zip='.format(upload_url),
         'meta': {}
     }
     return relationship_links
Esempio n. 31
0
def upload_attachment(user, node, attachment):
    attachment.seek(0)
    name = (attachment.filename or settings.MISSING_FILE_NAME)
    content = attachment.read()
    upload_url = util.waterbutler_api_url_for(
        node._id,
        'osfstorage',
        name=name,
        cookie=user.get_or_create_cookie(),
        _internal=True)

    requests.put(
        upload_url,
        data=content,
    )
Esempio n. 32
0
def get_file_object(node, path, provider, request):
    # Don't bother going to waterbutler for osfstorage
    if provider == 'osfstorage':
        # Kinda like /me for a user
        # The one odd case where path is not really path
        if path == '/':
            obj = node.get_addon('osfstorage').get_root()
        else:
            if path.endswith('/'):
                model = OsfStorageFolder
            else:
                model = OsfStorageFile
            obj = get_object_or_error(
                model,
                Q('node', 'eq', node.pk) & Q('_id', 'eq', path.strip('/')))
        return obj

    if not node.get_addon(provider) or not node.get_addon(provider).configured:
        raise NotFound(
            'The {} provider is not configured for this project.'.format(
                provider))

    url = waterbutler_api_url_for(node._id,
                                  provider,
                                  path,
                                  _internal=True,
                                  meta=True)
    waterbutler_request = requests.get(
        url,
        cookies=request.COOKIES,
        headers={'Authorization': request.META.get('HTTP_AUTHORIZATION')},
    )

    if waterbutler_request.status_code == 401:
        raise PermissionDenied

    if waterbutler_request.status_code == 404:
        raise NotFound

    if is_server_error(waterbutler_request.status_code):
        raise ServiceUnavailableError(
            detail='Could not retrieve files information at this time.')

    try:
        return waterbutler_request.json()['data']
    except KeyError:
        raise ServiceUnavailableError(
            detail='Could not retrieve files information at this time.')
Esempio n. 33
0
    def get_queryset(self):
        # Don't bother going to waterbutler for osfstorage
        if self.kwargs['provider'] == 'osfstorage':
            self.check_object_permissions(self.request, self.get_node())
            # Kinda like /me for a user
            # The one odd case where path is not really path
            if self.kwargs['path'] == '/':
                return list(self.get_node().get_addon('osfstorage').get_root().children)

            fobj = OsfStorageFileNode.find_one(
                Q('node', 'eq', self.get_node()._id) &
                Q('path', 'eq', self.kwargs['path'])
            )

            if fobj.is_file:
                return [fobj]

            return list(fobj.children)

        url = waterbutler_api_url_for(
            self.get_node()._id,
            self.kwargs['provider'],
            self.kwargs['path'],
            meta=True
        )

        waterbutler_request = requests.get(
            url,
            cookies=self.request.COOKIES,
            headers={'Authorization': self.request.META.get('HTTP_AUTHORIZATION')},
        )
        if waterbutler_request.status_code == 401:
            raise PermissionDenied
        try:
            files_list = waterbutler_request.json()['data']
        except KeyError:
            raise ValidationError(detail='detail: Could not retrieve files information.')

        if isinstance(files_list, dict):
            files_list = [files_list]

        return [self.get_file_item(file) for file in files_list]
Esempio n. 34
0
def get_file_object(node, path, provider, request):
    # Don't bother going to waterbutler for osfstorage
    if provider == 'osfstorage':
        # Kinda like /me for a user
        # The one odd case where path is not really path
        if path == '/':
            obj = node.get_addon('osfstorage').get_root()
        else:
            if path.endswith('/'):
                model = OsfStorageFolder
            else:
                model = OsfStorageFile
            obj = get_object_or_error(model, Q(node=node.pk, _id=path.strip('/')), request)
        return obj

    if not node.get_addon(provider) or not node.get_addon(provider).configured:
        raise NotFound('The {} provider is not configured for this project.'.format(provider))

    view_only = request.query_params.get('view_only', default=None)
    url = waterbutler_api_url_for(node._id, provider, path, _internal=True,
                                  meta=True, view_only=view_only)

    waterbutler_request = requests.get(
        url,
        cookies=request.COOKIES,
        headers={'Authorization': request.META.get('HTTP_AUTHORIZATION')},
    )

    if waterbutler_request.status_code == 401:
        raise PermissionDenied

    if waterbutler_request.status_code == 404:
        raise NotFound

    if is_server_error(waterbutler_request.status_code):
        raise ServiceUnavailableError(detail='Could not retrieve files information at this time.')

    try:
        return waterbutler_request.json()['data']
    except KeyError:
        raise ServiceUnavailableError(detail='Could not retrieve files information at this time.')
Esempio n. 35
0
    def test_get_files_has_links(self, app, user, url):
        res = app.get(url, auth=user.auth)
        file_detail_json = res.json['data'][0]
        quickfiles_node = quickfiles(user)
        waterbutler_url = website_utils.waterbutler_api_url_for(
            quickfiles_node._id, 'osfstorage',
            file_detail_json['attributes']['path'])

        assert 'delete' in file_detail_json['links']
        assert file_detail_json['links']['delete'] == waterbutler_url

        assert 'download' in file_detail_json['links']
        assert file_detail_json['links']['download'] == waterbutler_url

        assert 'info' in file_detail_json['links']

        assert 'move' in file_detail_json['links']
        assert file_detail_json['links']['move'] == waterbutler_url

        assert 'self' in file_detail_json['links']

        assert 'upload' in file_detail_json['links']
        assert file_detail_json['links']['upload'] == waterbutler_url
Esempio n. 36
0
def perform_wb_copy(reg, node_settings):
    src, dst, user = reg.archive_job.info()
    if dst.files.filter(name=node_settings.archive_folder_name.replace('/', '-')).exists():
        if not DELETE_COLLISIONS and not SKIP_COLLISIONS:
            raise Exception('Archive folder for {} already exists. Investigate manually and rerun with either --delete-collisions or --skip-collisions')
        if DELETE_COLLISIONS:
            archive_folder = dst.files.exclude(type='osf.trashedfolder').get(name=node_settings.archive_folder_name.replace('/', '-'))
            logger.info('Removing {}'.format(archive_folder))
            archive_folder.delete()
        if SKIP_COLLISIONS:
            complete_archive_target(reg, node_settings.short_name)
            return
    params = {'cookie': user.get_or_create_cookie()}
    data = {
        'action': 'copy',
        'path': '/',
        'rename': node_settings.archive_folder_name.replace('/', '-'),
        'resource': dst._id,
        'provider': ARCHIVE_PROVIDER,
    }
    url = waterbutler_api_url_for(src._id, node_settings.short_name, _internal=True, **params)
    res = requests.post(url, data=json.dumps(data))
    if res.status_code not in (http.OK, http.CREATED, http.ACCEPTED):
        raise HTTPError(res.status_code)
Esempio n. 37
0
def collect_timestamp_trees_to_json(auth, node, **kwargs):
    # admin call project to provider file list
    serialized = _view_project(node, auth, primary=True)
    serialized.update(rubeus.collect_addon_assets(node))
    user_info = OSFUser.objects.get(id=Guid.objects.get(_id=serialized['user']['id']).object_id)
    api_url = util.api_v2_url(api_url_path(kwargs.get('pid')))
    cookie = user_info.get_or_create_cookie()
    cookies = {settings.COOKIE_NAME: cookie}
    headers = {'content-type': 'application/json'}
    provider_json_res = None
    file_res = requests.get(api_url, headers=headers, cookies=cookies)
    provider_json_res = file_res.json()
    file_res.close()
    provider_list = []

    for provider_data in provider_json_res['data']:
        waterbutler_meta_url = util.waterbutler_api_url_for(
            kwargs.get('pid'),
            provider_data['attributes']['provider'],
            '/',
            **dict(waterbutler_meta_parameter())
        )
        waterbutler_json_res = None
        waterbutler_res = requests.get(waterbutler_meta_url, headers=headers, cookies=cookies)
        waterbutler_json_res = waterbutler_res.json()
        waterbutler_res.close()

        file_list = []
        child_file_list = []
        for file_data in waterbutler_json_res['data']:
            if file_data['attributes']['kind'] == 'folder':
                child_file_list.extend(
                    waterbutler_folder_file_info(
                        kwargs.get('pid'),
                        provider_data['attributes']['provider'],
                        file_data['attributes']['path'],
                        node, cookies, headers
                    )
                )
            else:
                file_info = None
                basefile_node = BaseFileNode.resolve_class(
                    provider_data['attributes']['provider'],
                    BaseFileNode.FILE
                ).get_or_create(
                    node,
                    file_data['attributes']['path']
                )
                basefile_node.save()
                if provider_data['attributes']['provider'] == 'osfstorage':
                    file_info = {
                        'file_name': file_data['attributes']['name'],
                        'file_path': file_data['attributes']['materialized'],
                        'file_kind': file_data['attributes']['kind'],
                        'file_id': basefile_node._id,
                        'version': file_data['attributes']['extra']['version']
                    }
                else:
                    file_info = {
                        'file_name': file_data['attributes']['name'],
                        'file_path': file_data['attributes']['materialized'],
                        'file_kind': file_data['attributes']['kind'],
                        'file_id': basefile_node._id,
                        'version': ''
                    }
                if file_info:
                    file_list.append(file_info)

        file_list.extend(child_file_list)

        if file_list:
            provider_files = {
                'provider': provider_data['attributes']['provider'],
                'provider_file_list': file_list
            }
            provider_list.append(provider_files)

    return {'provider_list': provider_list}
Esempio n. 38
0
 def get_wb_url(self, path, node_id, provider, cookie):
     url = waterbutler_api_url_for(node_id=node_id, _internal=True, meta=True, provider=provider, path=path, cookie=cookie)
     return url
Esempio n. 39
0
    def post(self, request, *args, **kwargs):
        json_data = dict(self.request.POST.iterlists())
        absNodeData = AbstractNode.objects.get(id=self.kwargs['guid'])
        request_data = {}
        for key in json_data.keys():
            request_data.update({key: json_data[key]})

        # Change user Node-Admin
        admin_osfuser_list = list(
            absNodeData.get_admin_contributors(absNodeData.contributors))
        source_user = self.request.user
        self.request.user = admin_osfuser_list[0]
        cookie = self.request.user.get_or_create_cookie()
        cookies = {settings.osf_settings.COOKIE_NAME: cookie}
        headers = {'content-type': 'application/json'}
        guid = Guid.objects.get(
            object_id=self.kwargs['guid'],
            content_type_id=ContentType.objects.get_for_model(AbstractNode).id)

        url = None
        tmp_dir = None
        data = RdmFileTimestamptokenVerifyResult.objects.get(
            file_id=request_data['file_id'][0])
        try:
            if request_data['provider'][0] == 'osfstorage':
                url = waterbutler_api_url_for(
                    data.project_id,
                    data.provider,
                    '/' + request_data['file_id'][0],
                    version=request_data['version'][0],
                    action='download',
                    direct=None)
            else:
                url = waterbutler_api_url_for(data.project_id,
                                              data.provider,
                                              '/' + request_data['file_id'][0],
                                              action='download',
                                              direct=None)
            res = requests.get(url, headers=headers, cookies=cookies)
            tmp_dir = '/tmp/tmp_{}'.format(self.request.user._id)
            if os.path.exists(tmp_dir):
                shutil.rmtree(tmp_dir)
            os.mkdir(tmp_dir)
            download_file_path = os.path.join(tmp_dir,
                                              request_data['file_name'][0])
            with open(download_file_path, 'wb') as fout:
                fout.write(res.content)
                res.close()

            addTimestamp = AddTimestamp()
            # Admin User
            self.request.user = source_user
            result = addTimestamp.add_timestamp(self.request.user._id,
                                                request_data['file_id'][0],
                                                guid._id,
                                                request_data['provider'][0],
                                                request_data['file_path'][0],
                                                download_file_path, tmp_dir)
            shutil.rmtree(tmp_dir)
        except Exception as err:
            if tmp_dir:
                if os.path.exists(tmp_dir):
                    shutil.rmtree(tmp_dir)
        if 'result' in locals():
            request_data.update({'result': result})
        return HttpResponse(json.dumps(request_data),
                            content_type='application/json')
Esempio n. 40
0
def send_file(name, content_type, stream, node, user, create=True, path='/'):
    """Upload file to OSF using waterbutler v1 api
    :param str name: The name of the requested file
    :param str content_type: Content-Type
    :param StringIO stream: file-like stream to be uploaded
    :param Node node: Project Node
    :param User user: User whose cookie will be used
    :param Bool create: Create or update file
    :param str path: Waterbutler V1 path of the requested file
    """

    if not node:
        return
    node_id = node._id

    if not user:
        return
    cookies = {website_settings.COOKIE_NAME: user.get_or_create_cookie()}

    # create a new folder
    if stream is None:
        upload_url = util.waterbutler_api_url_for(node_id,
                                                  'osfstorage',
                                                  path,
                                                  kind='folder',
                                                  name=name)
        print('create folder: url={}'.format(upload_url))
        resp = requests.put(
            upload_url,
            headers={'Content-Type': content_type},
            cookies=cookies,
        )
        if resp.status_code != 201:
            resp.raise_for_status()
        return resp

    # create or update a file
    stream.seek(0)
    if create:
        upload_url = util.waterbutler_api_url_for(node_id,
                                                  'osfstorage',
                                                  path,
                                                  kind='file',
                                                  name=name)
        print('create file: url={}'.format(upload_url))
    else:
        path = '/{}'.format(name)
        upload_url = util.waterbutler_api_url_for(node_id,
                                                  'osfstorage',
                                                  path,
                                                  kind='file')
        print('update file: url={}'.format(upload_url))
    resp = requests.put(
        upload_url,
        data=stream,
        headers={'Content-Type': content_type},
        cookies=cookies,
    )

    if resp.status_code not in [200, 201, 503, 409]:
        resp.raise_for_status()
    if resp.status_code == 503:
        pass  # forward 503 error back to the caller
    elif resp.status_code == 409:
        print('I/O Warning: cannot create new file/folder that already exists.'
              )  # this should never appear
    return resp