def waterbutler_folder_file_info(pid, provider, path, node, cookies, headers):
# get waterbutler folder file
if provider == 'osfstorage':
waterbutler_meta_url = util.waterbutler_api_url_for(
pid, provider,
'/' + path,
**dict(waterbutler_meta_parameter())
)
else:
waterbutler_meta_url = util.waterbutler_api_url_for(
pid, provider,
path,
**dict(waterbutler_meta_parameter())
)
waterbutler_res = requests.get(waterbutler_meta_url, headers=headers, cookies=cookies)
waterbutler_json_res = waterbutler_res.json()
waterbutler_res.close()
file_list = []
child_file_list = []
for file_data in waterbutler_json_res['data']:
if file_data['attributes']['kind'] == 'folder':
child_file_list.extend(waterbutler_folder_file_info(
pid, provider, file_data['attributes']['path'],
node, cookies, headers))
else:
basefile_node = BaseFileNode.resolve_class(
provider,
BaseFileNode.FILE
).get_or_create(
node,
file_data['attributes']['path']
)
basefile_node.save()
if provider == 'osfstorage':
file_info = {
'file_name': file_data['attributes']['name'],
'file_path': file_data['attributes']['materialized'],
'file_kind': file_data['attributes']['kind'],
'file_id': basefile_node._id,
'version': file_data['attributes']['extra']['version']
}
else:
file_info = {
'file_name': file_data['attributes']['name'],
'file_path': file_data['attributes']['materialized'],
'file_kind': file_data['attributes']['kind'],
'file_id': basefile_node._id,
'version': ''
}
file_list.append(file_info)
file_list.extend(child_file_list)
return file_list
def send_file(name, content_type, stream, node, user, create=True, path='/'):
"""Upload file to OSF using waterbutler v1 api
:param str name: The name of the requested file
:param str content_type: Content-Type
:param StringIO stream: file-like stream to be uploaded
:param Node node: Project Node
:param User user: User whose cookie will be used
:param Bool create: Create or update file
:param str path: Waterbutler V1 path of the requested file
"""
if not node:
return
node_id = node._id
if not user:
return
cookies = {website_settings.COOKIE_NAME:user.get_or_create_cookie()}
# create a new folder
if stream is None:
upload_url = util.waterbutler_api_url_for(node_id, 'osfstorage', path, kind='folder', name=name)
print('create folder: url={}'.format(upload_url))
resp = requests.put(
upload_url,
headers={'Content-Type': content_type},
cookies=cookies,
)
if resp.status_code != 201:
resp.raise_for_status()
return resp
# create or update a file
stream.seek(0)
if create:
upload_url = util.waterbutler_api_url_for(node_id, 'osfstorage', path, kind='file', name=name)
print('create file: url={}'.format(upload_url))
else:
path = '/{}'.format(name)
upload_url = util.waterbutler_api_url_for(node_id, 'osfstorage', path, kind='file')
print('update file: url={}'.format(upload_url))
resp = requests.put(
upload_url,
data=stream,
headers={'Content-Type': content_type},
cookies=cookies,
)
if resp.status_code not in [200, 201, 503, 409]:
resp.raise_for_status()
if resp.status_code == 503:
pass # forward 503 error back to the caller
elif resp.status_code == 409:
print('I/O Warning: cannot create new file/folder that already exists.') # this should never appear
return resp
def _get_fileobj_child_metadata(self,
filenode,
user,
cookie=None,
version=None):
kwargs = {}
if version:
kwargs['version'] = version
if cookie:
kwargs['cookie'] = cookie
elif user:
kwargs['cookie'] = user.get_or_create_cookie()
metadata_url = waterbutler_api_url_for(self.owner._id,
self.config.short_name,
path=filenode.get('path', '/'),
user=user,
view_only=True,
_internal=True,
**kwargs)
res = requests.get(metadata_url)
if res.status_code != 200:
raise HTTPError(res.status_code, data={'error': res.json()})
# TODO: better throttling?
time.sleep(1.0 / 5.0)
data = res.json().get('data', None)
if data:
return [child['attributes'] for child in data]
return []
def test_waterbutler_api_url_for(self):
with self.app.test_request_context():
url = waterbutler_api_url_for('fakeid', 'provider', '/path')
assert_in('/fakeid/', url)
assert_in('/path', url)
assert_in('/providers/provider/', url)
assert_in(settings.WATERBUTLER_URL, url)
def recursive_upload(dst, fs_path, parent, name=None):
is_folder = os.path.isdir(fs_path)
name = name or fs_path.rstrip('/')[-1]
params = {
'cookie': dst.creator.get_or_create_cookie(),
'kind': 'folder' if is_folder else 'file',
'name': name
}
url = waterbutler_api_url_for(dst._id,
'osfstorage',
parent.path,
_internal=True,
**params)
logger.info('Preparing to upload {} {}'.format(params['kind'],
params['name']))
if not DRY_RUN:
if is_folder:
resp = requests.put(url)
assert resp.status_code == 201
logger.info('Folder upload complete')
new_filenode = parent.children.get(name=name)
for child in os.listdir(fs_path):
recursive_upload(dst, child, new_filenode)
else:
with open(fs_path, 'r+') as fp:
resp = requests.put(url, data=fp)
assert resp.status_code == 201
logger.info('File upload complete')
def test_waterbutler_api_url_for(self):
with self.app.test_request_context():
url = waterbutler_api_url_for('fakeid', 'provider', '/path')
assert_in('/fakeid/', url)
assert_in('/path', url)
assert_in('/providers/provider/', url)
assert_in(settings.WATERBUTLER_URL, url)
def test_get_files_has_links(self, app, user, url):
res = app.get(url, auth=user.auth)
file_detail_json = res.json['data'][0]
quickfiles_node = quickfiles(user)
waterbutler_url = website_utils.waterbutler_api_url_for(
quickfiles_node._id,
'osfstorage',
file_detail_json['attributes']['path']
)
assert 'delete' in file_detail_json['links']
assert file_detail_json['links']['delete'] == waterbutler_url
assert 'download' in file_detail_json['links']
assert file_detail_json['links']['download'] == waterbutler_url
assert 'info' in file_detail_json['links']
assert 'move' in file_detail_json['links']
assert file_detail_json['links']['move'] == waterbutler_url
assert 'self' in file_detail_json['links']
assert 'upload' in file_detail_json['links']
assert file_detail_json['links']['upload'] == waterbutler_url
def _get_fileobj_child_metadata(self, filenode, user, cookie=None, version=None):
kwargs = {}
if version:
kwargs['version'] = version
if cookie:
kwargs['cookie'] = cookie
elif user:
kwargs['cookie'] = user.get_or_create_cookie()
metadata_url = waterbutler_api_url_for(
self.owner._id,
self.config.short_name,
path=filenode.get('path', '/'),
user=user,
view_only=True,
_internal=True,
**kwargs
)
res = requests.get(metadata_url)
if res.status_code != 200:
raise HTTPError(res.status_code, data={'error': res.json()})
# TODO: better throttling?
time.sleep(1.0 / 5.0)
data = res.json().get('data', None)
if data:
return [child['attributes'] for child in data]
return []
def get_file_object(node, path, provider, request):
if provider == 'osfstorage':
# Kinda like /me for a user
# The one odd case where path is not really path
if path == '/':
obj = node.get_addon('osfstorage').get_root()
else:
obj = get_object_or_error(
OsfStorageFileNode,
Q('node', 'eq', node._id) &
Q('_id', 'eq', path.strip('/')) &
Q('is_file', 'eq', not path.endswith('/'))
)
return obj
url = waterbutler_api_url_for(node._id, provider, path, meta=True)
waterbutler_request = requests.get(
url,
cookies=request.COOKIES,
headers={'Authorization': request.META.get('HTTP_AUTHORIZATION')},
)
if waterbutler_request.status_code == 401:
raise PermissionDenied
if waterbutler_request.status_code == 404:
raise NotFound
if is_server_error(waterbutler_request.status_code):
raise ServiceUnavailableError(detail='Could not retrieve files information at this time.')
try:
return waterbutler_request.json()['data']
except KeyError:
raise ServiceUnavailableError(detail='Could not retrieve files information at this time.')
def generate_waterbutler_url(self, **kwargs):
return util.waterbutler_api_url_for(
self.node._id,
self.provider,
self.path,
**kwargs
)
def generate_waterbutler_url(self, **kwargs):
return waterbutler_api_url_for(
self.node._id,
self.provider,
self.path,
**kwargs
)
def perform_wb_copy(reg, node_settings):
src, dst, user = reg.archive_job.info()
if dst.files.filter(
name=node_settings.archive_folder_name.replace('/', '-')).exists():
if not DELETE_COLLISIONS and not SKIP_COLLISIONS:
raise Exception(
'Archive folder for {} already exists. Investigate manually and rerun with either --delete-collisions or --skip-collisions'
)
if DELETE_COLLISIONS:
archive_folder = dst.files.exclude(type='osf.trashedfolder').get(
name=node_settings.archive_folder_name.replace('/', '-'))
logger.info('Removing {}'.format(archive_folder))
archive_folder.delete()
if SKIP_COLLISIONS:
complete_archive_target(reg, node_settings.short_name)
return
params = {'cookie': user.get_or_create_cookie()}
data = {
'action': 'copy',
'path': '/',
'rename': node_settings.archive_folder_name.replace('/', '-'),
'resource': dst._id,
'provider': ARCHIVE_PROVIDER,
}
url = waterbutler_api_url_for(src._id,
node_settings.short_name,
_internal=True,
**params)
res = requests.post(url, data=json.dumps(data))
if res.status_code not in (http.OK, http.CREATED, http.ACCEPTED):
raise HTTPError(res.status_code)
def export_files(node, user, current_dir):
"""
Creates a "files" directory within the current directory.
Exports all of the OSFStorage files for a given node.
Uses WB's download zip functionality to download osfstorage-archive.zip in a single request.
"""
files_dir = os.path.join(current_dir, 'files')
os.mkdir(files_dir)
response = requests.get(
url=waterbutler_api_url_for(
node_id=node._id,
_internal=True,
provider='osfstorage',
zip='',
cookie=user.get_or_create_cookie()
)
)
if response.status_code == 200:
with open(os.path.join(files_dir, 'osfstorage-archive.zip'), 'wb') as f:
f.write(response.content)
else:
ERRORS.append(
'Error exporting files for node {}. Waterbutler responded with a {} status code. Response: {}'
.format(node._id, response.status_code, response.json())
)
def archive_addon(addon_short_name, job_pk, stat_result):
"""Archive the contents of an addon by making a copy request to the
WaterBulter API
:param addon_short_name: AddonConfig.short_name of the addon to be archived
:param job_pk: primary key of ArchiveJob
:return: None
"""
create_app_context()
job = ArchiveJob.load(job_pk)
src, dst, user = job.info()
logger.info('Archiving addon: {0} on node: {1}'.format(addon_short_name, src._id))
cookie = user.get_or_create_cookie()
params = {'cookie': cookie}
rename_suffix = ''
# The dataverse API will not differentiate between published and draft files
# unless expcicitly asked. We need to create seperate folders for published and
# draft in the resulting archive.
#
# Additionally trying to run the archive without this distinction creates a race
# condition that non-deterministically caused archive jobs to fail.
if 'dataverse' in addon_short_name:
params['revision'] = 'latest' if addon_short_name.split('-')[-1] == 'draft' else 'latest-published'
rename_suffix = ' (draft)' if addon_short_name.split('-')[-1] == 'draft' else ' (published)'
addon_short_name = 'dataverse'
src_provider = src.get_addon(addon_short_name)
folder_name = src_provider.archive_folder_name
rename = '{}{}'.format(folder_name, rename_suffix)
url = waterbutler_api_url_for(src._id, addon_short_name, _internal=True, **params)
data = make_waterbutler_payload(dst._id, rename)
make_copy_request.delay(job_pk=job_pk, url=url, data=data)
def get_comment(self, check_permissions=True):
pk = self.kwargs[self.comment_lookup_url_kwarg]
try:
comment = Comment.find_one(Q('_id', 'eq', pk) & Q('root_target', 'ne', None))
except NoResultsFound:
raise NotFound
# Deleted root targets still appear as tuples in the database and are included in
# the above query, requiring an additional check
if isinstance(comment.root_target.referent, TrashedFileNode):
comment.root_target = None
comment.save()
if comment.root_target is None:
raise NotFound
if isinstance(comment.root_target.referent, StoredFileNode):
root_target = comment.root_target
referent = root_target.referent
if referent.provider == 'osfstorage':
try:
StoredFileNode.find(
Q('node', 'eq', comment.node._id) &
Q('_id', 'eq', referent._id) &
Q('is_file', 'eq', True)
)
except NoResultsFound:
Comment.update(Q('root_target', 'eq', root_target), data={'root_target': None})
raise NotFound
else:
if referent.provider == 'dropbox':
# referent.path is the absolute path for the db file, but wb requires the relative path
referent = DropboxFile.load(referent._id)
url = waterbutler_api_url_for(comment.node._id, referent.provider, referent.path, meta=True)
waterbutler_request = requests.get(
url,
cookies=self.request.COOKIES,
headers={'Authorization': self.request.META.get('HTTP_AUTHORIZATION')},
)
if waterbutler_request.status_code == 401:
raise PermissionDenied
if waterbutler_request.status_code == 404:
Comment.update(Q('root_target', 'eq', root_target), data={'root_target': None})
raise NotFound
if is_server_error(waterbutler_request.status_code):
raise ServiceUnavailableError(detail='Could not retrieve files information at this time.')
try:
waterbutler_request.json()['data']
except KeyError:
raise ServiceUnavailableError(detail='Could not retrieve files information at this time.')
if check_permissions:
# May raise a permission denied
self.check_object_permissions(self.request, comment)
return comment
def create_object(name, content_type, node, user, stream=None, kind=None, path='/'):
"""Create an object (file/folder) OSF using WaterButler v1 API
:param str name: The name of the requested file
:param str content_type: Content-Type
:param StringIO stream: file-like stream to be uploaded
:param Node node: Project Node
:param User user: User whose cookie will be used
:param str path: Waterbutler V1 path of the requested file
"""
assert(kind != 'file' or stream != None)
node_id = node._id
cookies = {website_settings.COOKIE_NAME: user.get_or_create_cookie()}
# create or update a file
url = util.waterbutler_api_url_for(node_id, 'osfstorage', path)
resp = requests.get(url, cookies=cookies)
data = resp.json()['data']
existing = None
for item in data:
if item['attributes']['name'] == name:
existing = item
if stream:
stream.seek(0)
# create a new file/folder?
if not existing:
url = util.waterbutler_api_url_for(node_id, 'osfstorage', path, kind=kind, name=name)
resp = requests.put(
url,
data=stream,
headers={'Content-Type': content_type},
cookies=cookies,
)
elif kind == 'file':
url = util.waterbutler_api_url_for(node_id, 'osfstorage', existing['attributes']['path'], kind=kind)
resp = requests.put(
url,
data=stream,
headers={'Content-Type': content_type},
cookies=cookies,
)
else:
return existing
return resp.json()['data']
def prepare_mock_wb_response(
node=None,
provider='github',
files=None,
folder=True,
path='/',
method=httpretty.GET,
status_code=200
):
"""Prepare a mock Waterbutler response with httpretty.
:param Node node: Target node.
:param str provider: Addon provider
:param list files: Optional list of files. You can specify partial data; missing values
will have defaults.
:param folder: True if mocking out a folder response, False if a file response.
:param path: Waterbutler path, passed to waterbutler_api_url_for.
:param str method: HTTP method.
:param int status_code: HTTP status.
"""
node = node
files = files or []
wb_url = waterbutler_api_url_for(
node._id, provider=provider, path=path, meta=True)
default_file = {
u'contentType': None,
u'extra': {u'downloads': 0, u'version': 1},
u'kind': u'file',
u'modified': None,
u'name': u'NewFile',
u'path': u'/NewFile',
u'provider': provider,
u'size': None,
u'materialized': '/',
}
if len(files):
data = [dict(default_file, **each) for each in files]
else:
data = [default_file]
jsonapi_data = []
for datum in data:
jsonapi_data.append({'attributes': datum})
if not folder:
jsonapi_data = jsonapi_data[0]
body = json.dumps({
u'data': jsonapi_data
})
httpretty.register_uri(
method,
wb_url,
body=body,
status=status_code,
content_type='application/json'
)
def resolve_url(self, obj):
"""Reverse URL lookup for WaterButler routes
"""
if self.must_be_folder is True and not obj.path.endswith('/'):
return None
if self.must_be_file is True and obj.path.endswith('/'):
return None
return waterbutler_api_url_for(obj.node._id, obj.provider, obj.path, **self.kwargs)
def resolve_url(self, obj):
"""Reverse URL lookup for WaterButler routes
"""
if self.must_be_folder is True and not obj.path.endswith('/'):
return None
if self.must_be_file is True and obj.path.endswith('/'):
return None
return waterbutler_api_url_for(obj.node._id, obj.provider, obj.path, **self.kwargs)
def prepare_mock_wb_response(node=None,
provider='github',
files=None,
folder=True,
path='/',
method=httpretty.GET,
status_code=200):
"""Prepare a mock Waterbutler response with httpretty.
:param Node node: Target node.
:param str provider: Addon provider
:param list files: Optional list of files. You can specify partial data; missing values
will have defaults.
:param folder: True if mocking out a folder response, False if a file response.
:param path: Waterbutler path, passed to waterbutler_api_url_for.
:param str method: HTTP method.
:param int status_code: HTTP status.
"""
node = node
files = files or []
wb_url = waterbutler_api_url_for(node._id,
provider=provider,
path=path,
meta=True)
default_file = {
u'contentType': None,
u'extra': {
u'downloads': 0,
u'version': 1
},
u'kind': u'file',
u'modified': None,
u'name': u'NewFile',
u'path': u'/NewFile',
u'provider': provider,
u'size': None,
u'materialized': '/',
}
if len(files):
data = [dict(default_file, **each) for each in files]
else:
data = [default_file]
jsonapi_data = []
for datum in data:
jsonapi_data.append({'attributes': datum})
if not folder:
jsonapi_data = jsonapi_data[0]
body = json.dumps({u'data': jsonapi_data})
httpretty.register_uri(method,
wb_url,
body=body,
status=status_code,
content_type='application/json')
def test_waterbutler_api_url_for_internal(self):
settings.WATERBUTLER_INTERNAL_URL = 'http://1.2.3.4:7777'
with self.app.test_request_context():
url = waterbutler_api_url_for('fakeid', 'provider', '/path', _internal=True)
assert_not_in(settings.WATERBUTLER_URL, url)
assert_in(settings.WATERBUTLER_INTERNAL_URL, url)
assert_in('/fakeid/', url)
assert_in('/path', url)
assert_in('/providers/provider', url)
def test_files_relationship_upload(self, app, user_one):
url = '/{}users/{}/'.format(API_BASE, user_one._id)
res = app.get(url, auth=user_one)
quickfiles = QuickFilesNode.objects.get(creator=user_one)
user_json = res.json['data']
upload_url = user_json['relationships']['quickfiles']['links']['upload']['href']
waterbutler_upload = website_utils.waterbutler_api_url_for(
quickfiles._id, 'osfstorage')
assert upload_url == waterbutler_upload
def test_waterbutler_invalid_data_returns_503(self):
wb_url = waterbutler_api_url_for(self.project._id, provider='github', path='/', meta=True)
httpretty.register_uri(
httpretty.GET,
wb_url,
body=json.dumps({}),
status=400
)
url = '/{}nodes/{}/files/github/'.format(API_BASE, self.project._id)
res = self.app.get(url, auth=self.user.auth, expect_errors=True)
assert_equal(res.status_code, 503)
def test_files_relationship_upload(self, app, user_one):
url = "/{}users/{}/".format(API_BASE, user_one._id)
res = app.get(url, auth=user_one)
quickfiles = QuickFilesNode.objects.get(creator=user_one)
user_json = res.json['data']
upload_url = user_json['relationships']['quickfiles']['links'][
'upload']['href']
waterbutler_upload = website_utils.waterbutler_api_url_for(
quickfiles._id, 'osfstorage')
assert upload_url == waterbutler_upload
def get_comment(self, check_permissions=True):
pk = self.kwargs[self.comment_lookup_url_kwarg]
try:
comment = Comment.find_one(Q('_id', 'eq', pk) & Q('root_target', 'ne', None))
except NoResultsFound:
raise NotFound
# Deleted root targets still appear as tuples in the database and are included in
# the above query, requiring an additional check
if comment.root_target is None:
raise NotFound
if isinstance(comment.root_target, StoredFileNode):
root_target = comment.root_target
if root_target.provider == 'osfstorage':
try:
StoredFileNode.find(
Q('node', 'eq', comment.node._id) &
Q('_id', 'eq', root_target._id) &
Q('is_file', 'eq', True)
)
except NoResultsFound:
Comment.update(Q('root_target', 'eq', root_target), data={'root_target': None})
del comment.node.commented_files[root_target._id]
raise NotFound
else:
if root_target.provider == 'dropbox':
root_target = DropboxFile.load(comment.root_target._id)
url = waterbutler_api_url_for(comment.node._id, root_target.provider, root_target.path, meta=True)
waterbutler_request = requests.get(
url,
cookies=self.request.COOKIES,
headers={'Authorization': self.request.META.get('HTTP_AUTHORIZATION')},
)
if waterbutler_request.status_code == 401:
raise PermissionDenied
if waterbutler_request.status_code == 404:
Comment.update(Q('root_target', 'eq', root_target), data={'root_target': None})
raise NotFound
if is_server_error(waterbutler_request.status_code):
raise ServiceUnavailableError(detail='Could not retrieve files information at this time.')
try:
waterbutler_request.json()['data']
except KeyError:
raise ServiceUnavailableError(detail='Could not retrieve files information at this time.')
if check_permissions:
# May raise a permission denied
self.check_object_permissions(self.request, comment)
return comment
def test_handles_bad_waterbutler_request(self):
wb_url = waterbutler_api_url_for(self.project._id, provider='github', path='/', meta=True)
httpretty.register_uri(
httpretty.GET,
wb_url,
body=json.dumps({}),
status=418
)
url = '/{}nodes/{}/files/github/'.format(API_BASE, self.project._id)
res = self.app.get(url, auth=self.user.auth, expect_errors=True)
assert_equal(res.status_code, 503)
assert_in('detail', res.json['errors'][0])
def resolve_url(self, obj):
"""Reverse URL lookup for WaterButler routes
"""
if self.must_be_folder is True and not obj.path.endswith('/'):
raise SkipField
if self.must_be_file is True and obj.path.endswith('/'):
raise SkipField
url = website_utils.waterbutler_api_url_for(obj.node._id, obj.provider, obj.path, **self.kwargs)
if not url:
raise SkipField
else:
return url
def resolve_url(self, obj):
"""Reverse URL lookup for WaterButler routes
"""
if self.must_be_folder is True and not obj.path.endswith('/'):
raise SkipField
if self.must_be_file is True and obj.path.endswith('/'):
raise SkipField
url = website_utils.waterbutler_api_url_for(obj.node._id, obj.provider, obj.path, **self.kwargs)
if not url:
raise SkipField
else:
return url
def test_waterbutler_api_url_for_internal(self):
settings.WATERBUTLER_INTERNAL_URL = 'http://1.2.3.4:7777'
with self.app.test_request_context():
url = waterbutler_api_url_for('fakeid',
'provider',
'/path',
_internal=True)
assert_not_in(settings.WATERBUTLER_URL, url)
assert_in(settings.WATERBUTLER_INTERNAL_URL, url)
assert_in('/fakeid/', url)
assert_in('/path', url)
assert_in('/providers/provider', url)
def to_representation(self, value):
relationship_links = super(QuickFilesRelationshipField, self).to_representation(value)
quickfiles_guid = value.nodes_created.filter(type=QuickFilesNode._typedmodels_type).values_list('guids___id', flat=True).get()
upload_url = website_utils.waterbutler_api_url_for(quickfiles_guid, 'osfstorage')
relationship_links['links']['upload'] = {
'href': upload_url,
'meta': {}
}
relationship_links['links']['download'] = {
'href': '{}?zip='.format(upload_url),
'meta': {}
}
return relationship_links
def upload_attachment(user, node, attachment):
attachment.seek(0)
name = (attachment.filename or settings.MISSING_FILE_NAME)
content = attachment.read()
upload_url = util.waterbutler_api_url_for(
node._id,
'osfstorage',
name=name,
cookie=user.get_or_create_cookie(),
_internal=True)
requests.put(
upload_url,
data=content,
)
def get_file_object(node, path, provider, request):
# Don't bother going to waterbutler for osfstorage
if provider == 'osfstorage':
# Kinda like /me for a user
# The one odd case where path is not really path
if path == '/':
obj = node.get_addon('osfstorage').get_root()
else:
if path.endswith('/'):
model = OsfStorageFolder
else:
model = OsfStorageFile
obj = get_object_or_error(
model,
Q('node', 'eq', node.pk) & Q('_id', 'eq', path.strip('/')))
return obj
if not node.get_addon(provider) or not node.get_addon(provider).configured:
raise NotFound(
'The {} provider is not configured for this project.'.format(
provider))
url = waterbutler_api_url_for(node._id,
provider,
path,
_internal=True,
meta=True)
waterbutler_request = requests.get(
url,
cookies=request.COOKIES,
headers={'Authorization': request.META.get('HTTP_AUTHORIZATION')},
)
if waterbutler_request.status_code == 401:
raise PermissionDenied
if waterbutler_request.status_code == 404:
raise NotFound
if is_server_error(waterbutler_request.status_code):
raise ServiceUnavailableError(
detail='Could not retrieve files information at this time.')
try:
return waterbutler_request.json()['data']
except KeyError:
raise ServiceUnavailableError(
detail='Could not retrieve files information at this time.')
def get_queryset(self):
# Don't bother going to waterbutler for osfstorage
if self.kwargs['provider'] == 'osfstorage':
self.check_object_permissions(self.request, self.get_node())
# Kinda like /me for a user
# The one odd case where path is not really path
if self.kwargs['path'] == '/':
return list(self.get_node().get_addon('osfstorage').get_root().children)
fobj = OsfStorageFileNode.find_one(
Q('node', 'eq', self.get_node()._id) &
Q('path', 'eq', self.kwargs['path'])
)
if fobj.is_file:
return [fobj]
return list(fobj.children)
url = waterbutler_api_url_for(
self.get_node()._id,
self.kwargs['provider'],
self.kwargs['path'],
meta=True
)
waterbutler_request = requests.get(
url,
cookies=self.request.COOKIES,
headers={'Authorization': self.request.META.get('HTTP_AUTHORIZATION')},
)
if waterbutler_request.status_code == 401:
raise PermissionDenied
try:
files_list = waterbutler_request.json()['data']
except KeyError:
raise ValidationError(detail='detail: Could not retrieve files information.')
if isinstance(files_list, dict):
files_list = [files_list]
return [self.get_file_item(file) for file in files_list]
def get_file_object(node, path, provider, request):
# Don't bother going to waterbutler for osfstorage
if provider == 'osfstorage':
# Kinda like /me for a user
# The one odd case where path is not really path
if path == '/':
obj = node.get_addon('osfstorage').get_root()
else:
if path.endswith('/'):
model = OsfStorageFolder
else:
model = OsfStorageFile
obj = get_object_or_error(model, Q(node=node.pk, _id=path.strip('/')), request)
return obj
if not node.get_addon(provider) or not node.get_addon(provider).configured:
raise NotFound('The {} provider is not configured for this project.'.format(provider))
view_only = request.query_params.get('view_only', default=None)
url = waterbutler_api_url_for(node._id, provider, path, _internal=True,
meta=True, view_only=view_only)
waterbutler_request = requests.get(
url,
cookies=request.COOKIES,
headers={'Authorization': request.META.get('HTTP_AUTHORIZATION')},
)
if waterbutler_request.status_code == 401:
raise PermissionDenied
if waterbutler_request.status_code == 404:
raise NotFound
if is_server_error(waterbutler_request.status_code):
raise ServiceUnavailableError(detail='Could not retrieve files information at this time.')
try:
return waterbutler_request.json()['data']
except KeyError:
raise ServiceUnavailableError(detail='Could not retrieve files information at this time.')
def test_get_files_has_links(self, app, user, url):
res = app.get(url, auth=user.auth)
file_detail_json = res.json['data'][0]
quickfiles_node = quickfiles(user)
waterbutler_url = website_utils.waterbutler_api_url_for(
quickfiles_node._id, 'osfstorage',
file_detail_json['attributes']['path'])
assert 'delete' in file_detail_json['links']
assert file_detail_json['links']['delete'] == waterbutler_url
assert 'download' in file_detail_json['links']
assert file_detail_json['links']['download'] == waterbutler_url
assert 'info' in file_detail_json['links']
assert 'move' in file_detail_json['links']
assert file_detail_json['links']['move'] == waterbutler_url
assert 'self' in file_detail_json['links']
assert 'upload' in file_detail_json['links']
assert file_detail_json['links']['upload'] == waterbutler_url
def perform_wb_copy(reg, node_settings):
src, dst, user = reg.archive_job.info()
if dst.files.filter(name=node_settings.archive_folder_name.replace('/', '-')).exists():
if not DELETE_COLLISIONS and not SKIP_COLLISIONS:
raise Exception('Archive folder for {} already exists. Investigate manually and rerun with either --delete-collisions or --skip-collisions')
if DELETE_COLLISIONS:
archive_folder = dst.files.exclude(type='osf.trashedfolder').get(name=node_settings.archive_folder_name.replace('/', '-'))
logger.info('Removing {}'.format(archive_folder))
archive_folder.delete()
if SKIP_COLLISIONS:
complete_archive_target(reg, node_settings.short_name)
return
params = {'cookie': user.get_or_create_cookie()}
data = {
'action': 'copy',
'path': '/',
'rename': node_settings.archive_folder_name.replace('/', '-'),
'resource': dst._id,
'provider': ARCHIVE_PROVIDER,
}
url = waterbutler_api_url_for(src._id, node_settings.short_name, _internal=True, **params)
res = requests.post(url, data=json.dumps(data))
if res.status_code not in (http.OK, http.CREATED, http.ACCEPTED):
raise HTTPError(res.status_code)
def collect_timestamp_trees_to_json(auth, node, **kwargs):
# admin call project to provider file list
serialized = _view_project(node, auth, primary=True)
serialized.update(rubeus.collect_addon_assets(node))
user_info = OSFUser.objects.get(id=Guid.objects.get(_id=serialized['user']['id']).object_id)
api_url = util.api_v2_url(api_url_path(kwargs.get('pid')))
cookie = user_info.get_or_create_cookie()
cookies = {settings.COOKIE_NAME: cookie}
headers = {'content-type': 'application/json'}
provider_json_res = None
file_res = requests.get(api_url, headers=headers, cookies=cookies)
provider_json_res = file_res.json()
file_res.close()
provider_list = []
for provider_data in provider_json_res['data']:
waterbutler_meta_url = util.waterbutler_api_url_for(
kwargs.get('pid'),
provider_data['attributes']['provider'],
'/',
**dict(waterbutler_meta_parameter())
)
waterbutler_json_res = None
waterbutler_res = requests.get(waterbutler_meta_url, headers=headers, cookies=cookies)
waterbutler_json_res = waterbutler_res.json()
waterbutler_res.close()
file_list = []
child_file_list = []
for file_data in waterbutler_json_res['data']:
if file_data['attributes']['kind'] == 'folder':
child_file_list.extend(
waterbutler_folder_file_info(
kwargs.get('pid'),
provider_data['attributes']['provider'],
file_data['attributes']['path'],
node, cookies, headers
)
)
else:
file_info = None
basefile_node = BaseFileNode.resolve_class(
provider_data['attributes']['provider'],
BaseFileNode.FILE
).get_or_create(
node,
file_data['attributes']['path']
)
basefile_node.save()
if provider_data['attributes']['provider'] == 'osfstorage':
file_info = {
'file_name': file_data['attributes']['name'],
'file_path': file_data['attributes']['materialized'],
'file_kind': file_data['attributes']['kind'],
'file_id': basefile_node._id,
'version': file_data['attributes']['extra']['version']
}
else:
file_info = {
'file_name': file_data['attributes']['name'],
'file_path': file_data['attributes']['materialized'],
'file_kind': file_data['attributes']['kind'],
'file_id': basefile_node._id,
'version': ''
}
if file_info:
file_list.append(file_info)
file_list.extend(child_file_list)
if file_list:
provider_files = {
'provider': provider_data['attributes']['provider'],
'provider_file_list': file_list
}
provider_list.append(provider_files)
return {'provider_list': provider_list}
def get_wb_url(self, path, node_id, provider, cookie):
url = waterbutler_api_url_for(node_id=node_id, _internal=True, meta=True, provider=provider, path=path, cookie=cookie)
return url
def post(self, request, *args, **kwargs):
json_data = dict(self.request.POST.iterlists())
absNodeData = AbstractNode.objects.get(id=self.kwargs['guid'])
request_data = {}
for key in json_data.keys():
request_data.update({key: json_data[key]})
# Change user Node-Admin
admin_osfuser_list = list(
absNodeData.get_admin_contributors(absNodeData.contributors))
source_user = self.request.user
self.request.user = admin_osfuser_list[0]
cookie = self.request.user.get_or_create_cookie()
cookies = {settings.osf_settings.COOKIE_NAME: cookie}
headers = {'content-type': 'application/json'}
guid = Guid.objects.get(
object_id=self.kwargs['guid'],
content_type_id=ContentType.objects.get_for_model(AbstractNode).id)
url = None
tmp_dir = None
data = RdmFileTimestamptokenVerifyResult.objects.get(
file_id=request_data['file_id'][0])
try:
if request_data['provider'][0] == 'osfstorage':
url = waterbutler_api_url_for(
data.project_id,
data.provider,
'/' + request_data['file_id'][0],
version=request_data['version'][0],
action='download',
direct=None)
else:
url = waterbutler_api_url_for(data.project_id,
data.provider,
'/' + request_data['file_id'][0],
action='download',
direct=None)
res = requests.get(url, headers=headers, cookies=cookies)
tmp_dir = '/tmp/tmp_{}'.format(self.request.user._id)
if os.path.exists(tmp_dir):
shutil.rmtree(tmp_dir)
os.mkdir(tmp_dir)
download_file_path = os.path.join(tmp_dir,
request_data['file_name'][0])
with open(download_file_path, 'wb') as fout:
fout.write(res.content)
res.close()
addTimestamp = AddTimestamp()
# Admin User
self.request.user = source_user
result = addTimestamp.add_timestamp(self.request.user._id,
request_data['file_id'][0],
guid._id,
request_data['provider'][0],
request_data['file_path'][0],
download_file_path, tmp_dir)
shutil.rmtree(tmp_dir)
except Exception as err:
if tmp_dir:
if os.path.exists(tmp_dir):
shutil.rmtree(tmp_dir)
if 'result' in locals():
request_data.update({'result': result})
return HttpResponse(json.dumps(request_data),
content_type='application/json')
def send_file(name, content_type, stream, node, user, create=True, path='/'):
"""Upload file to OSF using waterbutler v1 api
:param str name: The name of the requested file
:param str content_type: Content-Type
:param StringIO stream: file-like stream to be uploaded
:param Node node: Project Node
:param User user: User whose cookie will be used
:param Bool create: Create or update file
:param str path: Waterbutler V1 path of the requested file
"""
if not node:
return
node_id = node._id
if not user:
return
cookies = {website_settings.COOKIE_NAME: user.get_or_create_cookie()}
# create a new folder
if stream is None:
upload_url = util.waterbutler_api_url_for(node_id,
'osfstorage',
path,
kind='folder',
name=name)
print('create folder: url={}'.format(upload_url))
resp = requests.put(
upload_url,
headers={'Content-Type': content_type},
cookies=cookies,
)
if resp.status_code != 201:
resp.raise_for_status()
return resp
# create or update a file
stream.seek(0)
if create:
upload_url = util.waterbutler_api_url_for(node_id,
'osfstorage',
path,
kind='file',
name=name)
print('create file: url={}'.format(upload_url))
else:
path = '/{}'.format(name)
upload_url = util.waterbutler_api_url_for(node_id,
'osfstorage',
path,
kind='file')
print('update file: url={}'.format(upload_url))
resp = requests.put(
upload_url,
data=stream,
headers={'Content-Type': content_type},
cookies=cookies,
)
if resp.status_code not in [200, 201, 503, 409]:
resp.raise_for_status()
if resp.status_code == 503:
pass # forward 503 error back to the caller
elif resp.status_code == 409:
print('I/O Warning: cannot create new file/folder that already exists.'
) # this should never appear
return resp