def osf_upload_keywords(token, resource_id, keywords): """ Upload the keywords to a given resource id. Parameters ---------- token: str User's OSF token resource_id: str ID of the resource requested keywords: list List of new keywords to upload Returns ------- A dictionary object that represents the updated OSF resource keywords. Dictionary must be in the following format: { "updated_keywords": [ 'eggs', 'EGG', 'Breakfast' ] } """ try: osf_instance = OSF(token) except PresQTInvalidTokenError: raise PresQTResponseException( "Token is invalid. Response returned a 401 status code.", status.HTTP_401_UNAUTHORIZED) headers = { 'Authorization': 'Bearer {}'.format(token), 'Content-Type': 'application/json' } resource = get_osf_resource(resource_id, osf_instance) if resource.kind_name == 'project': patch_url = 'https://api.osf.io/v2/nodes/{}/'.format(resource_id) data = { "data": { "type": "nodes", "id": resource_id, "attributes": { "tags": keywords } } } elif resource.kind_name == 'file': patch_url = 'https://api.osf.io/v2/files/{}/'.format(resource_id) data = { "data": { "type": "files", "id": resource_id, "attributes": { "tags": keywords } } } elif resource.kind_name == 'folder': patch_url = 'https://api.osf.io/v2/nodes/{}/'.format(resource_id) data = { "data": { "type": "nodes", "id": resource_id, "attributes": { "tags": keywords } } } response = requests.patch(patch_url, headers=headers, data=json.dumps(data)) if response.status_code != 200: raise PresQTResponseException( "OSF returned a {} error trying to update keywords.".format( response.status_code), status.HTTP_400_BAD_REQUEST) return { "updated_keywords": response.json()['data']['attributes']['tags'], "project_id": get_project_id(resource) }
def osf_fetch_keywords(token, resource_id): """ Fetch the keywords of a given resource id. Parameters ---------- token: str User's OSF token resource_id: str ID of the resource requested Returns ------- A dictionary object that represents the OSF resource keywords. Dictionary must be in the following format: { "tags": [ "eggs", "ham", "bacon" ], "keywords": [ "eggs", "ham", "bacon" ] } """ try: osf_instance = OSF(token) except PresQTInvalidTokenError: raise PresQTResponseException( "Token is invalid. Response returned a 401 status code.", status.HTTP_401_UNAUTHORIZED) header = {'Authorization': 'Bearer {}'.format(token)} resource = get_osf_resource(resource_id, osf_instance) if resource.kind_name == 'storage': raise PresQTResponseException( "On OSF only projects, folders and files have keywords, not storages, therefore PresQT keyword features are not supported at OSF's storage level.", status.HTTP_400_BAD_REQUEST) # Find out if metadata exists for this project project_id = get_project_id(resource) project_data = osf_instance._get_all_paginated_data( 'https://api.osf.io/v2/nodes/{}/files/osfstorage'.format(project_id)) metadata = None for data in project_data: if data['attributes']['name'] == "PRESQT_FTS_METADATA.json": metadata_file = requests.get(data['links']['move'], headers=header).content # Update the existing metadata metadata = json.loads(metadata_file) if metadata: try: keywords = list(set(resource.tags + metadata['allKeywords'])) except KeyError: keywords = list(set(resource.tags)) else: keywords = list(set(resource.tags)) return {'tags': keywords, 'keywords': keywords}
def osf_download_resource(token, resource_id): """ Fetch the requested resource from OSF along with its hash information. Parameters ---------- token : str User's OSF token resource_id : str ID of the resource requested Returns ------- Dictionary with the following keys: values 'resources': List of dictionary objects that each hold a file and its information. Dictionary must be in the following format: { 'file': binary_file, 'hashes': {'hash_algorithm': 'the_hash'}, 'title': 'file.jpg', 'path': '/path/to/file', 'source_path: '/full/path/to/file', 'extra_metadata': {'any': 'extra'} } 'empty_containers: List of string paths representing empty containers that must be written. Example: ['empty/folder/to/write/', 'another/empty/folder/] 'action_metadata': Dictionary containing action metadata. Must be in the following format: { 'sourceUsername': '******', } """ try: osf_instance = OSF(token) except PresQTInvalidTokenError: raise PresQTResponseException( "Token is invalid. Response returned a 401 status code.", status.HTTP_401_UNAUTHORIZED) # Get contributor name contributor_name = requests.get( 'https://api.osf.io/v2/users/me/', headers={ 'Authorization': 'Bearer {}'.format(token) }).json()['data']['attributes']['full_name'] action_metadata = {"sourceUsername": contributor_name} # Get the resource resource = get_osf_resource(resource_id, osf_instance) # Get all files for the provided resources. # The 'path' value will be the path that the file is eventually saved in. The root of the # path should be the resource. files = [] empty_containers = [] if resource.kind_name == 'file': project = osf_instance.project(resource.parent_project_id) files.append({ "file": resource.download(), "hashes": resource.hashes, "title": resource.title, # If the file is the only resource we are downloading then we don't need it's full path "path": '/{}'.format(resource.title), "source_path": '/{}/{}{}'.format(project.title, resource.provider, resource.materialized_path), "extra_metadata": osf_download_metadata(resource) }) else: if resource.kind_name == 'project': resource.get_all_files('', files, empty_containers) project = resource elif resource.kind_name == 'storage': resource.get_all_files('/{}'.format(resource.title), files, empty_containers) project = osf_instance.project(resource.node) else: resource.get_all_files('', files, empty_containers) project = osf_instance.project(resource.parent_project_id) for file in files: # File Path needs to start at the folder and strip everything before it. # Example: If the resource is 'Docs2' and the starting path is # '/Project/Storage/Docs1/Docs2/file.jpeg' then the final path # needs to be '/Docs2/file.jpeg' path_to_strip = resource.materialized_path[:-( len(resource.title) + 2)] file['path'] = file['file'].materialized_path[len(path_to_strip ):] # Asynchronously make all download requests file_urls = [file['file'].download_url for file in files] loop = asyncio.new_event_loop() download_data = loop.run_until_complete(async_main(file_urls, token)) # Go through the file dictionaries and replace the file class with the binary_content for file in files: file['source_path'] = '/{}/{}{}'.format( project.title, file['file'].provider, file['file'].materialized_path) file['file'] = get_dictionary_from_list( download_data, 'url', file['file'].download_url)['binary_content'] return { 'resources': files, 'empty_containers': empty_containers, 'action_metadata': action_metadata }
def osf_upload_resource(token, resource_id, resource_main_dir, hash_algorithm, file_duplicate_action, process_info_path, action): """ Upload the files found in the resource_main_dir to OSF. Parameters ---------- token : str User's OSF token. resource_id : str ID of the resource requested. resource_main_dir : str Path to the main directory for the resources to be uploaded. hash_algorithm : str Hash algorithm we are using to check for fixity. file_duplicate_action : str The action to take when a duplicate file is found process_info_path: str Path to the process info file that keeps track of the action's progress action: str The action being performed Returns ------- Dictionary with the following keys: values 'resources_ignored' : Array of string file paths of files that were ignored when uploading the resource. Path should have the same base as resource_main_dir. Example: ['path/to/ignored/file.pg', 'another/ignored/file.jpg'] 'resources_updated' : Array of string file paths of files that were updated when uploading the resource. Path should have the same base as resource_main_dir. Example: ['path/to/updated/file.jpg'] 'action_metadata': Dictionary containing FTS action metadata. Must be in the following format: { 'destinationUsername': '******' } 'file_metadata_list': List of dictionaries for each file that contains FTS metadata and hash info. Must be in the following format: { "actionRootPath": '/path/on/disk', "destinationPath": '/path/on/target/destination', "title": 'file_title', "destinationHash": {'hash_algorithm': 'the_hash'}} } 'project_id': ID of the parent project for this upload. Needed for metadata upload. 'project_link': The link to either the resource or the home page of the user if not available through API """ try: osf_instance = OSF(token) except PresQTInvalidTokenError: raise PresQTResponseException( "Token is invalid. Response returned a 401 status code.", status.HTTP_401_UNAUTHORIZED) # Get contributor name contributor_name = requests.get( 'https://api.osf.io/v2/users/me/', headers={ 'Authorization': 'Bearer {}'.format(token) }).json()['data']['attributes']['full_name'] action_metadata = {"destinationUsername": contributor_name} hashes = {} resources_ignored = [] resources_updated = [] file_metadata_list = [] # Get total amount of files total_files = upload_total_files(resource_main_dir) update_process_info(process_info_path, total_files, action, 'upload') update_process_info_message(process_info_path, action, "Uploading files to OSF...") # If we are uploading to an existing container if resource_id: # Get the resource resource = get_osf_resource(resource_id, osf_instance) # Resource being uploaded to must not be a file if resource.kind_name == 'file': raise PresQTResponseException( "The Resource provided, {}, is not a container".format( resource_id), status.HTTP_400_BAD_REQUEST) elif resource.kind_name == 'project': project = resource project_id = project.id resource.storage('osfstorage').create_directory( resource_main_dir, file_duplicate_action, hashes, resources_ignored, resources_updated, file_metadata_list, process_info_path, action) else: # Folder or Storage resource.create_directory(resource_main_dir, file_duplicate_action, hashes, resources_ignored, resources_updated, file_metadata_list, process_info_path, action) # Get the project class for later metadata work if resource.kind_name == 'storage': project_id = resource.node else: project_id = resource.parent_project_id project = osf_instance.project(project_id) # else we are uploading a new project else: os_path = next(os.walk(resource_main_dir)) # Get the actual data we want to upload data_to_upload_path = '{}/{}'.format(os_path[0], os_path[1][0]) # Create a new project with the name being the top level directory's name. project = osf_instance.create_project(os_path[1][0]) project_id = project.id # Upload resources into OSFStorage for the new project. project.storage('osfstorage').create_directory( data_to_upload_path, file_duplicate_action, hashes, resources_ignored, resources_updated, file_metadata_list, process_info_path, action) for file_metadata in file_metadata_list: # Only send forward the hash we need based on the hash_algorithm provided file_metadata['destinationHash'] = file_metadata['destinationHash'][ hash_algorithm] # Prepend the project title to each resource's the metadata destinationPath file_metadata['destinationPath'] = '/{}/{}'.format( project.title, file_metadata['destinationPath']) return { 'resources_ignored': resources_ignored, 'resources_updated': resources_updated, 'action_metadata': action_metadata, 'file_metadata_list': file_metadata_list, 'project_id': project_id, "project_link": "https://osf.io/{}".format(project_id) }
def osf_fetch_resource(token, resource_id): """ Fetch the OSF resource matching the resource_id given. Parameters ---------- token : str User's OSF token resource_id : str ID of the resource requested Returns ------- A dictionary object that represents the OSF resource. Dictionary must be in the following format: { "kind": "item", "kind_name": "file", "id": "12345", "title": "23296359282_934200ec59_o.jpg", "date_created": "2019-05-13T14:54:17.129170Z", "date_modified": "2019-05-13T14:54:17.129170Z", "hashes": { "md5": "aaca7ef067dcab7cb8d79c36243823e4", "sha256": "ea94ce54261720c16abb508c6dcd1fd481c30c09b7f2f5ab0b79e3199b7e2b55" }, "extra": { "any": "extra", "values": "here" } } """ try: osf_instance = OSF(token) except PresQTInvalidTokenError: raise PresQTResponseException( "Token is invalid. Response returned a 401 status code.", status.HTTP_401_UNAUTHORIZED) def create_object(resource_object): resource_object_obj = { 'kind': resource_object.kind, 'kind_name': resource_object.kind_name, 'id': resource_object.id, 'title': resource_object.title, 'date_created': resource_object.date_created, 'date_modified': resource_object.date_modified, 'hashes': { 'md5': resource_object.md5, 'sha256': resource_object.sha256 }, 'extra': {} } if resource_object.kind_name in ['folder', 'file']: resource_object_obj['extra'] = { 'last_touched': resource_object.last_touched, 'materialized_path': resource_object.materialized_path, 'current_version': resource_object.current_version, 'provider': resource_object.provider, 'path': resource_object.path, 'current_user_can_comment': resource_object.current_user_can_comment, 'guid': resource_object.guid, 'checkout': resource_object.checkout, 'tags': resource_object.tags, 'size': resource_object.size } elif resource_object.kind_name == 'project': resource_object_obj['extra'] = { 'category': resource_object.category, 'fork': resource_object.fork, 'current_user_is_contributor': resource_object.current_user_is_contributor, 'preprint': resource_object.preprint, 'current_user_permissions': resource_object.current_user_permissions, 'custom_citation': resource_object.custom_citation, 'collection': resource_object.collection, 'public': resource_object.public, 'subjects': resource_object.subjects, 'registration': resource_object.registration, 'current_user_can_comment': resource_object.current_user_can_comment, 'wiki_enabled': resource_object.wiki_enabled, 'node_license': resource_object.node_license, 'tags': resource_object.tags, 'size': resource_object.size } return resource_object_obj # Get the resource resource = get_osf_resource(resource_id, osf_instance) return create_object(resource)