def should_create_file(): s = Mock(spec=session.Session) s.get = Mock() s.get.side_effect = [session.DataDict({'type': 'Project', '_id': make_uuid(), 'links': session.DataDict({'self': sentinel.parent_self})})] s.post = Mock() file = {'entities': [{'type': 'File', '_id': make_uuid()}]} s.post.return_value = file expected_name = 'file name' core.create_file(s, make_uuid(), expected_name) s.post.assert_called_once_with(sentinel.parent_self, data={'entities': [{'type': 'File', 'attributes': {'name': expected_name}}]})
def upload_file(session, parent, local_path_or_file_obj, name=None, content_type=None, progress=tqdm): """ Upload a file to Ovation :param session: Session :param parent: Project or Folder root :param local_path_or_file_obj: local path to file :param progress: if not None, wrap in a progress (i.e. tqdm). Default: tqdm :param content_type: revision content type (default: infer from file name) :param name: file name (default: local path basename) :return: created File entity dictionary """ if name is None: name = os.path.basename(local_path_or_file_obj) file = core.create_file(session, parent, name) return upload_revision(session, file, local_path_or_file_obj, content_type=content_type, file_name=name, progress=progress)
def copy_file(session, parent=None, file_key=None, file_name=None, source_bucket=None, destination_bucket=None, aws_access_key_id=None, aws_secret_access_key=None): """ Creates an Ovation 'File' and 'Revision' record. File is transferred from source_bucket to destination_bucket and then the Revision` version is set to S3 version_id of the object in the destination_bucket. :param session: ovation.connection.Session :param parent: Project or Folder (entity dict or ID) :param file_key: S3 Key of file to copy from source bucket :param file_name: Name of file :param source_bucket: the S3 bucket to copy file from :param destination_bucket: the destination_bucket to copy file to :param aws_access_key_id: id for key that must have read access to source and write access to destination :param aws_secret_access_key: AWS key :return: new `Revision` entity dicitonary """ content_type = upload.guess_content_type(file_name) # create file record new_file = core.create_file(session, parent, file_name, attributes={'original_s3_path': file_key}) # create revision record r = session.post(new_file.links.self, data={'entities': [{'type': 'Revision', 'attributes': {'name': file_name, 'content_type': content_type}}]}) # get key for where to copy existing file from create revision resonse revision = r['entities'][0] destination_s3_key = r['aws'][0]['aws']['key'] # Copy file from source bucket to destination location # Need to use key that has 'list' and 'get_object' privleges on source bucket as well as 'write' privleges # on destination bucket (can't use the temp aws key from the create revision response since this does # not have read access to the source bucket) aws_session = boto3.Session(aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key) s3 = aws_session.resource('s3') destination_file = s3.Object(destination_bucket, destination_s3_key) copy_source = "{0}/{1}".format(source_bucket, file_key) try: aws_response = destination_file.copy_from(CopySource=copy_source) # get version_id from AWS copy response and update revision record with aws version_id # revision['attributes']['version'] = aws_response['VersionId'] revision_response = session.put(revision['links']['upload-complete'], entity=None) return revision_response except Exception as e: logging.error(e, exc_info=True) return None
def upload_folder(session, parent, directory_path, progress=tqdm): """ Recursively uploads a folder to Ovation :param session: Session :param parent: Project or Folder root :param directory_path: local path to directory :param progress: if not None, wrap in a progress (i.e. tqdm). Default: tqdm """ root_folder = parent for root, dirs, files in os.walk(directory_path): root_name = os.path.basename(root) if len(root_name) == 0: root_name = os.path.basename(os.path.dirname(root)) root_folder = core.create_folder(session, root_folder, root_name) for f in files: path = os.path.join(root, f) file = core.create_file(session, root_folder, f) upload_revision(session, file, path, progress=progress)