def should_create_file():
s = Mock(spec=session.Session)
s.get = Mock()
s.get.side_effect = [session.DataDict({'type': 'Project',
'_id': make_uuid(),
'links': session.DataDict({'self': sentinel.parent_self})})]
s.post = Mock()
file = {'entities': [{'type': 'File',
'_id': make_uuid()}]}
s.post.return_value = file
expected_name = 'file name'
core.create_file(s, make_uuid(), expected_name)
s.post.assert_called_once_with(sentinel.parent_self, data={'entities': [{'type': 'File',
'attributes': {'name': expected_name}}]})
def upload_file(session,
parent,
local_path_or_file_obj,
name=None,
content_type=None,
progress=tqdm):
"""
Upload a file to Ovation
:param session: Session
:param parent: Project or Folder root
:param local_path_or_file_obj: local path to file
:param progress: if not None, wrap in a progress (i.e. tqdm). Default: tqdm
:param content_type: revision content type (default: infer from file name)
:param name: file name (default: local path basename)
:return: created File entity dictionary
"""
if name is None:
name = os.path.basename(local_path_or_file_obj)
file = core.create_file(session, parent, name)
return upload_revision(session,
file,
local_path_or_file_obj,
content_type=content_type,
file_name=name,
progress=progress)
def copy_file(session, parent=None, file_key=None, file_name=None, source_bucket=None,
destination_bucket=None, aws_access_key_id=None, aws_secret_access_key=None):
"""
Creates an Ovation 'File' and 'Revision' record. File is transferred from
source_bucket to destination_bucket and then the Revision` version is set to S3 version_id
of the object in the destination_bucket.
:param session: ovation.connection.Session
:param parent: Project or Folder (entity dict or ID)
:param file_key: S3 Key of file to copy from source bucket
:param file_name: Name of file
:param source_bucket: the S3 bucket to copy file from
:param destination_bucket: the destination_bucket to copy file to
:param aws_access_key_id: id for key that must have read access to source and write access to destination
:param aws_secret_access_key: AWS key
:return: new `Revision` entity dicitonary
"""
content_type = upload.guess_content_type(file_name)
# create file record
new_file = core.create_file(session, parent, file_name,
attributes={'original_s3_path': file_key})
# create revision record
r = session.post(new_file.links.self,
data={'entities': [{'type': 'Revision',
'attributes': {'name': file_name,
'content_type': content_type}}]})
# get key for where to copy existing file from create revision resonse
revision = r['entities'][0]
destination_s3_key = r['aws'][0]['aws']['key']
# Copy file from source bucket to destination location
# Need to use key that has 'list' and 'get_object' privleges on source bucket as well as 'write' privleges
# on destination bucket (can't use the temp aws key from the create revision response since this does
# not have read access to the source bucket)
aws_session = boto3.Session(aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key)
s3 = aws_session.resource('s3')
destination_file = s3.Object(destination_bucket, destination_s3_key)
copy_source = "{0}/{1}".format(source_bucket, file_key)
try:
aws_response = destination_file.copy_from(CopySource=copy_source)
# get version_id from AWS copy response and update revision record with aws version_id
# revision['attributes']['version'] = aws_response['VersionId']
revision_response = session.put(revision['links']['upload-complete'], entity=None)
return revision_response
except Exception as e:
logging.error(e, exc_info=True)
return None
def upload_folder(session, parent, directory_path, progress=tqdm):
"""
Recursively uploads a folder to Ovation
:param session: Session
:param parent: Project or Folder root
:param directory_path: local path to directory
:param progress: if not None, wrap in a progress (i.e. tqdm). Default: tqdm
"""
root_folder = parent
for root, dirs, files in os.walk(directory_path):
root_name = os.path.basename(root)
if len(root_name) == 0:
root_name = os.path.basename(os.path.dirname(root))
root_folder = core.create_folder(session, root_folder, root_name)
for f in files:
path = os.path.join(root, f)
file = core.create_file(session, root_folder, f)
upload_revision(session, file, path, progress=progress)
