def find_or_create_folder(entities, folder_path, project, session, checkpoint): with open(checkpoint, 'a') as checkpoint_file: checkpoint_writer = csv.DictWriter(checkpoint_file, fieldnames=FIELDNAMES) folder_list = folder_path.split('/')[:-1] # Drop trailing if folder_path not in entities: logging.info('Creating folder: ' + folder_path) # e.g. current_folder --> 'Folder3' current_folder = folder_list[-1] parent_folder_path = '/'.join(folder_list[:-1]) + '/' # if nested folder if len(folder_list) > 1: logging.info('Folder parent_folder is: ' + parent_folder_path) parent = find_or_create_folder(entities, parent_folder_path, project, session, checkpoint) else: logging.info('Folder parent_folder is: project') parent = project found_or_created_folder = core.create_folder(session, parent, current_folder, attributes={'original_s3_path': folder_path}) entities[folder_path] = found_or_created_folder['_id'] logging.info('Recording new folder to checkpoint file: ' + folder_path) checkpoint_writer.writerow({PATH: folder_path, ID: found_or_created_folder['_id']}) else: logging.info(folder_path + ' already in entities') found_or_created_folder = entities[folder_path] return found_or_created_folder
def should_create_folder(): s = Mock(spec=session.Session) s.get = Mock() s.get.side_effect = [session.DataDict({'type': 'Project', '_id': make_uuid(), 'links': session.DataDict({'self': sentinel.parent_self})})] s.post = Mock() folder = {'entities': [{'type': 'Folder', '_id': make_uuid()}]} s.post.return_value = folder expected_name = 'folder name' core.create_folder(s, make_uuid(), expected_name) s.post.assert_called_once_with(sentinel.parent_self, data={'entities': [{'type': 'Folder', 'attributes': {'name': expected_name}}]})
def upload_folder(session, parent, directory_path, progress=tqdm): """ Recursively uploads a folder to Ovation :param session: Session :param parent: Project or Folder root :param directory_path: local path to directory :param progress: if not None, wrap in a progress (i.e. tqdm). Default: tqdm """ root_folder = parent for root, dirs, files in os.walk(directory_path): root_name = os.path.basename(root) if len(root_name) == 0: root_name = os.path.basename(os.path.dirname(root)) root_folder = core.create_folder(session, root_folder, root_name) for f in files: path = os.path.join(root, f) file = core.create_file(session, root_folder, f) upload_revision(session, file, path, progress=progress)