def upload(args):
"""
Given a folder or file, upload all the folders and files contained
within it, skipping ones that already exist on the remote.
"""
base_remote_path, path_dict = Object.validate_full_path(
args.full_path, vault=args.vault, path=args.path)
# Assert the vault exists and is accessible
vault = Vault.get_by_full_path(path_dict['vault_full_path'])
# If not the vault root, validate remote path exists and is a folder
if path_dict['path'] != '/':
Object.get_by_full_path(base_remote_path, assert_type='folder')
for local_path in args.local_path:
local_path = local_path.rstrip('/')
local_start = os.path.basename(local_path)
if os.path.isdir(local_path):
_upload_folder(path_dict['domain'], vault,
base_remote_path, local_path, local_start)
else:
Object.upload_file(local_path, path_dict['path'],
vault.full_path)
def upload(args):
"""
Given a folder or file, upload all the folders and files contained
within it, skipping ones that already exist on the remote.
"""
base_remote_path, path_dict = Object.validate_full_path(args.full_path,
vault=args.vault,
path=args.path)
# Assert the vault exists and is accessible
vault = Vault.get_by_full_path(path_dict['vault_full_path'])
# If not the vault root, validate remote path exists and is a folder
if path_dict['path'] != '/':
Object.get_by_full_path(base_remote_path, assert_type='folder')
for local_path in args.local_path:
local_path = local_path.rstrip('/')
local_start = os.path.basename(local_path)
if os.path.isdir(local_path):
_upload_folder(path_dict['domain'], vault, base_remote_path,
local_path, local_start)
else:
Object.upload_file(local_path, path_dict['path'], vault.full_path)
def _upload_folder(domain, vault, base_remote_path, base_local_path,
local_start, exclude_paths=None, dry_run=False):
# Create the upload root folder if it does not exist on the remote
try:
upload_root_path, _ = Object.validate_full_path(
os.path.join(base_remote_path, local_start)
)
Object.get_by_full_path(upload_root_path, assert_type='folder')
except NotFoundError:
base_remote_path, path_dict = \
Object.validate_full_path(base_remote_path)
base_folder_path = os.path.join(base_remote_path, local_start)
if dry_run:
print('[Dry Run] Creating folder {}'.format(base_folder_path))
else:
_create_folder(vault, base_folder_path)
# Create folders and upload files
for abs_local_parent_path, folders, files in os.walk(base_local_path):
# Strips off the local path and adds the parent directory at
# each phase of the loop
local_parent_path = re.sub(
'^' + os.path.dirname(base_local_path), '', abs_local_parent_path
).lstrip('/')
if should_exclude(abs_local_parent_path, exclude_paths,
dry_run=dry_run):
continue
remote_folder_full_path = \
os.path.join(base_remote_path, local_parent_path)
# Create folders
for folder in folders:
new_folder_path = os.path.join(abs_local_parent_path, folder)
if should_exclude(new_folder_path, exclude_paths,
dry_run=dry_run):
continue
remote_path = os.path.join(remote_folder_full_path, folder)
if dry_run:
print('[Dry Run] Creating folder {}'.format(remote_path))
else:
_create_folder(vault, remote_path)
# Upload the files that do not yet exist on the remote
for f in files:
local_file_path = os.path.join(abs_local_parent_path, f)
if should_exclude(local_file_path, exclude_paths, dry_run=dry_run):
continue
if dry_run:
print('[Dry Run] Uploading {} to {}'
.format(local_file_path, remote_folder_full_path))
else:
remote_parent = Object.get_by_full_path(
remote_folder_full_path, assert_type='folder')
Object.upload_file(local_file_path, remote_parent.path,
vault.full_path)
def upload(args):
"""
Given a folder or file, upload all the folders and files contained
within it, skipping ones that already exist on the remote.
"""
base_remote_path, path_dict = Object.validate_full_path(args.full_path)
# Assert the vault exists and is accessible
vault = Vault.get_by_full_path(path_dict['vault_full_path'])
# If not the vault root, validate remote path exists and is a folder
if path_dict['path'] != '/':
try:
Object.get_by_full_path(base_remote_path, assert_type='folder')
except:
if not args.create_full_path:
raise
if args.dry_run:
print('[Dry Run] Creating {}'.format(base_remote_path))
else:
# Create the destination path (including subfolders)
# if not found
parent_folder_path = vault.full_path + ':'
folders = path_dict['path'].lstrip('/').split('/')
for folder in folders:
folder_full_path = os.path.join(parent_folder_path, folder)
parent_folder = _create_folder(vault, folder_full_path)
parent_folder_path = parent_folder.full_path
# Exit if there are multiple local paths and the
# exclude paths are not absolute
base_exclude_paths = args.exclude or []
if base_exclude_paths and len(args.local_path) > 1:
rel_exclude_paths = [p for p in base_exclude_paths
if not os.path.isabs(p)]
local_path_parents = set([os.path.dirname(os.path.abspath(p))
for p in args.local_path])
if rel_exclude_paths and len(local_path_parents) > 1:
sys.exit('Exiting. Cannot apply the --exclude relative paths when '
'multiple upload paths with different parent directories '
'are specified. Make --exclude paths absolute or run '
'upload paths one at a time.')
for local_path in args.local_path:
# Expand local path and strip trailing slash
local_path = os.path.abspath(local_path).rstrip('/')
local_name = os.path.basename(local_path)
# add basepath to excludes
exclude_paths = [
os.path.join(local_path, os.path.normpath(exclude_path))
for exclude_path in base_exclude_paths
]
if os.path.isdir(local_path):
_upload_folder(path_dict['domain'], vault,
base_remote_path, local_path,
local_name, exclude_paths=exclude_paths,
dry_run=args.dry_run)
else:
if args.dry_run:
print('[Dry Run] Uploading {} to {}'
.format(local_path, path_dict['path']))
else:
Object.upload_file(local_path, path_dict['path'],
vault.full_path)
def _upload_folder(domain, vault, base_remote_path,
base_local_path, local_start):
# Create the upload root folder if it does not exist on the remote
try:
upload_root_path, _ = Object.validate_full_path(
os.path.join(base_remote_path, local_start)
)
obj = Object.get_by_full_path(upload_root_path, assert_type='folder')
except NotFoundError:
base_remote_path, path_dict = \
Object.validate_full_path(base_remote_path)
if path_dict['path'] == '/':
parent_object_id = None
else:
obj = Object.get_by_full_path(base_remote_path,
assert_type='folder')
parent_object_id = obj.id
# Create base folder
new_folder = Object.create(
vault_id=vault.id,
parent_object_id=parent_object_id,
object_type='folder',
filename=local_start
)
print('Notice: Folder created for {0} at {1}'.format(
base_local_path,
new_folder.path,
))
for root, dirs, files in os.walk(base_local_path):
# Create the sub-folders that do not exist on the remote
for d in dirs:
dirpath = os.path.join(
base_remote_path,
re.sub('^' + os.path.dirname(base_local_path), '', root).lstrip('/'), # noqa
d
)
try:
Object.get_by_full_path(dirpath, object_type='folder')
except NotFoundError:
# Create the folder
if os.path.dirname(dirpath.split(':')[-1]) == '/':
parent_object_id = None
else:
parent_full_path = os.path.dirname(dirpath)
parent = Object.get_by_full_path(
parent_full_path, assert_type='folder')
parent_object_id = parent.id
# Make the API call
new_obj = Object.create(
vault_id=vault.id,
parent_object_id=parent_object_id,
object_type='folder',
filename=d,
)
print('Notice: Folder created for {0} at {1}'
.format(os.path.join(root, d), new_obj.path))
# Upload the files that do not yet exist on the remote
for f in files:
file_full_path = os.path.join(
base_remote_path,
re.sub('^' + os.path.dirname(base_local_path),
'',
root).lstrip('/'),
f,
)
try:
Object.get_by_full_path(file_full_path)
except NotFoundError:
parent_full_path = os.path.dirname(
os.path.join(
base_remote_path,
re.sub('^' + os.path.dirname(base_local_path),
'',
root).lstrip('/'),
f,
)
)
parent = Object.get_by_full_path(
parent_full_path, assert_type='folder')
Object.upload_file(os.path.join(root, f),
parent.path, vault.full_path)
def upload_file(self, local_path, remote_path, **kwargs):
from solvebio import Object
return Object.upload_file(local_path, remote_path, self.full_path,
**kwargs)
def upload_file(self, local_path, remote_path, **kwargs):
from solvebio import Object
return Object.upload_file(
local_path, remote_path, self.full_path, **kwargs)
def _upload_folder(domain, vault, base_remote_path, base_local_path,
local_start):
# Create the upload root folder if it does not exist on the remote
try:
upload_root_path, _ = Object.validate_full_path(
os.path.join(base_remote_path, local_start))
obj = Object.get_by_full_path(upload_root_path, assert_type='folder')
except NotFoundError:
base_remote_path, path_dict = \
Object.validate_full_path(base_remote_path)
if path_dict['path'] == '/':
parent_object_id = None
else:
obj = Object.get_by_full_path(base_remote_path,
assert_type='folder')
parent_object_id = obj.id
# Create base folder
new_folder = Object.create(vault_id=vault.id,
parent_object_id=parent_object_id,
object_type='folder',
filename=local_start)
print('Notice: Folder created for {0} at {1}'.format(
base_local_path,
new_folder.path,
))
for root, dirs, files in os.walk(base_local_path):
# Create the sub-folders that do not exist on the remote
for d in dirs:
dirpath = os.path.join(
base_remote_path,
re.sub('^' + os.path.dirname(base_local_path), '',
root).lstrip('/'), # noqa
d)
try:
Object.get_by_full_path(dirpath, object_type='folder')
except NotFoundError:
# Create the folder
if os.path.dirname(dirpath.split(':')[-1]) == '/':
parent_object_id = None
else:
parent_full_path = os.path.dirname(dirpath)
parent = Object.get_by_full_path(parent_full_path,
assert_type='folder')
parent_object_id = parent.id
# Make the API call
new_obj = Object.create(
vault_id=vault.id,
parent_object_id=parent_object_id,
object_type='folder',
filename=d,
)
print('Notice: Folder created for {0} at {1}'.format(
os.path.join(root, d), new_obj.path))
# Upload the files that do not yet exist on the remote
for f in files:
file_full_path = os.path.join(
base_remote_path,
re.sub('^' + os.path.dirname(base_local_path), '',
root).lstrip('/'),
f,
)
try:
Object.get_by_full_path(file_full_path)
except NotFoundError:
parent_full_path = os.path.dirname(
os.path.join(
base_remote_path,
re.sub('^' + os.path.dirname(base_local_path), '',
root).lstrip('/'),
f,
))
parent = Object.get_by_full_path(parent_full_path,
assert_type='folder')
Object.upload_file(os.path.join(root, f), parent.path,
vault.full_path)
