def download_files(cmdfile):
dds_client = DukeDSClient()
download_list = DownloadList(cmdfile)
items = download_list.items()
click.echo("Downloading {} files.".format(len(items)))
for key, dest in items:
click.echo("Downloading file {} to {}.".format(key, dest))
dds_file = dds_client.get_file_by_id(file_id=key)
dds_file.download_to_path(dest)
def main():
project_id = '20c1b14c-91c6-4a30-ab5e-aec4d632ee65'
client = Client()
project_name = get_project_name(client, project_id)
z = build_zip(client, project_id)
with open('{}.zip'.format(project_name), 'wb') as f:
for data in z:
print('WRITE {}'.format(len(data)))
f.write(data)
def make_client(user):
try:
dds_credential = DDSUserCredential.objects.get(user=user)
config = get_dds_config_for_credentials(dds_credential)
return Client(config=config)
except DDSUserCredential.DoesNotExist:
# No DDSUserCredential configured for this user, fall back to OAuth
# May raise an OAuthConfigurationException
endpoint = get_default_dds_endpoint()
config = Config()
config.update_properties({
Config.URL: endpoint.api_root,
})
authentication_service_id = endpoint.openid_provider_service_id
def create_data_service_auth(config, set_status_msg=print):
return CustomOAuthDataServiceAuth(user, authentication_service_id, config, set_status_msg=set_status_msg)
return Client(config=config, create_data_service_auth=create_data_service_auth)
def download(project_id):
client = Client() # This assumes it can authenticate
project_name = get_project_name(client, project_id)
def generate():
z = build_zip(client, project_id)
for chunk in z:
print('WRITE {}'.format(len(chunk)))
yield chunk
response = Response(generate(), mimetype='application/zip')
response.headers['Content-Disposition'] = 'attachment; filename={}'.format('{}.zip'.format(project_name))
print('RESPOND')
return response
class DDSFileUtil(object):
def __init__(self):
self.client = Client()
def find_file_for_path(self, duke_ds_file_path):
project_name, file_path = self.get_project_name_and_file_path(
duke_ds_file_path)
project = self.find_project_for_name(project_name)
if project:
try:
return project.get_child_for_path(file_path)
except ItemNotFound:
raise FileDoesNotExistException(
"File does not exist: {}".format(duke_ds_file_path))
else:
raise ProjectDoesNotExistException(
"Project does not exist: {}".format(duke_ds_file_path))
@staticmethod
def get_project_name_and_file_path(duke_ds_file_path):
if not duke_ds_file_path.startswith(PATH_PREFIX):
raise InvalidFilePathException("{}: {}".format(
DUKEDS_FILE_PATH_MISSING_PREFIX, duke_ds_file_path))
path = duke_ds_file_path.replace(PATH_PREFIX, '', 1)
path_parts = path.split(os.sep)
if len(path_parts) < 2:
raise InvalidFilePathException("{}: {}".format(
DUKEDS_FILE_PATH_MISSING_SLASH, duke_ds_file_path))
project_name = path_parts[0]
file_path = path.replace('{}/'.format(project_name), '', 1)
return project_name, file_path
def find_project_for_name(self, project_name):
for project in self.client.get_projects():
if project.name == project_name:
return project
return None
def give_download_permissions(self, project_id, dds_user_id):
self.client.dds_connection.data_service.set_user_project_permission(
project_id, dds_user_id, auth_role='file_downloader')
def __init__(self, cmdfile):
self.settings = Settings(cmdfile)
self.dds_client = DukeDSClient()
self.dds_config = self.dds_client.dds_connection.config
class UploadUtil(object):
def __init__(self, cmdfile):
self.settings = Settings(cmdfile)
self.dds_client = DukeDSClient()
self.dds_config = self.dds_client.dds_connection.config
def get_or_create_project(self):
"""
Find or create a project with the name self.settings.destination
:return: ddsc.sdk.client.Project
"""
project_name = self.settings.destination
for project in self.dds_client.get_projects():
if project.name == project_name:
return project
return self.dds_client.create_project(project_name,
description=project_name)
def upload_files(self, project):
"""
Upload files from local paths to the specified project
:param project: ddsc.sdk.client.Project: project to upload files to
:return: UploadedFileInfo: contains details about uploaded files
"""
project_upload = ProjectUpload(
self.dds_config,
ProjectNameOrId.create_from_project_id(project.id),
self.settings.paths)
click.echo(project_upload.get_differences_summary())
if project_upload.needs_to_upload():
click.echo("Uploading")
project_upload.run()
return UploadedFilesInfo(project_upload.local_project)
def create_provenance_activity(self, uploaded_files_info):
"""
Create a provenance activity in DukeDS API for our project.
:param uploaded_files_info: UploadedFilesInfo: contains details about uploaded files
"""
activity = DukeDSActivity(self.dds_client, self.settings,
uploaded_files_info)
activity.create()
def share_project(self, project):
"""
Share the specified project with some users.
:param project: ddsc.sdk.client.Project: project to share
"""
remote_store = RemoteStore(self.dds_config)
remote_project = remote_store.fetch_remote_project_by_id(project.id)
d4s2_project = D4S2Project(self.dds_config,
remote_store,
print_func=print)
for dds_user_id in self.settings.share_dds_user_ids:
d4s2_project.share(remote_project,
remote_store.fetch_user(dds_user_id),
force_send=True,
auth_role=self.settings.share_auth_role,
user_message=self.settings.share_user_message)
def create_annotate_project_details_script(self, project, outfile):
"""
Create a script to annotat a pod with details about the uploaded project
:param project: ddsc.sdk.client.Project: project to share
:param outfile: output file to write script into
"""
readme_file = project.get_child_for_path(
self.settings.readme_file_path)
click.echo(
"Writing annotate project details script project_id:{} readme_file_id:{} to {}"
.format(project.id, readme_file.id, outfile.name))
contents = "kubectl annotate pod $MY_POD_NAME " \
"project_id={} readme_file_id={}".format(project.id, readme_file.id)
outfile.write(contents)
outfile.close()
def create_json_project_details_file(self, project, outfile):
readme_file = project.get_child_for_path(
self.settings.readme_file_path)
click.echo(
"Writing JSON project details project_id:{} readme_file_id:{} to {}"
.format(project.id, readme_file.id, outfile.name))
outfile.write(
json.dumps({
"project_id": project.id,
"readme_file_id": readme_file.id
}))
def __init__(self):
self.client = Client()