def test_contained_in_dict(self):
item = ProjectNameOrId(value='mouse', is_name=True)
self.assertEqual(True, item.contained_in_dict({'name': 'mouse'}))
self.assertEqual(False, item.contained_in_dict({'name': 'mouse2'}))
item = ProjectNameOrId(value='576', is_name=False)
self.assertEqual(False, item.contained_in_dict({'id': '123'}))
self.assertEqual(True, item.contained_in_dict({'id': '576'}))
def test_upload_project_run(self):
mock_data_service = Mock()
mock_data_service.create_project.return_value = MagicMock()
mock_upload_context = Mock()
mock_upload_context.make_data_service.return_value = mock_data_service
mock_upload_context.project_name_or_id = ProjectNameOrId.create_from_name('mouse')
upload_project_run(mock_upload_context)
mock_data_service.create_project.assert_called_with('mouse', 'mouse')
def test_nothing_to_do(self, MockProjectUploadDryRun, MockProjectUpload,
MockRemoteStore):
MockProjectUploadDryRun().upload_items = []
name_or_id = ProjectNameOrId.create_from_name("someProject")
project_upload = ProjectUpload(MagicMock(), name_or_id, ["data"])
dry_run_report = project_upload.dry_run_report()
self.assertIn("No changes found. Nothing needs to be uploaded.",
dry_run_report)
def test_upload_project_run(self):
mock_data_service = Mock()
mock_data_service.create_project.return_value = MagicMock()
mock_upload_context = Mock()
mock_upload_context.make_data_service.return_value = mock_data_service
mock_upload_context.project_name_or_id = ProjectNameOrId.create_from_name('mouse')
upload_project_run(mock_upload_context)
mock_data_service.create_project.assert_called_with('mouse', 'mouse')
def test_two_files_to_upload(self, MockProjectUploadDryRun, MockProjectUpload, MockRemoteStore):
MockProjectUploadDryRun().upload_items = ['data.txt', 'data2.txt']
name_or_id = ProjectNameOrId.create_from_name("someProject")
project_upload = ProjectUpload(MagicMock(), name_or_id, ["data"])
dry_run_report = project_upload.dry_run_report()
self.assertIn("Files/Folders that need to be uploaded:", dry_run_report)
self.assertIn("data.txt", dry_run_report)
self.assertIn("data2.txt", dry_run_report)
def test_two_files_to_upload(self, MockProjectUploadDryRun,
MockProjectUpload, MockRemoteStore):
MockProjectUploadDryRun().upload_items = ['data.txt', 'data2.txt']
name_or_id = ProjectNameOrId.create_from_name("someProject")
project_upload = ProjectUpload(MagicMock(), name_or_id, ["data"])
dry_run_report = project_upload.dry_run_report()
self.assertIn("Files/Folders that need to be uploaded:",
dry_run_report)
self.assertIn("data.txt", dry_run_report)
self.assertIn("data2.txt", dry_run_report)
def test_contained_in_dict(self):
item = ProjectNameOrId(value='mouse', is_name=True)
self.assertEqual(True, item.contained_in_dict({'name': 'mouse'}))
self.assertEqual(False, item.contained_in_dict({'name': 'mouse2'}))
item = ProjectNameOrId(value='576', is_name=False)
self.assertEqual(False, item.contained_in_dict({'id': '123'}))
self.assertEqual(True, item.contained_in_dict({'id': '576'}))
def _upload_project(self, activity, project_name, temp_directory):
"""
Upload the contents of temp_directory into project_name
:param activity: CopyActivity: info about the copy activity are uploading for
:param project_name: str project name we will upload files to
:param temp_directory: str path to directory who's files we will upload
"""
self.print_func("Uploading to '{}'.".format(project_name))
items_to_send = [os.path.join(temp_directory, item) for item in os.listdir(os.path.abspath(temp_directory))]
project_name_or_id = ProjectNameOrId.create_from_name(project_name)
project_upload = ProjectUpload(self.config, project_name_or_id, items_to_send,
file_upload_post_processor=UploadedFileRelations(activity))
project_upload.run()
def test_constructors(self):
item = ProjectNameOrId.create_from_project_id('123')
self.assertEqual(item.is_name, False)
self.assertEqual(item.value, '123')
self.assertEqual('123', item.get_id_or_raise())
with self.assertRaises(ValueError):
item.get_name_or_raise()
item = ProjectNameOrId.create_from_name('mouse')
self.assertEqual(item.is_name, True)
self.assertEqual(item.value, 'mouse')
self.assertEqual('mouse', item.get_name_or_raise())
with self.assertRaises(ValueError):
item.get_id_or_raise()
mock_project = Mock()
mock_project.id = '456'
item = ProjectNameOrId.create_from_remote_project(mock_project)
self.assertEqual(item.is_name, False)
self.assertEqual(item.value, '456')
self.assertEqual('456', item.get_id_or_raise())
with self.assertRaises(ValueError):
item.get_name_or_raise()
def test_constructors(self):
item = ProjectNameOrId.create_from_project_id('123')
self.assertEqual(item.is_name, False)
self.assertEqual(item.value, '123')
self.assertEqual('123', item.get_id_or_raise())
with self.assertRaises(ValueError):
item.get_name_or_raise()
item = ProjectNameOrId.create_from_name('mouse')
self.assertEqual(item.is_name, True)
self.assertEqual(item.value, 'mouse')
self.assertEqual('mouse', item.get_name_or_raise())
with self.assertRaises(ValueError):
item.get_id_or_raise()
mock_project = Mock()
mock_project.id = '456'
item = ProjectNameOrId.create_from_remote_project(mock_project)
self.assertEqual(item.is_name, False)
self.assertEqual(item.value, '456')
self.assertEqual('456', item.get_id_or_raise())
with self.assertRaises(ValueError):
item.get_name_or_raise()
def upload_files(cmdfile):
upload_list = UploadList(cmdfile)
click.echo("Uploading {} paths to {}.".format(len(upload_list.paths),
upload_list.destination))
dds_client = DukeDSClient()
project_upload = ProjectUpload(
dds_client.dds_connection.config,
ProjectNameOrId.create_from_name(upload_list.destination),
upload_list.paths)
click.echo(project_upload.get_differences_summary())
if project_upload.needs_to_upload():
click.echo("Uploading")
project_upload.run()
else:
click.echo("Nothing needs to be done.")
def upload_files(self, project):
"""
Upload files from local paths to the specified project
:param project: ddsc.sdk.client.Project: project to upload files to
:return: UploadedFileInfo: contains details about uploaded files
"""
project_upload = ProjectUpload(
self.dds_config,
ProjectNameOrId.create_from_project_id(project.id),
self.settings.paths)
click.echo(project_upload.get_differences_summary())
if project_upload.needs_to_upload():
click.echo("Uploading")
project_upload.run()
return UploadedFilesInfo(project_upload.local_project)
def _copy_project(self, project, new_project_name, path_filter):
"""
Copy pre-existing project with name project_name to non-existing project new_project_name.
:param project: remotestore.RemoteProject project to copy from
:param new_project_name: str project to copy to
:param path_filter: PathFilter: filters what files are shared
:return: RemoteProject new project we copied data to
"""
temp_directory = tempfile.mkdtemp()
new_project_name_or_id = ProjectNameOrId.create_from_name(new_project_name)
remote_project = self.remote_store.fetch_remote_project(new_project_name_or_id)
if remote_project:
raise ValueError("A project with name '{}' already exists.".format(new_project_name))
activity = CopyActivity(self.remote_store.data_service, project, new_project_name)
self._download_project(activity, project, temp_directory, path_filter)
self._upload_project(activity, new_project_name, temp_directory)
activity.finished()
shutil.rmtree(temp_directory)
return self.remote_store.fetch_remote_project(new_project_name_or_id, must_exist=True)
def test_fetch_remote_project_exclude_response_fields(self, mock_data_service_api):
exclude_response_fields = ['audit', 'ancestors', 'project']
projects_resp = Mock()
projects_resp.json.return_value = {
'results': [
{
'id': '123',
'kind': 'dds-project',
'name': 'Project1',
'description': '',
'is_deleted': False,
},
]
}
mock_data_service_api.return_value.get_projects.return_value = projects_resp
remote_store = RemoteStore(config=MagicMock())
project_name_or_id = ProjectNameOrId.create_from_project_id('123')
remote_store.fetch_remote_project(project_name_or_id, must_exist=True, include_children=True)
mock_data_service_api.return_value.get_project_children.assert_called_with('123', '', exclude_response_fields)
def _upload_project(self, activity, project_name, temp_directory):
"""
Upload the contents of temp_directory into project_name
:param activity: CopyActivity: info about the copy activity are uploading for
:param project_name: str project name we will upload files to
:param temp_directory: str path to directory who's files we will upload
"""
self.print_func("Uploading to '{}'.".format(project_name))
items_to_send = [
os.path.join(temp_directory, item)
for item in os.listdir(os.path.abspath(temp_directory))
]
project_name_or_id = ProjectNameOrId.create_from_name(project_name)
project_upload = ProjectUpload(
self.config,
project_name_or_id,
items_to_send,
file_upload_post_processor=UploadedFileRelations(activity))
project_upload.run()
def test_fetch_remote_project_exclude_response_fields(
self, mock_data_service_api):
exclude_response_fields = ['audit', 'ancestors', 'project']
projects_resp = Mock()
projects_resp.json.return_value = {
'results': [
{
'id': '123',
'kind': 'dds-project',
'name': 'Project1',
'description': '',
'is_deleted': False,
},
]
}
mock_data_service_api.return_value.get_projects.return_value = projects_resp
remote_store = RemoteStore(config=MagicMock())
project_name_or_id = ProjectNameOrId.create_from_project_id('123')
remote_store.fetch_remote_project(project_name_or_id,
must_exist=True,
include_children=True)
mock_data_service_api.return_value.get_project_children.assert_called_with(
'123', '', exclude_response_fields)
def _copy_project(self, project, new_project_name, path_filter):
"""
Copy pre-existing project with name project_name to non-existing project new_project_name.
:param project: remotestore.RemoteProject project to copy from
:param new_project_name: str project to copy to
:param path_filter: PathFilter: filters what files are shared
:return: RemoteProject new project we copied data to
"""
temp_directory = tempfile.mkdtemp()
new_project_name_or_id = ProjectNameOrId.create_from_name(
new_project_name)
remote_project = self.remote_store.fetch_remote_project(
new_project_name_or_id)
if remote_project:
raise ValueError("A project with name '{}' already exists.".format(
new_project_name))
activity = CopyActivity(self.remote_store.data_service, project,
new_project_name)
self._download_project(activity, project, temp_directory, path_filter)
self._upload_project(activity, new_project_name, temp_directory)
activity.finished()
shutil.rmtree(temp_directory)
return self.remote_store.fetch_remote_project(new_project_name_or_id,
must_exist=True)
def test_nothing_to_do(self, MockProjectUploadDryRun, MockProjectUpload, MockRemoteStore):
MockProjectUploadDryRun().upload_items = []
name_or_id = ProjectNameOrId.create_from_name("someProject")
project_upload = ProjectUpload(MagicMock(), name_or_id, ["data"])
dry_run_report = project_upload.dry_run_report()
self.assertIn("No changes found. Nothing needs to be uploaded.", dry_run_report)
def test_description(self):
item = ProjectNameOrId(value='mouse', is_name=True)
self.assertEqual('name mouse', item.description())
item = ProjectNameOrId(value='123', is_name=False)
self.assertEqual('id 123', item.description())
def test_can_pickle(self):
"""Make sure we can pickle context since it must be passed to another process."""
settings = UploadSettings(None, FakeDataServiceApi(), None, ProjectNameOrId.create_from_name('mouse'), None)
params = ('one', 'two', 'three')
context = UploadContext(settings, params, multiprocessing.Manager().Queue(), 12)
pickle.dumps(context)
def create_project_name_or_id_from_args(args):
if args.project_name:
return ProjectNameOrId.create_from_name(args.project_name)
else:
return ProjectNameOrId.create_from_project_id(args.project_id)
def create_project_name_or_id_from_args(args):
if args.project_name:
return ProjectNameOrId.create_from_name(args.project_name)
else:
return ProjectNameOrId.create_from_project_id(args.project_id)
def test_constructor_fails_for_id(self):
mock_settings = Mock(project_name_or_id=ProjectNameOrId.create_from_project_id('123'))
mock_local_project = Mock()
with self.assertRaises(ValueError):
CreateProjectCommand(mock_settings, mock_local_project)
def test_description(self):
item = ProjectNameOrId(value='mouse', is_name=True)
self.assertEqual('name mouse', item.description())
item = ProjectNameOrId(value='123', is_name=False)
self.assertEqual('id 123', item.description())
def test_constructor_ok_for_name(self):
mock_settings = Mock(project_name_or_id=ProjectNameOrId.create_from_name('mouse'))
mock_local_project = Mock()
CreateProjectCommand(mock_settings, mock_local_project)
def test_constructor_fails_for_id(self):
mock_settings = Mock(project_name_or_id=ProjectNameOrId.create_from_project_id('123'))
mock_local_project = Mock()
with self.assertRaises(ValueError):
CreateProjectCommand(mock_settings, mock_local_project)
def test_can_pickle(self):
"""Make sure we can pickle context since it must be passed to another process."""
settings = UploadSettings(None, FakeDataServiceApi(), None, ProjectNameOrId.create_from_name('mouse'), None)
params = ('one', 'two', 'three')
context = UploadContext(settings, params, multiprocessing.Manager().Queue(), 12)
pickle.dumps(context)
def test_constructor_ok_for_name(self):
mock_settings = Mock(project_name_or_id=ProjectNameOrId.create_from_name('mouse'))
mock_local_project = Mock()
CreateProjectCommand(mock_settings, mock_local_project)