def test_upload_parallel(self):
print '--'
project_name = fake_name("TestRenameProject-")
print project_name
description = "Test project generated by automated test"
project = create_project(project_name, description)
self.assertIsNotNone(project)
self.assertIsNotNone(project.id)
self.assertIsNotNone(project.name)
self.assertEqual(project_name, project.name)
project.local_path = self.upload_dir_path
local_path = os_path.join(project.local_path, self.upload_dir)
parallel = True
verbose = False
limit = 1
# buik loader with very small limit to test for upload failure and shorten test length
loader = BulkFileUploader(parallel=parallel, limit=limit, verbose=verbose)
missed_files = loader.bulk_upload(project, local_path)
expected_missed_files = self.find_big_files(local_path, limit)
if (len(missed_files) > 2):
print missed_files
self.assertEqual(len(missed_files), len(expected_missed_files))
for i in range(0, len(expected_missed_files)):
self.assertTrue(expected_missed_files[i] in missed_files)
def test_create_dir_list_from_path_list(self):
project_name = fake_name("TestRenameProject-")
print project_name
description = "Test project generated by automated test"
project = create_project(project_name, description)
self.assertIsNotNone(project)
self.assertIsNotNone(project.id)
self.assertIsNotNone(project.name)
self.assertEqual(project_name, project.name)
directory_path_list = []
project.local_path = self.upload_dir_path
local_path = os_path.join(project.local_path, self.upload_dir)
for (dirpath, dirnames, filenames) in walk(local_path):
db_path = os_path.relpath(dirpath, project.local_path)
directory_path_list.append('/' + str(db_path))
print project.local_path
print directory_path_list
directory_table = project.add_directory_list(directory_path_list)
top_directory_name = project.name
prefix = len(top_directory_name)
for dir in directory_path_list:
self.assertIsNotNone(directory_table[dir])
self.assertEqual(directory_table[dir].name[prefix:], dir)
def buildProject(self):
name = "My Simple Project"
description = "A simple demo project from a script"
project = create_project(name, description)
self.project = project
experiment_name = "My experiment"
description = "A sample experiment"
experiment = project.create_experiment(experiment_name, description)
self.experiment = experiment
template_id = self._template_id_with(self.template_table, "Create Sample")
create_sample_process = experiment.create_process_from_template(template_id)
create_sample_process.rename("Create my sample")
sample_name = "Sample"
samples = create_sample_process.create_samples([sample_name])
sample = samples[0]
template_id = self._template_id_with(self.template_table, "Low Cycle Fatigue")
transform_process = experiment.create_process_from_template(template_id)
transform_process.rename("Subject to bending")
transform_process.add_input_samples_to_process([sample])
transform_process.decorate_with_output_samples()
transformed_sample = transform_process.output_samples[0]
template_id = self._template_id_with(self.template_table, "SEM")
measurement_process = experiment.create_process_from_template(template_id)
measurement_process.rename("Observe with SEM")
measurement_process.add_input_samples_to_process([transformed_sample])
project.local_path = BASE_DIRECTORY
project.add_file_by_local_path(os.path.join(project.local_path,"example_data/SEM.png"))
def setUpClass(cls):
cls.base_project_name = fake_name("TestDirectoryProject-")
description = "Test project generated by automated test"
project = create_project(cls.base_project_name, description)
cls.base_project_id = project.id
cls.base_project = project
cls.test_dir_path = "/TestDir1/TestDir2/TestDir3"
def setUpClass(cls):
print " -- "
cls.project_name = fake_name("TestProject-")
print cls.project_name
description = "Test project generated by automated test"
cls.project = create_project(cls.project_name, description)
cls.project_id = cls.project.id
def create_project_experiment(self):
time_stamp = '%s' % datetime.datetime.now()
project_name = "WE43 Fatigue Life: " + time_stamp
project_description = "WE43 Fatigue Life, created from script on " + time_stamp
self.project = create_project(project_name, project_description)
self.experiment = self.project.create_experiment(
"WE43 Fatigue Life", "Fatigue measurements on WE4s samples")
def setUpClass(cls):
cls.project_name = fake_name("TestMeasurementEPMASetupProject-")
description = "Test project generated by automated test"
cls.project = create_project(cls.project_name, description)
cls.project_id = cls.project.id
name = "TestMeasurementEPMASetup"
description = "Test experiment generated by automated test"
cls.experiment = cls.project.create_experiment(name, description)
cls.experiment_id = cls.experiment.id
cls.process = cls.experiment.create_process_from_template(
Template.create)
cls.sample_name = "setup-sample-1"
cls.samples = cls.process.create_samples(
sample_names=[cls.sample_name])
cls.sample = cls.samples[0]
cls.helper = TestFileProcessSampleHelper()
cls.helper.setup_test_file()
cls.filepath = cls.helper.filepath
project = cls.project
directory = project.get_top_directory()
path = Path(cls.filepath)
file_name = path.parts[-1]
input_path = str(path.absolute())
cls.byte_count = getsize(input_path)
cls.file = project.add_file_using_directory(directory, file_name,
input_path)
cls.measurement_process_name = "EPMA Process For Test"
epma_process = cls.helper.create_measurement_process(cls.experiment)
epma_process.rename(cls.measurement_process_name)
epma_process.add_input_samples_to_process([cls.sample])
epma_process.add_files([cls.file])
cls.measurement_process = epma_process
def create_project_experiment(self):
time_stamp = '%s' % datetime.datetime.now()
project_name = "Example-From-Script: " + time_stamp
project_description = "Example workflow, created from script on " + time_stamp
self.project = create_project(project_name, project_description)
self.project.local_path = BASE_DIRECTORY
self.experiment = self.project.create_experiment("EPMA-Melt5b", "EPMA measurements of Melt5b samples")
def setUpClass(cls):
cls.base_project_name = fake_name("TestDirectoryProject-")
print ''
print cls.base_project_name
description = "Test project generated by automated test"
project = create_project(cls.base_project_name, description)
cls.base_project_id = project.id
cls.base_project = project
def setUpClass(cls):
cls.project_name = fake_name("TestSampleCreateProject-")
description = "Test project generated by automated test"
cls.project = create_project(cls.project_name, description)
cls.project_id = cls.project.id
name = "Test Experiment - basic"
description = "Test experiment generated by automated test"
cls.experiment = cls.project.create_experiment(name, description)
cls.experiment_id = cls.experiment.id
def setUpClass(cls):
cls.project_name = fake_name("Project - ")
cls.project_description = "Test project, " + cls.project_name
cls.testDirPath = "/testDir1/testdir2/testdir3"
cls.project = create_project(name=cls.project_name,
description=cls.project_description)
cls.rootDir = cls.project.get_top_directory()
cls.testDir = cls.project.add_directory(cls.testDirPath)
def setUpClass(cls):
cls.base_project_name = "FileUploadTest01"
description = "Test project generated by automated test"
project = create_project(cls.base_project_name, description)
directory = project.get_top_directory()
cls.base_project_id = project.id
cls.base_project = project
cls.base_directory_id = directory.id
cls.base_directory = directory
def setUpClass(cls):
cls.base_project_name = fake_name("TestProject-")
description = "Test project generated by automated test"
cls.base_project = create_project(cls.base_project_name, description)
cls.base_project_id = cls.base_project.id
name = fake_name("TestExperiment-")
description = "Test experiment generated by automated test"
cls.base_experiment = cls.base_project.create_experiment(name, description)
cls.base_experiment_id = cls.base_experiment.id
cls.process_compute = cls.base_experiment.create_process_from_template(Template.compute)
def setUpClass(cls):
cls.base_project_name = fake_name("DirectoryUploadTest")
print cls.base_project_name
description = "Test project generated by automated test"
project = create_project(cls.base_project_name, description)
cls.base_project_id = project.id
cls.base_project = project
cls.single_file_dir_path = "test_upload_dir/sub_directory_a"
cls.path_of_empty_dir = "test_upload_dir/sub_directory_c/empty_dir"
cls.full_dir = 'test_upload_dir'
def _get_or_create_project(self, project_name, project_description):
projects = get_all_projects()
project = None
for p in projects:
if p.name == project_name:
project = p
if not project:
project = create_project(name=project_name,
description=project_description)
return project
def add_project(self, row):
name = self.source[row][1]
description = self.source[row][2]
name = self._make_unique_project_name(name)
print "Adding Project: " + name
project = create_project(name, description)
project.local_path = self.data_path
self.current_project = project
self.projects.append(project)
return row + 1
def test_create_project_object(self):
name = fake_name("TestProject-")
description = "Test project generated by automated test"
project = create_project(name, description)
self.assertIsNotNone(project.name)
self.assertTrue(isinstance(project, Project))
self.assertIsNotNone(project.description)
self.assertIsNotNone(project.id)
self.assertNotEqual(project.name, "")
self.assertEqual(name, project.name)
self.assertEqual(description, project.description)
def setUpClass(cls):
cls.project_name = fake_name("TestingDate-")
description = "Test project generated by automated test"
cls.project = create_project(cls.project_name, description)
cls.project_id = cls.project.id
cls.experiment_name = fake_name("TestExperiment-")
description = "Test experiment generated by automated test"
cls.experiment = cls.project.create_experiment(cls.experiment_name,
description)
cls.experiment_id = cls.experiment.id
cls.test_date_value = 1485977519347 #February 1, 2017
def setUpClass(cls):
cls.base_project_name = fake_name("TestDirectoryProject-")
description = "Test project generated by automated test"
project = create_project(cls.base_project_name, description)
cls.base_project_id = project.id
cls.base_project = project
cls.test_dir_path = "/TestDir1/TestDir2/TestDir3"
cls.directory_list = cls.base_project.create_or_get_all_directories_on_path(
cls.test_dir_path)
cls.another_path = "/TestDir1/TestDir2/TestDir4"
cls.base_project.create_or_get_all_directories_on_path(
cls.another_path)
def setUpClass(cls):
cls.project_name = fake_name("TestProject-")
description = "Test project generated by automated test"
cls.project = create_project(cls.project_name, description)
cls.project_id = cls.project.id
name = fake_name("TestExperiment-")
description = "Test experiment generated by automated test"
cls.experiment = cls.project.create_experiment(name, description)
cls.experiment_id = cls.experiment.id
cls.process = cls.experiment.create_process_from_template(
Template.primitive_crystal_structure)
cls.sample_name = "pcs-sample-1"
cls.sample = cls.process.create_samples(sample_names=[cls.sample_name])[0]
def setUpClass(cls):
cls.project_name = fake_name("ExperimentalProject-")
description = "Test project for experimental testing"
cls.project = create_project(cls.project_name, description)
cls.project_id = cls.project.id
name = fake_name("PlayingExperiment-")
description = "Using experiment this experiment to explore..."
cls.experiment = cls.project.create_experiment(name, description)
cls.experiment_id = cls.experiment.id
cls.process = cls.experiment.create_process_from_template(
Template.primitive_crystal_structure)
cls.sample_name = "pcs-sample-1"
cls.sample = cls.process.create_samples(
sample_names=[cls.sample_name])[0]
def setUpClass(cls):
cls.project_name = fake_name("TestMeasurementEPMASetupProject-")
description = "Test project generated by automated test"
cls.project = create_project(cls.project_name, description)
cls.project_id = cls.project.id
name = "TestMeasurementEPMASetup"
description = "Test experiment generated by automated test"
cls.experiment = cls.project.create_experiment(name, description)
cls.experiment_id = cls.experiment.id
cls.process = cls.experiment.create_process_from_template(
Template.create)
cls.sample_name = "setup-sample-1"
cls.samples = cls.process.create_samples(
sample_names=[cls.sample_name])
cls.sample = cls.samples[0]
def setUpClass(cls):
cls.base_project_name = "Project-TestSampleAssociate"
description = "Test project generated by automated test"
cls.base_project = create_project(cls.base_project_name, description)
cls.base_project_id = cls.base_project.id
name = fake_name("TestExperiment-")
description = "Test experiment generated by automated test"
cls.base_experiment = cls.base_project.create_experiment(
name, description)
cls.base_experiment_id = cls.base_experiment.id
cls.base_create_sample_process = \
cls.base_experiment.create_process_from_template(Template.create)
cls.base_sample = cls.base_create_sample_process.create_samples(
['Test Sample 1'])[0]
cls.base_compute_process = \
cls.base_experiment.create_process_from_template(Template.compute)
def setUpClass(cls):
cls.project_name = "Test file Project"
cls.project_description = "A test Project generated from api"
cls.experiment_name = fake_name("Experiment-")
cls.experiment_description = "A test Experiment generated from api"
cls.test_dir_path = "/testDir1/testdir2/testdir3"
cls.filename = "test.jpg"
cls.project = create_project(
name=cls.project_name,
description=cls.project_description)
cls.experiment = cls.project.create_experiment(
name=cls.experiment_name,
description=cls.experiment_description)
cls.process = cls.experiment.create_process_from_template(Template.create)
def setUpClass(cls):
cls.project_name = fake_name("TestMoveProject-")
description = "Test project generated by automated test"
project = create_project(cls.project_name, description)
cls.project_id = project.id
cls.project = project
print ''
print project.name
cls.top_directory = project.get_top_directory()
cls.test_dir_path_for_move = '/TestForMove'
cls.directory_for_move = project.add_directory(
cls.test_dir_path_for_move)
cls.test_dir_path_a = "/TestForMove/A"
cls.directory_a = project.add_directory(cls.test_dir_path_a)
cls.test_dir_path_b = "/TestForMove/A/B"
cls.directory_b = project.add_directory(cls.test_dir_path_b)
cls.test_dir_path_c = "/TestForMove/A/B/C"
cls.directory_c = project.add_directory(cls.test_dir_path_c)
cls.test_dir_path_d = "/TestForMove/A/B/D"
cls.directory_d = project.add_directory(cls.test_dir_path_d)
cls.test_dir_path_e = "/TestForMove/A/E"
cls.directory_e = project.add_directory(cls.test_dir_path_e)
cls.test_dir_path_f = "/TestForMove/F"
cls.directory_f = project.add_directory(cls.test_dir_path_f)
cls.directory_for_file_move_name = "/FilesForMove"
cls.directory_for_file_move = project.add_directory(
cls.directory_for_file_move_name)
if 'TEST_DATA_DIR' in environ:
test_path = os_path.abspath(environ['TEST_DATA_DIR'])
file_path = os_path.join(test_path, 'test_upload_data',
'fractal.jpg')
path = Path(file_path)
cls.file_path = str(path.absolute())
cls.byte_count = getsize(file_path)
cls.original_filename = "FileForMove.jpg"
cls.test_file = project.add_file_using_directory(
cls.directory_for_file_move, cls.original_filename,
cls.file_path)
cls.test_file_c = project.add_file_using_directory(
cls.directory_c, cls.original_filename, cls.file_path)
cls.test_file_d = project.add_file_using_directory(
cls.directory_d, cls.original_filename, cls.file_path)
def setUpClass(cls):
cls.project_name = fake_name("TestRenameProject-")
description = "Test project generated by automated test"
project = create_project(cls.project_name, description)
cls.project_id = project.id
cls.project = project
cls.test_base_path = "/TestForRename"
cls.top_directory = project.get_top_directory()
cls.test_dir_path_a = "/TestForRename/A"
cls.directory_a = project.add_directory(cls.test_dir_path_a)
cls.test_dir_path_b = "/TestForRename/A/B"
cls.directory_b = project.add_directory(cls.test_dir_path_b)
cls.test_dir_path_c = "/TestForRename/A/B/C"
cls.directory_c = project.add_directory(cls.test_dir_path_c)
cls.test_dir_path_d = "/TestForRename/A/B/D"
cls.directory_d = project.add_directory(cls.test_dir_path_d)
cls.test_dir_path_e = "/TestForRename/A/E"
cls.directory_e = project.add_directory(cls.test_dir_path_e)
cls.test_dir_path_f = "/TestForRename/F"
cls.directory_f = project.add_directory(cls.test_dir_path_f)
def setUpClass(cls):
cls.project_name = fake_name("TestRenameProject-")
description = "Test project generated by automated test"
project = create_project(cls.project_name, description)
cls.project_id = project.id
cls.project = project
cls.top_directory = project.get_top_directory()
cls.directory_for_rename_name = "/FilesForRename"
cls.directory_for_rename = project.add_directory(cls.directory_for_rename_name)
if 'TEST_DATA_DIR' in environ:
test_path = os_path.abspath(environ['TEST_DATA_DIR'])
file_path = os_path.join(test_path, 'test_upload_data', 'fractal.jpg')
path = Path(file_path)
cls.file_path = str(path.absolute())
cls.byte_count = getsize(file_path)
cls.original_filename = "FileForRename.jpg"
cls.test_file = project.add_file_using_directory(
cls.directory_for_rename,
cls.original_filename,
cls.file_path
)
def get_project():
project_name = fake_name("Repeat-Update")
project = create_project(project_name, "Project for redundent upload test")
print "Project: " + project.name
return project
def get_project(mode):
project_name = fake_name(mode + "-Stress-Test-")
project = create_project(project_name,
"Project for parallel upload stress test")
print "Project: " + project.name
return project
def setUpClass(cls):
cls.project_name = fake_name("TestDirectoryProject-")
description = "Test project generated by automated test"
project = create_project(cls.project_name, description)
cls.project = project
