def _retry_upload(self, evt=None):
"""Retry the upload after a failure is encountered.
Args:
evt: the event that fired this upload.
"""
self._prepare_for_automatic_upload()
pub.unsubscribe(self._finished_loading, DirectoryScannerTopics.finished_run_scan)
pub.subscribe(self._start_upload, DirectoryScannerTopics.finished_run_scan)
find_runs_in_directory(self._get_default_directory())
def test_upload_run_cached_projects(self, api):
self.sample_count = 0
# create a project first
project = Project("project", "description")
created_project = api.send_project(project)
project_id = created_project["resource"]["identifier"]
path_to_module = path.dirname(__file__)
run_dir = path.join(path_to_module, "fake_ngs_data_with_sample_name")
try:
remove(path.join(run_dir, ".miseqUploaderInfo"))
except:
pass
runs = find_runs_in_directory(run_dir)
assert 1 == len(runs)
run_to_upload = runs[0]
for sample in run_to_upload.sample_list:
sample.get_project_id = lambda: project_id
pub.subscribe(self.update_samples_counter,
sample.upload_completed_topic)
upload_run_to_server(api, run_to_upload)
assert 2 == self.sample_count
self.sample_count = 0
project = Project("project", "description")
created_project = api.send_project(project, clear_cache=False)
project_id = created_project["resource"]["identifier"]
try:
remove(path.join(run_dir, ".miseqUploaderInfo"))
except:
pass
runs = find_runs_in_directory(run_dir)
assert 1 == len(runs)
run_to_upload = runs[0]
for sample in run_to_upload.sample_list:
sample.get_project_id = lambda: project_id
pub.subscribe(self.update_samples_counter,
sample.upload_completed_topic)
upload_run_to_server(api, run_to_upload)
assert 2 == self.sample_count
def test_sample_names_spaces(self):
runs = find_runs_in_directory(path.join(path_to_module, "sample-names-with-spaces"))
self.assertEqual(1, len(runs))
samples = runs[0].sample_list
self.assertEqual(3, len(samples))
for sample in samples:
self.assertEqual(sample.get_id(), sample.get_id().strip())
def test_single_end(self):
runs = find_runs_in_directory(path.join(path_to_module, "single_end"))
self.assertEqual(1, len(runs))
self.assertEqual("SINGLE_END", runs[0].metadata["layoutType"])
samples = runs[0].sample_list
self.assertEqual(3, len(samples))
for sample in samples:
self.assertFalse(sample.is_paired_end())
def on_created(self, event):
"""Overrides `on_created` in `FileSystemEventHandler` to filter on
file creation events for `CompletedJobInfo.xml`."""
if isinstance(event, FileCreatedEvent) and event.src_path.endswith(
'CompletedJobInfo.xml'):
logging.info(
"Observed new run in {}, telling the UI to start uploading it."
.format(event.src_path))
directory = os.path.dirname(event.src_path)
# tell the UI to clean itself up before observing new runs
send_message(DirectoryMonitorTopics.new_run_observed)
# this will send a bunch of events that the UI is listening for, but
# unlike the UI (which runs this in a separate thread), we're going to do this
# in our own thread and block on it so we can tell the UI to start
# uploading once we've finished discovering the run
find_runs_in_directory(directory)
# now tell the UI to start
send_message(DirectoryMonitorTopics.finished_discovering_run)
else:
logging.debug("Ignoring file event [{}] with path [{}]".format(
str(event), event.src_path))
def test_get_and_send_sequence_files(self, api):
path_to_module = path.dirname(__file__)
if len(path_to_module) == 0:
path_to_module = '.'
run = find_runs_in_directory(path.join(path_to_module,
"fake_ngs_data")).pop()
samples_list = run.sample_list
# check that the sample with id 99-9999 (from SampleSheet.csv)
# has no sequence files
seq_file_list = []
for sample in samples_list:
res = api.get_sequence_files(sample)
if len(res) > 0:
seq_file_list.append(res)
assert len(seq_file_list) == 0
serv_res_list = api.send_sequence_files(samples_list)[0]
# check that the sample with id 99-9999 (from SampleSheet.csv)
# has the one that we just uploaded.
seq_file_list = []
for sample in samples_list:
res = api.get_sequence_files(sample)
logging.info(str(res))
if len(res) > 0:
seq_file_list.append(res)
assert len(seq_file_list) == 1
filename_list = [
serv_resp["fileName"] for serv_resp in seq_file_list[0]
]
assert len(filename_list) == 2
# check that the files in each sample are found
# in the server response
for sample in samples_list:
assert ntpath.basename(sample.get_files()[0]) in filename_list
assert ntpath.basename(sample.get_files()[1]) in filename_list
assert len(serv_res_list) == len(samples_list)
def test_resume_upload(self, api):
self.sample_count = 0
project = Project("project", "description")
created_project = api.send_project(project)
project_id = created_project["resource"]["identifier"]
path_to_module = path.dirname(__file__)
run_dir = path.join(path_to_module, "half-uploaded-run")
try:
remove(path.join(run_dir, ".miseqUploaderInfo"))
except:
pass
runs = find_runs_in_directory(run_dir)
assert 1 == len(runs)
run_to_upload = runs[0]
assert 2 == len(run_to_upload.sample_list)
for sample in run_to_upload.sample_list:
sample.get_project_id = lambda: project_id
original_files_method = run_to_upload.sample_list[1].get_files
run_to_upload.sample_list[1].get_files = lambda: (_ for _ in ()).throw(
Exception('foobar'))
try:
upload_run_to_server(api, run_to_upload)
except:
logging.info("Succeeded in failing to upload files.")
run_to_upload.sample_list[1].get_files = original_files_method
pub.subscribe(self.update_samples_counter,
run_to_upload.sample_list[1].upload_completed_topic)
upload_run_to_server(api, run_to_upload)
assert 1 == self.sample_count
def test_find_sample_sheet_name_variations(self):
runs = find_runs_in_directory(path.join(path_to_module, "sample-sheet-name-variations"))
self.assertEqual(1, len(runs))
def test_completed_upload(self):
runs = find_runs_in_directory(path.join(path_to_module, "completed"))
self.assertEqual(0, len(runs))
