def test_purging_output_keep_running_until_all_purged(self, history_id): job_state, outputs = self._setup_running_two_output_job( history_id, 120) # Pretty much right away after the job is running, these paths should be populated - # if they are grab them and make sure they are deleted at the end of the job. dataset_1 = self._get_history_item_as_admin(history_id, outputs[0]["id"]) dataset_2 = self._get_history_item_as_admin(history_id, outputs[1]["id"]) if "file_name" in dataset_1: output_dataset_paths = [ dataset_1["file_name"], dataset_2["file_name"] ] # This may or may not exist depending on if the test is local or not. output_dataset_paths_exist = os.path.exists( output_dataset_paths[0]) else: output_dataset_paths = [] output_dataset_paths_exist = False self._hack_to_skip_test_if_state_ok(job_state) current_state = job_state().json()["state"] assert current_state == "running", current_state # Purge one of the two outputs and make sure the job is still running. self._raw_update_history_item(history_id, outputs[0]["id"], {"purged": True}) time.sleep(1) self._hack_to_skip_test_if_state_ok(job_state) current_state = job_state().json()["state"] assert current_state == "running", current_state # Purge the second output and make sure the job is cancelled. self._raw_update_history_item(history_id, outputs[1]["id"], {"purged": True}) final_state = wait_on_state(job_state, assert_ok=False, timeout=15) assert final_state in ["deleted_new", "deleted"], final_state def paths_deleted(): if not os.path.exists( output_dataset_paths[0]) and not os.path.exists( output_dataset_paths[1]): return True if output_dataset_paths_exist: wait_on(paths_deleted, "path deletion")
def test_galaxy_task(self): history_id = self.dataset_populator.new_history() dataset = self.dataset_populator.new_dataset(history_id, wait=True) hda = self._latest_hda assert hda def hda_purged(): latest_details = self.dataset_populator.get_history_dataset_details( history_id, dataset=dataset, assert_ok=False, wait=False) return True if latest_details["purged"] else None assert not hda_purged() purge_hda.delay(hda_id=hda.id).get(timeout=10) wait_on(hda_purged, "dataset to become purged") assert hda_purged()
def wait_on_entry_points_active(self, job_id, expected_num=1): def active_entry_points(): entry_points = self.entry_points_for_job(job_id) if len(entry_points) != expected_num: return None elif any([not e["active"] for e in entry_points]): return None else: return entry_points return wait_on(active_entry_points, "entry points to become active")
def wait_on_entry_points_active(self, job_id, expected_num=1): def active_entry_points(): entry_points = self.entry_points_for_job(job_id) if len(entry_points) != expected_num: return None elif any(not e["active"] for e in entry_points): job_json = self._get(f"jobs/{job_id}?full=true").json() if job_json['state'] == 'error': raise Exception( f"Interactive tool job {job_id} failed: {job_json}") return None else: return entry_points # It currently takes at least 90 seconds until we can be sure the container monitor failed. # Can be decreased when galaxy_ext/container_monitor/monitor.py changes return wait_on(active_entry_points, "entry points to become active", timeout=120)
def wait_on_proxied_content(self, target): def get_hosted_content(): try: scheme, rest = target.split("://", 1) prefix, host_and_port = rest.split(".interactivetool.") faked_host = rest if "/" in rest: faked_host = rest.split("/", 1)[0] url = "%s://%s" % (scheme, host_and_port) response = requests.get(url, timeout=1, headers={"Host": faked_host}) return response.text except Exception as e: print(e) return None content = wait_on(get_hosted_content, "realtime hosted content at %s" % target) return content