def build_graph(self): self._build_node_map() for day_to_process in self._keys: node_id = self._get_next_node(day_to_process) carry_over_data = self._map_carry_over_data[node_id] list_frequency_groups = self._work_to_do[day_to_process] frequency_groups = make_groups_of_frequencies(list_frequency_groups, self._parallel_streams) add_output_s3 = [] if carry_over_data.drop_listobs is not None: add_output_s3.append(carry_over_data.drop_listobs) measurement_set, properties, drop_listobs = \ self._setup_measurement_set( day_to_process, carry_over_data.barrier_drop, add_output_s3, node_id ) carry_over_data.drop_listobs = drop_listobs outputs = [] for group in frequency_groups: last_element = None for frequency_pairs in group: last_element = self._split( last_element, frequency_pairs, measurement_set, properties, get_observation(day_to_process.full_tar_name), node_id ) if last_element is not None: outputs.append(last_element) barrier_drop = self.create_barrier_app(node_id) carry_over_data.barrier_drop = barrier_drop for output in outputs: if output is not None: barrier_drop.addInput(output) self.copy_logfiles_and_shutdown()
def build_graph(self): self._build_node_map() for day_to_process in self._keys: node_id = self._get_next_node(day_to_process) carry_over_data = self._map_carry_over_data[node_id] list_frequency_groups = self._work_to_do[day_to_process] frequency_groups = make_groups_of_frequencies( list_frequency_groups, self._parallel_streams) add_output_s3 = [] if carry_over_data.drop_listobs is not None: add_output_s3.append(carry_over_data.drop_listobs) measurement_set, properties, drop_listobs = \ self._setup_measurement_set( day_to_process, carry_over_data.barrier_drop, add_output_s3, node_id ) carry_over_data.drop_listobs = drop_listobs outputs = [] for group in frequency_groups: last_element = None for frequency_pairs in group: last_element = self._split( last_element, frequency_pairs, measurement_set, properties, get_observation(day_to_process.full_tar_name), node_id) if last_element is not None: outputs.append(last_element) barrier_drop = self.create_barrier_app(node_id) carry_over_data.barrier_drop = barrier_drop for output in outputs: if output is not None: barrier_drop.addInput(output) self.copy_logfiles_and_shutdown()
def test_get_observation(self): observation = get_observation('13B-266.sb28624226.eb28625769.56669.43262586805_calibrated_deepfield.ms') self.assertEquals('13B-266.sb28624226.eb28625769.56669.43262586805', observation)