Пример #1
0
    def build_graph(self):
        self._build_node_map()

        for day_to_process in self._keys:
            node_id = self._get_next_node(day_to_process)
            carry_over_data = self._map_carry_over_data[node_id]
            list_frequency_groups = self._work_to_do[day_to_process]
            frequency_groups = make_groups_of_frequencies(list_frequency_groups, self._parallel_streams)

            add_output_s3 = []
            if carry_over_data.drop_listobs is not None:
                add_output_s3.append(carry_over_data.drop_listobs)

            measurement_set, properties, drop_listobs = \
                self._setup_measurement_set(
                    day_to_process,
                    carry_over_data.barrier_drop,
                    add_output_s3,
                    node_id
                )

            carry_over_data.drop_listobs = drop_listobs

            outputs = []
            for group in frequency_groups:
                last_element = None
                for frequency_pairs in group:
                    last_element = self._split(
                        last_element,
                        frequency_pairs,
                        measurement_set,
                        properties,
                        get_observation(day_to_process.full_tar_name),
                        node_id
                    )

                if last_element is not None:
                    outputs.append(last_element)

            barrier_drop = self.create_barrier_app(node_id)
            carry_over_data.barrier_drop = barrier_drop

            for output in outputs:
                if output is not None:
                    barrier_drop.addInput(output)

        self.copy_logfiles_and_shutdown()
    def build_graph(self):
        self._build_node_map()

        for day_to_process in self._keys:
            node_id = self._get_next_node(day_to_process)
            carry_over_data = self._map_carry_over_data[node_id]
            list_frequency_groups = self._work_to_do[day_to_process]
            frequency_groups = make_groups_of_frequencies(
                list_frequency_groups, self._parallel_streams)

            add_output_s3 = []
            if carry_over_data.drop_listobs is not None:
                add_output_s3.append(carry_over_data.drop_listobs)

            measurement_set, properties, drop_listobs = \
                self._setup_measurement_set(
                    day_to_process,
                    carry_over_data.barrier_drop,
                    add_output_s3,
                    node_id
                )

            carry_over_data.drop_listobs = drop_listobs

            outputs = []
            for group in frequency_groups:
                last_element = None
                for frequency_pairs in group:
                    last_element = self._split(
                        last_element, frequency_pairs, measurement_set,
                        properties,
                        get_observation(day_to_process.full_tar_name), node_id)

                if last_element is not None:
                    outputs.append(last_element)

            barrier_drop = self.create_barrier_app(node_id)
            carry_over_data.barrier_drop = barrier_drop

            for output in outputs:
                if output is not None:
                    barrier_drop.addInput(output)

        self.copy_logfiles_and_shutdown()
Пример #3
0
 def test_get_observation(self):
     observation = get_observation('13B-266.sb28624226.eb28625769.56669.43262586805_calibrated_deepfield.ms')
     self.assertEquals('13B-266.sb28624226.eb28625769.56669.43262586805', observation)