Ejemplo n.º 1
0
    def get_time_idx(cls, coverage, timeval):
        temporal_variable = coverage.temporal_parameter_name
        uom = coverage.get_parameter_context(temporal_variable).uom
        
        units = TimeUtils.ts_to_units(uom, timeval)

        idx = TimeUtils.get_relative_time(coverage, units)
        return idx
Ejemplo n.º 2
0
    def get_time_idx(cls, coverage, timeval):
        temporal_variable = coverage.temporal_parameter_name
        uom = coverage.get_parameter_context(temporal_variable).uom

        units = TimeUtils.ts_to_units(uom, timeval)

        idx = TimeUtils.get_relative_time(coverage, units)
        return idx
 def dataset_temporal_bounds(self, dataset_id):
     coverage = self._get_coverage(dataset_id)
     temporal_param = coverage.temporal_parameter_name
     try:
         bounds = coverage.get_data_bounds(temporal_param)
     except ValueError:
         return (coverage.get_parameter_context(temporal_param).fill_value,) * 2
     uom = coverage.get_parameter_context(temporal_param).uom
     new_bounds = (TimeUtils.units_to_ts(uom,bounds[0]), TimeUtils.units_to_ts(uom,bounds[1]))
     return new_bounds
Ejemplo n.º 4
0
    def sync_rdt_with_coverage(self, coverage=None, tdoa=None, start_time=None, end_time=None, stride_time=None, parameters=None):
        '''
        Builds a granule based on the coverage
        '''
        if coverage is None:
            coverage = self.coverage

        slice_ = slice(None) # Defaults to all values
        if tdoa is not None and isinstance(tdoa,slice):
            slice_ = tdoa

        elif stride_time is not None:
            validate_is_instance(start_time, Number, 'start_time must be a number for striding.')
            validate_is_instance(end_time, Number, 'end_time must be a number for striding.')
            validate_is_instance(stride_time, Number, 'stride_time must be a number for striding.')
            ugly_range = np.arange(start_time, end_time, stride_time)
            idx_values = [TimeUtils.get_relative_time(coverage,i) for i in ugly_range]
            slice_ = [idx_values]

        elif not (start_time is None and end_time is None):
            time_var = coverage._temporal_param_name
            uom = coverage.get_parameter_context(time_var).uom
            if start_time is not None:
                start_units = TimeUtils.ts_to_units(uom,start_time)
                log.info('Units: %s', start_units)
                start_idx = TimeUtils.get_relative_time(coverage,start_units)
                log.info('Start Index: %s', start_idx)
                start_time = start_idx
            if end_time is not None:
                end_units   = TimeUtils.ts_to_units(uom,end_time)
                log.info('End units: %s', end_units)
                end_idx   = TimeUtils.get_relative_time(coverage,end_units)
                log.info('End index: %s',  end_idx)
                end_time = end_idx
            slice_ = slice(start_time,end_time,stride_time)
            log.info('Slice: %s', slice_)

        if parameters is not None:
            pdict = ParameterDictionary()
            params = set(coverage.list_parameters()).intersection(parameters)
            for param in params:
                pdict.add_context(coverage.get_parameter_context(param))
            rdt = RecordDictionaryTool(param_dictionary=pdict)
            self.pdict = pdict
        else:
            rdt = RecordDictionaryTool(param_dictionary=coverage.parameter_dictionary)
        
        fields = coverage.list_parameters()
        if parameters is not None:
            fields = set(fields).intersection(parameters)

        for d in fields:
            rdt[d] = coverage.get_parameter_values(d,tdoa=slice_)
        self.rdt = rdt # Sync
 def dataset_temporal_bounds(self, dataset_id):
     coverage = self._get_coverage(dataset_id)
     temporal_param = coverage.temporal_parameter_name
     try:
         bounds = coverage.get_data_bounds(temporal_param)
     except ValueError:
         return (coverage.get_parameter_context(temporal_param).fill_value,
                 ) * 2
     uom = coverage.get_parameter_context(temporal_param).uom
     new_bounds = (TimeUtils.units_to_ts(uom, bounds[0]),
                   TimeUtils.units_to_ts(uom, bounds[1]))
     return new_bounds
    def insert_values(self, coverage, rdt, stream_id):
        elements = len(rdt)

        start_index = coverage.num_timesteps - elements

        for k, v in rdt.iteritems():
            if isinstance(v, SparseConstantValue):
                continue
            slice_ = slice(start_index, None)
            try:
                coverage.set_parameter_values(param_name=k,
                                              tdoa=slice_,
                                              value=v)
            except IOError as e:
                log.error("Couldn't insert values for coverage: %s",
                          coverage.persistence_dir,
                          exc_info=True)
                try:
                    coverage.close()
                finally:
                    self._bad_coverages[stream_id] = 1
                    raise CorruptionError(e.message)

        if 'ingestion_timestamp' in coverage.list_parameters():
            t_now = time.time()
            ntp_time = TimeUtils.ts_to_units(
                coverage.get_parameter_context('ingestion_timestamp').uom,
                t_now)
            coverage.set_parameter_values(param_name='ingestion_timestamp',
                                          tdoa=slice_,
                                          value=ntp_time)
    def insert_values(self, coverage, rdt, stream_id):
        elements = len(rdt)

        start_index = coverage.num_timesteps - elements

        for k,v in rdt.iteritems():
            if isinstance(v, SparseConstantValue):
                continue
            slice_ = slice(start_index, None)
            try:
                coverage.set_parameter_values(param_name=k, tdoa=slice_, value=v)
            except IOError as e:
                log.error("Couldn't insert values for coverage: %s",
                          coverage.persistence_dir, exc_info=True)
                try:
                    coverage.close()
                finally:
                    self._bad_coverages[stream_id] = 1
                    raise CorruptionError(e.message)
            except IndexError as e:
                log.error("Value set: %s", v[:])
                data_products, _ = self.container.resource_registry.find_subjects(object=stream_id, predicate=PRED.hasStream, subject_type=RT.DataProduct)
                for data_product in data_products:
                    log.exception("Index exception with %s, trying to insert %s into coverage with shape %s", 
                                  data_product.name,
                                  k,
                                  v.shape)

    
        if 'ingestion_timestamp' in coverage.list_parameters():
            t_now = time.time()
            ntp_time = TimeUtils.ts_to_units(coverage.get_parameter_context('ingestion_timestamp').uom, t_now)
            coverage.set_parameter_values(param_name='ingestion_timestamp', tdoa=slice_, value=ntp_time)
 def dataset_temporal_bounds(self, dataset_id):
     dataset = self.read_dataset(dataset_id)
     if not dataset:
         return {}
     pdict = ParameterDictionary.load(dataset.parameter_dictionary)
     temporal_parameter = pdict.temporal_parameter_name
     units = pdict.get_temporal_context().uom
     bounds = self.dataset_bounds(dataset_id)
     if not bounds:
         return {}
     bounds = bounds[temporal_parameter or 'time']
     bounds = [TimeUtils.units_to_ts(units, i) for i in bounds]
     return bounds
    def dataset_temporal_bounds(self, dataset_id):
        dataset = self.read_dataset(dataset_id)

        parameter_dictionary_id = self.clients.resource_registry.find_objects(dataset_id, PRED.hasParameterDictionary, id_only=True)[0][0]
        pdict = self._coverage_parameter_dictionary(parameter_dictionary_id)

        if not dataset:
            return {}

        temporal_parameter = pdict.temporal_parameter_name
        units = pdict.get_temporal_context().uom
        bounds = self.dataset_bounds(dataset_id)
        if not bounds:
            return {}
        bounds = bounds[temporal_parameter or 'time']
        bounds = [TimeUtils.units_to_ts(units, i) for i in bounds]
        return bounds
    def recv_packet(self, msg, stream_route, stream_id):
        validate_is_instance(msg, Granule, "Incoming packet must be of type granule")

        cov = self.get_coverage(stream_id)
        if cov:
            cov.insert_timesteps(1)

            if "raw" in cov.list_parameters():
                gran = IonObjectSerializer().serialize(msg)
                cov.set_parameter_values(param_name="raw", value=[gran])

            if "ingestion_timestamp" in cov.list_parameters():
                t_now = time.time()
                ntp_time = TimeUtils.ts_to_units(cov.get_parameter_context("ingestion_timestamp").uom, t_now)
                cov.set_parameter_values(param_name="ingestion_timestamp", value=ntp_time)

            self.dataset_changed(self.get_dataset(stream_id), cov.num_timesteps)
Ejemplo n.º 11
0
    def dataset_temporal_bounds(self, dataset_id):
        dataset = self.read_dataset(dataset_id)

        parameter_dictionary_id = self.clients.resource_registry.find_objects(
            dataset_id, PRED.hasParameterDictionary, id_only=True)[0][0]
        pdict = self._coverage_parameter_dictionary(parameter_dictionary_id)

        if not dataset:
            return {}

        temporal_parameter = pdict.temporal_parameter_name
        units = pdict.get_temporal_context().uom
        bounds = self.dataset_bounds(dataset_id)
        if not bounds:
            return {}
        bounds = bounds[temporal_parameter or 'time']
        bounds = [TimeUtils.units_to_ts(units, i) for i in bounds]
        return bounds
    def recv_packet(self, msg, stream_route, stream_id):
        validate_is_instance(msg, Granule,
                             'Incoming packet must be of type granule')

        cov = self.get_coverage(stream_id)
        if cov:
            cov.insert_timesteps(1)

            if 'raw' in cov.list_parameters():
                gran = IonObjectSerializer().serialize(msg)
                cov.set_parameter_values(param_name='raw', value=[gran])

            if 'ingestion_timestamp' in cov.list_parameters():
                t_now = time.time()
                ntp_time = TimeUtils.ts_to_units(
                    cov.get_parameter_context('ingestion_timestamp').uom,
                    t_now)
                cov.set_parameter_values(param_name='ingestion_timestamp',
                                         value=ntp_time)

            self.dataset_changed(self.get_dataset(stream_id),
                                 cov.num_timesteps)
    def insert_values(self, coverage, rdt, stream_id):
        elements = len(rdt)

        start_index = coverage.num_timesteps - elements

        for k,v in rdt.iteritems():
            if isinstance(v, SparseConstantValue):
                continue
            slice_ = slice(start_index, None)
            try:
                coverage.set_parameter_values(param_name=k, tdoa=slice_, value=v)
            except IOError as e:
                log.error("Couldn't insert values for coverage: %s",
                          coverage.persistence_dir, exc_info=True)
                try:
                    coverage.close()
                finally:
                    self._bad_coverages[stream_id] = 1
                    raise CorruptionError(e.message)
    
        if 'ingestion_timestamp' in coverage.list_parameters():
            t_now = time.time()
            ntp_time = TimeUtils.ts_to_units(coverage.get_parameter_context('ingestion_timestamp').uom, t_now)
            coverage.set_parameter_values(param_name='ingestion_timestamp', tdoa=slice_, value=ntp_time)
Ejemplo n.º 14
0
    def execute(input=None,
                context=None,
                config=None,
                params=None,
                state=None):

        stream_definition_id = params

        #init stuff
        rdt_for_nones = {}
        data_table_content = []
        gdt_allowed_numerical_types = [
            'int8', 'int16', 'int32', 'int64', 'uint8', 'uint16', 'uint32',
            'uint64', 'float32', 'float64', 'str'
        ]
        # TODO : Move this in to container parameter
        default_precision = 5

        rdt = RecordDictionaryTool.load_from_granule(input)

        data_description = []
        # Buid a local precisions and fill value dictionary to use for parsing data correctly

        precisions = {}
        fill_values = {}
        for field in rdt.fields:
            precision_str = rdt.context(field).precision
            if not precision_str:
                precisions[field] = default_precision
            else:
                try:
                    precisions[field] = int(precision_str)
                except ValueError:
                    precisions[field] = default_precision

            fill_values[field] = rdt.fill_value(field)

        if stream_definition_id == None:
            log.error(
                "GoogleDT transform: Need a output stream definition to process graphs"
            )
            return None

        fields = []
        fields = rdt.fields

        # Ascertain temporal field. Use 'time' as backup
        time_field = rdt.temporal_parameter or 'time'
        #if 'time' not in rdt: return None
        if rdt[time_field] is None:
            return None

        #time_fill_value = 0.0 # should be derived from the granule's param dict.
        time_fill_value = fill_values[
            time_field]  # should be derived from the granule's param dict.
        total_num_of_records = len(rdt[time_field])
        data_description.append(('time', 'number', 'time'))

        ###### DEBUG ##########
        #for field in fields:
        #    if hasattr(rdt.context(field),'visible'):
        #        print "  >>>>>>>> '", field, "' [visible = ", rdt.context(field).visible,"] : ", rdt[field]
        #    else:
        #        print "  >>>>>>>> '", field, "' [visible = NOT SPECIFIED] : ", rdt[field]

        import re
        for field in fields:

            if field == time_field:
                continue

            # If a config block was passed, consider only the params listed in it
            if config and 'parameters' in config and len(
                    config['parameters']) > 0:
                if not field in config['parameters']:
                    log.info(
                        "Skipping ", field,
                        " since it was not present in the list of allowed parameters"
                    )
                    continue

            # If the value is none, assign it a small one fill_value array for now to generate description,
            # Actual array of fill_values will be assigned later
            rdt_field = rdt[field]
            if rdt_field == None:
                rdt_for_nones[field] = np.array([fill_values[field]] *
                                                total_num_of_records)
                rdt_field = rdt_for_nones[field]

            # Check if visibility is false (system generated params) or not specified explicitly
            if hasattr(rdt.context(field),
                       'visible') and not rdt.context(field).visible:
                continue

            # If it's a QC parameter ignore it
            if field.endswith('_qc'):
                continue

            # Handle string type or if its an unknown type, convert to string
            context = rdt.context(field)
            if (rdt_field.dtype == 'string'
                    or rdt_field.dtype not in gdt_allowed_numerical_types):
                data_description.append((field, 'string', field))
            elif (isinstance(context.param_type, ArrayType) or isinstance(
                    context.param_type,
                    ParameterFunctionType)) and len(rdt_field.shape) > 1:
                for i in xrange(rdt_field.shape[1]):
                    data_description.append(('%s[%s]' % (field, i), 'number',
                                             '%s[%s]' % (field, i), {
                                                 'precision':
                                                 str(precisions[field])
                                             }))
            else:
                data_description.append((field, 'number', field, {
                    'precision': str(precisions[field])
                }))

        for i in xrange(len(rdt)):
            varTuple = []

            # Put time first if its not zero. Retrieval returns 0 secs for malformed entries
            if rdt[time_field][i] == time_fill_value:
                continue
            # convert timestamp from instrument to UNIX time stamp since thats what Google DT expects
            varTuple.append(ntplib.ntp_to_system_time(rdt[time_field][i]))

            for dd in data_description:
                field = dd[0]
                field_type = dd[1]
                # ignore time since its been already added
                if field == None or field == time_field:
                    continue
                """
                rdt_field = rdt[field]
                if rdt_field == None:
                    rdt_field = np.array([fill_values[field]] * total_num_of_records)
                """

                if re.match(r'.*\[\d+\]', field):
                    field, j = re.match(r'(.*)\[(\d+)\]', field).groups()
                    j = int(j)
                    varTuple.append(float(rdt[field][i][j]))
                else:
                    if (field_type == 'number'):
                        if rdt[field] == None or rdt[field][i] == fill_values[
                                field]:
                            varTuple.append(None)
                        else:
                            # Adjust float for precision
                            if (precisions[field] == None):
                                varTuple.append(float(rdt[field][i]))
                            else:
                                varTuple.append(
                                    round(float(rdt[field][i]),
                                          precisions[field]))

                    # if field type is string, there are two possibilities. Either it really is a string or
                    # its an object that needs to be converted to string.
                    if (field_type == 'string'):
                        if rdt[field] == None or rdt[field][i] == fill_values[
                                field]:
                            varTuple.append(None)
                        else:
                            varTuple.append(str(rdt[field][i]))

            # Append the tuples to the data table
            if len(varTuple) > 0:
                data_table_content.append(varTuple)

        out_rdt = RecordDictionaryTool(
            stream_definition_id=stream_definition_id)

        # Prepare granule content
        out_dict = {
            "viz_product_type": "google_dt",
            "data_description": data_description,
            "data_content": data_table_content
        }

        out_rdt["google_dt_components"] = np.array([out_dict])
        out_rdt["viz_timestamp"] = TimeUtils.ts_to_units(
            rdt.context(time_field).uom, time.time())

        log.debug('Google DT transform: Sending a granule')
        out_granule = out_rdt.to_granule()

        return out_granule
 def ntp_from_iso(self, iso):
     return TimeUtils.ntp_from_iso(iso)
Ejemplo n.º 16
0
    def execute(input=None, context=None, config=None, params=None, state=None):

        stream_definition_id = params

        #init stuff
        data_description = []
        data_table_content = []
        gdt_allowed_numerical_types = ['int8', 'int16', 'int32', 'int64', 'uint8', 'uint16', 'uint32',
                                       'uint64', 'float32', 'float64','str']
        # TODO : Move this in to container parameter
        default_precision = 5

        rdt = RecordDictionaryTool.load_from_granule(input)


        data_description = []
        # Buid a local precisions and fill value dictionary to use for parsing data correctly

        precisions = {}
        fill_values = {}
        for field in rdt.fields:
            precision_str = rdt.context(field).precision
            if not precision_str:
                precisions[field] = default_precision
            else:
                try:
                    precisions[field] = int(precision_str)
                except ValueError:
                    precisions[field] = default_precision


            fill_values[field] = rdt.fill_value(field)

        if stream_definition_id == None:
            log.error("GoogleDT transform: Need a output stream definition to process graphs")
            return None

        fields = []
        fields = rdt.fields


        # if time was null or missing, do not process
        if 'time' not in rdt: return None
        if rdt['time'] is None:
            return None

        time_fill_value = 0.0 # should be derived from the granule's param dict.
        data_description.append(('time','number','time'))

        import re
        for field in fields:

            if field == rdt.temporal_parameter:
                continue

            # If a config block was passed, consider only the params listed in it
            if config and 'parameters' in config and len(config['parameters']) > 0:
                if not field in config['parameters']:
                    log.info("Skipping ", field, " since it was not present in the list of allowed parameters")
                    continue

            # only consider fields which are allowed.
            if rdt[field] == None:
                continue

            # Check if visibility is false (system generated params)
            if hasattr(rdt.context(field),'visible') and not rdt.context(field).visible:
                continue

            # If it's a QC parameter ignore it
            if field.endswith('_qc'):
                continue

            # Handle string type or if its an unknown type, convert to string
            context = rdt.context(field)
            if (rdt[field].dtype == 'string' or rdt[field].dtype not in gdt_allowed_numerical_types):
                data_description.append((field, 'string', field ))
            elif (isinstance(context.param_type, ArrayType) or isinstance(context.param_type,ParameterFunctionType)) and len(rdt[field].shape)>1:
                for i in xrange(rdt[field].shape[1]):
                    data_description.append(('%s[%s]' % (field,i), 'number', '%s[%s]' % (field,i), {'precision':str(precisions[field])}))
            else:
                data_description.append((field, 'number', field, {'precision':str(precisions[field])} ))


        for i in xrange(len(rdt)):
            varTuple = []

            # Put time first if its not zero. Retrieval returns 0 secs for malformed entries
            if rdt['time'][i] == time_fill_value:
                continue
            # convert timestamp from instrument to UNIX time stamp since thats what Google DT expects
            varTuple.append(ntplib.ntp_to_system_time(rdt['time'][i]))

            for dd in data_description:
                field = dd[0]
                field_type = dd[1]
                # ignore time since its been already added
                if field == None or field == 'time':
                    continue

                if re.match(r'.*\[\d+\]', field):
                    field, j = re.match(r'(.*)\[(\d+)\]', field).groups()
                    j = int(j)
                    varTuple.append(float(rdt[field][i][j]))
                elif rdt[field] == None or rdt[field][i] == None:
                    varTuple.append(None)
                else:
                    if(field_type == 'number'):
                        if rdt[field][i] == None or rdt[field][i] == fill_values[field]:
                            varTuple.append(None)
                        else:
                            # Adjust float for precision
                            if (precisions[field] == None):
                                varTuple.append(float(rdt[field][i]))
                            else:
                                varTuple.append(round(float(rdt[field][i]), precisions[field]))

                    # if field type is string, there are two possibilities. Either it really is a string or
                    # its an object that needs to be converted to string.
                    if(field_type == 'string'):
                        varTuple.append(str(rdt[field][i]))

            # Append the tuples to the data table
            if len(varTuple) > 0:
                data_table_content.append(varTuple)

        out_rdt = RecordDictionaryTool(stream_definition_id=stream_definition_id)

        # Prepare granule content
        out_dict = {"viz_product_type" : "google_dt",
                    "data_description" : data_description,
                    "data_content" : data_table_content}

        out_rdt["google_dt_components"] = np.array([out_dict])
        out_rdt["viz_timestamp"] = TimeUtils.ts_to_units(rdt.context(rdt.temporal_parameter).uom, time.time())

        log.debug('Google DT transform: Sending a granule')
        out_granule = out_rdt.to_granule()

        return out_granule
Ejemplo n.º 17
0
    def render_graphs(cls,
                      rdt,
                      graph_data,
                      stream_definition_id,
                      fileName=None,
                      resolution=None,
                      time_field='time'):
        # init Matplotlib with passsed parameters
        x_res = y_res = 100  # some default in case nothing is provided
        if resolution:
            x_res, y_res = resolution.split('x')
            x_res = int(x_res)
            y_res = int(y_res)
        fig = Figure(figsize=(x_res / 100, y_res / 100), dpi=100, frameon=True)
        ax = fig.add_subplot(111)
        canvas = FigureCanvas(fig)
        imgInMem = StringIO.StringIO()

        # Guess xticks for number of labels along x-axis. Place a label every 100 pixels
        ax.locator_params(x_res / 100)

        # If there's no data, wait
        # For the simple case of testing, lets plot all time variant variables one at a time
        xAxisVar = time_field

        # Prepare the set of y axis variables that will be plotted. This needs to be smarter and passed as
        # config variable to the transform
        yAxisVars = []
        for varName, varData in graph_data.iteritems():
            if varName == time_field or varName == 'height' or varName == 'longitude' or varName == 'latitude':
                continue
            yAxisVars.append(varName)

        # Do a error check for incorrect time values. Ignore all time == fill_values
        time_fill_value = 0.0
        clean_data_flag = False
        while not clean_data_flag:
            clean_data_flag = True
            # Go through the data and see if we can find empty time entries. Delete it along
            # with all corresponding variables
            for idx in xrange(len(graph_data[xAxisVar])):
                if graph_data[xAxisVar][idx] == time_fill_value:
                    graph_data[xAxisVar].pop(idx)
                    for varName in yAxisVars:
                        try:
                            graph_data[varName].pop(idx)
                        except IndexError:
                            # Do nothing really. Must be a malformed granule
                            pass
                    clean_data_flag = False
                    break

        #xAxisFloatData = graph_data[xAxisVar]
        xAxisFloatData = [
            datetime.fromtimestamp(ntplib.ntp_to_system_time(t))
            for t in graph_data[xAxisVar]
        ]
        #print " >>>>>>>>>>>>>>>> xAxisFloatData : ", xAxisFloatData

        idx = 0
        for varName in yAxisVars:
            yAxisFloatData = graph_data[varName]

            # Generate the plot
            ax.plot(xAxisFloatData,
                    yAxisFloatData,
                    cls.line_style(idx),
                    label=varName)
            idx += 1

        yAxisLabel = ""
        # generate a filename for the output image
        for varName in yAxisVars:
            if yAxisLabel:
                yAxisLabel = yAxisLabel + "-" + varName
            else:
                yAxisLabel = varName

        if not fileName:
            fileName = yAxisLabel + '_vs_' + xAxisVar

        fileName = fileName + "." + mpl_output_image_format

        # Choose a small font for the legend
        legend_font_prop = FontProperties()
        legend_font_prop.set_size('small')
        ax.set_xlabel(xAxisVar)
        ax.set_ylabel(yAxisLabel)
        ax.set_title(yAxisLabel + ' vs ' + xAxisVar)
        ax.set_autoscale_on(False)
        ax.legend(loc='upper left',
                  ncol=3,
                  fancybox=True,
                  shadow=True,
                  prop=legend_font_prop)

        # Date formatting
        # matplotlib date format object
        #hfmt = dates.DateFormatter('%m/%d %H:%M')
        #ax.xaxis.set_major_locator(dates.MinuteLocator())
        #ax.xaxis.set_major_formatter(hfmt)

        # Save the figure to the in memory file
        canvas.print_figure(imgInMem, format=mpl_output_image_format)
        imgInMem.seek(0)

        # Create output dictionary from the param dict
        out_rdt = RecordDictionaryTool(
            stream_definition_id=stream_definition_id)

        # Prepare granule content
        #out_dict = {}
        out_rdt["viz_product_type"] = ["matplotlib_graphs"]
        out_rdt["image_obj"] = [imgInMem.getvalue()]
        out_rdt["image_name"] = [fileName]
        out_rdt["content_type"] = ["image/png"]

        #print " >>>>>>>>>> OUT_IMAGE_NAME : ", out_rdt['image_name']
        #out_rdt["graph_image_param_dict"] = np.array([out_dict])
        out_rdt["viz_timestamp"] = TimeUtils.ts_to_units(
            rdt.context(rdt.temporal_parameter).uom, time.time())

        return out_rdt.to_granule()
Ejemplo n.º 18
0
    def render_graphs(cls, rdt, graph_data, stream_definition_id, fileName = None, resolution = None, time_field = 'time'):
        # init Matplotlib with passsed parameters
        x_res = y_res = 100 # some default in case nothing is provided
        if resolution:
            x_res, y_res = resolution.split('x')
            x_res = int(x_res)
            y_res = int(y_res)
        fig = Figure(figsize=(x_res / 100, y_res / 100), dpi=100, frameon=True)
        ax = fig.add_subplot(111)
        canvas = FigureCanvas(fig)
        imgInMem = StringIO.StringIO()

        # Guess xticks for number of labels along x-axis. Place a label every 100 pixels
        ax.locator_params(x_res/100)

        # If there's no data, wait
        # For the simple case of testing, lets plot all time variant variables one at a time
        xAxisVar = time_field

        # Prepare the set of y axis variables that will be plotted. This needs to be smarter and passed as
        # config variable to the transform
        yAxisVars = []
        for varName, varData in graph_data.iteritems():
            if varName == time_field or varName == 'height' or varName == 'longitude' or varName == 'latitude':
                continue
            yAxisVars.append(varName)


        # Do a error check for incorrect time values. Ignore all time == fill_values
        time_fill_value = 0.0
        clean_data_flag = False
        while not clean_data_flag:
            clean_data_flag = True
            # Go through the data and see if we can find empty time entries. Delete it along
            # with all corresponding variables
            for idx in xrange(len(graph_data[xAxisVar])):
                if graph_data[xAxisVar][idx] == time_fill_value:
                    graph_data[xAxisVar].pop(idx)
                    for varName in yAxisVars:
                        try:
                            graph_data[varName].pop(idx)
                        except IndexError:
                            # Do nothing really. Must be a malformed granule
                            pass
                    clean_data_flag = False
                    break

        #xAxisFloatData = graph_data[xAxisVar]
        xAxisFloatData = [datetime.fromtimestamp(ntplib.ntp_to_system_time(t)) for t in graph_data[xAxisVar]]
        #print " >>>>>>>>>>>>>>>> xAxisFloatData : ", xAxisFloatData

        idx = 0
        for varName in yAxisVars:
            yAxisFloatData = graph_data[varName]

            # Generate the plot
            ax.plot(xAxisFloatData, yAxisFloatData, cls.line_style(idx), label=varName)
            idx += 1

        yAxisLabel = ""
        # generate a filename for the output image
        for varName in yAxisVars:
            if yAxisLabel:
                yAxisLabel = yAxisLabel + "-" + varName
            else:
                yAxisLabel = varName

        if not fileName:
            fileName = yAxisLabel + '_vs_' + xAxisVar

        fileName = fileName + "." + mpl_output_image_format

        # Choose a small font for the legend
        legend_font_prop = FontProperties()
        legend_font_prop.set_size('small')
        ax.set_xlabel(xAxisVar)
        ax.set_ylabel(yAxisLabel)
        ax.set_title(yAxisLabel + ' vs ' + xAxisVar)
        ax.set_autoscale_on(False)
        ax.legend(loc='upper left', ncol=3, fancybox=True, shadow=True, prop=legend_font_prop)

        # Date formatting
        # matplotlib date format object
        #hfmt = dates.DateFormatter('%m/%d %H:%M')
        #ax.xaxis.set_major_locator(dates.MinuteLocator())
        #ax.xaxis.set_major_formatter(hfmt)

        # Save the figure to the in memory file
        canvas.print_figure(imgInMem, format=mpl_output_image_format)
        imgInMem.seek(0)

        # Create output dictionary from the param dict
        out_rdt = RecordDictionaryTool(stream_definition_id=stream_definition_id)

        # Prepare granule content
        #out_dict = {}
        out_rdt["viz_product_type"] = ["matplotlib_graphs"]
        out_rdt["image_obj"] = [imgInMem.getvalue()]
        out_rdt["image_name"] = [fileName]
        out_rdt["content_type"] = ["image/png"]

        #print " >>>>>>>>>> OUT_IMAGE_NAME : ", out_rdt['image_name']
        #out_rdt["graph_image_param_dict"] = np.array([out_dict])
        out_rdt["viz_timestamp"] = TimeUtils.ts_to_units(rdt.context(rdt.temporal_parameter).uom, time.time())

        return out_rdt.to_granule()
Ejemplo n.º 19
0
    def get_time_idx(cls, coverage, timeval):
        corrected_time = cls.convert_time(coverage, timeval)

        idx = TimeUtils.get_relative_time(coverage, corrected_time)
        return idx
Ejemplo n.º 20
0
    def convert_time(cls, coverage, timeval):
        tname = coverage.temporal_parameter_name
        uom = coverage.get_parameter_context(tname).uom

        corrected_time = TimeUtils.ts_to_units(uom, timeval)
        return corrected_time
Ejemplo n.º 21
0
    def execute(input=None, context=None, config=None, params=None, state=None):

        stream_definition_id = params

        #init stuff
        rdt_for_nones = {}
        hc_data = []
        normalized_ts = []
        hc_allowed_numerical_types = ['int8', 'int16', 'int32', 'int64', 'uint8', 'uint16', 'uint32',
                                       'uint64', 'float32', 'float64','str']
        # TODO : Move this in to container parameter
        default_precision = 5

        rdt = RecordDictionaryTool.load_from_granule(input)
        # Buid a local precisions and fill value dictionary to use for parsing data correctly

        precisions = {}
        fill_values = {}
        for field in rdt.fields:
            precision_str = rdt.context(field).precision
            if not precision_str:
                precisions[field] = default_precision
            else:
                try:
                    precisions[field] = int(precision_str)
                except ValueError:
                    precisions[field] = default_precision


            fill_values[field] = rdt.fill_value(field)

        if stream_definition_id == None:
            log.error("HighCharts transform: Need a output stream definition to process graphs")
            return None

        fields = []
        fields = rdt.fields

        # Ascertain temporal field. Use 'time' as backup
        time_field = rdt.temporal_parameter or 'time'
        #if 'time' not in rdt: return None
        if rdt[time_field] is None:
            return None

        #time_fill_value = 0.0 # should be derived from the granule's param dict.
        time_fill_value = fill_values[time_field] # should be derived from the granule's param dict.
        total_num_of_records = len(rdt[time_field])

        # convert timestamps from ntp to system
        count = 0
        normalized_ts = [None] * total_num_of_records
        for ts in rdt[time_field]:
            if ts == time_fill_value:
                normalized_ts[count] = time_fill_value
            else:
                normalized_ts[count] = float(ntplib.ntp_to_system_time(ts))
            count += 1

        ###### DEBUG ##########
        #for field in fields:
        #    if hasattr(rdt.context(field),'visible'):
        #        print "  >>>>>>>> '", field, "' [visible = ", rdt.context(field).visible,"] : ", rdt[field]
        #    else:
        #        print "  >>>>>>>> '", field, "' [visible = NOT SPECIFIED] : ", rdt[field]

        # Convert the fields in to HC series format
        import re
        for field in fields:
            field_precision = precisions[field]

            if field == time_field:
                continue

            # If a config block was passed, consider only the params listed in it
            if config and 'parameters' in config and len(config['parameters']) > 0:
                if not field in config['parameters']:
                    log.info("Skipping ", field, " since it was not present in the list of allowed parameters")
                    continue

            # If the value is none, assign it a small one fill_value array for now to generate description,
            # Actual array of fill_values will be assigned later
            rdt_field =rdt[field]
            if rdt_field == None:
                rdt_for_nones[field] = np.array([fill_values[field]] * total_num_of_records)
                #rdt_for_nones[field] = [fill_values[field]] * total_num_of_records
                rdt_field = rdt_for_nones[field]

            # Check if visibility is false (system generated params) or not specified explicitly
            if hasattr(rdt.context(field),'visible') and not rdt.context(field).visible:
                continue

            # If it's a QC parameter ignore it
            if field.endswith('_qc'):
                continue

            # NOTE: The reason why the following libes of code seem to make a lot of branches are to pull the major decisions
            #       outside the primary loops responsible for arranging data in the data structures to be sent to the charts.
            #       This is better than having to make the decisions on a per record entry.

            # Handle arrays by spliting them them in to individual parameters
            context = rdt.context(field)
            if (isinstance(context.param_type, ArrayType) or isinstance(context.param_type,ParameterFunctionType)) and len(rdt_field.shape)>1:
                if (rdt_field.dtype == 'string' or rdt_field.dtype not in hc_allowed_numerical_types):
                    for i in xrange(rdt_field.shape[1]):
                        series = {}
                        series["name"] = field + "[" + str(i) + "]"
                        series["visible"] = False
                        series["data"] = VizTransformHighChartsAlgorithm.form_series_data_str(normalized_ts, rdt_field, i, time_fill_value, fill_values[field])
                        hc_data.append(series)

                else: # Assume its a float or number
                    for i in xrange(rdt_field.shape[1]):
                        series = {}
                        series["name"] = field + "[" + str(i) + "]"
                        series["visible"] = True
                        series["tooltip"] = {"valueDecimals":field_precision}
                        series["data"] = VizTransformHighChartsAlgorithm.form_series_data_num(normalized_ts, rdt_field, i, time_fill_value, fill_values[field], field_precision)
                        hc_data.append(series)
            else:
                if (rdt_field.dtype == 'string' or rdt_field.dtype not in hc_allowed_numerical_types):
                    series = {}
                    series["name"] = field
                    series["visible"] = False
                    series["data"] = VizTransformHighChartsAlgorithm.form_series_data_str(normalized_ts, rdt_field, None, time_fill_value, fill_values[field])

                else:
                    series = {}
                    series["name"] = field
                    series["tooltip"] = {"valueDecimals":field_precision}
                    series["visible"] = True
                    series["data"] = VizTransformHighChartsAlgorithm.form_series_data_num(normalized_ts, rdt_field, None, time_fill_value, fill_values[field], field_precision)

                # Append series to the hc data
                hc_data.append(series)

        # Prep the outgoing granule
        out_rdt = RecordDictionaryTool(stream_definition_id=stream_definition_id)

        #out_dict = {"hc_data": hc_data}
        #out_rdt["hc_data"] = np.array([out_dict])

        out_rdt["hc_data"] = [hc_data]
        out_rdt["viz_timestamp"] = TimeUtils.ts_to_units(rdt.context(time_field).uom, time.time())

        log.debug('HighCharts Transform: Sending a granule')
        out_granule = out_rdt.to_granule()

        return out_granule
    def add_granule(self,stream_id, rdt):
        ''' Appends the granule's data to the coverage and persists it. '''
        debugging = log.isEnabledFor(DEBUG)
        timer = Timer() if debugging else None
        if stream_id in self._bad_coverages:
            log.info('Message attempting to be inserted into bad coverage: %s',
                     DatasetManagementService._get_coverage_path(self.get_dataset(stream_id)))
            

        #--------------------------------------------------------------------------------
        # Coverage determiniation and appending
        #--------------------------------------------------------------------------------
        dataset_id = self.get_dataset(stream_id)
        if not dataset_id:
            log.error('No dataset could be determined on this stream: %s', stream_id)
            return
        try:
            coverage = self.get_coverage(stream_id)
        except IOError as e:
            log.error("Couldn't open coverage: %s",
                      DatasetManagementService._get_coverage_path(self.get_dataset(stream_id)))
            raise CorruptionError(e.message)

        if debugging:
            path = DatasetManagementService._get_coverage_path(dataset_id)
            log.debug('%s: add_granule stream %s dataset %s coverage %r file %s',
                      self._id, stream_id, dataset_id, coverage, path)

        if not coverage:
            log.error('Could not persist coverage from granule, coverage is None')
            return
        #--------------------------------------------------------------------------------
        # Actual persistence
        #--------------------------------------------------------------------------------
        elements = len(rdt)
        if debugging:
            timer.complete_step('checks') # lightweight ops, should be zero
        try:
            coverage.insert_timesteps(elements, oob=False)
        except IOError as e:
            log.error("Couldn't insert time steps for coverage: %s",
                      DatasetManagementService._get_coverage_path(self.get_dataset(stream_id)), exc_info=True)
            try:
                coverage.close()
            finally:
                self._bad_coverages[stream_id] = 1
                raise CorruptionError(e.message)
        if debugging:
            timer.complete_step('insert')

        start_index = coverage.num_timesteps - elements

        for k,v in rdt.iteritems():
            slice_ = slice(start_index, None)
            try:
                coverage.set_parameter_values(param_name=k, tdoa=slice_, value=v)
            except IOError as e:
                log.error("Couldn't insert values for coverage: %s",
                          DatasetManagementService._get_coverage_path(self.get_dataset(stream_id)), exc_info=True)
                try:
                    coverage.close()
                finally:
                    self._bad_coverages[stream_id] = 1
                    raise CorruptionError(e.message)
        if 'ingestion_timestamp' in coverage.list_parameters():
            t_now = time.time()
            ntp_time = TimeUtils.ts_to_units(coverage.get_parameter_context('ingestion_timestamp').uom, t_now)
            coverage.set_parameter_values(param_name='ingestion_timestamp', tdoa=slice_, value=ntp_time)
        if debugging:
            timer.complete_step('keys')
        DatasetManagementService._save_coverage(coverage)
        if debugging:
            timer.complete_step('save')
        self.dataset_changed(dataset_id,coverage.num_timesteps,(start_index,start_index+elements))
        if debugging:
            timer.complete_step('notify')
            self._add_timing_stats(timer)
 def ntp_from_iso(self, iso):
     return TimeUtils.ntp_from_iso(iso)
Ejemplo n.º 24
0
    def get_time_idx(cls, coverage, timeval):
        corrected_time = cls.convert_time(coverage, timeval)

        idx = TimeUtils.get_relative_time(coverage, corrected_time)
        return idx
Ejemplo n.º 25
0
    def convert_time(cls, coverage, timeval):
        tname = coverage.temporal_parameter_name
        uom = coverage.get_parameter_context(tname).uom

        corrected_time = TimeUtils.ts_to_units(uom, timeval)
        return corrected_time
Ejemplo n.º 26
0
    def sync_rdt_with_coverage(self,
                               coverage=None,
                               tdoa=None,
                               start_time=None,
                               end_time=None,
                               stride_time=None,
                               parameters=None):
        '''
        Builds a granule based on the coverage
        '''
        if coverage is None:
            coverage = self.coverage

        slice_ = slice(None)  # Defaults to all values
        if tdoa is not None and isinstance(tdoa, slice):
            slice_ = tdoa

        elif stride_time is not None:
            validate_is_instance(start_time, Number,
                                 'start_time must be a number for striding.')
            validate_is_instance(end_time, Number,
                                 'end_time must be a number for striding.')
            validate_is_instance(stride_time, Number,
                                 'stride_time must be a number for striding.')
            ugly_range = np.arange(start_time, end_time, stride_time)
            idx_values = [
                TimeUtils.get_relative_time(coverage, i) for i in ugly_range
            ]
            slice_ = [idx_values]

        elif not (start_time is None and end_time is None):
            time_var = coverage._temporal_param_name
            uom = coverage.get_parameter_context(time_var).uom
            if start_time is not None:
                start_units = TimeUtils.ts_to_units(uom, start_time)
                log.info('Units: %s', start_units)
                start_idx = TimeUtils.get_relative_time(coverage, start_units)
                log.info('Start Index: %s', start_idx)
                start_time = start_idx
            if end_time is not None:
                end_units = TimeUtils.ts_to_units(uom, end_time)
                log.info('End units: %s', end_units)
                end_idx = TimeUtils.get_relative_time(coverage, end_units)
                log.info('End index: %s', end_idx)
                end_time = end_idx
            slice_ = slice(start_time, end_time, stride_time)
            log.info('Slice: %s', slice_)

        if parameters is not None:
            pdict = ParameterDictionary()
            params = set(coverage.list_parameters()).intersection(parameters)
            for param in params:
                pdict.add_context(coverage.get_parameter_context(param))
            rdt = RecordDictionaryTool(param_dictionary=pdict)
            self.pdict = pdict
        else:
            rdt = RecordDictionaryTool(
                param_dictionary=coverage.parameter_dictionary)

        fields = coverage.list_parameters()
        if parameters is not None:
            fields = set(fields).intersection(parameters)

        for d in fields:
            rdt[d] = coverage.get_parameter_values(d, tdoa=slice_)
        self.rdt = rdt  # Sync
Ejemplo n.º 27
0
    def add_granule(self, stream_id, rdt):
        ''' Appends the granule's data to the coverage and persists it. '''
        debugging = log.isEnabledFor(DEBUG)
        timer = Timer() if debugging else None
        if stream_id in self._bad_coverages:
            log.info(
                'Message attempting to be inserted into bad coverage: %s',
                DatasetManagementService._get_coverage_path(
                    self.get_dataset(stream_id)))

        #--------------------------------------------------------------------------------
        # Coverage determiniation and appending
        #--------------------------------------------------------------------------------
        dataset_id = self.get_dataset(stream_id)
        if not dataset_id:
            log.error('No dataset could be determined on this stream: %s',
                      stream_id)
            return
        try:
            coverage = self.get_coverage(stream_id)
        except IOError as e:
            log.error(
                "Couldn't open coverage: %s",
                DatasetManagementService._get_coverage_path(
                    self.get_dataset(stream_id)))
            raise CorruptionError(e.message)

        if debugging:
            path = DatasetManagementService._get_coverage_path(dataset_id)
            log.debug(
                '%s: add_granule stream %s dataset %s coverage %r file %s',
                self._id, stream_id, dataset_id, coverage, path)

        if not coverage:
            log.error(
                'Could not persist coverage from granule, coverage is None')
            return
        #--------------------------------------------------------------------------------
        # Actual persistence
        #--------------------------------------------------------------------------------
        elements = len(rdt)
        if debugging:
            timer.complete_step('checks')  # lightweight ops, should be zero
        try:
            coverage.insert_timesteps(elements, oob=False)
        except IOError as e:
            log.error("Couldn't insert time steps for coverage: %s",
                      DatasetManagementService._get_coverage_path(
                          self.get_dataset(stream_id)),
                      exc_info=True)
            try:
                coverage.close()
            finally:
                self._bad_coverages[stream_id] = 1
                raise CorruptionError(e.message)
        if debugging:
            timer.complete_step('insert')

        start_index = coverage.num_timesteps - elements

        for k, v in rdt.iteritems():
            slice_ = slice(start_index, None)
            try:
                coverage.set_parameter_values(param_name=k,
                                              tdoa=slice_,
                                              value=v)
            except IOError as e:
                log.error("Couldn't insert values for coverage: %s",
                          DatasetManagementService._get_coverage_path(
                              self.get_dataset(stream_id)),
                          exc_info=True)
                try:
                    coverage.close()
                finally:
                    self._bad_coverages[stream_id] = 1
                    raise CorruptionError(e.message)
        if 'ingestion_timestamp' in coverage.list_parameters():
            t_now = time.time()
            ntp_time = TimeUtils.ts_to_units(
                coverage.get_parameter_context('ingestion_timestamp').uom,
                t_now)
            coverage.set_parameter_values(param_name='ingestion_timestamp',
                                          tdoa=slice_,
                                          value=ntp_time)
        if debugging:
            timer.complete_step('keys')
        DatasetManagementService._save_coverage(coverage)
        if debugging:
            timer.complete_step('save')
        self.dataset_changed(dataset_id, coverage.num_timesteps,
                             (start_index, start_index + elements))
        if debugging:
            timer.complete_step('notify')
            self._add_timing_stats(timer)