Esempio n. 1
0
def spatial_markov_trend(
    subquery,
    time_cols,
    num_classes=7,
    w_type="knn",
    num_ngbrs=5,
    permutations=0,
    geom_col="the_geom",
    id_col="cartodb_id",
):
    """
        Predict the trends of a unit based on:
        1. history of its transitions to different classes (e.g., 1st quantile -> 2nd quantile)
        2. average class of its neighbors

        Inputs:
        @param subquery string: e.g., SELECT the_geom, cartodb_id,
          interesting_time_column FROM table_name
        @param time_cols list of strings: list of strings of column names
        @param num_classes (optional): number of classes to break distribution
          of values into. Currently uses quantile bins.
        @param w_type string (optional): weight type ('knn' or 'queen')
        @param num_ngbrs int (optional): number of neighbors (if knn type)
        @param permutations int (optional): number of permutations for test
          stats
        @param geom_col string (optional): name of column which contains the
          geometries
        @param id_col string (optional): name of column which has the ids of
          the table

        Outputs:
        @param trend_up float: probablity that a geom will move to a higher
          class
        @param trend_down float: probablity that a geom will move to a lower
          class
        @param trend float: (trend_up - trend_down) / trend_static
        @param volatility float: a measure of the volatility based on
          probability stddev(prob array)
    """

    if len(time_cols) < 2:
        plpy.error("More than one time column needs to be passed")

    qvals = {
        "id_col": id_col,
        "time_cols": time_cols,
        "geom_col": geom_col,
        "subquery": subquery,
        "num_ngbrs": num_ngbrs,
    }

    try:
        query_result = plpy.execute(pu.construct_neighbor_query(w_type, qvals))
        if len(query_result) == 0:
            return zip([None], [None], [None], [None], [None])
    except plpy.SPIError, e:
        plpy.debug("Query failed with exception %s: %s" % (err, pu.construct_neighbor_query(w_type, qvals)))
        plpy.error("Analysis failed: %s" % e)
        return zip([None], [None], [None], [None], [None])
Esempio n. 2
0
def spatial_markov_trend(subquery,
                         time_cols,
                         num_classes=7,
                         w_type='knn',
                         num_ngbrs=5,
                         permutations=0,
                         geom_col='the_geom',
                         id_col='cartodb_id'):
    """
        Predict the trends of a unit based on:
        1. history of its transitions to different classes (e.g., 1st quantile -> 2nd quantile)
        2. average class of its neighbors

        Inputs:
        @param subquery string: e.g., SELECT the_geom, cartodb_id,
          interesting_time_column FROM table_name
        @param time_cols list of strings: list of strings of column names
        @param num_classes (optional): number of classes to break distribution
          of values into. Currently uses quantile bins.
        @param w_type string (optional): weight type ('knn' or 'queen')
        @param num_ngbrs int (optional): number of neighbors (if knn type)
        @param permutations int (optional): number of permutations for test
          stats
        @param geom_col string (optional): name of column which contains the
          geometries
        @param id_col string (optional): name of column which has the ids of
          the table

        Outputs:
        @param trend_up float: probablity that a geom will move to a higher
          class
        @param trend_down float: probablity that a geom will move to a lower
          class
        @param trend float: (trend_up - trend_down) / trend_static
        @param volatility float: a measure of the volatility based on
          probability stddev(prob array)
    """

    if len(time_cols) < 2:
        plpy.error('More than one time column needs to be passed')

    qvals = {
        "id_col": id_col,
        "time_cols": time_cols,
        "geom_col": geom_col,
        "subquery": subquery,
        "num_ngbrs": num_ngbrs
    }

    try:
        query_result = plpy.execute(pu.construct_neighbor_query(w_type, qvals))
        if len(query_result) == 0:
            return zip([None], [None], [None], [None], [None])
    except plpy.SPIError, e:
        plpy.debug('Query failed with exception %s: %s' %
                   (err, pu.construct_neighbor_query(w_type, qvals)))
        plpy.error('Analysis failed: %s' % e)
        return zip([None], [None], [None], [None], [None])
Esempio n. 3
0
def moran_local(subquery, attr,
                w_type, num_ngbrs, permutations, geom_col, id_col):
    """
    Moran's I implementation for PL/Python
    Andy Eschbacher
    """

    # geometries with attributes that are null are ignored
    # resulting in a collection of not as near neighbors

    qvals = OrderedDict([("id_col", id_col),
                         ("attr1", attr),
                         ("geom_col", geom_col),
                         ("subquery", subquery),
                         ("num_ngbrs", num_ngbrs)])

    query = pu.construct_neighbor_query(w_type, qvals)

    try:
        result = plpy.execute(query)
        # if there are no neighbors, exit
        if len(result) == 0:
            return pu.empty_zipped_array(5)
    except plpy.SPIError, e:
        plpy.error('Analysis failed: %s' % e)
        return pu.empty_zipped_array(5)
Esempio n. 4
0
def moran_local(subquery, attr,
                w_type, num_ngbrs, permutations, geom_col, id_col):
    """
    Moran's I implementation for PL/Python
    Andy Eschbacher
    """

    # geometries with attributes that are null are ignored
    # resulting in a collection of not as near neighbors

    qvals = OrderedDict([("id_col", id_col),
                         ("attr1", attr),
                         ("geom_col", geom_col),
                         ("subquery", subquery),
                         ("num_ngbrs", num_ngbrs)])

    query = pu.construct_neighbor_query(w_type, qvals)

    try:
        result = plpy.execute(query)
        # if there are no neighbors, exit
        if len(result) == 0:
            return pu.empty_zipped_array(5)
    except plpy.SPIError, e:
        plpy.error('Analysis failed: %s' % e)
        return pu.empty_zipped_array(5)
Esempio n. 5
0
def moran_local_bv(subquery, attr1, attr2, permutations, geom_col, id_col,
                   w_type, num_ngbrs):
    """
        Moran's I (local) Bivariate (untested)
    """
    plpy.notice('** Constructing query')

    qvals = {
        "num_ngbrs": num_ngbrs,
        "attr1": attr1,
        "attr2": attr2,
        "subquery": subquery,
        "geom_col": geom_col,
        "id_col": id_col
    }

    query = pu.construct_neighbor_query(w_type, qvals)

    try:
        result = plpy.execute(query)
        # if there are no neighbors, exit
        if len(result) == 0:
            return pu.empty_zipped_array(4)
    except plpy.SPIError:
        plpy.error("Error: areas of interest query failed, " \
                   "check input parameters")
        plpy.notice('** Query failed: "%s"' % query)
        return pu.empty_zipped_array(4)

    ## collect attributes
    attr1_vals = pu.get_attributes(result, 1)
    attr2_vals = pu.get_attributes(result, 2)

    # create weights
    weight = pu.get_weight(result, w_type, num_ngbrs)

    # calculate LISA values
    lisa = ps.esda.moran.Moran_Local_BV(attr1_vals,
                                        attr2_vals,
                                        weight,
                                        permutations=permutations)

    plpy.notice("len of Is: %d" % len(lisa.Is))

    # find clustering of significance
    lisa_sig = quad_position(lisa.q)

    plpy.notice('** Finished calculations')

    return zip(lisa.Is, lisa_sig, lisa.p_sim, weight.id_order)
Esempio n. 6
0
def moran_local_rate(subquery, numerator, denominator, w_type, num_ngbrs,
                     permutations, geom_col, id_col):
    """
        Moran's I Local Rate
        Andy Eschbacher
    """
    # geometries with values that are null are ignored
    # resulting in a collection of not as near neighbors

    query = pu.construct_neighbor_query(
        w_type, {
            "id_col": id_col,
            "numerator": numerator,
            "denominator": denominator,
            "geom_col": geom_col,
            "subquery": subquery,
            "num_ngbrs": num_ngbrs
        })

    try:
        result = plpy.execute(query)
        # if there are no neighbors, exit
        if len(result) == 0:
            return pu.empty_zipped_array(5)
    except plpy.SPIError:
        plpy.error(
            'Error: areas of interest query failed, check input parameters')
        plpy.notice('** Query failed: "%s"' % query)
        plpy.notice('** Error: %s' % plpy.SPIError)
        return pu.empty_zipped_array(5)

    ## collect attributes
    numer = pu.get_attributes(result, 1)
    denom = pu.get_attributes(result, 2)

    weight = pu.get_weight(result, w_type, num_ngbrs)

    # calculate LISA values
    lisa = ps.esda.moran.Moran_Local_Rate(numer,
                                          denom,
                                          weight,
                                          permutations=permutations)

    # find units of significance
    quads = quad_position(lisa.q)

    return zip(lisa.Is, quads, lisa.p_sim, weight.id_order, lisa.y)
Esempio n. 7
0
def moran_local_bv(subquery, attr1, attr2,
                   permutations, geom_col, id_col, w_type, num_ngbrs):
    """
        Moran's I (local) Bivariate (untested)
    """
    plpy.notice('** Constructing query')

    qvals = OrderedDict([("id_col", id_col),
                         ("attr1", attr1),
                         ("attr2", attr2),
                         ("geom_col", geom_col),
                         ("subquery", subquery),
                         ("num_ngbrs", num_ngbrs)])

    query = pu.construct_neighbor_query(w_type, qvals)

    try:
        result = plpy.execute(query)
        # if there are no neighbors, exit
        if len(result) == 0:
            return pu.empty_zipped_array(4)
    except plpy.SPIError:
        plpy.error("Error: areas of interest query failed, " \
                   "check input parameters")
        plpy.notice('** Query failed: "%s"' % query)
        return pu.empty_zipped_array(4)

    ## collect attributes
    attr1_vals = pu.get_attributes(result, 1)
    attr2_vals = pu.get_attributes(result, 2)

    # create weights
    weight = pu.get_weight(result, w_type, num_ngbrs)

    # calculate LISA values
    lisa = ps.esda.moran.Moran_Local_BV(attr1_vals, attr2_vals, weight,
                                        permutations=permutations)

    plpy.notice("len of Is: %d" % len(lisa.Is))

    # find clustering of significance
    lisa_sig = quad_position(lisa.q)

    plpy.notice('** Finished calculations')

    return zip(lisa.Is, lisa_sig, lisa.p_sim, weight.id_order)
Esempio n. 8
0
def moran(subquery, attr_name, w_type, num_ngbrs, permutations, geom_col,
          id_col):
    """
    Moran's I (global)
    Implementation building neighbors with a PostGIS database and Moran's I
     core clusters with PySAL.
    Andy Eschbacher
    """
    qvals = {
        "id_col": id_col,
        "attr1": attr_name,
        "geom_col": geom_col,
        "subquery": subquery,
        "num_ngbrs": num_ngbrs
    }

    query = pu.construct_neighbor_query(w_type, qvals)

    plpy.notice('** Query: %s' % query)

    try:
        result = plpy.execute(query)
        # if there are no neighbors, exit
        if len(result) == 0:
            return pu.empty_zipped_array(2)
        plpy.notice('** Query returned with %d rows' % len(result))
    except plpy.SPIError:
        plpy.error(
            'Error: areas of interest query failed, check input parameters')
        plpy.notice('** Query failed: "%s"' % query)
        plpy.notice('** Error: %s' % plpy.SPIError)
        return pu.empty_zipped_array(2)

    ## collect attributes
    attr_vals = pu.get_attributes(result)

    ## calculate weights
    weight = pu.get_weight(result, w_type, num_ngbrs)

    ## calculate moran global
    moran_global = ps.esda.moran.Moran(attr_vals,
                                       weight,
                                       permutations=permutations)

    return zip([moran_global.I], [moran_global.EI])
Esempio n. 9
0
def moran_rate(subquery, numerator, denominator, w_type, num_ngbrs,
               permutations, geom_col, id_col):
    """
    Moran's I Rate (global)
    Andy Eschbacher
    """
    qvals = {
        "id_col": id_col,
        "attr1": numerator,
        "attr2": denominator,
        "geom_col": geom_col,
        "subquery": subquery,
        "num_ngbrs": num_ngbrs
    }

    query = pu.construct_neighbor_query(w_type, qvals)

    plpy.notice('** Query: %s' % query)

    try:
        result = plpy.execute(query)
        # if there are no neighbors, exit
        if len(result) == 0:
            return pu.empty_zipped_array(2)
        plpy.notice('** Query returned with %d rows' % len(result))
    except plpy.SPIError:
        plpy.error(
            'Error: areas of interest query failed, check input parameters')
        plpy.notice('** Query failed: "%s"' % query)
        plpy.notice('** Error: %s' % plpy.SPIError)
        return pu.empty_zipped_array(2)

    ## collect attributes
    numer = pu.get_attributes(result, 1)
    denom = pu.get_attributes(result, 2)

    weight = pu.get_weight(result, w_type, num_ngbrs)

    ## calculate moran global rate
    lisa_rate = ps.esda.moran.Moran_Rate(numer,
                                         denom,
                                         weight,
                                         permutations=permutations)

    return zip([lisa_rate.I], [lisa_rate.EI])
Esempio n. 10
0
def moran_local(subquery, attr, w_type, num_ngbrs, permutations, geom_col,
                id_col):
    """
    Moran's I implementation for PL/Python
    Andy Eschbacher
    """

    # geometries with attributes that are null are ignored
    # resulting in a collection of not as near neighbors

    qvals = {
        "id_col": id_col,
        "attr1": attr,
        "geom_col": geom_col,
        "subquery": subquery,
        "num_ngbrs": num_ngbrs
    }

    query = pu.construct_neighbor_query(w_type, qvals)

    try:
        result = plpy.execute(query)
        # if there are no neighbors, exit
        if len(result) == 0:
            return pu.empty_zipped_array(5)
    except plpy.SPIError:
        plpy.error(
            'Error: areas of interest query failed, check input parameters')
        plpy.notice('** Query failed: "%s"' % query)
        return pu.empty_zipped_array(5)

    attr_vals = pu.get_attributes(result)
    weight = pu.get_weight(result, w_type, num_ngbrs)

    # calculate LISA values
    lisa = ps.esda.moran.Moran_Local(attr_vals,
                                     weight,
                                     permutations=permutations)

    # find quadrants for each geometry
    quads = quad_position(lisa.q)

    return zip(lisa.Is, quads, lisa.p_sim, weight.id_order, lisa.y)
Esempio n. 11
0
def moran_local_rate(subquery, numerator, denominator,
                     w_type, num_ngbrs, permutations, geom_col, id_col):
    """
        Moran's I Local Rate
        Andy Eschbacher
    """
    # geometries with values that are null are ignored
    # resulting in a collection of not as near neighbors

    qvals = OrderedDict([("id_col", id_col),
                         ("numerator", numerator),
                         ("denominator", denominator),
                         ("geom_col", geom_col),
                         ("subquery", subquery),
                         ("num_ngbrs", num_ngbrs)])

    query = pu.construct_neighbor_query(w_type, qvals)

    try:
        result = plpy.execute(query)
        # if there are no neighbors, exit
        if len(result) == 0:
            return pu.empty_zipped_array(5)
    except plpy.SPIError:
        plpy.error('Error: areas of interest query failed, check input parameters')
        plpy.notice('** Query failed: "%s"' % query)
        plpy.notice('** Error: %s' % plpy.SPIError)
        return pu.empty_zipped_array(5)

    ## collect attributes
    numer = pu.get_attributes(result, 1)
    denom = pu.get_attributes(result, 2)

    weight = pu.get_weight(result, w_type, num_ngbrs)

    # calculate LISA values
    lisa = ps.esda.moran.Moran_Local_Rate(numer, denom, weight,
                                          permutations=permutations)

    # find quadrants for each geometry
    quads = quad_position(lisa.q)

    return zip(lisa.Is, quads, lisa.p_sim, weight.id_order, lisa.y)
Esempio n. 12
0
def moran_rate(subquery, numerator, denominator, w_type, num_ngbrs,
               permutations, geom_col, id_col):
    """
    Moran's I Rate (global)
    Andy Eschbacher
    """
    qvals = OrderedDict([("id_col", id_col), ("attr1", numerator),
                         ("attr2", denominator)("geom_col", geom_col),
                         ("subquery", subquery), ("num_ngbrs", num_ngbrs)])

    query = pu.construct_neighbor_query(w_type, qvals)

    try:
        result = plpy.execute(query)
        # if there are no neighbors, exit
        if len(result) == 0:
            return pu.empty_zipped_array(2)
    except plpy.SPIError, e:
        plpy.error('Analysis failed: %s' % e)
        return pu.empty_zipped_array(2)
Esempio n. 13
0
def moran(subquery, attr_name,
          w_type, num_ngbrs, permutations, geom_col, id_col):
    """
    Moran's I (global)
    Implementation building neighbors with a PostGIS database and Moran's I
     core clusters with PySAL.
    Andy Eschbacher
    """
    qvals = OrderedDict([("id_col", id_col),
                         ("attr1", attr_name),
                         ("geom_col", geom_col),
                         ("subquery", subquery),
                         ("num_ngbrs", num_ngbrs)])

    query = pu.construct_neighbor_query(w_type, qvals)

    plpy.notice('** Query: %s' % query)

    try:
        result = plpy.execute(query)
        # if there are no neighbors, exit
        if len(result) == 0:
            return pu.empty_zipped_array(2)
        plpy.notice('** Query returned with %d rows' % len(result))
    except plpy.SPIError:
        plpy.error('Error: areas of interest query failed, check input parameters')
        plpy.notice('** Query failed: "%s"' % query)
        plpy.notice('** Error: %s' % plpy.SPIError)
        return pu.empty_zipped_array(2)

    ## collect attributes
    attr_vals = pu.get_attributes(result)

    ## calculate weights
    weight = pu.get_weight(result, w_type, num_ngbrs)

    ## calculate moran global
    moran_global = ps.esda.moran.Moran(attr_vals, weight,
                                       permutations=permutations)

    return zip([moran_global.I], [moran_global.EI])
Esempio n. 14
0
def moran_rate(subquery, numerator, denominator,
               w_type, num_ngbrs, permutations, geom_col, id_col):
    """
    Moran's I Rate (global)
    Andy Eschbacher
    """
    qvals = OrderedDict([("id_col", id_col),
                         ("attr1", numerator),
                         ("attr2", denominator)
                         ("geom_col", geom_col),
                         ("subquery", subquery),
                         ("num_ngbrs", num_ngbrs)])

    query = pu.construct_neighbor_query(w_type, qvals)

    plpy.notice('** Query: %s' % query)

    try:
        result = plpy.execute(query)
        # if there are no neighbors, exit
        if len(result) == 0:
            return pu.empty_zipped_array(2)
        plpy.notice('** Query returned with %d rows' % len(result))
    except plpy.SPIError:
        plpy.error('Error: areas of interest query failed, check input parameters')
        plpy.notice('** Query failed: "%s"' % query)
        plpy.notice('** Error: %s' % plpy.SPIError)
        return pu.empty_zipped_array(2)

    ## collect attributes
    numer = pu.get_attributes(result, 1)
    denom = pu.get_attributes(result, 2)

    weight = pu.get_weight(result, w_type, num_ngbrs)

    ## calculate moran global rate
    lisa_rate = ps.esda.moran.Moran_Rate(numer, denom, weight,
                                         permutations=permutations)

    return zip([lisa_rate.I], [lisa_rate.EI])
Esempio n. 15
0
def moran_local_bv(subquery, attr1, attr2,
                   permutations, geom_col, id_col, w_type, num_ngbrs):
    """
        Moran's I (local) Bivariate (untested)
    """

    qvals = OrderedDict([("id_col", id_col),
                         ("attr1", attr1),
                         ("attr2", attr2),
                         ("geom_col", geom_col),
                         ("subquery", subquery),
                         ("num_ngbrs", num_ngbrs)])

    query = pu.construct_neighbor_query(w_type, qvals)

    try:
        result = plpy.execute(query)
        # if there are no neighbors, exit
        if len(result) == 0:
            return pu.empty_zipped_array(4)
    except plpy.SPIError:
        plpy.error("Error: areas of interest query failed, "
                   "check input parameters")
        return pu.empty_zipped_array(4)

    # collect attributes
    attr1_vals = pu.get_attributes(result, 1)
    attr2_vals = pu.get_attributes(result, 2)

    # create weights
    weight = pu.get_weight(result, w_type, num_ngbrs)

    # calculate LISA values
    lisa = ps.esda.moran.Moran_Local_BV(attr1_vals, attr2_vals, weight,
                                        permutations=permutations)

    # find clustering of significance
    lisa_sig = quad_position(lisa.q)

    return zip(lisa.Is, lisa_sig, lisa.p_sim, weight.id_order)
Esempio n. 16
0
def moran(subquery, attr_name, w_type, num_ngbrs, permutations, geom_col,
          id_col):
    """
    Moran's I (global)
    Implementation building neighbors with a PostGIS database and Moran's I
     core clusters with PySAL.
    Andy Eschbacher
    """
    qvals = OrderedDict([("id_col", id_col), ("attr1", attr_name),
                         ("geom_col", geom_col), ("subquery", subquery),
                         ("num_ngbrs", num_ngbrs)])

    query = pu.construct_neighbor_query(w_type, qvals)

    try:
        result = plpy.execute(query)
        # if there are no neighbors, exit
        if len(result) == 0:
            return pu.empty_zipped_array(2)
    except plpy.SPIError, e:
        plpy.error('Analysis failed: %s' % e)
        return pu.empty_zipped_array(2)
Esempio n. 17
0
def moran_rate(subquery, numerator, denominator,
               w_type, num_ngbrs, permutations, geom_col, id_col):
    """
    Moran's I Rate (global)
    Andy Eschbacher
    """
    qvals = OrderedDict([("id_col", id_col),
                         ("attr1", numerator),
                         ("attr2", denominator)
                         ("geom_col", geom_col),
                         ("subquery", subquery),
                         ("num_ngbrs", num_ngbrs)])

    query = pu.construct_neighbor_query(w_type, qvals)

    try:
        result = plpy.execute(query)
        # if there are no neighbors, exit
        if len(result) == 0:
            return pu.empty_zipped_array(2)
    except plpy.SPIError, e:
        plpy.error('Analysis failed: %s' % e)
        return pu.empty_zipped_array(2)
Esempio n. 18
0
def moran(subquery, attr_name,
          w_type, num_ngbrs, permutations, geom_col, id_col):
    """
    Moran's I (global)
    Implementation building neighbors with a PostGIS database and Moran's I
     core clusters with PySAL.
    Andy Eschbacher
    """
    qvals = OrderedDict([("id_col", id_col),
                         ("attr1", attr_name),
                         ("geom_col", geom_col),
                         ("subquery", subquery),
                         ("num_ngbrs", num_ngbrs)])

    query = pu.construct_neighbor_query(w_type, qvals)

    try:
        result = plpy.execute(query)
        # if there are no neighbors, exit
        if len(result) == 0:
            return pu.empty_zipped_array(2)
    except plpy.SPIError, e:
        plpy.error('Analysis failed: %s' % e)
        return pu.empty_zipped_array(2)
Esempio n. 19
0
    def test_construct_neighbor_query(self):
        """Test construct_neighbor_query"""

        # Compare to raw knn query
        self.assertEqual(pu.construct_neighbor_query("knn", self.params), pu.knn(self.params))
Esempio n. 20
0
    def test_construct_neighbor_query(self):
        """Test construct_neighbor_query"""

        # Compare to raw knn query
        self.assertEqual(pu.construct_neighbor_query('knn', self.params),
                         pu.knn(self.params))