def _send_to_plpy(self, level, text, exception=None):
# exception might also be a tuple generated by sys.exc_info
if exception:
if isinstance(exception, tuple) and len(exception) > 1:
exception = exception[1]
exception_message = '. Exception: {}'.format(exception)
else:
exception_message = ''
# Adding trace breaks tests
# trace = traceback.format_exc(15)
# message = '{}{}. Trace: {}'.format(text, exception_message, trace)
message = '{}{}'.format(text, exception_message)
if self._check_plpy():
if level == 'debug':
plpy.debug(message)
elif level == 'info':
plpy.info(message)
elif level == 'warning':
plpy.warning(message)
elif level == 'error':
# Plpy.error and fatal raises exceptions and we only want to
# log an error, exceptions should be raise explicitly
plpy.warning(message)
def spatial_markov_trend(
subquery,
time_cols,
num_classes=7,
w_type="knn",
num_ngbrs=5,
permutations=0,
geom_col="the_geom",
id_col="cartodb_id",
):
"""
Predict the trends of a unit based on:
1. history of its transitions to different classes (e.g., 1st quantile -> 2nd quantile)
2. average class of its neighbors
Inputs:
@param subquery string: e.g., SELECT the_geom, cartodb_id,
interesting_time_column FROM table_name
@param time_cols list of strings: list of strings of column names
@param num_classes (optional): number of classes to break distribution
of values into. Currently uses quantile bins.
@param w_type string (optional): weight type ('knn' or 'queen')
@param num_ngbrs int (optional): number of neighbors (if knn type)
@param permutations int (optional): number of permutations for test
stats
@param geom_col string (optional): name of column which contains the
geometries
@param id_col string (optional): name of column which has the ids of
the table
Outputs:
@param trend_up float: probablity that a geom will move to a higher
class
@param trend_down float: probablity that a geom will move to a lower
class
@param trend float: (trend_up - trend_down) / trend_static
@param volatility float: a measure of the volatility based on
probability stddev(prob array)
"""
if len(time_cols) < 2:
plpy.error("More than one time column needs to be passed")
qvals = {
"id_col": id_col,
"time_cols": time_cols,
"geom_col": geom_col,
"subquery": subquery,
"num_ngbrs": num_ngbrs,
}
try:
query_result = plpy.execute(pu.construct_neighbor_query(w_type, qvals))
if len(query_result) == 0:
return zip([None], [None], [None], [None], [None])
except plpy.SPIError, e:
plpy.debug("Query failed with exception %s: %s" % (err, pu.construct_neighbor_query(w_type, qvals)))
plpy.error("Analysis failed: %s" % e)
return zip([None], [None], [None], [None], [None])
def spatial_markov_trend(subquery,
time_cols,
num_classes=7,
w_type='knn',
num_ngbrs=5,
permutations=0,
geom_col='the_geom',
id_col='cartodb_id'):
"""
Predict the trends of a unit based on:
1. history of its transitions to different classes (e.g., 1st quantile -> 2nd quantile)
2. average class of its neighbors
Inputs:
@param subquery string: e.g., SELECT the_geom, cartodb_id,
interesting_time_column FROM table_name
@param time_cols list of strings: list of strings of column names
@param num_classes (optional): number of classes to break distribution
of values into. Currently uses quantile bins.
@param w_type string (optional): weight type ('knn' or 'queen')
@param num_ngbrs int (optional): number of neighbors (if knn type)
@param permutations int (optional): number of permutations for test
stats
@param geom_col string (optional): name of column which contains the
geometries
@param id_col string (optional): name of column which has the ids of
the table
Outputs:
@param trend_up float: probablity that a geom will move to a higher
class
@param trend_down float: probablity that a geom will move to a lower
class
@param trend float: (trend_up - trend_down) / trend_static
@param volatility float: a measure of the volatility based on
probability stddev(prob array)
"""
if len(time_cols) < 2:
plpy.error('More than one time column needs to be passed')
qvals = {
"id_col": id_col,
"time_cols": time_cols,
"geom_col": geom_col,
"subquery": subquery,
"num_ngbrs": num_ngbrs
}
try:
query_result = plpy.execute(pu.construct_neighbor_query(w_type, qvals))
if len(query_result) == 0:
return zip([None], [None], [None], [None], [None])
except plpy.SPIError, e:
plpy.debug('Query failed with exception %s: %s' %
(err, pu.construct_neighbor_query(w_type, qvals)))
plpy.error('Analysis failed: %s' % e)
return zip([None], [None], [None], [None], [None])
def _send_to_plpy(self, level, text):
if self._check_plpy():
if level == 'debug':
plpy.debug(text)
elif level == 'info':
plpy.info(text)
elif level == 'warning':
plpy.warning(text)
elif level == 'error':
# Plpy.error and fatal raises exceptions and we only want to
# log an error, exceptions should be raise explicitly
plpy.warning(text)
def _send_to_plpy(self, level, text):
if self._check_plpy():
if level == 'debug':
plpy.debug(text)
elif level == 'info':
plpy.info(text)
elif level == 'warning':
plpy.warning(text)
elif level == 'error':
# Plpy.error and fatal raises exceptions and we only want to
# log an error, exceptions should be raise explicitly
plpy.warning(text)
def get_dyn_transfo_params_form_1(params_column, params, time):
''' Return the dynamic transfo parameters.
'''
if isinstance(time, datetime.datetime):
plpy.error('times as strings unsupported for dynamic transforms of form 1')
schema, table, column = tuple(map(plpy.quote_ident, params_column.split('.')))
params = params[0]
select = []
for param in params.values():
if isinstance(param, list):
for dim in param:
append_dim_select(dim, select)
else:
dim = param
append_dim_select(dim, select)
select = ', '.join(select)
q = ('''
with patch as (
select pc_interpolate({column}, 'time', {time:f}, true) point
from {schema}.{table}
where pc_patchmin({column}, 'time') <= {time:f} and
pc_patchmax({column}, 'time') > {time:f}
) select %s from patch
''' % select).format(schema=schema, table=table, column=column, time=time)
plpy.debug(q)
rv = plpy.execute(q)
if len(rv) == 0:
plpy.warning('no parameters for the provided time ({:f})'.format(time))
return None
if len(rv) != 1:
plpy.error('multiple rows returned from time interpolation')
values = rv[0]
for key, param in params.items():
if isinstance(param, list):
for i, dim in enumerate(param):
val = values[dim]
param[i] = val
else:
dim = param
val = values[dim]
params[key] = val
return params
def get_transform(transfoid, time):
''' Return information about the transfo whose id is transfoid. A dict with keys "name",
"params", "func_name", and "func_sign".
'''
if not isinstance(time, (float, str)):
plpy.error('unexpected type for "time" parameter ({})'.format(type(time)))
if isinstance(time, str):
# if time is a string parse it to a datetime object
time = dateutil.parser.parse(time)
q = '''
select t.name as name,
t.parameters_column as params_column, t.parameters as params,
tt.name as func_name, tt.func_signature as func_sign
from li3ds.transfo t
join li3ds.transfo_type tt on t.transfo_type = tt.id
where t.id = {:d}
'''.format(transfoid)
plpy.debug(q)
rv = plpy.execute(q)
if len(rv) < 1:
plpy.error('no transfo with id {:d}'.format(transfoid))
transfo = rv[0]
params_column = transfo['params_column']
params = json.loads(transfo['params'])
if params_column:
# dynamic transform form 1
if not time:
plpy.error('no time value provided for dynamic transfo "{}"'
.format(transfo['name']))
params = get_dyn_transfo_params_form_1(params_column, params, time)
elif params:
if len(params) > 1:
# dynamic tranform form 2
if not time:
plpy.error('no time value provided for dynamic transfo "{}"'
.format(transfo['name']))
params = get_dyn_transfo_params_form_2(params, time)
else:
# static transform
params = params[0]
if params is None:
return None
return transfo['name'], params, transfo['func_name'], transfo['func_sign']
def _transform(obj, type_, func_name, func_sign, params):
''' Transform obj, whose type is type_, using func_name, func_sign and params.
'''
if func_name not in func_names:
plpy.error('function {} is unknown'.format(func_name))
func_name = func_names[func_name]
if isinstance(params, basestring): # NOQA
params = json.loads(params)
args = [params[p] for p in func_sign if p != '_time']
args_str, args_val = args_to_array_string(args)
q = 'select {}(\'{}\'::{}{}) r'.format(func_name, obj, type_, args_str)
plpy.debug(q, args_val)
plan = plpy.prepare(q, ['numeric'] * len(args_val))
rv = plpy.execute(plan, args_val)
if len(rv) != 1:
plpy.error('unexpected number of rows ({}) returned from {}'.format(len(rv), q))
result = rv[0].get('r')
if result is None:
plpy.error('unexpected value None returned from {}'.format(q))
return result
)
if len(query_result) == 0:
return zip([None], [None], [None], [None], [None])
except plpy.SPIError, err:
plpy.debug('Query failed with exception %s: %s' % (err, pu.construct_neighbor_query(w_type, qvals)))
plpy.error('Query failed, check the input parameters')
return zip([None], [None], [None], [None], [None])
## build weight
weights = pu.get_weight(query_result, w_type)
weights.transform = 'r'
## prep time data
t_data = get_time_data(query_result, time_cols)
plpy.debug('shape of t_data %d, %d' % t_data.shape)
plpy.debug('number of weight objects: %d, %d' % (weights.sparse).shape)
plpy.debug('first num elements: %f' % t_data[0, 0])
sp_markov_result = ps.Spatial_Markov(t_data,
weights,
k=num_classes,
fixed=False,
permutations=permutations)
## get lag classes
lag_classes = ps.Quantiles(
ps.lag_spatial(weights, t_data[:, -1]),
k=num_classes).yb
## look up probablity distribution for each unit according to class and lag class
query_result = plpy.execute(pu.construct_neighbor_query(w_type, qvals))
if len(query_result) == 0:
return zip([None], [None], [None], [None], [None])
except plpy.SPIError, e:
plpy.debug("Query failed with exception %s: %s" % (err, pu.construct_neighbor_query(w_type, qvals)))
plpy.error("Analysis failed: %s" % e)
return zip([None], [None], [None], [None], [None])
## build weight
weights = pu.get_weight(query_result, w_type)
weights.transform = "r"
## prep time data
t_data = get_time_data(query_result, time_cols)
plpy.debug("shape of t_data %d, %d" % t_data.shape)
plpy.debug("number of weight objects: %d, %d" % (weights.sparse).shape)
plpy.debug("first num elements: %f" % t_data[0, 0])
sp_markov_result = ps.Spatial_Markov(t_data, weights, k=num_classes, fixed=False, permutations=permutations)
## get lag classes
lag_classes = ps.Quantiles(ps.lag_spatial(weights, t_data[:, -1]), k=num_classes).yb
## look up probablity distribution for each unit according to class and lag class
prob_dist = get_prob_dist(sp_markov_result.P, lag_classes, sp_markov_result.classes[:, -1])
## find the ups and down and overall distribution of each cell
trend_up, trend_down, trend, volatility = get_prob_stats(prob_dist, sp_markov_result.classes[:, -1])
## output the results
if len(query_result) == 0:
return zip([None], [None], [None], [None], [None])
except plpy.SPIError, e:
plpy.debug('Query failed with exception %s: %s' %
(err, pu.construct_neighbor_query(w_type, qvals)))
plpy.error('Analysis failed: %s' % e)
return zip([None], [None], [None], [None], [None])
## build weight
weights = pu.get_weight(query_result, w_type)
weights.transform = 'r'
## prep time data
t_data = get_time_data(query_result, time_cols)
plpy.debug('shape of t_data %d, %d' % t_data.shape)
plpy.debug('number of weight objects: %d, %d' % (weights.sparse).shape)
plpy.debug('first num elements: %f' % t_data[0, 0])
sp_markov_result = ps.Spatial_Markov(t_data,
weights,
k=num_classes,
fixed=False,
permutations=permutations)
## get lag classes
lag_classes = ps.Quantiles(ps.lag_spatial(weights, t_data[:, -1]),
k=num_classes).yb
## look up probablity distribution for each unit according to class and lag class
prob_dist = get_prob_dist(sp_markov_result.P, lag_classes,