def clear_statistic_data(widget_url_or_stat,
label=None,
statistic_url=None,
pval=None):
"""Clear data for the requested statistic (which must be scalar: non-list)
Interprets arguments as for the get_statistic function.
Raises LoaderException if the requested Statistic does not exist or is a list.
"""
stat = get_statistic(widget_url_or_stat, label, statistic_url)
if stat.tile.widget.parametisation:
if not pval:
raise LoaderException(
"Must provide parameter values to clear statistic data in a parametised widget"
)
else:
if pval:
raise LoaderException(
"Parameter values provided to clear_statistic_data for a non-parametised widget"
)
if stat.is_data_list():
raise LoaderException("Statistic %s is a list statistic" %
statistic_url)
data = stat.get_data(pval=pval)
if data:
data.delete()
def add_graph_dyncluster(graph,
label,
sort_order,
name,
hyperlink=None,
pval=None):
if not graph.dynamic_clusters:
raise LoaderException("Graph %s does not support dynamic clusters" %
graph.tile.url)
if graph.widget().parametisation and not pval:
raise LoaderException(
"Graph %s is parametised, but no parametisation value supplied when creating dynamic cluster"
% graph.tile.url)
if not graph.widget().parametisation and pval:
raise LoaderException(
"Graph %s is not parametised, but a parametisation value was supplied when creating dynamic cluster"
% graph.tile.url)
cluster = DynamicGraphCluster(graph=graph,
param_value=pval,
url=label,
sort_order=sort_order,
label=name,
hyperlink=hyperlink)
try:
cluster.save()
except Exception, e:
raise LoaderException("Add Cluster %s to graph %s failed: %s" %
(label, graph.tile.url, unicode(e)))
def get_traffic_light_code(stat, value):
"""Lookup traffic light code for a statistic by value"""
if not stat.traffic_light_scale:
raise LoaderException(
"Statistic %s does not have a traffic light scale" % stat.url)
try:
return stat.traffic_light_scale.codes.get(value=value)
except TrafficLightScaleCode.DoesNotExist:
raise LoaderException(
"Traffic light code %s not found in scale %s for statistic %s" %
(value, stat.traffic_light_scale.name, stat.url))
def parse_datetime(dt):
"""Parse a string into a datetime.datetime
(Assume default timezone as per settings file)
Supported formats: YYYY-MM-DDThh:mm:ss
YYYY-MM-DD
YYYY
DD mmm YYYY hh:mm:ss
YYYYQq
"""
if dt is None:
return None
elif isinstance(dt, datetime.date):
return tz.localize(dt)
elif isinstance(dt, datetime.date):
return tz.localize(datetime.datetime.combine(dt, datetime.time()))
for fmt in (
"%Y-%m-%dT%H:%M:%S",
"%Y-%m-%d",
"%Y",
"%d %b %Y %H:%M:%S",
):
try:
dt = datetime.datetime.strptime(dt, fmt)
return tz.localize(dt)
except ValueError:
pass
try:
dt = datetime.datetime.strptime(dt, "%YQ%m")
dt = dt.replace(month=(dt.month - 1) * 3 + 1)
return tz.localize(dt)
except ValueError:
raise LoaderException("Not a valid date string: %s" % repr(dt))
def get_paramval(parametisation, **kwargs):
if isinstance(parametisation, Parametisation):
param = parametisation
else:
param = Parametisation.objects.get(url=parametisation)
try:
for pval in param.parametisationvalue_set.all():
if pval.matches_parameters(kwargs):
return pval
except ViewDoesNotHaveAllKeys:
raise LoaderException(
"Parametisation error. Insuffient parameter specification for %s: %s"
% (param.url, repr(kwargs)))
raise LoaderException(
"Parametisation %s has no value set that matches %s" %
(param.url, repr(kwargs)))
def load_rss(url, process_item, environment={}, verbosity=0):
"""Utility function for processing RSS feeds with the ElementTree XML API.
url: The url of the RSS feed.
environment: a state object to passed to each call to process_item
verbosity: The verbosity level, 0-3. Higher numbers = more verbosity.
(default=0)
process_item: A callback function for processing each rss item. Returns
a list of logging messages on success or raises a
LoaderException on error, and takes the following
arguments:
elem: An ElementTree element representing an RSS item.
environment: The state object.
verbosity: The verbosity level.
Returns a list of logging messages on success, or raises a LoaderException
on error.
"""
messages = []
try:
response = urllib.urlopen(url)
xml = ET.parse(response)
for elem in xml.getroot()[0]:
if elem.tag == 'item':
messages.extend(process_item(elem, environment, verbosity))
response.close()
except LoaderException:
raise
except Exception, e:
raise LoaderException(
"Unexpected error processing RSS feed at %s: (%s)%s",
(url, e.__class__.__name__, unicode(e)))
def get_rawdataset(widget_url, label, rds_url):
"""Get a RawDataSet object by urls."""
try:
return RawDataSet.objects.get(widget__family__url=widget_url,
widget__label=label,
url=rds_url)
except RawDataSet.DoesNotExist:
raise LoaderException(
"Raw Dataset %s of widget %s(%s) does not exist" %
(rds_url, widget_url, label))
def get_icon(library, lookup):
"""Lookup icon code by icon library name and icon name or sort_order"""
try:
if isinstance(lookup, int):
return IconCode.objects.get(scale__name=library, sort_order=lookup)
else:
return IconCode.objects.get(scale__name=library, value=lookup)
except IconCode.DoesNotExist:
raise LoaderException("Icon code %s:%s does not exist" %
(library, lookup))
def get_graph(widget_url, label, tile_url):
"""Look up a graph by urls."""
try:
return GraphDefinition.objects.get(
tile__url=tile_url,
tile__widget__family__url=widget_url,
tile__widget__label=label)
except GraphDefinition.DoesNotExist:
raise LoaderException(
"Graph for tile %s of widget %s:(%s) does not exist" %
(tile_url, widget_url, label))
def clear_statistic_list(widget_url_or_stat,
label=None,
statistic_url=None,
pval=None):
"""Clear data for the requested statistic (which must be a list)
Interprets arguments as for the get_statistic function.
Raises LoaderException if the requested Statistic does not exist or is not a list.
"""
stat = get_statistic(widget_url_or_stat, label, statistic_url)
if stat.tile.widget.parametisation:
if not pval:
raise LoaderException(
"Must provide parameter values to clear statistic list data in a parametised widget"
)
else:
if pval:
raise LoaderException(
"Parameter values provided to clear_statistic_list for a non-parametised widget"
)
stat.statisticlistitem_set.filter(param_value=pval).delete()
def set_dataset_override(graph, dataset, display_name, pval=None):
if not isinstance(dataset, GraphDataset):
try:
dataset = GraphDataset.objects.get(graph=graph, url=dataset)
except GraphDataset.DoesNotExist:
raise LoaderException("Dataset %s for graph %s does not exist" %
(str(dataset), graph.tile.url))
(dd, created) = GraphDatasetData.objects.get_or_create(
dataset=dataset,
param_value=pval,
defaults={'display_name': display_name})
if not created:
dd.display_name = display_name
dd.save()
def parse_time(t):
"""Parse a string into a datetime.time
Supported formats: hh:mm:ss
hh-mm-ss
hh:mm
hh-mm
"""
if t is None:
return None
elif isinstance(t, datetime.time):
return t
elif isinstance(t, datetime.datetime):
return t.time()
for fmt in ("%H:%M:%S", "%H-%M-%S", "%H:%M", "%H-%M"):
try:
dt = datetime.datetime.strptime(t, fmt)
return dt.time()
except ValueError:
pass
except TypeError:
raise LoaderException("Cannot parse t as a time: %s" % repr(t))
raise LoaderException("Not a valid time string: %s" % repr(t))
def set_text_block(widget_url,
label,
text,
view=None,
pval=None,
pval_id=None):
"""Set the text block for the indicated widget"""
try:
wdef = WidgetDefinition.objects.get(family__url=widget_url,
label=label)
except WidgetDefinition.DoesNotExist:
raise LoaderException("Widget %s(%s) does not exist" %
(widget_url, label))
set_widget_text_block(wdef, text, view=view, pval=pval, pval_id=pval_id)
def apply_traffic_light_automation(stat, pval=None):
if stat.tile.widget.parametisation:
if not pval:
raise LoaderException(
"Must provide parameter values to apply traffic light automation in a parametised widget"
)
else:
if pval:
raise LoaderException(
"Parameter values provided to apply_traffic_light_automation for a non-parametised widget"
)
for auto in stat.trafficlightautomation_set.all():
for autostat in auto.statistic_set.all():
apply_traffic_light_automation(autostat, pval)
auto = stat.traffic_light_automation
if not auto:
return
metric_data = stat.get_data(pval)
if not metric_data:
return
if not stat.is_data_list():
metric_data = [metric_data]
if auto.target_statistic:
target_value = decimal.Decimal(
auto.target_statistic.get_data(pval).value())
elif auto.target_value:
target_value = auto.target_value
else:
target_value = None
for datum in metric_data:
metric_value = datum.value()
if stat.is_numeric():
metric_value = decimal.Decimal(metric_value)
datum.traffic_light_code = auto.strategy.traffic_light_for(
metric_value, target_value)
datum.save()
return
def parse_date(d):
"""Parse a string into a datetime.date
Supported formats: YYYY-MM-DD
"""
if d is None:
return None
elif isinstance(d, datetime.date):
return d
elif isinstance(d, datetime.date):
return d.date()
try:
dt = datetime.datetime.strptime(d, "%Y-%m-%d")
return dt.date()
except ValueError:
raise LoaderException("Not a valid date string: %s" % repr(d))
def clear_graph_suppdata(graph, pval=None, dataset=None, clusters=False):
if dataset is None:
GraphDatasetData.objects.filter(dataset__graph=graph,
param_value=pval).delete()
else:
if not isinstance(dataset, GraphDataset):
try:
dataset = GraphDataset.objects.get(graph=graph, url=dataset)
except GraphDataset.DoesNotExist:
raise LoaderException(
"Dataset %s for graph %s does not exist" %
(str(dataset), graph.tile.url))
GraphDatasetData.objects.filter(dataset=dataset,
param_value=pval).delete()
if clusters:
DynamicGraphCluster.objects.filter(graph=graph,
param_value=pval).delete()
def set_actual_frequency_display_text(widget_url,
label,
display_text,
view=None,
pval=None,
pval_id=None):
"""Set the actual frequency display text of the indicated widget"""
try:
wdef = WidgetDefinition.objects.get(family__url=widget_url,
label=label)
except WidgetDefinition.DoesNotExist:
raise LoaderException("Widget %s(%s) does not exist" %
(widget_url, label))
set_widget_actual_frequency_display_text(wdef,
display_text,
view=view,
pval=pval,
pval_id=pval_id)
def get_statistic(widget_url_or_stat, label=None, statistic_url=None):
"""If called with a Statistic object as the first argument, simply return the first
argument. Otherwise interpret the arguments as defined below, look up the indicated
Statistic object and return it.
widget_url_or_stat: The url of the widget to which the requested Statistic belong.
label: The label for the widget definition to which the requested statistic belongs.
statistic_url: The url of the requested statistic.
Raises LoaderException if the requested Statistic does not exist.
"""
if isinstance(widget_url_or_stat, Statistic):
return widget_url_or_stat
try:
return Statistic.objects.get(
url=statistic_url,
tile__widget__family__url=widget_url_or_stat,
tile__widget__label=label)
except Statistic.DoesNotExist:
raise LoaderException("Statistic %s for Widget %s(%s) does not exist" %
(statistic_url, widget_url_or_stat, label))
def new_geofeature(ds, geom, *args, **kwargs):
"""Add a new geospatial feature to a GeoDataset
ds: The GeoDataset
geom: A GDAL geometry of the correct type for the dataset.
Properties may be set by either positional or keyword arguments,
or individually by calling ther set_geoproperty method.
Returns the newly created feature on success.
"""
if geom.__class__ not in ds.gdal_datatypes():
raise LoaderException("Expected geometry type: %s Got %s" %
(ds.datatype(), geom.__class__.__name__))
feat = GeoFeature(dataset=ds)
if geom:
feat.geometry = geom.wkt
feat.save()
(proparray, propdict) = ds.prop_array_dict()
for i in range(len(args)):
set_geoproperty(feat, proparray[i], args[i])
for k, v in kwargs.items():
set_geoproperty(feat, propdict[k], v)
return feat
def clear_graph_data(graph,
cluster=None,
dataset=None,
pval=None,
clusters=False):
"""Clear all graph data, or partially by cluster or dataset"""
data = graph.get_data(pval=pval)
if clusters and not graph.dynamic_clusters:
raise LoaderException("Graph %s does not use clusters" %
graph.tile.url)
if graph.use_clusters():
if cluster:
if clusters:
raise LoaderException(
"Cannot delete clusters for Graph %s unless deleting all data"
% graph.tile.url)
if not isinstance(cluster, GraphCluster):
try:
cluster = GraphCluster.objects.get(graph=graph,
url=cluster)
except GraphCluster.DoesNotExist:
raise LoaderException(
"Cluster %s for graph %s does not exist" %
(str(cluster), graph.tile.url))
data = data.filter(cluster=cluster)
if dataset:
raise LoaderException(
"Graph %s uses clusters - cannot delete by dataset" %
graph.tile.url)
elif dataset:
if not isinstance(dataset, GraphDataset):
try:
dataset = GraphDataset.objects.get(graph=graph, url=dataset)
except GraphDataset.DoesNotExist:
raise LoaderException(
"Dataset %s for graph %s does not exist" %
(str(cluster), graph.tile.url))
data = data.filter(dataset=dataset)
if cluster or clusters:
raise LoaderException("Graph %s does not use clusters" %
graph.tile.url)
clear_graph_suppdata(graph, pval, dataset, clusters)
data.delete()
def add_stat_list_item(stat,
value,
sort_order,
pval=None,
datetimekey=None,
datetimekey_level=None,
datekey=None,
label=None,
traffic_light_code=None,
icon_code=None,
trend=None,
url=None):
"""Add an item for a list statistic.
stat: A list Statistic object
value: The value for the new item. Must be of the appropriate type for the statistic.
datetimekey: The datetime key for the item
datetimekey_level: The datetime "level" at which the datetimekey is to be interpreted.
(Must be one of the numeric values defined in widget_data.models.stat_list.StatisticListItem)
datekey: The date key for the item
traffic_light_code: The TrafficLightScaleCode object (or value string) for the new item
icon_code: The IconCode object (or value string, or numeric id) for the new item
trend: The trend indicator value (+1: up, -1: down, 0: steady) for the new item
label: The label (string key) for the item.
url: The hyperlink (url) for the item.
If any arguments are required according to the statistic meta-data and not supplied (or vice versa), a
LoaderException is raised.
"""
if not stat.is_data_list():
raise LoaderException("Not a list statistic %s" % stat.url)
if stat.is_kvlist() and not label:
raise LoaderException(
"Must provide a label for list items for statistic %s" % stat.url)
elif not stat.is_display_list() and not stat.name_as_label and not label:
raise LoaderException(
"Must provide a label for list items for statistic %s" % stat.url)
elif not stat.is_display_list() and stat.name_as_label and label:
raise LoaderException(
"Cannot provide a label for list items for statistic %s" %
stat.url)
elif stat.is_display_list() and not stat.is_kvlist() and label:
raise LoaderException(
"Cannot provide a label for list items for statistic %s" %
stat.url)
if stat.use_datekey() and not datekey:
raise LoaderException(
"Must provide a datekey for list items for statistic %s" %
stat.url)
elif datekey and not stat.use_datekey():
raise LoaderException(
"Cannot provide a datekey for list items for statistic %s" %
stat.url)
if stat.use_datetimekey() and not datetimekey:
raise LoaderException(
"Must provide a datetimekey for list items for statistic %s" %
stat.url)
elif datetimekey and not stat.use_datetimekey():
raise LoaderException(
"Cannot provide a datetimekey for list items for statistic %s" %
stat.url)
if stat.use_datetimekey_level() and not datetimekey_level:
raise LoaderException(
"Must provide a datetimekey level for list items for statistic %s"
% stat.url)
elif datetimekey_level and not stat.use_datetimekey_level():
raise LoaderException(
"Cannot provide a datetimekey level for list items for statistic %s"
% stat.url)
if stat.trend and trend is None:
raise LoaderException("Must provide a trend for statistic %s" %
stat.url)
if not stat.hyperlinkable and url is not None:
raise LoaderException(
"Cannnot provide a url for statistic %s (not hyperlinkable)" %
stat.url)
if stat.traffic_light_scale and not traffic_light_code:
raise LoaderException(
"Must provide a traffic light code for statistic %s" % stat.url)
if stat.icon_library and not icon_code:
raise LoaderException("Must provide a icon code for statistic %s" %
stat.url)
if stat.tile.widget.parametisation:
if not pval:
raise LoaderException(
"Must provide parameter values to add a statistic list item in a parametised widget"
)
else:
if pval:
raise LoaderException(
"Parameter values provided to add_stat_list_item for a non-parametised widget"
)
if stat.traffic_light_scale and isinstance(traffic_light_code,
TrafficLightScaleCode):
tlc = traffic_light_code
elif stat.traffic_light_scale:
tlc = get_traffic_light_code(stat, traffic_light_code)
else:
tlc = None
if stat.icon_library and isinstance(icon_code, IconCode):
ic = icon_code
elif stat.icon_library:
ic = get_icon(stat.icon_library.name, icon_code)
else:
ic = None
if datekey:
datetimekey = datekey
if datetimekey_level:
if not isinstance(datetimekey_level, int):
for lc in StatisticListItem.level_choices:
if lc[1] == datetimekey_level:
datetimekey_level = lc[0]
break
item = StatisticListItem(statistic=stat,
param_value=pval,
keyval=label,
trend=trend,
sort_order=sort_order,
datetime_key=datetimekey,
datetime_keylevel=datetimekey_level,
traffic_light_code=tlc,
icon_code=ic,
url=url)
if stat.is_numeric():
if stat.num_precision == 0:
item.intval = value
else:
item.decval = value
else:
item.strval = value
item.save()
apply_traffic_light_automation(stat, pval)
def set_stat_data(stat,
value,
pval=None,
traffic_light_code=None,
icon_code=None,
trend=None,
label=None):
"""Set the data for a scalar statistic.
stat: A non-list Statistic object
value: The value to set the data to. Must be of the appropriate type for the statistic.
traffic_light_code: The TrafficLightScaleCode object (or value string) to set the traffic light value to
icon_code: The IconCode object (or value string, or numeric id) to set the icon value to
trend: The trend indicator value (+1: up, -1: down, 0: steady)
label: The dynamic label for the statistic.
If any arguments are required according to the statistic meta-data and not supplied (or vice versa), a
LoaderException is raised.
"""
if stat.is_data_list():
raise LoaderException("Statistic %s is a list statistic" % stat.url)
if stat.tile.widget.parametisation:
if not pval:
raise LoaderException(
"Must provide parameter values to set statistic data in a parametised widget"
)
else:
if pval:
raise LoaderException(
"Parameter values provided to set_statistic_data for a non-parametised widget"
)
data = stat.get_data(pval=pval)
if not data:
data = StatisticData(statistic=stat, param_value=pval)
if not stat.name_as_label and not label:
raise LoaderException("Must provide a label for statistic %s" %
stat.url)
else:
data.label = label
if stat.trend and trend is None:
raise LoaderException("Must provide a trend for statistic %s" %
stat.url)
else:
data.trend = trend
if stat.traffic_light_scale and not traffic_light_code:
raise LoaderException(
"Must provide a traffic light code for statistic %s" % stat.url)
if stat.traffic_light_scale and isinstance(traffic_light_code,
TrafficLightScaleCode):
tlc = traffic_light_code
elif stat.traffic_light_scale:
tlc = get_traffic_light_code(stat, traffic_light_code)
else:
tlc = None
data.traffic_light_code = tlc
if stat.icon_library and not icon_code:
raise LoaderException("Must provide a icon code for statistic %s" %
stat.url)
if stat.icon_library and isinstance(icon_code, IconCode):
ic = icon_code
elif stat.icon_library:
ic = get_icon(stat.icon_library.name, icon_code)
else:
ic = None
data.icon_code = ic
if stat.is_numeric():
if stat.num_precision == 0:
data.intval = value + 0.5
else:
data.decval = value
elif stat.stat_type != stat.NULL_STAT:
data.strval = value
try:
data.save()
except Exception, e:
raise LoaderException(str(e))
def get_geodataset(url):
"""Lookup a GeoDataset by url"""
try:
return GeoDataset.objects.get(url=url)
except GeoDataset.DoesNotExist:
raise LoaderException("GeoDataset %s does not exist" % url)
def add_graph_data(graph,
dataset,
value,
cluster=None,
horiz_value=None,
pval=None,
val_min=None,
val_max=None):
"""Add a graph datapoint.
Return the newly created GraphData object on success.
Raise a LoaderException on error, or if the provided arguments are not valid for the graph.
"""
if not isinstance(dataset, GraphDataset):
try:
dataset = GraphDataset.objects.get(graph=graph, url=dataset)
except GraphDataset.DoesNotExist:
raise LoaderException("Dataset %s for graph %s does not exist" %
(str(dataset), graph.tile.url))
if dataset.use_error_bars:
if val_min is None or val_max is None:
raise LoaderException(
"Dataset %s for graph %s requires error bar limits" %
(dataset.url, graph.tile.url))
else:
if val_min is not None or val_max is not None:
raise LoaderException(
"Dataset %s for graph %s does not use error bars" %
(dataset.url, graph.tile.url))
value = decimal.Decimal(value).quantize(decimal.Decimal("0.0001"),
rounding=decimal.ROUND_HALF_UP)
gd = GraphData(graph=graph,
param_value=pval,
dataset=dataset,
value=value,
err_valmin=val_min,
err_valmax=val_max)
if graph.use_clusters():
if not cluster:
raise LoaderException("Must supply cluster for data for graph %s" %
graph.tile.url)
elif not isinstance(cluster, GraphClusterBase):
try:
if graph.dynamic_clusters:
cluster = DynamicGraphCluster.objects.get(graph=graph,
url=cluster,
param_value=pval)
else:
cluster = GraphCluster.objects.get(graph=graph,
url=cluster)
except ObjectDoesNotExist:
raise LoaderException(
"Cluster %s for graph %s does not exist" %
(str(cluster), graph.tile.url))
if graph.dynamic_clusters:
gd.dynamic_cluster = cluster
else:
gd.cluster = cluster
else:
if graph.horiz_axis_type == graph.NUMERIC:
gd.horiz_numericval = decimal.Decimal(horiz_value).quantize(
decimal.Decimal("0.0001"), rounding=decimal.ROUND_HALF_UP)
elif graph.horiz_axis_type == graph.DATE:
gd.horiz_dateval = horiz_value
elif graph.horiz_axis_type == graph.TIME:
gd.horiz_timeval = horiz_value
elif graph.horiz_axis_type == graph.DATETIME:
gd.horiz_dateval = horiz_value.date()
gd.horiz_timeval = horiz_value.time()
gd.save()
return gd