def d0_basic(self, weekly=False):
cherrypy.response.headers['Content-Type'] = 'image/png'
url = self.metadata.get('d0_csv', 'http://physics.lunet.edu/~snow/' \
'd0osgprod.csv')
url_fp = urllib2.urlopen(url)
metadata = {'croptime':False, 'span':86400, 'pivot_name': 'Date', \
'column_names': 'Merged Events', \
'title': 'D0 OSG Production'}
if weekly:
metadata['span'] *= 7
fp = cStringIO.StringIO()
data = {}
for line in url_fp.readlines():
info = line.strip().split(',')
grouping_name = time.strptime(info[0], '%y%m%d')
group = datetime.datetime(*grouping_name[:6])
# Change the date to the first day of that week; groups things
# by week instead of the default by day.
if weekly:
weekday = group.isoweekday()
group -= datetime.timedelta(weekday, 0)
group = to_timestamp(group)
results = int(info[3])
value = data.setdefault(group, 0)
data[group] = value + results
metadata['starttime'] = min(data.values())
time_max = max(data.values())
if weekly:
metadata['endtime'] = time_max + 7*86400
else:
metadata['endtime'] = time_max + 1*86400
GB = GratiaBar()
GB(data, fp, metadata)
return fp.getvalue()
def parse_type(self, string, my_type):
if my_type == 'int':
return int(string)
elif my_type == 'offset_now':
value = int(string)
if value > 0:
return value
else:
return time.time() + value
elif my_type == 'float':
return float(string)
elif my_type == 'eval':
raise ValueError("sorry eval type not supported")
elif my_type == 'datetime':
return convert_to_datetime(string)
elif my_type == 'timestamp':
return to_timestamp(string)
elif my_type == 'bool' or my_type == 'boolean':
if not isinstance(string, str):
return bool(string)
if string.lower().strip() == 'false':
return False
elif string.lower().strip() == 'true':
return True
else:
raise TypeError("Cannot convert string %s to boolean; valid "
"inputs are 'true' or 'false'." % string)
else:
return str(string)
def groupingAttrs( self, grouping_name, grouping ):
grouping_attrs = {}
if grouping_name and str(grouping_name).lower()=='time':
grouping_attrs['value'] = str(datetime.datetime.utcfromtimestamp(to_timestamp(grouping)))
else:
grouping_attrs['value'] = str(grouping)
return grouping_attrs
def parse_type( self, string, my_type ):
if my_type == 'int':
return int( string )
elif my_type == 'offset_now':
value = int( string )
if value > 0:
return value
else:
return time.time() + value
elif my_type == 'float':
return float( string )
elif my_type == 'eval':
raise ValueError("sorry eval type not supported")
elif my_type == 'datetime':
return convert_to_datetime( string )
elif my_type == 'timestamp':
return to_timestamp( string )
elif my_type == 'bool' or my_type == 'boolean':
if not isinstance(string, str):
return bool(string)
if string.lower().strip() == 'false':
return False
elif string.lower().strip() == 'true':
return True
else:
raise TypeError("Cannot convert string %s to boolean; valid "
"inputs are 'true' or 'false'." % string )
else:
return str( string )
def d0_basic(self, weekly=False):
cherrypy.response.headers['Content-Type'] = 'image/png'
url = self.metadata.get('d0_csv', 'http://physics.lunet.edu/~snow/' \
'd0osgprod.csv')
url_fp = urllib2.urlopen(url)
metadata = {'croptime':False, 'span':86400, 'pivot_name': 'Date', \
'column_names': 'Merged Events', \
'title': 'D0 OSG Production'}
if weekly:
metadata['span'] *= 7
fp = cStringIO.StringIO()
data = {}
for line in url_fp.readlines():
info = line.strip().split(',')
grouping_name = time.strptime(info[0], '%y%m%d')
group = datetime.datetime(*grouping_name[:6])
# Change the date to the first day of that week; groups things
# by week instead of the default by day.
if weekly:
weekday = group.isoweekday()
group -= datetime.timedelta(weekday, 0)
group = to_timestamp(group)
results = int(info[3])
value = data.setdefault(group, 0)
data[group] = value + results
metadata['starttime'] = min(data.values())
time_max = max(data.values())
if weekly:
metadata['endtime'] = time_max + 7 * 86400
else:
metadata['endtime'] = time_max + 1 * 86400
GB = GratiaBar()
GB(data, fp, metadata)
return fp.getvalue()
def groupingAttrs(self, grouping_name, grouping):
grouping_attrs = {}
if grouping_name and str(grouping_name).lower() == 'time':
grouping_attrs['value'] = str(
datetime.datetime.utcfromtimestamp(to_timestamp(grouping)))
else:
grouping_attrs['value'] = str(grouping)
return grouping_attrs
def make_hash_str( self, query, **kw ):
if 'starttime' in kw.keys():
kw['starttime'] = int(to_timestamp(kw['starttime']))
if 'endtime' in kw.keys():
kw['endtime'] = int(to_timestamp(kw['endtime']))
if 'starttime' in kw and 'endtime' in kw:
if kw['endtime'] - kw['starttime'] > 300*5:
kw['endtime'] -= kw['endtime'] % 300
kw['starttime'] -= kw['starttime'] % 300
elif kw['endtime'] - kw['starttime'] > 300:
kw['endtime'] -= kw['endtime'] % 10
kw['starttime'] -= kw['starttime'] % 10
hash_str = str(query)
keys = kw.keys(); keys.sort()
for key in keys:
hash_str += ',' + str(key) + ',' + str(kw[key])
return hash_str
def adjust_time( mytime, **kw ):
if 'adjust_hours' in kw.keys():
timechange = float(kw['adjust_hours'])*3600
else:
timechange = 0
timestamp = to_timestamp(mytime)
timestamp += timechange
#print mytime, datetime.datetime.utcfromtimestamp( timestamp )
return datetime.datetime.utcfromtimestamp( timestamp )
def adjust_time(mytime, **kw):
if 'adjust_hours' in kw.keys():
timechange = float(kw['adjust_hours']) * 3600
else:
timechange = 0
timestamp = to_timestamp(mytime)
timestamp += timechange
#print mytime, datetime.datetime.utcfromtimestamp( timestamp )
return datetime.datetime.utcfromtimestamp(timestamp)
def groupingAttrs( self, grouping_name, grouping, metadata):
grouping_attrs = {}
if metadata.get('translate_mp_2_gc',False):
grouping_attrs['value'] = json.dumps(grouping,separators=(',',':'), cls=CustomDecimalDateObjectJSONEncoder)
elif grouping_name and str(grouping_name).lower()=='time':
grouping_attrs['value'] = str(datetime.datetime.utcfromtimestamp(to_timestamp(grouping)))
else:
grouping_attrs['value'] = str(grouping)
return grouping_attrs
def make_hash_str(self, query, **kw):
if 'starttime' in kw.keys():
kw['starttime'] = int(to_timestamp(kw['starttime']))
if 'endtime' in kw.keys():
kw['endtime'] = int(to_timestamp(kw['endtime']))
if 'starttime' in kw and 'endtime' in kw:
if kw['endtime'] - kw['starttime'] > 300 * 5:
kw['endtime'] -= kw['endtime'] % 300
kw['starttime'] -= kw['starttime'] % 300
elif kw['endtime'] - kw['starttime'] > 300:
kw['endtime'] -= kw['endtime'] % 10
kw['starttime'] -= kw['starttime'] % 10
hash_str = str(query)
keys = kw.keys()
keys.sort()
for key in keys:
hash_str += ',' + str(key) + ',' + str(kw[key])
return hash_str
def addResults_pg(self, data, metadata, gen, **kw):
try:
if 'grapher' in metadata:
coords = metadata['grapher'].get_coords(
metadata['query'], metadata, **metadata['given_kw'])
else:
coords = None
except Exception as e:
print(e)
traceback.print_exc(sys.stdout)
coords = None
attrs = {'kind': 'pivot-group'}
pivot_name = str(metadata['pivot_name'])
if pivot_name and len(pivot_name) > 0:
attrs['pivot'] = pivot_name
grouping_name = str(metadata.get('grouping_name', ''))
if grouping_name and len(grouping_name) > 0:
attrs['group'] = grouping_name
if coords:
attrs['coords'] = 'True'
else:
attrs['coords'] = 'False'
self.write_columns(metadata, gen)
gen.startElement('data', attrs)
for pivot in data.keys():
gen.characters("\n\t\t\t")
gen.startElement(*self.pivotName(pivot, attrs))
my_groups = data[pivot].keys()
my_groups.sort()
my_groups.reverse()
for grouping in my_groups:
gen.characters("\n\t\t\t\t")
grouping_attrs = {}
gen.startElement('group',
self.groupingAttrs(grouping_name, grouping))
if coords:
try:
groups = coords[pivot]
if isinstance(grouping, datetime.datetime) and (
not (grouping in groups.keys())):
kw['coords'] = groups[to_timestamp(grouping)]
else:
kw['coords'] = groups[grouping]
except Exception as e:
#print "Missing coords", pivot, grouping
#print e
pass
self.addData(data[pivot][grouping], gen, **kw)
gen.endElement('group')
gen.endElement(self.pivotName(pivot, attrs)[0])
gen.characters("\n\t\t")
gen.endElement('data')
gen.characters("\n\t")
def parse_type( self, string, my_type ):
if my_type == 'int':
return int( string )
elif my_type == 'float':
return float( string )
elif my_type == 'eval':
raise ValueError("sorry eval type not supported")
elif my_type == 'datetime':
return convert_to_datetime( string )
elif my_type == 'timestamp':
return to_timestamp( string )
else:
return str( string )
def parse_type(self, string, my_type):
if my_type == 'int':
return int(string)
elif my_type == 'float':
return float(string)
elif my_type == 'eval':
raise ValueError("sorry eval type not supported")
elif my_type == 'datetime':
return convert_to_datetime(string)
elif my_type == 'timestamp':
return to_timestamp(string)
else:
return str(string)
def parse_type( self, string, my_type ):
if my_type == 'int':
return int( string )
elif my_type == 'float':
return float( string )
elif my_type == 'eval':
return eval(str(string),{'__builtins__':None,'time':time},{})
elif my_type == 'datetime':
return convert_to_datetime( string )
elif my_type == 'timestamp':
return to_timestamp( string )
else:
return str( string )
def groupingAttrs(self, grouping_name, grouping, metadata):
grouping_attrs = {}
if metadata.get('translate_mp_2_gc', False):
grouping_attrs['value'] = json.dumps(
grouping,
separators=(',', ':'),
cls=CustomDecimalDateObjectJSONEncoder)
elif grouping_name and str(grouping_name).lower() == 'time':
grouping_attrs['value'] = str(
datetime.datetime.utcfromtimestamp(to_timestamp(grouping)))
else:
grouping_attrs['value'] = str(grouping)
return grouping_attrs
def parse_type( self, string, my_type ):
if my_type == 'int':
return int( string )
elif my_type == 'float':
return float( string )
elif my_type == 'eval':
return eval(str(string),{'__builtins__':None,'time':time},{})
elif my_type == 'datetime':
return convert_to_datetime( string )
elif my_type == 'timestamp':
return to_timestamp( string )
else:
return str( string )
def addResults_pg( self, data, metadata, gen, **kw ):
try:
if 'grapher' in metadata:
coords = metadata['grapher'].get_coords( metadata['query'], metadata, **metadata['given_kw'] )
else: coords = None
except Exception as e:
print(e)
traceback.print_exc( sys.stdout )
coords = None
attrs = {'kind':'pivot-group'}
pivot_name = str(metadata['pivot_name'])
if pivot_name and len(pivot_name) > 0:
attrs['pivot'] = pivot_name
grouping_name = str(metadata.get('grouping_name',''))
if grouping_name and len(grouping_name) > 0:
attrs['group'] = grouping_name
if coords:
attrs['coords'] = 'True'
else:
attrs['coords'] = 'False'
self.write_columns( metadata, gen )
gen.startElement('data',attrs)
for pivot in data.keys():
gen.characters("\n\t\t\t")
gen.startElement( *self.pivotName( pivot, attrs ) )
my_groups = data[pivot].keys(); my_groups.sort(); my_groups.reverse()
for grouping in my_groups:
gen.characters("\n\t\t\t\t")
grouping_attrs = {}
gen.startElement('group', self.groupingAttrs( grouping_name, grouping ) )
if coords:
try:
groups = coords[pivot]
if isinstance(grouping, datetime.datetime) and (not (grouping in groups.keys()) ):
kw['coords'] = groups[to_timestamp(grouping)]
else: kw['coords'] = groups[grouping]
except Exception as e:
#print "Missing coords", pivot, grouping
#print e
pass
self.addData( data[pivot][grouping], gen, **kw )
gen.endElement('group')
gen.endElement( self.pivotName( pivot, attrs )[0] )
gen.characters("\n\t\t")
gen.endElement('data')
gen.characters("\n\t")
def d0_basic(self, weekly=False):
cherrypy.response.headers['Content-Type'] = 'image/png'
srchUrl = 'InitD0Url'
modName = 'd0'
print "%s: srchUrl: %s" % (modName, srchUrl)
try:
D0Url = GratiaURLS().GetUrl(srchUrl)
print "%s: SUCCESS: GratiaURLS().GetUrl(url = %s)" % (modName,
srchUrl)
print "%s: retUrl: %s" % (modName, D0Url)
except:
print "%s: FAILED: GratiaURLS().GetUrl(url = %s)" % (modName,
srchUrl)
pass
url_fp = urllib2.urlopen(D0Url)
metadata = {'croptime':False, 'span':86400, 'pivot_name': 'Date', \
'column_names': 'Merged Events', \
'title': 'D0 OSG Production'}
if weekly:
metadata['span'] *= 7
fp = cStringIO.StringIO()
data = {}
for line in url_fp.readlines():
info = line.strip().split(',')
grouping_name = time.strptime(info[0], '%y%m%d')
group = datetime.datetime(*grouping_name[:6])
# Change the date to the first day of that week; groups things
# by week instead of the default by day.
if weekly:
weekday = group.isoweekday()
group -= datetime.timedelta(weekday, 0)
group = to_timestamp(group)
results = int(info[3])
value = data.setdefault(group, 0)
data[group] = value + results
metadata['starttime'] = min(data.values())
time_max = max(data.values())
if weekly:
metadata['endtime'] = time_max + 7 * 86400
else:
metadata['endtime'] = time_max + 1 * 86400
GB = GratiaBar()
GB(data, fp, metadata)
return fp.getvalue()
def d0_basic(self, weekly=False):
cherrypy.response.headers['Content-Type'] = 'image/png'
srchUrl = 'InitD0Url'
modName = 'd0'
print "%s: srchUrl: %s" % (modName, srchUrl)
try:
D0Url = GratiaURLS().GetUrl(srchUrl)
print "%s: SUCCESS: GratiaURLS().GetUrl(url = %s)" % (modName,srchUrl)
print "%s: retUrl: %s" % (modName, D0Url)
except:
print "%s: FAILED: GratiaURLS().GetUrl(url = %s)" % (modName,srchUrl)
pass
url_fp = urllib2.urlopen(D0Url)
metadata = {'croptime':False, 'span':86400, 'pivot_name': 'Date', \
'column_names': 'Merged Events', \
'title': 'D0 OSG Production'}
if weekly:
metadata['span'] *= 7
fp = cStringIO.StringIO()
data = {}
for line in url_fp.readlines():
info = line.strip().split(',')
grouping_name = time.strptime(info[0], '%y%m%d')
group = datetime.datetime(*grouping_name[:6])
# Change the date to the first day of that week; groups things
# by week instead of the default by day.
if weekly:
weekday = group.isoweekday()
group -= datetime.timedelta(weekday, 0)
group = to_timestamp(group)
results = int(info[3])
value = data.setdefault(group, 0)
data[group] = value + results
metadata['starttime'] = min(data.values())
time_max = max(data.values())
if weekly:
metadata['endtime'] = time_max + 7*86400
else:
metadata['endtime'] = time_max + 1*86400
GB = GratiaBar()
GB(data, fp, metadata)
return fp.getvalue()
def parse_type( self, string, my_type ):
if my_type == 'int':
return int( string )
elif my_type == 'float':
return float( string )
elif my_type == 'eval':
return eval(str(string),{'__builtins__':None,'time':time},{})
elif my_type == 'datetime':
return convert_to_datetime( string )
elif my_type == 'timestamp':
return to_timestamp( string )
elif my_type == 'bool' or my_type == 'boolean':
if type(string) != types.StringType:
return bool(string)
if string.lower().strip() == 'false':
return False
elif string.lower().strip() == 'true':
return True
else:
raise TypeError("Cannot convert string %s to boolean; valid "
"inputs are 'true' or 'false'." % string )
else:
return str( string )
def parse_type( self, string, my_type ):
if my_type == 'int':
return int( string )
elif my_type == 'float':
return float( string )
elif my_type == 'eval':
return eval(str(string),{'__builtins__':None,'time':time},{})
elif my_type == 'datetime':
return convert_to_datetime( string )
elif my_type == 'timestamp':
return to_timestamp( string )
elif my_type == 'bool' or my_type == 'boolean':
if type(string) != types.StringType:
return bool(string)
if string.lower().strip() == 'false':
return False
elif string.lower().strip() == 'true':
return True
else:
raise TypeError("Cannot convert string %s to boolean; valid "
"inputs are 'true' or 'false'." % string )
else:
return str( string )
class XmlGenerator(QueryHandler):
XSLT_NAME = 'xml_results.xsl'
def _do_multiprocess_child(self, q, results, metadata, kw):
try:
xml = self.handle_results_internal(results, metadata, **kw)
q.put(xml)
except:
q.put(None)
raise
def do_multiprocess(self, results, metadata, **kw):
q = multiprocessing.Queue()
p = multiprocessing.Process(target=self._do_multiprocess_child,
args=(q, results, metadata, kw),
name="GraphTool XmlGenerator")
p.daemon = True
p.start()
xml = q.get()
p.join()
return xml
def handle_results_internal(self, results, metadata, **kw):
output = cStringIO.StringIO()
gen = self.startPlot(output, results, metadata)
kind = metadata.get('kind', 'Type not specified!')
if kind == 'pivot-group':
self.addResults_pg(results, metadata, gen)
elif kind == 'pivot':
self.addResults_p(results, metadata, gen)
elif kind == 'complex-pivot':
self.addResults_c_p(results, metadata, gen)
else:
raise Exception("Unknown data type! (%s)" % kind)
self.endPlot(gen)
return output.getvalue()
def handle_results(self, results, metadata, **kw):
if has_multiprocessing:
return self.do_multiprocess(results, metadata, **kw)
else:
return self.handle_results_internal(results, metadata, **kw)
def handle_list(self, *args, **kw):
output = cStringIO.StringIO()
gen = self.startDocument(output)
base_url = ''
if 'base_url' in self.metadata:
base_url = self.metadata['base_url']
if base_url[-1] != '/':
base_url += '/'
i = 0
for query_obj in self.objs:
i += 1
if 'display_name' in query_obj.__dict__:
name = query_obj.display_name
else:
name = "Query"
gen.startElement("pagelist", {'name': name, 'id': str(i)})
for page in query_obj.commands.keys():
gen.characters("\t\t\n")
attrs = {}
my_page = self.known_commands[page]
if 'title' in my_page.__dict__.keys():
attrs['title'] = my_page.title
else:
attrs['title'] = page
gen.startElement('page', attrs)
gen.characters(base_url + page)
gen.endElement('page')
gen.characters("\t\n")
gen.endElement("pagelist")
gen.characters("\t\n")
self.endDocument(gen)
return output.getvalue()
def startDocument(self, output, encoding='UTF-8'):
gen = XMLGenerator(output, encoding)
gen.startDocument()
try:
static_location = '/static/content'
static_object = self.globals['static']
static_location = static_object.metadata.get('base_url', '/static')
static_location += '/content'
except:
pass
output.write('<?xml-stylesheet type="text/xsl" href="%s/%s"?>\n' % \
(static_location, self.XSLT_NAME) )
output.write('<!DOCTYPE graphtool-data>\n')
gen.startElement('graphtool', {})
gen.characters("\n\t")
return gen
def startPlot(self, output, results, metadata, encoding='UTF-8'):
gen = self.startDocument(output, encoding)
query_attrs = {}
name = metadata.get('name', '')
if name and len(name) > 0:
query_attrs['name'] = name
gen.startElement('query', query_attrs)
gen.characters("\n\t\t")
title = expand_string(metadata.get('title', ''),
metadata.get('sql_vars', ''))
if title and len(title) > 0:
gen.startElement('title', {})
gen.characters(title)
gen.endElement('title')
gen.characters("\n\t\t")
graph_type = metadata.get('graph_type', False)
if graph_type and len(graph_type) > 0:
gen.startElement('graph', {})
gen.characters(graph_type)
gen.endElement('graph')
gen.characters("\n\t\t")
sql_string = str(metadata.get('sql', ''))
gen.startElement('sql', {})
gen.characters(sql_string)
gen.characters("\n\t\t")
gen.endElement('sql')
gen.characters("\n\t\t")
self.write_sql_vars(results, metadata, gen)
gen.characters("\n\t\t")
base_url = None
graphs = metadata.get('grapher', None)
if graphs and 'base_url' in graphs.metadata:
base_url = graphs.metadata['base_url']
else:
print "Base URL not specified!"
print metadata
if graphs:
print graphs.metadata
else:
print "Graphs not specified"
pass
my_base_url = self.metadata.get('base_url', '')
gen.startElement('attr', {'name': 'base_url'})
gen.characters(my_base_url)
gen.endElement('attr')
gen.characters('\n\t\t')
try:
static_location = '/static/content'
static_object = self.globals['static']
static_location = static_object.metadata.get('base_url', '/static')
static_location += '/content'
except:
pass
gen.startElement('attr', {'name': 'static_base_url'})
gen.characters(static_location)
gen.endElement('attr')
gen.characters('\n\t\t')
self.write_graph_url(results, metadata, gen, base_url=base_url)
gen.characters('\n\t\t')
return gen
def write_graph_url(self, results, metadata, gen, base_url=None):
if base_url != None:
base = base_url + '/' + metadata.get('name', '') + '?'
kw = metadata.get('given_kw', {})
for key, item in kw.items():
base += str(key) + '=' + str(item) + '&'
gen.startElement("url", {})
gen.characters(base)
gen.endElement("url")
def write_sql_vars(self, data, metadata, gen):
sql_vars = metadata['sql_vars']
for key, item in metadata['given_kw'].items():
sql_vars[key] = item
gen.startElement('sqlvars', {})
for var in sql_vars:
gen.characters("\n\t\t\t")
gen.startElement('var', {'name': var})
gen.characters(str(sql_vars[var]))
gen.endElement('var')
gen.characters("\n\t\t")
gen.endElement('sqlvars')
gen.characters("\n\t\t")
def endPlot(self, gen):
gen.endElement('query')
gen.characters("\n")
self.endDocument(gen)
def endDocument(self, gen):
gen.endElement('graphtool')
gen.characters("\n")
gen.endDocument()
def write_columns(self, metadata, gen):
column_names = str(metadata.get('column_names', ''))
column_units = str(metadata.get('column_units', ''))
names = [i.strip() for i in column_names.split(',')]
units = [i.strip() for i in column_units.split(',')]
columns = {}
num_columns = min(len(names), len(units))
for idx in range(num_columns):
columns[names[idx]] = units[idx]
if len(columns.items()) > 0:
gen.startElement('columns', {})
i = 1
for header in names:
gen.characters("\n\t\t\t")
gen.startElement('column', {
'unit': columns[header],
'index': str(i)
})
i += 1
gen.characters(header)
gen.endElement('column')
gen.characters("\n\t\t")
gen.endElement('columns')
gen.characters("\n\t\t")
def addResults_pg(self, data, metadata, gen, **kw):
try:
if 'grapher' in metadata:
coords = metadata['grapher'].get_coords(
metadata['query'], metadata, **metadata['given_kw'])
else:
coords = None
except Exception, e:
print e
traceback.print_exc(sys.stdout)
coords = None
attrs = {'kind': 'pivot-group'}
pivot_name = str(metadata['pivot_name'])
if pivot_name and len(pivot_name) > 0:
attrs['pivot'] = pivot_name
grouping_name = str(metadata.get('grouping_name', ''))
if grouping_name and len(grouping_name) > 0:
attrs['group'] = grouping_name
if coords:
attrs['coords'] = 'True'
else:
attrs['coords'] = 'False'
self.write_columns(metadata, gen)
gen.startElement('data', attrs)
for pivot in data.keys():
gen.characters("\n\t\t\t")
gen.startElement(*self.pivotName(pivot, attrs))
my_groups = data[pivot].keys()
my_groups.sort()
my_groups.reverse()
for grouping in my_groups:
gen.characters("\n\t\t\t\t")
grouping_attrs = {}
gen.startElement('group',
self.groupingAttrs(grouping_name, grouping))
if coords:
try:
groups = coords[pivot]
if type(grouping) == datetime.datetime and (
not (grouping in groups.keys())):
kw['coords'] = groups[to_timestamp(grouping)]
else:
kw['coords'] = groups[grouping]
except Exception, e:
#print "Missing coords", pivot, grouping
#print e
pass
self.addData(data[pivot][grouping], gen, **kw)
gen.endElement('group')
gen.endElement(self.pivotName(pivot, attrs)[0])
def pivot_group_parser_plus(sql_results,
pivots="0,1",
grouping="2",
results="3",
pivot_transform="echo",
grouping_transform="echo",
data_transform="echo",
globals=globals(),
suppress_zeros=True,
**kw):
metadata = {}
pivot_cols = [int(i.strip()) for i in pivots.split(',')]
grouping_cols = [int(i.strip()) for i in grouping.split(',')]
results_cols = [int(i.strip()) for i in results.split(',')]
len_results_cols = len(results_cols)
if len(sql_results) > 0:
row_size = len(sql_results[0])
if callable(pivot_transform):
pivot_transform_func = pivot_transform
elif pivot_transform == 'echo':
pivot_transform_func = echo
else:
pivot_transform_func = globals[pivot_transform.strip()]
if grouping_transform == 'echo':
grouping_transform_func = echo
else:
grouping_transform_func = globals[grouping_transform.strip()]
if data_transform == 'echo':
data_transform_func = echo
else:
data_transform_func = globals[data_transform.strip()]
parsed_results = {}
groups = set()
pivots = set()
for row in sql_results:
my_pivot = make_entry(row, pivot_cols, pivot_transform_func, row_size,
**kw)
if my_pivot == None: continue
my_group = make_entry(row, grouping_cols, grouping_transform_func,
row_size, **kw)
my_group = to_timestamp(my_group)
groups.add(my_group)
pivots.add(my_pivot)
groups = list(groups)
groups.sort()
if len(groups) > 0:
min_span = groups[-1]
for i in range(len(groups) - 1):
min_span = min(groups[i + 1] - groups[i], min_span)
for row in sql_results:
my_pivot = make_entry(row, pivot_cols, pivot_transform_func, row_size,
**kw)
if my_pivot == None: continue
my_group = make_entry(row, grouping_cols, grouping_transform_func,
row_size, **kw)
my_group = to_timestamp(my_group)
if not (my_pivot in parsed_results.keys()):
parsed_results[my_pivot] = {}
if my_group in parsed_results[my_pivot].keys():
parsed_results[my_pivot][my_group] = add_data(
parsed_results[my_pivot][my_group], row, results_cols)
else:
parsed_results[my_pivot][my_group] = new_data(
row, results_cols, len_results_cols)
filtered_results = {}
metadata['kind'] = 'pivot-group'
#metadata['is_cumulative'] = True
for pivot in parsed_results.keys():
data = parsed_results[pivot]
tmp_group = {}
#pivot_has_nonzero = True
for grouping, info in data.items():
info = check_tuple(info, len_results_cols)
#if has_nonzero( info, len_results_cols ):
tmp_group[grouping] = info
#pivot_has_nonzero = True
#if pivot_has_nonzero:
filtered_results[pivot] = tmp_group
if len(groups) == 0:
return filtered_results, metadata
results = filtered_results
filtered_results = {}
current_group = groups.pop(0)
csum = {}
for pivot in results.keys():
csum[pivot] = 0
filtered_results[pivot] = {}
def add_cumulative_data(current_group):
for pivot in results.keys():
if current_group in results[pivot].keys():
csum[pivot] = float(results[pivot][current_group])
filtered_results[pivot][current_group] = csum[pivot]
while len(groups) > 0:
next_group = groups[0]
add_cumulative_data(current_group)
while current_group + min_span < next_group:
current_group += min_span
add_cumulative_data(current_group)
current_group = groups.pop(0)
add_cumulative_data(current_group)
for pivot, groups in filtered_results.items():
for group, data in groups.items():
groups[group] = data_transform_func(data, **kw)
return filtered_results, metadata
def pivot_group_parser_plus( sql_results, pivots="0,1", grouping="2", results="3", pivot_transform="echo", grouping_transform="echo", data_transform="echo", globals=globals(), suppress_zeros=True, **kw ):
metadata = {}
pivot_cols = [int(i.strip()) for i in pivots.split(',')]
grouping_cols = [int(i.strip()) for i in grouping.split(',')]
results_cols = [int(i.strip()) for i in results.split(',')]
len_results_cols = len(results_cols)
if len(sql_results) > 0:
row_size = len(sql_results[0])
if callable(pivot_transform):
pivot_transform_func = pivot_transform
elif pivot_transform == 'echo':
pivot_transform_func = echo
else:
pivot_transform_func = globals[pivot_transform.strip()]
if grouping_transform == 'echo':
grouping_transform_func = echo
else:
grouping_transform_func = globals[grouping_transform.strip()]
if data_transform == 'echo':
data_transform_func = echo
else:
data_transform_func = globals[data_transform.strip()]
parsed_results = {}
groups = set()
pivots = set()
for row in sql_results:
my_pivot = make_entry( row, pivot_cols, pivot_transform_func, row_size, **kw )
if my_pivot == None: continue
my_group = make_entry( row, grouping_cols, grouping_transform_func, row_size, **kw )
my_group = to_timestamp( my_group )
groups.add( my_group )
pivots.add( my_pivot )
groups = list(groups)
groups.sort()
if len(groups) > 0:
min_span = groups[-1]
for i in range( len(groups)-1 ):
min_span = min( groups[i+1] - groups[i], min_span )
for row in sql_results:
my_pivot = make_entry( row, pivot_cols, pivot_transform_func, row_size, **kw )
if my_pivot == None: continue
my_group = make_entry( row, grouping_cols, grouping_transform_func, row_size, **kw )
my_group = to_timestamp( my_group )
if not (my_pivot in parsed_results.keys()): parsed_results[my_pivot] = {}
if my_group in parsed_results[my_pivot].keys():
parsed_results[my_pivot][my_group] = add_data( parsed_results[my_pivot][my_group], row, results_cols )
else:
parsed_results[my_pivot][my_group] = new_data( row, results_cols, len_results_cols )
filtered_results = {}
metadata['kind'] = 'pivot-group'
#metadata['is_cumulative'] = True
for pivot in parsed_results.keys():
data = parsed_results[pivot]
tmp_group = {}
#pivot_has_nonzero = True
for grouping, info in data.items():
info = check_tuple( info, len_results_cols )
#if has_nonzero( info, len_results_cols ):
tmp_group[grouping] = info
#pivot_has_nonzero = True
#if pivot_has_nonzero:
filtered_results[pivot] = tmp_group
if len(groups) == 0:
return filtered_results, metadata
results = filtered_results
filtered_results = {};
current_group = groups.pop(0)
csum = {}
for pivot in results.keys():
csum[ pivot ] = 0
filtered_results[pivot] = {}
def add_cumulative_data( current_group ):
for pivot in results.keys():
if current_group in results[pivot].keys():
csum[ pivot ] = float(results[pivot][current_group])
filtered_results[pivot][current_group] = csum[ pivot ]
while len(groups) > 0:
next_group = groups[0]
add_cumulative_data( current_group )
while current_group + min_span < next_group:
current_group += min_span
add_cumulative_data( current_group )
current_group = groups.pop(0)
add_cumulative_data( current_group )
for pivot, groups in filtered_results.items():
for group, data in groups.items():
groups[group] = data_transform_func(data, **kw)
return filtered_results, metadata
