def __init__(self, **kwargs):
if not 'container_type' in kwargs:
raise Exception('need to define a container element')
if not 'data' in kwargs or kwargs['data'] is None:
raise Exception('you need to define at least one data variable')
self.configuration = kwargs.copy()
if not 'visualization_css' in kwargs:
self.configuration['visualization_css'] = '{0}/templates/{1}.css'.format(SRC_DIR, self.configuration['visualization_name'])
if not 'visualization_js' in kwargs:
self.configuration['visualization_js'] = '{0}/templates/{1}.js'.format(SRC_DIR, self.configuration['visualization_name'])
self.configuration['visualization_name'] = self.configuration['visualization_name'].replace('-', '_').replace('.', '_')
self.configuration['variables'] = valmap(_dump_json, self.configuration['variables'])
if 'attributes' in self.configuration:
self.configuration['attributes'] = valmap(self.process_attribute, self.configuration['attributes'])
else:
self.configuration['attributes'] = {}
if 'colorables' in self.configuration:
self.configuration['colorables'] = valmap(self.process_colorable, self.configuration['colorables'])
else:
self.configuration['colorables'] = {}
self.configuration['__data_variables__'] = self.configuration['data'].keys()
self.configuration['data'] = _dump_json(self.configuration['data'])
if 'facets' in self.configuration:
self.configuration['facets'] = _dump_json(self.configuration['facets'])
def __init__(self, **kwargs):
if not 'container_type' in kwargs:
raise Exception('need to define a container element')
if not 'data' in kwargs or kwargs['data'] is None:
raise Exception('you need to define at least one data variable')
self.configuration = kwargs.copy()
if not 'visualization_css' in kwargs:
self.configuration['visualization_css'] = '{0}/templates/{1}.css'.format(SRC_DIR, self.configuration['visualization_name'])
if not 'visualization_js' in kwargs:
self.configuration['visualization_js'] = '{0}/templates/{1}.js'.format(SRC_DIR, self.configuration['visualization_name'])
self.configuration['visualization_name'] = self.configuration['visualization_name'].replace('-', '_').replace('.', '_')
self.configuration['variables'] = valmap(_dump_json, self.configuration['variables'])
if 'attributes' in self.configuration:
self.configuration['attributes'] = valmap(self.process_attribute, self.configuration['attributes'])
else:
self.configuration['attributes'] = {}
if 'colorables' in self.configuration:
self.configuration['colorables'] = valmap(self.process_colorable, self.configuration['colorables'])
else:
self.configuration['colorables'] = {}
self.configuration['__data_variables__'] = self.configuration['data'].keys()
self.configuration['data'] = _dump_json(self.configuration['data'])
if 'facets' in self.configuration:
self.configuration['facets'] = _dump_json(self.configuration['facets'])
def __init__(self, **kwargs):
if not 'container_type' in kwargs:
raise Exception('need to define a container element')
if not 'data' in kwargs or kwargs['data'] is None:
raise Exception('you need to define at least one data variable')
self.configuration = kwargs.copy()
if not 'visualization_css' in kwargs:
self.configuration[
'visualization_css'] = '{0}/templates/{1}.css'.format(
SRC_DIR, self.configuration['visualization_name'])
if not 'visualization_js' in kwargs:
self.configuration[
'visualization_js'] = '{0}/templates/{1}.js'.format(
SRC_DIR, self.configuration['visualization_name'])
self.configuration['visualization_name'] = self.configuration[
'visualization_name'].replace('-', '_').replace('.', '_')
self.configuration['variables'] = valmap(
self.process_variable, self.configuration['variables'])
if 'objects' in self.configuration:
self.configuration['objects'] = valmap(
self.process_objects, self.configuration['objects'])
if 'attributes' in self.configuration:
self.configuration['attributes'] = valmap(
self.process_attribute, self.configuration['attributes'])
else:
self.configuration['attributes'] = {}
if 'colorables' in self.configuration:
self.configuration['colorables'] = valmap(
self.process_colorable, self.configuration['colorables'])
else:
self.configuration['colorables'] = {}
if 'allowed_events' in self.configuration[
'options'] and self.configuration['options']['allowed_events']:
if 'events' in self.configuration:
self.configuration['events'] = {
k: self.process_event(k, v)
for k, v in self.configuration['events'].items()
if v is not None
}
self.configuration['__data_variables__'] = list(
self.configuration['data'].keys())
self.configuration['data'] = _dump_json(self.configuration['data'])
if 'facets' in self.configuration:
self.configuration['facets'] = _dump_json(
self.configuration['facets'])
self.configuration['datagram_events'] = self.datagram_events
def process_colorable(self, colorable):
if 'n_colors' not in colorable or colorable['n_colors'] is None:
# this is the seaborn default
colorable['n_colors'] = 6
if 'palette' in colorable and colorable['palette'] is not None:
if type(colorable['palette']) in (str, unicode):
# a palette name
palette = color_palette(colorable['palette'],
n_colors=colorable['n_colors'])
colorable['palette'] = map(rgb2hex, palette)
else:
# a list of colors. we override n_colors
colorable['palette'] = map(rgb2hex, colorable['palette'])
colorable['n_colors'] = len(colorable['palette'])
else:
colorable['palette'] = None
if 'domain' not in colorable:
colorable['domain'] = None
if 'legend' not in colorable:
colorable['legend'] = False
return valmap(_dump_json, colorable)
def process_colorable(self, colorable):
if 'domain' not in colorable:
colorable['domain'] = None
if 'n_colors' not in colorable or colorable['n_colors'] is None:
if colorable['domain'] is None:
# this is the seaborn default
colorable['n_colors'] = 6
else:
colorable['n_colors'] = len(colorable['domain'])
else:
if type(colorable['n_colors']) != int or colorable['n_colors'] < 1:
raise Exception('Number of colors must be an integer greater or equal than 1.')
if 'palette' in colorable and colorable['palette'] is not None:
if type(colorable['palette']) == str:
# a palette name
palette = color_palette(colorable['palette'], n_colors=colorable['n_colors'])
colorable['palette'] = list(map(rgb2hex, palette))
else:
# a list of colors. we override n_colors
colorable['palette'] = list(map(rgb2hex, colorable['palette']))
colorable['n_colors'] = len(colorable['palette'])
else:
colorable['palette'] = None
if 'legend' not in colorable:
colorable['legend'] = False
return valmap(self.process_variable, colorable)
def process_attribute(self, attribute):
if 'legend' not in attribute:
attribute['legend'] = False
if 'legend_location' not in attribute:
attribute['legend_location'] = None
if 'legend_orientation' not in attribute:
attribute['legend_orientation'] = None
return valmap(self.process_variable, attribute)
def process_attribute(self, attribute):
if 'legend' not in attribute:
attribute['legend'] = False
if 'legend_location' not in attribute:
attribute['legend_location'] = None
if 'legend_orientation' not in attribute:
attribute['legend_orientation'] = None
return valmap(self.process_variable, attribute)
def __init__(self, **kwargs):
if not 'container_type' in kwargs:
raise Exception('need to define a container element')
if not 'data' in kwargs or kwargs['data'] is None:
raise Exception('you need to define at least one data variable')
self.configuration = kwargs.copy()
if not 'visualization_css' in kwargs:
self.configuration['visualization_css'] = '{0}/templates/{1}.css'.format(SRC_DIR, self.configuration['visualization_name'])
if not 'visualization_js' in kwargs:
self.configuration['visualization_js'] = '{0}/templates/{1}.js'.format(SRC_DIR, self.configuration['visualization_name'])
self.configuration['visualization_name'] = self.configuration['visualization_name'].replace('-', '_').replace('.', '_')
self.configuration['variables'] = valmap(self.process_variable, self.configuration['variables'])
if 'objects' in self.configuration:
self.configuration['objects'] = valmap(self.process_objects, self.configuration['objects'])
if 'attributes' in self.configuration:
self.configuration['attributes'] = valmap(self.process_attribute, self.configuration['attributes'])
else:
self.configuration['attributes'] = {}
if 'colorables' in self.configuration:
self.configuration['colorables'] = valmap(self.process_colorable, self.configuration['colorables'])
else:
self.configuration['colorables'] = {}
if 'allowed_events' in self.configuration['options'] and self.configuration['options']['allowed_events']:
if 'events' in self.configuration:
self.configuration['events'] = {k: self.process_event(k, v) for k, v in self.configuration['events'].items() if v is not None}
self.configuration['__data_variables__'] = list(self.configuration['data'].keys())
self.configuration['data'] = _dump_json(self.configuration['data'])
if 'facets' in self.configuration:
self.configuration['facets'] = _dump_json(self.configuration['facets'])
self.configuration['datagram_events'] = self.datagram_events
def load_datasets(cache_dir, dbc, dsets, parse_all=False, resume=True):
import cytoolz.dicttoolz as dz
cache = dask.cache.Cache(8e9)
cache.register()
dask.set_options(get=dask.threaded.get, pool=ThreadPool())
configs = list(
map(lambda x: os.path.join('config/data', x),
os.listdir('config/data')))
cmap = {k: DatasetConfig.from_yaml(k) for k in configs}
dsids = dz.valmap(lambda ds: ds.id, cmap)
cmap = dz.merge(cmap, {d.id: d for d in cmap.values()})
if parse_all:
dsets = dsids
for d in dsets:
process_dataset(cmap[d], dbc, cache_dir)
def _get_wf_call_statuses(metadata):
calls = metadata['calls'].keys()
states = set([])
call_stats = {}
for c in calls:
tasks = metadata['calls'][c]
counts = pipe(tasks, map(get('executionStatus')), frequencies)
new_states = list(filter(lambda x: x not in states, counts.keys()))
if new_states:
for s in new_states:
states.add(s)
call_stats[c] = counts
base_states = {s: 0 for s in states}
final_stats = valmap(lambda d: merge(base_states, d), call_stats)
return (calls, sorted(states), final_stats)
def process_colorable(self, colorable):
if 'domain' not in colorable:
colorable['domain'] = None
if 'n_colors' not in colorable or colorable['n_colors'] is None:
if colorable['domain'] is None:
# this is the seaborn default
colorable['n_colors'] = 6
else:
colorable['n_colors'] = len(colorable['domain'])
else:
if type(colorable['n_colors']) != int or colorable['n_colors'] < 1:
raise Exception(
'Number of colors must be an integer greater or equal than 1.'
)
if 'palette' in colorable and colorable['palette'] is not None:
if type(colorable['palette']) == str:
# a palette name
palette = color_palette(colorable['palette'],
n_colors=colorable['n_colors'])
colorable['palette'] = list(map(rgb2hex, palette))
else:
# a list of colors. we override n_colors
colorable['palette'] = list(map(rgb2hex, colorable['palette']))
colorable['n_colors'] = len(colorable['palette'])
else:
colorable['palette'] = None
if 'legend' not in colorable:
colorable['legend'] = False
if 'legend_location' not in colorable:
colorable['legend_location'] = None
if 'legend_orientation' not in colorable:
colorable['legend_orientation'] = None
return valmap(self.process_variable, colorable)
def process_colorable(self, colorable):
if 'n_colors' not in colorable or colorable['n_colors'] is None:
# this is the seaborn default
colorable['n_colors'] = 6
if 'palette' in colorable and colorable['palette'] is not None:
if type(colorable['palette']) in (str, unicode):
# a palette name
palette = color_palette(colorable['palette'], n_colors=colorable['n_colors'])
colorable['palette'] = map(rgb2hex, palette)
else:
# a list of colors. we override n_colors
colorable['palette'] = map(rgb2hex, colorable['palette'])
colorable['n_colors'] = len(colorable['palette'])
else:
colorable['palette'] = None
if 'domain' not in colorable:
colorable['domain'] = None
if 'legend' not in colorable:
colorable['legend'] = False
return valmap(_dump_json, colorable)
def parse_surveyvars_spss(spss_file, lgr=logger):
"""Extracts dat metadata from CDC YRBS SPSS files
Extracts the survey questions and responses from the
CDC YRBS SPSS file.
Args:
spss_file: SPSS file path
Returns:
survey_vars: an OrderedDict with survey variables
The resulting `survey_vars` is an OrderedDict with
variable names as keys and dict values with fields of
`question` containing the survey question, and `responses`
containing a list of tuples of the form (resp_num, resp_label)
"""
# if arg is filename, open it
if not getattr(spss_file, 'read', False):
x = dl.fetch_data_from_url(spss_file)
t = x.read()
t = t.decode('utf-8', errors='ignore') if type(t) is bytes else t
spss_file = t.split('\n')
survey_vars = OrderedDict()
vars_flag = False
vals_flag = False
var = None
vals = []
# extract fixed-width-field length rows
for line in spss_file:
if line.startswith('VARIABLE LABELS'):
vars_flag = True
continue
elif vars_flag and strip_line(line) == '':
vars_flag = False
continue
elif vars_flag:
# parse variable label
# and add tuple (var, question/label)
(var, q) = strip_line(line).split(' ', 1)
var = var.lower()
survey_vars[var] = {'question': q,
'responses': [],
'is_integer': False}
elif line.startswith('VALUE LABELS'):
vals_flag = True
vars_flag = False
vals = []
var = None
continue
elif vals_flag and line.startswith('/.'):
# save the last var and val lbls
# reset vals_flag
survey_vars[var]['responses'] = vals[:]
survey_vars[var]['is_integer'] = all(x[0].isdigit() for x in vals)
var = None
vals = []
vals_flag = False
continue
elif vals_flag and line.strip() == '/':
# save the last var and val lbls
# reset
survey_vars[var]['responses'] = vals[:]
survey_vars[var]['is_integer'] = all(x[0].isdigit() for x in vals)
var = None
vals = []
continue
elif vals_flag and not var:
# set the current var
var = strip_line(line).lower()
continue
elif vals_flag and var:
# add (num, label) to current list of val labels
vals.append(tuple(strip_line(line).split(' ', 1)))
continue
else:
# default
continue
# end for line in fh.readline()...
survey_vars = {k: v['responses'] for k, v in survey_vars.items()}
survey_vars = valmap(lambda v: {1.0: "Yes", 2.0: "No"} if len(v)==0 else
{eager_float(t[0]): t[1] for t in v}, survey_vars)
lgr.info('parsed survey vars', n_keys=len(survey_vars), v=survey_vars)
return survey_vars
def process_attribute(self, attribute):
if 'legend' not in attribute:
attribute['legend'] = False
return valmap(_dump_json, attribute)
def test_valmap(self):
D, kw = self.D, self.kw
assert valmap(inc, D({1: 1, 2: 2}), **kw) == D({1: 2, 2: 3})
def process_attribute(self, attribute):
if 'legend' not in attribute:
attribute['legend'] = False
return valmap(_dump_json, attribute)
def test_valmap():
assert valmap(inc, {1: 1, 2: 2}) == {1: 2, 2: 3}
def test_environ():
# See: https://github.com/pycytoolz/cycytoolz/issues/127
assert keymap(identity, os.environ) == os.environ
assert valmap(identity, os.environ) == os.environ
assert itemmap(identity, os.environ) == os.environ
def process_attribute(self, attribute):
if 'legend' not in attribute:
attribute['legend'] = False
return valmap(self.process_variable, attribute)
def test_valmap():
assert valmap(inc, {1: 1, 2: 2}) == {1: 2, 2: 3}