Beispiel #1
0
    def __init__(self, **kwargs):
        if not 'container_type' in kwargs:
            raise Exception('need to define a container element')

        if not 'data' in kwargs or kwargs['data'] is None:
            raise Exception('you need to define at least one data variable')

        self.configuration = kwargs.copy()

        if not 'visualization_css' in kwargs:
            self.configuration['visualization_css'] =  '{0}/templates/{1}.css'.format(SRC_DIR, self.configuration['visualization_name'])
        if not 'visualization_js' in kwargs:
            self.configuration['visualization_js'] = '{0}/templates/{1}.js'.format(SRC_DIR, self.configuration['visualization_name'])

        self.configuration['visualization_name'] = self.configuration['visualization_name'].replace('-', '_').replace('.', '_')
        self.configuration['variables'] = valmap(_dump_json, self.configuration['variables'])

        if 'attributes' in self.configuration:
            self.configuration['attributes'] = valmap(self.process_attribute, self.configuration['attributes'])
        else:
            self.configuration['attributes'] = {}

        if 'colorables' in self.configuration:
            self.configuration['colorables'] = valmap(self.process_colorable, self.configuration['colorables'])
        else:
            self.configuration['colorables'] = {}

        self.configuration['__data_variables__'] = self.configuration['data'].keys()
        self.configuration['data'] = _dump_json(self.configuration['data'])
        if 'facets' in self.configuration:
            self.configuration['facets'] = _dump_json(self.configuration['facets'])
Beispiel #2
0
    def __init__(self, **kwargs):
        if not 'container_type' in kwargs:
            raise Exception('need to define a container element')

        if not 'data' in kwargs or kwargs['data'] is None:
            raise Exception('you need to define at least one data variable')

        self.configuration = kwargs.copy()

        if not 'visualization_css' in kwargs:
            self.configuration['visualization_css'] =  '{0}/templates/{1}.css'.format(SRC_DIR, self.configuration['visualization_name'])
        if not 'visualization_js' in kwargs:
            self.configuration['visualization_js'] = '{0}/templates/{1}.js'.format(SRC_DIR, self.configuration['visualization_name'])

        self.configuration['visualization_name'] = self.configuration['visualization_name'].replace('-', '_').replace('.', '_')
        self.configuration['variables'] = valmap(_dump_json, self.configuration['variables'])

        if 'attributes' in self.configuration:
            self.configuration['attributes'] = valmap(self.process_attribute, self.configuration['attributes'])
        else:
            self.configuration['attributes'] = {}

        if 'colorables' in self.configuration:
            self.configuration['colorables'] = valmap(self.process_colorable, self.configuration['colorables'])
        else:
            self.configuration['colorables'] = {}

        self.configuration['__data_variables__'] = self.configuration['data'].keys()
        self.configuration['data'] = _dump_json(self.configuration['data'])
        if 'facets' in self.configuration:
            self.configuration['facets'] = _dump_json(self.configuration['facets'])
Beispiel #3
0
    def __init__(self, **kwargs):
        if not 'container_type' in kwargs:
            raise Exception('need to define a container element')

        if not 'data' in kwargs or kwargs['data'] is None:
            raise Exception('you need to define at least one data variable')

        self.configuration = kwargs.copy()

        if not 'visualization_css' in kwargs:
            self.configuration[
                'visualization_css'] = '{0}/templates/{1}.css'.format(
                    SRC_DIR, self.configuration['visualization_name'])
        if not 'visualization_js' in kwargs:
            self.configuration[
                'visualization_js'] = '{0}/templates/{1}.js'.format(
                    SRC_DIR, self.configuration['visualization_name'])

        self.configuration['visualization_name'] = self.configuration[
            'visualization_name'].replace('-', '_').replace('.', '_')

        self.configuration['variables'] = valmap(
            self.process_variable, self.configuration['variables'])

        if 'objects' in self.configuration:
            self.configuration['objects'] = valmap(
                self.process_objects, self.configuration['objects'])

        if 'attributes' in self.configuration:
            self.configuration['attributes'] = valmap(
                self.process_attribute, self.configuration['attributes'])
        else:
            self.configuration['attributes'] = {}

        if 'colorables' in self.configuration:
            self.configuration['colorables'] = valmap(
                self.process_colorable, self.configuration['colorables'])
        else:
            self.configuration['colorables'] = {}

        if 'allowed_events' in self.configuration[
                'options'] and self.configuration['options']['allowed_events']:
            if 'events' in self.configuration:
                self.configuration['events'] = {
                    k: self.process_event(k, v)
                    for k, v in self.configuration['events'].items()
                    if v is not None
                }

        self.configuration['__data_variables__'] = list(
            self.configuration['data'].keys())
        self.configuration['data'] = _dump_json(self.configuration['data'])
        if 'facets' in self.configuration:
            self.configuration['facets'] = _dump_json(
                self.configuration['facets'])

        self.configuration['datagram_events'] = self.datagram_events
Beispiel #4
0
    def process_colorable(self, colorable):
        if 'n_colors' not in colorable or colorable['n_colors'] is None:
            # this is the seaborn default
            colorable['n_colors'] = 6

        if 'palette' in colorable and colorable['palette'] is not None:
            if type(colorable['palette']) in (str, unicode):
                # a palette name
                palette = color_palette(colorable['palette'],
                                        n_colors=colorable['n_colors'])
                colorable['palette'] = map(rgb2hex, palette)
            else:
                # a list of colors. we override n_colors
                colorable['palette'] = map(rgb2hex, colorable['palette'])
                colorable['n_colors'] = len(colorable['palette'])
        else:
            colorable['palette'] = None

        if 'domain' not in colorable:
            colorable['domain'] = None

        if 'legend' not in colorable:
            colorable['legend'] = False

        return valmap(_dump_json, colorable)
Beispiel #5
0
    def process_colorable(self, colorable):
        if 'domain' not in colorable:
            colorable['domain'] = None

        if 'n_colors' not in colorable or colorable['n_colors'] is None:
            if colorable['domain'] is None:
                # this is the seaborn default
                colorable['n_colors'] = 6
            else:
                colorable['n_colors'] = len(colorable['domain'])
        else:
            if type(colorable['n_colors']) != int or colorable['n_colors'] < 1:
                raise Exception('Number of colors must be an integer greater or equal than 1.')

        if 'palette' in colorable and colorable['palette'] is not None:
            if type(colorable['palette']) == str:
                # a palette name
                palette = color_palette(colorable['palette'], n_colors=colorable['n_colors'])
                colorable['palette'] = list(map(rgb2hex, palette))
            else:
                # a list of colors. we override n_colors
                colorable['palette'] = list(map(rgb2hex, colorable['palette']))
                colorable['n_colors'] = len(colorable['palette'])
        else:
            colorable['palette'] = None

        if 'legend' not in colorable:
            colorable['legend'] = False

        return valmap(self.process_variable, colorable)
Beispiel #6
0
    def process_attribute(self, attribute):
        if 'legend' not in attribute:
            attribute['legend'] = False

        if 'legend_location' not in attribute:
            attribute['legend_location'] = None

        if 'legend_orientation' not in attribute:
            attribute['legend_orientation'] = None

        return valmap(self.process_variable, attribute)
Beispiel #7
0
    def process_attribute(self, attribute):
        if 'legend' not in attribute:
            attribute['legend'] = False

        if 'legend_location' not in attribute:
            attribute['legend_location'] = None

        if 'legend_orientation' not in attribute:
            attribute['legend_orientation'] = None

        return valmap(self.process_variable, attribute)
Beispiel #8
0
    def __init__(self, **kwargs):
        if not 'container_type' in kwargs:
            raise Exception('need to define a container element')

        if not 'data' in kwargs or kwargs['data'] is None:
            raise Exception('you need to define at least one data variable')

        self.configuration = kwargs.copy()

        if not 'visualization_css' in kwargs:
            self.configuration['visualization_css'] =  '{0}/templates/{1}.css'.format(SRC_DIR, self.configuration['visualization_name'])
        if not 'visualization_js' in kwargs:
            self.configuration['visualization_js'] = '{0}/templates/{1}.js'.format(SRC_DIR, self.configuration['visualization_name'])

        self.configuration['visualization_name'] = self.configuration['visualization_name'].replace('-', '_').replace('.', '_')

        self.configuration['variables'] = valmap(self.process_variable, self.configuration['variables'])

        if 'objects' in self.configuration:
            self.configuration['objects'] = valmap(self.process_objects, self.configuration['objects'])

        if 'attributes' in self.configuration:
            self.configuration['attributes'] = valmap(self.process_attribute, self.configuration['attributes'])
        else:
            self.configuration['attributes'] = {}

        if 'colorables' in self.configuration:
            self.configuration['colorables'] = valmap(self.process_colorable, self.configuration['colorables'])
        else:
            self.configuration['colorables'] = {}

        if 'allowed_events' in self.configuration['options'] and self.configuration['options']['allowed_events']:
            if 'events' in self.configuration:
                self.configuration['events'] = {k: self.process_event(k, v) for k, v in self.configuration['events'].items() if v is not None}

        self.configuration['__data_variables__'] = list(self.configuration['data'].keys())
        self.configuration['data'] = _dump_json(self.configuration['data'])
        if 'facets' in self.configuration:
            self.configuration['facets'] = _dump_json(self.configuration['facets'])

        self.configuration['datagram_events'] = self.datagram_events
Beispiel #9
0
def load_datasets(cache_dir, dbc, dsets, parse_all=False, resume=True):
    import cytoolz.dicttoolz as dz
    cache = dask.cache.Cache(8e9)
    cache.register()
    dask.set_options(get=dask.threaded.get, pool=ThreadPool())
    configs = list(
        map(lambda x: os.path.join('config/data', x),
            os.listdir('config/data')))
    cmap = {k: DatasetConfig.from_yaml(k) for k in configs}
    dsids = dz.valmap(lambda ds: ds.id, cmap)
    cmap = dz.merge(cmap, {d.id: d for d in cmap.values()})
    if parse_all:
        dsets = dsids
    for d in dsets:
        process_dataset(cmap[d], dbc, cache_dir)
Beispiel #10
0
def _get_wf_call_statuses(metadata):
    calls = metadata['calls'].keys()
    states = set([])
    call_stats = {}

    for c in calls:
        tasks = metadata['calls'][c]
        counts = pipe(tasks, map(get('executionStatus')), frequencies)
        new_states = list(filter(lambda x: x not in states, counts.keys()))
        if new_states:
            for s in new_states:
                states.add(s)
        call_stats[c] = counts

    base_states = {s: 0 for s in states}

    final_stats = valmap(lambda d: merge(base_states, d), call_stats)
    return (calls, sorted(states), final_stats)
Beispiel #11
0
    def process_colorable(self, colorable):
        if 'domain' not in colorable:
            colorable['domain'] = None

        if 'n_colors' not in colorable or colorable['n_colors'] is None:
            if colorable['domain'] is None:
                # this is the seaborn default
                colorable['n_colors'] = 6
            else:
                colorable['n_colors'] = len(colorable['domain'])
        else:
            if type(colorable['n_colors']) != int or colorable['n_colors'] < 1:
                raise Exception(
                    'Number of colors must be an integer greater or equal than 1.'
                )

        if 'palette' in colorable and colorable['palette'] is not None:
            if type(colorable['palette']) == str:
                # a palette name
                palette = color_palette(colorable['palette'],
                                        n_colors=colorable['n_colors'])
                colorable['palette'] = list(map(rgb2hex, palette))
            else:
                # a list of colors. we override n_colors
                colorable['palette'] = list(map(rgb2hex, colorable['palette']))
                colorable['n_colors'] = len(colorable['palette'])
        else:
            colorable['palette'] = None

        if 'legend' not in colorable:
            colorable['legend'] = False

        if 'legend_location' not in colorable:
            colorable['legend_location'] = None

        if 'legend_orientation' not in colorable:
            colorable['legend_orientation'] = None

        return valmap(self.process_variable, colorable)
Beispiel #12
0
    def process_colorable(self, colorable):
        if 'n_colors' not in colorable or colorable['n_colors'] is None:
            # this is the seaborn default
            colorable['n_colors'] = 6

        if 'palette' in colorable and colorable['palette'] is not None:
            if type(colorable['palette']) in (str, unicode):
                # a palette name
                palette = color_palette(colorable['palette'], n_colors=colorable['n_colors'])
                colorable['palette'] = map(rgb2hex, palette)
            else:
                # a list of colors. we override n_colors
                colorable['palette'] = map(rgb2hex, colorable['palette'])
                colorable['n_colors'] = len(colorable['palette'])
        else:
            colorable['palette'] = None

        if 'domain' not in colorable:
            colorable['domain'] = None

        if 'legend' not in colorable:
            colorable['legend'] = False

        return valmap(_dump_json, colorable)
Beispiel #13
0
def parse_surveyvars_spss(spss_file, lgr=logger):
    """Extracts dat metadata from CDC YRBS SPSS files

    Extracts the survey questions and responses from the
    CDC YRBS SPSS file.

    Args:
        spss_file: SPSS file path

    Returns:
        survey_vars: an OrderedDict with survey variables

        The resulting `survey_vars` is an OrderedDict with
        variable names as keys and dict values with fields of
        `question` containing the survey question, and `responses`
        containing a list of tuples of the form (resp_num, resp_label)
    """
    # if arg is filename, open it
    if not getattr(spss_file, 'read', False):
        x = dl.fetch_data_from_url(spss_file)
        t = x.read()
        t = t.decode('utf-8', errors='ignore') if type(t) is bytes else t
        spss_file = t.split('\n')

    survey_vars = OrderedDict()
    vars_flag = False
    vals_flag = False
    var = None
    vals = []
    # extract fixed-width-field length rows
    for line in spss_file:
        if line.startswith('VARIABLE LABELS'):
            vars_flag = True
            continue
        elif vars_flag and strip_line(line) == '':
            vars_flag = False
            continue
        elif vars_flag:
            # parse variable label
            # and add tuple (var, question/label)
            (var, q) = strip_line(line).split(' ', 1)
            var = var.lower()
            survey_vars[var] = {'question': q,
                                'responses': [],
                                'is_integer': False}
        elif line.startswith('VALUE LABELS'):
            vals_flag = True
            vars_flag = False
            vals = []
            var = None
            continue
        elif vals_flag and line.startswith('/.'):
            # save the last var and val lbls
            # reset vals_flag
            survey_vars[var]['responses'] = vals[:]
            survey_vars[var]['is_integer'] = all(x[0].isdigit() for x in vals)
            var = None
            vals = []
            vals_flag = False
            continue
        elif vals_flag and line.strip() == '/':
            # save the last var and val lbls
            # reset
            survey_vars[var]['responses'] = vals[:]
            survey_vars[var]['is_integer'] = all(x[0].isdigit() for x in vals)
            var = None
            vals = []
            continue
        elif vals_flag and not var:
            # set the current var
            var = strip_line(line).lower()
            continue
        elif vals_flag and var:
            # add (num, label) to current list of val labels
            vals.append(tuple(strip_line(line).split(' ', 1)))
            continue
        else:
            # default
            continue
    # end for line in fh.readline()...
    survey_vars = {k: v['responses'] for k, v in survey_vars.items()}
    survey_vars = valmap(lambda v: {1.0: "Yes", 2.0: "No"} if len(v)==0 else 
           {eager_float(t[0]): t[1] for t in v}, survey_vars)
    lgr.info('parsed survey vars', n_keys=len(survey_vars), v=survey_vars)
    return survey_vars
Beispiel #14
0
    def process_attribute(self, attribute):
        if 'legend' not in attribute:
            attribute['legend'] = False

        return valmap(_dump_json, attribute)
Beispiel #15
0
 def test_valmap(self):
     D, kw = self.D, self.kw
     assert valmap(inc, D({1: 1, 2: 2}), **kw) == D({1: 2, 2: 3})
Beispiel #16
0
    def process_attribute(self, attribute):
        if 'legend' not in attribute:
            attribute['legend'] = False

        return valmap(_dump_json, attribute)
Beispiel #17
0
def test_valmap():
    assert valmap(inc, {1: 1, 2: 2}) == {1: 2, 2: 3}
def test_environ():
    # See: https://github.com/pycytoolz/cycytoolz/issues/127
    assert keymap(identity, os.environ) == os.environ
    assert valmap(identity, os.environ) == os.environ
    assert itemmap(identity, os.environ) == os.environ
Beispiel #19
0
    def process_attribute(self, attribute):
        if 'legend' not in attribute:
            attribute['legend'] = False

        return valmap(self.process_variable, attribute)
def test_valmap():
    assert valmap(inc, {1: 1, 2: 2}) == {1: 2, 2: 3}