예제 #1
0
 def expand(self, rule, ruleinfo, **kwargs):
     overrides = self.rule_overrides.get(rule.name, {})
     for attr_name, values in overrides.items():
         if attr_name not in self.types:
             raise YmpConfigError(overrides,
                                  f'Cannot override "{attr_name}" field',
                                  key=attr_name)
         attr = getattr(ruleinfo, attr_name)
         if not isinstance(values, self.types[attr_name]):
             raise YmpConfigError(
                 overrides,
                 f'Overrides for "{attr_name}" must be of type "{self.types[attr_name].__name__}"'
                 f' (found type "{type(values).__name__}").',
                 key=attr_name,
             )
         if isinstance(values, Mapping):
             for val_name, value in values.items():
                 log.debug("Overriding {}.{}={} in {} with {}".format(
                     attr_name, val_name, attr[1][val_name], rule.name,
                     value))
                 attr[1][val_name] = value
         if isinstance(values, int):
             log.debug("Overriding {}={} in {} with {}".format(
                 attr_name, attr, rule.name, values))
             setattr(ruleinfo, attr_name, values)
예제 #2
0
파일: reference.py 프로젝트: epruesse/ymp
    def __init__(self, name, cfg):
        super().__init__("ref_" + name, cfg)
        #: Files provided by the reference. Keys are the file names
        #: within ymp ("target.extension"), symlinked into dir.ref/ref_name/ and
        #: values are the path to the reference file from workspace root.
        self.files: Dict[str, str] = {}
        self.archives = []
        self._ids: Set[str] = set()
        self._outputs = None

        import ymp
        self.dir = os.path.join(ymp.get_config().dir.references, name)

        if isinstance(cfg, Mapping):
            self.add_resource(cfg)
        elif isinstance(cfg, Sequence) and not isinstance(cfg, str):
            for item in cfg:
                self.add_resource(item)
        else:
            raise YmpConfigError(
                cfg, "Reference config must list or key-value mapping")

        # Copy rules defined in primary references stage
        stage_references = Stage.get_registry().get("references")
        if not stage_references:
            raise YmpConfigError(
                cfg, "Reference base stage not found. Main rules not loaded?")
        self.rules = stage_references.rules.copy()
예제 #3
0
파일: pipeline.py 프로젝트: epruesse/ymp
    def __init__(self, name: str, cfg) -> None:
        super().__init__(name, cfg)
        self._params = None
        self._outputs: Optional[Dict[str, str]] = None

        #: If true, outputs of stages are hidden by default
        self.hide_outputs = getattr(cfg, "hide", False)
        if 'params' in cfg and cfg.params is not None:
            if not isinstance(cfg.params, Mapping):
                raise YmpConfigError(cfg,
                                     "Params must contain a mapping",
                                     key="params")
            self._init_params(cfg.params)

        #: Dictionary of stages with configuration options for each
        self.stages = OrderedDict()
        path = ""
        if not "stages" in cfg:
            raise YmpConfigError(cfg, "Pipeline must have stages entry")
        for stage in cfg.stages:
            if stage is None:
                raise YmpConfigError(self,
                                     f"Empty stage name in pipeline '{name}'")
            if isinstance(stage, str):
                stage_name = stage
                stage_cfg = {}
            else:
                stage_name = next(iter(stage))
                stage_cfg = stage[stage_name]
            path = ".".join((path, stage_name))
            self.stages[path] = stage_cfg

        #: Path fragment describing this pipeline
        self.pipeline = path
예제 #4
0
파일: reference.py 프로젝트: epruesse/ymp
    def add_resource(self, rsc):
        if not isinstance(rsc, Mapping):
            raise YmpConfigError(
                rsc, "Reference resource config must be a key-value mapping")

        if not "url" in rsc:
            raise YmpConfigError(rsc,
                                 "Reference resource must have 'url' field")
        maybeurl = str(rsc["url"])
        import ymp
        local_path = make_local_path(ymp.get_config(), maybeurl)
        isurl = local_path != maybeurl
        if not isurl:
            local_path = rsc.get_path("url")

        type_name = rsc.get('type', 'fasta').lower()
        if 'id' in rsc:
            self._ids.add(rsc['id'])

        if type_name in ("fasta", "fastp"):
            self.files[f"ALL.{type_name}.gz"] = local_path
        elif type_name in ("gtf", "snp", "tsv", "csv"):
            self.files[f"ALL.{type_name}"] = local_path
        elif type_name == 'dir':
            archive = Archive(name=self.name,
                              dirname=self.dir,
                              tar=local_path,
                              url=maybeurl,
                              files=rsc['files'],
                              strip=rsc.get('strip_components', 0))
            self.files.update(archive.get_files())
            self.archives.append(archive)
        elif type_name == 'dirx':
            self.files.update({
                key: os.path.join(local_path, val)
                for key, val in rsc.get('files', {}).items()
            })
        elif type_name == 'path':
            self.dir = local_path.rstrip("/")
            try:
                filenames = os.listdir(local_path)
            except FileNotFoundError:
                log.error("Directory %s required by %s %s does not exist",
                          local_path, self.__class__.__name__, self.name)
                filenames = []
            for filename in filenames:
                for regex in rsc.get('match', []):
                    match = re.fullmatch(regex, filename)
                    if not match:
                        continue
                    self._ids.add(match.group('sample'))
                    self.files[filename] = os.path.join(local_path, filename)
        else:
            raise YmpConfigError(rsc, f"Unknown type {type_name}", key="type")
예제 #5
0
파일: config.py 프로젝트: ConnorJacobs/ymp
    def choose_id_column(self):
        """Configures column to use as index on runs

        If explicitly configured via KEY_IDCOL, verifies that the column
        exists and that it is unique. Otherwise chooses the leftmost
        unique column in the data.
        """
        import pandas as pd

        column_frequencies = self._runs.apply(pd.Series.nunique)
        log.debug("Column frequencies: {}".format(column_frequencies))
        nrows = self._runs.shape[0]
        log.debug("Row count: {}".format(nrows))
        unique_columns = self._runs.columns[column_frequencies == nrows]

        if unique_columns.empty:
            raise YmpConfigError(
                self.cfg,
                "Project data has no column containing unique values for "
                "each row. At least one is needed to identify samples!"
            )

        if self.KEY_IDCOL in self.cfg:
            idcol = self.cfg[self.KEY_IDCOL]
            if idcol not in self._runs.columns:
                raise YmpConfigError(
                    self.cfg, key=self.KEY_IDCOL,
                    msg="Configured column not found in data. "
                    "Possible spelling error? "
                    "Available columns: " +
                    ", ".join(str(c) for c in self._runs.columns))

            if idcol not in unique_columns:
                duplicated = self._runs.duplicated(subset=[idcol], keep=False)
                dup_rows = self._runs[duplicated].sort_values(by=idcol)
                raise YmpConfigError(
                    self.cfg, key=self.KEY_IDCOL,
                    msg="Configured id_col column '{}' is not unique.\n"
                    "Duplicated rows:\n {}\n"
                    "Unique columns: {}"
                    "".format(
                        idcol, dup_rows, list(unique_columns)
                    )
                )
        else:
            self.cfg[self.KEY_IDCOL] = unique_columns[0]
            log.info("Autoselected column %s=%s",
                     self.KEY_IDCOL, self.cfg[self.KEY_IDCOL])

        self._runs.set_index(self.cfg[self.KEY_IDCOL],
                             drop=False, inplace=True)
예제 #6
0
파일: pipeline.py 프로젝트: epruesse/ymp
 def _init_params(self, params):
     for param, data in params.items():
         if not isinstance(data, Mapping):
             raise YmpConfigError(data,
                                  "Param must contain a mapping",
                                  key=param)
         try:
             key = data['key']
             typ = data['type']
         except KeyError as exc:
             raise YmpConfigError(
                 data,
                 "Param must have at least key and type defined") from exc
         self.add_param(key, typ, param, data.get("value"),
                        data.get("default"))
예제 #7
0
파일: config.py 프로젝트: ConnorJacobs/ymp
    def group_by(self):
        if self._group_by is not None:
            return self._group_by

        df = self.dcfg.run_data
        import pandas as pd

        groupbys = []
        # extract groupby column from dir or by key, with by having preference
        for key in ['_YMP_DIR', 'dir', '_YMP_VRT', 'by']:
            if hasattr(self.wc, key):
                groupbys += self.RE_BY.findall(getattr(self.wc, key))

        if len(groupbys) == 0 or groupbys[-1] == "ALL":
            # no grouping desired
            # fake by grouping with virtual column containing "ALL" as value
            self._group_by = df.groupby(pd.Series("ALL", index=df.index))
        elif groupbys[-1] == "ID":
            # individual grouping desired
            # fake by grouping according to index
            self._group_by = df.groupby(df.index)
        else:
            try:
                self._group_by = df.groupby(groupbys[-1])
            except KeyError:
                raise YmpConfigError("Unkown column in groupby: {}"
                                     "".format(groupbys[-1]))
        return self._group_by
예제 #8
0
 def _load_file(self, cfg, key):
     fname = cfg.get_path(key)
     try:
         data = self.pd.read_csv(
             fname, sep=None, engine='python', dtype='str'
         )
     except FileNotFoundError:
         parts = fname.split('%', maxsplit=1)
         try:
             data = self.pd.read_excel(
                 parts[0], parts[1] if len(parts) > 1 else 0)
         except ImportError as exc:
             raise YmpConfigError(
                 cfg,
                 "Could not load specified data file."
                 " If this is an Excel file, you might need"
                 " to install 'openpyxl'.",
                 key=key
             ) from exc
     # prefix fq files with name of config file's directory
     rdir = os.path.dirname(fname)
     data = data.applymap(
         lambda s: os.path.join(rdir, s)
         if is_fq(s) and os.path.exists(os.path.join(rdir, s))
         else s
     )
     self.files.append(cfg)
     return data
예제 #9
0
 def parse_config(self, cfg):
     """Parses limits config"""
     limits = OrderedDict()
     for name, params in cfg.items():
         lconf = {}
         format_name = params.get("format")
         lconf["parser"] = self.parsers.get(format_name) or (
             lambda x, unit=None: x)
         lconf["formatter"] = self.formatters.get(format_name) or (
             lambda x, unit=None: x)
         unit = params.get("unit")
         if unit:
             if not format:
                 raise YmpConfigError(
                     cfg,
                     'Resource "unit" only valid with formatter',
                     key=name)
             lconf["unit"] = unit
         source = params.get("from")
         if source:
             if source not in cfg:
                 raise YmpConfigError(
                     cfg, f'Resource "from" ({source}) must reference'
                     f' previously defined resource (have {", ".join(cfg.keys())})',
                     key=name)
             lconf["from"] = source
         for opt in params:
             if opt in ("format", "unit", "from"):
                 continue
             if opt not in ("default", "scale", "min", "max"):
                 raise YmpConfigError(
                     params,
                     f'Unknown parameter "{opt}" in "{name}" resource_limits',
                     opt)
             try:
                 lconf[opt] = lconf['parser'](params.get(opt))
             except ValueError:
                 raise YmpConfigError(
                     params,
                     f'Failed to parse "{params.get(opt)}"',
                     key=opt) from None
         limits[name] = lconf
     for key in list(limits.keys()):
         if limits[key].get("from"):
             limits.move_to_end(key)
     return limits
예제 #10
0
 def load_data(self, cfg, key):
     if not (key in cfg or isinstance(cfg, Sequence)):
         raise YmpConfigError(cfg, f"Missing key '{key}' in project data config", key=key)
     value = cfg[key]
     if isinstance(value, str):
         return self._load_file(cfg, key)
     if isinstance(value, Sequence):
         return self._rowbind(cfg, key)
     if isinstance(value, Mapping):
         command = next(iter(value), None)
         if len(value) != 1 or command not in ("join", "paste", "table"):
             raise YmpConfigError(cfg, "Expecting exactly one of join, paste or table", key=key)
         if command == "join":
             return self._join(value["join"])
         if command == "paste":
             return self._paste(value["paste"])
         if command == "table":
             return self._table(value["table"])
     raise YmpConfigError(cfg, "Unrecognized statement in data config", key=key)
예제 #11
0
파일: config.py 프로젝트: ConnorJacobs/ymp
    def __init__(self, cfgmgr, project, cfg):
        self.project = project
        self.cfgmgr = cfgmgr
        self.cfg = cfg
        self.fieldnames = None
        self._runs = None
        self._source_cfg = None

        if self.KEY_DATA not in self.cfg:
            raise YmpConfigError(self.cfg, "Missing key '{}'".format(self.KEY_DATA))
예제 #12
0
    def source_path(self, target, pair, nosplit=False):
        """Get path for FQ file for ``run`` and ``pair``"""
        source = self.source_cfg.get(target)
        cfg = ymp.get_config()
        if not source:
            raise YmpConfigError(self.cfg,
                                 "No run '{}' in source config".format(target))

        if isinstance(pair, str):
            pair = self.pairnames.index(pair)

        if self.bccol and not nosplit:
            barcode_file, = self.data.fetch(self.bccol, self.idcol, target)[0]
            if barcode_file:
                return self.encode_barcode_path(barcode_file, target, pair)

        kind = source[0]
        if kind == 'srr':
            srr, = self.data.fetch(source[1], self.idcol, target)[0]
            f = os.path.join(cfg.dir.scratch,
                             "SRR",
                             "{}_{}.fastq.gz".format(srr, pair+1))
            return f

        fq_col = source[pair+1]
        if not isinstance(fq_col, str):
            return (
                "Configuration Error: no source for sample {} and read {} "
                "found.".format(target, pair+1))

        fn, = self.data.fetch(fq_col, self.idcol, target)[0]
        if kind == 'file':
            return fn

        if kind == 'remote':
            return make_local_path(cfg, fn)

        raise YmpConfigError(
            self.cfg,
            "Configuration Error: no source for sample {} and read {} found."
            "".format(target, pair+1))
예제 #13
0
    def __init__(self, name, cfg):
        super().__init__(name, cfg)
        self.pairnames = ymp.get_config().pairnames
        self.fieldnames = None
        self._data = None
        self._source_cfg = None
        self._idcol = None
        self.bccol = cfg.get(self.KEY_BCCOL)

        if self.KEY_DATA not in self.cfg:
            raise YmpConfigError(
                self.cfg, "Missing key '{}'".format(self.KEY_DATA))
예제 #14
0
 def _join(self, cfg):
     tables = list(map(self.load_data, [cfg]*len(cfg), range(len(cfg))))
     try:
         return self.pd.merge(*tables)
     except self.MergeError as e:
         raise YmpConfigError(
             cfg,
             "Failed to `join` configured data.\n"
             "Joined table indices:\n{}\n\n"
             "".format("\n".join(", ".join(table.columns.tolist())
                                 for table in tables))
         )
예제 #15
0
    def choose_id_column(self):
        """Configures column to use as index on runs

        If explicitly configured via KEY_IDCOL, verifies that the column
        exists and that it is unique. Otherwise chooses the leftmost
        unique column in the data.
        """
        all_columns = self.data.columns()
        unique_columns = self.data.identifying_columns()

        if not unique_columns:
            raise YmpConfigError(
                self.cfg,
                "Project data has no column containing unique values for "
                "each row. At least one is needed to identify samples!"
            )

        if self.KEY_IDCOL in self.cfg:
            idcol = self.cfg[self.KEY_IDCOL]
            if idcol not in all_columns:
                raise YmpConfigError(self.cfg, key=self.KEY_IDCOL, msg=(
                    "Configured column not found in data. "
                    "Possible spelling error? Available columns: "
                    ", ".join(all_columns)
                    ))

            if idcol not in unique_columns:
                raise YmpConfigError(self.cfg, key=self.KEY_IDCOL, msg=(
                    "Configured id_col column '{}' is not unique.\n"
                    "Duplicated rows:\n {}\n"
                    "Unique columns: {}".format(
                        idcol, self.data.duplicate_rows(idcol), unique_columns
                    )
                ))
        else:
            idcol = unique_columns[0]
            log.warning("Project '%s' using column '%s' to identify units",
                        self.name, idcol)

        return idcol
예제 #16
0
    def load_data(self, cfg):

        if isinstance(cfg, str):
            return self._load_file(cfg)
        if isinstance(cfg, Sequence):
            return self._rowbind(cfg)
        if isinstance(cfg, Mapping):
            if 'join' in cfg:
                return self._join(cfg['join'])
            if 'paste' in cfg:
                return self._paste(cfg['paste'])
            if 'table' in cfg:
                return self._table(cfg['table'])
        raise YmpConfigError(cfg, "Unrecognized statement in data config")
예제 #17
0
 def _paste(self, cfg):
     tables = list(map(self.load_data, cfg))
     manyrow = [table for table in tables if len(table) > 1]
     if manyrow:
         nrows = len(manyrow[0])
         if any(len(table) != nrows for table in manyrow[1:]):
             raise YmpConfigError(
                 cfg, "Failed to `paste` configured data. "
                 "Row counts differ and are not 1."
                 "Row counts: {}\n"
                 "".format(", ".join(str(len(table)) for table in manyrow)))
         tables = [
             table if len(table) > 1 else self.pd.concat([table] * nrows,
                                                         ignore_index=True)
             for table in tables
         ]
     return self.pd.concat(tables, axis=1)
예제 #18
0
파일: config.py 프로젝트: ConnorJacobs/ymp
    def choose_fq_columns(self):
        """
        Configures the columns referencing the fastq sources
        """
        import pandas as pd

        # get only columns containing string data
        string_cols = self.run_data.select_dtypes(include=['object'])
        # turn NaN into '' so they don't bother us later
        string_cols.fillna('', inplace=True)

        # if barcode column specified, omit that
        if self.KEY_BCCOL in self.cfg:
            string_cols.drop([self.cfg[self.KEY_BCCOL]], axis=1, inplace=True)

        # if read columns specified, constrain to those
        if self.KEY_READCOLS in self.cfg:
            read_cols = self.cfg[self.KEY_READCOLS]
            if isinstance(read_cols, str):
                read_cols = [read_cols]
            try:
                string_cols = string_cols[read_cols]
            except KeyError as e:
                raise YmpConfigError("{}={} references invalid columns: {}"
                                     "".format(self.KEY_READCOLS,
                                               read_cols,
                                               e.args))

        # select type to use for each row
        source_cfg = pd.DataFrame(index=self.runs,
                                  columns=['type', 'r1', 'r2'])

        # prepare array indicating which columns to use for each
        # row, and what type the row source data is
        for pat, nmax, msg, func in (
                (self.RE_FILE, 2, "fastq files", "file"),
                (self.RE_FILE, 1, "fastq files", "file"),
                (self.RE_REMOTE, 2, "remote URLs", "remote"),
                (self.RE_REMOTE, 1, "remote URLs", "remote"),
                (self.RE_SRR, 1, "SRR numbers", "srr")):
            # collect rows not yet assigned values
            no_type_yet = string_cols[source_cfg['type'].isnull()]
            # match the regex to each value
            match = no_type_yet.apply(lambda x: x.str.contains(pat))
            # check if we have more values than allowed for that
            # data source type
            broken_rows = match.sum(axis=1) > nmax
            if any(broken_rows):
                rows = list(self.runs[broken_rows])
                cols = list(self.run_data.columns[match[broken_rows].any])
                raise YmpConfigError(
                    "Some rows contain more than two {}. "
                    "Use {} to specify the desired rows. "
                    "Rows in question: {} "
                    "Columns in question: {} "
                    "".format(msg, self.KEY_READCOLS, rows, cols))
            # collect rows with matched data
            good_rows = match.sum(axis=1).eq(nmax)
            # prepare output matrix
            out = match[good_rows]
            out = out.apply(lambda x: (func,) + tuple(match.columns[x]),
                            axis=1)
            outm = out.apply(pd.Series, index=source_cfg.columns[0:nmax+1])
            source_cfg.update(outm, overwrite=False)

        return source_cfg
예제 #19
0
    def choose_fq_columns(self):
        """
        Configures the columns referencing the fastq sources
        """
        # get only columns containing string data
        string_cols = self.data.string_columns()

        # if barcode column specified, omit that
        if self.bccol:
            string_cols.remove(self.bccol)

        # if read columns specified, constrain to those
        read_cols = self.cfg.get(self.KEY_READCOLS)
        if read_cols:
            if isinstance(read_cols, str):
                read_cols = [read_cols]
            typo_cols = set(read_cols) - set(string_cols)
            if typo_cols:
                log.warning("%s=%s references invalid columns: %s",
                            self.KEY_READCOLS, read_cols, typo_cols)
                read_cols = [col for col in read_cols if col not in typo_cols]
        else:
            read_cols = string_cols

        if not read_cols:
            raise YmpConfigError(self.cfg, key=self.KEY_READCOLS, msg=(
                "No columns containing read files found"
            ))

        err = False
        source_config = {}
        for row in self.data.rows([self.idcol] + read_cols):
            cols = []
            for i, val in enumerate(row[2:]):
                if val is None:
                    val = ""
                if self.RE_FILE.match(val):
                    cols.append(("file", read_cols[i]))
                elif self.RE_REMOTE.match(val):
                    cols.append(("remote", read_cols[i]))
                elif self.RE_SRR.match(val):
                    cols.append(("srr", read_cols[i]))
            types = set(col[0] for col in cols)
            if not types:
                log.error("No data sources found in row %s.",
                          row[1])
                err = True
            elif len(types) > 1 or len(cols) > 2 or \
                 (cols[0] == 'srr' and len(cols) > 1):
                log.error("Ambiguous data sources found in row %s. "
                          "You may need to constrain the columns allowed "
                          "to contain read data using '%'.",
                          row[1], self.KEY_READCOLS)
                err = True
            elif len(cols) == 2:
                source_config[row[1]] = (cols[0][0], cols[0][1], cols[1][1])
            elif len(cols) == 1:
                source_config[row[1]] = (cols[0][0], cols[0][1], None)
            else:
                raise RuntimeError("this should not have happened")
        if err:
            raise YmpConfigError(self.cfg, msg=(
                "Failed to identify source data in project data config. "
                "See above log messages for details."
            ))

        return source_config
예제 #20
0
 def __init__(self, cfg: Optional[Mapping]) -> None:
     if not isinstance(cfg, Mapping):
         raise YmpConfigError(cfg,
                              "Limits section must be a map (key: value)")
     self.limits = self.parse_config(cfg)
예제 #21
0
파일: config.py 프로젝트: ConnorJacobs/ymp
def load_data(cfg):
    """Recursively loads csv/tsv type data as defined by yaml structure

    Format:
     - string items are files
     - lists of files are concatenated top to bottom
     - dicts must have one "command" value:

       - 'join' contains a two-item list
         the two items are joined 'naturally' on shared headers
       - 'table' contains a list of one-item dicts
         dicts have form ``key:value[,value...]``
         a in-place table is created from the keys
         list-of-dict is necessary as dicts are unordered
       - 'paste' contains a list of tables pasted left to right
         tables pasted must be of equal length or length 1
     - if a value is a valid path relative to the csv/tsv/xls file's
       location, it is expanded to a path relative to CWD

    Example:
     .. code-block:: yaml

      - top.csv
      - join:
        - bottom_left.csv
        - bottom_right.csv
      - table:
        - sample: s1,s2,s3
        - fq1: s1.1.fq, s2.1.fq, s3.1.fq
        - fq2: s1.2.fq, s2.2.fq, s3.2.fq

    """
    import pandas as pd
    from pandas.core.reshape.merge import MergeError

    if isinstance(cfg, str):
        try:
            data = pd.read_csv(cfg, sep=None, engine='python', dtype='str')
        except FileNotFoundError:
            parts = cfg.split('%')
            try:
                data = pd.read_excel(parts[0],
                                     parts[1] if len(parts) > 1 else 0)
            except ImportError:
                raise YmpConfigError(
                    cfg,
                    "Could not load specified data file."
                    " If this is an Excel file, you might need"
                    " to install 'xlrd'."
                )
        rdir = os.path.dirname(cfg)
        data = data.applymap(
            lambda s: os.path.join(rdir, s)
            if is_fq(s) and os.path.exists(os.path.join(rdir, s))
            else s)
        return data

    if isinstance(cfg, Sequence):
        return pd.concat(list(map(load_data, cfg)), ignore_index=True)
    if isinstance(cfg, Mapping):
        # JOIN
        if 'join' in cfg:
            tables = list(map(load_data, cfg['join']))
            try:
                return pd.merge(*tables)
            except MergeError as e:
                raise YmpConfigError(
                    cfg,
                    "Failed to `join` configured data.\n"
                    "Joined table indices:\n{}\n\n"
                    "".format("\n".join(", ".join(table.columns.tolist())
                                        for table in tables)),
                    exc=e)
        # PASTE
        if 'paste' in cfg:
            tables = list(map(load_data, cfg['paste']))
            manyrow = [table for table in tables if len(table) > 1]
            if manyrow:
                nrows = len(manyrow[0])
                if any(len(table) != nrows for table in manyrow[1:]):
                    raise YmpConfigError(
                        cfg,
                        "Failed to `paste` configured data. "
                        "Row counts differ and are not 1."
                        "Row counts: {}\n"
                        "".format(", ".join(str(len(table))
                                            for table in manyrow)))
                tables = [
                    table if len(table) > 1
                    else pd.concat([table]*nrows, ignore_index=True)
                    for table in tables
                ]
            return pd.concat(tables, axis=1)
        # TABLE
        if 'table' in cfg:
            return pd.DataFrame.from_dict({
                key: value.split(',')
                for row in cfg['table']
                for key, value in row.items()
            })
    raise YmpConfigError(cfg, "Unrecognized statement in data config")