Example #1
0
def traverse_and_remove_path(obj, path=None, match="First"):
    if path is None:
        path = []
    if isinstance(obj, dict):
        res = OrderedDict()
        for k, v in obj.items():
            cmatch = False
            if(match and len(path) > 0 and path[0] == k):
                cmatch = True
            res.update({k: traverse_and_remove_path(v, path=path[1:], match=cmatch)})
        if(len(path) == 1 and path[0] in res.keys() and match):
            del res[path[0]]
        return res
    elif isinstance(obj, list):
        res = []
        for i, elem in enumerate(obj):
            cmatch = False
            if(match and len(path) >= 1 and isinstance(path[0], int) and path[0] < len(obj) and i == path[0]):
                cmatch = True
            res.append(traverse_and_remove_path(elem, path=path[1:], match=cmatch))
        if(len(path) == 1 and isinstance(path[0], int) and path[0] < len(res) and match):
            res.pop(path[0])
        return res
    else:
        return obj  # no container, just values (str, int, float)
Example #2
0
def get_data(task, key, separate_cols=True):
    data_dir = results_dir

    if separate_cols:
        data = OrderedDict()
    else:
        data = []
    dirs = [d for d in sorted(os.listdir(data_dir))
            if '.sim' in d]
    sims = [os.path.basename(d).split('.')[0] for d in dirs]

    for dir_, sim in zip(dirs, sims):
        if separate_cols:
            data[sim] = []

        paths = glob(os.path.join(data_dir, dir_, "test_%s*.txt" % task))
        for path in paths:
            with open(path, 'r') as fp:
                try:
                    if separate_cols:
                        data[sim].append(json.load(fp)[key])
                    else:
                        data.append((sim, json.load(fp)[key]))
                except:
                    print(path)
                    raise
    if separate_cols:
        return pd.DataFrame(data)
    else:
        return pd.DataFrame(data, columns=['Backend', key])
Example #3
0
    def get_hidden_layers(cls,
                          pins,
                          phidden={
                              "dim": 800,
                              "nlayers": 2,
                              "batch_norm": False
                          },
                          nn_lin="ReLU",
                          name=""):
        '''outputs the hidden module of the layer.
        :param input_dim: dimension of the input
        :type input_dim: int
        :param phidden: parameters of hidden layers
        :type phidden: dict
        :param nn_lin: non-linearity name
        :type nn_lin: str
        :param name: name of module
        :type name: str
        :returns: nn.Sequential'''
        # Hidden layers
        input_dim = 0
        for p in pins:
            new_input = p['dim'] if issubclass(type(p), dict) else p
            input_dim += new_input

        residual = phidden.get("residual", False)
        LayerModule = MLPResidualLayer if residual else MLPLayer

        if not issubclass(type(phidden), list):
            modules = OrderedDict()
            nlayers = phidden.get('nlayers', 1)
            hidden_dim = phidden.get('dim', 800)
            for i in range(nlayers):
                n_in = int(input_dim) if i == 0 else int(hidden_dim)
                modules['layer_%d' % i] = LayerModule(
                    n_in,
                    int(hidden_dim),
                    nn_lin=nn_lin,
                    batch_norm=phidden.get('batch_norm', False),
                    dropout=phidden.get('dropout'),
                    name_suffix="_%d" % i)
            return nn.Sequential(modules)

        else:
            modules = nn.ModuleList()
            for i, ph in enumerate(phidden):
                mod = OrderedDict()
                nlayers = ph.get('nlayers', 1)
                hidden_dim = phidden.get('dim', 800)
                for i in range(ph['nlayers']):
                    n_in = int(input_dim) if i == 0 else int(hidden_dim)
                    mod['layer_%d' % i] = LayerModule(
                        n_in,
                        int(hidden_dim),
                        nn_lin=nn_lin,
                        batch_norm=ph.get('batch_norm', False),
                        dropout=ph.get('dropout', None),
                        name_suffix="_%d" % i)
                modules.append(nn.Sequential(mod))
            return modules
Example #4
0
    def check_for_add_to_toolbars(self, plugin):
        from calibre.gui2.preferences.toolbar import ConfigWidget
        from calibre.customize import InterfaceActionBase

        if not isinstance(plugin, InterfaceActionBase):
            return

        all_locations = OrderedDict(ConfigWidget.LOCATIONS)
        plugin_action = plugin.load_actual_plugin(self.gui)
        installed_actions = OrderedDict([
            (key, list(gprefs.get('action-layout-' + key, [])))
            for key in all_locations
        ])

        # If already installed in a GUI container, do nothing
        for action_names in installed_actions.itervalues():
            if plugin_action.name in action_names:
                return

        allowed_locations = [(key, text)
                             for key, text in all_locations.iteritems()
                             if key not in plugin_action.dont_add_to]
        if not allowed_locations:
            return  # This plugin doesn't want to live in the GUI

        from calibre.gui2.dialogs.choose_plugin_toolbars import ChoosePluginToolbarsDialog
        d = ChoosePluginToolbarsDialog(self, plugin_action, allowed_locations)
        if d.exec_() == d.Accepted:
            for key, text in d.selected_locations():
                installed_actions = list(gprefs.get('action-layout-' + key,
                                                    []))
                installed_actions.append(plugin_action.name)
                gprefs['action-layout-' + key] = tuple(installed_actions)
Example #5
0
    def export(self, sheet, errorfmt=None, warningfmt=None):
        data = OrderedDict()
        visit_dates = [i.data.get('date') for i in self.patient.entries if i.form.slug == 'nurtureckd']

        within_range = False
        for result in self.results:
            datestr = result.date.strftime(DATETIMEFMT)
            key = (result.patient_id, result.source_group.name, result.source_type, datestr)
            if key not in data:
                data[key] = {}

            data[key][result.observation.name] = result.value_label_or_value

            for visit in visit_dates:
                visit_date = datetime.datetime.strptime(visit, '%Y-%m-%d')
                if not within_range and in_date_range(visit_date, result.date.replace(tzinfo=None)):
                    within_range = True

        if not within_range:
            msg = 'NO RESULTS WITHIN {} DAYS OF VISIT'.format(DAYS_RESULTS_SHOULD_BE_WITHIN)
            sheet.write_row([self.patient.id, msg], errorfmt)

        for key, results in data.items():
            data = list(key)
            for test in self.observations:
                data.append(results.get(test, None))

            sheet.write_row(data)
Example #6
0
def get_replacement_dict(rules):
    """Given a string with replacement rules, produces a dict of from: to"""

    if isinstance(rules, (binary_type, text_type)):
        rules = [rules]

    pairs = OrderedDict()
    for rule in rules:
        if isinstance(rule, (list, tuple)):
            if len(rule) == 2:
                pairs.append(rule)
            else:
                raise ValueError("Got a rule %s which is not a string or a pair of values (from, to)"
                                 % repr(rule))
        if len(rule) <= 2:
            raise ValueError("")
        rule_split = rule[1:].split(rule[0])
        if len(rule_split) != 2:
            raise ValueError(
                "Rename string must be of format '/pat1/replacement', "
                "where / is an arbitrary character to decide replacement. "
                "Got %s when trying to separate %s" % (rule_split, rule)
            )
        pairs[rule_split[0]] = rule_split[1]
    return pairs
Example #7
0
    def __call__(self, im, im_info=None, label=None):
        """
        Args:
            im (np.ndarray): 图像np.ndarray数据。
            im_info (list): 存储图像reisze或padding前的shape信息,如
                [('resize', [200, 300]), ('padding', [400, 600])]表示
                图像在过resize前shape为(200, 300), 过padding前shape为
                (400, 600)
            label (np.ndarray): 标注图像np.ndarray数据。

        Returns:
            tuple: 当label为空时,返回的tuple为(im, im_info),分别对应图像np.ndarray数据、存储与图像相关信息的字典;
                当label不为空时,返回的tuple为(im, im_info, label),分别对应图像np.ndarray数据、
                存储与图像相关信息的字典和标注图像np.ndarray数据。
                其中,im_info新增字段为:
                    -shape_before_padding (tuple): 保存padding之前图像的形状(h, w)。

        Raises:
            ValueError: 输入图像im或label的形状大于目标值
        """
        if im_info is None:
            im_info = OrderedDict()
        im_info.append(('padding', im.shape[:2]))

        im_height, im_width = im.shape[0], im.shape[1]
        if isinstance(self.target_size, int):
            target_height = self.target_size
            target_width = self.target_size
        else:
            target_height = self.target_size[1]
            target_width = self.target_size[0]
        pad_height = target_height - im_height
        pad_width = target_width - im_width
        pad_height = max(pad_height, 0)
        pad_width = max(pad_width, 0)
        if (pad_height > 0 or pad_width > 0):
            im_channel = im.shape[2]
            import copy
            orig_im = copy.deepcopy(im)
            im = np.zeros((im_height + pad_height, im_width + pad_width,
                           im_channel)).astype(orig_im.dtype)
            for i in range(im_channel):
                im[:, :,
                   i] = np.pad(orig_im[:, :, i],
                               pad_width=((0, pad_height), (0, pad_width)),
                               mode='constant',
                               constant_values=(self.im_padding_value[i],
                                                self.im_padding_value[i]))

            if label is not None:
                label = np.pad(label,
                               pad_width=((0, pad_height), (0, pad_width)),
                               mode='constant',
                               constant_values=(self.label_padding_value,
                                                self.label_padding_value))

        if label is None:
            return (im, im_info)
        else:
            return (im, im_info, label)
Example #8
0
def parse(tokens):
    if tokens[0] == '{':
        ret = OrderedDict()
        tokens = tokens[1:]
        while tokens[0] != '}':
            key = tokens[0]
            tokens = tokens[1:]

            tokens = tokens[1:]  # Skip ':'

            value, tokens = parse(tokens)

            if tokens[0] == ',':
                tokens = tokens[1:]

            ret[key] = value
        tokens = tokens[1:]
        return ret, tokens
    elif tokens[0] == '[':
        ret = []
        tokens = tokens[1:]
        while tokens[0] != ']':
            value, tokens = parse(tokens)
            if tokens[0] == ',':
                tokens = tokens[1:]
            ret.append(value)
        tokens = tokens[1:]
        return ret, tokens
    else:
        return tokens[0], tokens[1:]
Example #9
0
 def recurse(pair, out=None, indent=0):
     if isinstance(pair, dict):
         (k, v) = pair.popitem()
         if out is None:
             if isinstance(k, int) or k == '':
                 out = []
             else:
                 out = OrderedDict()
         if isinstance(out, list) and (k == ''):
             out.append(recurse(v, indent=indent + 1))
         elif isinstance(out, list) and (k != ''):
             out = dict(enumerate(out))
             out[k] = recurse(v, out.get(k), indent=indent + 1)
         elif isinstance(out, dict) and k == '':
             numeric_indices = [
                 k for k in out.keys() if isinstance(k, int)
             ]
             out[1 + max(numeric_indices
                         ) if numeric_indices else 0] = recurse(
                             v, out.get(k))
         elif isinstance(out, dict):
             out[k] = recurse(v, out.get(k), indent=indent + 1)
         else:
             return recurse({k: v}, indent=indent + 1)
         return out
     return pair
Example #10
0
def read_node(node):
    """Recursively read the given YAML piece, returning an appropriate collection."""
    line = node.start_mark.line + 1
    if isinstance(node, nodes.ScalarNode):
        tag = node.tag.split(":")[-1]
        if tag in ["int", "float"]:
            constructor = eval(tag)
            value = constructor(node.value)
        elif tag == "str":
            value = node.value
        else:
            raise RuntimeError(
                "cannot parse this scalar at line {}".format(line))

        return (value, line)

    if isinstance(node, nodes.MappingNode):
        col = OrderedDict()
        for node_name, node_value in node.value:
            name = read_node(node_name)[0]
            value = read_node(node_value)
            col[name] = value

        col["--begin"] = line
        return col

    if isinstance(node, nodes.SequenceNode):
        col = []
        for node_value in node.value:
            value = read_node(node_value)
            col.append(value)

        return col

    raise RuntimeError("cannot parse the node at line {}".format(line))
Example #11
0
def decode(message):
    ''' Decodes bencoded text, recursively parses lists and dictionaries '''
    ''' All dictionaries are represented as OrderedDicts, lists as Python lists, strings as strings, ints as ints '''
    ''' Returns 'highest' parsed unit and the remaining bencoded text '''
    current = message[0]
    if current is 'd':
        token = OrderedDict()

        i = 1
        while message[i] is not 'e':
            key, rest = _decode_next_string(message[i : ])
            value, rest = decode(rest)
            token[key] = value
            message = rest
            i = 0
        return token, rest[1 : ] # skip the dict's 'e'

    elif current is 'l':
        token = []

        i = 1
        while message[i] is not 'e':
            item, rest = decode(message[i : ])
            token.append(item)
            message = rest
            i = 0
        return token, rest[1 : ] # skip the list's 'e'

    else:
        token, rest = _decode_next_unit(message)
        return token, rest
Example #12
0
def elementToValue(field, element, default=_marker):
    """Read the contents of an element that is assumed to represent a value
    allowable by the given field.

    If converter is given, it should be an IToUnicode instance.

    If not, the field will be adapted to this interface to obtain a converter.
    """
    value = default

    if IDict.providedBy(field):
        key_converter = IFromUnicode(field.key_type)
        value = OrderedDict()
        for child in element.iterchildren(tag=etree.Element):
            if noNS(child.tag.lower()) != 'element':
                continue
            parseinfo.stack.append(child)

            key_text = child.attrib.get('key', None)
            if key_text is None:
                k = None
            else:
                k = key_converter.fromUnicode(unicode(key_text))

            value[k] = elementToValue(field.value_type, child)
            parseinfo.stack.pop()
        value = fieldTypecast(field, value)

    elif ICollection.providedBy(field):
        value = []
        for child in element.iterchildren(tag=etree.Element):
            if noNS(child.tag.lower()) != 'element':
                continue
            parseinfo.stack.append(child)
            v = elementToValue(field.value_type, child)
            value.append(v)
            parseinfo.stack.pop()
        value = fieldTypecast(field, value)

    # Unicode
    else:
        text = element.text
        if text is None:
            value = field.missing_value
        else:
            converter = IFromUnicode(field)
            value = converter.fromUnicode(unicode(text))

        # handle i18n
        if isinstance(value, unicode) and parseinfo.i18n_domain is not None:
            translate_attr = ns('translate', I18N_NAMESPACE)
            domain_attr = ns('domain', I18N_NAMESPACE)
            msgid = element.attrib.get(translate_attr)
            domain = element.attrib.get(domain_attr, parseinfo.i18n_domain)
            if msgid:
                value = Message(msgid, domain=domain, default=value)
            elif translate_attr in element.attrib:
                value = Message(value, domain=domain)

    return value
Example #13
0
    def __call__(self, im, im_info=None, label=None):
        """
        Args:
            im (np.ndarray): 图像np.ndarray数据。
            im_info (list): 存储图像reisze或padding前的shape信息,如
                [('resize', [200, 300]), ('padding', [400, 600])]表示
                图像在过resize前shape为(200, 300), 过padding前shape为
                (400, 600)
            label (np.ndarray): 标注图像np.ndarray数据。

        Returns:
            tuple: 当label为空时,返回的tuple为(im, im_info),分别对应图像np.ndarray数据、存储与图像相关信息的字典;
                当label不为空时,返回的tuple为(im, im_info, label),分别对应图像np.ndarray数据、
                存储与图像相关信息的字典和标注图像np.ndarray数据。
                其中,im_info新增字段为:
                    -shape_before_resize (tuple): 保存resize之前图像的形状(h, w)。
        """
        if im_info is None:
            im_info = OrderedDict()

        im_info.append(('resize', im.shape[:2]))
        im = resize_long(im, self.long_size)
        if label is not None:
            label = resize_long(label, self.long_size, cv2.INTER_NEAREST)

        if label is None:
            return (im, im_info)
        else:
            return (im, im_info, label)
Example #14
0
def more_heuristic_attempts(blocksizes):
    # Ramp up to higher block sizes
    handle = OrderedDict([(i, options['at_blocksize'][-1])
                          for i in blocksizes[0]])
    for i in range(3):
        new_bs = OrderedDict([(k, v * 2) for k, v in handle.items()])
        blocksizes.insert(blocksizes.index(handle) + 1, new_bs)
        handle = new_bs

    handle = []
    # Extended shuffling for the smaller block sizes
    for bs in blocksizes[:4]:
        for i in blocksizes:
            handle.append(
                OrderedDict(list(bs.items())[:-1] + [list(i.items())[-1]]))
    # Some more shuffling for all block sizes
    for bs in list(blocksizes):
        ncombs = len(bs)
        for i in range(ncombs):
            for j in combinations(bs, i + 1):
                item = [(k, bs[k] * 2 if k in j else v) for k, v in bs.items()]
                handle.append(OrderedDict(item))

    unique = []
    for i in blocksizes + handle:
        if i not in unique:
            unique.append(i)

    return unique
Example #15
0
def parse(tokens):
    if tokens[0] == '{':
        ret = OrderedDict()
        tokens = tokens[1:]
        while tokens[0] != '}':
            key = tokens[0]
            tokens = tokens[1:]

            tokens = tokens[1:]  # Skip ':'

            value, tokens = parse(tokens)

            if tokens[0] == ',':
                tokens = tokens[1:]

            ret[key] = value
        tokens = tokens[1:]
        return ret, tokens
    elif tokens[0] == '[':
        ret = []
        tokens = tokens[1:]
        while tokens[0] != ']':
            value, tokens = parse(tokens)
            if tokens[0] == ',':
                tokens = tokens[1:]
            ret.append(value)
        tokens = tokens[1:]
        return ret, tokens
    else:
        return tokens[0], tokens[1:]
Example #16
0
    def _serialize(self):
        """
        Serialize the sheet in a JSON-ready format.
        """
        obj = OrderedDict() 

        if 'key' in self._columns and 'value' in self._columns:
            for row in self:
                obj[row['key']] = row['value']
        elif 'key' in self._columns:
            for row in self:
                obj[row['key']] = OrderedDict() 

                for column in self._columns:
                    if column == 'key':
                        continue

                    value = row[column]

                    obj[row['key']][column] = value
        else:
            obj = []

            for row in self:
                row_obj = OrderedDict() 

                for i, column in enumerate(row):
                    row_obj[self._columns[i]] = column

                obj.append(row_obj)

        return obj 
Example #17
0
    def insert(self,intervals,newinterval,merge=1):
        '''
        O(logn) time implementation of insert method
        ans and intervals are of type OrderedDict
        a in ans, i in intervals is of form {start time:(end time, index}
        new interval is given via a tuple (start time, end time, index)
        if merge=1, the overlapping intervals will be printed out and then merge.
        '''
        # suppose OrderedDict is implemented using double linked list
        ans = OrderedDict()

        # find left bound
        keys = list(intervals.keys())
        left_key=keys[bisect.bisect_left(keys,newinterval[0])]
        right_key=keys[bisect.bisect_left(keys,newinterval[1])]
        ......
        for i in range(len(intervals)):
            if newInterval[0] < intervals[i][0]:
                index = i
                break

        intervals.insert(index, newInterval) # this insert is built_in method for list

        # then repeat the same prodecure as Interval.merge(self, intervals)
        for interval in intervals:
            if not ans or interval[0] > ans[-1][1]:
                ans.append(interval)
            else:
                if(merge==0):
                    print(interval, ans[-1])
                    continue
                elif(merge==1):
                    ans[-1][1] = max(ans[-1][1], interval[1])
        return ans
Example #18
0
def followup_targets(events, fields, area):
    """A generator of event/astroplan target pairs given events and fields

    All ra and dec values should be in decimal degrees.

    Args:
       - events :: a pandas.DataFrame with event coordinates
                   in ra and decl columns
       - fields :: a pandas.DataFrame with field coordinates
                   in fieldRA and fieldDec columns
       - area :: the search area in square degrees

    Returns:
       a succession of event/astroplan.FixedTarget pairs
    """

    event_fields = find_event_fields(events, fields, area)
    targets = OrderedDict()

    # there should be a faster (vectorized) way to do this.
    for idx, event in events.iterrows():
        targets = []
        for field_idx, field in event_fields.query(
                f'event_id=={event.event_id}').iterrows():
            field_coords = SkyCoord(ra=field.fieldRA * u.deg,
                                    dec=field.fieldDec * u.deg)
            field_name = "%d" % field.fieldID
            target = astroplan.FixedTarget(coord=field_coords, name=field_name)
            targets.append(target)

        yield event, targets
Example #19
0
class MultiKeyInfo(BaseKeyInfo):
    def __init__(self, name, datatype, minOccurs, maxOccurs, handler,
                 attribute):
        BaseKeyInfo.__init__(self, name, datatype, minOccurs, maxOccurs,
                             handler, attribute)
        if self.name == "+":
            self._default = OrderedDict()
        else:
            self._default = []

    def add_valueinfo(self, vi, key):
        if self.name == "+":
            # This is a keyed value, not a simple value:
            if key in self._default:
                self._default[key].append(vi)
            else:
                self._default[key] = [vi]
        else:
            self._default.append(vi)

    def computedefault(self, keytype):
        self.prepare_raw_defaults()
        for k, vlist in self._rawdefaults.items():
            key = ValueInfo(k, vlist[0].position).convert(keytype)
            for vi in vlist:
                self.add_valueinfo(vi, key)

    def getdefault(self):
        return copy.copy(self._default)
Example #20
0
def get_data(task, key, separate_cols=True):
    data_dir = results_dir

    if separate_cols:
        data = OrderedDict()
    else:
        data = []
    dirs = [d for d in sorted(os.listdir(data_dir)) if '.sim' in d]
    sims = [os.path.basename(d).split('.')[0] for d in dirs]

    for dir_, sim in zip(dirs, sims):
        if separate_cols:
            data[sim] = []

        paths = glob(os.path.join(data_dir, dir_, "test_%s*.txt" % task))
        for path in paths:
            with open(path, 'r') as fp:
                try:
                    if separate_cols:
                        data[sim].append(json.load(fp)[key])
                    else:
                        data.append((sim, json.load(fp)[key]))
                except:
                    print(path)
                    raise
    if separate_cols:
        return pd.DataFrame(data)
    else:
        return pd.DataFrame(data, columns=['Backend', key])
Example #21
0
class MultiKeyInfo(BaseKeyInfo):

    def __init__(self, name, datatype, minOccurs, maxOccurs, handler,
                 attribute):
        BaseKeyInfo.__init__(self, name, datatype, minOccurs, maxOccurs,
                             handler, attribute)
        if self.name == "+":
            self._default = OrderedDict()
        else:
            self._default = []

    def add_valueinfo(self, vi, key):
        if self.name == "+":
            # This is a keyed value, not a simple value:
            if key in self._default:
                self._default[key].append(vi)
            else:
                self._default[key] = [vi]
        else:
            self._default.append(vi)

    def computedefault(self, keytype):
        self.prepare_raw_defaults()
        for k, vlist in self._rawdefaults.items():
            key = ValueInfo(k, vlist[0].position).convert(keytype)
            for vi in vlist:
                self.add_valueinfo(vi, key)

    def getdefault(self):
        return copy.copy(self._default)
Example #22
0
    def enrich(self, value):
        """
        Given a YAML value, performs our registered transformations on it.
        """
        if hasattr(value, 'enrich'):
            return value.enrich(self)

        # need to special-case str before Sequence because str \subset Sequence
        elif isinstance(value, str):
            return value

        elif isinstance(value, Mapping):
            out = OrderedDict()
            for key, val in value.items():
                val = self.enrich(val)
                if val is not Void:
                    out[key] = val
            return out
        elif isinstance(value, Sequence):
            out = []
            for item in value:
                item = self.enrich(item)
                if item is not Void:
                    out.append(item)
            return out
        else:
            return value
Example #23
0
def _get_sorteddict(object, dictwithhash=False):
    if isinstance(object, dict):
        out = OrderedDict()
        for key, val in sorted(object.items()):
            if val:
                out[key] = _get_sorteddict(val, dictwithhash)
    elif isinstance(object, (list, tuple)):
        out = []
        for val in object:
            if val:
                out.append(_get_sorteddict(val, dictwithhash))
        if isinstance(object, tuple):
            out = tuple(out)
    else:
        if isinstance(object, str) and os.path.isfile(object):
            hash = hash_infile(object)
            if dictwithhash:
                out = (object, hash)
            else:
                out = hash
        elif isinstance(object, float):
            out = '%.10f'.format(object)
        else:
            out = object
    return out
Example #24
0
 def get_semodule(self, exisse, blocks):
     """
         以多种方式添加SeModule到MoBlock上,默认reduction=4
     """
     if exisse == () or isinstance(exisse[0], tuple):
         # 指定要添加的block-id 和 reduce-val
         exisse = OrderedDict(exisse)
     elif isinstance(exisse, (list, tuple)):
         # 指定要添加的block-id
         exisse_dict = OrderedDict()
         for id in exisse:
             exisse_dict.setdefault(id, 4)
         exisse = exisse_dict
     elif exisse == 'all':
         # 所有block都添加SeModule
         exisse = OrderedDict()
         for i in range(1, blocks + 1):
             exisse.setdefault(i, 4)
     elif exisse.startswith('ft-'):
         # 指定block添加SeModule
         # ft-3:5-7:10-15:18, from 3-5, 7-10, 15-18.
         # ft-0:0 不添加
         ft = exisse[3:].split('-')
         exisse = []
         for idx in ft:
             idx = [int(x) for x in idx.split(':')]
             for i in range(idx[0], idx[1] + 1):
                 exisse.append((i, 4))
         exisse = OrderedDict(exisse)
     return exisse
Example #25
0
def read_xlsx(file_path, sheet_metaes):
    workbook = xlrd.open_workbook(file_path)
    xlsx_content = {}
    for sheetname, sheet_meta in sheet_metaes.iteritems():
        sheet = workbook.sheet_by_name(sheetname)
        col_index = {}
        for x in xrange(sheet.ncols):
            colname = sheet.cell(0, x).value
            if colname in sheet_meta.fields:
                col_index[colname] = x

        sheet_content = OrderedDict() if sheet_meta.key else []
        for x in xrange(1, sheet.nrows):
            if x == sheet_meta.descrow - 1:
                continue
            key_value = None
            row_value = {}
            for colname, field in sheet_meta.fields.iteritems():
                col = col_index[colname]
                field_value = refine(field.type, field.element,
                                     _str(sheet.cell(x, col).value))
                row_value[field.name] = field_value
                if field.name == sheet_meta.key:
                    assert field.type != 'array'
                    key_value = field_value
            if type(sheet_content) == list:
                sheet_content.append(row_value)
            else:
                sheet_content[key_value] = row_value

        xlsx_content[sheet_meta.dest] = sheet_content

    return xlsx_content
Example #26
0
        def _getter(obj):
            obj = getter(obj)
            if isinstance(obj, dict):
                obj = [{'key': k, 'value': v} for k, v in obj.items()]

            if isinstance(schema, dict):
                res = OrderedDict()
                _key, _value = list(schema.items())[0]
                for item in obj:
                    if condition and not self.__get_value(
                            condition, obj, item):
                        continue
                    res[self.__get_value(_key, obj, item)] = self.__get_value(
                        _value, obj, item)
                return res
            if isinstance(schema, (tuple, list)):
                res = []
                _value = schema[0]

                for item in obj:
                    if condition and not self.__get_value(
                            condition, obj, item):
                        continue
                    res.append(self.__get_value(_value, obj, item))
                return type(schema)(res)
Example #27
0
    def __to_plain_containers(self,
                              container: Union[CommentedSeq, CommentedMap]
                              ) -> Union[OrderedDict, list]:
        """Converts any sequence or mapping to list or OrderedDict

        Stops at anything that isn't a sequence or a mapping.

        One day, we'll extract the comments and formatting and store
        them out-of-band.

        Args:
            mapping: The mapping of constructed subobjects to edit
        """
        if isinstance(container, CommentedMap):
            new_container = OrderedDict()  # type: Union[OrderedDict, list]
            for key, value_obj in container.items():
                if (isinstance(value_obj, CommentedMap)
                        or isinstance(value_obj, CommentedSeq)):
                    new_container[key] = self.__to_plain_containers(value_obj)
                else:
                    new_container[key] = value_obj

        elif isinstance(container, CommentedSeq):
            new_container = list()
            for value_obj in container:
                if (isinstance(value_obj, CommentedMap)
                        or isinstance(value_obj, CommentedSeq)):
                    new_container.append(self.__to_plain_containers(value_obj))
                else:
                    new_container.append(value_obj)
        return new_container
Example #28
0
def split_data_by_k(data, k, dic):
    if dic:
        data1, data2 = OrderedDict(), OrderedDict()

        for k1, v1 in data.items():
            assert len(v1) >= k
            mid = len(v1) // k

            data1[k1] = v1[mid:]
            data2[k1] = v1[:mid]

        print("{} total images".format(
            sum([len(data[key]) for key in data.keys()])))
        print("{} (1) images".format(
            sum([len(data1[key]) for key in data1.keys()])))
        print("{} (2) images".format(
            sum([len(data2[key]) for key in data2.keys()])))

    else:
        data1, data2 = [], []

        for v1 in data:
            assert len(v1) >= k
            mid = len(v1) / k
            data1.append(v1[mid:])
            data2.append(v1[:mid])

        print("{} total images".format(sum([len(d) for d in data])))
        print("{} (1) images".format(sum([len(d) for d in data1])))
        print("{} (2) images".format(sum([len(d) for d in data2])))

    return data1, data2
Example #29
0
def traverse_and_remove_path(obj, path=None, match="First"):
    if path is None:
        path = []
    if isinstance(obj, dict):
        res = OrderedDict()
        for k, v in obj.items():
            cmatch = False
            if (match and len(path) > 0 and path[0] == k):
                cmatch = True
            res.update(
                {k: traverse_and_remove_path(v, path=path[1:], match=cmatch)})
        if (len(path) == 1 and path[0] in res.keys() and match):
            del res[path[0]]
        return res
    elif isinstance(obj, list):
        res = []
        for i, elem in enumerate(obj):
            cmatch = False
            if (match and len(path) >= 1 and isinstance(path[0], int)
                    and path[0] < len(obj) and i == path[0]):
                cmatch = True
            res.append(
                traverse_and_remove_path(elem, path=path[1:], match=cmatch))
        if (len(path) == 1 and isinstance(path[0], int) and path[0] < len(res)
                and match):
            res.pop(path[0])
        return res
    else:
        return obj  # no container, just values (str, int, float)
Example #30
0
def parse(tokens):
    if tokens[0] == "{":
        ret = OrderedDict()
        tokens = tokens[1:]
        while tokens[0] != "}":
            key = tokens[0]
            tokens = tokens[1:]

            tokens = tokens[1:]  # Skip ':'

            value, tokens = parse(tokens)

            if tokens[0] == ",":
                tokens = tokens[1:]

            ret[key] = value
        tokens = tokens[1:]
        return ret, tokens
    elif tokens[0] == "[":
        ret = []
        tokens = tokens[1:]
        while tokens[0] != "]":
            value, tokens = parse(tokens)
            if tokens[0] == ",":
                tokens = tokens[1:]
            ret.append(value)
        tokens = tokens[1:]
        return ret, tokens
    else:
        return tokens[0], tokens[1:]
Example #31
0
    def check_for_add_to_toolbars(self, plugin):
        from calibre.gui2.preferences.toolbar import ConfigWidget
        from calibre.customize import InterfaceActionBase

        if not isinstance(plugin, InterfaceActionBase):
            return

        all_locations = OrderedDict(ConfigWidget.LOCATIONS)
        plugin_action = plugin.load_actual_plugin(self.gui)
        installed_actions = OrderedDict([
            (key, list(gprefs.get('action-layout-'+key, [])))
            for key in all_locations])

        # If already installed in a GUI container, do nothing
        for action_names in installed_actions.itervalues():
            if plugin_action.name in action_names:
                return

        allowed_locations = [(key, text) for key, text in
                all_locations.iteritems() if key
                not in plugin_action.dont_add_to]
        if not allowed_locations:
            return # This plugin doesn't want to live in the GUI

        from calibre.gui2.dialogs.choose_plugin_toolbars import ChoosePluginToolbarsDialog
        d = ChoosePluginToolbarsDialog(self, plugin_action, allowed_locations)
        if d.exec_() == d.Accepted:
            for key, text in d.selected_locations():
                installed_actions = list(gprefs.get('action-layout-'+key, []))
                installed_actions.append(plugin_action.name)
                gprefs['action-layout-'+key] = tuple(installed_actions)
Example #32
0
    def test_plugins(self):
        "Test that plugins without dependencies work"
        localrc = {'test_localrc': '1'}
        local_conf = {'install':
                      {'nova.conf':
                       {'main':
                        {'test_conf': '2'}}}}
        services = {'cinder': True}
        # We use ordereddict here to make sure the plugins are in the
        # *wrong* order for testing.
        plugins = OrderedDict([
            ('bar', 'https://git.openstack.org/openstack/bar-plugin'),
            ('foo', 'https://git.openstack.org/openstack/foo-plugin'),
            ('baz', 'https://git.openstack.org/openstack/baz-plugin'),
            ])
        p = dict(localrc=localrc,
                 local_conf=local_conf,
                 base_services=[],
                 services=services,
                 plugins=plugins,
                 base_dir='./test',
                 path=os.path.join(self.tmpdir, 'test.local.conf'))
        lc = self._init_localconf(p)
        lc.write(p['path'])

        plugins = []
        with open(p['path']) as f:
            for line in f:
                if line.startswith('enable_plugin'):
                    plugins.append(line.split()[1])
        self.assertEqual(['bar', 'baz', 'foo'], plugins)
Example #33
0
    def test_plugins(self):
        "Test that plugins without dependencies work"
        localrc = {'test_localrc': '1'}
        local_conf = {'install': {'nova.conf': {'main': {'test_conf': '2'}}}}
        services = {'cinder': True}
        # We use ordereddict here to make sure the plugins are in the
        # *wrong* order for testing.
        plugins = OrderedDict([
            ('bar', 'https://git.openstack.org/openstack/bar-plugin'),
            ('foo', 'https://git.openstack.org/openstack/foo-plugin'),
            ('baz', 'https://git.openstack.org/openstack/baz-plugin'),
        ])
        p = dict(localrc=localrc,
                 local_conf=local_conf,
                 base_services=[],
                 services=services,
                 plugins=plugins,
                 base_dir='./test',
                 path=os.path.join(self.tmpdir, 'test.local.conf'))
        lc = self._init_localconf(p)
        lc.write(p['path'])

        plugins = []
        with open(p['path']) as f:
            for line in f:
                if line.startswith('enable_plugin'):
                    plugins.append(line.split()[1])
        self.assertEqual(['bar', 'baz', 'foo'], plugins)
Example #34
0
def csv(input, dialect=None, header=True, key=None, sort=False):
    obj = []
    fields = None
    if dialect:
        dialect = ocsv.get_dialect(dialect)
    else:
        sniffer = ocsv.Sniffer()
        dialect = sniffer.sniff(input.readline())()
        input.seek(0)
    reader = ocsv.reader(input, dialect=dialect)
    if key:
        header = True
    if header:
        fields = next(reader)
        if key and key <= len(fields):
            obj = OrderedDict()
        else:
            key = None
    for row in reader:
        if header:
            row = OrderedDict(zip(fields, row))
            if key:
                rowkey = row.pop(fields[key - 1])
                obj[rowkey] = row
            else:
                obj.append(row)
        else:
            obj.append(row)
    if sort:
        obj = sort_ordereddict(obj)
    return obj
Example #35
0
def issmmumpssolver(*args):
    #ISSMSOLVE - return issm solver options
    #
    #   Usage:
    #      options=issmsolver;

    #retrieve options provided in varargin
    arguments = pairoptions.pairoptions(*args)

    options = OrderedDict()
    options['toolkit'] = 'issm'
    options['mat_type'] = 'mpidense'
    options['vec_type'] = 'mpi'
    options['solver_type'] = 'mumps'

    #now, go through our arguments, and write over default options.
    for i in range(len(arguments.list)):
        arg1 = arguments.list[i][0]
        arg2 = arguments.list[i][1]
        found = 0
        for j in range(len(options)):
            joption = options[j][0]
            if joption == arg1:
                joption[1] = arg2
                options[j] = joption
                found = 1
                break
        if not found:
            #this option did not exist, add it:
            options.append([arg1, arg2])
    return options
Example #36
0
def unescape_json(element):
    """
    Remove escape character '\' in a JSON object.
    :param element: JSON object
    :type element: JSON object
    :return: unescaped JSON object
    :rtype: JSON object
    """
    if isinstance(element, str):
        return unescape_string(element)
    elif isinstance(element, Number):
        return element
    elif isinstance(element, bool):
        return element
    elif element is None:
        return element
    elif isinstance(element, dict):
        new_element = OrderedDict()
        for key, value in element.items():
            new_key = unescape_json(key)
            new_value = unescape_json(value)
            new_element[new_key] = new_value
        return new_element
    elif isinstance(element, list):
        new_element = list()
        for index, item in enumerate(element):
            new_item = unescape_json(item)
            new_element.append(new_item)
        return new_element
    raise TemplateEngineException("Unknown data type {} of {}.".format(
        type(element), element))
Example #37
0
def _get_sorteddict(object, dictwithhash=False):
    if isinstance(object, dict):
        out = OrderedDict()
        for key, val in sorted(object.items()):
            if val:
                out[key] = _get_sorteddict(val, dictwithhash)
    elif isinstance(object, (list, tuple)):
        out = []
        for val in object:
            if val:
                out.append(_get_sorteddict(val, dictwithhash))
        if isinstance(object, tuple):
            out = tuple(out)
    else:
        if isinstance(object, str) and os.path.isfile(object):
            hash = hash_infile(object)
            if dictwithhash:
                out = (object, hash)
            else:
                out = hash
        elif isinstance(object, float):
            out = '%.10f' % object
        else:
            out = object
    return out
Example #38
0
    def _serialize(self):
        """
        Serialize the sheet in a JSON-ready format.
        """
        obj = OrderedDict()

        if 'key' in self._columns and 'value' in self._columns:
            for row in self:
                obj[row['key']] = row['value']
        elif 'key' in self._columns:
            for row in self:
                obj[row['key']] = OrderedDict()

                for column in self._columns:
                    if column == 'key':
                        continue

                    value = row[column]

                    obj[row['key']][column] = value
        else:
            obj = []

            for row in self:
                row_obj = OrderedDict()

                for i, column in enumerate(row):
                    row_obj[self._columns[i]] = column

                obj.append(row_obj)

        return obj
Example #39
0
def _marshall_item(rawitem, template, options):
    if isinstance(template, dict):
        result = OrderedDict()
        for key, value in template.items():
            nkey = _marshall_item(rawitem, key, options)
            nvalue = _marshall_item(rawitem, value, options)
            if (nkey is not None) and (nvalue is not None):
                result[nkey] = nvalue
        return result

    elif isinstance(template, list):
        result = []
        for value in template:
            nvalue = _marshall_item(rawitem, value, options)
            if nvalue is not None:
                result.append(nvalue)
        return result

    elif type(template) in (str, unicode):
        if _is_field(template):
            name, ftype = _get_field_comps(template)
            if name in rawitem:
                value = rawitem[name]
            else:
                return None
            if options['ignore_empty_item'] and len(value.strip()) == 0:
                return None
            value = _convert_to(ftype, value, options)
            return value
        elif options['ignore_empty_item'] and len(template.strip()) == 0:
                return None

    return template
Example #40
0
    def _parse(self, data, base_url):
        """
        Given some parsed JSON data, returns the corresponding DocJSON objects.
        """
        if isinstance(data, OrderedDict) and data.get('_type') == 'link':
            # Link objects
            url = data.get('url')
            method = data.get('method')
            fields = data.get('fields')
            try:
                return Link(url, method, fields, _base_url=base_url, _transport=self)
            except (TypeError, ValueError) as exc:
                raise ParseError(str(exc))

        elif isinstance(data, OrderedDict):
            # Any unknown types should be ignored and treaded as a regular object.
            data.pop('_type', None)

            # Parse all the items in the dict and wrap them in a `Object`.
            parsed = OrderedDict()
            for key, value in data.items():
                parsed[key] = self._parse(value, base_url)
            return Object(parsed)

        elif isinstance(data, list):
            # Parse all the items in the list and wrap them in a `List`.
            parsed = []
            for item in data:
                value = self._parse(item, base_url)
                # Ignore 'Link' objects contained in a list.
                if not isinstance(value, Link):
                    parsed.append(value)
            return List(parsed)

        return data
Example #41
0
def ordUnUnicode(value):
    """ this is completely a python2 thing and should be able to just
    return value in python 3"""

    def chkNum(rawval):
        val = str(rawval)
        if val.isdigit():
            val = int(val)
        else:
            try:
                val = float(val)
            except ValueError:
                val = str(val)
        return val

    #if isinstance(value, OrderedDict)or isinstance(value, dict):
    if type(value) in (OrderedDict, dict):
        newobj = OrderedDict()
        for key, value in value.items():
            #print ('doing dict',key)
            key = ordUnUnicode(key)
            newobj[key] = ordUnUnicode(value)
    elif isinstance(value, list):
        newobj = []
        for i in value:
            #print ('in i loop',i,' ck:',ordUnUnicode(i), ' val:', value,)
            newobj.append(ordUnUnicode(i))
        #print ('newobh now', newobj)
    elif isinstance(value, unicode):
        newobj = str(value)  # chkNum(value)
    else:
        newobj = value

    return newobj
Example #42
0
def construct(obj):
    """Used for preventing of not expected changing of class attributes"""
    if isinstance(obj, OrderedDict):
        new_obj = OrderedDict()
        for key, value in obj.items():
            new_obj[key] = construct(value)
    elif not isinstance(obj, OrderedDict) and isinstance(obj, dict):
        new_obj = dict()
        for key, value in obj.items():
            new_obj[key] = construct(value)
    elif isinstance(obj, list):
        new_obj = list()
        for value in obj:
            new_obj.append(construct(value))
    elif isinstance(obj, tuple):
        base = list()
        for value in obj:
            base.append(construct(value))
        new_obj = tuple(base)
    elif isinstance(obj, str):
        new_obj = str(obj)
    elif isinstance(obj,
                    (int, float, complex, type(None))) or inspect.isclass(obj):
        new_obj = obj
    else:
        raise TypeError(
            "Object of unsupported type was passed to construct function: %s" %
            type(obj))
    return new_obj
Example #43
0
def apply_dict_depth_first(nodes, func, as_dict=True, parents=None):
    '''
    This function is similar to the `apply_depth_first` except that it operates
    on the `OrderedDict`-based structure returned from `apply_depth_first` when
    `as_dict=True`.

    Note that if `as_dict` is `False`, the result of this function is given in
    the entry/tuple form.
    '''
    if as_dict:
        items = OrderedDict()
    else:
        items = []

    if parents is None:
        parents = []

    for i, (k, node) in enumerate(nodes.iteritems()):
        item = func(k, node, parents)
        item_parents = parents + [(k, node)]
        if node.children is not None:
            children = apply_dict_depth_first(node.children, func,
                                              as_dict=as_dict,
                                              parents=item_parents)
        else:
            children = None
        if as_dict:
            items[k] = Node(item, children)
        elif children:
            items.append((item, children))
        else:
            items.append(item)
    return items
Example #44
0
def to_json_stat(input_df, value="value", output="list"):
    """Encode pandas.DataFrame object into JSON-stat format. The DataFrames
       must have exactly one value column.

    Args:
      df(pandas.DataFrame): pandas data frame (or list of data frames) to
      encode.
       value (string, optional): name of the value column. Defaults to 'value'.
      output(string): accepts two values: 'list' or 'dict'. Produce list of\
                      dicts or dict of dicts as output.

    Returns:
      output(string): String with JSON-stat object.

    """

    data = []
    if output == "list":
        result = []
    elif output == "dict":
        result = OrderedDict({})
    if isinstance(input_df, pd.DataFrame):
        data.append(input_df)
    else:
        data = input_df
    for row, dataframe in enumerate(data):
        pd.notnull(dataframe[value])
        dims = data[row].filter([item for item in data[row].columns.values if item not in value])
        if len(dims.columns.values) != len(set(dims.columns.values)):
            raise ValueError("Non-value columns must constitute a unique ID")
        dim_names = list(dims)
        categories = [
            {
                to_int(i): {
                    "label": to_str(i),
                    "category": {
                        "index": OrderedDict([(to_str(j), to_int(k)) for k, j in enumerate(uniquify(dims[i]))]),
                        "label": OrderedDict([(to_str(j), to_str(j)) for k, j in enumerate(uniquify(dims[i]))]),
                    },
                }
            }
            for i in dims.columns.values
        ]
        dataset = {"dataset" + str(row + 1): {"dimension": OrderedDict(), value: [x for x in dataframe[value].values]}}
        for category in categories:
            dataset["dataset" + str(row + 1)]["dimension"].update(category)
        dataset["dataset" + str(row + 1)]["dimension"].update({"id": dim_names})
        dataset["dataset" + str(row + 1)]["dimension"].update(
            {"size": [len(dims[i].unique()) for i in dims.columns.values]}
        )
        for category in categories:
            dataset["dataset" + str(row + 1)]["dimension"].update(category)
        if output == "list":
            result.append(dataset)
        elif output == "dict":
            result.update(dataset)
        else:
            result = None
    return json.dumps(result)
Example #45
0
	def __init__(self, data = None, items = None, post = None, **args):

		super().__init__(data, items, post, **args)

		config = self.config
		config_data = config['data']
		field_config = config_data.get('field_config', {})
		
		set = self['configset']
		
		
		options = OrderedDict()
		
		options['autocomplete'] = s('Auto complete')
		options['select'] = s('Select list')
		options['checkboxes'] = s('Check boxes')
		options['radioboxes'] = s('Radio boxes')
		
		# max limit
		set.add('HTMLSelect', {
			'id' : 'field_selection_type',
			'value' : post.get('field_selection_type', None) or field_config.get('field_selection_type', None),
			'title' : s('Field selection type'),
			'options' : options,
			'required' : True,
			'css' : ['i-form-large'],
			'multiple' : False,
			'weight' : 1,
		})
		
		field_value_options = post.get('field_value_options', None)
		if field_value_options is None:
			keys = field_config.get('field_value_option_keys', [])
			values = field_config.get('field_value_option_values', {})
			options = []
			for key in keys:
				if key in values:
					options.append(':'.join((str(key), values[key])))
			field_value_options = '\n'.join(options)
		
		set.add('TextArea', {
			'id' : 'field_value_options',
			'value' : field_value_options,
			'title' : s('Field allowed values'),
			'required' : True,
			'css' : ['i-form-large i-width-1-1'],
			'multiple' : False,
			'weight' : 2,
			'info' : s('''Colon separated key:value pair of allowed values, one per line. Key should be a number.
example:<br>
0:Female<br>
1:Male<br>
2:Shemale<br>
'''),
			'validation_rule' : ['Not', [['Empty']], 'Values field is required']
		})
Example #46
0
 def deal_fields(self):
     l = OrderedDict()
     if self.is_responsible:
         deals = Deal.objects.filter(partnership__responsible=self).all()
     else:
         deals = Deal.objects.filter(partnership=self).all()
     l = list()
     for deal in deals:
         l.append(deal.fields)
     return l
    def getLastRecordsByThreshold(self, minTimestamp = None, maxTimestamp = None, orderAsc = True, asDict = False, noOfRecords = None):
        if asDict:
            result = OrderedDict()
        else:
            result = []

        if(self.database.isOpen()):
            cursor = None

            order = "ASC"
            if not orderAsc:
                order = "DESC"

            limit = ""
            if noOfRecords:
                limit = " LIMIT %d" % noOfRecords

            if minTimestamp is None and maxTimestamp is None:
                cursor = self.database.execute("SELECT MAX(forecastID), * FROM mixerData GROUP BY timestamp ORDER BY timestamp " + order + limit)
            elif minTimestamp is None:
                cursor = self.database.execute("SELECT MAX(forecastID), * FROM mixerData WHERE timestamp<=? GROUP BY timestamp ORDER BY timestamp " + order + limit, (maxTimestamp, ))
            elif maxTimestamp is None:
                cursor = self.database.execute("SELECT MAX(forecastID), * FROM mixerData WHERE ?<=timestamp GROUP BY timestamp ORDER BY timestamp " + order + limit, (minTimestamp, ))
            else:
                cursor = self.database.execute("SELECT MAX(forecastID), * FROM mixerData WHERE ?<=timestamp AND timestamp<=? GROUP BY timestamp ORDER BY timestamp " + order + limit,
                                    (minTimestamp, maxTimestamp, ))

            for row in cursor:
                mixerData = RMMixerData(row[3])
                mixerData.temperature = row[4]
                mixerData.rh = row[5]
                mixerData.wind = row[6]
                mixerData.solarRad = row[7]
                mixerData.skyCover = row[8]
                mixerData.rain = row[9]
                mixerData.et0 = row[10]
                mixerData.pop = row[11]
                mixerData.qpf = row[12]
                mixerData.condition = row[13]
                mixerData.pressure = row[14]
                mixerData.dewPoint = row[15]
                mixerData.minTemp = row[16]
                mixerData.maxTemp = row[17]
                mixerData.minRH = row[18]
                mixerData.maxRH = row[19]
                mixerData.et0calc = row[20]
                mixerData.et0final = row[21]

                if asDict:
                    result[row[3]] = mixerData
                else:
                    result.append(mixerData)

        return result
Example #48
0
    def _handle_one_attr_one_env(self, obj, key, mc_attr, env, attributes_overriding_property, dir_entries, names_only):
        attr_inf = []
        try:
            val = mc_attr.env_values[env]
            if key in dir_entries:
                attributes_overriding_property.add(key)
                attr_inf = [(' #overrides @property', True)]
        except KeyError as ex:
            # mc_attribute overriding @property OR the value for env has not yet been set
            try:
                val = obj.getattr(key, env)
                attr_inf = [(' #value for {env} provided by @property'.format(env=env), True)]
            except AttributeError:
                val = McInvalidValue.MC_NO_VALUE

        if self.user_filter_callable:
            try:
                key, val = self.user_filter_callable(obj, key, val)
                if key is False:
                    return key, []
            except Exception as ex:
                self.num_errors += 1
                traceback.print_exception(*sys.exc_info())
                attr_inf.append((' #json_error calling filter', repr(ex)),)

        val = self._check_nesting(obj, val)
        if val == McInvalidValue.MC_NO_VALUE:
            return key, [(' #no value for {env}'.format(env=env), True)]

        if isinstance(val, Mapping):
            new_val = OrderedDict()
            for inner_key, maybeitem in val.items():
                if not isinstance(maybeitem, self.multiconf_base_type):
                    new_val[str(inner_key)] = maybeitem
                    continue
                new_val[inner_key] = self._ref_mc_item_str(maybeitem)
            return key, [('', new_val)] + attr_inf

        try:
            iterable = iter(val)
            # TODO?: Include type of iterable in json meta info
        except TypeError:
            return key, [('', val)] + attr_inf

        if isinstance(val, str):
            return key, [('', val)] + attr_inf

        new_val = []
        for maybeitem in val:
            if not isinstance(maybeitem, self.multiconf_base_type):
                new_val.append(maybeitem)
                continue
            new_val.append(self._ref_mc_item_str(maybeitem))
        return key, [('', new_val)] + attr_inf
Example #49
0
def get_categories(url):
    soup = bs(urlopen(url))
    cats = OrderedDict() 
    for opt in soup.findAll('option')[1:]:
	c = opt.text.split('&')[0].lower().replace(' ', '-')
	cat_url = url + 'category/' + c + '/'
	cats.append(c)
	"""
	if isValid(cat_url):
	    cats.append(c)
	"""
    return cats
Example #50
0
def parseADL(lines):
    objects = []
    parent_object = [objects]
    while lines:
        line = lines.pop().strip()

        # opening
        m = re.match('^(.*) {', line)
        if m:
            name = m.groups()[0]
            if name in ['children', 'colors', 'points']:
                current = []
            else:
                current = OrderedDict()
                current['object_type'] = name

            if isinstance(parent_object[-1], list):
                parent_object[-1].append(current)
            else:
                parent_object[-1][name] = current
            parent_object.append(current)

        # attr = value
        m = re.match('^(.*?)=(.*)', line)
        if m:
            attr = m.groups()[0]
            value= m.groups()[1]
            current[attr] = value

        # color,
        m = re.match('([0-9a-fA-F]{6}),', line)
        if m:
            color = m.groups()[0]
            current.append(color)
        # (x,y) point pair
        m = re.match('\((-?[0-9]+),(-?[0-9]+)\)', line)
        if m:
            x = m.groups()[0]
            y = m.groups()[1]
            current.append((int(x),int(y)))
        # closing
        if line == '}':
            parent_object.pop()
            current = parent_object[-1]
    # restructure to adl dictionary
    adl = {'objects':[]}
    for o in objects:
        type_name = o['object_type']
        if type_name in ['file', '"color map"', 'display']:
            adl[type_name] = o
        else:
            adl['objects'].append(o)
    return adl
Example #51
0
def _insert_parent_items(parent, items):
    if isinstance(parent, dict):
        result = OrderedDict()
        for key, value in parent.iteritems():
            result[key] = _insert_parent_items(value, items)
        return result
    elif isinstance(parent, list):
        result = OrderedDict()
        for item in parent:
            result.append(_insert_parent_items(parent, items))
        return result
    elif parent == "${template}":
        return items
Example #52
0
    def test_plugin_deps(self):
        "Test that plugins with dependencies work"
        os.makedirs(os.path.join(self.tmpdir, 'foo-plugin', 'devstack'))
        os.makedirs(os.path.join(self.tmpdir, 'foo-plugin', '.git'))
        os.makedirs(os.path.join(self.tmpdir, 'bar-plugin', 'devstack'))
        os.makedirs(os.path.join(self.tmpdir, 'bar-plugin', '.git'))
        with open(os.path.join(
                self.tmpdir,
                'foo-plugin', 'devstack', 'settings'), 'w') as f:
            f.write('define_plugin foo-plugin\n')
        with open(os.path.join(
                self.tmpdir,
                'bar-plugin', 'devstack', 'settings'), 'w') as f:
            f.write('define_plugin bar-plugin\n')
            f.write('plugin_requires bar-plugin foo-plugin\n')

        localrc = {'test_localrc': '1'}
        local_conf = {'install':
                      {'nova.conf':
                       {'main':
                        {'test_conf': '2'}}}}
        services = {'cinder': True}
        # We use ordereddict here to make sure the plugins are in the
        # *wrong* order for testing.
        plugins = OrderedDict([
            ('bar-plugin', 'git://git.openstack.org/openstack/bar-plugin'),
            ('foo-plugin', 'git://git.openstack.org/openstack/foo-plugin'),
            ])
        p = dict(localrc=localrc,
                 local_conf=local_conf,
                 base_services=[],
                 services=services,
                 plugins=plugins,
                 base_dir=self.tmpdir,
                 path=os.path.join(self.tmpdir, 'test.local.conf'))
        lc = LocalConf(p.get('localrc'),
                       p.get('local_conf'),
                       p.get('base_services'),
                       p.get('services'),
                       p.get('plugins'),
                       p.get('base_dir'),
                       p.get('projects'),
                       p.get('project'))
        lc.write(p['path'])

        plugins = []
        with open(p['path']) as f:
            for line in f:
                if line.startswith('enable_plugin'):
                    plugins.append(line.split()[1])
        self.assertEqual(['foo-plugin', 'bar-plugin'], plugins)
Example #53
0
def _sort_recurse(item):
    if isinstance(item, str):
        result = item
    elif isinstance(item, Mapping):
        result = OrderedDict()
        for key in sorted(item.keys()):
            result[key] = _sort_recurse(item[key])
    elif isinstance(item, Sequence):
        result = list()
        for element in item:
            result.append(_sort_recurse(element))
    else:
        result = item
    return result
Example #54
0
def _parse_sig(sig, arg_names, validate=False):
    """
    Parses signatures into a ``OrderedDict`` of paramName => type.
    Numerically-indexed arguments that do not correspond to an argument
    name in python (ie: it takes a variable number of arguments) will be
    keyed as the stringified version of it's index.

      sig         the signature to be parsed
      arg_names   a list of argument names extracted from python source

    Returns a tuple of (method name, types dict, return type)
    """
    d = SIG_RE.match(sig)
    if not d:
        raise ValueError('Invalid method signature %s' % sig)
    d = d.groupdict()
    ret = [(n, Any) for n in arg_names]
    if 'args_sig' in d and type(
            d['args_sig']) is str and d['args_sig'].strip():
        for i, arg in enumerate(d['args_sig'].strip().split(',')):
            _type_checking_available(sig, validate)
            if '=' in arg:
                if not type(ret) is OrderedDict:
                    ret = OrderedDict(ret)
                dk = KWARG_RE.match(arg)
                if not dk:
                    raise ValueError('Could not parse arg type %s in %s' %
                                     (arg, sig))
                dk = dk.groupdict()
                if not sum(
                    [(k in dk and type(dk[k]) is str and bool(dk[k].strip()))
                         for k in ('arg_name', 'arg_type')]):
                    raise ValueError('Invalid kwarg value %s in %s' %
                                     (arg, sig))
                ret[dk['arg_name']] = _eval_arg_type(dk['arg_type'], None, arg,
                                                     sig)
            else:
                if type(ret) is OrderedDict:
                    raise ValueError('Positional arguments must occur '
                                     'before keyword arguments in %s' % sig)
                if len(ret) < i + 1:
                    ret.append((str(i), _eval_arg_type(arg, None, arg, sig)))
                else:
                    ret[i] = (ret[i][0], _eval_arg_type(arg, None, arg, sig))
    if not type(ret) is OrderedDict:
        ret = OrderedDict(ret)
    return (d['method_name'], ret,
            (_eval_arg_type(d['return_sig'], Any, 'return', sig) if
             d['return_sig'] else Any))
Example #55
0
	def getProducerInterval(self, dataType, aggregationType = None):
		if aggregationType == None:
			print(dataType.subtrees["avg"].subtrees['producer_interval'].value)
			if not self._root.subtrees['data'].subtrees[dataType].subtrees.get(aggregationType) == None:
				return self._root.subtrees['data'].subtrees[dataType].subtrees[aggregationType].value
			else:
				return False
		else:
			result = OrderedDict()
			for i in range(len(self._root.subtrees['data'].subtrees[dataType].subtrees)):
				string = self._root.subtrees['data'].subtrees[dataType].subtrees.items()[i][0]
				print(string)
				result.append(string)
				# =  self._root.subtrees['data'].subtrees[dataType].subtrees[string].value
			return result
Example #56
0
 def ununicodify(obj):
     result = None
     if isinstance(obj, OrderedDict):
         result = OrderedDict()
         for k, v in obj.iteritems():
             k1 = str(k) if isinstance(k, unicode) else k
             result[k1] = ununicodify(v)
     elif isinstance(obj, list):
         result = []
         for v in obj:
             result.append(ununicodify(v))
     elif isinstance(obj, unicode):
         result = str(obj)
     else:
         result = obj
     return result
    def getAllRecords(self, asDict = True):
        if self.database.isOpen():
            if asDict:
                records = OrderedDict()
            else:
                records = []

            rows = self.database.execute("SELECT token, expiration FROM authTokens")
            for row in rows:
                if asDict:
                    records[str(row[0])] = RMAuthToken(str(row[0]), row[1])
                else:
                    records.append(RMAuthToken(str(row[0]), row[1]))

            return records

        return None