def traverse_and_remove_path(obj, path=None, match="First"): if path is None: path = [] if isinstance(obj, dict): res = OrderedDict() for k, v in obj.items(): cmatch = False if(match and len(path) > 0 and path[0] == k): cmatch = True res.update({k: traverse_and_remove_path(v, path=path[1:], match=cmatch)}) if(len(path) == 1 and path[0] in res.keys() and match): del res[path[0]] return res elif isinstance(obj, list): res = [] for i, elem in enumerate(obj): cmatch = False if(match and len(path) >= 1 and isinstance(path[0], int) and path[0] < len(obj) and i == path[0]): cmatch = True res.append(traverse_and_remove_path(elem, path=path[1:], match=cmatch)) if(len(path) == 1 and isinstance(path[0], int) and path[0] < len(res) and match): res.pop(path[0]) return res else: return obj # no container, just values (str, int, float)
def get_data(task, key, separate_cols=True): data_dir = results_dir if separate_cols: data = OrderedDict() else: data = [] dirs = [d for d in sorted(os.listdir(data_dir)) if '.sim' in d] sims = [os.path.basename(d).split('.')[0] for d in dirs] for dir_, sim in zip(dirs, sims): if separate_cols: data[sim] = [] paths = glob(os.path.join(data_dir, dir_, "test_%s*.txt" % task)) for path in paths: with open(path, 'r') as fp: try: if separate_cols: data[sim].append(json.load(fp)[key]) else: data.append((sim, json.load(fp)[key])) except: print(path) raise if separate_cols: return pd.DataFrame(data) else: return pd.DataFrame(data, columns=['Backend', key])
def get_hidden_layers(cls, pins, phidden={ "dim": 800, "nlayers": 2, "batch_norm": False }, nn_lin="ReLU", name=""): '''outputs the hidden module of the layer. :param input_dim: dimension of the input :type input_dim: int :param phidden: parameters of hidden layers :type phidden: dict :param nn_lin: non-linearity name :type nn_lin: str :param name: name of module :type name: str :returns: nn.Sequential''' # Hidden layers input_dim = 0 for p in pins: new_input = p['dim'] if issubclass(type(p), dict) else p input_dim += new_input residual = phidden.get("residual", False) LayerModule = MLPResidualLayer if residual else MLPLayer if not issubclass(type(phidden), list): modules = OrderedDict() nlayers = phidden.get('nlayers', 1) hidden_dim = phidden.get('dim', 800) for i in range(nlayers): n_in = int(input_dim) if i == 0 else int(hidden_dim) modules['layer_%d' % i] = LayerModule( n_in, int(hidden_dim), nn_lin=nn_lin, batch_norm=phidden.get('batch_norm', False), dropout=phidden.get('dropout'), name_suffix="_%d" % i) return nn.Sequential(modules) else: modules = nn.ModuleList() for i, ph in enumerate(phidden): mod = OrderedDict() nlayers = ph.get('nlayers', 1) hidden_dim = phidden.get('dim', 800) for i in range(ph['nlayers']): n_in = int(input_dim) if i == 0 else int(hidden_dim) mod['layer_%d' % i] = LayerModule( n_in, int(hidden_dim), nn_lin=nn_lin, batch_norm=ph.get('batch_norm', False), dropout=ph.get('dropout', None), name_suffix="_%d" % i) modules.append(nn.Sequential(mod)) return modules
def check_for_add_to_toolbars(self, plugin): from calibre.gui2.preferences.toolbar import ConfigWidget from calibre.customize import InterfaceActionBase if not isinstance(plugin, InterfaceActionBase): return all_locations = OrderedDict(ConfigWidget.LOCATIONS) plugin_action = plugin.load_actual_plugin(self.gui) installed_actions = OrderedDict([ (key, list(gprefs.get('action-layout-' + key, []))) for key in all_locations ]) # If already installed in a GUI container, do nothing for action_names in installed_actions.itervalues(): if plugin_action.name in action_names: return allowed_locations = [(key, text) for key, text in all_locations.iteritems() if key not in plugin_action.dont_add_to] if not allowed_locations: return # This plugin doesn't want to live in the GUI from calibre.gui2.dialogs.choose_plugin_toolbars import ChoosePluginToolbarsDialog d = ChoosePluginToolbarsDialog(self, plugin_action, allowed_locations) if d.exec_() == d.Accepted: for key, text in d.selected_locations(): installed_actions = list(gprefs.get('action-layout-' + key, [])) installed_actions.append(plugin_action.name) gprefs['action-layout-' + key] = tuple(installed_actions)
def export(self, sheet, errorfmt=None, warningfmt=None): data = OrderedDict() visit_dates = [i.data.get('date') for i in self.patient.entries if i.form.slug == 'nurtureckd'] within_range = False for result in self.results: datestr = result.date.strftime(DATETIMEFMT) key = (result.patient_id, result.source_group.name, result.source_type, datestr) if key not in data: data[key] = {} data[key][result.observation.name] = result.value_label_or_value for visit in visit_dates: visit_date = datetime.datetime.strptime(visit, '%Y-%m-%d') if not within_range and in_date_range(visit_date, result.date.replace(tzinfo=None)): within_range = True if not within_range: msg = 'NO RESULTS WITHIN {} DAYS OF VISIT'.format(DAYS_RESULTS_SHOULD_BE_WITHIN) sheet.write_row([self.patient.id, msg], errorfmt) for key, results in data.items(): data = list(key) for test in self.observations: data.append(results.get(test, None)) sheet.write_row(data)
def get_replacement_dict(rules): """Given a string with replacement rules, produces a dict of from: to""" if isinstance(rules, (binary_type, text_type)): rules = [rules] pairs = OrderedDict() for rule in rules: if isinstance(rule, (list, tuple)): if len(rule) == 2: pairs.append(rule) else: raise ValueError("Got a rule %s which is not a string or a pair of values (from, to)" % repr(rule)) if len(rule) <= 2: raise ValueError("") rule_split = rule[1:].split(rule[0]) if len(rule_split) != 2: raise ValueError( "Rename string must be of format '/pat1/replacement', " "where / is an arbitrary character to decide replacement. " "Got %s when trying to separate %s" % (rule_split, rule) ) pairs[rule_split[0]] = rule_split[1] return pairs
def __call__(self, im, im_info=None, label=None): """ Args: im (np.ndarray): 图像np.ndarray数据。 im_info (list): 存储图像reisze或padding前的shape信息,如 [('resize', [200, 300]), ('padding', [400, 600])]表示 图像在过resize前shape为(200, 300), 过padding前shape为 (400, 600) label (np.ndarray): 标注图像np.ndarray数据。 Returns: tuple: 当label为空时,返回的tuple为(im, im_info),分别对应图像np.ndarray数据、存储与图像相关信息的字典; 当label不为空时,返回的tuple为(im, im_info, label),分别对应图像np.ndarray数据、 存储与图像相关信息的字典和标注图像np.ndarray数据。 其中,im_info新增字段为: -shape_before_padding (tuple): 保存padding之前图像的形状(h, w)。 Raises: ValueError: 输入图像im或label的形状大于目标值 """ if im_info is None: im_info = OrderedDict() im_info.append(('padding', im.shape[:2])) im_height, im_width = im.shape[0], im.shape[1] if isinstance(self.target_size, int): target_height = self.target_size target_width = self.target_size else: target_height = self.target_size[1] target_width = self.target_size[0] pad_height = target_height - im_height pad_width = target_width - im_width pad_height = max(pad_height, 0) pad_width = max(pad_width, 0) if (pad_height > 0 or pad_width > 0): im_channel = im.shape[2] import copy orig_im = copy.deepcopy(im) im = np.zeros((im_height + pad_height, im_width + pad_width, im_channel)).astype(orig_im.dtype) for i in range(im_channel): im[:, :, i] = np.pad(orig_im[:, :, i], pad_width=((0, pad_height), (0, pad_width)), mode='constant', constant_values=(self.im_padding_value[i], self.im_padding_value[i])) if label is not None: label = np.pad(label, pad_width=((0, pad_height), (0, pad_width)), mode='constant', constant_values=(self.label_padding_value, self.label_padding_value)) if label is None: return (im, im_info) else: return (im, im_info, label)
def parse(tokens): if tokens[0] == '{': ret = OrderedDict() tokens = tokens[1:] while tokens[0] != '}': key = tokens[0] tokens = tokens[1:] tokens = tokens[1:] # Skip ':' value, tokens = parse(tokens) if tokens[0] == ',': tokens = tokens[1:] ret[key] = value tokens = tokens[1:] return ret, tokens elif tokens[0] == '[': ret = [] tokens = tokens[1:] while tokens[0] != ']': value, tokens = parse(tokens) if tokens[0] == ',': tokens = tokens[1:] ret.append(value) tokens = tokens[1:] return ret, tokens else: return tokens[0], tokens[1:]
def recurse(pair, out=None, indent=0): if isinstance(pair, dict): (k, v) = pair.popitem() if out is None: if isinstance(k, int) or k == '': out = [] else: out = OrderedDict() if isinstance(out, list) and (k == ''): out.append(recurse(v, indent=indent + 1)) elif isinstance(out, list) and (k != ''): out = dict(enumerate(out)) out[k] = recurse(v, out.get(k), indent=indent + 1) elif isinstance(out, dict) and k == '': numeric_indices = [ k for k in out.keys() if isinstance(k, int) ] out[1 + max(numeric_indices ) if numeric_indices else 0] = recurse( v, out.get(k)) elif isinstance(out, dict): out[k] = recurse(v, out.get(k), indent=indent + 1) else: return recurse({k: v}, indent=indent + 1) return out return pair
def read_node(node): """Recursively read the given YAML piece, returning an appropriate collection.""" line = node.start_mark.line + 1 if isinstance(node, nodes.ScalarNode): tag = node.tag.split(":")[-1] if tag in ["int", "float"]: constructor = eval(tag) value = constructor(node.value) elif tag == "str": value = node.value else: raise RuntimeError( "cannot parse this scalar at line {}".format(line)) return (value, line) if isinstance(node, nodes.MappingNode): col = OrderedDict() for node_name, node_value in node.value: name = read_node(node_name)[0] value = read_node(node_value) col[name] = value col["--begin"] = line return col if isinstance(node, nodes.SequenceNode): col = [] for node_value in node.value: value = read_node(node_value) col.append(value) return col raise RuntimeError("cannot parse the node at line {}".format(line))
def decode(message): ''' Decodes bencoded text, recursively parses lists and dictionaries ''' ''' All dictionaries are represented as OrderedDicts, lists as Python lists, strings as strings, ints as ints ''' ''' Returns 'highest' parsed unit and the remaining bencoded text ''' current = message[0] if current is 'd': token = OrderedDict() i = 1 while message[i] is not 'e': key, rest = _decode_next_string(message[i : ]) value, rest = decode(rest) token[key] = value message = rest i = 0 return token, rest[1 : ] # skip the dict's 'e' elif current is 'l': token = [] i = 1 while message[i] is not 'e': item, rest = decode(message[i : ]) token.append(item) message = rest i = 0 return token, rest[1 : ] # skip the list's 'e' else: token, rest = _decode_next_unit(message) return token, rest
def elementToValue(field, element, default=_marker): """Read the contents of an element that is assumed to represent a value allowable by the given field. If converter is given, it should be an IToUnicode instance. If not, the field will be adapted to this interface to obtain a converter. """ value = default if IDict.providedBy(field): key_converter = IFromUnicode(field.key_type) value = OrderedDict() for child in element.iterchildren(tag=etree.Element): if noNS(child.tag.lower()) != 'element': continue parseinfo.stack.append(child) key_text = child.attrib.get('key', None) if key_text is None: k = None else: k = key_converter.fromUnicode(unicode(key_text)) value[k] = elementToValue(field.value_type, child) parseinfo.stack.pop() value = fieldTypecast(field, value) elif ICollection.providedBy(field): value = [] for child in element.iterchildren(tag=etree.Element): if noNS(child.tag.lower()) != 'element': continue parseinfo.stack.append(child) v = elementToValue(field.value_type, child) value.append(v) parseinfo.stack.pop() value = fieldTypecast(field, value) # Unicode else: text = element.text if text is None: value = field.missing_value else: converter = IFromUnicode(field) value = converter.fromUnicode(unicode(text)) # handle i18n if isinstance(value, unicode) and parseinfo.i18n_domain is not None: translate_attr = ns('translate', I18N_NAMESPACE) domain_attr = ns('domain', I18N_NAMESPACE) msgid = element.attrib.get(translate_attr) domain = element.attrib.get(domain_attr, parseinfo.i18n_domain) if msgid: value = Message(msgid, domain=domain, default=value) elif translate_attr in element.attrib: value = Message(value, domain=domain) return value
def __call__(self, im, im_info=None, label=None): """ Args: im (np.ndarray): 图像np.ndarray数据。 im_info (list): 存储图像reisze或padding前的shape信息,如 [('resize', [200, 300]), ('padding', [400, 600])]表示 图像在过resize前shape为(200, 300), 过padding前shape为 (400, 600) label (np.ndarray): 标注图像np.ndarray数据。 Returns: tuple: 当label为空时,返回的tuple为(im, im_info),分别对应图像np.ndarray数据、存储与图像相关信息的字典; 当label不为空时,返回的tuple为(im, im_info, label),分别对应图像np.ndarray数据、 存储与图像相关信息的字典和标注图像np.ndarray数据。 其中,im_info新增字段为: -shape_before_resize (tuple): 保存resize之前图像的形状(h, w)。 """ if im_info is None: im_info = OrderedDict() im_info.append(('resize', im.shape[:2])) im = resize_long(im, self.long_size) if label is not None: label = resize_long(label, self.long_size, cv2.INTER_NEAREST) if label is None: return (im, im_info) else: return (im, im_info, label)
def more_heuristic_attempts(blocksizes): # Ramp up to higher block sizes handle = OrderedDict([(i, options['at_blocksize'][-1]) for i in blocksizes[0]]) for i in range(3): new_bs = OrderedDict([(k, v * 2) for k, v in handle.items()]) blocksizes.insert(blocksizes.index(handle) + 1, new_bs) handle = new_bs handle = [] # Extended shuffling for the smaller block sizes for bs in blocksizes[:4]: for i in blocksizes: handle.append( OrderedDict(list(bs.items())[:-1] + [list(i.items())[-1]])) # Some more shuffling for all block sizes for bs in list(blocksizes): ncombs = len(bs) for i in range(ncombs): for j in combinations(bs, i + 1): item = [(k, bs[k] * 2 if k in j else v) for k, v in bs.items()] handle.append(OrderedDict(item)) unique = [] for i in blocksizes + handle: if i not in unique: unique.append(i) return unique
def _serialize(self): """ Serialize the sheet in a JSON-ready format. """ obj = OrderedDict() if 'key' in self._columns and 'value' in self._columns: for row in self: obj[row['key']] = row['value'] elif 'key' in self._columns: for row in self: obj[row['key']] = OrderedDict() for column in self._columns: if column == 'key': continue value = row[column] obj[row['key']][column] = value else: obj = [] for row in self: row_obj = OrderedDict() for i, column in enumerate(row): row_obj[self._columns[i]] = column obj.append(row_obj) return obj
def insert(self,intervals,newinterval,merge=1): ''' O(logn) time implementation of insert method ans and intervals are of type OrderedDict a in ans, i in intervals is of form {start time:(end time, index} new interval is given via a tuple (start time, end time, index) if merge=1, the overlapping intervals will be printed out and then merge. ''' # suppose OrderedDict is implemented using double linked list ans = OrderedDict() # find left bound keys = list(intervals.keys()) left_key=keys[bisect.bisect_left(keys,newinterval[0])] right_key=keys[bisect.bisect_left(keys,newinterval[1])] ...... for i in range(len(intervals)): if newInterval[0] < intervals[i][0]: index = i break intervals.insert(index, newInterval) # this insert is built_in method for list # then repeat the same prodecure as Interval.merge(self, intervals) for interval in intervals: if not ans or interval[0] > ans[-1][1]: ans.append(interval) else: if(merge==0): print(interval, ans[-1]) continue elif(merge==1): ans[-1][1] = max(ans[-1][1], interval[1]) return ans
def followup_targets(events, fields, area): """A generator of event/astroplan target pairs given events and fields All ra and dec values should be in decimal degrees. Args: - events :: a pandas.DataFrame with event coordinates in ra and decl columns - fields :: a pandas.DataFrame with field coordinates in fieldRA and fieldDec columns - area :: the search area in square degrees Returns: a succession of event/astroplan.FixedTarget pairs """ event_fields = find_event_fields(events, fields, area) targets = OrderedDict() # there should be a faster (vectorized) way to do this. for idx, event in events.iterrows(): targets = [] for field_idx, field in event_fields.query( f'event_id=={event.event_id}').iterrows(): field_coords = SkyCoord(ra=field.fieldRA * u.deg, dec=field.fieldDec * u.deg) field_name = "%d" % field.fieldID target = astroplan.FixedTarget(coord=field_coords, name=field_name) targets.append(target) yield event, targets
class MultiKeyInfo(BaseKeyInfo): def __init__(self, name, datatype, minOccurs, maxOccurs, handler, attribute): BaseKeyInfo.__init__(self, name, datatype, minOccurs, maxOccurs, handler, attribute) if self.name == "+": self._default = OrderedDict() else: self._default = [] def add_valueinfo(self, vi, key): if self.name == "+": # This is a keyed value, not a simple value: if key in self._default: self._default[key].append(vi) else: self._default[key] = [vi] else: self._default.append(vi) def computedefault(self, keytype): self.prepare_raw_defaults() for k, vlist in self._rawdefaults.items(): key = ValueInfo(k, vlist[0].position).convert(keytype) for vi in vlist: self.add_valueinfo(vi, key) def getdefault(self): return copy.copy(self._default)
def enrich(self, value): """ Given a YAML value, performs our registered transformations on it. """ if hasattr(value, 'enrich'): return value.enrich(self) # need to special-case str before Sequence because str \subset Sequence elif isinstance(value, str): return value elif isinstance(value, Mapping): out = OrderedDict() for key, val in value.items(): val = self.enrich(val) if val is not Void: out[key] = val return out elif isinstance(value, Sequence): out = [] for item in value: item = self.enrich(item) if item is not Void: out.append(item) return out else: return value
def _get_sorteddict(object, dictwithhash=False): if isinstance(object, dict): out = OrderedDict() for key, val in sorted(object.items()): if val: out[key] = _get_sorteddict(val, dictwithhash) elif isinstance(object, (list, tuple)): out = [] for val in object: if val: out.append(_get_sorteddict(val, dictwithhash)) if isinstance(object, tuple): out = tuple(out) else: if isinstance(object, str) and os.path.isfile(object): hash = hash_infile(object) if dictwithhash: out = (object, hash) else: out = hash elif isinstance(object, float): out = '%.10f'.format(object) else: out = object return out
def get_semodule(self, exisse, blocks): """ 以多种方式添加SeModule到MoBlock上,默认reduction=4 """ if exisse == () or isinstance(exisse[0], tuple): # 指定要添加的block-id 和 reduce-val exisse = OrderedDict(exisse) elif isinstance(exisse, (list, tuple)): # 指定要添加的block-id exisse_dict = OrderedDict() for id in exisse: exisse_dict.setdefault(id, 4) exisse = exisse_dict elif exisse == 'all': # 所有block都添加SeModule exisse = OrderedDict() for i in range(1, blocks + 1): exisse.setdefault(i, 4) elif exisse.startswith('ft-'): # 指定block添加SeModule # ft-3:5-7:10-15:18, from 3-5, 7-10, 15-18. # ft-0:0 不添加 ft = exisse[3:].split('-') exisse = [] for idx in ft: idx = [int(x) for x in idx.split(':')] for i in range(idx[0], idx[1] + 1): exisse.append((i, 4)) exisse = OrderedDict(exisse) return exisse
def read_xlsx(file_path, sheet_metaes): workbook = xlrd.open_workbook(file_path) xlsx_content = {} for sheetname, sheet_meta in sheet_metaes.iteritems(): sheet = workbook.sheet_by_name(sheetname) col_index = {} for x in xrange(sheet.ncols): colname = sheet.cell(0, x).value if colname in sheet_meta.fields: col_index[colname] = x sheet_content = OrderedDict() if sheet_meta.key else [] for x in xrange(1, sheet.nrows): if x == sheet_meta.descrow - 1: continue key_value = None row_value = {} for colname, field in sheet_meta.fields.iteritems(): col = col_index[colname] field_value = refine(field.type, field.element, _str(sheet.cell(x, col).value)) row_value[field.name] = field_value if field.name == sheet_meta.key: assert field.type != 'array' key_value = field_value if type(sheet_content) == list: sheet_content.append(row_value) else: sheet_content[key_value] = row_value xlsx_content[sheet_meta.dest] = sheet_content return xlsx_content
def _getter(obj): obj = getter(obj) if isinstance(obj, dict): obj = [{'key': k, 'value': v} for k, v in obj.items()] if isinstance(schema, dict): res = OrderedDict() _key, _value = list(schema.items())[0] for item in obj: if condition and not self.__get_value( condition, obj, item): continue res[self.__get_value(_key, obj, item)] = self.__get_value( _value, obj, item) return res if isinstance(schema, (tuple, list)): res = [] _value = schema[0] for item in obj: if condition and not self.__get_value( condition, obj, item): continue res.append(self.__get_value(_value, obj, item)) return type(schema)(res)
def __to_plain_containers(self, container: Union[CommentedSeq, CommentedMap] ) -> Union[OrderedDict, list]: """Converts any sequence or mapping to list or OrderedDict Stops at anything that isn't a sequence or a mapping. One day, we'll extract the comments and formatting and store them out-of-band. Args: mapping: The mapping of constructed subobjects to edit """ if isinstance(container, CommentedMap): new_container = OrderedDict() # type: Union[OrderedDict, list] for key, value_obj in container.items(): if (isinstance(value_obj, CommentedMap) or isinstance(value_obj, CommentedSeq)): new_container[key] = self.__to_plain_containers(value_obj) else: new_container[key] = value_obj elif isinstance(container, CommentedSeq): new_container = list() for value_obj in container: if (isinstance(value_obj, CommentedMap) or isinstance(value_obj, CommentedSeq)): new_container.append(self.__to_plain_containers(value_obj)) else: new_container.append(value_obj) return new_container
def split_data_by_k(data, k, dic): if dic: data1, data2 = OrderedDict(), OrderedDict() for k1, v1 in data.items(): assert len(v1) >= k mid = len(v1) // k data1[k1] = v1[mid:] data2[k1] = v1[:mid] print("{} total images".format( sum([len(data[key]) for key in data.keys()]))) print("{} (1) images".format( sum([len(data1[key]) for key in data1.keys()]))) print("{} (2) images".format( sum([len(data2[key]) for key in data2.keys()]))) else: data1, data2 = [], [] for v1 in data: assert len(v1) >= k mid = len(v1) / k data1.append(v1[mid:]) data2.append(v1[:mid]) print("{} total images".format(sum([len(d) for d in data]))) print("{} (1) images".format(sum([len(d) for d in data1]))) print("{} (2) images".format(sum([len(d) for d in data2]))) return data1, data2
def traverse_and_remove_path(obj, path=None, match="First"): if path is None: path = [] if isinstance(obj, dict): res = OrderedDict() for k, v in obj.items(): cmatch = False if (match and len(path) > 0 and path[0] == k): cmatch = True res.update( {k: traverse_and_remove_path(v, path=path[1:], match=cmatch)}) if (len(path) == 1 and path[0] in res.keys() and match): del res[path[0]] return res elif isinstance(obj, list): res = [] for i, elem in enumerate(obj): cmatch = False if (match and len(path) >= 1 and isinstance(path[0], int) and path[0] < len(obj) and i == path[0]): cmatch = True res.append( traverse_and_remove_path(elem, path=path[1:], match=cmatch)) if (len(path) == 1 and isinstance(path[0], int) and path[0] < len(res) and match): res.pop(path[0]) return res else: return obj # no container, just values (str, int, float)
def parse(tokens): if tokens[0] == "{": ret = OrderedDict() tokens = tokens[1:] while tokens[0] != "}": key = tokens[0] tokens = tokens[1:] tokens = tokens[1:] # Skip ':' value, tokens = parse(tokens) if tokens[0] == ",": tokens = tokens[1:] ret[key] = value tokens = tokens[1:] return ret, tokens elif tokens[0] == "[": ret = [] tokens = tokens[1:] while tokens[0] != "]": value, tokens = parse(tokens) if tokens[0] == ",": tokens = tokens[1:] ret.append(value) tokens = tokens[1:] return ret, tokens else: return tokens[0], tokens[1:]
def check_for_add_to_toolbars(self, plugin): from calibre.gui2.preferences.toolbar import ConfigWidget from calibre.customize import InterfaceActionBase if not isinstance(plugin, InterfaceActionBase): return all_locations = OrderedDict(ConfigWidget.LOCATIONS) plugin_action = plugin.load_actual_plugin(self.gui) installed_actions = OrderedDict([ (key, list(gprefs.get('action-layout-'+key, []))) for key in all_locations]) # If already installed in a GUI container, do nothing for action_names in installed_actions.itervalues(): if plugin_action.name in action_names: return allowed_locations = [(key, text) for key, text in all_locations.iteritems() if key not in plugin_action.dont_add_to] if not allowed_locations: return # This plugin doesn't want to live in the GUI from calibre.gui2.dialogs.choose_plugin_toolbars import ChoosePluginToolbarsDialog d = ChoosePluginToolbarsDialog(self, plugin_action, allowed_locations) if d.exec_() == d.Accepted: for key, text in d.selected_locations(): installed_actions = list(gprefs.get('action-layout-'+key, [])) installed_actions.append(plugin_action.name) gprefs['action-layout-'+key] = tuple(installed_actions)
def test_plugins(self): "Test that plugins without dependencies work" localrc = {'test_localrc': '1'} local_conf = {'install': {'nova.conf': {'main': {'test_conf': '2'}}}} services = {'cinder': True} # We use ordereddict here to make sure the plugins are in the # *wrong* order for testing. plugins = OrderedDict([ ('bar', 'https://git.openstack.org/openstack/bar-plugin'), ('foo', 'https://git.openstack.org/openstack/foo-plugin'), ('baz', 'https://git.openstack.org/openstack/baz-plugin'), ]) p = dict(localrc=localrc, local_conf=local_conf, base_services=[], services=services, plugins=plugins, base_dir='./test', path=os.path.join(self.tmpdir, 'test.local.conf')) lc = self._init_localconf(p) lc.write(p['path']) plugins = [] with open(p['path']) as f: for line in f: if line.startswith('enable_plugin'): plugins.append(line.split()[1]) self.assertEqual(['bar', 'baz', 'foo'], plugins)
def csv(input, dialect=None, header=True, key=None, sort=False): obj = [] fields = None if dialect: dialect = ocsv.get_dialect(dialect) else: sniffer = ocsv.Sniffer() dialect = sniffer.sniff(input.readline())() input.seek(0) reader = ocsv.reader(input, dialect=dialect) if key: header = True if header: fields = next(reader) if key and key <= len(fields): obj = OrderedDict() else: key = None for row in reader: if header: row = OrderedDict(zip(fields, row)) if key: rowkey = row.pop(fields[key - 1]) obj[rowkey] = row else: obj.append(row) else: obj.append(row) if sort: obj = sort_ordereddict(obj) return obj
def issmmumpssolver(*args): #ISSMSOLVE - return issm solver options # # Usage: # options=issmsolver; #retrieve options provided in varargin arguments = pairoptions.pairoptions(*args) options = OrderedDict() options['toolkit'] = 'issm' options['mat_type'] = 'mpidense' options['vec_type'] = 'mpi' options['solver_type'] = 'mumps' #now, go through our arguments, and write over default options. for i in range(len(arguments.list)): arg1 = arguments.list[i][0] arg2 = arguments.list[i][1] found = 0 for j in range(len(options)): joption = options[j][0] if joption == arg1: joption[1] = arg2 options[j] = joption found = 1 break if not found: #this option did not exist, add it: options.append([arg1, arg2]) return options
def unescape_json(element): """ Remove escape character '\' in a JSON object. :param element: JSON object :type element: JSON object :return: unescaped JSON object :rtype: JSON object """ if isinstance(element, str): return unescape_string(element) elif isinstance(element, Number): return element elif isinstance(element, bool): return element elif element is None: return element elif isinstance(element, dict): new_element = OrderedDict() for key, value in element.items(): new_key = unescape_json(key) new_value = unescape_json(value) new_element[new_key] = new_value return new_element elif isinstance(element, list): new_element = list() for index, item in enumerate(element): new_item = unescape_json(item) new_element.append(new_item) return new_element raise TemplateEngineException("Unknown data type {} of {}.".format( type(element), element))
def _get_sorteddict(object, dictwithhash=False): if isinstance(object, dict): out = OrderedDict() for key, val in sorted(object.items()): if val: out[key] = _get_sorteddict(val, dictwithhash) elif isinstance(object, (list, tuple)): out = [] for val in object: if val: out.append(_get_sorteddict(val, dictwithhash)) if isinstance(object, tuple): out = tuple(out) else: if isinstance(object, str) and os.path.isfile(object): hash = hash_infile(object) if dictwithhash: out = (object, hash) else: out = hash elif isinstance(object, float): out = '%.10f' % object else: out = object return out
def _marshall_item(rawitem, template, options): if isinstance(template, dict): result = OrderedDict() for key, value in template.items(): nkey = _marshall_item(rawitem, key, options) nvalue = _marshall_item(rawitem, value, options) if (nkey is not None) and (nvalue is not None): result[nkey] = nvalue return result elif isinstance(template, list): result = [] for value in template: nvalue = _marshall_item(rawitem, value, options) if nvalue is not None: result.append(nvalue) return result elif type(template) in (str, unicode): if _is_field(template): name, ftype = _get_field_comps(template) if name in rawitem: value = rawitem[name] else: return None if options['ignore_empty_item'] and len(value.strip()) == 0: return None value = _convert_to(ftype, value, options) return value elif options['ignore_empty_item'] and len(template.strip()) == 0: return None return template
def _parse(self, data, base_url): """ Given some parsed JSON data, returns the corresponding DocJSON objects. """ if isinstance(data, OrderedDict) and data.get('_type') == 'link': # Link objects url = data.get('url') method = data.get('method') fields = data.get('fields') try: return Link(url, method, fields, _base_url=base_url, _transport=self) except (TypeError, ValueError) as exc: raise ParseError(str(exc)) elif isinstance(data, OrderedDict): # Any unknown types should be ignored and treaded as a regular object. data.pop('_type', None) # Parse all the items in the dict and wrap them in a `Object`. parsed = OrderedDict() for key, value in data.items(): parsed[key] = self._parse(value, base_url) return Object(parsed) elif isinstance(data, list): # Parse all the items in the list and wrap them in a `List`. parsed = [] for item in data: value = self._parse(item, base_url) # Ignore 'Link' objects contained in a list. if not isinstance(value, Link): parsed.append(value) return List(parsed) return data
def ordUnUnicode(value): """ this is completely a python2 thing and should be able to just return value in python 3""" def chkNum(rawval): val = str(rawval) if val.isdigit(): val = int(val) else: try: val = float(val) except ValueError: val = str(val) return val #if isinstance(value, OrderedDict)or isinstance(value, dict): if type(value) in (OrderedDict, dict): newobj = OrderedDict() for key, value in value.items(): #print ('doing dict',key) key = ordUnUnicode(key) newobj[key] = ordUnUnicode(value) elif isinstance(value, list): newobj = [] for i in value: #print ('in i loop',i,' ck:',ordUnUnicode(i), ' val:', value,) newobj.append(ordUnUnicode(i)) #print ('newobh now', newobj) elif isinstance(value, unicode): newobj = str(value) # chkNum(value) else: newobj = value return newobj
def construct(obj): """Used for preventing of not expected changing of class attributes""" if isinstance(obj, OrderedDict): new_obj = OrderedDict() for key, value in obj.items(): new_obj[key] = construct(value) elif not isinstance(obj, OrderedDict) and isinstance(obj, dict): new_obj = dict() for key, value in obj.items(): new_obj[key] = construct(value) elif isinstance(obj, list): new_obj = list() for value in obj: new_obj.append(construct(value)) elif isinstance(obj, tuple): base = list() for value in obj: base.append(construct(value)) new_obj = tuple(base) elif isinstance(obj, str): new_obj = str(obj) elif isinstance(obj, (int, float, complex, type(None))) or inspect.isclass(obj): new_obj = obj else: raise TypeError( "Object of unsupported type was passed to construct function: %s" % type(obj)) return new_obj
def apply_dict_depth_first(nodes, func, as_dict=True, parents=None): ''' This function is similar to the `apply_depth_first` except that it operates on the `OrderedDict`-based structure returned from `apply_depth_first` when `as_dict=True`. Note that if `as_dict` is `False`, the result of this function is given in the entry/tuple form. ''' if as_dict: items = OrderedDict() else: items = [] if parents is None: parents = [] for i, (k, node) in enumerate(nodes.iteritems()): item = func(k, node, parents) item_parents = parents + [(k, node)] if node.children is not None: children = apply_dict_depth_first(node.children, func, as_dict=as_dict, parents=item_parents) else: children = None if as_dict: items[k] = Node(item, children) elif children: items.append((item, children)) else: items.append(item) return items
def to_json_stat(input_df, value="value", output="list"): """Encode pandas.DataFrame object into JSON-stat format. The DataFrames must have exactly one value column. Args: df(pandas.DataFrame): pandas data frame (or list of data frames) to encode. value (string, optional): name of the value column. Defaults to 'value'. output(string): accepts two values: 'list' or 'dict'. Produce list of\ dicts or dict of dicts as output. Returns: output(string): String with JSON-stat object. """ data = [] if output == "list": result = [] elif output == "dict": result = OrderedDict({}) if isinstance(input_df, pd.DataFrame): data.append(input_df) else: data = input_df for row, dataframe in enumerate(data): pd.notnull(dataframe[value]) dims = data[row].filter([item for item in data[row].columns.values if item not in value]) if len(dims.columns.values) != len(set(dims.columns.values)): raise ValueError("Non-value columns must constitute a unique ID") dim_names = list(dims) categories = [ { to_int(i): { "label": to_str(i), "category": { "index": OrderedDict([(to_str(j), to_int(k)) for k, j in enumerate(uniquify(dims[i]))]), "label": OrderedDict([(to_str(j), to_str(j)) for k, j in enumerate(uniquify(dims[i]))]), }, } } for i in dims.columns.values ] dataset = {"dataset" + str(row + 1): {"dimension": OrderedDict(), value: [x for x in dataframe[value].values]}} for category in categories: dataset["dataset" + str(row + 1)]["dimension"].update(category) dataset["dataset" + str(row + 1)]["dimension"].update({"id": dim_names}) dataset["dataset" + str(row + 1)]["dimension"].update( {"size": [len(dims[i].unique()) for i in dims.columns.values]} ) for category in categories: dataset["dataset" + str(row + 1)]["dimension"].update(category) if output == "list": result.append(dataset) elif output == "dict": result.update(dataset) else: result = None return json.dumps(result)
def __init__(self, data = None, items = None, post = None, **args): super().__init__(data, items, post, **args) config = self.config config_data = config['data'] field_config = config_data.get('field_config', {}) set = self['configset'] options = OrderedDict() options['autocomplete'] = s('Auto complete') options['select'] = s('Select list') options['checkboxes'] = s('Check boxes') options['radioboxes'] = s('Radio boxes') # max limit set.add('HTMLSelect', { 'id' : 'field_selection_type', 'value' : post.get('field_selection_type', None) or field_config.get('field_selection_type', None), 'title' : s('Field selection type'), 'options' : options, 'required' : True, 'css' : ['i-form-large'], 'multiple' : False, 'weight' : 1, }) field_value_options = post.get('field_value_options', None) if field_value_options is None: keys = field_config.get('field_value_option_keys', []) values = field_config.get('field_value_option_values', {}) options = [] for key in keys: if key in values: options.append(':'.join((str(key), values[key]))) field_value_options = '\n'.join(options) set.add('TextArea', { 'id' : 'field_value_options', 'value' : field_value_options, 'title' : s('Field allowed values'), 'required' : True, 'css' : ['i-form-large i-width-1-1'], 'multiple' : False, 'weight' : 2, 'info' : s('''Colon separated key:value pair of allowed values, one per line. Key should be a number. example:<br> 0:Female<br> 1:Male<br> 2:Shemale<br> '''), 'validation_rule' : ['Not', [['Empty']], 'Values field is required'] })
def deal_fields(self): l = OrderedDict() if self.is_responsible: deals = Deal.objects.filter(partnership__responsible=self).all() else: deals = Deal.objects.filter(partnership=self).all() l = list() for deal in deals: l.append(deal.fields) return l
def getLastRecordsByThreshold(self, minTimestamp = None, maxTimestamp = None, orderAsc = True, asDict = False, noOfRecords = None): if asDict: result = OrderedDict() else: result = [] if(self.database.isOpen()): cursor = None order = "ASC" if not orderAsc: order = "DESC" limit = "" if noOfRecords: limit = " LIMIT %d" % noOfRecords if minTimestamp is None and maxTimestamp is None: cursor = self.database.execute("SELECT MAX(forecastID), * FROM mixerData GROUP BY timestamp ORDER BY timestamp " + order + limit) elif minTimestamp is None: cursor = self.database.execute("SELECT MAX(forecastID), * FROM mixerData WHERE timestamp<=? GROUP BY timestamp ORDER BY timestamp " + order + limit, (maxTimestamp, )) elif maxTimestamp is None: cursor = self.database.execute("SELECT MAX(forecastID), * FROM mixerData WHERE ?<=timestamp GROUP BY timestamp ORDER BY timestamp " + order + limit, (minTimestamp, )) else: cursor = self.database.execute("SELECT MAX(forecastID), * FROM mixerData WHERE ?<=timestamp AND timestamp<=? GROUP BY timestamp ORDER BY timestamp " + order + limit, (minTimestamp, maxTimestamp, )) for row in cursor: mixerData = RMMixerData(row[3]) mixerData.temperature = row[4] mixerData.rh = row[5] mixerData.wind = row[6] mixerData.solarRad = row[7] mixerData.skyCover = row[8] mixerData.rain = row[9] mixerData.et0 = row[10] mixerData.pop = row[11] mixerData.qpf = row[12] mixerData.condition = row[13] mixerData.pressure = row[14] mixerData.dewPoint = row[15] mixerData.minTemp = row[16] mixerData.maxTemp = row[17] mixerData.minRH = row[18] mixerData.maxRH = row[19] mixerData.et0calc = row[20] mixerData.et0final = row[21] if asDict: result[row[3]] = mixerData else: result.append(mixerData) return result
def _handle_one_attr_one_env(self, obj, key, mc_attr, env, attributes_overriding_property, dir_entries, names_only): attr_inf = [] try: val = mc_attr.env_values[env] if key in dir_entries: attributes_overriding_property.add(key) attr_inf = [(' #overrides @property', True)] except KeyError as ex: # mc_attribute overriding @property OR the value for env has not yet been set try: val = obj.getattr(key, env) attr_inf = [(' #value for {env} provided by @property'.format(env=env), True)] except AttributeError: val = McInvalidValue.MC_NO_VALUE if self.user_filter_callable: try: key, val = self.user_filter_callable(obj, key, val) if key is False: return key, [] except Exception as ex: self.num_errors += 1 traceback.print_exception(*sys.exc_info()) attr_inf.append((' #json_error calling filter', repr(ex)),) val = self._check_nesting(obj, val) if val == McInvalidValue.MC_NO_VALUE: return key, [(' #no value for {env}'.format(env=env), True)] if isinstance(val, Mapping): new_val = OrderedDict() for inner_key, maybeitem in val.items(): if not isinstance(maybeitem, self.multiconf_base_type): new_val[str(inner_key)] = maybeitem continue new_val[inner_key] = self._ref_mc_item_str(maybeitem) return key, [('', new_val)] + attr_inf try: iterable = iter(val) # TODO?: Include type of iterable in json meta info except TypeError: return key, [('', val)] + attr_inf if isinstance(val, str): return key, [('', val)] + attr_inf new_val = [] for maybeitem in val: if not isinstance(maybeitem, self.multiconf_base_type): new_val.append(maybeitem) continue new_val.append(self._ref_mc_item_str(maybeitem)) return key, [('', new_val)] + attr_inf
def get_categories(url): soup = bs(urlopen(url)) cats = OrderedDict() for opt in soup.findAll('option')[1:]: c = opt.text.split('&')[0].lower().replace(' ', '-') cat_url = url + 'category/' + c + '/' cats.append(c) """ if isValid(cat_url): cats.append(c) """ return cats
def parseADL(lines): objects = [] parent_object = [objects] while lines: line = lines.pop().strip() # opening m = re.match('^(.*) {', line) if m: name = m.groups()[0] if name in ['children', 'colors', 'points']: current = [] else: current = OrderedDict() current['object_type'] = name if isinstance(parent_object[-1], list): parent_object[-1].append(current) else: parent_object[-1][name] = current parent_object.append(current) # attr = value m = re.match('^(.*?)=(.*)', line) if m: attr = m.groups()[0] value= m.groups()[1] current[attr] = value # color, m = re.match('([0-9a-fA-F]{6}),', line) if m: color = m.groups()[0] current.append(color) # (x,y) point pair m = re.match('\((-?[0-9]+),(-?[0-9]+)\)', line) if m: x = m.groups()[0] y = m.groups()[1] current.append((int(x),int(y))) # closing if line == '}': parent_object.pop() current = parent_object[-1] # restructure to adl dictionary adl = {'objects':[]} for o in objects: type_name = o['object_type'] if type_name in ['file', '"color map"', 'display']: adl[type_name] = o else: adl['objects'].append(o) return adl
def _insert_parent_items(parent, items): if isinstance(parent, dict): result = OrderedDict() for key, value in parent.iteritems(): result[key] = _insert_parent_items(value, items) return result elif isinstance(parent, list): result = OrderedDict() for item in parent: result.append(_insert_parent_items(parent, items)) return result elif parent == "${template}": return items
def test_plugin_deps(self): "Test that plugins with dependencies work" os.makedirs(os.path.join(self.tmpdir, 'foo-plugin', 'devstack')) os.makedirs(os.path.join(self.tmpdir, 'foo-plugin', '.git')) os.makedirs(os.path.join(self.tmpdir, 'bar-plugin', 'devstack')) os.makedirs(os.path.join(self.tmpdir, 'bar-plugin', '.git')) with open(os.path.join( self.tmpdir, 'foo-plugin', 'devstack', 'settings'), 'w') as f: f.write('define_plugin foo-plugin\n') with open(os.path.join( self.tmpdir, 'bar-plugin', 'devstack', 'settings'), 'w') as f: f.write('define_plugin bar-plugin\n') f.write('plugin_requires bar-plugin foo-plugin\n') localrc = {'test_localrc': '1'} local_conf = {'install': {'nova.conf': {'main': {'test_conf': '2'}}}} services = {'cinder': True} # We use ordereddict here to make sure the plugins are in the # *wrong* order for testing. plugins = OrderedDict([ ('bar-plugin', 'git://git.openstack.org/openstack/bar-plugin'), ('foo-plugin', 'git://git.openstack.org/openstack/foo-plugin'), ]) p = dict(localrc=localrc, local_conf=local_conf, base_services=[], services=services, plugins=plugins, base_dir=self.tmpdir, path=os.path.join(self.tmpdir, 'test.local.conf')) lc = LocalConf(p.get('localrc'), p.get('local_conf'), p.get('base_services'), p.get('services'), p.get('plugins'), p.get('base_dir'), p.get('projects'), p.get('project')) lc.write(p['path']) plugins = [] with open(p['path']) as f: for line in f: if line.startswith('enable_plugin'): plugins.append(line.split()[1]) self.assertEqual(['foo-plugin', 'bar-plugin'], plugins)
def _sort_recurse(item): if isinstance(item, str): result = item elif isinstance(item, Mapping): result = OrderedDict() for key in sorted(item.keys()): result[key] = _sort_recurse(item[key]) elif isinstance(item, Sequence): result = list() for element in item: result.append(_sort_recurse(element)) else: result = item return result
def _parse_sig(sig, arg_names, validate=False): """ Parses signatures into a ``OrderedDict`` of paramName => type. Numerically-indexed arguments that do not correspond to an argument name in python (ie: it takes a variable number of arguments) will be keyed as the stringified version of it's index. sig the signature to be parsed arg_names a list of argument names extracted from python source Returns a tuple of (method name, types dict, return type) """ d = SIG_RE.match(sig) if not d: raise ValueError('Invalid method signature %s' % sig) d = d.groupdict() ret = [(n, Any) for n in arg_names] if 'args_sig' in d and type( d['args_sig']) is str and d['args_sig'].strip(): for i, arg in enumerate(d['args_sig'].strip().split(',')): _type_checking_available(sig, validate) if '=' in arg: if not type(ret) is OrderedDict: ret = OrderedDict(ret) dk = KWARG_RE.match(arg) if not dk: raise ValueError('Could not parse arg type %s in %s' % (arg, sig)) dk = dk.groupdict() if not sum( [(k in dk and type(dk[k]) is str and bool(dk[k].strip())) for k in ('arg_name', 'arg_type')]): raise ValueError('Invalid kwarg value %s in %s' % (arg, sig)) ret[dk['arg_name']] = _eval_arg_type(dk['arg_type'], None, arg, sig) else: if type(ret) is OrderedDict: raise ValueError('Positional arguments must occur ' 'before keyword arguments in %s' % sig) if len(ret) < i + 1: ret.append((str(i), _eval_arg_type(arg, None, arg, sig))) else: ret[i] = (ret[i][0], _eval_arg_type(arg, None, arg, sig)) if not type(ret) is OrderedDict: ret = OrderedDict(ret) return (d['method_name'], ret, (_eval_arg_type(d['return_sig'], Any, 'return', sig) if d['return_sig'] else Any))
def getProducerInterval(self, dataType, aggregationType = None): if aggregationType == None: print(dataType.subtrees["avg"].subtrees['producer_interval'].value) if not self._root.subtrees['data'].subtrees[dataType].subtrees.get(aggregationType) == None: return self._root.subtrees['data'].subtrees[dataType].subtrees[aggregationType].value else: return False else: result = OrderedDict() for i in range(len(self._root.subtrees['data'].subtrees[dataType].subtrees)): string = self._root.subtrees['data'].subtrees[dataType].subtrees.items()[i][0] print(string) result.append(string) # = self._root.subtrees['data'].subtrees[dataType].subtrees[string].value return result
def ununicodify(obj): result = None if isinstance(obj, OrderedDict): result = OrderedDict() for k, v in obj.iteritems(): k1 = str(k) if isinstance(k, unicode) else k result[k1] = ununicodify(v) elif isinstance(obj, list): result = [] for v in obj: result.append(ununicodify(v)) elif isinstance(obj, unicode): result = str(obj) else: result = obj return result
def getAllRecords(self, asDict = True): if self.database.isOpen(): if asDict: records = OrderedDict() else: records = [] rows = self.database.execute("SELECT token, expiration FROM authTokens") for row in rows: if asDict: records[str(row[0])] = RMAuthToken(str(row[0]), row[1]) else: records.append(RMAuthToken(str(row[0]), row[1])) return records return None