def test_encode_images(): # invalid data, but the header and footer are from real files pngdata = b'\x89PNG\r\n\x1a\nblahblahnotactuallyvalidIEND\xaeB`\x82' jpegdata = b'\xff\xd8\xff\xe0\x00\x10JFIFblahblahjpeg(\xa0\x0f\xff\xd9' pdfdata = b'%PDF-1.\ntrailer<</Root<</Pages<</Kids[<</MediaBox[0 0 3 3]>>]>>>>>>' fmt = { 'image/png': pngdata, 'image/jpeg': jpegdata, 'application/pdf': pdfdata } encoded = encode_images(fmt) for key, value in iteritems(fmt): # encoded has unicode, want bytes decoded = decodestring(encoded[key].encode('ascii')) nt.assert_equal(decoded, value) encoded2 = encode_images(encoded) nt.assert_equal(encoded, encoded2) b64_str = {} for key, encoded in iteritems(encoded): b64_str[key] = unicode_to_str(encoded) encoded3 = encode_images(b64_str) nt.assert_equal(encoded3, b64_str) for key, value in iteritems(fmt): # encoded3 has str, want bytes decoded = decodestring(str_to_bytes(encoded3[key])) nt.assert_equal(decoded, value)
def test_encode_images(): # invalid data, but the header and footer are from real files pngdata = b'\x89PNG\r\n\x1a\nblahblahnotactuallyvalidIEND\xaeB`\x82' jpegdata = b'\xff\xd8\xff\xe0\x00\x10JFIFblahblahjpeg(\xa0\x0f\xff\xd9' pdfdata = b'%PDF-1.\ntrailer<</Root<</Pages<</Kids[<</MediaBox[0 0 3 3]>>]>>>>>>' fmt = { 'image/png' : pngdata, 'image/jpeg' : jpegdata, 'application/pdf' : pdfdata } encoded = encode_images(fmt) for key, value in iteritems(fmt): # encoded has unicode, want bytes decoded = decodestring(encoded[key].encode('ascii')) nt.assert_equal(decoded, value) encoded2 = encode_images(encoded) nt.assert_equal(encoded, encoded2) b64_str = {} for key, encoded in iteritems(encoded): b64_str[key] = unicode_to_str(encoded) encoded3 = encode_images(b64_str) nt.assert_equal(encoded3, b64_str) for key, value in iteritems(fmt): # encoded3 has str, want bytes decoded = decodestring(str_to_bytes(encoded3[key])) nt.assert_equal(decoded, value)
def _add_arguments(self, aliases=None, flags=None): self.alias_flags = {} # print aliases, flags if aliases is None: aliases = self.aliases if flags is None: flags = self.flags paa = self.parser.add_argument for key,value in iteritems(aliases): if key in flags: # flags nargs = '?' else: nargs = None if len(key) is 1: paa('-'+key, '--'+key, type=unicode_type, dest=value, nargs=nargs) else: paa('--'+key, type=unicode_type, dest=value, nargs=nargs) for key, (value, help) in iteritems(flags): if key in self.aliases: # self.alias_flags[self.aliases[key]] = value continue if len(key) is 1: paa('-'+key, '--'+key, action='append_const', dest='_flags', const=value) else: paa('--'+key, action='append_const', dest='_flags', const=value)
def _load_config(self, cfg, section_names=None, traits=None): """load traits from a Config object""" if traits is None: traits = self.traits(config=True) if section_names is None: section_names = self.section_names() my_config = self._find_my_config(cfg) # hold trait notifications until after all config has been loaded with self.hold_trait_notifications(): for name, config_value in iteritems(my_config): if name in traits: if isinstance(config_value, LazyConfigValue): # ConfigValue is a wrapper for using append / update on containers # without having to copy the initial value initial = getattr(self, name) config_value = config_value.get_value(initial) # We have to do a deepcopy here if we don't deepcopy the entire # config object. If we don't, a mutable config_value will be # shared by all instances, effectively making it a class attribute. setattr(self, name, deepcopy(config_value)) elif isinstance(self, LoggingConfigurable): from difflib import get_close_matches matches = get_close_matches(name, traits) if len(matches) == 1: self.log.warning(u"Config option `{option}` not recognized by `{klass}`, do you mean : `{matches}`" .format(option=name, klass=type(self).__name__, matches=matches[0])) elif len(matches) >= 1: self.log.warning(u"Config option `{option}` not recognized by `{klass}`, do you mean one of : `{matches}`" .format(option=name, klass=type(self).__name__, matches=' ,'.join(matches)))
def _convert_to_config(self): """self.parsed_data->self.config, parse unrecognized extra args via KVLoader.""" # remove subconfigs list from namespace before transforming the Namespace if '_flags' in self.parsed_data: subcs = self.parsed_data._flags del self.parsed_data._flags else: subcs = [] for k, v in iteritems(vars(self.parsed_data)): if v is None: # it was a flag that shares the name of an alias subcs.append(self.alias_flags[k]) else: # eval the KV assignment self._exec_config_str(k, v) for subc in subcs: self._load_flag(subc) if self.extra_args: sub_parser = KeyValueConfigLoader(log=self.log) sub_parser.load_config(self.extra_args) self.config.merge(sub_parser.config) self.extra_args = sub_parser.extra_args
def print_alias_help(self): """Print the alias part of the help.""" if not self.aliases: return lines = [] classdict = {} for cls in self.classes: # include all parents (up to, but excluding Configurable) in available names for c in cls.mro()[:-3]: classdict[c.__name__] = c for alias, longname in iteritems(self.aliases): classname, traitname = longname.split('.',1) cls = classdict[classname] trait = cls.class_traits(config=True)[traitname] help = cls.class_get_trait_help(trait).splitlines() # reformat first line help[0] = help[0].replace(longname, alias) + ' (%s)'%longname if len(alias) == 1: help[0] = help[0].replace('--%s='%alias, '-%s '%alias) lines.extend(help) # lines.append('') print(os.linesep.join(lines))
def class_config_section(cls): """Get the config class config section""" def c(s): """return a commented, wrapped block.""" s = '\n\n'.join(wrap_paragraphs(s, 78)) return '# ' + s.replace('\n', '\n# ') # section header breaker = '#' + '-' * 78 s = "# %s configuration" % cls.__name__ lines = [breaker, s, breaker, ''] # get the description trait desc = cls.class_traits().get('description') if desc: desc = desc.default_value else: # no description trait, use __doc__ desc = getattr(cls, '__doc__', '') if desc: lines.append(c(desc)) lines.append('') for name, trait in iteritems(cls.class_own_traits(config=True)): lines.append(c(trait.help)) lines.append('# c.%s.%s = %r' % (cls.__name__, name, trait.default_value)) lines.append('') return '\n'.join(lines)
def class_config_section(cls): """Get the config class config section""" def c(s): """return a commented, wrapped block.""" s = '\n\n'.join(wrap_paragraphs(s, 78)) return '# ' + s.replace('\n', '\n# ') # section header breaker = '#' + '-'*78 s = "# %s configuration" % cls.__name__ lines = [breaker, s, breaker, ''] # get the description trait desc = cls.class_traits().get('description') if desc: desc = desc.default_value else: # no description trait, use __doc__ desc = getattr(cls, '__doc__', '') if desc: lines.append(c(desc)) lines.append('') for name, trait in iteritems(cls.class_own_traits(config=True)): help = trait.get_metadata('help') or '' lines.append(c(help)) lines.append('# c.%s.%s = %r'%(cls.__name__, name, trait.get_default_value())) lines.append('') return '\n'.join(lines)
def _render_expression(self, check): """Turn a mongodb-style search dict into an SQL query.""" expressions = [] args = [] skeys = set(check.keys()) skeys.difference_update(set(self._keys)) skeys.difference_update(set(['buffers', 'result_buffers'])) if skeys: raise KeyError("Illegal testing key(s): %s" % skeys) for name, sub_check in iteritems(check): if isinstance(sub_check, dict): for test, value in iteritems(sub_check): try: op = operators[test] except KeyError: raise KeyError("Unsupported operator: %r" % test) if isinstance(op, tuple): op, join = op if value is None and op in null_operators: expr = "%s %s" % (name, null_operators[op]) else: expr = "%s %s ?" % (name, op) if isinstance(value, (tuple, list)): if op in null_operators and any( [v is None for v in value]): # equality tests don't work with NULL raise ValueError( "Cannot use %r test with NULL values on SQLite backend" % test) expr = '( %s )' % (join.join([expr] * len(value))) args.extend(value) else: args.append(value) expressions.append(expr) else: # it's an equality check if sub_check is None: expressions.append("%s IS NULL" % name) else: expressions.append("%s = ?" % name) args.append(sub_check) expr = " AND ".join(expressions) return expr, args
def uncan_dict(obj, g=None): if istype(obj, dict): newobj = {} for k, v in iteritems(obj): newobj[k] = uncan(v, g) return newobj else: return obj
def _flags_changed(self, name, old, new): """ensure flags dict is valid""" for key, value in iteritems(new): assert len(value) == 2, "Bad flag: %r:%s" % (key, value) assert isinstance(value[0], (dict, Config)), "Bad flag: %r:%s" % (key, value) assert isinstance(value[1], string_types), "Bad flag: %r:%s" % (key, value)
def uncan_dict(obj, g=None): if istype(obj, dict): newobj = {} for k, v in iteritems(obj): newobj[k] = uncan(v,g) return newobj else: return obj
def test_get_dict(self): n = len(self.client) ar = self.client[:].apply_async(lambda : 5) self.assertEqual(ar.get(), [5]*n) d = ar.get_dict() self.assertEqual(sorted(d.keys()), sorted(self.client.ids)) for eid,r in iteritems(d): self.assertEqual(r, 5)
def test_get_dict(self): n = len(self.client) ar = self.client[:].apply_async(lambda: 5) self.assertEqual(ar.get(), [5] * n) d = ar.get_dict() self.assertEqual(sorted(d.keys()), sorted(self.client.ids)) for eid, r in iteritems(d): self.assertEqual(r, 5)
def _load_flag(self, cfg): """update self.config from a flag, which can be a dict or Config""" if isinstance(cfg, (dict, Config)): # don't clobber whole config sections, update # each section from config: for sec, c in iteritems(cfg): self.config[sec].update(c) else: raise TypeError("Invalid flag: %r" % cfg)
def _load_flag(self, cfg): """update self.config from a flag, which can be a dict or Config""" if isinstance(cfg, (dict, Config)): # don't clobber whole config sections, update # each section from config: for sec,c in iteritems(cfg): self.config[sec].update(c) else: raise TypeError("Invalid flag: %r" % cfg)
def validate_string_dict(dct): """Validate that the input is a dict with string keys and values. Raises ValueError if not.""" for k,v in iteritems(dct): if not isinstance(k, string_types): raise ValueError('key %r in dict must be a string' % k) if not isinstance(v, string_types): raise ValueError('value %r in dict must be a string' % v)
def can_dict(obj): """can the *values* of a dict""" if istype(obj, dict): newobj = {} for k, v in iteritems(obj): newobj[k] = can(v) return newobj else: return obj
def validate_string_dict(dct): """Validate that the input is a dict with string keys and values. Raises ValueError if not.""" for k, v in iteritems(dct): if not isinstance(k, string_types): raise ValueError('key %r in dict must be a string' % k) if not isinstance(v, string_types): raise ValueError('value %r in dict must be a string' % v)
def get_env_vars(self): env_vars = ET.Element('EnvironmentVariables') for k, v in iteritems(self.environment_variables): variable = ET.SubElement(env_vars, "Variable") name = ET.SubElement(variable, "Name") name.text = k value = ET.SubElement(variable, "Value") value.text = v return env_vars
def flatten_flags(self): """flatten flags and aliases, so cl-args override as expected. This prevents issues such as an alias pointing to InteractiveShell, but a config file setting the same trait in TerminalInteraciveShell getting inappropriate priority over the command-line arg. Only aliases with exactly one descendent in the class list will be promoted. """ # build a tree of classes in our list that inherit from a particular # it will be a dict by parent classname of classes in our list # that are descendents mro_tree = defaultdict(list) for cls in self.classes: clsname = cls.__name__ for parent in cls.mro()[1:-3]: # exclude cls itself and Configurable,HasTraits,object mro_tree[parent.__name__].append(clsname) # flatten aliases, which have the form: # { 'alias' : 'Class.trait' } aliases = {} for alias, cls_trait in iteritems(self.aliases): cls,trait = cls_trait.split('.',1) children = mro_tree[cls] if len(children) == 1: # exactly one descendent, promote alias cls = children[0] aliases[alias] = '.'.join([cls,trait]) # flatten flags, which are of the form: # { 'key' : ({'Cls' : {'trait' : value}}, 'help')} flags = {} for key, (flagdict, help) in iteritems(self.flags): newflag = {} for cls, subdict in iteritems(flagdict): children = mro_tree[cls] # exactly one descendent, promote flag section if len(children) == 1: cls = children[0] newflag[cls] = subdict flags[key] = (newflag, help) return flags, aliases
def _render_expression(self, check): """Turn a mongodb-style search dict into an SQL query.""" expressions = [] args = [] skeys = set(check.keys()) skeys.difference_update(set(self._keys)) skeys.difference_update(set(['buffers', 'result_buffers'])) if skeys: raise KeyError("Illegal testing key(s): %s"%skeys) for name,sub_check in iteritems(check): if isinstance(sub_check, dict): for test,value in iteritems(sub_check): try: op = operators[test] except KeyError: raise KeyError("Unsupported operator: %r"%test) if isinstance(op, tuple): op, join = op if value is None and op in null_operators: expr = "%s %s" % (name, null_operators[op]) else: expr = "%s %s ?"%(name, op) if isinstance(value, (tuple,list)): if op in null_operators and any([v is None for v in value]): # equality tests don't work with NULL raise ValueError("Cannot use %r test with NULL values on SQLite backend"%test) expr = '( %s )'%( join.join([expr]*len(value)) ) args.extend(value) else: args.append(value) expressions.append(expr) else: # it's an equality check if sub_check is None: expressions.append("%s IS NULL" % name) else: expressions.append("%s = ?"%name) args.append(sub_check) expr = " AND ".join(expressions) return expr, args
def squash_dates(obj): """squash datetime objects into ISO8601 strings""" if isinstance(obj, dict): obj = dict(obj) # don't clobber for k,v in iteritems(obj): obj[k] = squash_dates(v) elif isinstance(obj, (list, tuple)): obj = [ squash_dates(o) for o in obj ] elif isinstance(obj, datetime): obj = obj.isoformat() return obj
def squash_dates(obj): """squash datetime objects into ISO8601 strings""" if isinstance(obj, dict): obj = dict(obj) # don't clobber for k, v in iteritems(obj): obj[k] = squash_dates(v) elif isinstance(obj, (list, tuple)): obj = [squash_dates(o) for o in obj] elif isinstance(obj, datetime): obj = obj.isoformat() return obj
def extract_dates(obj): """extract ISO8601 dates from unpacked JSON""" if isinstance(obj, dict): new_obj = {} # don't clobber for k,v in iteritems(obj): new_obj[k] = extract_dates(v) obj = new_obj elif isinstance(obj, (list, tuple)): obj = [ extract_dates(o) for o in obj ] elif isinstance(obj, string_types): obj = _parse_date(obj) return obj
def extract_dates(obj): """extract ISO8601 dates from unpacked JSON""" if isinstance(obj, dict): new_obj = {} # don't clobber for k,v in iteritems(obj): new_obj[k] = extract_dates(v) obj = new_obj elif isinstance(obj, (list, tuple)): obj = [ extract_dates(o) for o in obj ] elif isinstance(obj, string_types): obj = parse_date(obj) return obj
def print_flag_help(self): """Print the flag part of the help.""" if not self.flags: return lines = [] for m, (cfg,help) in iteritems(self.flags): prefix = '--' if len(m) > 1 else '-' lines.append(prefix+m) lines.append(indent(dedent(help.strip()))) # lines.append('') print(os.linesep.join(lines))
def __new__(mcls, name, bases, classdict): # FIXME: this duplicates the code from MetaHasTraits. # I don't think a super() call will help me here. for k, v in iteritems(classdict): if isinstance(v, TraitType): v.name = k elif inspect.isclass(v): if issubclass(v, TraitType): vinst = v() vinst.name = k classdict[k] = vinst cls = MetaQObject.__new__(mcls, name, bases, classdict) return cls
def __new__(mcls, name, bases, classdict): # FIXME: this duplicates the code from MetaHasTraits. # I don't think a super() call will help me here. for k,v in iteritems(classdict): if isinstance(v, TraitType): v.name = k elif inspect.isclass(v): if issubclass(v, TraitType): vinst = v() vinst.name = k classdict[k] = vinst cls = MetaQObject.__new__(mcls, name, bases, classdict) return cls
def _match(self, check): """Find all the matches for a check dict.""" matches = [] tests = {} for k, v in iteritems(check): if isinstance(v, dict): tests[k] = CompositeFilter(v) else: tests[k] = lambda o: o == v for rec in itervalues(self._records): if self._match_one(rec, tests): matches.append(deepcopy(rec)) return matches
def _match(self, check): """Find all the matches for a check dict.""" matches = [] tests = {} for k,v in iteritems(check): if isinstance(v, dict): tests[k] = CompositeFilter(v) else: tests[k] = lambda o: o==v for rec in itervalues(self._records): if self._match_one(rec, tests): matches.append(deepcopy(rec)) return matches
def merge(self, other): """merge another config object into this one""" to_update = {} for k, v in iteritems(other): if k not in self: to_update[k] = v else: # I have this key if isinstance(v, Config) and isinstance(self[k], Config): # Recursively merge common sub Configs self[k].merge(v) else: # Plain updates for non-Configs to_update[k] = v self.update(to_update)
def uncan(obj, g=None): """Invert canning.""" import_needed = False for cls, uncanner in iteritems(uncan_map): if isinstance(cls, string_types): import_needed = True break elif isinstance(obj, cls): return uncanner(obj, g) if import_needed: # perform uncan_map imports, then try again # this will usually only happen once _import_mapping(uncan_map, _original_uncan_map) return uncan(obj, g) return obj
def print_subcommands(self): """Print the subcommand part of the help.""" if not self.subcommands: return lines = ["Subcommands"] lines.append('-'*len(lines[0])) lines.append('') for p in wrap_paragraphs(self.subcommand_description.format( app=self.name)): lines.append(p) lines.append('') for subc, (cls, help) in iteritems(self.subcommands): lines.append(subc) if help: lines.append(indent(dedent(help.strip()))) lines.append('') print(os.linesep.join(lines))
def can(obj): """Prepare an object for pickling.""" import_needed = False for cls, canner in iteritems(can_map): if isinstance(cls, string_types): import_needed = True break elif istype(obj, cls): return canner(obj) if import_needed: # perform can_map imports, then try again # this will usually only happen once _import_mapping(can_map, _original_can_map) return can(obj) return obj
def uncan(obj, g=None): """invert canning""" import_needed = False for cls,uncanner in iteritems(uncan_map): if isinstance(cls, string_types): import_needed = True break elif isinstance(obj, cls): return uncanner(obj, g) if import_needed: # perform uncan_map imports, then try again # this will usually only happen once _import_mapping(uncan_map, _original_uncan_map) return uncan(obj, g) return obj
def can(obj): """prepare an object for pickling""" import_needed = False for cls,canner in iteritems(can_map): if isinstance(cls, string_types): import_needed = True break elif istype(obj, cls): return canner(obj) if import_needed: # perform can_map imports, then try again # this will usually only happen once _import_mapping(can_map, _original_can_map) return can(obj) return obj
def set_flags(self, **kwargs): """set my attribute flags by keyword. Views determine behavior with a few attributes (`block`, `track`, etc.). These attributes can be set all at once by name with this method. Parameters ---------- block : bool whether to wait for results track : bool whether to create a MessageTracker to allow the user to safely edit after arrays and buffers during non-copying sends. """ for name, value in iteritems(kwargs): if name not in self._flag_names: raise KeyError("Invalid name: %r" % name) else: setattr(self, name, value)
def _load_config(self, cfg, section_names=None, traits=None): """load traits from a Config object""" if traits is None: traits = self.traits(config=True) if section_names is None: section_names = self.section_names() my_config = self._find_my_config(cfg) # hold trait notifications until after all config has been loaded with self.hold_trait_notifications(): for name, config_value in iteritems(my_config): if name in traits: if isinstance(config_value, LazyConfigValue): # ConfigValue is a wrapper for using append / update on containers # without having to copy the initial value initial = getattr(self, name) config_value = config_value.get_value(initial) # We have to do a deepcopy here if we don't deepcopy the entire # config object. If we don't, a mutable config_value will be # shared by all instances, effectively making it a class attribute. setattr(self, name, deepcopy(config_value)) elif not _is_section_key(name) and not isinstance( config_value, Config): from difflib import get_close_matches if isinstance(self, LoggingConfigurable): warn = self.log.warning else: warn = lambda msg: warnings.warn(msg, stacklevel=9) matches = get_close_matches(name, traits) msg = u"Config option `{option}` not recognized by `{klass}`.".format( option=name, klass=self.__class__.__name__) if len(matches) == 1: msg += u" Did you mean `{matches}`?".format( matches=matches[0]) elif len(matches) >= 1: msg += " Did you mean one of: `{matches}`?".format( matches=', '.join(sorted(matches))) warn(msg)
def set_flags(self, **kwargs): """set my attribute flags by keyword. Views determine behavior with a few attributes (`block`, `track`, etc.). These attributes can be set all at once by name with this method. Parameters ---------- block : bool whether to wait for results track : bool whether to create a MessageTracker to allow the user to safely edit after arrays and buffers during non-copying sends. """ for name, value in iteritems(kwargs): if name not in self._flag_names: raise KeyError("Invalid name: %r"%name) else: setattr(self, name, value)
def _load_config(self, cfg, section_names=None, traits=None): """load traits from a Config object""" if traits is None: traits = self.traits(config=True) if section_names is None: section_names = self.section_names() my_config = self._find_my_config(cfg) # hold trait notifications until after all config has been loaded with self.hold_trait_notifications(): for name, config_value in iteritems(my_config): if name in traits: if isinstance(config_value, LazyConfigValue): # ConfigValue is a wrapper for using append / update on containers # without having to copy the initial value initial = getattr(self, name) config_value = config_value.get_value(initial) # We have to do a deepcopy here if we don't deepcopy the entire # config object. If we don't, a mutable config_value will be # shared by all instances, effectively making it a class attribute. setattr(self, name, deepcopy(config_value))
def assertSubset(self, da, db): """assert that da is a subset of db, ignoring self.ignored_keys. Called recursively on containers, ultimately comparing individual elements. """ if isinstance(da, dict): for k,v in iteritems(da): if k in self.ignored_keys: continue self.assertTrue(k in db) self.assertSubset(v, db[k]) elif isinstance(da, list): for a,b in zip(da, db): self.assertSubset(a,b) else: if isinstance(da, string_types) and isinstance(db, string_types): # pyfile is not sensitive to preserving leading/trailing # newlines in blocks through roundtrip da = da.strip('\n') db = db.strip('\n') self.assertEqual(da, db) return True
def assertSubset(self, da, db): """assert that da is a subset of db, ignoring self.ignored_keys. Called recursively on containers, ultimately comparing individual elements. """ if isinstance(da, dict): for k, v in iteritems(da): if k in self.ignored_keys: continue self.assertTrue(k in db) self.assertSubset(v, db[k]) elif isinstance(da, list): for a, b in zip(da, db): self.assertSubset(a, b) else: if isinstance(da, string_types) and isinstance(db, string_types): # pyfile is not sensitive to preserving leading/trailing # newlines in blocks through roundtrip da = da.strip('\n') db = db.strip('\n') self.assertEqual(da, db) return True
def __init__(self, msg_dict): dct = self.__dict__ for k, v in iteritems(dict(msg_dict)): if isinstance(v, dict): v = Message(v) dct[k] = v
def __iter__(self): return iter(iteritems(self.__dict__))
def _data_changed(self, name, old, new): for k,v in iteritems(new): assert mime_pat.match(k) nt.assert_is_instance(v, string_types)
def __init__(self, *args, **kwargs): dict.__init__(self, *args, **kwargs) self._reverse = dict() for key, value in iteritems(self): self._reverse[value] = key
def _flags_changed(self, name, old, new): """ensure flags dict is valid""" for key,value in iteritems(new): assert len(value) == 2, "Bad flag: %r:%s"%(key,value) assert isinstance(value[0], (dict, Config)), "Bad flag: %r:%s"%(key,value) assert isinstance(value[1], string_types), "Bad flag: %r:%s"%(key,value)