def iterkeys(self, raw=False): for doc in self.coll.find(no_cursor_timeout=False, projection=['spec']): if raw: yield doc['_id'], doc['spec'] else: yield Spec.dict2spec(doc['spec'])
def test_replace(self): for spec in self.instances: for field_name, field_spec in spec.get_fields(): spec_dict = spec.to_dict(include_all=True) if isinstance(field_spec, PrimitiveField): spec_dict[field_name] = replace_val = 1 else: self.assertRaises(InvalidSpecInstance, spec.replace, **{field_name: 1}) if isinstance(field_spec, BaseSpecField): replace_val = [ e for e in self.instances if field_spec.check_valid_value(e) ][0] spec_dict[field_name] = replace_val.to_dict( include_all=True) else: replace_val = [Spec()] spec_dict[field_name] = [ replace_val[0].to_dict(include_all=True) ] replaced_spec_dict = spec.replace(**{ field_name: replace_val }).to_dict(include_all=True) assert replaced_spec_dict == spec_dict
def _get(self, spec): if isinstance(spec, dict): # assume that if spec is a dictionary, then must be loadable spec = Spec.dict2spec(spec) if spec not in self.data: raise KeyError("Spec not found: {}".format(spec)) return self.data.get(spec)
def get(self, name): res = self._get_raw(name) if isinstance(res, dict) and 'type' in res: res = Spec.dict2spec(res) return res
def iteritems(self): for k, v in self.fdf.iteritems(): try: op = Spec.key2spec(k) except ValueError: op = GetOperation(series_name=k) yield op, v
def operations(self): res = [] for k in self.fdf.keys(): try: res.append(Spec.key2spec(k)) except ValueError: res.append(GetOperation(k)) return res
def _get_key(self, spec_or_dict): if isinstance(spec_or_dict, Spec): return spec_or_dict.key elif isinstance(spec_or_dict, dict): return Spec._dict2key(spec_or_dict) else: # assume it's an id return spec_or_dict
def test_reference(self): for spec in self.instances: for use_relative_paths in True, False: try: tmp_dir = mkdtemp() fnames = splitted_serialize( spec, tmp_dir, use_relative_paths=use_relative_paths) assert spec == Spec.from_yaml().load(fnames[spec]) finally: shutil.rmtree(tmp_dir)
def __init__(self, *args, **kwargs): super(FileDataStore, self).__init__(*args, **kwargs) if not os.path.exists(self.path): os.makedirs(self.path) conf_file = os.path.join(self.path, 'conf.yaml') if os.path.exists(conf_file): with open(conf_file) as f: conf = yaml.load(f) if 'serializer' not in conf: warnings.warn( "Old conf.yaml format. Please update it to the new format") conf_serializer = Spec.dict2spec(conf) conf_use_class_name = False else: conf_serializer = Spec.dict2spec(conf['serializer']) conf_use_class_name = conf.get('use_class_name', False) if conf_use_class_name != self.use_class_name: raise RuntimeError( 'This store was initialized with use_class_name = {} and now was instanced with {}' .format(conf_use_class_name, self.use_class_name)) if self.serializer is not None and self.serializer != conf_serializer: raise RuntimeError( "This store was initialized with this serializer:\n{}\n\n" + "But was now instanced with this one:\n{}".format( json.dumps(conf['serializer'], indent=2), json.dumps(self.serializer.to_dict(), indent=2))) self.serializer = conf_serializer self.use_class_name = conf_use_class_name else: if self.serializer is None: self.serializer = PickleSerializer() with open(conf_file, 'w') as f: yaml.dump( { 'serializer': self.serializer.to_dict(), 'use_class_name': self.use_class_name }, f)
def refactor(self, refactor_operation, out_data_store, permissive=False): # TODO: rewrite iterkeys, it's horrible! for id, doc in self.iterkeys(raw=True): try: refactored_doc = refactor_operation.bind(doc=doc).execute() spec = Spec.dict2spec(refactored_doc) out_data_store[spec] = self[id] except Exception, e: if permissive: warnings.warn(' '.join(e.args)) else: raise e
def iterkeys(self, raw=False): for subdir, _, _ in os.walk(self.path): key_fname = os.path.join(subdir, 'key') if not os.path.exists(key_fname): continue with open(key_fname) as f: key = f.read() if raw: yield subdir, Spec.key2dict(key) else: try: spec = Spec.key2spec(key) except Exception, e: # there might be a key that is not a valid json if len(e.args) > 0 and isinstance( e.args[0], basestring) and e.args[0].startswith( 'Unknown spec type'): raise e traceback.print_exc() warnings.warn('Unable to load spec key: {}'.format(key)) continue yield spec
def find_similar(self, spec): res = [] spec_dict = spec.to_dict() if isinstance(spec, Spec) else spec for id, other_spec_dict in self.iterkeys(raw=True): similarity = matching_fields(spec_dict, other_spec_dict) if similarity > 0: try: res.append((Spec.dict2spec(other_spec_dict), similarity)) except: # TODO: improve how exceptions are risen res.append((other_spec_dict, similarity)) res.sort(key=lambda x: -x[1]) return res
def test_get_by_id(self): for i, ds in enumerate(self.data_stores): for j, (id, doc) in enumerate(ds.iterkeys(raw=True)): # Not gonna perform this test on these kind of specs, I might even remove them in the future if isinstance(doc['type'], basestring) and '@' in doc['type']: continue spec = Spec.dict2spec(doc) v = ds[spec] assert ds[id] == v assert ds[doc] == v assert ds[ds.get_id(spec)] == v for spec in self.not_indexed_specs: self.assertRaises(KeyError, ds.get_id, spec) ds[spec] = 1 assert ds[ds.get_id(spec)] == 1
def test_create_refactor(self): specs = get_test_specs(easy=True) + get_test_data_stores() specs = [ e for e in specs if not isinstance(e, SpecB) and not isinstance(e, SpecWithDefault) ] specs.append(func(Spec())) specs.append(func(A())) specs.append(func(A(H(0)))) specs.append(func(A(H(1)))) for s1 in specs: for s2 in specs: s1_dict = s1.to_dict() s2_dict = s2.to_dict() diff = Diff.build(s1_dict, s2_dict) r = diff.create_refactor() refactored_dict = r.bind(s1_dict).execute() assert refactored_dict == s2_dict
def get_key(cls, spec): if isinstance(spec, Spec): return spec.key else: assert isinstance(spec, dict) return Spec._dict2key(spec)
def test_key(self): for spec in self.instances: assert spec.to_dict() == Spec.key2spec(spec.key).to_dict()
def test_type2spec_class(self): assert Spec == Spec.type2spec_class('fito:Spec') assert Spec == Spec.type2spec_class('fito.specs.base:Spec')
def _dict2spec(self, d): d = d.copy() return Spec.dict2spec(d)
def _parse_doc(self, doc): values = doc['values'] if self.use_gridfs: values = self.gridfs.get(values).read() spec = Spec.dict2spec(doc['spec']) return spec, values
def test_serialize(self): s = SpecA(0, verbose=True) assert 'verbose' not in s.to_dict() assert 'verbose' in s.to_dict(include_all=True) assert Spec.dict2spec(s.to_dict()) == s assert Spec.dict2spec(s.to_dict(include_all=True)) == s
def test_empty_load(self): assert SpecWithDefault() == Spec.dict2spec({'type': 'SpecWithDefault'})
def test_to_dict(self): for ds in self.data_stores: assert ds == Spec.dict2spec(ds.to_dict())