def test_get(): # get before init with pytest.raises(peeringdb.BackendError): B = peeringdb.get_backend() peeringdb.initialize_backend('_mock') B = peeringdb.get_backend()
def fetch_latest(self, R, pk, depth, since=None): backend = peeringdb.get_backend() if since is None: since = backend.last_change(backend.get_concrete(R)) if since: since = since+1 return self._req(lambda: self.get(R.tag, pk, since=since, depth=depth))
def test_dry_run(client_empty): client = peeringdb.PeeringDB(helper.CONFIG, dry_run=True) rs = all_resources() client.update_all(rs) # still empty? with pytest.raises(peeringdb.get_backend().object_missing_error()): client.get(Network, FIRST_NET)
def _handle_duplicate(self, res, obj, dup_fields): B = get_backend() # Try to update conflicting object field = dup_fields[0] value = getattr(obj, field) try: dup = B.get_object_by(B.get_concrete(res), field, value) dup_id = dup.id dup.delete(hard=True) except B.object_missing_error(B.get_concrete(res)): # shouldn't happen raise Exception("internal error") self._log.debug("dup: %s = %s", field, value) def fetch_dup(): data, err = self.fetcher.fetch_latest(res, dup_id, 0) if isinstance(err, _fetch.NotFoundException): # case where the local duplicate has been deleted return {}, None return data, err fetch_job = _tasks.UpdateTask(self.fetch_and_index(fetch_dup), (res, dup_id)) return self.set_job((res, dup_id), self.update_after, (res, dup_id, 0, fetch_job))
def _transaction(self): try: with get_backend().atomic_transaction(): yield if self.dry_run: raise _CancelSave self._log.debug("Committing transaction") except _CancelSave: self._log.info("Transaction commit was cancelled (dry run)")
def test_reversed(client_empty): client = client_empty # sanity check for empty client B = peeringdb.get_backend() with pytest.raises(B.object_missing_error()): client.get(Network, FIRST_NET) rs = all_resources() rs = reversed(rs) client.update_all(rs)
def _atomic_update(self, sync_func): try: with get_backend().atomic_transaction(): sync_func() if self.dry_run: raise _CancelSave self._log.debug('Committing transaction') except _CancelSave: self._log.info('Transaction commit was cancelled (dry run)') pass
def initialize_object(B, res, row): """ Do a shallow initialization of an object Arguments: - row<dict>: dict of data like depth=1, i.e. many_refs are only ids """ B = get_backend() field_groups = FieldGroups(B.get_concrete(res)) try: obj = B.get_object(B.get_concrete(res), row['id']) except B.object_missing_error(B.get_concrete(res)): tbl = B.get_concrete(res) obj = tbl() # Set attributes, refs for fname, field in field_groups['scalars'].items(): value = row.get(fname, getattr(obj, fname, None)) value = B.convert_field(obj.__class__, fname, value) setattr(obj, fname, value) # _debug('res, row: %s, %s', res, row) # Already-fetched, and id-only refs fetched, dangling = defaultdict(dict), defaultdict(set) # To handle subrows that might be shallow (id) or deep (dict) def _handle_subrow(R, subrow): if isinstance(subrow, dict): pk = subrow['id'] fetched[R][pk] = subrow else: pk = subrow dangling[R].add(pk) return pk for fname, field in field_groups['one_refs'].items(): fieldres = _field_resource(B, B.get_concrete(res), fname) key = field.column subrow = row.get(key) if subrow is None: # e.g. use "org" if "org_id" is missing key = fname subrow = row[key] pk = _handle_subrow(fieldres, subrow) setattr(obj, key, pk) for fname, field in field_groups['many_refs'].items(): fieldres = _field_resource(B, B.get_concrete(res), fname) pks = [ _handle_subrow(fieldres, subrow) for subrow in row.get(fname, []) ] return obj, fetched, dangling
def fetch_all_latest(self, R, depth, params={}, since=None): backend = peeringdb.get_backend() if since is None: since = backend.last_change(backend.get_concrete(R)) if since: since = since + 1 params = { k: ",".join(map(str, v)) if isinstance(v, (list, tuple)) else v for k, v in params.items() } return self._req( lambda: self.all(R.tag, since=since, depth=depth, **params))
def handle(config, poids, output_format, depth, remote, **_): client = Client(config) for poid in poids: (tag, pk) = util.split_ref(poid) res = resource.get_resource(tag) B = peeringdb.get_backend() try: obj = client.get(res, pk) except B.object_missing_error(B.get_concrete(res)): if remote: obj = client.fetch(res, pk, depth)[0] else: print(f"Not found: {tag}-{pk}", file=sys.stderr) return 1 dump(obj, depth, sys.stdout)
def test_delete_all(client): from django.db import connection def _count(): # returns (int,) with connection.cursor() as c: return c.execute(helper.SQL_COUNT_ROWS).fetchone() ct = _count() assert ct[0] > 0, ct client = peeringdb.client.Client(helper.CONFIG) B = peeringdb.get_backend() B.delete_all() ct = _count() assert ct[0] == 0, ct
def reset_data(filename=None): client = peeringdb.client.Client(CONFIG) # Make sure db is empty B = peeringdb.get_backend() B.delete_all() if filename is None: print("Resetting database to empty") return print("Resetting database from", filename) # Insert our stuff # FIXME django-specific from django.db import connection path = _path.join(data_path(), filename) sql = open(path).read() with connection.cursor() as c: c.executescript(sql)
def test_nonunique(client_dup): client = client_dup # sanity check - do we actually have a duplicate swapdup = client.get(Network, 9) d = client._fetcher.fetch_latest(Network, FIRST_NET, 0, since=0) assert d[0][0]['name'] == swapdup.name # obj that doesn't exist remotely assert client.get(Network, 12) rs = all_resources() client.update_all(rs, since=0) assert client.get(Network, FIRST_NET) # remotely deleted dup should be gone B = peeringdb.get_backend() with pytest.raises(B.object_missing_error()): client.get(Network, 12)
def group_fields(concrete): "Partition a concrete's fields into groups based on type" GROUPS = ("scalars", "single_refs", "many_refs") B = get_backend() ret = {kind: {} for kind in GROUPS} fields = B.get_fields(concrete) for field in fields: name = field.name related, multiple = B.is_field_related(concrete, name) if related: if multiple: group = ret["many_refs"] else: group = ret["single_refs"] else: group = ret["scalars"] group[name] = field return ret
def __init__(self, concrete): backend = get_backend() kinds = {kind: {} for kind in FieldGroups.GROUPS} fields = backend.get_fields(concrete) for field in fields: name = field.name related, multiple = backend.is_field_related(concrete, name) if related: if multiple: group = kinds['many_refs'] else: group = kinds['one_refs'] else: group = kinds['scalars'] group[name] = field self._fields = kinds for kind, fs in kinds.items(): setattr(self, kind, lambda _fs=fs: _fs.items())
def sync(self): settings.USE_TZ = False config = { "sync": { "url": self.pdburl, "user": "", "password": "", "strip_tz": 1, "timeout": 0, "only": [], }, "orm": { "database": settings.DATABASES["default"], "backend": "django_peeringdb", "migrate": True, }, } initialize_backend("django_peeringdb", **config["orm"]) b = get_backend() client = Client(config, **config) print("Syncing from", config["sync"]["url"]) client.update_all(resource.all_resources())
def all(self, res): "Get resources using a filter condition" B = get_backend() return B.get_objects(B.get_concrete(res))
def dump(obj, depth, file): _init() if get_backend().is_concrete(type(obj)): obj = YamlWrap(obj, depth) yaml.safe_dump(obj, stream=file, default_flow_style=False)
def _init(): dumper = yaml.SafeDumper for cls in get_backend().CUSTOM_FIELDS: dumper.add_representer(cls, default_representer) dumper.add_representer(YamlWrap, represent_wrapped)
def sync_row(self, res, row, depth): B = get_backend() def _have(R, pks): have = B.get_objects(B.get_concrete(R), pks) return set(have.values_list("id", flat=True)) self._log.debug("sync_row(%s, %s, %s)", res.tag, row["id"], depth) if self.disable_partial and depth > 0: raise ValueError("depth > 0 sync is disabled") # Before attempting to set the related-object fields, ensure they are synced # Skip all ref'd objects that we already have, or have scheduled to update fetched, dangling = _sync.extract_relations(B, res, row) sync_jobs = [] for R, sub in fetched.items(): have = _have(R, set(sub.keys())) sync_jobs.extend( self.set_job((R, pk), self.sync_row, (R, subrow, depth - 1)) for pk, subrow in sub.items() if not (pk in have)) for R, pks in dangling.items(): pks = pks.difference(_have(R, pks)) pending = self.pending_jobs(R) needpks = [] for pk in pks: if pk in pending: sync_jobs.append(self.get_task((R, pk))) else: needpks.append(pk) if not needpks: continue def fetch_dangling(_R=R, _pks=needpks): return self.fetcher.fetch_all(_R, 0, dict(id__in=_pks)) fetch_job = _tasks.UpdateTask(self.fetch_and_index(fetch_dangling), (R, None)) sync_jobs.extend( self.set_job((R, pk), self.update_after, (R, pk, depth - 1, fetch_job)) for pk in pks) obj = _sync.initialize_object(B, res, row) _sync.set_scalars(B, res, obj, row) # Resolve refs and then save full object self._log.debug(" waiting for: %s", sync_jobs) yield _tasks.gather(sync_jobs) self._log.debug("sync_row(%s, %s, %s) (resumed)", res.tag, row["id"], depth) _sync.set_single_relations(B, res, obj, row) # Detect integrity gaps dup_fields, missing = _sync.clean_helper(B, obj, B.clean) if dup_fields: job = self._handle_duplicate(res, obj, dup_fields) sync_jobs.append(job) # ignore expected missing refs for R, pks in missing.items(): for pk in pks: if pk not in dangling[R]: # shouldn't happen raise RuntimeError("Unexpected missing relation", (R, pk)) _sync.patch_object(B, res, obj, self.updater.strip_tz) # Ignore new objects for consistency - TODO: test if obj.updated >= self.start_time: self._log.info("Ignoring object updated after sync began: (%s-%s)", res.tag, row["id"]) return # Preliminary save so this object exists for its dependents # XXX Doesn't work in MySQL, single-refs must exist if not self.updater.dry_run: B.save(obj) # B.get_objects(res, pk=row['id']).update() def finish(): _sync.set_single_relations(B, res, obj, row) _sync.set_many_relations(B, res, obj, row) if self.updater.dry_run: return B.clean(obj) B.save(obj) self.set_job((res, row["id"]), self.sync_row_finish, (res, row["id"], finish, sync_jobs))
def get(self, res, pk): "Get a resource instance by primary key (id)" B = get_backend() return B.get_object(B.get_concrete(res), pk)
def sync_row(self, res, row, depth): self._log.debug("sync_row(%s, %s, %s)", res, row['id'], depth) if self.disable_partial and depth > 0: raise ValueError('depth > 0 sync is disabled') B = get_backend() obj, fetched, dangling = _sync.initialize_object(B, res, row) # self._log.debug(' fetched: %s', fetched) # self._log.debug(' dangling: %s', dangling) def _have(R, pks): have = B.get_objects(B.get_concrete(R), pks) return set(have.values_list('id', flat=True)) # Before attempting to set the related-object fields, ensure they are synced # Skip all ref'd objects that we already have, or have scheduled to update sync_jobs = [] for R, sub in fetched.items(): have = _have(R, set(sub.keys())) sync_jobs.extend( self.set_job((R, pk), self.sync_row, (R, subrow, depth - 1)) for pk, subrow in sub.items() if not (pk in have)) for R, pks in dangling.items(): pks = pks.difference(_have(R, pks)) pending = self.pending_tasks(R) needpks = [] for pk in pks: if pk in pending: sync_jobs.append(self.get_task((R, pk))) else: needpks.append(pk) if not needpks: continue def fetch(_R=R, _pks=needpks): return self.fetcher.fetch_all_latest(_R, 0, dict(id__in=_pks)) fetch_job = _tasks.UpdateTask(self.fetch_and_index(fetch), (R, None)) sync_jobs.extend( self.set_job((R, pk), self.update_after, (R, pk, depth - 1, fetch_job)) for pk in pks) # Detect integrity gaps dup_fields, missing = _sync.clean_helper(B, obj, B.clean) if dup_fields: # For non-uniques, try to update conflicting object field = dup_fields[0] value = getattr(obj, field) try: dup = B.get_object_by(B.get_concrete(res), field, value) dup_id = dup.id dup.delete(hard=True) except B.object_missing_error( B.get_concrete(res)): # shouldn't happen raise Exception('internal error') self._log.debug('dup: %s = %s', field, value) def fetch(): data, err = self.fetcher.fetch_latest(res, dup_id, 0) if isinstance(err, _fetch.NotFoundException): # case where the local duplicate has been deleted return {}, None return data, err fetch_job = _tasks.UpdateTask(self.fetch_and_index(fetch), (res, dup_id)) job = self.set_job((res, dup_id), self.update_after, (res, dup_id, 0, fetch_job)) sync_jobs.append(job) if missing: # ignore expected missing refs for R, pks in missing.items(): for pk in pks: if pk not in dangling[R]: # shouldn't happen raise RuntimeError("Unexpected missing relation", (R, pk)) # Preliminary save so this object exists for its dependencies _sync.patch_object(B, res, obj, self.updater.strip_tz) # Ignore new objects for consistency - TODO: test if obj.updated >= self.start_time: self._log.info('Ignoring object updated after sync began: (%s-%s)', res.tag, row['id']) return if not self.updater.dry_run: B.save(obj) # Now, resolve refs and then save full object self._log.debug(' waiting for (%s-%s): %s', res.tag, row['id'], sync_jobs) yield _tasks.gather(sync_jobs) self._log.debug("sync_row(%s, %s, %s) (resumed)", res.tag, row['id'], depth) if self.updater.dry_run: return _sync.set_object_relations(B, res, obj, row) B.clean(obj) B.save(obj)
def handle(config, **_): Client(config) B = peeringdb.get_backend() B.delete_all()
def backend(self): return get_backend()