def test_get():
# get before init
with pytest.raises(peeringdb.BackendError):
B = peeringdb.get_backend()
peeringdb.initialize_backend('_mock')
B = peeringdb.get_backend()
def fetch_latest(self, R, pk, depth, since=None):
backend = peeringdb.get_backend()
if since is None:
since = backend.last_change(backend.get_concrete(R))
if since:
since = since+1
return self._req(lambda: self.get(R.tag, pk, since=since, depth=depth))
def test_dry_run(client_empty):
client = peeringdb.PeeringDB(helper.CONFIG, dry_run=True)
rs = all_resources()
client.update_all(rs)
# still empty?
with pytest.raises(peeringdb.get_backend().object_missing_error()):
client.get(Network, FIRST_NET)
def _handle_duplicate(self, res, obj, dup_fields):
B = get_backend()
# Try to update conflicting object
field = dup_fields[0]
value = getattr(obj, field)
try:
dup = B.get_object_by(B.get_concrete(res), field, value)
dup_id = dup.id
dup.delete(hard=True)
except B.object_missing_error(B.get_concrete(res)): # shouldn't happen
raise Exception("internal error")
self._log.debug("dup: %s = %s", field, value)
def fetch_dup():
data, err = self.fetcher.fetch_latest(res, dup_id, 0)
if isinstance(err, _fetch.NotFoundException):
# case where the local duplicate has been deleted
return {}, None
return data, err
fetch_job = _tasks.UpdateTask(self.fetch_and_index(fetch_dup),
(res, dup_id))
return self.set_job((res, dup_id), self.update_after,
(res, dup_id, 0, fetch_job))
def _transaction(self):
try:
with get_backend().atomic_transaction():
yield
if self.dry_run:
raise _CancelSave
self._log.debug("Committing transaction")
except _CancelSave:
self._log.info("Transaction commit was cancelled (dry run)")
def test_reversed(client_empty):
client = client_empty
# sanity check for empty client
B = peeringdb.get_backend()
with pytest.raises(B.object_missing_error()):
client.get(Network, FIRST_NET)
rs = all_resources()
rs = reversed(rs)
client.update_all(rs)
def _atomic_update(self, sync_func):
try:
with get_backend().atomic_transaction():
sync_func()
if self.dry_run:
raise _CancelSave
self._log.debug('Committing transaction')
except _CancelSave:
self._log.info('Transaction commit was cancelled (dry run)')
pass
def initialize_object(B, res, row):
"""
Do a shallow initialization of an object
Arguments:
- row<dict>: dict of data like depth=1, i.e. many_refs are only ids
"""
B = get_backend()
field_groups = FieldGroups(B.get_concrete(res))
try:
obj = B.get_object(B.get_concrete(res), row['id'])
except B.object_missing_error(B.get_concrete(res)):
tbl = B.get_concrete(res)
obj = tbl()
# Set attributes, refs
for fname, field in field_groups['scalars'].items():
value = row.get(fname, getattr(obj, fname, None))
value = B.convert_field(obj.__class__, fname, value)
setattr(obj, fname, value)
# _debug('res, row: %s, %s', res, row)
# Already-fetched, and id-only refs
fetched, dangling = defaultdict(dict), defaultdict(set)
# To handle subrows that might be shallow (id) or deep (dict)
def _handle_subrow(R, subrow):
if isinstance(subrow, dict):
pk = subrow['id']
fetched[R][pk] = subrow
else:
pk = subrow
dangling[R].add(pk)
return pk
for fname, field in field_groups['one_refs'].items():
fieldres = _field_resource(B, B.get_concrete(res), fname)
key = field.column
subrow = row.get(key)
if subrow is None: # e.g. use "org" if "org_id" is missing
key = fname
subrow = row[key]
pk = _handle_subrow(fieldres, subrow)
setattr(obj, key, pk)
for fname, field in field_groups['many_refs'].items():
fieldres = _field_resource(B, B.get_concrete(res), fname)
pks = [
_handle_subrow(fieldres, subrow) for subrow in row.get(fname, [])
]
return obj, fetched, dangling
def fetch_all_latest(self, R, depth, params={}, since=None):
backend = peeringdb.get_backend()
if since is None:
since = backend.last_change(backend.get_concrete(R))
if since:
since = since + 1
params = {
k: ",".join(map(str, v)) if isinstance(v, (list, tuple)) else v
for k, v in params.items()
}
return self._req(
lambda: self.all(R.tag, since=since, depth=depth, **params))
def handle(config, poids, output_format, depth, remote, **_):
client = Client(config)
for poid in poids:
(tag, pk) = util.split_ref(poid)
res = resource.get_resource(tag)
B = peeringdb.get_backend()
try:
obj = client.get(res, pk)
except B.object_missing_error(B.get_concrete(res)):
if remote:
obj = client.fetch(res, pk, depth)[0]
else:
print(f"Not found: {tag}-{pk}", file=sys.stderr)
return 1
dump(obj, depth, sys.stdout)
def test_delete_all(client):
from django.db import connection
def _count(): # returns (int,)
with connection.cursor() as c:
return c.execute(helper.SQL_COUNT_ROWS).fetchone()
ct = _count()
assert ct[0] > 0, ct
client = peeringdb.client.Client(helper.CONFIG)
B = peeringdb.get_backend()
B.delete_all()
ct = _count()
assert ct[0] == 0, ct
def reset_data(filename=None):
client = peeringdb.client.Client(CONFIG)
# Make sure db is empty
B = peeringdb.get_backend()
B.delete_all()
if filename is None:
print("Resetting database to empty")
return
print("Resetting database from", filename)
# Insert our stuff
# FIXME django-specific
from django.db import connection
path = _path.join(data_path(), filename)
sql = open(path).read()
with connection.cursor() as c:
c.executescript(sql)
def test_nonunique(client_dup):
client = client_dup
# sanity check - do we actually have a duplicate
swapdup = client.get(Network, 9)
d = client._fetcher.fetch_latest(Network, FIRST_NET, 0, since=0)
assert d[0][0]['name'] == swapdup.name
# obj that doesn't exist remotely
assert client.get(Network, 12)
rs = all_resources()
client.update_all(rs, since=0)
assert client.get(Network, FIRST_NET)
# remotely deleted dup should be gone
B = peeringdb.get_backend()
with pytest.raises(B.object_missing_error()):
client.get(Network, 12)
def group_fields(concrete):
"Partition a concrete's fields into groups based on type"
GROUPS = ("scalars", "single_refs", "many_refs")
B = get_backend()
ret = {kind: {} for kind in GROUPS}
fields = B.get_fields(concrete)
for field in fields:
name = field.name
related, multiple = B.is_field_related(concrete, name)
if related:
if multiple:
group = ret["many_refs"]
else:
group = ret["single_refs"]
else:
group = ret["scalars"]
group[name] = field
return ret
def __init__(self, concrete):
backend = get_backend()
kinds = {kind: {} for kind in FieldGroups.GROUPS}
fields = backend.get_fields(concrete)
for field in fields:
name = field.name
related, multiple = backend.is_field_related(concrete, name)
if related:
if multiple:
group = kinds['many_refs']
else:
group = kinds['one_refs']
else:
group = kinds['scalars']
group[name] = field
self._fields = kinds
for kind, fs in kinds.items():
setattr(self, kind, lambda _fs=fs: _fs.items())
def sync(self):
settings.USE_TZ = False
config = {
"sync": {
"url": self.pdburl,
"user": "",
"password": "",
"strip_tz": 1,
"timeout": 0,
"only": [],
},
"orm": {
"database": settings.DATABASES["default"],
"backend": "django_peeringdb",
"migrate": True,
},
}
initialize_backend("django_peeringdb", **config["orm"])
b = get_backend()
client = Client(config, **config)
print("Syncing from", config["sync"]["url"])
client.update_all(resource.all_resources())
def all(self, res):
"Get resources using a filter condition"
B = get_backend()
return B.get_objects(B.get_concrete(res))
def dump(obj, depth, file):
_init()
if get_backend().is_concrete(type(obj)):
obj = YamlWrap(obj, depth)
yaml.safe_dump(obj, stream=file, default_flow_style=False)
def _init():
dumper = yaml.SafeDumper
for cls in get_backend().CUSTOM_FIELDS:
dumper.add_representer(cls, default_representer)
dumper.add_representer(YamlWrap, represent_wrapped)
def sync_row(self, res, row, depth):
B = get_backend()
def _have(R, pks):
have = B.get_objects(B.get_concrete(R), pks)
return set(have.values_list("id", flat=True))
self._log.debug("sync_row(%s, %s, %s)", res.tag, row["id"], depth)
if self.disable_partial and depth > 0:
raise ValueError("depth > 0 sync is disabled")
# Before attempting to set the related-object fields, ensure they are synced
# Skip all ref'd objects that we already have, or have scheduled to update
fetched, dangling = _sync.extract_relations(B, res, row)
sync_jobs = []
for R, sub in fetched.items():
have = _have(R, set(sub.keys()))
sync_jobs.extend(
self.set_job((R, pk), self.sync_row, (R, subrow, depth - 1))
for pk, subrow in sub.items() if not (pk in have))
for R, pks in dangling.items():
pks = pks.difference(_have(R, pks))
pending = self.pending_jobs(R)
needpks = []
for pk in pks:
if pk in pending:
sync_jobs.append(self.get_task((R, pk)))
else:
needpks.append(pk)
if not needpks:
continue
def fetch_dangling(_R=R, _pks=needpks):
return self.fetcher.fetch_all(_R, 0, dict(id__in=_pks))
fetch_job = _tasks.UpdateTask(self.fetch_and_index(fetch_dangling),
(R, None))
sync_jobs.extend(
self.set_job((R, pk), self.update_after, (R, pk, depth - 1,
fetch_job))
for pk in pks)
obj = _sync.initialize_object(B, res, row)
_sync.set_scalars(B, res, obj, row)
# Resolve refs and then save full object
self._log.debug(" waiting for: %s", sync_jobs)
yield _tasks.gather(sync_jobs)
self._log.debug("sync_row(%s, %s, %s) (resumed)", res.tag, row["id"],
depth)
_sync.set_single_relations(B, res, obj, row)
# Detect integrity gaps
dup_fields, missing = _sync.clean_helper(B, obj, B.clean)
if dup_fields:
job = self._handle_duplicate(res, obj, dup_fields)
sync_jobs.append(job)
# ignore expected missing refs
for R, pks in missing.items():
for pk in pks:
if pk not in dangling[R]: # shouldn't happen
raise RuntimeError("Unexpected missing relation", (R, pk))
_sync.patch_object(B, res, obj, self.updater.strip_tz)
# Ignore new objects for consistency - TODO: test
if obj.updated >= self.start_time:
self._log.info("Ignoring object updated after sync began: (%s-%s)",
res.tag, row["id"])
return
# Preliminary save so this object exists for its dependents
# XXX Doesn't work in MySQL, single-refs must exist
if not self.updater.dry_run:
B.save(obj)
# B.get_objects(res, pk=row['id']).update()
def finish():
_sync.set_single_relations(B, res, obj, row)
_sync.set_many_relations(B, res, obj, row)
if self.updater.dry_run:
return
B.clean(obj)
B.save(obj)
self.set_job((res, row["id"]), self.sync_row_finish,
(res, row["id"], finish, sync_jobs))
def get(self, res, pk):
"Get a resource instance by primary key (id)"
B = get_backend()
return B.get_object(B.get_concrete(res), pk)
def sync_row(self, res, row, depth):
self._log.debug("sync_row(%s, %s, %s)", res, row['id'], depth)
if self.disable_partial and depth > 0:
raise ValueError('depth > 0 sync is disabled')
B = get_backend()
obj, fetched, dangling = _sync.initialize_object(B, res, row)
# self._log.debug(' fetched: %s', fetched)
# self._log.debug(' dangling: %s', dangling)
def _have(R, pks):
have = B.get_objects(B.get_concrete(R), pks)
return set(have.values_list('id', flat=True))
# Before attempting to set the related-object fields, ensure they are synced
# Skip all ref'd objects that we already have, or have scheduled to update
sync_jobs = []
for R, sub in fetched.items():
have = _have(R, set(sub.keys()))
sync_jobs.extend(
self.set_job((R, pk), self.sync_row, (R, subrow, depth - 1))
for pk, subrow in sub.items() if not (pk in have))
for R, pks in dangling.items():
pks = pks.difference(_have(R, pks))
pending = self.pending_tasks(R)
needpks = []
for pk in pks:
if pk in pending:
sync_jobs.append(self.get_task((R, pk)))
else:
needpks.append(pk)
if not needpks:
continue
def fetch(_R=R, _pks=needpks):
return self.fetcher.fetch_all_latest(_R, 0, dict(id__in=_pks))
fetch_job = _tasks.UpdateTask(self.fetch_and_index(fetch),
(R, None))
sync_jobs.extend(
self.set_job((R, pk), self.update_after, (R, pk, depth - 1,
fetch_job))
for pk in pks)
# Detect integrity gaps
dup_fields, missing = _sync.clean_helper(B, obj, B.clean)
if dup_fields:
# For non-uniques, try to update conflicting object
field = dup_fields[0]
value = getattr(obj, field)
try:
dup = B.get_object_by(B.get_concrete(res), field, value)
dup_id = dup.id
dup.delete(hard=True)
except B.object_missing_error(
B.get_concrete(res)): # shouldn't happen
raise Exception('internal error')
self._log.debug('dup: %s = %s', field, value)
def fetch():
data, err = self.fetcher.fetch_latest(res, dup_id, 0)
if isinstance(err, _fetch.NotFoundException):
# case where the local duplicate has been deleted
return {}, None
return data, err
fetch_job = _tasks.UpdateTask(self.fetch_and_index(fetch),
(res, dup_id))
job = self.set_job((res, dup_id), self.update_after,
(res, dup_id, 0, fetch_job))
sync_jobs.append(job)
if missing:
# ignore expected missing refs
for R, pks in missing.items():
for pk in pks:
if pk not in dangling[R]: # shouldn't happen
raise RuntimeError("Unexpected missing relation",
(R, pk))
# Preliminary save so this object exists for its dependencies
_sync.patch_object(B, res, obj, self.updater.strip_tz)
# Ignore new objects for consistency - TODO: test
if obj.updated >= self.start_time:
self._log.info('Ignoring object updated after sync began: (%s-%s)',
res.tag, row['id'])
return
if not self.updater.dry_run:
B.save(obj)
# Now, resolve refs and then save full object
self._log.debug(' waiting for (%s-%s): %s', res.tag, row['id'],
sync_jobs)
yield _tasks.gather(sync_jobs)
self._log.debug("sync_row(%s, %s, %s) (resumed)", res.tag, row['id'],
depth)
if self.updater.dry_run:
return
_sync.set_object_relations(B, res, obj, row)
B.clean(obj)
B.save(obj)
def handle(config, **_):
Client(config)
B = peeringdb.get_backend()
B.delete_all()
def backend(self):
return get_backend()
