def __init__(self, cfg=None, **kwargs):
"""
Arguments:
- cfg <dict>: dict of complete config options (see config.ClientSchema),
by default loads from DEFAULT_CONFIG_DIR
- url<str>: URL to connect to
- user<str>: username to connect to api with
- password<str>: password
- timeout<float>: timeout to fail after
"""
if cfg is None:
cfg = config.load_config()
self.config = cfg
orm_config = cfg['orm']
orm_name = orm_config['backend']
if not peeringdb.backend_initialized():
peeringdb.initialize_backend(orm_name, **orm_config)
sync_config = cfg['sync']
# override config with kwargs
munge.util.recursive_update(sync_config, kwargs)
self._fetcher = Fetcher(**sync_config)
self._updater = Updater(self._fetcher, **sync_config)
self.update_all = self._updater.update_all
self.update = self._updater.update
self.update_where = self._updater.update_where
tag_attrs = {
res.tag: _Query(self, res)
for res in resource.all_resources()
}
self._Tags = type('_Tags', (), tag_attrs)
self.tags = self._Tags()
def test_dry_run(client_empty):
client = peeringdb.PeeringDB(helper.CONFIG, dry_run=True)
rs = all_resources()
client.update_all(rs)
# still empty?
with pytest.raises(peeringdb.get_backend().object_missing_error()):
client.get(Network, FIRST_NET)
def __init__(self, updater):
self.updater = updater
self.fetcher = updater._fetcher
self._log = updater._log
self._jobs = {R: {}
for R in resource.all_resources()}, threading.Lock()
self.disable_partial = True # TODO
self.start_time = datetime.utcnow()
def update_all(self, rs=None, since=None):
"Sync all objects for the relations rs (if None, sync all resources)"
self._log.info("Updating resources: %s", ' '.join(r.tag for r in rs))
if rs is None:
rs = resource.all_resources()
ctx = self._ContextClass(self)
for r in rs:
self._atomic_update(lambda: ctx.sync_resource(r, since=since))
def test_reversed(client_empty):
client = client_empty
# sanity check for empty client
B = peeringdb.get_backend()
with pytest.raises(B.object_missing_error()):
client.get(Network, FIRST_NET)
rs = all_resources()
rs = reversed(rs)
client.update_all(rs)
def handle(self, *args, **options):
if settings.RELEASE_ENV != "dev" and not settings.TUTORIAL_MODE:
self.stdout.write(
"Command can only be run on dev instances and instances "
"with tutorial mode enabled")
return
if not options.get("commit"):
self.stdout.write(
"This will sync data from {url} to this instance, and will take "
"roughly 20 minutes to complete on a fresh db. "
"Run the command with `--commit` if you are sure you want "
"to do this.".format(**options))
return
# settings.USE_TZ = True
db_settings = settings.DATABASES.get("default")
config = {
"sync": {
"url": options.get("url")
},
"orm": {
"secret_key": settings.SECRET_KEY,
"backend": "peeringdb_server",
"migrate": False,
"database": {k.lower(): v
for k, v in db_settings.items()},
},
}
apply_defaults(ClientSchema(), config)
pre_save.disconnect(signals.addressmodel_save,
sender=pdb_models.Facility)
djpdb_models.all_models = [
pdb_models.Organization,
pdb_models.Facility,
pdb_models.Network,
pdb_models.InternetExchange,
pdb_models.InternetExchangeFacility,
pdb_models.IXLan,
pdb_models.IXLanPrefix,
pdb_models.NetworkContact,
pdb_models.NetworkFacility,
pdb_models.NetworkIXLan,
]
SUPPORTED_BACKENDS[
"peeringdb_server"] = "peeringdb_server.client_adaptor"
client = Client(config)
client.update_all(resource.all_resources(), since=None)
def update_all(self, rs=None, since=None):
"Sync all objects for the relations rs (if None, sync all resources)"
self._log.info("Updating resources: %s",
" ".join(r.tag for r in (rs or [])))
if rs is None:
rs = resource.all_resources()
ctx = self._ContextClass(self)
for r in rs:
with self._transaction():
ctx.sync_resource(r, since=since)
def test_auth(client_empty):
with pytest.raises(ValueError):
config = helper.CONFIG
config["sync"]["user"] = "******"
config["sync"]["password"] = "******"
config["sync"]["api_key"] = "test"
client = Client(config, dry_run=True)
rs = all_resources()
client.update_all(rs)
client.get(Network, FIRST_NET)
def handle(config, verbose, quiet, init, **kwargs):
rs = resource.all_resources()
# if only: rs = [resource.get_resource(tag) for tag in only]
loglvl = 1 + (verbose or 0) - (quiet or 0)
if loglvl > 1:
peeringdb._config_logs(logging.DEBUG)
if loglvl < 1:
peeringdb._config_logs(logging.WARNING)
client = Client(config, **kwargs)
# todo verify table schema
if init: return
if loglvl >= 0:
print("Syncing to", config['sync']['url'])
client.update_all(rs)
def test_nonunique(client_dup):
client = client_dup
# sanity check - do we actually have a duplicate
swapdup = client.get(Network, 9)
d = client._fetcher.fetch_latest(Network, FIRST_NET, 0, since=0)
assert d[0][0]['name'] == swapdup.name
# obj that doesn't exist remotely
assert client.get(Network, 12)
rs = all_resources()
client.update_all(rs, since=0)
assert client.get(Network, FIRST_NET)
# remotely deleted dup should be gone
B = peeringdb.get_backend()
with pytest.raises(B.object_missing_error()):
client.get(Network, 12)
def __init__(self, cfg=None, **kwargs):
"""
Arguments:
- cfg <dict>: dict of complete config options (see config.ClientSchema),
by default loads from DEFAULT_CONFIG_DIR
- url<str>: URL to connect to
- user<str>: username to connect to api with
- password<str>: password
- timeout<float>: timeout to fail after
"""
if cfg is None:
cfg = config.load_config()
self.config = cfg
orm_config = cfg["orm"]
orm_name = orm_config["backend"]
if not backend_initialized():
initialize_backend(orm_name, **orm_config)
sync_config = cfg["sync"]
# override config with kwargs
munge.util.recursive_update(sync_config, kwargs)
self._fetcher = Fetcher(**sync_config)
self._updater = Updater(self._fetcher, **sync_config)
self.update_all = self._updater.update_all
self.update = self._updater.update
self.update_where = self._updater.update_where
tag_res = OrderedDict(
[(res.tag, _Query(self, res)) for res in resource.all_resources()]
)
tag_attrs = {
**tag_res,
**{
"keys": lambda self: list(tag_res.keys()),
"all": lambda self: list(tag_res.values()),
},
}
self._Tags = type("_Tags", (), tag_attrs)
self.tags = self._Tags()
def sync(self):
self.log_info(f"Syncing from {self.pdburl}")
settings.USE_TZ = False
config = {
"sync": {
"url": self.pdburl,
"user": self.username,
"password": self.password,
"strip_tz": 1,
"timeout": 0,
"only": [],
},
"orm": {
"database": settings.DATABASES["default"],
"backend": "django_peeringdb",
"migrate": True,
},
}
initialize_backend("django_peeringdb", **config["orm"])
client = Client(config, **config)
client.update_all(resource.all_resources())
def test_full(client_empty):
client = get_client()
rs = all_resources()
client.update_all(rs)
def test_full(client_empty):
client = get_client()
rs = all_resources()
client.update_all(rs)
assert client.get(Network, FIRST_NET)
