def __init__(self, cfg=None, **kwargs): """ Arguments: - cfg <dict>: dict of complete config options (see config.ClientSchema), by default loads from DEFAULT_CONFIG_DIR - url<str>: URL to connect to - user<str>: username to connect to api with - password<str>: password - timeout<float>: timeout to fail after """ if cfg is None: cfg = config.load_config() self.config = cfg orm_config = cfg['orm'] orm_name = orm_config['backend'] if not peeringdb.backend_initialized(): peeringdb.initialize_backend(orm_name, **orm_config) sync_config = cfg['sync'] # override config with kwargs munge.util.recursive_update(sync_config, kwargs) self._fetcher = Fetcher(**sync_config) self._updater = Updater(self._fetcher, **sync_config) self.update_all = self._updater.update_all self.update = self._updater.update self.update_where = self._updater.update_where tag_attrs = { res.tag: _Query(self, res) for res in resource.all_resources() } self._Tags = type('_Tags', (), tag_attrs) self.tags = self._Tags()
def test_dry_run(client_empty): client = peeringdb.PeeringDB(helper.CONFIG, dry_run=True) rs = all_resources() client.update_all(rs) # still empty? with pytest.raises(peeringdb.get_backend().object_missing_error()): client.get(Network, FIRST_NET)
def __init__(self, updater): self.updater = updater self.fetcher = updater._fetcher self._log = updater._log self._jobs = {R: {} for R in resource.all_resources()}, threading.Lock() self.disable_partial = True # TODO self.start_time = datetime.utcnow()
def update_all(self, rs=None, since=None): "Sync all objects for the relations rs (if None, sync all resources)" self._log.info("Updating resources: %s", ' '.join(r.tag for r in rs)) if rs is None: rs = resource.all_resources() ctx = self._ContextClass(self) for r in rs: self._atomic_update(lambda: ctx.sync_resource(r, since=since))
def test_reversed(client_empty): client = client_empty # sanity check for empty client B = peeringdb.get_backend() with pytest.raises(B.object_missing_error()): client.get(Network, FIRST_NET) rs = all_resources() rs = reversed(rs) client.update_all(rs)
def handle(self, *args, **options): if settings.RELEASE_ENV != "dev" and not settings.TUTORIAL_MODE: self.stdout.write( "Command can only be run on dev instances and instances " "with tutorial mode enabled") return if not options.get("commit"): self.stdout.write( "This will sync data from {url} to this instance, and will take " "roughly 20 minutes to complete on a fresh db. " "Run the command with `--commit` if you are sure you want " "to do this.".format(**options)) return # settings.USE_TZ = True db_settings = settings.DATABASES.get("default") config = { "sync": { "url": options.get("url") }, "orm": { "secret_key": settings.SECRET_KEY, "backend": "peeringdb_server", "migrate": False, "database": {k.lower(): v for k, v in db_settings.items()}, }, } apply_defaults(ClientSchema(), config) pre_save.disconnect(signals.addressmodel_save, sender=pdb_models.Facility) djpdb_models.all_models = [ pdb_models.Organization, pdb_models.Facility, pdb_models.Network, pdb_models.InternetExchange, pdb_models.InternetExchangeFacility, pdb_models.IXLan, pdb_models.IXLanPrefix, pdb_models.NetworkContact, pdb_models.NetworkFacility, pdb_models.NetworkIXLan, ] SUPPORTED_BACKENDS[ "peeringdb_server"] = "peeringdb_server.client_adaptor" client = Client(config) client.update_all(resource.all_resources(), since=None)
def update_all(self, rs=None, since=None): "Sync all objects for the relations rs (if None, sync all resources)" self._log.info("Updating resources: %s", " ".join(r.tag for r in (rs or []))) if rs is None: rs = resource.all_resources() ctx = self._ContextClass(self) for r in rs: with self._transaction(): ctx.sync_resource(r, since=since)
def test_auth(client_empty): with pytest.raises(ValueError): config = helper.CONFIG config["sync"]["user"] = "******" config["sync"]["password"] = "******" config["sync"]["api_key"] = "test" client = Client(config, dry_run=True) rs = all_resources() client.update_all(rs) client.get(Network, FIRST_NET)
def handle(config, verbose, quiet, init, **kwargs): rs = resource.all_resources() # if only: rs = [resource.get_resource(tag) for tag in only] loglvl = 1 + (verbose or 0) - (quiet or 0) if loglvl > 1: peeringdb._config_logs(logging.DEBUG) if loglvl < 1: peeringdb._config_logs(logging.WARNING) client = Client(config, **kwargs) # todo verify table schema if init: return if loglvl >= 0: print("Syncing to", config['sync']['url']) client.update_all(rs)
def test_nonunique(client_dup): client = client_dup # sanity check - do we actually have a duplicate swapdup = client.get(Network, 9) d = client._fetcher.fetch_latest(Network, FIRST_NET, 0, since=0) assert d[0][0]['name'] == swapdup.name # obj that doesn't exist remotely assert client.get(Network, 12) rs = all_resources() client.update_all(rs, since=0) assert client.get(Network, FIRST_NET) # remotely deleted dup should be gone B = peeringdb.get_backend() with pytest.raises(B.object_missing_error()): client.get(Network, 12)
def __init__(self, cfg=None, **kwargs): """ Arguments: - cfg <dict>: dict of complete config options (see config.ClientSchema), by default loads from DEFAULT_CONFIG_DIR - url<str>: URL to connect to - user<str>: username to connect to api with - password<str>: password - timeout<float>: timeout to fail after """ if cfg is None: cfg = config.load_config() self.config = cfg orm_config = cfg["orm"] orm_name = orm_config["backend"] if not backend_initialized(): initialize_backend(orm_name, **orm_config) sync_config = cfg["sync"] # override config with kwargs munge.util.recursive_update(sync_config, kwargs) self._fetcher = Fetcher(**sync_config) self._updater = Updater(self._fetcher, **sync_config) self.update_all = self._updater.update_all self.update = self._updater.update self.update_where = self._updater.update_where tag_res = OrderedDict( [(res.tag, _Query(self, res)) for res in resource.all_resources()] ) tag_attrs = { **tag_res, **{ "keys": lambda self: list(tag_res.keys()), "all": lambda self: list(tag_res.values()), }, } self._Tags = type("_Tags", (), tag_attrs) self.tags = self._Tags()
def sync(self): self.log_info(f"Syncing from {self.pdburl}") settings.USE_TZ = False config = { "sync": { "url": self.pdburl, "user": self.username, "password": self.password, "strip_tz": 1, "timeout": 0, "only": [], }, "orm": { "database": settings.DATABASES["default"], "backend": "django_peeringdb", "migrate": True, }, } initialize_backend("django_peeringdb", **config["orm"]) client = Client(config, **config) client.update_all(resource.all_resources())
def test_full(client_empty): client = get_client() rs = all_resources() client.update_all(rs)
def test_full(client_empty): client = get_client() rs = all_resources() client.update_all(rs) assert client.get(Network, FIRST_NET)