def connect(self, url, **kwargs): self.rpc = RestClient(url, **kwargs)
class Command(BaseCommand): help = "synchronize local tables to PeeringDB" def add_arguments(self, parser): parser.add_argument('-n', '--dry-run', action='store_true', default=False, help='enable extra debug output') parser.add_argument('--debug', action='store_true', default=False, help='enable extra debug output') parser.add_argument('--only', action='store', default=False, help='only process this table') parser.add_argument('--id', action='store', default=0, help='only process this id') parser.add_argument( '--limit', type=int, default=0, help="limit objects retrieved, retrieve all if 0 (default)") def handle(self, *args, **options): self.log = logging.getLogger('peeringdb.sync') kwargs = {} if settings.SYNC_USERNAME: kwargs['user'] = settings.SYNC_USERNAME kwargs['password'] = settings.SYNC_PASSWORD self.log.debug("syncing from %s", settings.SYNC_URL) self.connect(settings.SYNC_URL, **kwargs) # get models if limited by config only = options.get('only', settings.SYNC_ONLY) self.log.debug("only tables %s", only) pk = options.get('id', 0) tables = self.get_class_list(only) limit = options.get("limit", 0) self.dry_run = options.get('dry_run', False) # disable auto now for model in tables: for field in model._meta.fields: if field.name == "created": field.auto_now_add = False if field.name == "updated": field.auto_now = False self.sync(tables, pk, limit=limit) def connect(self, url, **kwargs): self.rpc = RestClient(url, **kwargs) def sync(self, tables, pk=0, **kwargs): for cls in tables: self.update_db(cls, self.get_objs(cls, pk=pk, **kwargs)) def get_class_list(self, only=None): tables = [] if only: for name in only: tables.append(get_model(name)) else: tables = django_peeringdb.models.all_models return tables def get_since(self, cls): upd = cls.handleref.last_change() if upd: return int(calendar.timegm(upd.timetuple())) return 0 def get_data(self, cls, since): return self.rpc.all(cls._handleref.tag, since=since) def get_objs(self, cls, **kwargs): pk = int(kwargs.pop('pk', 0)) if pk: self.log.debug("getting single id=%d", pk) data = self.rpc.all(cls._handleref.tag, id=pk, **kwargs) print("%s==%d %d changed" % (cls._handleref.tag, pk, len(data))) else: since = self.get_since(cls) data = self.rpc.all(cls._handleref.tag, since=since, **kwargs) print("%s last update %s %d changed" % (cls._handleref.tag, str(since), len(data))) return data def cls_from_tag(self, tag): tables = self.get_class_list() for cls in tables: if cls._handleref.tag == tag: return cls raise Exception("Unknown reftag: %s" % tag) def _sync(self, cls, row): """ Try to sync an object to the local database, in case of failure where a referenced object is not found, attempt to fetch said object from the REST api """ if self.dry_run: return try: sync.sync_obj(cls, row) except django.core.exceptions.ObjectDoesNotExist as e: # thrown by subquery on single row print(e) raise except django.core.exceptions.ValidationError as inst: # There were validation errors for field, errlst in inst.error_dict.items(): # check if it was a relationship that doesnt exist locally m = re.match(".+ with id (\d+) does not exist.+", str(errlst)) if m: print( "%s.%s not found locally, trying to fetch object... " % (field, m.group(1))) # fetch missing object r = self.rpc.get(field, int(m.group(1)), depth=0) # sync missing object self._sync(self.cls_from_tag(field), r[0]) else: raise # try to sync initial object once more sync.sync_obj(cls, row) def update_db(self, cls, data): print("data to be processed", len(data)) if not data: return for row in data: self._sync(cls, row)
class Command(BaseCommand): help = "synchronize local tables to PeeringDB" option_list = BaseCommand.option_list + ( make_option('-n', '--dry-run', action='store_true', default=False, help='enable extra debug output'), make_option('--debug', action='store_true', default=False, help='enable extra debug output'), make_option('--only', action='store', default=False, help='only process this ixp (id)'), ) # progress # quiet def handle(self, *args, **options): log = logging.getLogger('peeringdb.sync') kwargs = {} if settings.SYNC_USERNAME: kwargs['user'] = settings.SYNC_USERNAME kwargs['password'] = settings.SYNC_PASSWORD self.connect(settings.SYNC_URL, **kwargs) # get models if limited by config tables = self.get_class_list(settings.SYNC_ONLY) # tables = self.get_class_list(['organization']) #print(tables) self.sync(tables) def connect(self, url, **kwargs): self.rpc = RestClient(url, **kwargs) def sync(self, tables): for cls in tables: self.update_db(cls, self.get_objs(cls)) def get_class_list(self, only=None): tables = [] if only: for name in only: tables.append(get_model(name)) else: tables = django_peeringdb.models.all_models return tables def get_since(self, cls): upd = cls.handleref.last_change() if upd: return int(time.mktime(upd.timetuple())) return 0 def get_data(self, cls, since): return self.rpc.all(cls._handleref.tag, since=since) def get_objs(self, cls, **kwargs): since = self.get_since(cls) data = self.rpc.all(cls._handleref.tag, since=since, **kwargs) #data = self.rpc.all(cls._handleref.tag, since=since, limit=20) print("%s last update %s %d changed" % (cls._handleref.tag, str(since), len(data))) #print(data) return data def update_db(self, cls, data): print("data to be processed", len(data)) if not data: return for row in data: sync.sync_obj(cls, row)
class Command(BaseCommand): help = "synchronize local tables to PeeringDB" option_list = BaseCommand.option_list + ( make_option('-n', '--dry-run', action='store_true', default=False, help='enable extra debug output'), make_option('--debug', action='store_true', default=False, help='enable extra debug output'), make_option('--only', action='store', default=False, help='only process this ixp (id)'), ) # progress # quiet def handle(self, *args, **options): self.log = logging.getLogger('peeringdb.sync') kwargs = {} if settings.SYNC_USERNAME: kwargs['user'] = settings.SYNC_USERNAME kwargs['password'] = settings.SYNC_PASSWORD self.connect(settings.SYNC_URL, **kwargs) # get models if limited by config tables = self.get_class_list(settings.SYNC_ONLY) # disable auto now for model in tables: for field in model._meta.fields: if field.name == "created": field.auto_now_add = False if field.name == "updated": field.auto_now = False self.sync(tables) def connect(self, url, **kwargs): self.rpc = RestClient(url, **kwargs) def sync(self, tables): for cls in tables: self.update_db(cls, self.get_objs(cls)) def get_class_list(self, only=None): tables = [] if only: for name in only: tables.append(get_model(name)) else: tables = django_peeringdb.models.all_models return tables def get_since(self, cls): upd = cls.handleref.last_change() if upd: return int(time.mktime(upd.timetuple())) return 0 def get_data(self, cls, since): return self.rpc.all(cls._handleref.tag, since=since) def get_objs(self, cls, **kwargs): since = self.get_since(cls) data = self.rpc.all(cls._handleref.tag, since=since, **kwargs) #data = self.rpc.all(cls._handleref.tag, since=since, limit=20) print("%s last update %s %d changed" % (cls._handleref.tag, str(since), len(data))) #print(data) return data def cls_from_tag(self, tag): tables = self.get_class_list() for cls in tables: if cls._handleref.tag == tag: return cls raise Exception("Unknown reftag: %s" % tag) def _sync(self, cls, row): """ Try to sync an object to the local database, in case of failure where a referenced object is not found, attempt to fetch said object from the REST api """ try: sync.sync_obj(cls, row) except django.core.exceptions.ValidationError, inst: # There were validation errors for field, errlst in inst.error_dict.items(): # check if it was a relationship that doesnt exist locally m = re.match(".+ with id (\d+) does not exist.+", str(errlst)) if m: print("%s.%s not found locally, trying to fetch object... " % (field, m.group(1))) # fetch missing object r = self.rpc.get(field, int(m.group(1)), depth=0) # sync missing object self._sync(self.cls_from_tag(field), r[0]) else: raise # try to sync initial object once more sync.sync_obj(cls, row)
class Command(BaseCommand): help = "synchronize local tables to PeeringDB" option_list = BaseCommand.option_list + ( make_option('-n', '--dry-run', action='store_true', default=False, help='enable extra debug output'), make_option('--debug', action='store_true', default=False, help='enable extra debug output'), make_option('--only', action='store', default=False, help='only process this ixp (id)'), ) # progress # quiet def handle(self, *args, **options): self.log = logging.getLogger('peeringdb.sync') kwargs = {} if settings.SYNC_USERNAME: kwargs['user'] = settings.SYNC_USERNAME kwargs['password'] = settings.SYNC_PASSWORD self.connect(settings.SYNC_URL, **kwargs) # get models if limited by config tables = self.get_class_list(settings.SYNC_ONLY) # disable auto now for model in tables: for field in model._meta.fields: if field.name == "created": field.auto_now_add = False if field.name == "updated": field.auto_now = False self.sync(tables) def connect(self, url, **kwargs): self.rpc = RestClient(url, **kwargs) def sync(self, tables): for cls in tables: self.update_db(cls, self.get_objs(cls)) def get_class_list(self, only=None): tables = [] if only: for name in only: tables.append(get_model(name)) else: tables = django_peeringdb.models.all_models return tables def get_since(self, cls): upd = cls.handleref.last_change() if upd: return int(time.mktime(upd.timetuple())) return 0 def get_data(self, cls, since): return self.rpc.all(cls._handleref.tag, since=since) def get_objs(self, cls, **kwargs): since = self.get_since(cls) data = self.rpc.all(cls._handleref.tag, since=since, **kwargs) #data = self.rpc.all(cls._handleref.tag, since=since, limit=20) print("%s last update %s %d changed" % (cls._handleref.tag, str(since), len(data))) #print(data) return data def cls_from_tag(self, tag): tables = self.get_class_list() for cls in tables: if cls._handleref.tag == tag: return cls raise Exception("Unknown reftag: %s" % tag) def _sync(self, cls, row): """ Try to sync an object to the local database, in case of failure where a referenced object is not found, attempt to fetch said object from the REST api """ try: sync.sync_obj(cls, row) except django.core.exceptions.ValidationError, inst: # There were validation errors for field, errlst in inst.error_dict.items(): # check if it was a relationship that doesnt exist locally m = re.match(".+ with id (\d+) does not exist.+", str(errlst)) if m: print( "%s.%s not found locally, trying to fetch object... " % (field, m.group(1))) # fetch missing object r = self.rpc.get(field, int(m.group(1)), depth=0) # sync missing object self._sync(self.cls_from_tag(field), r[0]) else: raise # try to sync initial object once more sync.sync_obj(cls, row)