def load(self, truncate=False, skip=False): to_update = [s.table for s in Status.objects.filter(ver__ver__lt=self._version.ver)] for table_name in set(to_update) & set(FIAS_TABLES): try: table = self.tables[table_name] except KeyError: log.debug('Table `{0}` not found in archive'.format(table_name)) continue status = Status.objects.get(table=table.full_name) log.info('Updating table `{0}` from {1} to {2}...'.format(table.full_name, status.ver.ver, self._version.ver)) ldr = loader(table) try: ldr.load(truncate=False, update=True) except XMLSyntaxError as e: msg = 'XML file for table `{0}` is broken. Data not loaded!'.format(table.full_name) if skip: log.error(msg) else: raise BadArchiveError(msg) else: status.ver = self._version status.save() self._process_deleted_table(table_name)
def load(self, truncate=False, skip=False): to_update = [ s.table for s in Status.objects.filter(ver__ver__lt=self._version.ver) ] for table_name in set(to_update) & set(FIAS_TABLES): try: table = self.tables[table_name] except KeyError: log.debug( 'Table `{0}` not found in archive'.format(table_name)) continue status = Status.objects.get(table=table.full_name) log.info('Updating table `{0}` from {1} to {2}...'.format( table.full_name, status.ver.ver, self._version.ver)) ldr = loader(table) try: ldr.load(truncate=False, update=True) except XMLSyntaxError as e: msg = 'XML file for table `{0}` is broken. Data not loaded!'.format( table.full_name) if skip: log.error(msg) else: raise BadArchiveError(msg) else: status.ver = self._version status.save() self._process_deleted_table(table_name)
def update_data(path=None, version=None, skip=False, data_format='xml', limit=1000, tables=None, tempdir=None): tablelist = get_tablelist(path=path, version=version, data_format=data_format, tempdir=tempdir) for tbl in get_table_names(tables): # Пропускаем таблицы, которых нет в архиве if tbl not in tablelist.tables: continue st = Status.objects.get(table=tbl) if st.ver.ver >= tablelist.version.ver: log.info('Update of the table `{0}` is not needed [{1} <= {2}]. Skipping...'.format( tbl, st.ver.ver, tablelist.version.ver )) continue for table in tablelist.tables[tbl]: loader = TableUpdater(limit=limit) try: loader.load(tablelist=tablelist, table=table) except BadTableError as e: if skip: log.error(str(e)) else: raise st.ver = tablelist.version st.save()
def load_delta_xml(skip=False): min_version = Status.objects.filter(table__in=FIAS_TABLES).aggregate(Min('ver'))['ver__min'] if min_version is not None: for version in Version.objects.filter(ver__gt=min_version).order_by('ver'): try: arch = DeltaArchive(version=version) except BadArchiveError as e: if skip: log.error(e.message) else: raise else: arch.load(truncate=False, skip=skip) else: log.error('Not available. Please import the data before updating')
def load_delta_xml(skip=False): min_version = Status.objects.filter(table__in=FIAS_TABLES).aggregate( Min('ver'))['ver__min'] if min_version is not None: for version in Version.objects.filter( ver__gt=min_version).order_by('ver'): try: arch = DeltaArchive(version=version) except BadArchiveError as e: if skip: log.error(e.message) else: raise else: arch.load(truncate=False, skip=skip) else: log.error('Not available. Please import the data before updating')
def update_data(path=None, version=None, skip=False, data_format='xml', limit=1000, tables=None, tempdir=None): tablelist = get_tablelist(path=path, version=version, data_format=data_format, tempdir=tempdir) for tbl in get_table_names(tables): # Пропускаем таблицы, которых нет в архиве if tbl not in tablelist.tables: continue st = Status.objects.get(table=tbl) if st.ver.ver >= tablelist.version.ver: log.info( 'Update of the table `{0}` is not needed [{1} <= {2}]. Skipping...' .format(tbl, st.ver.ver, tablelist.version.ver)) continue for table in tablelist.tables[tbl]: loader = TableUpdater(limit=limit) try: loader.load(tablelist=tablelist, table=table) except BadTableError as e: if skip: log.error(str(e)) else: raise st.ver = tablelist.version st.save()