def load(self, truncate=False, skip=False):
to_update = [s.table for s in Status.objects.filter(ver__ver__lt=self._version.ver)]
for table_name in set(to_update) & set(FIAS_TABLES):
try:
table = self.tables[table_name]
except KeyError:
log.debug('Table `{0}` not found in archive'.format(table_name))
continue
status = Status.objects.get(table=table.full_name)
log.info('Updating table `{0}` from {1} to {2}...'.format(table.full_name,
status.ver.ver,
self._version.ver))
ldr = loader(table)
try:
ldr.load(truncate=False, update=True)
except XMLSyntaxError as e:
msg = 'XML file for table `{0}` is broken. Data not loaded!'.format(table.full_name)
if skip:
log.error(msg)
else:
raise BadArchiveError(msg)
else:
status.ver = self._version
status.save()
self._process_deleted_table(table_name)
def load(self, truncate=False, skip=False):
to_update = [
s.table
for s in Status.objects.filter(ver__ver__lt=self._version.ver)
]
for table_name in set(to_update) & set(FIAS_TABLES):
try:
table = self.tables[table_name]
except KeyError:
log.debug(
'Table `{0}` not found in archive'.format(table_name))
continue
status = Status.objects.get(table=table.full_name)
log.info('Updating table `{0}` from {1} to {2}...'.format(
table.full_name, status.ver.ver, self._version.ver))
ldr = loader(table)
try:
ldr.load(truncate=False, update=True)
except XMLSyntaxError as e:
msg = 'XML file for table `{0}` is broken. Data not loaded!'.format(
table.full_name)
if skip:
log.error(msg)
else:
raise BadArchiveError(msg)
else:
status.ver = self._version
status.save()
self._process_deleted_table(table_name)
def update_data(path=None, version=None, skip=False, data_format='xml', limit=1000, tables=None, tempdir=None):
tablelist = get_tablelist(path=path, version=version, data_format=data_format, tempdir=tempdir)
for tbl in get_table_names(tables):
# Пропускаем таблицы, которых нет в архиве
if tbl not in tablelist.tables:
continue
st = Status.objects.get(table=tbl)
if st.ver.ver >= tablelist.version.ver:
log.info('Update of the table `{0}` is not needed [{1} <= {2}]. Skipping...'.format(
tbl, st.ver.ver, tablelist.version.ver
))
continue
for table in tablelist.tables[tbl]:
loader = TableUpdater(limit=limit)
try:
loader.load(tablelist=tablelist, table=table)
except BadTableError as e:
if skip:
log.error(str(e))
else:
raise
st.ver = tablelist.version
st.save()
def load_delta_xml(skip=False):
min_version = Status.objects.filter(table__in=FIAS_TABLES).aggregate(Min('ver'))['ver__min']
if min_version is not None:
for version in Version.objects.filter(ver__gt=min_version).order_by('ver'):
try:
arch = DeltaArchive(version=version)
except BadArchiveError as e:
if skip:
log.error(e.message)
else:
raise
else:
arch.load(truncate=False, skip=skip)
else:
log.error('Not available. Please import the data before updating')
def load_delta_xml(skip=False):
min_version = Status.objects.filter(table__in=FIAS_TABLES).aggregate(
Min('ver'))['ver__min']
if min_version is not None:
for version in Version.objects.filter(
ver__gt=min_version).order_by('ver'):
try:
arch = DeltaArchive(version=version)
except BadArchiveError as e:
if skip:
log.error(e.message)
else:
raise
else:
arch.load(truncate=False, skip=skip)
else:
log.error('Not available. Please import the data before updating')
def update_data(path=None,
version=None,
skip=False,
data_format='xml',
limit=1000,
tables=None,
tempdir=None):
tablelist = get_tablelist(path=path,
version=version,
data_format=data_format,
tempdir=tempdir)
for tbl in get_table_names(tables):
# Пропускаем таблицы, которых нет в архиве
if tbl not in tablelist.tables:
continue
st = Status.objects.get(table=tbl)
if st.ver.ver >= tablelist.version.ver:
log.info(
'Update of the table `{0}` is not needed [{1} <= {2}]. Skipping...'
.format(tbl, st.ver.ver, tablelist.version.ver))
continue
for table in tablelist.tables[tbl]:
loader = TableUpdater(limit=limit)
try:
loader.load(tablelist=tablelist, table=table)
except BadTableError as e:
if skip:
log.error(str(e))
else:
raise
st.ver = tablelist.version
st.save()
