def app_after_request(response):
"""
После каждого запроса надо инвалидировать реестр справочников, ибо неизвестно, что произошло с ними в
других процессах
"""
from nvesta.library.rb.registry import RefBookRegistry
RefBookRegistry.invalidate()
return response
def update_nsi_dicts():
parser = argparse.ArgumentParser()
parser.add_argument('--all',
action='store_const',
const=True,
default=False)
parser.add_argument('--list',
action='store_const',
const=True,
default=False)
parser.add_argument('--host', default=None)
parser.add_argument('--port', default=None)
parser.add_argument('--url', default=url)
parser.add_argument('--key', default=key)
parser.add_argument('--db', default=nvesta_db)
args = parser.parse_args(sys.argv[1:])
mongo = pymongo.MongoClient(
host=args.host,
port=args.port,
)
RefBookRegistry.bootstrap(mongo[args.db])
client = NsiClient(url=args.url, user_key=args.key)
print('Retrieving data from NSI')
listed = list_nsi_dictionaries(client)
cooked = [(their['code'], their['name'],
safe_traverse(our, 'version',
default='?'), their.get('version', 0), their)
for our, their in ((safe_dict(desc['our']),
safe_dict(desc['their']))
for desc in listed)]
if args.list:
for code, name, our, their, nsi_dict in cooked:
print('%s %s %s->%s' % (code, name, our, their))
elif args.all:
to_update = [(code, name, our, their, nsi_dict)
for code, name, our, their, nsi_dict in cooked
if their != our]
for code, name, our, their, nsi_dict in to_update:
print('%s %s %s->%s' % (code, name, our, their))
print 'Updating (%s) %s...' % (code, name)
nsi_dict['version'] = their
import_nsi_dict(nsi_dict, client)
if not to_update:
print('Nothing to update')
def list_nsi_dictionaries(nsi_client):
result = nsi_client.getRefbookList() or []
final = []
for nsi_dict_raw in result:
if nsi_dict_raw.key == 'errors':
raise ApiException(
500, u'Ошибка доступа к НСИ:\n%s' %
(u'\n'.join(u'%s: %s' % (item.key, item.value)
for item in nsi_dict_raw.children.item)))
nsi_dict = prepare_dictionary(nsi_dict_raw)
code = nsi_dict['code']
try:
# Пытаемся понять, какая версия справочника нынче актуальна
nsi_dict['version'] = prepare_dictionary(
nsi_client.getVersionList(code)[-1])['version']
except (IndexError, ValueError):
continue
meta = None
try:
rb = RefBookRegistry.get(code)
meta = rb.meta
except KeyError:
pass
final.append({
'their': nsi_dict,
'our': meta,
})
return final
def get_data_hs(code, field, field_value):
rb = RefBookRegistry.get(code)
find = {field: field_value}
doc = rb.find_one(find)
if rb.meta.oid:
# Работаем с НСИ справочником
data = doc
oid = rb.meta.oid
else:
# Переключаемся на НСИ справочник
link_meta = rb.meta.primary_link
linked_rb = rb.get_primary_linked_rb()
if not linked_rb or not linked_rb.meta.oid:
raise ApiException(404, u'Нет связанного справочника НСИ')
data = linked_rb.find_one(
{link_meta.right_field: doc[link_meta.left_field]})
oid = linked_rb.meta.oid
if data:
data = _prepare_hs_response(data, code)
else:
data = {}
return {
'oid': oid,
'data': data,
}
def as_json(self, edge=True, with_meta=False):
from nvesta.library.rb.registry import RefBookRegistry
if edge:
data = self.meta.edit or self.data
else:
data = self.data
result = {}
for description in self.rb.meta.fields:
key = description.key
value = data.get(key)
result[key] = value
if description.link:
add_key = description.link['key']
rb_code = description.link['code']
rb_field = description.link['linked_field']
as_list = description.link.get('list')
ref_book = RefBookRegistry.get(rb_code)
if as_list:
result[add_key] = ref_book.find({rb_field: value})
else:
result[add_key] = ref_book.find_one({rb_field: value})
if self._id:
result['_id'] = str(self._id)
if with_meta:
_meta = self.meta.__json__()
dirty = bool(_meta.pop('edit', False))
result['_meta'] = dict(_meta, dirty=dirty)
return result
def dictionary_put(rb_code, document_id):
data = force_json(request)
rb = RefBookRegistry.get(rb_code)
data['_id'] = bson.ObjectId(document_id)
record = rb.record_factory(data)
rb.save(record)
return document_id
def v1_get_document_by_field(rb_code, field, value):
if field == 'id':
value = int(value)
elif field == '_id':
value = bson.ObjectId(value)
rb = RefBookRegistry.get(rb_code)
return rb.find_one(field, value)
def find_data_hs(code):
data = force_json(request)
rb = RefBookRegistry.get(code)
ret_data = rb.meta.__json__()
ret_data['data'] = rb.find_one(prepare_find_params(data))
return ret_data
def rb_post():
"""
Создание нового справочника
:return:
"""
j = flask.request.get_json()
rb = RefBookRegistry.create(j)
return rb.meta
def kladr_maintenance():
with log_context(['kladr', 'maintenance']) as log:
log.log(u'Проверяем индексы на справочнике STR172 (улицы)')
rb = RefBookRegistry.get('STR172')
log.log(u' name')
rb.collection.create_index([('name', ASCENDING)])
log.log(u' identcode')
rb.collection.create_index([('identcode', ASCENDING)])
log.log(u' identparent')
rb.collection.create_index([('identparent', ASCENDING)])
log.log(u'Проверяем индексы на справочнике KLD172 (регионы)')
rb = RefBookRegistry.get('KLD172')
log.log(u' name + level')
rb.collection.create_index([('name', ASCENDING), ('level', ASCENDING)])
log.log(u' identparent')
rb.collection.create_index([('identcode', ASCENDING)])
def kladr_maintenance():
parser = argparse.ArgumentParser()
parser.add_argument('--host', default=None)
parser.add_argument('--port', default=None)
parser.add_argument('--db', default=nvesta_db)
args = parser.parse_args(sys.argv[1:])
mongo = pymongo.MongoClient(
host=args.host,
port=args.port,
)
RefBookRegistry.bootstrap(mongo[args.db])
from nvesta.library.nsi.data import kladr_maintenance
kladr_maintenance()
def get_primary_linked_rb(self):
"""
Получение справочника по первичной связке
@return: Справочник
@rtype: RefBook | NoneType
"""
if self.meta.primary_link.right_rb_code:
from nvesta.library.rb.registry import RefBookRegistry
return RefBookRegistry.get(self.meta.primary_link.right_rb_code)
def rb_get(rb_code):
"""
Получение метаданных справочника
:param rb_code:
:return:
"""
rb = RefBookRegistry.get(rb_code)
if rb_code:
return rb.meta
raise ApiException(404, 'Reference Book not found')
def rb_put(rb_code):
"""
Изменение метаданных справочника
:param rb_code:
:return:
"""
j = flask.request.get_json()
rb = RefBookRegistry.get(rb_code)
rb.meta.update(j)
rb.meta.reshape()
return rb.meta
def _get_parents(city):
rb = RefBookRegistry.get(CITY_CODE)
result = []
def _get_parent(c, f=False):
if c['identparent']:
_get_parent(rb.find_one({'identcode': c['identparent']}))
if not f:
result.append(_safe_city(c))
_get_parent(city, True)
return result
def rb_list_get():
"""
Список всех имеющихся справочников
:return:
"""
rb_list = RefBookRegistry.list()
result = [
d.meta
for d in rb_list
]
result.sort(key=lambda meta: meta.code)
return result
def v1_dictionary_post(rb_code):
data = force_json(request)
rb = RefBookRegistry.get(rb_code)
if isinstance(data, list):
id_list = []
for doc in data:
record = rb.record_factory(doc)
rb.save(record)
id_list.append(record.data.get('_id'))
return {'_id': id_list}
record = rb.record_factory(data)
rb.save(record)
return {'_id': record.data.get('_id')}
def search_city_country(value, limit=None):
rb = RefBookRegistry.get(CITY_CODE)
find = {
'is_actual': '1',
'$or': [{
'name': prepare_find_params(value)
}, {
'identcode': value
}]
}
cities = rb.find(find, 'level', limit)
result = _set_cities_parents(cities)
return result
def rb_record_get_id(rb_code, field, rec_id):
"""
Получение записи из справочника
:param rb_code:
:param rec_id:
:return:
"""
rb = RefBookRegistry.get(rb_code)
edge = bool(flask.request.args.get('edge', False))
with_meta = bool(flask.request.args.get('with-meta', False))
version = edge and EdgeVersion or flask.request.args.get('version') or None
record = rb.find_one(field, rec_id, version=version)
return record.as_json(edge, with_meta)
def autofix():
parser = argparse.ArgumentParser()
parser.add_argument('--all',
action='store_const',
const=True,
default=False)
parser.add_argument('--host', default=None)
parser.add_argument('--port', default=None)
parser.add_argument('--db', default=nvesta_db)
parser.add_argument('--list',
action='store_const',
const=True,
default=False)
parser.add_argument('version')
args = parser.parse_args(sys.argv[1:])
mongo = pymongo.MongoClient(
host=args.host,
port=args.port,
)
RefBookRegistry.bootstrap(mongo[args.db])
refbooks = RefBookRegistry.list()
to_fix = set(rb for rb in refbooks if not rb.meta.version)
if args.list:
for rb in to_fix:
print(u"%s -- %s" % (rb.code, rb.name))
elif args.all:
for rb in refbooks:
if rb in to_fix:
rb.fixate(args.version)
else:
rb._fix_meta(args.version)
else:
parser.print_help()
def rb_records_get(rb_code):
"""
Получение всех данных из справочника
:param rb_code:
:return:
"""
skip = flask.request.args.get('skip') or None
limit = flask.request.args.get('limit') or 100
edge = bool(flask.request.args.get('edge', False))
with_meta = bool(flask.request.args.get('with-meta', False))
version = edge and EdgeVersion or flask.request.args.get('version') or None
rb = RefBookRegistry.get(rb_code)
return [
record.as_json(edge, with_meta)
for record in rb.find({}, 'code', limit=limit, skip=skip, version=version)
]
def search_street(city_code, value=None, limit=None):
rb = RefBookRegistry.get(STREET_CODE)
find = {'identparent': city_code, 'is_actual': '1'}
if value:
prepared = prepare_find_params(value)
find.update({
'$or': [{
'name': prepared
}, {
'fulltype': prepared
}, {
'shorttype': prepared
}, {
'identcode': value
}]
})
result = rb.find(find, 'name', limit)
return list(result)
def refbooks_dict(self):
from nvesta.library.rb.registry import RefBookRegistry
return {
code: RefBookRegistry.get(code)
for code in self.codes if RefBookRegistry.get(code)
}
def migrate_from_v1():
def blocks(iterable, max_size=2000):
result = []
i = 0
for item in iterable:
result.append(item)
i += 1
if i >= max_size:
yield result
result = []
i = 0
if result:
yield result
def get_field_codes(collection):
codes = set()
for row in collection.find():
codes |= set(row.keys())
codes.discard('_id')
return codes
parser = argparse.ArgumentParser()
parser.add_argument('--all',
action='store_const',
const=True,
default=False)
parser.add_argument('--host', default=None)
parser.add_argument('--port', default=None)
parser.add_argument('--from-db', default='vesta')
parser.add_argument('--db', default=nvesta_db)
args = parser.parse_args(sys.argv[1:])
mongo = pymongo.MongoClient(
host=args.host,
port=args.port,
)
RefBookRegistry.bootstrap(mongo[args.db])
db_vesta = mongo[args.from_db]
RefBookRegistry.bootstrap(mongo[args.db])
processed_dicts = {'dict_names'}
processed_dicts.update(set(RefBookRegistry.names()))
for v_description in db_vesta['dict_names'].find():
code = v_description.get('code')
if code in processed_dicts:
continue
v_collection = db_vesta[code]
count = v_collection.count()
print 'Transferring', str(count).rjust(6), code
primary_link = None
linked = v_description.get('linked')
if linked:
primary_link = {
'left_field': linked['origin_field'],
'right_field': linked['linked_field'],
'right_rb': linked['collection']['code'],
}
rb = RefBookRegistry.create({
'code':
code,
'name':
v_description.get('name') or code,
'description':
v_description.get('description'),
'oid':
v_description.get('oid'),
'fields': [{
'key': fc,
'mandatory': fc == 'code',
} for fc in sorted(get_field_codes(v_collection))],
'primary_link':
primary_link,
'version':
safe_traverse(v_description, 'version', 'version', default=None)
})
for block in blocks(v_collection.find()):
rb.save_bulk(rb.record_factory(raw) for raw in block)
processed_dicts.add(code)
for code in db_vesta.collection_names(False):
if code in processed_dicts:
continue
v_collection = db_vesta[code]
count = v_collection.count()
print 'Transferring', str(count).rjust(6), code
rb = RefBookRegistry.create({
'code':
code,
'name':
code,
'description':
'',
'oid':
'',
'fields': [{
'key': fc,
'mandatory': fc == 'code',
} for fc in sorted(get_field_codes(v_collection))],
'primary_link':
None,
})
for block in blocks(v_collection.find()):
rb.save_bulk(rb.record_factory(raw) for raw in block)
RefBookRegistry.bootstrap(mongo[args.db])
def rb_version_fix(rb_code):
version = flask.request.args.get('version') or bail_out(ApiException(400, 'Need "version" argument'))
rb = RefBookRegistry.get(rb_code)
rb.fixate(version)
return rb.meta
def rb_diff(rb_code):
rb = RefBookRegistry.get(rb_code)
return [
record.as_json(True, True)
for record in rb.find({}, version=DiffVersion)
]
def rb_record_post(rb_code):
rb = RefBookRegistry.get(rb_code)
record = rb.record_factory()
record.update(flask.request.get_json())
record.save()
return record.as_json(True, True)
def rb_record_delete(rb_code, field, rec_id):
rb = RefBookRegistry.get(rb_code)
record = rb.find_one(field, rec_id, version=EdgeVersion)
record.delete()
record.save()
return record.as_json(True, True)
def rb_record_put_id(rb_code, field, rec_id):
rb = RefBookRegistry.get(rb_code)
record = rb.find_one(field, rec_id, version=EdgeVersion)
record.update(flask.request.get_json())
record.save()
return record.as_json(True, True)
def refbooks_metas(self):
from nvesta.library.rb.registry import RefBookRegistry
return [
RefBookRegistry.get(code).meta for code in self.codes
if RefBookRegistry.get(code)
]
