def update_nsi_dicts():
parser = argparse.ArgumentParser()
parser.add_argument('--all',
action='store_const',
const=True,
default=False)
parser.add_argument('--list',
action='store_const',
const=True,
default=False)
parser.add_argument('--host', default=None)
parser.add_argument('--port', default=None)
parser.add_argument('--url', default=url)
parser.add_argument('--key', default=key)
parser.add_argument('--db', default=nvesta_db)
args = parser.parse_args(sys.argv[1:])
mongo = pymongo.MongoClient(
host=args.host,
port=args.port,
)
RefBookRegistry.bootstrap(mongo[args.db])
client = NsiClient(url=args.url, user_key=args.key)
print('Retrieving data from NSI')
listed = list_nsi_dictionaries(client)
cooked = [(their['code'], their['name'],
safe_traverse(our, 'version',
default='?'), their.get('version', 0), their)
for our, their in ((safe_dict(desc['our']),
safe_dict(desc['their']))
for desc in listed)]
if args.list:
for code, name, our, their, nsi_dict in cooked:
print('%s %s %s->%s' % (code, name, our, their))
elif args.all:
to_update = [(code, name, our, their, nsi_dict)
for code, name, our, their, nsi_dict in cooked
if their != our]
for code, name, our, their, nsi_dict in to_update:
print('%s %s %s->%s' % (code, name, our, their))
print 'Updating (%s) %s...' % (code, name)
nsi_dict['version'] = their
import_nsi_dict(nsi_dict, client)
if not to_update:
print('Nothing to update')
def kladr_maintenance():
parser = argparse.ArgumentParser()
parser.add_argument('--host', default=None)
parser.add_argument('--port', default=None)
parser.add_argument('--db', default=nvesta_db)
args = parser.parse_args(sys.argv[1:])
mongo = pymongo.MongoClient(
host=args.host,
port=args.port,
)
RefBookRegistry.bootstrap(mongo[args.db])
from nvesta.library.nsi.data import kladr_maintenance
kladr_maintenance()
def autofix():
parser = argparse.ArgumentParser()
parser.add_argument('--all',
action='store_const',
const=True,
default=False)
parser.add_argument('--host', default=None)
parser.add_argument('--port', default=None)
parser.add_argument('--db', default=nvesta_db)
parser.add_argument('--list',
action='store_const',
const=True,
default=False)
parser.add_argument('version')
args = parser.parse_args(sys.argv[1:])
mongo = pymongo.MongoClient(
host=args.host,
port=args.port,
)
RefBookRegistry.bootstrap(mongo[args.db])
refbooks = RefBookRegistry.list()
to_fix = set(rb for rb in refbooks if not rb.meta.version)
if args.list:
for rb in to_fix:
print(u"%s -- %s" % (rb.code, rb.name))
elif args.all:
for rb in refbooks:
if rb in to_fix:
rb.fixate(args.version)
else:
rb._fix_meta(args.version)
else:
parser.print_help()
def migrate_from_v1():
def blocks(iterable, max_size=2000):
result = []
i = 0
for item in iterable:
result.append(item)
i += 1
if i >= max_size:
yield result
result = []
i = 0
if result:
yield result
def get_field_codes(collection):
codes = set()
for row in collection.find():
codes |= set(row.keys())
codes.discard('_id')
return codes
parser = argparse.ArgumentParser()
parser.add_argument('--all',
action='store_const',
const=True,
default=False)
parser.add_argument('--host', default=None)
parser.add_argument('--port', default=None)
parser.add_argument('--from-db', default='vesta')
parser.add_argument('--db', default=nvesta_db)
args = parser.parse_args(sys.argv[1:])
mongo = pymongo.MongoClient(
host=args.host,
port=args.port,
)
RefBookRegistry.bootstrap(mongo[args.db])
db_vesta = mongo[args.from_db]
RefBookRegistry.bootstrap(mongo[args.db])
processed_dicts = {'dict_names'}
processed_dicts.update(set(RefBookRegistry.names()))
for v_description in db_vesta['dict_names'].find():
code = v_description.get('code')
if code in processed_dicts:
continue
v_collection = db_vesta[code]
count = v_collection.count()
print 'Transferring', str(count).rjust(6), code
primary_link = None
linked = v_description.get('linked')
if linked:
primary_link = {
'left_field': linked['origin_field'],
'right_field': linked['linked_field'],
'right_rb': linked['collection']['code'],
}
rb = RefBookRegistry.create({
'code':
code,
'name':
v_description.get('name') or code,
'description':
v_description.get('description'),
'oid':
v_description.get('oid'),
'fields': [{
'key': fc,
'mandatory': fc == 'code',
} for fc in sorted(get_field_codes(v_collection))],
'primary_link':
primary_link,
'version':
safe_traverse(v_description, 'version', 'version', default=None)
})
for block in blocks(v_collection.find()):
rb.save_bulk(rb.record_factory(raw) for raw in block)
processed_dicts.add(code)
for code in db_vesta.collection_names(False):
if code in processed_dicts:
continue
v_collection = db_vesta[code]
count = v_collection.count()
print 'Transferring', str(count).rjust(6), code
rb = RefBookRegistry.create({
'code':
code,
'name':
code,
'description':
'',
'oid':
'',
'fields': [{
'key': fc,
'mandatory': fc == 'code',
} for fc in sorted(get_field_codes(v_collection))],
'primary_link':
None,
})
for block in blocks(v_collection.find()):
rb.save_bulk(rb.record_factory(raw) for raw in block)
RefBookRegistry.bootstrap(mongo[args.db])