def test_file_is_empty(self):
file_path = os.path.join(self.DATA_PATH, 'test_file')
with open(file_path, 'w'):
pass
self.assertEqual(file_is_empty(file_path), True)
with open(file_path, 'w') as test_file:
test_file.write('some_data')
self.assertEqual(file_is_empty(file_path), False)
os.remove(file_path)
def update_json_files(self):
while True:
now = get_now()
registry_last_modified = get_file_last_modified(self.registry_xml)
last_check_dict = {
'inn': self.inn_json_last_check,
'atc': self.atc_json_last_check,
'inn2atc': self.inn2atc_json_last_check,
'atc2inn': self.atc2inn_json_last_check
}
files_dict = {
'inn': self.inn_json,
'atc': self.atc_json,
'inn2atc': self.inn2atc_json,
'atc2inn': self.atc2inn_json
}
eq_valid_names = (('mnn', 'inn'), ('atc1', 'atc'),
('inn2atc', 'inn2atc'), ('atc2inn', 'atc2inn'))
for name, eq_name in eq_valid_names:
if file_is_empty(files_dict.get(eq_name)):
self.update_json(name)
else:
if now.date() >= registry_last_modified.date():
last_check = last_check_dict.get(eq_name)
if not last_check or registry_last_modified.date(
) > last_check.date():
self.update_json(name)
gevent.sleep(self.json_files_delay)
def test_save_registry(self):
self.worker = Registry(config.get('source_registry'),
config.get('time_update_at'),
config.get('delay'),
config.get('registry_delay'),
config.get('services_not_available'))
self.worker.DATA_PATH = self.DATA_PATH
self.assertTrue(
file_exists(os.path.join(self.worker.DATA_PATH, 'registry.xml')))
self.assertTrue(
file_is_empty(os.path.join(self.worker.DATA_PATH, 'registry.xml')))
self.worker.save_registry(
'<?xml version="1.0" encoding="Windows-1251"?>')
self.assertFalse(
file_is_empty(os.path.join(self.worker.DATA_PATH, 'registry.xml')))
def test_get_registry(self):
self.worker = Registry(config.get('source_registry'),
config.get('time_update_at'),
config.get('delay'),
config.get('registry_delay'),
config.get('services_not_available'))
self.worker.DATA_PATH = self.DATA_PATH
self.assertTrue(
file_is_empty(os.path.join(self.worker.DATA_PATH, 'registry.xml')))
self.worker.source_registry = 'some_url'
registry = self.worker.get_registry()
self.assertEqual(registry, None)
self.assertTrue(
file_is_empty(os.path.join(self.worker.DATA_PATH, 'registry.xml')))
self.worker.source_registry = 'http://www.drlz.com.ua/'
self.worker.get_registry()
self.assertFalse(
file_is_empty(os.path.join(self.worker.DATA_PATH, 'registry.xml')))
def update_local_registry(self):
while self.INFINITY_LOOP:
now = get_now()
last_modified = get_file_last_modified(self.registry_xml)
conditions = (now.date() > last_modified.date()
and self.registry_update_time,
file_is_empty(self.registry_xml))
if any(conditions):
logger.info('Update local registry file...',
extra=journal_context(
{'MESSAGE_ID': BRIDGE_REGISTER}, {}))
self.get_registry()
else:
logger.info('Registry file is updated.',
extra=journal_context(
{'MESSAGE_ID': BRIDGE_REGISTER}, {}))
gevent.sleep(self.registry_delay)
def _update_cache(self, name):
logger.info('Update cache for {}...'.format(name),
extra=journal_context({'MESSAGE_ID': BRIDGE_INFO}, {}))
file_path = os.path.join(self.DATA_PATH, '{}.json'.format(name))
with open(file_path, 'r') as f:
if not file_is_empty(file_path):
data = json.loads(f.read())
self.db.remove(name)
self.db.put(name, data)
logger.info('Cache updated for {}.'.format(name),
extra=journal_context({'MESSAGE_ID': BRIDGE_CACHE},
{}))
else:
logger.warn(
'Cache not updated for {}. Registry file is empty.'.format(
name),
extra=journal_context({'MESSAGE_ID': BRIDGE_CACHE}, {}))
def test_files_init(self):
self.worker = MedicinesRegistryBridge(config)
self.worker.BASE_DIR = self.BASE_DIR
self.DATA_PATH = os.path.join(self.worker.BASE_DIR, 'data')
self.assertFalse(os.path.exists(self.DATA_PATH))
self.worker._files_init()
self.registry_xml = os.path.join(self.DATA_PATH, 'registry.xml')
self.inn_json = os.path.join(self.DATA_PATH, 'inn.json')
self.atc_json = os.path.join(self.DATA_PATH, 'atc.json')
self.inn2atc_json = os.path.join(self.DATA_PATH, 'inn2atc.json')
self.atc2inn_json = os.path.join(self.DATA_PATH, 'atc2inn.json')
self.DATA_PATH = os.path.join(self.worker.BASE_DIR, 'data')
self.assertTrue(os.path.exists(self.DATA_PATH))
self.assertTrue(file_exists(self.registry_xml))
self.assertTrue(file_exists(self.inn_json))
self.assertTrue(file_exists(self.atc_json))
self.assertTrue(file_exists(self.inn2atc_json))
self.assertTrue(file_exists(self.atc2inn_json))
self.assertTrue(file_is_empty(self.registry_xml))
rm_dir(self.DATA_PATH)
def update_json(self, name):
logger.info('Update local {}.json file...'.format(
self.eq_valid_names.get(name)),
extra=journal_context({'MESSAGE_ID': BRIDGE_INFO}, {}))
file_path = os.path.join(
self.DATA_PATH, '{}.json'.format(self.eq_valid_names.get(name)))
if file_is_empty(self.registry_xml):
logger.info(
'Local {}.json file not updated. Registry file is empty.'.
format(self.eq_valid_names.get(name)),
extra=journal_context({'MESSAGE_ID': BRIDGE_FILE}, {}))
return
with open(self.registry_xml) as registry:
xml_parser = XMLParser(registry.read())
if name == 'mnn':
values = {
v.decode('utf-8').lower(): v.decode('utf-8')
for v in xml_parser.get_values(name) if v
}
self.inn_json_last_check = get_now()
elif name == 'atc1':
values = {
v.decode('utf-8'): v.decode('utf-8')
for v in xml_parser.get_values(name) if v
}
self.atc_json_last_check = get_now()
elif name == 'inn2atc':
values = xml_parser.inn2atc_atc2inn(root='inn')
values = {k.lower(): v for k, v in values.items()}
self.inn2atc_json_last_check = get_now()
elif name == 'atc2inn':
values = xml_parser.inn2atc_atc2inn(root='atc')
values = {k: [i.lower() for i in v] for k, v in values.items()}
self.atc2inn_json_last_check = get_now()
else:
logger.warn('Error! Incorrect xml tag.',
extra=journal_context(
{'MESSAGE_ID': BRIDGE_PARSER_ERROR}, {}))
return
name = self.eq_valid_names.get(name)
with open(file_path, 'r') as f:
if file_is_empty(file_path):
data = dict()
else:
data = json.loads(f.read())
if 'data' in data and set(values.keys()) == set(
data.get('data').keys()):
if name in ['inn2atc', 'atc2inn']:
last_modified = lambda key: {
'inn2atc': get_file_last_modified(self.inn_json),
'atc2inn': get_file_last_modified(self.atc_json)
}.get(key)
data = dict(data=values, dateModified=str(last_modified(name)))
with open(file_path, 'w') as f:
f.write(json.dumps(data))
logger.info('DONE. Local {}.json file updated.'.format(name),
extra=journal_context({'MESSAGE_ID': BRIDGE_FILE},
{}))
self._update_cache(name)
else:
logger.info(
'{} values in remote registry not changed. Skipping update local {}.json file'
.format(name, name),
extra=journal_context({'MESSAGE_ID': BRIDGE_FILE}, {}))
else:
with open(file_path, 'w') as f:
data = dict(data=values,
dateModified=str(
get_file_last_modified(self.registry_xml)))
f.write(json.dumps(data))
logger.info('DONE. Local {}.json file updated.'.format(name),
extra=journal_context({'MESSAGE_ID': BRIDGE_FILE}, {}))
self._update_cache(name)
def test_update_json(self):
self.worker = JsonFormer(self.db, config.get('delay'),
config.get('json_files_delay'),
config.get('cache_monitoring_delay'),
config.get('services_not_available'))
self.worker.DATA_PATH = self.DATA_PATH
self.worker.registry_xml = os.path.join(self.worker.DATA_PATH,
'registry.xml')
self.worker.inn_json = os.path.join(self.worker.DATA_PATH, 'inn.json')
self.worker.atc_json = os.path.join(self.worker.DATA_PATH, 'atc.json')
self.worker.inn2atc_json = os.path.join(self.worker.DATA_PATH,
'inn2atc.json')
self.worker.atc2inn_json = os.path.join(self.worker.DATA_PATH,
'atc2inn.json')
# updates json`s with empty registry
self.assertTrue(
file_exists(os.path.join(self.worker.DATA_PATH, 'registry.xml')))
self.assertTrue(
file_is_empty(os.path.join(self.worker.DATA_PATH, 'registry.xml')))
json_names = ['mnn', 'atc1', 'inn2atc', 'atc2inn']
for name in json_names:
self.assertEqual(self.worker.update_json(name), None)
# updates json`s with registry with invalid data
with open(os.path.join(self.BASE_DIR, 'test_registry.xml'), 'r') as f:
xml = f.read()
with open(os.path.join(self.worker.DATA_PATH, 'registry.xml'),
'w') as f:
f.write(xml)
self.assertFalse(
file_is_empty(os.path.join(self.worker.DATA_PATH, 'registry.xml')))
# update json`s with invalid xml tag
self.assertEqual(self.worker.update_json('xxx'), None)
# update inn json with valid xml tag
self.assertEqual(file_exists(self.worker.inn_json), False)
self.assertEqual(file_exists(self.worker.atc_json), False)
create_file(os.path.join(self.worker.DATA_PATH, 'inn.json'))
create_file(os.path.join(self.worker.DATA_PATH, 'atc.json'))
create_file(os.path.join(self.worker.DATA_PATH, 'inn2atc.json'))
create_file(os.path.join(self.worker.DATA_PATH, 'atc2inn.json'))
self.assertEqual(file_exists(self.worker.inn_json), True)
self.assertEqual(file_is_empty(self.worker.inn_json), True)
self.assertEqual(file_exists(self.worker.atc_json), True)
self.assertEqual(file_is_empty(self.worker.atc_json), True)
self.assertFalse(file_is_empty(self.worker.registry_xml))
self.worker.update_json('mnn')
self.assertFalse(file_is_empty(self.worker.inn_json))
self.worker.update_json('atc1')
self.assertFalse(file_is_empty(self.worker.atc_json))
self.worker.update_json('inn2atc')
self.assertFalse(file_is_empty(self.worker.inn2atc_json))
self.worker.update_json('atc2inn')
self.assertFalse(file_is_empty(self.worker.atc2inn_json))
# check cache
cache = self.db.get('inn')
self.assertEqual(
str_to_obj(cache).get('data'), {u'methyluracil': u'Methyluracil'})
cache = self.db.get('atc')
self.assertEqual(str_to_obj(cache).get('data'), {})
cache = self.db.get('inn2atc')
self.assertIn(u'methyluracil', str_to_obj(cache).get('data'))
if file_is_empty(self.worker.atc2inn_json):
cache = self.db.get('atc2inn')
self.assertEqual(str_to_obj(cache).get('data'), {})
else:
cache = self.db.get('atc2inn')
with open(self.worker.atc2inn_json) as f:
data = f.read()
if str_to_obj(data).get('data'):
self.assertEqual(
str_to_obj(data).get('data'),
str_to_obj(cache).get('data'))
else:
self.assertEqual(str_to_obj(data).get('data'), dict())
self.db.flushall()
self.assertEqual(self.db.has('inn'), False)
self.worker._update_cache('inn')
self.assertEqual(self.db.has('inn'), True)
