def update_json_files(self):
while True:
now = get_now()
registry_last_modified = get_file_last_modified(self.registry_xml)
last_check_dict = {
'inn': self.inn_json_last_check,
'atc': self.atc_json_last_check,
'inn2atc': self.inn2atc_json_last_check,
'atc2inn': self.atc2inn_json_last_check
}
files_dict = {
'inn': self.inn_json,
'atc': self.atc_json,
'inn2atc': self.inn2atc_json,
'atc2inn': self.atc2inn_json
}
eq_valid_names = (('mnn', 'inn'), ('atc1', 'atc'),
('inn2atc', 'inn2atc'), ('atc2inn', 'atc2inn'))
for name, eq_name in eq_valid_names:
if file_is_empty(files_dict.get(eq_name)):
self.update_json(name)
else:
if now.date() >= registry_last_modified.date():
last_check = last_check_dict.get(eq_name)
if not last_check or registry_last_modified.date(
) > last_check.date():
self.update_json(name)
gevent.sleep(self.json_files_delay)
def __init__(self, db, delay, json_files_delay, cache_monitoring_delay,
services_not_available):
super(JsonFormer, self).__init__(services_not_available)
self.start_time = get_now()
self.INFINITY_LOOP = True
self.DATA_PATH = DATA_PATH
self.registry_xml = os.path.join(self.DATA_PATH, 'registry.xml')
self.db = db
self.delay = delay
self.json_files_delay = json_files_delay
self.cache_monitoring_delay = cache_monitoring_delay
self.inn_json_last_check = None
self.atc_json_last_check = None
self.inn2atc_json_last_check = None
self.atc2inn_json_last_check = None
self.eq_valid_names = {
'mnn': 'inn',
'atc1': 'atc',
'inn2atc': 'inn2atc',
'atc2inn': 'atc2inn'
}
self.inn_json = os.path.join(self.DATA_PATH, 'inn.json')
self.atc_json = os.path.join(self.DATA_PATH, 'atc.json')
self.inn2atc_json = os.path.join(self.DATA_PATH, 'inn2atc.json')
self.atc2inn_json = os.path.join(self.DATA_PATH, 'atc2inn.json')
def __init__(self,
source_registry,
time_update_at,
delay,
registry_delay,
services_not_available,
source_registry_proxy=None):
super(Registry, self).__init__(services_not_available)
self.start_time = get_now()
self.INFINITY_LOOP = True
self.DATA_PATH = DATA_PATH
self.registry_xml = os.path.join(self.DATA_PATH, 'registry.xml')
self.source_registry = source_registry
self.source_registry_proxy = source_registry_proxy
if type(time_update_at).__name__ == 'str':
self.time_update_at = string_time_to_datetime(time_update_at
or '05:30:00')
else:
self.time_update_at = time_update_at
self.delay = delay
self.registry_delay = registry_delay
if source_registry_proxy:
proxy = ProxyHandler({
'http': source_registry_proxy,
'https': source_registry_proxy
})
self.urlopen = build_opener(proxy).open
else:
self.urlopen = urlopen
def save_registry(self, xml):
logger.info('Save response to \'registry.xml\' file...',
extra=journal_context({'MESSAGE_ID': BRIDGE_REGISTER}, {}))
with open(os.path.join(self.DATA_PATH, 'registry.xml'), 'w') as f:
f.write(xml)
logger.info(
'File \'registry.xml\' saved at: {}.'.format(get_now()),
extra=journal_context({'MESSAGE_ID': BRIDGE_REGISTER}, {}))
def test_registry_update_time(self):
self.worker = Registry(config.get('source_registry'),
config.get('time_update_at'),
config.get('delay'),
config.get('registry_delay'),
config.get('services_not_available'))
self.worker.time_update_at = string_time_to_datetime('00:00:00')
self.assertFalse(self.worker.registry_update_time)
self.worker.time_update_at = get_now()
self.assertTrue(self.worker.registry_update_time)
def registry_update_time(self):
now = get_now()
if self.time_update_at and (now.hour == self.time_update_at.hour):
check_to_time = self.time_update_at + timedelta(minutes=30)
if self.time_update_at.minute <= now.minute and now.replace(
tzinfo=None).time() <= check_to_time.time():
return True
else:
return False
else:
return False
def update_local_registry(self):
while self.INFINITY_LOOP:
now = get_now()
last_modified = get_file_last_modified(self.registry_xml)
conditions = (now.date() > last_modified.date()
and self.registry_update_time,
file_is_empty(self.registry_xml))
if any(conditions):
logger.info('Update local registry file...',
extra=journal_context(
{'MESSAGE_ID': BRIDGE_REGISTER}, {}))
self.get_registry()
else:
logger.info('Registry file is updated.',
extra=journal_context(
{'MESSAGE_ID': BRIDGE_REGISTER}, {}))
gevent.sleep(self.registry_delay)
def __init__(self, source_registry, time_update_at, delay, registry_delay,
services_not_available):
super(Registry, self).__init__(services_not_available)
self.start_time = get_now()
self.INFINITY_LOOP = True
self.DATA_PATH = DATA_PATH
self.registry_xml = os.path.join(self.DATA_PATH, 'registry.xml')
self.source_registry = source_registry
if type(time_update_at).__name__ == 'str':
self.time_update_at = string_time_to_datetime(time_update_at
or '05:30:00')
else:
self.time_update_at = time_update_at
self.delay = delay
self.registry_delay = registry_delay
def update_json(self, name):
logger.info('Update local {}.json file...'.format(
self.eq_valid_names.get(name)),
extra=journal_context({'MESSAGE_ID': BRIDGE_INFO}, {}))
file_path = os.path.join(
self.DATA_PATH, '{}.json'.format(self.eq_valid_names.get(name)))
if file_is_empty(self.registry_xml):
logger.info(
'Local {}.json file not updated. Registry file is empty.'.
format(self.eq_valid_names.get(name)),
extra=journal_context({'MESSAGE_ID': BRIDGE_FILE}, {}))
return
with open(self.registry_xml) as registry:
xml_parser = XMLParser(registry.read())
if name == 'mnn':
values = {
v.decode('utf-8').lower(): v.decode('utf-8')
for v in xml_parser.get_values(name) if v
}
self.inn_json_last_check = get_now()
elif name == 'atc1':
values = {
v.decode('utf-8'): v.decode('utf-8')
for v in xml_parser.get_values(name) if v
}
self.atc_json_last_check = get_now()
elif name == 'inn2atc':
values = xml_parser.inn2atc_atc2inn(root='inn')
values = {k.lower(): v for k, v in values.items()}
self.inn2atc_json_last_check = get_now()
elif name == 'atc2inn':
values = xml_parser.inn2atc_atc2inn(root='atc')
values = {k: [i.lower() for i in v] for k, v in values.items()}
self.atc2inn_json_last_check = get_now()
else:
logger.warn('Error! Incorrect xml tag.',
extra=journal_context(
{'MESSAGE_ID': BRIDGE_PARSER_ERROR}, {}))
return
name = self.eq_valid_names.get(name)
with open(file_path, 'r') as f:
if file_is_empty(file_path):
data = dict()
else:
data = json.loads(f.read())
if 'data' in data and set(values.keys()) == set(
data.get('data').keys()):
if name in ['inn2atc', 'atc2inn']:
last_modified = lambda key: {
'inn2atc': get_file_last_modified(self.inn_json),
'atc2inn': get_file_last_modified(self.atc_json)
}.get(key)
data = dict(data=values, dateModified=str(last_modified(name)))
with open(file_path, 'w') as f:
f.write(json.dumps(data))
logger.info('DONE. Local {}.json file updated.'.format(name),
extra=journal_context({'MESSAGE_ID': BRIDGE_FILE},
{}))
self._update_cache(name)
else:
logger.info(
'{} values in remote registry not changed. Skipping update local {}.json file'
.format(name, name),
extra=journal_context({'MESSAGE_ID': BRIDGE_FILE}, {}))
else:
with open(file_path, 'w') as f:
data = dict(data=values,
dateModified=str(
get_file_last_modified(self.registry_xml)))
f.write(json.dumps(data))
logger.info('DONE. Local {}.json file updated.'.format(name),
extra=journal_context({'MESSAGE_ID': BRIDGE_FILE}, {}))
self._update_cache(name)