def __init__(self, config, agent_config):
super().__init__(config=agent_config)
if not (isinstance(config, dict) and all(section in config for section in ('integrator', 'config'))):
raise ValueError(
'Configuration invalid / missing required section')
# Whilst the integrator core requires particular configuration, top-level sections could be defined to provide
# parameters specific to this integrator.
self.__integrator = Integrator(config['integrator'], self.client, self)
self.__assets = set()
self.__config = config
# data cache used to check that the asset has been changed or not before publishing the event
self.__data_cache = get_cache(config, config_path='integrator.asset.cache.method')
self.__loop_time = NestedConfig.get(self.__config, 'config.loop_time', required=False, default=False)
self.__sap_config_info = SapConfig(
hi_endp=NestedConfig.get(
self.__config, 'config.sap.hierarchy_endpoint', required=True, check=non_empty_str
),
md_endp=NestedConfig.get(
self.__config, 'config.sap.master_endpoint', required=True, check=non_empty_str
),
md_usr=NestedConfig.get(
self.__config, 'config.sap.usr', required=True, check=non_empty_str
),
md_pwd=NestedConfig.get(
self.__config, 'config.sap.pwd', required=True, check=non_empty_str
),
md_timeout=int(NestedConfig.get(
self.__config, 'config.sap.timeout', required=False, default=10, check=non_negative_int
))
)
def __init__(self, config, agent_config):
super().__init__(config=agent_config)
if not (isinstance(config, dict) and all(section in config for section in ('integrator', 'config'))):
raise ValueError('Configuration invalid / missing required section')
# Whilst the integrator core requires particular configuration, top-level sections could be defined to provide
# parameters specific to this integrator.
self.__integrator = Integrator(config['integrator'], self.client, self)
self.__assets = set()
self.__config = config
# data cache used to check that the asset has been changed or not before publishing the event
self.__data_cache = get_cache(config, config_path='integrator.asset.cache.method')
self.__use_mock_data = NestedConfig.get(self.__config, 'config.use_mock_data', required=False, default=False)
self.__workers = NestedConfig.get(self.__config,
'config.workers', required=False, default=1, check=non_negative_int)
self.__loop_time = NestedConfig.get(self.__config,
'config.loop_time', required=False, default=5, check=non_negative_int)
self.__req_pool = ThreadPoolExecutor(max_workers=self.__workers)
self.__talend_config_info = TalendConfig(
endpoint=NestedConfig.get(self.__config,
'config.talend.endpoint', required=True, check=non_empty_str),
endpoint_single=NestedConfig.get(self.__config,
'config.talend.endpoint_single', required=True, check=non_empty_str),
usr=NestedConfig.get(self.__config, 'config.talend.usr', required=True, check=non_empty_str),
pwd=NestedConfig.get(self.__config, 'config.talend.pwd', required=True, check=non_empty_str),
timeout=int(NestedConfig.get(self.__config,
'config.talend.timeout', required=False, default=10, check=non_negative_int))
)
def __init__(self, config, agent_config):
super().__init__(config=agent_config)
if not (isinstance(config, Mapping)
and all(section in config
for section in ('integrator', 'config'))):
raise ValueError(
'Configuration invalid / missing required section')
# parameters specific to this integrator.
self.__integrator = Integrator(config['integrator'], self.client, self)
self.__assets = set()
self.__config = config
# data cache used to check that the asset has been changed or not before publishing the event
self.__data_cache = get_cache(
self.__config, config_path='integrator.asset.cache.method')
# Pool of workers to execture type2 requests
workers = NestedConfig.get(self.__config,
'config.workers',
required=False,
default=1,
check=non_negative_int)
self.__req_pool = ThreadPoolExecutor(max_workers=workers)
# Validate config
self.__sap_config_info = SapConfig(
eq_hist_endp=NestedConfig.get(
self.__config,
'config.sap.equipment_history_endpoint',
required=True,
check=non_empty_str),
eq_doc_endp=NestedConfig.get(
self.__config,
'config.sap.equipment_document_endpoint',
required=True,
check=non_empty_str),
eq_doc_single=NestedConfig.get(
self.__config,
'config.sap.equipment_document_single',
required=True,
check=non_empty_str),
eq_doc_test=NestedConfig.get(self.__config,
'config.sap.equipment_document_test',
required=True,
check=non_empty_str),
usr=NestedConfig.get(self.__config,
'config.sap.usr',
required=True,
check=non_empty_str),
pwd=NestedConfig.get(self.__config,
'config.sap.pwd',
required=True,
check=non_empty_str),
timeout=int(
NestedConfig.get(self.__config,
'config.sap.timeout',
required=False,
default=10,
check=non_negative_int)))
def __init__(self, config, agent_config):
super().__init__(config=agent_config)
if not (isinstance(config, dict) and all(section in config for section in ('integrator', 'config'))):
raise ValueError('Configuration invalid / missing required section')
# Whilst the integrator core requires particular configuration, top-level sections could be defined to provide
# parameters specific to this integrator.
self.__integrator = Integrator(config['integrator'], self.client, self)
self.__assets = set()
self.__config = config
self.__data_cache = self.__config['config']['data-cache']
def __init__(self, config, agent_config): ### where does config, agent_config came from...?????
super().__init__(config=agent_config) ### reinitializing class 'ThingRunner' using super(), in nested
if not (isinstance(config, dict) and all(section in config for section in ('integrator', 'config'))):
raise ValueError(
'Configuration invalid / missing required section')
#?? # Whilst the integrator core requires particular configuration, top-level sections could be defined to provide ###?? which top level section???
# parameters specific to this integrator.
self.__integrator = Integrator(config['integrator'], self.client, self)
self.__assets = set() ### setting up asset, but where and how ????
self.__config = config
self.__data_cache = self.__config['config']['data-cache'] ### this might be nested dict, 'self.__data_cache = value'
def __init__(self, config, agent_config):
super().__init__(config=agent_config)
if not (isinstance(config, dict) and all(section in config for section in ('integrator', 'config'))):
raise ValueError(
'Configuration invalid / missing required section')
# Whilst the integrator core requires particular configuration, top-level sections could be defined to provide
# parameters specific to this integrator.
self.__integrator = Integrator(config['integrator'], self.client, self)
self.__assets = set()
self.__config = config
# data cache used to check that the asset has been changed or not before publishing the event
self.__data_cache = get_cache(config, config_path='integrator.asset.cache.method')
class IntegratorCB(IntegratorCallbacks):
def __init__(self, config, client):
self.__integrator = Integrator(config, client, self)
self.__assets = set()
def start(self):
self.__integrator.start()
def stop(self):
self.__integrator.stop()
def publish_event(self, event):
self.__integrator.publish_event(event)
# for IntegratorCallbacks
def on_asset_created(self, asset_id):
log.critical('IntegratorCallbacks Asset created: %s', asset_id)
# for IntegratorCallbacks
def on_asset_deleted(self, asset_id):
log.critical('IntegratorCallbacks Asset deleted: %s', asset_id)
# for IntegratorCallbacks
def on_t2_request(self, request):
pass
def __init__(self, config, client):
self.__integrator = Integrator(config, client, self)
self.__assets = set()
class TalendFirmwareIntegrator(IntegratorCallbacks, ThingRunner):
def __init__(self, config, agent_config):
super().__init__(config=agent_config)
if not (isinstance(config, dict) and all(section in config for section in ('integrator', 'config'))):
raise ValueError(
'Configuration invalid / missing required section')
# Whilst the integrator core requires particular configuration, top-level sections could be defined to provide
# parameters specific to this integrator.
self.__integrator = Integrator(config['integrator'], self.client, self)
self.__assets = set()
self.__config = config
self.__data_cache = self.__config['config']['data-cache']
def on_startup(self):
log.info('Talend Firmware Integrator Startup')
self.__integrator.start()
def main(self):
log.info('Talend Firmware Integrator Running')
self._process_data()
loop_time = self.__config['config']['loop_time']
while not self.wait_for_shutdown(loop_time):
self._process_data()
def on_shutdown(self, exc_info):
log.info('Talend Firmware Integrator Shutdown')
self.__integrator.stop()
# for IntegratorCallbacks
def on_asset_created(self, asset_id):
log.info('Asset created: %s', asset_id)
self.__assets.add(asset_id)
# for IntegratorCallbacks
def on_asset_deleted(self, asset_id):
log.info('Asset deleted: %s', asset_id)
self.__assets.discard(asset_id)
# for IntegratorCallbacks
def on_t2_request(self, request):
pass
def _get_data_for_asset(self, asset_id):
log.info("Get Talend data for: %s", asset_id)
data = None
if self.__config['config']['use_mock_data'] == 1:
log.debug("Using mock data")
with open(self.MOCK_DATA_FILE, mode="r", encoding="utf-8") as f:
data = json.load(f)
else:
endpoint = self.__config['config']['talend']['endpoint']
usr = self.__config['config']['talend']['usr']
pwd = self.__config['config']['talend']['pwd']
key = 'config.enable_sap_sample_serial_hack'
if NestedConfig.get(self.__config, key, required=False, default=False, check=bool):
if asset_id == '1000021' or asset_id == '1000015':
asset_id = '16701003340'
endpoint = endpoint.replace('XXX_ASSET_ID_XXX', asset_id)
timeout = int(self.__config['config']['talend']['timeout'])
log.debug("Calling: %s", endpoint)
try:
resp = requests.get(endpoint, auth=(usr, pwd), verify=False, timeout=timeout)
log.debug("Response status: %s", resp.status_code)
if resp.status_code == requests.codes['ok']:
data = resp.json()
except requests.exceptions.RequestException as ex:
log.error(ex)
return data
def _process_data(self):
log.info("Processing Talend Firmware")
for asset_id in list(self.__assets):
log.debug("Processing asset: %s", asset_id)
data = self._get_data_for_asset(asset_id)
if data is not None:
if self._has_asset_data_changed_for(asset_id, data):
event = TalendFirmwareSet(asset_id, data=data)
log.debug("Publishing event: %s", event)
try:
self.__integrator.publish_event(event)
self._cache_asset_data_for(asset_id, data)
# These will all retry
except EventPublishFailure as ex:
log.error("Event Publish Failure: %s", ex)
except AssetUnknown as ex:
pass
# Checks to see if the given data for the asset has changed
# since it was last processed.
def _has_asset_data_changed_for(self, asset_id, data):
log.info("Checking asset cache for: %s", asset_id)
if not os.path.exists(self.__data_cache):
# No cache so this is new data
return True
filename = asset_id + '.json'
file_path = os.path.join(self.__data_cache, filename)
if not os.path.isfile(file_path):
# No file exists so this is new data
return True
with open(file_path, mode="r", encoding="utf-8") as f:
cached_data_hash = f.read()
data_hash = self.__compute_data_hash(data)
if cached_data_hash != data_hash:
os.remove(file_path)
# The data has changed so flush the cache
return True
# Nothing has changed for this data
return False
@classmethod
def __compute_data_hash(cls, data):
jdata = json.dumps(data, sort_keys=True, separators=(',', ':'))
return hashlib_md5(jdata.encode('utf8')).hexdigest()
def _cache_asset_data_for(self, asset_id, data):
log.info("Cache asset for: %s", asset_id)
if not os.path.exists(self.__data_cache):
log.debug("Creating data cache")
os.makedirs(self.__data_cache, exist_ok=True)
filename = asset_id + '.json'
file_path = os.path.join(self.__data_cache, filename)
if not os.path.isfile(file_path):
with open(file_path, mode="w", encoding="utf-8") as f:
f.write(self.__compute_data_hash(data))
log.debug("Caching data for asset %s", asset_id)
MOCK_DATA_FILE = os.path.join('cfg', 'mock-data.json')
class SAPMasterDataIntegrator(IntegratorCallbacks, RetryingThingRunner):
def __init__(self, config, agent_config):
super().__init__(config=agent_config)
if not (isinstance(config, dict)
and all(section in config
for section in ('integrator', 'config'))):
raise ValueError(
'Configuration invalid / missing required section')
# Whilst the integrator core requires particular configuration, top-level sections could be defined to provide
# parameters specific to this integrator.
self.__integrator = Integrator(config['integrator'], self.client, self)
self.__assets = set()
self.__config = config
# data cache used to check that the asset has been changed or not before publishing the event
self.__data_cache = get_cache(
config, config_path='integrator.asset.cache.method')
self.__loop_time = NestedConfig.get(self.__config,
'config.loop_time',
required=False,
default=False)
self.__enable_dev_mapping = NestedConfig.get(
self.__config,
'config.enable_dev_mapping',
required=False,
default=False,
check=non_empty_str)
self.__sap_config_info = SapConfig(
hi_endp=NestedConfig.get(self.__config,
'config.sap.hierarchy_endpoint',
required=True,
check=non_empty_str),
md_endp=NestedConfig.get(self.__config,
'config.sap.master_endpoint',
required=True,
check=non_empty_str),
md_usr=NestedConfig.get(self.__config,
'config.sap.usr',
required=True,
check=non_empty_str),
md_pwd=NestedConfig.get(self.__config,
'config.sap.pwd',
required=True,
check=non_empty_str),
md_timeout=int(
NestedConfig.get(self.__config,
'config.sap.timeout',
required=False,
default=10,
check=non_negative_int)))
def on_startup(self):
log.debug('SAP Master Data Integrator Startup')
self.__data_cache.start()
self.__integrator.start()
def main(self):
log.debug('SAP Master Data Integrator Running')
loop_time = self.__loop_time
while not self.wait_for_shutdown(loop_time):
self.__process_master_data()
def on_shutdown(self, exc_info):
log.debug('SAP Master Data Integrator Shutdown')
self.__integrator.stop()
self.__data_cache.stop()
# for IntegratorCallbacks
def on_asset_created(self, asset_id):
log.debug('Asset created: %s', asset_id)
self.__assets.add(asset_id)
# for IntegratorCallbacks
def on_asset_deleted(self, asset_id):
log.debug('Asset deleted: %s', asset_id)
self.__assets.discard(asset_id)
# for IntegratorCallbacks
def on_t2_request(self, request):
pass
def __get_data_for_asset(self, asset_id):
""" returns sapmasterdata for asset_id """
log.info("Get Master Data for: %s", asset_id)
# asset_id hack for RR Dev environment
# The dev environment sapmasterdata API uses a specific asset_id we will swap two of our test IDs.
if self.__enable_dev_mapping:
asset_id = '1000015'
usr = self.__sap_config_info.md_usr
pwd = self.__sap_config_info.md_pwd
timeout = int(self.__sap_config_info.md_timeout)
# aggregate_equipment_number, results = self.__get_aggregate_equipment_num(asset_id, usr, pwd, timeout)
# engine_serial = self.__get_the_engine_serial_number(asset_id, aggregate_equipment_number, results)
engine_serial = self.__get_the_engine_serial_number(
asset_id, usr, pwd, timeout)
aggregate_data = self.__get_the_aggregate_data(asset_id, usr, pwd,
timeout)
data = self.__get_the_engine_data(engine_serial, aggregate_data, usr,
pwd, timeout)
if data:
return data
return None
def __get_the_engine_serial_number(self, asset_id, usr, pwd, timeout):
""" returns __get_the_engine_serial_number for asset_id """
log.info("Get the Aggregate Equipment Number for asset_id: %s",
asset_id)
try:
hi_endpoint = self.__sap_config_info.hi_endp.format(
asset_id=asset_id)
log.debug("Calling hi_endpoint: %s", hi_endpoint)
resp = requests.get(hi_endpoint,
auth=(usr, pwd),
verify=False,
timeout=timeout)
log.debug("Response status: %s", resp.status_code)
resp.raise_for_status()
if resp.ok:
try:
data = resp.json()
results = data['d']['results']
aggregate_equipment_number = None
for _data in results:
if _data['Hequi'] == "":
aggregate_equipment_number = _data['Equnr']
log.debug("Getting aggregate_equipment_number: %s",
aggregate_equipment_number)
# return aggregate_equipment_number, results
for _data in results:
if _data[
"Hequi"] == aggregate_equipment_number and _data[
"Eqart"] == "ENG":
engine_serial = str(_data["Sernr"])
log.debug("Getting engine_serial: %s",
engine_serial)
return engine_serial
except: # pylint: disable=broad-except
log.error(
"Could not parse JSON from response for asset %s",
asset_id,
exc_info=DEBUG_ENABLED)
else:
log.error("Endpoint response failed: %s", resp.status_code)
except requests.exceptions.HTTPError as ex:
log.error("Get the Engine Serial %s with asset_id: %s", ex,
asset_id)
except requests.exceptions.RequestException as ex:
log.error(ex, exc_info=DEBUG_ENABLED)
return None
# @staticmethod
# def __get_the_engine_serial_number(asset_id, aggregate_equipment_number, results):
# """ returns engine_serial_number for asset_id """
# log.info("Get the Engine Serial Number for asset_id: %s", asset_id)
#
# for _data in results:
# try:
# if _data["Hequi"] == aggregate_equipment_number and _data["Eqart"] == "ENG":
# engine_serial = str(_data["Sernr"])
# log.debug("Getting engine_serial: %s", engine_serial)
# return engine_serial
# except KeyError:
# log.error("\'Hequi\' or \'Eqart\' key not found for this item")
# return None
#
# return None
def __get_the_aggregate_data(self, asset_id, usr, pwd, timeout):
""" returns aggregate_data for asset_id """
log.info("Get the Aggregate Data for asset_id: %s", asset_id)
try:
md_endpoint = self.__sap_config_info.md_endp.format(
asset_id=asset_id)
log.debug("Calling md_endpoint: %s", md_endpoint)
resp = requests.get(md_endpoint,
auth=(usr, pwd),
verify=False,
timeout=timeout)
log.debug("Response status: %s", resp.status_code)
resp.raise_for_status()
if resp.ok:
try:
data = resp.json()
except: # pylint: disable=broad-except
log.error(
"Could not parse JSON from response for asset %s",
asset_id,
exc_info=DEBUG_ENABLED)
else:
event_data = None
try:
event_data = data['d']['results'][0]
event_data['Datab'] = math.floor(
datetime.datetime.strptime(
event_data['Datab'],
'%Y-%m-%d').timestamp()) * 1000
event_data['Datbi'] = math.floor(
datetime.datetime.strptime(
event_data['Datbi'],
'%Y-%m-%d').timestamp()) * 1000
event_data['EqunrAgg'] = event_data['Equnr']
event_data['MaktxAgg'] = event_data['Maktx']
event_data['MatnrAgg'] = event_data['Matnr']
event_data['SernrAgg'] = event_data['Sernr']
for field in ('Equnr', 'Maktx', 'Matnr', 'Sernr'):
del event_data[field]
except KeyError:
log.error("Could not find response for asset %s",
asset_id)
return event_data
else:
log.error("Endpoint response failed: %s", resp.status_code)
except requests.exceptions.HTTPError as ex:
log.error("Get the Aggregate Data %s with asset_id: %s", ex,
asset_id)
except requests.exceptions.RequestException as ex:
log.error(ex, exc_info=DEBUG_ENABLED)
return None
def __get_the_engine_data(self, engine_serial, event_data, usr, pwd,
timeout):
""" returns engine_data for engine_serial """
log.info("Get the Engine Data for engine_serial: %s", engine_serial)
try:
md_endpoint = self.__sap_config_info.md_endp.format(
asset_id=engine_serial)
log.debug("Calling md_endpoint: %s", md_endpoint)
resp = requests.get(md_endpoint,
auth=(usr, pwd),
verify=False,
timeout=timeout)
log.debug("Response status: %s", resp.status_code)
resp.raise_for_status()
if resp.ok:
try:
data = resp.json()
except: # pylint: disable=broad-except
log.error(
"Could not parse JSON from response for asset %s",
engine_serial,
exc_info=DEBUG_ENABLED)
else:
try:
engine_data = data['d']['results'][0]
event_data['SernrEng'] = engine_data['Sernr']
event_data['MatnrEng'] = engine_data['Matnr']
event_data['MaktxEng'] = engine_data['Maktx']
event_data['EqunrEng'] = engine_data['Equnr']
except KeyError:
log.error(
"Could not find response for engine_serial %s",
engine_serial)
return event_data
else:
log.error("Endpoint response failed: %s", resp.status_code)
except requests.exceptions.HTTPError as ex:
log.error("Get the Engine Data %s with engine_serial: %s", ex,
engine_serial)
except requests.exceptions.RequestException as ex:
log.error(ex, exc_info=DEBUG_ENABLED)
return None
@classmethod
def __calc_eventtime(cls, master_data):
""" calculate the event time with appropriate format """
event_time = master_data.get('Datab', None)
if event_time:
try:
event_time = datetime.datetime.utcfromtimestamp(event_time //
1000)
except (OverflowError, ValueError) as exc:
log.error("Could not create a valid datetime from %s",
event_time)
log.error(exc)
return event_time
def __process_master_data(self):
""" Check whether the dat has changed or not. If its changed, create the event and publish """
log.debug("Processing Master Data")
for asset_id in list(self.__assets):
log.debug("Processing asset: %s", asset_id)
master_data = self.__get_data_for_asset(asset_id)
if master_data and self.__has_asset_data_changed_for(
asset_id, master_data):
log.info("Publish event for %s", asset_id)
event_time = self.__calc_eventtime(master_data)
if master_data and event_time:
event = SapMasterDataSet(asset_id,
data=master_data,
time=event_time)
log.debug("Event: %s", event)
else:
continue
try:
self.__integrator.publish_event(event, retry=True)
self.__cache_assetdata_for(asset_id, master_data)
except ShutdownRequested:
log.debug("Shutdown requested while publishing event")
return
except AssetUnknown:
pass
@classmethod
def __compute_data_hash(cls, data):
""" compute the md5 hash for the particular data"""
jdata = json.dumps(data, sort_keys=True, separators=(',', ':'))
return hashlib_md5(jdata.encode('utf8')).hexdigest()
# Checks to see if the given data for the asset has changed
# since it was last processed.
def __has_asset_data_changed_for(self, asset_id, data):
""" checking whether asset data has changed or not """
log.info("Checking asset cache for: %s", asset_id)
try:
asset_id_hash = self.__data_cache.get_attr(asset_id, 'hash')
except KeyError:
# No cache so this is new data
return True
data_hash = self.__compute_data_hash(data)
if asset_id_hash['hash'] != data_hash:
# data has changed
return True
# Nothing has changed for this data
return False
# After publishing the event, update the cache
def __cache_assetdata_for(self, asset_id, data):
""" update the cache for the asset id """
log.info("Cache asset for: %s", asset_id)
data_hash = self.__compute_data_hash(data)
self.__data_cache.mark_as_known(asset_id, hash=data_hash)
class SAPEquipmentHistoryIntegrator(IntegratorCallbacks, RetryingThingRunner):
def __init__(self, config, agent_config):
super().__init__(config=agent_config)
if not (isinstance(config, Mapping)
and all(section in config
for section in ('integrator', 'config'))):
raise ValueError(
'Configuration invalid / missing required section')
# parameters specific to this integrator.
self.__integrator = Integrator(config['integrator'], self.client, self)
self.__assets = set()
self.__config = config
# data cache used to check that the asset has been changed or not before publishing the event
self.__data_cache = get_cache(
self.__config, config_path='integrator.asset.cache.method')
# Pool of workers to execture type2 requests
workers = NestedConfig.get(self.__config,
'config.workers',
required=False,
default=1,
check=non_negative_int)
self.__req_pool = ThreadPoolExecutor(max_workers=workers)
# Validate config
self.__sap_config_info = SapConfig(
eq_hist_endp=NestedConfig.get(
self.__config,
'config.sap.equipment_history_endpoint',
required=True,
check=non_empty_str),
eq_doc_endp=NestedConfig.get(
self.__config,
'config.sap.equipment_document_endpoint',
required=True,
check=non_empty_str),
eq_doc_single=NestedConfig.get(
self.__config,
'config.sap.equipment_document_single',
required=True,
check=non_empty_str),
eq_doc_test=NestedConfig.get(self.__config,
'config.sap.equipment_document_test',
required=True,
check=non_empty_str),
usr=NestedConfig.get(self.__config,
'config.sap.usr',
required=True,
check=non_empty_str),
pwd=NestedConfig.get(self.__config,
'config.sap.pwd',
required=True,
check=non_empty_str),
timeout=int(
NestedConfig.get(self.__config,
'config.sap.timeout',
required=False,
default=10,
check=non_negative_int)))
def on_startup(self):
log.info('Startup')
self.__data_cache.start()
self.__integrator.start()
def main(self):
log.info('Running')
loop_time = NestedConfig.get(self.__config,
'config.loop_time',
required=False,
default=5,
check=non_negative_int)
while not self.wait_for_shutdown(loop_time):
self.__process_data()
def on_shutdown(self, exc_info):
log.info('Shutdown')
self.__integrator.stop()
self.__data_cache.stop()
# for IntegratorCallbacks
def on_asset_created(self, asset_id):
log.info('Asset created: %s', asset_id)
self.__assets.add(asset_id)
# for IntegratorCallbacks
def on_asset_deleted(self, asset_id):
log.info('Asset deleted: %s', asset_id)
self.__assets.discard(asset_id)
# for IntegratorCallbacks
def on_t2_request(self, request):
self.__req_pool.submit(self.__process_t2, request)
# Wrap since run via thread pool without handling return/exception
@log_exceptions(log)
def __process_t2(self, request):
log.info('New T2 req for %s - %s(%r)', request.asset_id, request.type_,
request.data)
if request.type_ == T2_REQUEST_SAP_DOCUMENTSINGLE:
self.__t2_do_document_req(request)
elif request.type_ == T2_REQUEST_SAP_SDOCK:
self.__t2_do_test_document_req(request)
else:
log.warning('Ignoring unknown request type %s', request.type_)
return
def __integrator_t2_respond_error(self,
request,
reason=T2ProviderFailureReason.
REQ_UNHANDLED):
try:
self.__integrator.t2_respond_error(request, reason)
except ShutdownRequested:
pass
except TypeError:
log.error(
'Could not send T2 error response, invalid request or reason',
exc_info=DEBUG_ENABLED)
except T2ResponseFailure:
log.error('Could not send T2 error response',
exc_info=DEBUG_ENABLED)
def __integrator_t2_test_respond(self, result, request):
try:
value = result['d']['results'][0]['FileName']
except KeyError:
log.error("Error in finding mime type")
self.__integrator_t2_respond_error(request)
return
mime = guess_type(value)[0]
if not mime:
log.error('Unknown file type')
self.__integrator_t2_respond_error(request)
return
try:
file_name = json.dumps(value).encode('utf8')
self.__integrator.t2_respond(
request, mime,
a2b_base64(result['d']['results'][0]['Xstring']))
print("\n", mime, "\n")
print("\n", file_name, "\n")
# print("\n", a2b_base64(result['d']['results'][0]['Xstring']), "\n")
# doc_data = result['d']['results']
# response_data = []
# for doc in doc_data:
# doc_template = {"FileName": "", "Xstring": ""}
#
# doc_template["FileName"] = doc['FileName']
# doc_template["Xstring"] = doc['Xstring']
# response_data.append(doc_template)
#
# # self.__integrator.t2_respond(
# # request, mime, json.dumps(response_data).encode('utf-8'))
# self.__integrator.t2_respond(
# request, mime, a2b_base64(response_data))
# print("\nrequest\n", request)
# print("\nmime\n", mime)
# print("\nvalue\n", value)
# # print("\nresult['d']['results'][0]['Xstring']\n", result['d']['results'][0]['Xstring'])
# # print("\njson.dumps(value).encode('utf8')\n", json.dumps(value).encode('utf8'))
# # print("\na2b_base64(result['d']['results'][0]['Xstring']\n", a2b_base64(result['d']['results'][0]['Xstring']))
# # self.__integrator.t2_respond(
# # request, mime, json.dumps(value).encode('utf8'), a2b_base64(result['d']['results'][0]['Xstring']))
except binascii_Error:
log.error("Failed to a2b_base64 data")
self.__integrator_t2_respond_error(request)
except ShutdownRequested:
pass
except AssetUnknown:
log.error('Could not send T2 response, asset unknown',
exc_info=DEBUG_ENABLED)
except T2ResponseFailure:
log.error('Could not send T2 response', exc_info=DEBUG_ENABLED)
def __integrator_t2_respond(self, result, request):
try:
value = result['d']['results'][0]['FileName']
except KeyError:
log.error("Error in finding mime type")
self.__integrator_t2_respond_error(request)
return
mime = guess_type(value)[0]
if not mime:
log.error('Unknown file type')
self.__integrator_t2_respond_error(request)
return
try:
self.__integrator.t2_respond(
request, mime, a2b_base64(result['d']['results'][0]['String']))
except binascii_Error:
log.error("Failed to a2b_base64 data")
self.__integrator_t2_respond_error(request)
except ShutdownRequested:
pass
except AssetUnknown:
log.error('Could not send T2 response, asset unknown',
exc_info=DEBUG_ENABLED)
except T2ResponseFailure:
log.error('Could not send T2 response', exc_info=DEBUG_ENABLED)
def __t2_do_test_document_req(self, request):
decoded = json.loads(request.data.decode('utf-8'))
try:
equnr = decoded['equnr']
dokar = decoded['document_type']
except KeyError:
log.warning('Equnr or Dokar not in request')
self.__integrator_t2_respond_error(request)
return
log.debug("Getting document by Equnr: %s", equnr)
equipment_document_test = self.__sap_config_info.eq_doc_test.format(
Equnr=equnr, Dokar=dokar)
log.debug("Calling Document Test endpoint: %s",
equipment_document_test)
try:
resp = requests.get(equipment_document_test,
auth=(self.__sap_config_info.usr,
self.__sap_config_info.pwd),
verify=False,
timeout=self.__sap_config_info.timeout)
if resp.status_code == requests.codes.not_found: # pylint: disable=no-member
self.__integrator_t2_respond_error(
request, reason=T2ProviderFailureReason.RESOURCE_UNKNOWN)
return
log.debug("Response status: %s", resp.status_code)
resp.raise_for_status()
if resp.ok:
try:
result = resp.json()
except: # pylint: disable=broad-except
log.error(
"Could not parse JSON from response for Equnr %s",
equnr,
exc_info=DEBUG_ENABLED)
else:
self.__integrator_t2_test_respond(result, request)
return
except requests.exceptions.HTTPError as ex:
log.error("__get_document_by_type %s with Equnr: %s", ex, equnr)
except requests.exceptions.RequestException as ex:
log.error(ex, exc_info=DEBUG_ENABLED)
self.__integrator_t2_respond_error(request)
def __t2_do_document_req(self, request):
decoded = json.loads(request.data.decode('utf-8'))
try:
instid = decoded['instid']
except KeyError:
log.warning('instid not in request')
self.__integrator_t2_respond_error(request)
return
log.debug("Getting document by instid: %s", instid)
equipment_document_single = self.__sap_config_info.eq_doc_single.format(
instid=instid)
log.debug("Calling Document Single endpoint: %s",
equipment_document_single)
try:
resp = requests.get(equipment_document_single,
auth=(self.__sap_config_info.usr,
self.__sap_config_info.pwd),
verify=False,
timeout=self.__sap_config_info.timeout)
if resp.status_code == requests.codes.not_found: # pylint: disable=no-member
self.__integrator_t2_respond_error(
request, reason=T2ProviderFailureReason.RESOURCE_UNKNOWN)
return
log.debug("Response status: %s", resp.status_code)
resp.raise_for_status()
if resp.ok:
try:
result = resp.json()
except: # pylint: disable=broad-except
log.error(
"Could not parse JSON from response for instid %s",
instid,
exc_info=DEBUG_ENABLED)
else:
self.__integrator_t2_respond(result, request)
return
except requests.exceptions.HTTPError as ex:
log.error("__get_document_by_instid %s with instid: %s", ex,
instid)
except requests.exceptions.RequestException as ex:
log.error(ex, exc_info=DEBUG_ENABLED)
self.__integrator_t2_respond_error(request)
def __get_values(self, results, enable_document_hack=False, mock=False):
"""Get values from the dictionary and update it in a formatted way"""
for item in results:
item['Datum'] = int(search(r'\d+', item['Datum']).group())
# If there is an equpiment number, fetch the document details
# and append them to this item
if enable_document_hack:
equnr = '10000018'
enable_document_hack = False
elif not mock:
try:
equnr = item['Equnr']
log.debug("Getting documents for Equnr: %s", equnr)
except KeyError:
log.error("\'Equnr\' key not found for this item")
item['Documents'] = []
continue
if mock:
with open(MOCK_DOCUMENT_FILE, 'r') as dfp:
results = json.load(dfp)
else:
results = self.__get_document_by_equnr(equnr)
if results is None:
item['Documents'] = []
continue
try:
documents = results['d']['results']
except KeyError:
log.error("KeyError exception in __get_data_for_asset()")
item['Documents'] = []
continue
# Strip date string values and convert to epoch longs
for document in documents:
document['Crdat'] = int(
search(r'\d+', document['Crdat']).group())
document['Chdat'] = int(
search(r'\d+', document['Chdat']).group())
item['Documents'] = documents
def __get_data_for_asset(self, asset_id):
log.debug("Get Equipment History Data for: %s", asset_id)
enable_document_hack = False
if asset_id in ('1000021', '1000015'):
enable_document_hack = True
asset_id = 'WEBER-TEST-01'
equipment_history_endpoint = self.__sap_config_info.eq_hist_endp.format(
asset_id=asset_id)
log.debug("Calling Equipment History endpoint: %s",
equipment_history_endpoint)
try:
resp = requests.get(equipment_history_endpoint,
auth=(self.__sap_config_info.usr,
self.__sap_config_info.pwd),
verify=False,
timeout=self.__sap_config_info.timeout)
except requests.exceptions.RequestException:
log.error("RequestException in __get_data_for_asset()")
return None
results = None
log.debug("Response status: %s", resp.status_code)
if resp.status_code == requests.codes['ok']:
try:
data = resp.json()
except: # pylint: disable=broad-except
log.error("Could not parse JSON from response for asset %s",
asset_id,
exc_info=DEBUG_ENABLED)
return None
try:
results = data['d']['results']
except KeyError:
log.error("Could not find results in response for asset %s",
asset_id)
return None
self.__get_values(results,
enable_document_hack=enable_document_hack)
else:
log.error("Endpoint response failed: %s", resp.status_code)
return results
def __get_document_by_equnr(self, equnr):
if not equnr:
return None
log.debug("Getting documents for Equnr: %s", equnr)
equipment_document_endpoint = self.__sap_config_info.eq_doc_endp.format(
equnr=equnr)
log.debug("Calling Equipment Document endpoint: %s",
equipment_document_endpoint)
try:
resp = requests.get(equipment_document_endpoint,
auth=(self.__sap_config_info.usr,
self.__sap_config_info.pwd),
verify=False,
timeout=self.__sap_config_info.timeout)
log.debug("Response status: %s", resp.status_code)
if resp.status_code == requests.codes.not_found: # pylint: disable=no-member
return None
resp.raise_for_status()
if resp.ok:
try:
return resp.json()
except: # pylint: disable=broad-except
log.error(
"Could not parse JSON from response for equnr %s",
equnr,
exc_info=DEBUG_ENABLED)
else:
log.error("Endpoint response failed: %s", resp.status_code)
except requests.exceptions.HTTPError as ex:
log.error("__get_document_by_equnr %s with equnr: %s", ex, equnr)
except requests.exceptions.RequestException as ex:
log.error(ex, exc_info=DEBUG_ENABLED)
return None
def __create_event_and_publish(self, asset_id, data):
log.debug("Publish event for %s", asset_id)
# Publish the event based on the document type
for item in data:
if not item.get('Documents'):
log.error(
"No documents found in equipment history for asset %s",
asset_id)
continue
try:
doctype = item['Doctype']
except KeyError:
log.error("KeyError exception in __process_data")
continue
event = self.__create_document_event(asset_id, doctype, item)
if not event:
log.error("Could not create document event for this asset")
continue
log.debug("Event: %s", event)
self.__integrator.publish_event(event, retry=True)
self.__cache_asset_data_for(asset_id, data)
def __process_data(self):
log.debug("Processing Equipment History")
for asset_id in list(self.__assets):
log.debug("Processing asset: %s", asset_id)
data = self.__get_data_for_asset(asset_id)
if data and self.__has_asset_data_changed_for(asset_id, data):
try:
self.__create_event_and_publish(asset_id, data)
except ShutdownRequested:
log.debug("Shutdown requested while publishing event")
return
except AssetUnknown:
pass
@classmethod
def __create_document_event(cls, asset_id, doctype, item):
try:
event_time = item['Datum']
except KeyError:
log.error("Datum KeyError for asset_id %s", asset_id)
return None
try:
event_time = datetime.datetime.utcfromtimestamp(event_time // 1000)
except OverflowError:
log.error("Could not create a valid datetime from %s", event_time)
return None
log.info("Creating document event for: %s", doctype)
doctypes = {
'DELI': SapEquipmentHistoryDeliverySet,
'MAIN': SapEquipmentHistoryMaintenanceContractSet,
'MOVE': SapEquipmentHistoryMaterialMovementSet,
'INLO': SapEquipmentHistoryInspectionLotSet,
'PROD': SapEquipmentHistoryProductionOrderSet,
'INVE': SapEquipmentHistoryPhysicalInventorySet,
'PURO': SapEquipmentHistoryPurchaseOrderSet,
'PMOD': SapEquipmentHistoryPmOrderSet,
'NOTI': SapEquipmentHistoryNotificationSet,
'HIST': SapEquipmentHistoryInstallationHistorySet
}
try:
return doctypes[doctype](asset_id, data=item, time=event_time)
except KeyError:
log.error("Unknown document type: %s", doctype)
return None
@classmethod
def __compute_data_hash(cls, data):
jdata = json.dumps(data, sort_keys=True, separators=(',', ':'))
return hashlib_md5(jdata.encode('utf8')).hexdigest()
# Checks to see if the given data for the asset has changed
# since it was last processed.
def __has_asset_data_changed_for(self, asset_id, data):
log.info("Checking asset cache for: %s", asset_id)
try:
asset_id_hash = self.__data_cache.get_attr(asset_id, 'hash')
except KeyError:
# No cache so this is new data
return True
data_hash = self.__compute_data_hash(data)
if asset_id_hash['hash'] != data_hash:
# data has changed
return True
# Nothing has changed for this data
return False
# After publishing the event, update the cache
def __cache_asset_data_for(self, asset_id, data):
log.info("Cache asset for: %s", asset_id)
data_hash = self.__compute_data_hash(data)
self.__data_cache.mark_as_known(asset_id, hash=data_hash)
class SAPMasterDataIntegrator(
IntegratorCallbacks, ThingRunner
): #### IntegratorCallbacks is might be for feed data and action requests stoaring, ?@ where does this feed data and action requests stoaring...????
def __init__(self, config, agent_config
): ### where does config, agent_config came from...?????
super().__init__(
config=agent_config
) ### reinitializing class 'ThingRunner' using super(), in nested
if not (isinstance(config, dict)
and all(section in config
for section in ('integrator', 'config'))):
raise ValueError(
'Configuration invalid / missing required section')
#?? # Whilst the integrator core requires particular configuration, top-level sections could be defined to provide ###?? which top level section???
# parameters specific to this integrator.
self.__integrator = Integrator(config['integrator'], self.client, self)
self.__assets = set() ### setting up asset, but where and how ????
self.__config = config
self.__data_cache = self.__config['config'][
'data-cache'] ### this might be nested dict, 'self.__data_cache = value'
def on_startup(self):
log.debug('SAP Master Data Integrator Startup'
) ### what data and how it comes...? what is a format...???
self.__integrator.start()
def main(self):
log.debug('SAP Master Data Integrator Running'
) ### what data and how it comes...? what is a format...??
loop_time = self.__config['config'][
'loop_time'] ### how loop_time set up...???
while not self.wait_for_shutdown(
loop_time
): ### waiting for shut down untill loop_time, might be stopping the searching of asset
self._process_master_data(
) ### calling '_process_master_data()', @down.
def on_shutdown(
self, exc_info
): ### if commanded to shutdown then asking integrator to stop.
log.debug('SAP Master Data Integrator Shutdown'
) ### what data and how it comes...? what is a format...??
self.__integrator.stop() ### stop command for integrator.
# for IntegratorCallbacks
def on_asset_created(self, asset_id): ### when it gets called...???
log.debug('Asset created: %s', asset_id)
self.__assets.add(asset_id)
# for IntegratorCallbacks
def on_asset_deleted(self, asset_id): ### when it gets called...???
log.debug('Asset deleted: %s', asset_id)
self.__assets.discard(asset_id)
# for IntegratorCallbacks
def on_t2_request(
self, request
): ### when it gets called...??? ### T2 request comes from app to sys
pass
def _get_data_for_asset(
self,
asset_id): ### geting call from _process_master_data() @down.
log.info("Get Master Data for: %s", asset_id)
data = None
if self.__config['config'][
'use_mock_data'] == 1: ### cheacking for mock data present or not
log.debug("Using mock master data")
with open(self.MOCK_DATA_FILE, mode="r", encoding="utf-8") as f:
data = json.load(f)
# Strip junk from dates (string) and convert to epoch ints
datab = data['d']['results'][0]['Datab'] ### might be in binary
data['d']['results'][0]['Datab'] = int(
re.findall(r'\d+', datab)[0])
datbi = data['d']['results'][0]['Datbi'] ### might be in int
data['d']['results'][0]['Datbi'] = int(
re.findall(r'\d+', datbi)[0])
else:
endpoint = self.__config['config']['bomgar']['endpoint']
usr = self.__config['config']['bomgar']['usr']
pwd = self.__config['config']['bomgar']['pwd']
key = 'config.enable_sap_sample_serial_hack'
if NestedConfig.get(self.__config,
key,
required=False,
default=False,
check=bool):
if asset_id == '1000021' or asset_id == '1000015':
asset_id = '4711-001'
endpoint = endpoint.replace('XXX_ASSET_ID_XXX', asset_id)
timeout = int(self.__config['config']['bomgar']['timeout'])
log.debug("Calling: %s", endpoint)
try:
resp = requests.get(endpoint,
auth=(usr, pwd),
verify=False,
timeout=timeout)
log.debug("Response status: %s", resp.status_code)
if resp.status_code == requests.codes['ok']:
data = resp.json()
if len(data['d']['results']) == 0:
return data
# Strip junk from dates (string) and convert to ints
datab = data['d']['results'][0]['Datab']
data['d']['results'][0]['Datab'] = int(
re.findall(r'\d+', datab)[0])
datbi = data['d']['results'][0]['Datbi']
data['d']['results'][0]['Datbi'] = int(
re.findall(r'\d+', datbi)[0])
except requests.exceptions.RequestException as ex:
log.error(ex)
return data
def _process_master_data(
self
): ### getting call from 'main()' @up, (waiting for shutdown - within loop_time)
log.debug("Processing Master Data")
for asset_id in list(
self.__assets
): ## 'asset_id' came from where? # might be cheacking asset by asset_id in asset pool OR in event list i.e. kafka
log.debug("Processing asset: %s", asset_id)
data = self._get_data_for_asset(
asset_id) ### calling '_get_data_for_asset () @up
if data is not None:
### means might be kafka having something
import ipdb
ipdb.set_trace()
if self._has_asset_data_changed_for(
asset_id, data
): #### calling it to '_has_asset_data_changed_for ()' @down
log.info("Publish event for %s", asset_id)
if len(data['d']['results']) == 0:
continue
master_data = data['d']['results'][0]
event_time = master_data.get('Datab', None)
if event_time is not None:
try:
event_time = datetime.datetime.utcfromtimestamp(
event_time // 1000)
except Exception as ex:
log.error(
"Could not create a valid datetime from %s",
event_time)
log.error(ex)
event = SapMasterDataSet(
asset_id, data=master_data, time=event_time
) ### SapMasterDataSet @(kafka) extracting event history
log.debug("Event: %s", event)
try:
self.__integrator.publish_event(
event) ### publish_event(event) @integrator API
self._cache_asset_data_for(
asset_id,
data) ### calling '_cache_asset_data_for()
# These will all retry
except EventPublishFailure as ex:
log.error("Event Publish Failure: %s", ex)
except AssetUnknown as ex:
pass
# Checks to see if the given data for the asset has changed
# since it was last processed.
##### _has_asset_data_changed_for will be same for all integrator
def _has_asset_data_changed_for(
self, asset_id, data
): ### 'data' came from _get_data_for_asset(). ### getting call from '_process_master_data()' @up
log.info("Checking asset cache for: %s", asset_id
) ### what is 'asset cache' here, why we are checking...???
if not os.path.exists(
self.__data_cache
): ### asking whether 'asset cache', if not might be need to append. (might be in kafka(this might be an event))
# No cache so this is new data
return True
filename = asset_id + '.json' ### created file with '.json' extension
file_path = os.path.join(
self.__data_cache,
filename) ### might be putting '__data_cache' in 'filename'
###
if not os.path.isfile(
file_path
): ### checking file in the existed file path (location)
# No file exists so this is new data
return True
with open(file_path, mode="r",
encoding="utf-8") as f: ### opening the newfile
cached_data_hash = f.read()
data_hash = self.__compute_data_hash(
data) ### calling '__compute_data_hash(data)' @ down.
if cached_data_hash != data_hash: ### this might be cross cheacking for format
os.remove(file_path)
# The data has changed so flush the cache
return True
# Nothing has changed for this data
return False
@classmethod
def __compute_data_hash(cls, data):
jdata = json.dumps(data, sort_keys=True, separators=(',', ':'))
return hashlib_md5(jdata.encode('utf8')).hexdigest()
def _cache_asset_data_for(self, asset_id, data):
log.info("Cache asset for: %s", asset_id)
if not os.path.exists(self.__data_cache):
log.debug("Creating data cache")
os.makedirs(self.__data_cache, exist_ok=True)
filename = asset_id + '.json'
file_path = os.path.join(self.__data_cache, filename)
if not os.path.isfile(file_path):
with open(file_path, mode="w", encoding="utf-8") as f:
f.write(self.__compute_data_hash(data))
log.debug("Caching data for asset %s", asset_id)
MOCK_DATA_FILE = os.path.join('cfg', 'mock-master-data.json')
class SapWarrantyRecallIntegrator(IntegratorCallbacks, ThingRunner):
""" SapWarrantyRecallIntegrator initializing """
def __init__(self, config, agent_config):
super().__init__(config=agent_config)
if not (isinstance(config, dict)
and all(section in config
for section in ('integrator', 'config'))):
raise ValueError(
'Configuration invalid / missing required section')
# Whilst the integrator core requires particular configuration, top-level sections could be defined to provide
# parameters specific to this integrator.
self.__integrator = Integrator(config['integrator'], self.client, self)
self.__assets = set()
self.__config = config
# data cache used to check that the asset has been changed or not before publishing the event
self.__data_cache = get_cache(
config, config_path='integrator.asset.cache.method')
self.__use_mock_data = NestedConfig.get(self.__config,
'config.use_mock_data',
required=False,
default=False)
self.__loop_time = NestedConfig.get(self.__config,
'config.loop_time',
required=False,
default=5,
check=non_negative_int)
self.__sap_config_info = SapConfig(
equnr_endpoint=NestedConfig.get(self.__config,
'config.sap.equnr_endpoint',
required=True,
check=non_empty_str),
sernr_endpoint=NestedConfig.get(self.__config,
'config.sap.sernr_endpoint',
required=True,
check=non_empty_str),
usr=NestedConfig.get(self.__config,
'config.sap.usr',
required=True,
check=non_empty_str),
pwd=NestedConfig.get(self.__config,
'config.sap.pwd',
required=True,
check=non_empty_str),
timeout=int(
NestedConfig.get(self.__config,
'config.sap.timeout',
required=False,
default=10,
check=non_negative_int)))
def on_startup(self):
""" Starts the integrator. Other public methods herein must not be called beforehand.
Should be called after starting Iotic agent.
Alternatively use with keyword on the instance. """
log.info('Sap Warranty Recall Integrator Startup')
self.__data_cache.start()
self.__integrator.start()
def main(self):
""" Sap Warranty Recall Integrator Started Running """
log.info('Sap Warranty Recall Integrator Running')
self.__process_data()
loop_time = self.__loop_time
while not self.wait_for_shutdown(loop_time):
self.__process_data()
def on_shutdown(self, exc_info):
""" Stops the integrator. Should be called before shutting down the Iotic agent. """
log.info('Sap Warranty Recall Integrator Shutdown')
self.__integrator.stop()
self.__data_cache.stop()
# for IntegratorCallbacks
def on_asset_created(self, asset_id):
""" A new asset has been created.
Called once for each known asset on startup as well as whenever
a new asset appears whilst the integrator is running. """
log.info('Asset created: %s', asset_id)
self.__assets.add(asset_id)
# for IntegratorCallbacks
def on_asset_deleted(self, asset_id):
""" An asset has been deleted.
Called whenever an asset has been removed and should no longer be considered by the integrator.
Note: This is NOT called if an asset has been deleted whilst the integrator is not running. """
log.info('Asset deleted: %s', asset_id)
self.__assets.discard(asset_id)
# for IntegratorCallbacks
def on_t2_request(self, request):
""" A new type2 request has been made for a particular asset.
request - instance of T2Request """
log.info('T2 request: %s', request)
def __get_data_for_asset(self, asset_id):
""" returns Sap Warranty Recall data for asset_id """
log.info("Get Sap Warranty Recall data for: %s", asset_id)
data = None
if self.__use_mock_data == 1:
log.debug("Using mock data")
with open(MOCK_DATA_FILE, mode="r", encoding="utf-8") as f:
results = json.load(f)
try:
data = results['d']['results']
# Strip date string values and convert to epoch longs
for item in data:
item['Refdt'] = int(re.findall(r'\d+', item['Refdt'])[0])
except KeyError as ex:
log.error(ex)
else:
# asset_id hack for RR Dev environment
# The dev environment SapWarrantyRecall API uses a specific asset_id we will swap two of our test IDs.
if asset_id in ('1000021', '1000015'):
asset_id = '5242454668'
endpoint = self.__sap_config_info.sernr_endpoint.format(
asset_id=asset_id)
usr = self.__sap_config_info.usr
pwd = self.__sap_config_info.pwd
timeout = self.__sap_config_info.timeout
log.debug("Calling: %s", endpoint)
data = self.__call_data(endpoint, usr, pwd, timeout)
return data
@classmethod
def __call_data(cls, endpoint, usr, pwd, timeout):
try:
resp = requests.get(endpoint,
auth=(usr, pwd),
verify=False,
timeout=timeout)
log.debug("Response status: %s", resp.status_code)
if resp.status_code == requests.codes['ok']:
results = resp.json()
try:
data = results['d']['results']
# Strip date string values and convert to epoch longs
for item in data:
item['Refdt'] = int(
re.findall(r'\d+', item['Refdt'])[0])
except ValueError as ex:
log.error(ex)
else:
log.error("Endpoint response failed: %s", resp.status_code)
except requests.exceptions.RequestException as ex:
log.error(ex)
return data
def __process_data(self):
""" Processing Sap Warranty Recalls data"""
log.info("Processing Sap Warranty Recalls")
for asset_id in list(self.__assets):
log.debug("Processing asset: %s", asset_id)
data = self.__get_data_for_asset(asset_id)
if data and self.__has_asset_data_changed_for(asset_id, data):
event = SapWarrantyRecallSet(asset_id, data=data)
log.debug("Publishing event: %s", event)
try:
self.__integrator.publish_event(event)
self.__cache_asset_data_for(asset_id, data)
# These will all retry
except EventPublishFailure as ex:
log.error("Event Publish Failure: %s", ex)
except AssetUnknown:
pass
@classmethod
def __compute_data_hash(cls, data):
""" computing data"""
jdata = json.dumps(data, sort_keys=True, separators=(',', ':'))
return hashlib_md5(jdata.encode('utf8')).hexdigest()
# Checks to see if the given data for the asset has changed
# since it was last processed.
def __has_asset_data_changed_for(self, asset_id, data):
""" Checking wheather asset cache data has changed for asset_id or not """
log.info("Checking asset cache for: %s", asset_id)
try:
asset_id_hash = self.__data_cache.get_attr(asset_id, 'hash')
except KeyError:
# No cache so this is new data
return True
data_hash = self.__compute_data_hash(data)
if asset_id_hash['hash'] != data_hash:
# data has changed
return True
# Nothing has changed for this data
return False
# After publishing the event, update the cache
def __cache_asset_data_for(self, asset_id, data):
""" updating Cache asset data for asset_id """
log.info("Cache asset for: %s", asset_id)
data_hash = self.__compute_data_hash(data)
self.__data_cache.mark_as_known(asset_id, hash=data_hash)
class SAPBomAsBuiltIntegrator(IntegratorCallbacks, ThingRunner):
__TRANSFER_KEYS = frozenset(
("ParRecno", "SonRecno", "Matnr", "Descr", "Valfr", "Valto"))
def __init__(self, config, agent_config):
super().__init__(config=agent_config)
if not (isinstance(config, dict)
and all(section in config
for section in ('integrator', 'config'))):
raise ValueError(
'Configuration invalid / missing required section')
# Whilst the integrator core requires particular configuration, top-level sections could be defined to provide
# parameters specific to this integrator.
self.__integrator = Integrator(config['integrator'], self.client, self)
self.__assets = set()
self.__config = config
self.__data_cache = self.__config['config']['data-cache']
self.__req_pool = ThreadPoolExecutor(
max_workers=self.__config['config']['workers'])
def on_startup(self):
log.debug('SAP Bom As Built Integrator Startup')
self.__integrator.start()
def main(self):
log.debug('SAP Bom As Built Integrator Running')
loop_time = self.__config['config']['loop_time']
while not self.wait_for_shutdown(loop_time):
self._process_sap_data()
def on_shutdown(self, exc_info):
log.debug('SAP Bom As Built Integrator Shutdown')
self.__integrator.stop()
# for IntegratorCallbacks
def on_asset_created(self, asset_id):
log.debug('Asset created: %s', asset_id)
self.__assets.add(asset_id)
# for IntegratorCallbacks
def on_asset_deleted(self, asset_id):
log.debug('Asset deleted: %s', asset_id)
self.__assets.discard(asset_id)
# for IntegratorCallbacks
def on_t2_request(self, request):
self.__req_pool.submit(self.__process_t2, request)
# Wrap since run via thread pool without handling return/exception
@log_exceptions(log)
def __process_t2(self, request):
log.info('New T2 req for %s - %s(%r)', request.asset_id, request.type_,
request.data)
if request.type_ != T2_REQUEST_SAP_BOMASMAINT:
log.warning('Ignoring unknown request type %s', request.type_)
return
self.__t2_do_bommaint(request)
def __t2_do_bommaint(self, request):
decoded = json.loads(request.data.decode('utf-8'))
data = self._get_bom_as_maintained(request.asset_id, decoded["Valfr"])
if data:
to_send = self.__tidy_dict(data['d']['results'])
self.__integrator.t2_respond(request, "application/json",
json.dumps(to_send).encode('utf8'))
else:
self.__integrator.t2_respond_error(
request, T2ProviderFailureReason.REQ_UNHANDLED)
def __tidy_dict(self, results):
ret = []
for row in results:
temp = {}
for key in self.__TRANSFER_KEYS:
temp[key] = row[key]
ret.append(temp)
return ret
def _get_bom_as_maintained(self, asset_id, valid_from):
log.info("Get Bom As Maintained Data for: %s %s", asset_id, valid_from)
data = None
if self.__config['config']['use_mock_data'] == 1:
log.debug("Using mock bom data")
with open(self.MOCK_DATA_FILE, mode="r", encoding="utf-8") as f:
data = json.load(f)
else:
endpoint = self.__config['config']['bomgar']['endpoint_maint']
usr = self.__config['config']['bomgar']['usr']
pwd = self.__config['config']['bomgar']['pwd']
key = 'config.enable_sap_sample_serial_hack'
if NestedConfig.get(self.__config,
key,
required=False,
default=False,
check=bool):
if asset_id == '1000021' or asset_id == '1000015':
asset_id = '526104875'
endpoint = endpoint.replace('XXX_ASSET_ID_XXX', asset_id)
endpoint = endpoint.replace('XXX_VALID_FROM_XXX', valid_from)
timeout = int(self.__config['config']['bomgar']['timeout'])
log.debug("Calling: %s", endpoint)
try:
resp = requests.get(endpoint,
auth=(usr, pwd),
verify=False,
timeout=timeout)
log.debug("Response status: %s", resp.status_code)
if resp.status_code == requests.codes['ok']:
data = resp.json()
except requests.exceptions.RequestException as ex:
log.error(ex)
return data
def _get_data_for_asset(self, asset_id):
log.info("Get Bom As Built Data for: %s", asset_id)
data = None
if self.__config['config']['use_mock_data'] == 1:
log.debug("Using mock bom data")
with open(self.MOCK_DATA_FILE, mode="r", encoding="utf-8") as f:
data = json.load(f)
else:
endpoint = self.__config['config']['bomgar']['endpoint']
usr = self.__config['config']['bomgar']['usr']
pwd = self.__config['config']['bomgar']['pwd']
key = 'config.enable_sap_sample_serial_hack'
if NestedConfig.get(self.__config,
key,
required=False,
default=False,
check=bool):
if asset_id == '1000021' or asset_id == '1000015':
asset_id = '526104875'
endpoint = endpoint.replace('XXX_ASSET_ID_XXX', asset_id)
timeout = int(self.__config['config']['bomgar']['timeout'])
log.debug("Calling: %s", endpoint)
try:
resp = requests.get(endpoint,
auth=(usr, pwd),
verify=False,
timeout=timeout)
log.debug("Response status: %s", resp.status_code)
if resp.status_code == requests.codes['ok']:
data = resp.json()
except requests.exceptions.RequestException as ex:
log.error(ex)
return data
def _process_sap_data(self):
log.debug("Processing Bom As Built")
for asset_id in list(self.__assets):
log.debug("Processing asset: %s", asset_id)
data = self._get_data_for_asset(asset_id)
if data is not None:
if self._has_asset_data_changed_for(asset_id, data):
log.info("Publish event for %s", asset_id)
items = data['d']['results']
parents = [
i for i in items if i.get('ParRecno', None) == ''
]
event_time = None
if parents:
event_time = parents[0].get('Valfr', None)
if event_time:
try:
event_time = datetime.datetime.strptime(
event_time, '%Y%m%d%H%M%S')
except Exception as ex:
log.error(
"Could not create a valid datetime from %s",
event_time)
log.error(ex)
event = SapBomAsBuiltSet(asset_id,
data=data['d']['results'],
time=event_time)
log.debug("Event: %s", event)
try:
self.__integrator.publish_event(event)
self._cache_asset_data_for(asset_id, data)
# These will all retry
except EventPublishFailure as ex:
log.error("Event Publish Failure: %s", ex)
except AssetUnknown as ex:
pass
# Checks to see if the given data for the asset has changed
# since it was last processed.
def _has_asset_data_changed_for(self, asset_id, data):
log.info("Checking asset cache for: %s", asset_id)
if not os.path.exists(self.__data_cache):
# No cache so this is new data
return True
filename = asset_id + '.json'
file_path = os.path.join(self.__data_cache, filename)
if not os.path.isfile(file_path):
# No file exists so this is new data
return True
with open(file_path, mode="r", encoding="utf-8") as f:
cached_data_hash = f.read()
data_hash = self.__compute_data_hash(data)
if cached_data_hash != data_hash:
os.remove(file_path)
# The data has changed so flush the cache
return True
# Nothing has changed for this data
return False
@classmethod
def __compute_data_hash(cls, data):
jdata = json.dumps(data, sort_keys=True, separators=(',', ':'))
return hashlib_md5(jdata.encode('utf8')).hexdigest()
def _cache_asset_data_for(self, asset_id, data):
log.info("Cache asset for: %s", asset_id)
if not os.path.exists(self.__data_cache):
log.debug("Creating data cache")
os.makedirs(self.__data_cache, exist_ok=True)
filename = asset_id + '.json'
file_path = os.path.join(self.__data_cache, filename)
if not os.path.isfile(file_path):
with open(file_path, mode="w", encoding="utf-8") as f:
f.write(self.__compute_data_hash(data))
log.debug("Caching data for asset %s", asset_id)
MOCK_DATA_FILE = os.path.join('cfg', 'mock-ibase-data.json')
class SAPMasterDataIntegrator(IntegratorCallbacks, ThingRunner): #### IntegratorCallbacks is might be for feed data and action requests stoaring, ?@ where does this feed data and action requests stoaring...????
def __init__(self, config, agent_config): ### where does config, agent_config came from...?????
super().__init__(config=agent_config) ### reinitializing class 'ThingRunner' using super(), in nested
if not (isinstance(config, dict) and all(section in config for section in ('integrator', 'config'))):
raise ValueError(
'Configuration invalid / missing required section')
#?? # Whilst the integrator core requires particular configuration, top-level sections could be defined to provide ###?? which top level section???
# parameters specific to this integrator.
self.__integrator = Integrator(config['integrator'], self.client, self)
self.__assets = set() ### setting up asset, but where and how ????
self.__config = config
self.__data_cache = self.__config['config']['data-cache'] ### this might be nested dict, 'self.__data_cache = value'
def on_startup(self):
log.debug('SAP Master Data Integrator Startup') ### what data and how it comes...? what is a format...???
self.__integrator.start()
def main(self):
log.debug('SAP Master Data Integrator Running') ### what data and how it comes...? what is a format...??
loop_time = self.__config['config']['loop_time'] ### how loop_time set up...???
while not self.wait_for_shutdown(loop_time): ### waiting for shut down untill loop_time, might be stopping the searching of asset
self._process_master_data() ### calling '_process_master_data()', @down.
def on_shutdown(self, exc_info): ### if commanded to shutdown then asking integrator to stop.
log.debug('SAP Master Data Integrator Shutdown') ### what data and how it comes...? what is a format...??
self.__integrator.stop() ### stop command for integrator.
# for IntegratorCallbacks
def on_asset_created(self, asset_id): ### when it gets called...???
log.debug('Asset created: %s', asset_id)
self.__assets.add(asset_id)
# for IntegratorCallbacks
def on_asset_deleted(self, asset_id): ### when it gets called...???
log.debug('Asset deleted: %s', asset_id)
self.__assets.discard(asset_id)
# for IntegratorCallbacks
def on_t2_request(self, request): ### when it gets called...??? ### T2 request comes from app to sys
pass
def _get_data_for_asset(self, asset_id): ### geting call from _process_master_data() @down.
log.info("Get Master Data for: %s", asset_id)
data = None
if self.__config['config']['use_mock_data'] == 1:
log.debug("Using mock master data")
with open(self.MOCK_DATA_FILE, mode="r", encoding="utf-8") as f:
data = json.load(f)
# Strip junk from dates (string) and convert to epoch ints
datab = data['d']['results'][0]['Datab'] ### might be in binary
data['d']['results'][0]['Datab'] = int(
re.findall(r'\d+', datab)[0])
datbi = data['d']['results'][0]['Datbi'] ### might be in int
data['d']['results'][0]['Datbi'] = int(
re.findall(r'\d+', datbi)[0])
else:
endpoint = self.__config['config']['bomgar']['endpoint']
usr = self.__config['config']['bomgar']['usr']
pwd = self.__config['config']['bomgar']['pwd']
key = 'config.enable_sap_sample_serial_hack'
if NestedConfig.get(self.__config, key, required=False, default=False, check=bool):
if asset_id == '1000021' or asset_id == '1000015':
asset_id = '4711-001'
endpoint = endpoint.replace('XXX_ASSET_ID_XXX', asset_id)
timeout = int(self.__config['config']['bomgar']['timeout'])
log.debug("Calling: %s", endpoint)
try:
resp = requests.get(endpoint, auth=(
usr, pwd), verify=False, timeout=timeout)
log.debug("Response status: %s", resp.status_code)
if resp.status_code == requests.codes['ok']:
data = resp.json()
if len(data['d']['results']) == 0:
return data
# Strip junk from dates (string) and convert to ints
datab = data['d']['results'][0]['Datab']
data['d']['results'][0]['Datab'] = int(
re.findall(r'\d+', datab)[0])
datbi = data['d']['results'][0]['Datbi']
data['d']['results'][0]['Datbi'] = int(
re.findall(r'\d+', datbi)[0])
except requests.exceptions.RequestException as ex:
log.error(ex)
return data
def _process_master_data(self): ### getting call from 'main()' @up, (waiting for shutdown - within loop_time)
log.debug("Processing Master Data")
for asset_id in list(self.__assets): ## 'asset_id' came from where? # might be cheacking asset by asset_id in asset pool OR in event list i.e. kafka
log.debug("Processing asset: %s", asset_id)
data = self._get_data_for_asset(asset_id) ### calling '_get_data_for_asset () @up
if data is not None: ### means might be kafka having something
if self._has_asset_data_changed_for(asset_id, data): #### calling it to '_has_asset_data_changed_for ()' @down
log.info("Publish event for %s", asset_id)
if len(data['d']['results']) == 0:
continue
master_data = data['d']['results'][0]
event_time = master_data.get('Datab', None)
if event_time is not None:
try:
event_time = datetime.datetime.utcfromtimestamp(event_time // 1000)
except Exception as ex:
log.error("Could not create a valid datetime from %s", event_time)
log.error(ex)
event = SapMasterDataSet(asset_id, data=master_data, time=event_time) ### SapMasterDataSet @(kafka) extracting event history
log.debug("Event: %s", event)
try:
self.__integrator.publish_event(event) ### publish_event(event) @integrator API
self._cache_asset_data_for(asset_id, data) ### calling '_cache_asset_data_for()
class TalendTimDocumentIntegrator(IntegratorCallbacks, ThingRunner):
""" TalendTimDocumentIntegrator initializing """
def __init__(self, config, agent_config):
super().__init__(config=agent_config)
if not (isinstance(config, dict) and all(section in config for section in ('integrator', 'config'))):
raise ValueError('Configuration invalid / missing required section')
# Whilst the integrator core requires particular configuration, top-level sections could be defined to provide
# parameters specific to this integrator.
self.__integrator = Integrator(config['integrator'], self.client, self)
self.__assets = set()
self.__config = config
# data cache used to check that the asset has been changed or not before publishing the event
self.__data_cache = get_cache(config, config_path='integrator.asset.cache.method')
self.__use_mock_data = NestedConfig.get(self.__config, 'config.use_mock_data', required=False, default=False)
self.__workers = NestedConfig.get(self.__config,
'config.workers', required=False, default=1, check=non_negative_int)
self.__loop_time = NestedConfig.get(self.__config,
'config.loop_time', required=False, default=5, check=non_negative_int)
self.__req_pool = ThreadPoolExecutor(max_workers=self.__workers)
self.__talend_config_info = TalendConfig(
endpoint=NestedConfig.get(self.__config,
'config.talend.endpoint', required=True, check=non_empty_str),
endpoint_single=NestedConfig.get(self.__config,
'config.talend.endpoint_single', required=True, check=non_empty_str),
usr=NestedConfig.get(self.__config, 'config.talend.usr', required=True, check=non_empty_str),
pwd=NestedConfig.get(self.__config, 'config.talend.pwd', required=True, check=non_empty_str),
timeout=int(NestedConfig.get(self.__config,
'config.talend.timeout', required=False, default=10, check=non_negative_int))
)
def on_startup(self):
""" Starts the integrator. Other public methods herein must not be called beforehand.
Should be called after starting Iotic agent.
Alternatively use with keyword on the instance. """
log.info('Talend Tim Document Integrator Startup')
self.__data_cache.start()
self.__integrator.start()
def main(self):
""" Talend Tim Document Integrator Started Running """
log.info('Talend Tim Document Integrator Running')
self.__process_data()
loop_time = self.__loop_time
while not self.wait_for_shutdown(loop_time):
self.__process_data()
def on_shutdown(self, exc_info):
""" Stops the integrator. Should be called before shutting down the Iotic agent. """
log.info('Talend Tim Document Integrator Shutdown')
self.__integrator.stop()
self.__data_cache.stop()
# for IntegratorCallbacks
def on_asset_created(self, asset_id):
""" A new asset has been created.
Called once for each known asset on startup as well as whenever
a new asset appears whilst the integrator is running. """
log.info('Asset created: %s', asset_id)
self.__assets.add(asset_id)
# for IntegratorCallbacks
def on_asset_deleted(self, asset_id):
""" An asset has been deleted.
Called whenever an asset has been removed and should no longer be considered by the integrator.
Note: This is NOT called if an asset has been deleted whilst the integrator is not running. """
log.info('Asset deleted: %s', asset_id)
self.__assets.discard(asset_id)
# for IntegratorCallbacks
def on_t2_request(self, request):
""" A new type2 request has been made for a particular asset.
request - instance of T2Request """
self.__req_pool.submit(self.__process_t2, request)
# Wrap since run via thread pool without handling return/exception
@log_exceptions(log)
def __process_t2(self, request):
""" processing t2 request """
log.info('New T2 req for %s - %s(%r)', request.asset_id, request.type_, request.data)
if request.type_ != T2_REQUEST_TALEND_DOCUMENT:
log.warning('Ignoring unknown request type %s', request.type_)
return
self.__t2_do_tlddoc(request)
def __t2_do_tlddoc(self, request):
decoded = json.loads(request.data.decode('utf-8'))
data = self.__get_tim_doc(decoded["serialNumber"], decoded["documentLabel"], decoded["documentName"])
if data:
try:
self.__integrator.t2_respond(request, "application/pdf", b64decode(data))
except binascii_Error:
log.error("Failed to b64decode data")
self.__integrator.t2_respond_error(request, T2ProviderFailureReason.REQ_UNHANDLED)
else:
self.__integrator.t2_respond_error(request, T2ProviderFailureReason.REQ_UNHANDLED)
def __get_tim_doc(self, serial_no, document_label, document_name):
log.info("Get Talend doc for: %s", serial_no)
data = None
try:
if self.__use_mock_data == 1:
return mockpdf.data
except ValueError as ex:
log.error(ex)
endpoint = self.__talend_config_info.endpoint_single.\
format(asset_id=serial_no, doc_label=document_label, doc_name=document_name)
usr = self.__talend_config_info.usr
pwd = self.__talend_config_info.pwd
timeout = self.__talend_config_info.timeout
log.debug("Calling: %s", endpoint)
try:
resp = requests.get(endpoint, auth=(usr, pwd), verify=False, timeout=timeout)
log.debug("Response status: %s", resp.status_code)
if resp.text and resp.status_code == requests.codes['ok']:
try:
data = resp.json()['document']
except Exception as ex: # pylint: disable=broad-except
log.error("Could not parse JSON from response: %s", resp.text)
log.error(ex)
except requests.exceptions.RequestException as ex:
log.error(ex)
return data
def __get_data_for_asset(self, asset_id):
""" returns Talend data for asset_id """
log.info("Get Talend data for: %s", asset_id)
data = None
if self.__use_mock_data == 1:
log.debug("Using mock data")
with open(MOCK_DATA_FILE, mode="r", encoding="utf-8") as f:
data = json.load(f)
else:
# asset_id hack for RR Dev environment
# The dev environment TalendTimDocument API uses a specific asset_id we will swap two of our test IDs.
if asset_id in ('1000021', '1000015'):
asset_id = '16701003340'
usr = self.__talend_config_info.usr
pwd = self.__talend_config_info.pwd
endpoint = self.__talend_config_info.endpoint_single.format(asset_id=asset_id)
timeout = self.__talend_config_info.timeout
log.debug("Calling: %s", endpoint)
try:
resp = requests.get(endpoint, auth=(usr, pwd), verify=False, timeout=timeout)
log.debug("Response status: %s", resp.status_code)
if resp.text and resp.status_code == requests.codes['ok']:
try:
data = resp.json()
except Exception as ex:
log.error("Could not parse JSON from response: %s", resp.text)
raise ex
except requests.exceptions.RequestException as ex:
log.error(ex)
return data
def __process_data(self):
""" Processing Talend Tim Documents """
log.info("Processing Talend Tim Documents")
for asset_id in list(self.__assets):
log.debug("Processing asset: %s", asset_id)
data = self.__get_data_for_asset(asset_id)
if data and self.__has_asset_data_changed_for(asset_id, data):
event = TalendTimDocumentSet(asset_id, data=data["documentList"])
log.debug("Publishing event: %s", event)
try:
self.__integrator.publish_event(event)
self.__cache_asset_data_for(asset_id, data)
# These will all retry
except EventPublishFailure as ex:
log.error("Event Publish Failure: %s", ex)
except AssetUnknown:
pass
# Checks to see if the given data for the asset has changed
# since it was last processed.
def __has_asset_data_changed_for(self, asset_id, data):
""" Checking wheather asset cache data has changed for asset_id or not """
log.info("Checking asset cache for: %s", asset_id)
try:
asset_id_hash = self.__data_cache.get_attr(asset_id, 'hash')
except KeyError:
# No cache so this is new data
return True
data_hash = self.__compute_data_hash(data)
if asset_id_hash['hash'] != data_hash:
# data has changed
return True
# Nothing has changed for this data
return False
@classmethod
def __compute_data_hash(cls, data):
""" computing data"""
jdata = json.dumps(data, sort_keys=True, separators=(',', ':'))
return hashlib_md5(jdata.encode('utf8')).hexdigest()
# After publishing the event, update the cache
def __cache_asset_data_for(self, asset_id, data):
""" updating Cache asset data for asset_id """
log.info("Cache asset for: %s", asset_id)
data_hash = self.__compute_data_hash(data)
self.__data_cache.mark_as_known(asset_id, hash=data_hash)
class SAPEquipmentHistoryIntegrator(IntegratorCallbacks, ThingRunner):
def __init__(self, config, agent_config):
super().__init__(config=agent_config)
if not (isinstance(config, Mapping) and all(section in config for section in ('integrator', 'config'))):
raise ValueError(
'Configuration invalid / missing required section')
# parameters specific to this integrator.
self.__integrator = Integrator(config['integrator'], self.client, self)
self.__assets = set()
self.__config = config
self.__data_cache = NestedConfig.get(self.__config, 'config.data-cache', required=True, check=non_empty_str)
# Pool of workers to execture type2 requests
workers = NestedConfig.get(self.__config, 'config.workers', required=False, default=1, check=non_negative_int)
self.__req_pool = ThreadPoolExecutor(max_workers=workers)
# Validate config
self.__use_mock_data = NestedConfig.get(self.__config, 'config.use_mock_data', required=False, default=False)
self.__sap_config_info = SapConfig(
eq_hist_endp=NestedConfig.get(
self.__config, 'config.sap.equipment_history_endpoint', required=True, check=non_empty_str
),
eq_doc_endp=NestedConfig.get(
self.__config, 'config.sap.equipment_document_endpoint', required=True, check=non_empty_str
),
eq_doc_single=NestedConfig.get(
self.__config, 'config.sap.equipment_document_single', required=True, check=non_empty_str
),
usr=NestedConfig.get(
self.__config, 'config.sap.usr', required=True, check=non_empty_str
),
pwd=NestedConfig.get(
self.__config, 'config.sap.pwd', required=True, check=non_empty_str
),
timeout=int(NestedConfig.get(
self.__config, 'config.sap.timeout', required=False, default=10, check=non_negative_int
))
)
def on_startup(self):
log.info('Startup')
self.__integrator.start()
def main(self):
log.info('Running')
loop_time = NestedConfig.get(
self.__config, 'config.loop_time', required=False, default=5, check=non_negative_int
)
while not self.wait_for_shutdown(loop_time):
self.__process_data()
def on_shutdown(self, exc_info):
log.info('Shutdown')
self.__integrator.stop()
# for IntegratorCallbacks
def on_asset_created(self, asset_id):
log.info('Asset created: %s', asset_id)
self.__assets.add(asset_id)
# for IntegratorCallbacks
def on_asset_deleted(self, asset_id):
log.info('Asset deleted: %s', asset_id)
self.__assets.discard(asset_id)
# for IntegratorCallbacks
def on_t2_request(self, request):
self.__req_pool.submit(self.__process_t2, request)
# Wrap since run via thread pool without handling return/exception
@log_exceptions(log)
def __process_t2(self, request):
log.info('New T2 req for %s - %s(%r)', request.asset_id, request.type_, request.data)
if request.type_ != T2_REQUEST_SAP_DOCUMENTSINGLE:
log.warning('Ignoring unknown request type %s', request.type_)
return
self.__t2_do_document_req(request)
def __integrator_respond_error(self, request):
try:
self.__integrator.t2_respond_error(request, T2ProviderFailureReason.REQ_UNHANDLED)
except AssetUnknown:
pass
def __t2_do_document_req(self, request):
decoded = json.loads(request.data.decode('utf-8'))
try:
instid = decoded['instid']
except KeyError:
log.warning('instid not in request')
self.__integrator_respond_error(request)
result = self.__get_document_by_instid(instid)
if result:
try:
result['d']['results'][0]['String']
except KeyError:
log.error('Unknown file type')
self.__integrator_respond_error(request)
return
try:
value = result['d']['results'][0]['FileName']
except KeyError:
log.error("Error in finding mime type")
self.__integrator_respond_error(request)
return
mime = guess_type(value)[0]
if not mime:
log.error('Unknown file type')
self.__integrator_respond_error(request)
return
try:
self.__integrator.t2_respond(
request, mime, a2b_base64(result['d']['results'][0]['String'], validate=True)
)
except binascii_Error:
log.error("Failed to a2b_base64 data")
self.__integrator_respond_error(request)
else:
log.error("Failed to get Document Single for: %s. %s", instid, result)
self.__integrator_respond_error(request)
def __get_document_by_instid(self, instid):
log.debug("Getting document by instid: %s", instid)
if self.__use_mock_data:
return {"d": {"results": [
{"Instid": "123456", "FileName": "text.txt", "FileType": "ASC",
"String": "VGltIHdhcyBoZXJlISAyMDE4LTExLTMwIDE4OjA5IQ=="}
]}} # noqa
equipment_document_single = self.__sap_config_info.eq_doc_single.format(instid=instid)
log.debug("Calling Document Single endpoint: %s", equipment_document_single)
try:
resp = requests.get(
equipment_document_single,
auth=(self.__sap_config_info.usr, self.__sap_config_info.pwd),
verify=False,
timeout=self.__sap_config_info.timeout
)
log.debug("Response status: %s", resp.status_code)
except requests.exceptions.RequestException:
log.error("__get_document_by_instid() error")
return None
if resp.status_code == requests.codes['ok']:
try:
results = resp.json()
except ValueError:
log.error('Decoding JSON has failed')
return None
return results
def __get_values(self, results, enable_document_hack=False, mock=False):
"""Get values from the dictionary and update it in a formatted way"""
for item in results:
item['Datum'] = int(search(r'\d+', item['Datum']).group())
# If there is an equpiment number, fetch the document details
# and append them to this item
if enable_document_hack:
equnr = '10000018'
enable_document_hack = False
elif not mock:
try:
equnr = item['Equnr']
log.debug("Getting documents for Equnr: %s", equnr)
except KeyError:
log.error("\'Equnr\' key not found for this item")
item['Documents'] = []
continue
if mock:
with open(MOCK_DOCUMENT_FILE, 'r') as dfp:
results = json.load(dfp)
else:
results = self.__get_document_by_equnr(equnr)
try:
documents = results['d']['results']
except KeyError:
log.error("KeyError exception in __get_data_for_asset()")
item['Documents'] = []
continue
# Strip date string values and convert to epoch longs
for document in documents:
document['Crdat'] = int(search(r'\d+', document['Crdat']).group())
document['Chdat'] = int(search(r'\d+', document['Chdat']).group())
item['Documents'] = documents
def __get_mock_data(self):
log.debug("Using mock Equipment History Data data")
with open(MOCK_DATA_FILE, 'r') as f:
data = json.load(f)
try:
results = data['d']['results']
except KeyError:
log.error("KeyError exception in __get_data_for_asset")
return None
self.__get_values(results, mock=True)
return results
def __get_data_for_asset(self, asset_id):
log.debug("Get Equipment History Data for: %s", asset_id)
if self.__use_mock_data:
return self.__get_mock_data()
enable_document_hack = False
if asset_id in ('1000021', '1000015'):
enable_document_hack = True
asset_id = 'WEBER-TEST-01'
equipment_history_endpoint = self.__sap_config_info.eq_hist_endp.format(asset_id=asset_id)
log.debug("Calling Equipment History endpoint: %s", equipment_history_endpoint)
try:
resp = requests.get(
equipment_history_endpoint,
auth=(self.__sap_config_info.usr, self.__sap_config_info.pwd),
verify=False,
timeout=self.__sap_config_info.timeout
)
except requests.exceptions.RequestException:
log.error("RequestException in __get_data_for_asset()")
return None
results = None
log.debug("Response status: %s", resp.status_code)
if resp.status_code == requests.codes['ok']:
data = resp.json()
try:
results = data['d']['results']
except KeyError:
log.error("KeyError exception in __get_data_for_asset()")
return None
self.__get_values(results, enable_document_hack=enable_document_hack)
return results
def __get_document_by_equnr(self, equnr):
if not equnr:
return None
log.debug("Getting documents for Equnr: %s", equnr)
equipment_document_endpoint = self.__sap_config_info.eq_doc_endp.format(equnr=equnr)
log.debug("Calling Equipment Document endpoint: %s", equipment_document_endpoint)
try:
resp = requests.get(
equipment_document_endpoint,
auth=(self.__sap_config_info.usr, self.__sap_config_info.pwd),
verify=False,
timeout=self.__sap_config_info.timeout
)
log.debug("Response status: %s", resp.status_code)
except requests.exceptions.RequestException:
log.error("__get_document_by_equnr error")
if resp.status_code == requests.codes['ok']:
try:
results = resp.json()
except ValueError:
log.error('Decoding JSON has failed')
return None
return results
def __process_data(self):
log.debug("Processing Equipment History")
for asset_id in list(self.__assets):
log.debug("Processing asset: %s", asset_id)
data = self.__get_data_for_asset(asset_id)
if data and self.__has_asset_data_changed_for(asset_id, data):
log.debug("Publish event for %s", asset_id)
# Publish the event based on the document type
for item in data:
if not item.get('Documents'):
log.error("No documents found in equipment history for asset %s", asset_id)
continue
try:
doctype = item['Doctype']
except KeyError:
log.error("KeyError exception in __process_data")
continue
event = self.__create_document_event(asset_id, doctype, item)
if not event:
log.error("Could not create document event for this asset")
continue
log.debug("Event: %s", event)
try:
self.__integrator.publish_event(event)
# These will all retry
except EventPublishFailure as ex:
log.error("Event Publish Failure: %s", ex)
except AssetUnknown:
pass
self.__cache_asset_data_for(asset_id, data)
@classmethod
def __create_document_event(cls, asset_id, doctype, item):
try:
event_time = item['Datum']
except KeyError:
log.error("Datum KeyError for asset_id %s", asset_id)
return None
try:
event_time = datetime.datetime.utcfromtimestamp(event_time // 1000)
except OverflowError:
log.error("Could not create a valid datetime from %s", event_time)
return None
log.info("Creating document event for: %s", doctype)
doctypes = {
'DELI': SapEquipmentHistoryDeliverySet,
'MAIN': SapEquipmentHistoryMaintenanceContractSet,
'MOVE': SapEquipmentHistoryMaterialMovementSet,
'INLO': SapEquipmentHistoryInspectionLotSet,
'PROD': SapEquipmentHistoryProductionOrderSet,
'INVE': SapEquipmentHistoryPhysicalInventorySet,
'PURO': SapEquipmentHistoryPurchaseOrderSet,
'PMOD': SapEquipmentHistoryPmOrderSet,
'NOTI': SapEquipmentHistoryNotificationSet,
'HIST': SapEquipmentHistoryInstallationHistorySet
}
try:
return doctypes[doctype](asset_id, data=item, time=event_time)
except KeyError:
log.error("Unknown document type: %s", doctype)
return None
def __has_asset_data_changed_for(self, asset_id, data):
"""Checks whether the given data for the asset has changed since it was last processed"""
log.debug("Checking asset cache for: %s", asset_id)
if not os.path.exists(self.__data_cache):
# No cache so this is new data
return True
file_path = os.path.join(self.__data_cache, asset_id + '.json')
if not os.path.isfile(file_path):
# No file exists so this is new data
return True
with open(file_path, mode="r", encoding="utf-8") as f:
cached_data_hash = f.read()
data_hash = self.__compute_data_hash(data)
if cached_data_hash != data_hash:
os.remove(file_path)
# The data has changed so flush the cache
return True
# Nothing has changed for this data
log.debug("Asset %s already in cache", asset_id)
return False
@classmethod
def __compute_data_hash(cls, data):
jdata = json.dumps(data, sort_keys=True, separators=(',', ':'))
return hashlib_md5(jdata.encode('utf8')).hexdigest()
def __cache_asset_data_for(self, asset_id, data):
log.debug("Cache asset for: %s", asset_id)
if not os.path.exists(self.__data_cache):
log.debug("Creating data cache")
os.makedirs(self.__data_cache, exist_ok=True)
file_path = os.path.join(self.__data_cache, asset_id + '.json')
if not os.path.isfile(file_path):
with open(file_path, mode="w", encoding="utf-8") as f:
f.write(self.__compute_data_hash(data))
log.debug("Caching data for asset %s", asset_id)
class SAPSupersessionIntegrator(IntegratorCallbacks, ThingRunner):
def __init__(self, config, agent_config):
super().__init__(config=agent_config)
if not (isinstance(config, dict)
and all(section in config
for section in ('integrator', 'config'))):
raise ValueError(
'Configuration invalid / missing required section')
# Whilst the integrator core requires particular configuration, top-level sections could be defined to provide
# parameters specific to this integrator.
self.__integrator = Integrator(config['integrator'], self.client, self)
self.__config = config
self.__data_cache = self.__config['config']['data-cache']
self.__req_pool = ThreadPoolExecutor(
max_workers=self.__config['config']['workers'])
def on_startup(self):
log.debug('SAP Supersession Integrator Startup')
self.__integrator.start()
def main(self):
log.debug('SAP Supersession Integrator Running')
loop_time = self.__config['config']['loop_time']
while not self.wait_for_shutdown(loop_time):
pass
def on_shutdown(self, exc_info):
log.debug('SAP Supersession Integrator Shutdown')
self.__integrator.stop()
# for IntegratorCallbacks
def on_asset_created(self, asset_id):
log.debug('Asset created: %s', asset_id)
# for IntegratorCallbacks
def on_asset_deleted(self, asset_id):
log.debug('Asset deleted: %s', asset_id)
# for IntegratorCallbacks
def on_t2_request(self, request):
self.__req_pool.submit(self.__process_t2, request)
# Wrap since run via thread pool without handling return/exception
@log_exceptions(log)
def __process_t2(self, request):
log.info('New T2 req for %s - %s(%r)', request.asset_id, request.type_,
request.data)
if request.type_ != T2_REQUEST_SAP_SUPERSESSION:
log.warning('Ignoring unknown request type %s', request.type_)
return
self.__t2_do_sapspc(request)
def __t2_do_sapspc(self, request):
decoded = json.loads(request.data.decode('utf-8'))
data = self._get_data_for_material(decoded["Matnr"])
if data:
self.__integrator.t2_respond(request, "application/json",
json.dumps(data).encode('utf8'))
else:
self.__integrator.t2_respond_error(
request, T2ProviderFailureReason.REQ_UNHANDLED)
def _get_data_for_material(self, material_no):
log.info("Get Supersession Data for: %s", material_no)
data = None
if self.__config['config']['use_mock_data'] == 1:
log.debug("Using mock data")
with open(self.MOCK_DATA_FILE, mode="r", encoding="utf-8") as f:
results = json.load(f)
data = results['d']['results']
for item in data:
item.pop("__metadata",
{}) # remove the unnecessary metadata if it's there
item['Brgew'] = self._convert_to_double(item['Brgew'])
item['Ntgew'] = self._convert_to_double(item['Ntgew'])
item['Laeng'] = self._convert_to_double(item['Laeng'])
item['Breit'] = self._convert_to_double(item['Breit'])
item['Hoehe'] = self._convert_to_double(item['Hoehe'])
else:
endpoint = self.__config['config']['sap']['endpoint']
usr = self.__config['config']['sap']['usr']
pwd = self.__config['config']['sap']['pwd']
endpoint = endpoint.replace('XXX_MATERIAL_NO_XXX', material_no)
timeout = int(self.__config['config']['sap']['timeout'])
log.debug("Calling: %s", endpoint)
try:
resp = requests.get(endpoint,
auth=(usr, pwd),
verify=False,
timeout=timeout)
log.debug("Response status: %s", resp.status_code)
if resp.status_code == requests.codes['ok']:
results = resp.json()
data = results['d']['results']
for item in data:
item.pop(
"__metadata", {}
) # remove the unnecessary metadata if it's there
item['Brgew'] = self._convert_to_double(item['Brgew'])
item['Ntgew'] = self._convert_to_double(item['Ntgew'])
item['Laeng'] = self._convert_to_double(item['Laeng'])
item['Breit'] = self._convert_to_double(item['Breit'])
item['Hoehe'] = self._convert_to_double(item['Hoehe'])
except requests.exceptions.RequestException as ex:
log.error(ex)
return data
@classmethod
def _convert_to_double(cls, str_val):
try:
value = float(str_val)
return value
except:
log.error("Error converting str %s to float", str_val)
return None
MOCK_DATA_FILE = os.path.join('cfg', 'mock-data.json')
class TalendTimDocumentIntegrator(IntegratorCallbacks, ThingRunner):
def __init__(self, config, agent_config):
super().__init__(config=agent_config)
if not (isinstance(config, dict)
and all(section in config
for section in ('integrator', 'config'))):
raise ValueError(
'Configuration invalid / missing required section')
# Whilst the integrator core requires particular configuration, top-level sections could be defined to provide
# parameters specific to this integrator.
self.__integrator = Integrator(config['integrator'], self.client, self)
self.__assets = set()
self.__config = config
self.__data_cache = self.__config['config']['data-cache']
self.__req_pool = ThreadPoolExecutor(
max_workers=self.__config['config']['workers'])
def on_startup(self):
log.info('Talend Tim Document Integrator Startup')
self.__integrator.start()
def main(self):
log.info('Talend Tim Document Integrator Running')
self._process_data()
loop_time = self.__config['config']['loop_time']
while not self.wait_for_shutdown(loop_time):
self._process_data()
def on_shutdown(self, exc_info):
log.info('Talend Tim Document Integrator Shutdown')
self.__integrator.stop()
# for IntegratorCallbacks
def on_asset_created(self, asset_id):
log.info('Asset created: %s', asset_id)
self.__assets.add(asset_id)
# for IntegratorCallbacks
def on_asset_deleted(self, asset_id):
log.info('Asset deleted: %s', asset_id)
self.__assets.discard(asset_id)
# for IntegratorCallbacks
def on_t2_request(self, request):
self.__req_pool.submit(self.__process_t2, request)
# Wrap since run via thread pool without handling return/exception
@log_exceptions(log)
def __process_t2(self, request):
log.info('New T2 req for %s - %s(%r)', request.asset_id, request.type_,
request.data)
if request.type_ != T2_REQUEST_TALEND_DOCUMENT:
log.warning('Ignoring unknown request type %s', request.type_)
return
self.__t2_do_tlddoc(request)
def __t2_do_tlddoc(self, request):
decoded = json.loads(request.data.decode('utf-8'))
data = self._get_tim_doc(decoded["serialNumber"],
decoded["documentLabel"],
decoded["documentName"])
if data:
try:
self.__integrator.t2_respond(request, "application/pdf",
b64decode(data))
except binascii_Error:
log.error("Failed to b64decode data")
self.__integrator.t2_respond_error(
request, T2ProviderFailureReason.REQ_UNHANDLED)
else:
self.__integrator.t2_respond_error(
request, T2ProviderFailureReason.REQ_UNHANDLED)
def _get_tim_doc(self, serial_no, document_label, document_name):
log.info("Get Talend doc for: %s", serial_no)
data = None
if self.__config['config']['use_mock_data'] == 1:
return mockpdf.data
endpoint = self.__config['config']['talend']['endpoint_single']
usr = self.__config['config']['talend']['usr']
pwd = self.__config['config']['talend']['pwd']
endpoint = endpoint.replace('XXX_ASSET_ID_XXX', serial_no)
endpoint = endpoint.replace('XXX_DOC_LABEL_XXX', document_label)
endpoint = endpoint.replace('XXX_DOC_NAME_XXX', document_name)
timeout = int(self.__config['config']['talend']['timeout'])
log.debug("Calling: %s", endpoint)
try:
resp = requests.get(endpoint,
auth=(usr, pwd),
verify=False,
timeout=timeout)
log.debug("Response status: %s", resp.status_code)
if resp.text and resp.status_code == requests.codes['ok']:
try:
data = resp.json()['document']
except Exception as ex: # pylint: disable=broad-except
log.error("Could not parse JSON from response: %s",
resp.text)
except requests.exceptions.RequestException as ex:
log.error(ex)
return data
def _get_data_for_asset(self, asset_id):
log.info("Get Talend data for: %s", asset_id)
data = None
if self.__config['config']['use_mock_data'] == 1:
log.debug("Using mock data")
with open(self.MOCK_DATA_FILE, mode="r", encoding="utf-8") as f:
data = json.load(f)
else:
endpoint = self.__config['config']['talend']['endpoint']
usr = self.__config['config']['talend']['usr']
pwd = self.__config['config']['talend']['pwd']
key = 'config.enable_sap_sample_serial_hack'
if NestedConfig.get(self.__config,
key,
required=False,
default=False,
check=bool):
if asset_id == '1000021' or asset_id == '1000015':
asset_id = '16701003340'
endpoint = endpoint.replace('XXX_ASSET_ID_XXX', asset_id)
timeout = int(self.__config['config']['talend']['timeout'])
log.debug("Calling: %s", endpoint)
try:
resp = requests.get(endpoint,
auth=(usr, pwd),
verify=False,
timeout=timeout)
log.debug("Response status: %s", resp.status_code)
if resp.text and resp.status_code == requests.codes['ok']:
try:
data = resp.json()
except Exception as ex:
log.error("Could not parse JSON from response: %s",
resp.text)
raise ex
except requests.exceptions.RequestException as ex:
log.error(ex)
return data
def _process_data(self):
log.info("Processing Talend Tim Documents")
for asset_id in list(self.__assets):
log.debug("Processing asset: %s", asset_id)
data = self._get_data_for_asset(asset_id)
if data is not None:
if self._has_asset_data_changed_for(asset_id, data):
event = TalendTimDocumentSet(asset_id,
data=data["documentList"])
log.debug("Publishing event: %s", event)
try:
self.__integrator.publish_event(event)
self._cache_asset_data_for(asset_id, data)
# These will all retry
except EventPublishFailure as ex:
log.error("Event Publish Failure: %s", ex)
except AssetUnknown as ex:
pass
# Checks to see if the given data for the asset has changed
# since it was last processed.
def _has_asset_data_changed_for(self, asset_id, data):
log.info("Checking asset cache for: %s", asset_id)
if not os.path.exists(self.__data_cache):
# No cache so this is new data
return True
filename = asset_id + '.json'
file_path = os.path.join(self.__data_cache, filename)
if not os.path.isfile(file_path):
# No file exists so this is new data
return True
with open(file_path, mode="r", encoding="utf-8") as f:
cached_data_hash = f.read()
data_hash = self.__compute_data_hash(data)
if cached_data_hash != data_hash:
os.remove(file_path)
# The data has changed so flush the cache
return True
# Nothing has changed for this data
return False
@classmethod
def __compute_data_hash(cls, data):
jdata = json.dumps(data, sort_keys=True, separators=(',', ':'))
return hashlib_md5(jdata.encode('utf8')).hexdigest()
def _cache_asset_data_for(self, asset_id, data):
log.info("Cache asset for: %s", asset_id)
if not os.path.exists(self.__data_cache):
log.debug("Creating data cache")
os.makedirs(self.__data_cache, exist_ok=True)
filename = asset_id + '.json'
file_path = os.path.join(self.__data_cache, filename)
if not os.path.isfile(file_path):
with open(file_path, mode="w", encoding="utf-8") as f:
f.write(self.__compute_data_hash(data))
log.debug("Caching data for asset %s", asset_id)
MOCK_DATA_FILE = os.path.join('cfg', 'mock-data.json')
class SapWarrantyRecallIntegrator(IntegratorCallbacks, RetryingThingRunner):
""" SapWarrantyRecallIntegrator initializing """
def __init__(self, config, agent_config):
super().__init__(config=agent_config)
if not (isinstance(config, dict)
and all(section in config
for section in ('integrator', 'config'))):
raise ValueError(
'Configuration invalid / missing required section')
# Whilst the integrator core requires particular configuration, top-level sections could be defined to provide
# parameters specific to this integrator.
self.__integrator = Integrator(config['integrator'], self.client, self)
self.__assets = set()
self.__config = config
# data cache used to check that the asset has been changed or not before publishing the event
self.__data_cache = get_cache(
config, config_path='integrator.asset.cache.method')
self.__loop_time = NestedConfig.get(self.__config,
'config.loop_time',
required=False,
default=5,
check=non_negative_int)
self.__enable_dev_mapping = NestedConfig.get(
self.__config,
'config.enable_dev_mapping',
required=False,
default=False)
self.__sap_config_info = SapConfig(
md_endpoint=NestedConfig.get(self.__config,
'config.sap.sernr_endpoint',
required=True,
check=non_empty_str),
hi_endpoint=NestedConfig.get(self.__config,
'config.sap.hierarchy_endpoint',
required=True,
check=non_empty_str),
usr=NestedConfig.get(self.__config,
'config.sap.usr',
required=True,
check=non_empty_str),
pwd=NestedConfig.get(self.__config,
'config.sap.pwd',
required=True,
check=non_empty_str),
timeout=int(
NestedConfig.get(self.__config,
'config.sap.timeout',
required=False,
default=10,
check=non_negative_int)))
def on_startup(self):
""" Starts the integrator. Other public methods herein must not be called beforehand.
Should be called after starting Iotic agent.
Alternatively use with keyword on the instance. """
log.info('Sap Warranty Recall Integrator Startup')
self.__data_cache.start()
self.__integrator.start()
def main(self):
""" Sap Warranty Recall Integrator Started Running """
log.info('Sap Warranty Recall Integrator Running')
self.__process_data()
loop_time = self.__loop_time
while not self.wait_for_shutdown(loop_time):
self.__process_data()
def on_shutdown(self, exc_info):
""" Stops the integrator. Should be called before shutting down the Iotic agent. """
log.info('Sap Warranty Recall Integrator Shutdown')
self.__integrator.stop()
self.__data_cache.stop()
# for IntegratorCallbacks
def on_asset_created(self, asset_id):
""" A new asset has been created.
Called once for each known asset on startup as well as whenever
a new asset appears whilst the integrator is running. """
log.info('Asset created: %s', asset_id)
self.__assets.add(asset_id)
# for IntegratorCallbacks
def on_asset_deleted(self, asset_id):
""" An asset has been deleted.
Called whenever an asset has been removed and should no longer be considered by the integrator.
Note: This is NOT called if an asset has been deleted whilst the integrator is not running. """
log.info('Asset deleted: %s', asset_id)
self.__assets.discard(asset_id)
# for IntegratorCallbacks
def on_t2_request(self, request):
""" A new type2 request has been made for a particular asset.
request - instance of T2Request """
log.info('T2 request: %s', request)
# def __get_data_for_asset(self, asset_id):
# """ returns Sap Warranty Recall data for asset_id """
# log.info("Get Sap Warranty Recall data for: %s", asset_id)
#
# # asset_id hack for RR Dev environment
# # The dev environment SapWarrantyRecall API uses a specific asset_id we will swap two of our test IDs.
# if asset_id in ('1000021', '1000015'):
# asset_id = '5242454668'
#
# url = self.__sap_config_info.sernr_endpoint
# hi_url = self.__sap_config_info.hi_endpoint
# usr = self.__sap_config_info.usr
# pwd = self.__sap_config_info.pwd
# timeout = self.__sap_config_info.timeout
# log.debug("Calling: %s", url)
#
# data = self.__call_data(hi_url, asset_id, usr, pwd, timeout)
#
# return data
def __get_data_for_asset(self, asset_id):
""" returns Sap Warranty Recall data for asset_id """
log.info("Get Sap Warranty Recall data for: %s", asset_id)
# asset_id hack for RR Dev environment
# The dev environment SapWarrantyRecall API uses a specific asset_id we will swap two of our test IDs.
if self.__enable_dev_mapping:
log.warning("Swapping asset_id %s for test id 5242454668",
asset_id)
asset_id = '5242454668'
usr = self.__sap_config_info.usr
pwd = self.__sap_config_info.pwd
timeout = int(self.__sap_config_info.timeout)
engine_serial = self.__get_engine_serial_number(
asset_id, usr, pwd, timeout)
aggregate_data = self.__get_aggregate_data(asset_id, usr, pwd, timeout)
data = self.__get_engine_data(engine_serial, aggregate_data, usr, pwd,
timeout)
if data:
return data
return None
def __get_engine_serial_number(self, asset_id, usr, pwd, timeout):
""" returns Engine Serial Number for asset_id """
log.info("Get the Engine Serial Number for asset_id: %s", asset_id)
try:
hi_endpoint = self.__sap_config_info.hi_endpoint.format(
asset_id=asset_id)
log.debug("Calling hi_endpoint: %s", hi_endpoint)
resp = requests.get(hi_endpoint,
auth=(usr, pwd),
verify=False,
timeout=timeout)
log.debug("Response status: %s", resp.status_code)
resp.raise_for_status()
if resp.ok:
try:
data = resp.json()
results = data['d']['results']
aggregate_equipment_number = None
for _data in results:
if _data['Hequi'] == "":
aggregate_equipment_number = _data['Equnr']
log.debug("Getting aggregate_equipment_number: %s",
aggregate_equipment_number)
for _data in results:
if _data[
"Hequi"] == aggregate_equipment_number and _data[
"Eqart"] == "ENG":
engine_serial = str(_data["Sernr"])
log.debug("Getting engine_serial: %s",
engine_serial)
return engine_serial
except: # pylint: disable=broad-except
log.error("Could not find response for asset %s",
asset_id,
exc_info=DEBUG_ENABLED)
else:
log.error("Endpoint response failed: %s", resp.status_code)
except requests.exceptions.HTTPError as ex:
log.error("Get the Engine Serial %s with asset_id: %s", ex,
asset_id)
except requests.exceptions.RequestException as ex:
log.error(ex, exc_info=DEBUG_ENABLED)
return None
def __get_aggregate_data(self, asset_id, usr, pwd, timeout):
""" returns aggregate_data for asset_id """
log.info("Get the Aggregate Data for asset_id: %s", asset_id)
try:
md_endpoint = self.__sap_config_info.md_endpoint.format(
asset_id=asset_id)
log.debug("Calling md_endpoint: %s", md_endpoint)
resp = requests.get(md_endpoint,
auth=(usr, pwd),
verify=False,
timeout=timeout)
log.debug("Response status: %s", resp.status_code)
resp.raise_for_status()
if resp.ok:
try:
data = resp.json()
except: # pylint: disable=broad-except
log.error(
"Could not parse JSON from response for asset %s",
asset_id,
exc_info=DEBUG_ENABLED)
else:
event_data = None
try:
event_data = data['d']['results'][0]
# event_data['Datab'] = math.floor(
# datetime.datetime.strptime(event_data['Datab'],
# '%Y-%m-%d').timestamp()) * 1000
# event_data['Datbi'] = math.floor(
# datetime.datetime.strptime(event_data['Datbi'],
# '%Y-%m-%d').timestamp()) * 1000
event_data['AssetType'] = 'AGG'
event_data['EqunrAgg'] = event_data['Equnr']
# event_data['MaktxAgg'] = event_data['Maktx']
# event_data['MatnrAgg'] = event_data['Matnr']
event_data['SernrAgg'] = event_data['Sernr']
for field in ('Equnr', 'Maktx', 'Matnr', 'Sernr'):
del event_data[field]
except KeyError:
log.error("Could not find response for asset %s",
asset_id)
return event_data
else:
log.error("Endpoint response failed: %s", resp.status_code)
except requests.exceptions.HTTPError as ex:
log.error("Get the Aggregate Data %s with asset_id: %s", ex,
asset_id)
except requests.exceptions.RequestException as ex:
log.error(ex, exc_info=DEBUG_ENABLED)
return None
def __get_engine_data(self, engine_serial, event_data, usr, pwd, timeout):
""" returns Engine Data for Engine Serial """
log.info("Get the Engine Data for engine_serial: %s", engine_serial)
try:
md_endpoint = self.__sap_config_info.md_endpoint.format(
asset_id=engine_serial)
log.debug("Calling md_endpoint: %s", md_endpoint)
resp = requests.get(md_endpoint,
auth=(usr, pwd),
verify=False,
timeout=timeout)
log.debug("Response status: %s", resp.status_code)
resp.raise_for_status()
if resp.ok:
try:
data = resp.json()
except: # pylint: disable=broad-except
log.error(
"Could not parse JSON from response for asset %s",
engine_serial,
exc_info=DEBUG_ENABLED)
else:
try:
engine_data = data['d']['results'][0]
event_data['AssetType'] = 'ENG'
event_data['SernrEng'] = engine_data['Sernr']
event_data['MatnrEng'] = engine_data['Matnr']
event_data['MaktxEng'] = engine_data['Maktx']
event_data['EqunrEng'] = engine_data['Equnr']
except KeyError:
log.error(
"Could not find response for engine_serial %s",
engine_serial)
return event_data
else:
log.error("Endpoint response failed: %s", resp.status_code)
except requests.exceptions.HTTPError as ex:
log.error("Get the Engine Data %s with engine_serial: %s", ex,
engine_serial)
except requests.exceptions.RequestException as ex:
log.error(ex, exc_info=DEBUG_ENABLED)
return None
# @classmethod
# def __call_data(cls, url, asset_id, usr, pwd, timeout):
#
# endpoint = url.format(asset_id=asset_id)
#
# try:
# resp = requests.get(endpoint, auth=(usr, pwd), verify=False, timeout=timeout)
# log.debug("Response status: %s", resp.status_code)
# resp.raise_for_status()
#
# if resp.ok:
# try:
# results = resp.json()
# except: # pylint: disable=broad-except
# log.error("Could not parse JSON from response for asset %s", asset_id, exc_info=DEBUG_ENABLED)
# else:
# try:
# data = results['d']['results']
# # Strip date string values and convert to epoch longs
# for item in data:
# item['Refdt'] = int(re.findall(r'\d+', item['Refdt'])[0])
# return data
# except: # pylint: disable=broad-except
# log.error("Could not parse response for asset %s", asset_id)
#
# except requests.exceptions.HTTPError as ex:
# log.error("%s with asset_id: %s", ex, asset_id)
#
# except requests.exceptions.RequestException as ex:
# log.error(ex, exc_info=DEBUG_ENABLED)
#
# return None
def __process_data(self):
""" Processing Sap Warranty Recalls data"""
log.info("Processing Sap Warranty Recalls")
for asset_id in list(self.__assets):
log.debug("Processing asset: %s", asset_id)
data = self.__get_data_for_asset(asset_id)
if data and self.__has_asset_data_changed_for(asset_id, data):
event = SapWarrantyRecallSet(asset_id, data=data)
log.debug("Publishing event: %s", event)
try:
self.__integrator.publish_event(event, retry=True)
self.__cache_asset_data_for(asset_id, data)
except ShutdownRequested:
log.debug("Shutdown requested while publishing event")
return
except AssetUnknown:
pass
@classmethod
def __compute_data_hash(cls, data):
""" computing data"""
jdata = json.dumps(data, sort_keys=True, separators=(',', ':'))
return hashlib_md5(jdata.encode('utf8')).hexdigest()
# Checks to see if the given data for the asset has changed
# since it was last processed.
def __has_asset_data_changed_for(self, asset_id, data):
""" Checking wheather asset cache data has changed for asset_id or not """
log.info("Checking asset cache for: %s", asset_id)
try:
asset_id_hash = self.__data_cache.get_attr(asset_id, 'hash')
except KeyError:
# No cache so this is new data
return True
data_hash = self.__compute_data_hash(data)
if asset_id_hash['hash'] != data_hash:
# data has changed
return True
# Nothing has changed for this data
return False
# After publishing the event, update the cache
def __cache_asset_data_for(self, asset_id, data):
""" updating Cache asset data for asset_id """
log.info("Cache asset for: %s", asset_id)
data_hash = self.__compute_data_hash(data)
self.__data_cache.mark_as_known(asset_id, hash=data_hash)
class SAPMasterDataIntegrator(IntegratorCallbacks, ThingRunner):
def __init__(self, config, agent_config):
super().__init__(config=agent_config)
if not (isinstance(config, dict) and all(section in config for section in ('integrator', 'config'))):
raise ValueError(
'Configuration invalid / missing required section')
# Whilst the integrator core requires particular configuration, top-level sections could be defined to provide
# parameters specific to this integrator.
self.__integrator = Integrator(config['integrator'], self.client, self)
self.__assets = set()
self.__config = config
# data cache used to check that the asset has been changed or not before publishing the event
self.__data_cache = get_cache(config, config_path='integrator.asset.cache.method')
def on_startup(self):
log.debug('SAP Master Data Integrator Startup')
self.__data_cache.start()
self.__integrator.start()
def main(self):
log.debug('SAP Master Data Integrator Running')
loop_time = self.__config['config']['loop_time']
while not self.wait_for_shutdown(loop_time):
self._process_master_data()
def on_shutdown(self, exc_info):
log.debug('SAP Master Data Integrator Shutdown')
self.__integrator.stop()
self.__data_cache.stop()
# for IntegratorCallbacks
def on_asset_created(self, asset_id):
log.debug('Asset created: %s', asset_id)
self.__assets.add(asset_id)
# for IntegratorCallbacks
def on_asset_deleted(self, asset_id):
log.debug('Asset deleted: %s', asset_id)
self.__assets.discard(asset_id)
# for IntegratorCallbacks
def on_t2_request(self, request):
pass
MOCK_DATA_FILE = os.path.join('cfg', 'mock-master-data.json')
def _get_data_for_asset(self, asset_id):
log.info("Get Master Data for: %s", asset_id)
data = None
if self.__config['config']['use_mock_data'] == 1:
log.debug("Using mock master data")
with open(self.MOCK_DATA_FILE, mode="r", encoding="utf-8") as f:
data = json.load(f)
# Strip junk from dates (string) and convert to epoch ints
datab = data['d']['results'][0]['Datab']
data['d']['results'][0]['Datab'] = int(
re.findall(r'\d+', datab)[0])
datbi = data['d']['results'][0]['Datbi']
data['d']['results'][0]['Datbi'] = int(
re.findall(r'\d+', datbi)[0])
else:
endpoint = self.__config['config']['bomgar']['endpoint']
usr = self.__config['config']['bomgar']['usr']
pwd = self.__config['config']['bomgar']['pwd']
key = 'config.enable_sap_sample_serial_hack'
if NestedConfig.get(self.__config, key, required=False, default=False, check=bool):
if asset_id == '1000021' or asset_id == '1000015':
asset_id = '4711-001'
endpoint = endpoint.replace('XXX_ASSET_ID_XXX', asset_id)
timeout = int(self.__config['config']['bomgar']['timeout'])
log.debug("Calling: %s", endpoint)
try:
resp = requests.get(endpoint, auth=(
usr, pwd), verify=False, timeout=timeout)
log.debug("Response status: %s", resp.status_code)
if resp.status_code == requests.codes['ok']:
data = resp.json()
if len(data['d']['results']) == 0:
return data
# Strip junk from dates (string) and convert to ints
datab = data['d']['results'][0]['Datab']
data['d']['results'][0]['Datab'] = int(
re.findall(r'\d+', datab)[0])
datbi = data['d']['results'][0]['Datbi']
data['d']['results'][0]['Datbi'] = int(
re.findall(r'\d+', datbi)[0])
except requests.exceptions.RequestException as ex:
log.error(ex)
return data
def _process_master_data(self):
log.debug("Processing Master Data")
for asset_id in list(self.__assets):
log.debug("Processing asset: %s", asset_id)
data = self._get_data_for_asset(asset_id)
if data is not None:
if self._has_asset_data_changed_for(asset_id, data):
log.info("Publish event for %s", asset_id)
if len(data['d']['results']) == 0:
continue
master_data = data['d']['results'][0]
event_time = master_data.get('Datab', None)
if event_time is not None:
try:
event_time = datetime.datetime.utcfromtimestamp(event_time // 1000)
except Exception as ex:
log.error("Could not create a valid datetime from %s", event_time)
log.error(ex)
event = SapMasterDataSet(asset_id, data=master_data, time=event_time)
log.debug("Event: %s", event)
try:
self.__integrator.publish_event(event)
self._cache_asset_data_for(asset_id, data)
# These will all retry
except EventPublishFailure as ex:
log.error("Event Publish Failure: %s", ex)
except AssetUnknown as ex:
pass
@classmethod
def __compute_data_hash(cls, data):
jdata = json.dumps(data, sort_keys=True, separators=(',', ':'))
return hashlib_md5(jdata.encode('utf8')).hexdigest()
# Checks to see if the given data for the asset has changed
# since it was last processed.
def _has_asset_data_changed_for(self, asset_id, data):
log.info("Checking asset cache for: %s", asset_id)
try:
asset_id_hash = self.__data_cache.get_attr(asset_id, 'hash')
except KeyError:
# No cache so this is new data
return True
data_hash = self.__compute_data_hash(data)
if asset_id_hash['hash'] != data_hash:
# data has changed
return True
# Nothing has changed for this data
return False
# After publishing the event, update the cache
def _cache_asset_data_for(self, asset_id, data):
log.info("Cache asset for: %s", asset_id)
data_hash = self.__compute_data_hash(data)
self.__data_cache.mark_as_known(asset_id, hash=data_hash)