def from_config(cls, config):
return cls(
# Local id (lid) prefix for Iotic Thing
lid_prefix=NestedConfig.get(
config,
'asset.thing.prefix.lid',
required=False,
default=Asset.Thing.LID_PREFIX,
check=NestedConfig.Check.non_empty_str),
# Unique tag prefix
tag_prefix=NestedConfig.get(
config,
'asset.thing.prefix.tag',
required=False,
default=Asset.Thing.TAG_PREFIX,
check=NestedConfig.Check.non_empty_str),
# Number of recent data samples to save
recent_samples=NestedConfig.get(
config,
'asset.thing.recent.samples',
required=False,
default=Asset.Point.Fielddata.RECENT_DATA,
check=lambda x: isinstance(x, int) and x >= -1),
# Metadata for field data things
meta=ThingMeta.from_config(config,
'asset.thing.meta',
tag_default=Asset.Thing.TAGS))
def __init__(self, config, agent_config):
super().__init__(config=agent_config)
if not (isinstance(config, dict) and all(section in config for section in ('integrator', 'config'))):
raise ValueError(
'Configuration invalid / missing required section')
# Whilst the integrator core requires particular configuration, top-level sections could be defined to provide
# parameters specific to this integrator.
self.__integrator = Integrator(config['integrator'], self.client, self)
self.__assets = set()
self.__config = config
# data cache used to check that the asset has been changed or not before publishing the event
self.__data_cache = get_cache(config, config_path='integrator.asset.cache.method')
self.__req_pool = ThreadPoolExecutor(max_workers=self.__config['config']['workers'])
self.__talend_config_info = talendConfig(
endpoint=NestedConfig.get(self.__config,
'config.talend.endpoint', required=True, check=non_empty_str),
endpoint_single=NestedConfig.get(self.__config,
'config.talend.endpoint_single', required=True, check=non_empty_str)
usr=NestedConfig.get(self.__config, 'config.talend.usr', required=True, check=non_empty_str),
pwd=NestedConfig.get(self.__config, 'config.talend.pwd', required=True, check=non_empty_str),
timeout=int(NestedConfig.get(self.__config,
'config.talend.timeout', required=False, default=10, check=non_negative_int))
)
def _get_tim_doc(self, serial_no, document_label, document_name):
log.info("Get Talend doc for: %s", serial_no)
data = None
if self.__config['config']['use_mock_data'] == 1:
return mockpdf.data
endpoint = NestedConfig.get(self.__config, 'config.talend.endpoint_single', required=True, check=non_empty_str)
usr = NestedConfig.get(self.__config, 'config.talend.usr', required=True, check=non_empty_str)
pwd = NestedConfig.get(self.__config, 'config.talend.pwd', required=True, check=non_empty_str)
endpoint = endpoint.replace('XXX_ASSET_ID_XXX', serial_no)
endpoint = endpoint.replace('XXX_DOC_LABEL_XXX', document_label)
endpoint = endpoint.replace('XXX_DOC_NAME_XXX', document_name)
timeout = int(self.__config['config']['talend']['timeout'])
log.debug("Calling: %s", endpoint)
try:
resp = requests.get(endpoint, auth=(usr, pwd), verify=False, timeout=timeout)
log.debug("Response status: %s", resp.status_code)
if resp.text and resp.status_code == requests.codes['ok']:
try:
data = resp.json()['document']
except Exception as ex: # pylint: disable=broad-except
log.error("Could not parse JSON from response: %s", resp.text)
log.error(ex)
except requests.exceptions.RequestException as ex:
log.error(ex)
return data
def __init__(self, config, agent_config):
super().__init__(config=agent_config)
if not (isinstance(config, Mapping)
and all(section in config
for section in ('integrator', 'config'))):
raise ValueError(
'Configuration invalid / missing required section')
# parameters specific to this integrator.
self.__integrator = Integrator(config['integrator'], self.client, self)
self.__assets = set()
self.__config = config
# data cache used to check that the asset has been changed or not before publishing the event
self.__data_cache = get_cache(
self.__config, config_path='integrator.asset.cache.method')
# Pool of workers to execture type2 requests
workers = NestedConfig.get(self.__config,
'config.workers',
required=False,
default=1,
check=non_negative_int)
self.__req_pool = ThreadPoolExecutor(max_workers=workers)
# Validate config
self.__sap_config_info = SapConfig(
eq_hist_endp=NestedConfig.get(
self.__config,
'config.sap.equipment_history_endpoint',
required=True,
check=non_empty_str),
eq_doc_endp=NestedConfig.get(
self.__config,
'config.sap.equipment_document_endpoint',
required=True,
check=non_empty_str),
eq_doc_single=NestedConfig.get(
self.__config,
'config.sap.equipment_document_single',
required=True,
check=non_empty_str),
eq_doc_test=NestedConfig.get(self.__config,
'config.sap.equipment_document_test',
required=True,
check=non_empty_str),
usr=NestedConfig.get(self.__config,
'config.sap.usr',
required=True,
check=non_empty_str),
pwd=NestedConfig.get(self.__config,
'config.sap.pwd',
required=True,
check=non_empty_str),
timeout=int(
NestedConfig.get(self.__config,
'config.sap.timeout',
required=False,
default=10,
check=non_negative_int)))
def _get_data_for_asset(self, asset_id):
log.info("Get Talend data for: %s", asset_id)
data = None
if self.__config['config']['use_mock_data'] == 1:
log.debug("Using mock data")
with open(self.MOCK_DATA_FILE, mode="r", encoding="utf-8") as f:
data = json.load(f)
else:
endpoint = self.__config['config']['talend']['endpoint']
usr = self.__config['config']['talend']['usr']
pwd = self.__config['config']['talend']['pwd']
key = 'config.enable_sap_sample_serial_hack'
if NestedConfig.get(self.__config, key, required=False, default=False, check=bool):
if asset_id == '1000021' or asset_id == '1000015':
asset_id = '16701003340'
endpoint = endpoint.replace('XXX_ASSET_ID_XXX', asset_id)
timeout = int(self.__config['config']['talend']['timeout'])
log.debug("Calling: %s", endpoint)
try:
resp = requests.get(endpoint, auth=(usr, pwd), verify=False, timeout=timeout)
log.debug("Response status: %s", resp.status_code)
if resp.status_code == requests.codes['ok']:
data = resp.json()
except requests.exceptions.RequestException as ex:
log.error(ex)
return data
def main(self):
log.info('Running')
loop_time = NestedConfig.get(
self.__config, 'config.loop_time', required=False, default=5, check=non_negative_int
)
while not self.wait_for_shutdown(loop_time):
self.__process_data()
def _get_data_for_asset(self, asset_id):
log.info("Get Sap Warranty Recall data for: %s", asset_id)
data = None
if self.__config['config']['use_mock_data'] == 1:
log.debug("Using mock data")
with open(self.MOCK_DATA_FILE, mode="r", encoding="utf-8") as f:
results = json.load(f)
data = results['d']['results']
# Strip date string values and convert to epoch longs
for item in data:
item['Refdt'] = int(re.findall(r'\d+', item['Refdt'])[0])
else:
endpoint = self.__config['config']['sap']['sernr_endpoint']
key = 'config.enable_sap_sample_serial_hack'
if NestedConfig.get(self.__config,
key,
required=False,
default=False,
check=bool):
if asset_id == '1000021' or asset_id == '1000015':
asset_id = '5242454668'
endpoint = endpoint.replace('XXX_ASSET_ID_XXX', asset_id)
usr = self.__config['config']['sap']['usr']
pwd = self.__config['config']['sap']['pwd']
timeout = int(self.__config['config']['sap']['timeout'])
log.debug("Calling: %s", endpoint)
try:
resp = requests.get(endpoint,
auth=(usr, pwd),
verify=False,
timeout=timeout)
log.debug("Response status: %s", resp.status_code)
if resp.status_code == requests.codes['ok']:
results = resp.json()
data = results['d']['results']
# Strip date string values and convert to epoch longs
for item in data:
item['Refdt'] = int(
re.findall(r'\d+', item['Refdt'])[0])
else:
log.error("Endpoint response failed: %s", resp.status_code)
except requests.exceptions.RequestException as ex:
log.error(ex)
return data
def _get_data_for_asset(self, asset_id): ### geting call from _process_master_data() @down.
log.info("Get Master Data for: %s", asset_id)
data = None
if self.__config['config']['use_mock_data'] == 1:
log.debug("Using mock master data")
with open(self.MOCK_DATA_FILE, mode="r", encoding="utf-8") as f:
data = json.load(f)
# Strip junk from dates (string) and convert to epoch ints
datab = data['d']['results'][0]['Datab'] ### might be in binary
data['d']['results'][0]['Datab'] = int(
re.findall(r'\d+', datab)[0])
datbi = data['d']['results'][0]['Datbi'] ### might be in int
data['d']['results'][0]['Datbi'] = int(
re.findall(r'\d+', datbi)[0])
else:
endpoint = self.__config['config']['bomgar']['endpoint']
usr = self.__config['config']['bomgar']['usr']
pwd = self.__config['config']['bomgar']['pwd']
key = 'config.enable_sap_sample_serial_hack'
if NestedConfig.get(self.__config, key, required=False, default=False, check=bool):
if asset_id == '1000021' or asset_id == '1000015':
asset_id = '4711-001'
endpoint = endpoint.replace('XXX_ASSET_ID_XXX', asset_id)
timeout = int(self.__config['config']['bomgar']['timeout'])
log.debug("Calling: %s", endpoint)
try:
resp = requests.get(endpoint, auth=(
usr, pwd), verify=False, timeout=timeout)
log.debug("Response status: %s", resp.status_code)
if resp.status_code == requests.codes['ok']:
data = resp.json()
if len(data['d']['results']) == 0:
return data
# Strip junk from dates (string) and convert to ints
datab = data['d']['results'][0]['Datab']
data['d']['results'][0]['Datab'] = int(
re.findall(r'\d+', datab)[0])
datbi = data['d']['results'][0]['Datbi']
data['d']['results'][0]['Datbi'] = int(
re.findall(r'\d+', datbi)[0])
except requests.exceptions.RequestException as ex:
log.error(ex)
return data
def __init__(self, config, agent_config):
super().__init__(config=agent_config)
if not (isinstance(config, dict) and all(section in config for section in ('integrator', 'config'))):
raise ValueError(
'Configuration invalid / missing required section')
# Whilst the integrator core requires particular configuration, top-level sections could be defined to provide
# parameters specific to this integrator.
self.__integrator = Integrator(config['integrator'], self.client, self)
self.__assets = set()
self.__config = config
# data cache used to check that the asset has been changed or not before publishing the event
self.__data_cache = get_cache(config, config_path='integrator.asset.cache.method')
self.__loop_time = NestedConfig.get(self.__config, 'config.loop_time', required=False, default=False)
self.__sap_config_info = SapConfig(
hi_endp=NestedConfig.get(
self.__config, 'config.sap.hierarchy_endpoint', required=True, check=non_empty_str
),
md_endp=NestedConfig.get(
self.__config, 'config.sap.master_endpoint', required=True, check=non_empty_str
),
md_usr=NestedConfig.get(
self.__config, 'config.sap.usr', required=True, check=non_empty_str
),
md_pwd=NestedConfig.get(
self.__config, 'config.sap.pwd', required=True, check=non_empty_str
),
md_timeout=int(NestedConfig.get(
self.__config, 'config.sap.timeout', required=False, default=10, check=non_negative_int
))
)
def _get_data_for_asset(self, asset_id):
log.info("Get Talend data for: %s", asset_id)
data = None
if self.__config['config']['use_mock_data'] == 1:
log.debug("Using mock data")
with open(self.MOCK_DATA_FILE, mode="r", encoding="utf-8") as f:
data = json.load(f)
else:
endpoint = NestedConfig.get(self.__config, 'config.talend.endpoint_single', required=True, check=non_empty_str)
usr = NestedConfig.get(self.__config, 'config.talend.usr', required=True, check=non_empty_str)
pwd = NestedConfig.get(self.__config, 'config.talend.pwd', required=True, check=non_empty_str)
key = 'config.enable_talend_sample_serial_hack'
if NestedConfig.get(self.__config, key, required=False, default=False, check=bool):
if asset_id in ('1000021', '16701003340'):
asset_id = '16701003340'
endpoint = endpoint.replace('XXX_ASSET_ID_XXX', asset_id)
timeout = int(self.__config['config']['talend']['timeout'])
log.debug("Calling: %s", endpoint)
try:
resp = requests.get(endpoint, auth=(usr, pwd), verify=False, timeout=timeout)
log.debug("Response status: %s", resp.status_code)
if resp.text and resp.status_code == requests.codes['ok']:
try:
data = resp.json()
except Exception as ex:
log.error("Could not parse JSON from response: %s", resp.text)
raise ex
except requests.exceptions.RequestException as ex:
log.error(ex)
return data