''' To test the execution of kpi calculations defined for the entity type locally use this function. A local test will not update the server job log or write kpi data to the AS data lake. Instead kpi data is written to the local filesystem in csv form. ''' entity.exec_local_pipeline() ''' view entity data ''' # can get entity metadata with following # # TODO, for some reason this returns data but also throws exception. ''' meta = db.get_entity_type(entity_name) db_schema = meta['schemaName'] table_name = meta['metricTableName'] ''' db_schema = settings.DB_SCHEMA or "BLUADMIN" table_name = settings.TABLE_NAME or "kalbuildingstest" df = db.read_table(table_name=table_name, schema=db_schema) # df = db.read_table(table_name=entity_name, schema=db_schema) print(df.head()) print ( 'Done registering entity %s ' %db.register_functions([MaximoAssetHTTP]) )
'_db_schema': db_schema }) # dimension columns dimension_columns = [] dimension_columns.append(Column('business', String(50))) dimension_columns.append(Column('site', String(50))) dimension_columns.append(Column('equipment_type', String(50))) dimension_columns.append(Column('train', String(50))) dimension_columns.append(Column('service', String(50))) dimension_columns.append(Column('asset_id', String(50))) entity.register(raise_error=False) db.register_functions([TurbineHTTPPreload]) df = db.read_table(table_name=entity_name, schema=db_schema) print(df.head()) meta = db.get_entity_type(entityType) jobsettings = { '_production_mode': False, '_start_ts_override': dt.datetime.utcnow() - dt.timedelta(days=10), '_end_ts_override': (dt.datetime.utcnow() - dt.timedelta(days=1)), # .strftime('%Y-%m-%d %H:%M:%S'), '_db_schema': 'BLUADMIN', 'save_trace_to_file': True } logging.info('Instantiated create compressor job')
['firmware', 'firmware_version']: ['1.0', '1.12', '1.13', '2.1'] ['manufacturer']: ['Rentech', 'GHI Industries'] ['zone']: ['27A', '27B', '27C'] ['status', 'status_code']: ['inactive', 'active'] ['operator', 'operator_id', 'person', 'employee']: ['Fred', 'Joe', 'Mary', 'Steve', 'Henry', 'Jane', 'Hillary', 'Justin', 'Rod'] 3.3 Other dimension name Any other dimension name other than the one above will generate random values ''' db.drop_table(dim_table_name, schema=schema) entity_type.make_dimension(dim_table_name, Column('company', String(50)), Column('status', String(50)), Column('operator', String(50)), **{'schema': schema}) entity_type.register() ''' To test the execution of kpi calculations defined for the entity type locally use this function. A local test will not update the server job log or write kpi data to the AS data lake. Instead kpi data is written to the local filesystem in csv form. ''' # entity_type.exec_local_pipeline(**{'_production_mode': False}) ef = db.read_table(entity_type.logical_name, schema=schema, columns=[entity_type._entity_id]) ids = set(ef[entity_type._entity_id].unique()) entity_type.generate_dimension_data(entities=ids)
import datetime as dt import json import os import pandas as pd from sqlalchemy import Column, Integer, String, Float, DateTime, Boolean, func from iotfunctions.preprocessor import BaseTransformer from iotfunctions.bif import IoTExpression from iotfunctions.metadata import EntityType, make_sample_entity from iotfunctions.db import Database #replace with a credentials dictionary or provide a credentials file with open('credentials.json', encoding='utf-8') as F: credentials = json.loads(F.read()) #create a sample entity to work with db_schema = None #set if you are not using the default db = Database(credentials=credentials) entity = make_sample_entity(db=db, schema = db_schema) #examine the sample entity df = db.read_table(entity.name,schema=db_schema) df.head(1).transpose() #configure an expression function expression = 'df["throttle"]/df["grade"]' fn = IoTExpression(expression=expression, output_name='expression_out') df = entity.exec_pipeline(fn) df.head(1).transpose()
class IotEntity(object): ''' Query Entity objects to establish Events Streams connectivity, manage Stream metadata and sessions Parameters: ----------- credentials: dict (optional) Database credentials. If none specified use DB_CONNECTION_STRING environment variable start_session: bool Start a session when establishing connection echo: bool Output sql to log ''' def __init__(self, entity_type_name=None, entity_name=None): # replace with valid table and column names self.entity_type_name = entity_type_name self.entity_name = entity_name self.db_schema = "public" # only required if you are not using the default self.table_name = entity_type_name.upper( ) # change to a valid entity time series table name self.dim_table_name = "DM_" + self.table_name # change to a entity dimenstion table name self.timestamp = 'evt_timestamp' self.credentials = settings.CREDENTIALS # logging.info('username %s' %self.credentials['db2']['username']) # logging.info('password %s' %self.credentials['db2']['password']) # logging.info('host %s' %self.credentials['db2']['host']) # logging.info('port %s' %self.credentials['db2']['port']) # logging.info('databaseName%s' %self.credentials['db2']['databaseName']) self.db = Database(credentials=self.credentials) def query(self, metrics=None, timestamp='evt_timestamp', agg_dict=None, to_csv=True): logging.info("Query %s and output format is %s" % (agg_dict, to_csv)) # Retrieve a single data item using a standard aggregation function #agg = {metrics[0]: ['mean']} df = self.db.read_agg(table_name=self.table_name, schema=self.db_schema, timestamp='evt_timestamp', agg_dict=agg, to_csv=to_csv) return (df) # Works def query_entity_data(self, columns=None, start_ts=None, end_ts=None): ''' Read whole table and return chosen metrics for selected start and end time as a dataframe Parameters ----------- table_name: str Source table name schema: str Schema name where table is located columns: list of strs Projection list timestamp_col: str Name of timestamp column in the table. Required for time filters. start_ts: datetime Retrieve data from this date end_ts: datetime Retrieve data up until date entities: list of strs Retrieve data for a list of deviceids dimension: str Table name for dimension table. Dimension table will be joined on deviceid. parse_dates: list of strs Column names to parse as dates ''' logging.info('table_name %s' % self.table_name) logging.info('db_schema %s' % self.db_schema) columns.append("RCV_TIMESTAMP_UTC") logging.info('columns %s' % columns) df = self.db.read_table(table_name="IOT_" + self.table_name, schema=self.db_schema, parse_dates=None, columns=columns, timestamp_col='RCV_TIMESTAMP_UTC', start_ts=start_ts, end_ts=end_ts) logging.info(df) query_data = df.to_json() return query_data
To also register the functions and constants associated with the entity type, specify 'publish_kpis' = True. ''' entity.register(raise_error=False) # When creating a custom preload function you can register it by uncommenting the following lines # You must unregister_functions if you change the method signature or required inputs. db.unregister_functions(['Issue455HTTPPreload']) db.register_functions([Issue455HTTPPreload]) ''' To test the execution of kpi calculations defined for the entity type locally use this function. A local test will not update the server job log or write kpi data to the AS data lake. Instead kpi data is written to the local filesystem in csv form. ''' entity.exec_local_pipeline(**{'_production_mode': False}) ''' view entity data ''' print("Read Table of new entity") df = db.read_table(table_name=entity_name, schema=db_schema) print(df.head()) print("Read Table of new dimension") print(entity_dimension) df = db.read_table(table_name=entity_dimension, schema=db_schema) print(df.head()) print("Done registering entity")