def test_init(self, mocked_parse_expression): test_output_name = "some_output" mocked_parse_expression.return_value = EXPRESSION py_exp = bif.PythonExpression(expression=EXPRESSION, output_name=test_output_name) mocked_parse_expression.assert_called_once_with(EXPRESSION) assert py_exp.output_name == test_output_name assert py_exp.expression == EXPRESSION assert py_exp.constants == ['expression'] assert py_exp.outputs == ['output_name']
def test_execute(self, mocked_input_items, mocked_trace_append, expression, sample_df, input_items, expected_output): output_name = "test_output_name" mocked_input_items.return_value = input_items sample_df_copy = sample_df.copy() py_exp = bif.PythonExpression(expression=expression, output_name=output_name) py_exp._entity_type = MagicMock() df = py_exp.execute(sample_df) assert py_exp._entity_type.get_attributes_dict.call_count == 1 assert mocked_trace_append.call_count == 2 assert output_name in list(df.columns) assert sample_df_copy.equals(sample_df) assert df[output_name].equals(eval(expression)) assert df[output_name].array == expected_output
entity = EntityType(entity_name, db, # following columns can be dynamically generated based on meters associated with each asset Column('building',String(50)), # Column('energy_value',Float()), # Column('energy_unit',String(50)), Column('temperature',Float()), Column('temperature_unit',String(50)), MaximoAssetHTTP( username = USERNAME, password = PASSWORD, request='GET', url=URL, output_item = 'http_preload_done'), # bif.PythonExpression(expression='df["energy_value"]*df["energy_value"]', bif.PythonExpression(expression='df["temperature"]', output_name = 'F'), **{ '_timestamp' : 'evt_timestamp', '_db_schema' : db_schema }) ''' When creating an EntityType object you will need to specify the name of the entity, the database object that will contain entity data After creating an EntityType you will need to register it so that it visible in the Add Data to Entity Function UI. To also register the functions and constants associated with the entity type, specify 'publish_kpis' = True. ''' entity.register(raise_error=False)
It also has a function EntityDataGenerator The keyword args dict specifies extra properties. The database schema is only needed if you are not using the default schema. You can also rename the timestamp. ''' entity_name = 'test_http_preload' db_schema = None # replace if you are not using the default schema db.drop_table(entity_name, schema=db_schema) entity = EntityType( entity_name, db, Column('company_code', String(50)), Column('temp', Float()), Column('pressure', Float()), sample.HTTPPreload(request='GET', url='internal_test', output_item='http_preload_done'), bif.PythonExpression(expression='df["temp"]*df["pressure"]', output_name='volume'), **{ '_timestamp': 'evt_timestamp', '_db_schema': db_schema }) ''' When creating an EntityType object you will need to specify the name of the entity, the database object that will contain entity data After creating an EntityType you will need to register it so that it visible in the UI. To also register the functions and constants associated with the entity type, specify 'publish_kpis' = True. ''' entity.register(raise_error=False) db.register_functions([sample.HTTPPreload]) ''' To test the execution of kpi calculations defined for the entity type locally
db.drop_table(entity_name, schema=db_schema) entity = EntityType( entity_name, db, Column('TURBINE_ID', String(50)), Column('TEMPERATURE', Float()), Column('PRESSURE', Float()), Column('STEP', Float()), Column('PRESS_X', Float()), Column('PRESS_Y', Float()), Column('TEMP_X', Float()), Column('TEMP_Y', Float()), DemoHTTPPreload( username=BI_USERNAME, password=BI_PASSWORD, request='GET', url="https://turbine-simulator.mybluemix.net/v1/api/reading", output_item='http_preload_done'), bif.PythonExpression(expression='df["TEMPERATURE"]*df["PRESSURE"]', output_name='VOLUME'), **{ '_timestamp': 'evt_timestamp', '_db_schema': db_schema }) ''' When creating an EntityType object you will need to specify the name of the entity, the database object that will contain entity data After creating an EntityType you will need to register it so that it visible in the Add Data to Entity Function UI. To also register the functions and constants associated with the entity type, specify 'publish_kpis' = True. ''' entity.register(raise_error=False) # You must unregister_functions if you change the mehod signature or required inputs. #db.unregister_functions(["DataHTTPPreload"]) #db.register_functions([DemoHTTPPreload])
The database schema is only needed if you are not using the default schema. You can also rename the timestamp. ''' entity_name = 'Turbines' # dash100462 Used in dev2 db_schema = 'dash100462' # db_schema = None # replace if you are not using the default schema #db.drop_table(entity_name, schema = db_schema) entity = EntityType( entity_name, db, Column('Turbine_ID', String(50)), Column('Temperature', Float()), Column('drvn_p1', Float()), Column('Volume', Float()), TurbineHTTPPreload(request='GET', url='internal_test', output_item='http_preload_done'), bif.PythonExpression(expression='df["Temperature"]*df["drvn_p1"]', output_name='Volume'), **{ '_timestamp': 'evt_timestamp', '_db_schema': db_schema }) ''' When creating an EntityType object you will need to specify the name of the entity, the database object that will contain entity data After creating an EntityType you will need to register it so that it visible in the UI. To also register the functions and constants associated with the entity type, specify 'publish_kpis' = True. ''' entity.register(raise_error=False) db.register_functions([TurbineHTTPPreload]) ''' To test the execution of kpi calculations defined for the entity type locally use 'test_local_pipeline'.
With what you have seen so far, generate_data and the AS function EntityDataGenerator allow for the simulation independent variables. Real world systems have a mix of independent and dependent variables. You can use AS functions to simulate dependent variables. Consider and extension to this example where operating temperature is dependent ambient temperature (temp) and load. We can model the relationship between these variables using an AS function. In this example the relationship is simple enough to be modeled using a PythonExpression. You could PythonFunctions or custom functions to model more complex relationships. We will also add some random noise to the result of the expression. This will allow our simulation to retain some of the random variation typically seen in the real world. ''' temp_function = bif.PythonExpression(expression='df["temp"]+df["pressure"]/300*5', output_name='operating_temperature_work') entity = EntityType(entity_name, db, Column('temp', Float()), Column('pressure', Float()), Column('company_code', String(50)), Column('category_code', String(5)), bif.EntityDataGenerator(parameters=sim_parameters, data_item='is_generated'), temp_function, bif.RandomNoise(input_items=['operating_temperature_work'], standard_deviation=1, output_items=['operating_temperature']), **{'_timestamp': 'evt_timestamp', '_db_schema': db_schema}) entity.exec_local_pipeline() ''' Note: entity.generate_data only writes simulated random data to the AS input table. It does not retrieve this data and apply AS functions to it.
print(BI_USERNAME) print(BI_PASSWORD) print(BI_TENANT_ID) entity = EntityType( entity_name, db, Column('building', String(50)), Column('energy_value', Float()), Column('energy_compare_percent', Float()), Column('energy_unit', String(50)), Column('energy_trend', String(50)), Column('energy_trend_status', String(50)), BIAssetHTTPPreload(username=BI_USERNAME, password=BI_PASSWORD, request='GET', url=BI_TENANT_ID, output_item='http_preload_done'), bif.PythonExpression( expression='df["energy_value"]*df["energy_compare_percent"]', output_name='volume'), **{ '_timestamp': 'evt_timestamp', '_db_schema': db_schema }) ''' When creating an EntityType object you will need to specify the name of the entity, the database object that will contain entity data After creating an EntityType you will need to register it so that it visible in the Add Data to Entity Function UI. To also register the functions and constants associated with the entity type, specify 'publish_kpis' = True. ''' entity.register(raise_error=False) db.register_functions([BIAssetHTTPPreload]) '''
Column('building', String(50)), Column('energy_value', Float()), Column('energy_unit', String(50)), Column('temperature', Float()), Column('temperature_unit', String(50)), # Column('energy_value',Float()), # Column('energy_compare_percent', Float()), # Column('energy_unit',String(50)), # Column('energy_trend',String(50)), # Column('energy_trend_status',String(50)), MaximoAssetHTTPPreload(username=BI_USERNAME, password=BI_PASSWORD, request='GET', url=BI_TENANT_ID, output_item='http_preload_done'), bif.PythonExpression(expression='df["energy_value"]*df["energy_value"]', output_name='volume'), **{ '_timestamp': 'evt_timestamp', '_db_schema': db_schema }) ''' When creating an EntityType object you will need to specify the name of the entity, the database object that will contain entity data After creating an EntityType you will need to register it so that it visible in the Add Data to Entity Function UI. To also register the functions and constants associated with the entity type, specify 'publish_kpis' = True. ''' entity.register(raise_error=False) db.register_functions([MaximoAssetHTTPPreload]) '''
each child device. ''' entity = EntityType( consolidated, db, bif.GetEntityData(source_entity_type_name=child1[0], key_map_column='work_area', input_items=child1[1], output_items=child1[1]), bif.GetEntityData(source_entity_type_name=child2[0], key_map_column='work_area', input_items=child2[1], output_items=child2[1]), bif.PythonExpression(expression='df["%s"]-21.5' % child1[1], output_name='comfort_level'), bif.PythonExpression(expression='df["%s"]' % child2[1], output_name='is_occupied'), **{ '_timestamp': 'evt_timestamp', '_db_schema': db_schema }) ''' We also added two expression to the entity: comfort_level and is_occupied. These are there to show how to work with child_entity_data. Execute the local pipeline. This will read the child entity type data and compute comfort_level and is_occupied. ''' entity.exec_local_pipeline()
'pressure': 320 }, "data_item_sd": { 'temp': 2, 'pressure': 5 } } entity = EntityType( entity_name, db, Column('temp', Float()), Column('pressure', Float()), Column('company_code', String(50)), bif.EntityDataGenerator(parameters=sim_parameters, data_item='is_generated'), TestMetadataProvider(108, 'custom_metadata_added'), bif.PythonExpression( expression='df["temp"] + self._entity_type.custom_metadata', output_name='adjusted_temp1'), bif.PythonExpression(expression='df["temp"] *2 + c["custom_metadata"]*2', output_name='adjusted_temp2'), **{ '_timestamp': 'evt_timestamp', '_db_schema': db_schema }) entity.exec_local_pipeline(start_ts=dt.datetime.utcnow() - dt.timedelta(days=30)) ''' Execution results id evt_timestamp temp deviceid _timestamp entitydatagenerator custom_metadata_added adjusted_temp1 adjusted_temp2 73004 8/21/2019 20:50 24.46272725 73004 8/21/2019 20:50 TRUE TRUE 132.4627273 264.9254545 73003 8/21/2019 20:55 22.28387595 73003 8/21/2019 20:55 TRUE TRUE 130.283876 260.5677519
use 'test_local_pipeline'. A local test will not update the server job log or write kpi data to the AS data lake. Instead kpi data is written to the local filesystem in csv form. Test missing data item dependency for function ''' entity = EntityType( entity_name, db, Column('company_code', String(50)), Column('temp', Float()), Column('pressure', Float()), bif.EntityDataGenerator(ids=['A01', 'A02', 'B01'], data_item='is_generated'), bif.PythonExpression('df["pressure"]*df["missing_item"]', 'unable_to_execute'), **{ '_timestamp': 'evt_timestamp', '_db_schema': db_schema }) entity.exec_local_pipeline(_abort_on_fail=False) ''' Test broken expression ''' entity = EntityType( entity_name, db, Column('company_code', String(50)), Column('temp', Float()), Column('pressure', Float()), bif.EntityDataGenerator(ids=['A01', 'A02', 'B01'],
y1 is a direct linear function of x1 and x2 - it will be a breeze to predict y2 is y1 with some added noise thrown in to make it more difficult to fit a model y3 has a non linear relation to x1 and x2, but will be easy to predict with the right estimator y4 is y3 with some noise We will start by trying to predict the easy on: y1 using the SimpleRegressor function. ''' entity_name = 'predict_test' # you can give your entity type a better nane db = Database(credentials=credentials) db_schema = None # set if you are not using the default db.drop_table(entity_name) fn_gen = bif.EntityDataGenerator(output_item='generator_ok') fn_dep1 = bif.PythonExpression( # linear relatoionship '5*df["x1"]-df["x2"]', 'y1') fn_dep2 = bif.PythonExpression( 'df["x1"]*df["x1"]-df["x2"]', # non-linear relationship 'y3') fn_noise = bif.RandomNoise( # add noise to y1 and y3 to produce y2 and y4 input_items=['y1', 'y3'], standard_deviation=.5, output_items=['y2', 'y4']) job_settings = { 'delete_existing_models': True, } entity = EntityType( entity_name, db, Column('x1', Float()), Column('x2', Float()), Column('x3', Float()), Column('y0', Float()), fn_gen, fn_dep1, fn_dep2,
"schedule": {}, "backtrack": {}, "enabled": True } print(f"appending function {payload}") rest_functions.append(payload) # functions.append(f) elif function_name == 'ratio': # expression = "df['%s'].iloc[-1] / df['%s'].iloc[-1]" % (input_metrics[0], input_metrics[1]) function_name = "PythonExpression" expression = "df['%s'] / df['%s']" % (input_metrics[0], input_metrics[1]) input = {"expression": expression} output_name = entity_type_name.lower() + "ratio" f = bif.PythonExpression(expression=expression, output_name=output_name) functions.append(f) continue elif function_name == 'multiply': # expression = "df['%s'].iloc[-1] / df['%s'].iloc[-1]" % (input_metrics[0], input_metrics[1]) function_name = "PythonExpression" expression = "df['%s'] / df['%s']" % (input_metrics[0], input_metrics[1]) input = {"expression": expression} output_name = entity_type_name.lower() + "multiply" f = bif.PythonExpression(expression=expression, output_name=output_name) functions.append(f) continue else: function_name = "PythonExpression"