def test_init(self, mocked_parse_expression):
test_output_name = "some_output"
mocked_parse_expression.return_value = EXPRESSION
py_exp = bif.PythonExpression(expression=EXPRESSION,
output_name=test_output_name)
mocked_parse_expression.assert_called_once_with(EXPRESSION)
assert py_exp.output_name == test_output_name
assert py_exp.expression == EXPRESSION
assert py_exp.constants == ['expression']
assert py_exp.outputs == ['output_name']
def test_execute(self, mocked_input_items, mocked_trace_append, expression,
sample_df, input_items, expected_output):
output_name = "test_output_name"
mocked_input_items.return_value = input_items
sample_df_copy = sample_df.copy()
py_exp = bif.PythonExpression(expression=expression,
output_name=output_name)
py_exp._entity_type = MagicMock()
df = py_exp.execute(sample_df)
assert py_exp._entity_type.get_attributes_dict.call_count == 1
assert mocked_trace_append.call_count == 2
assert output_name in list(df.columns)
assert sample_df_copy.equals(sample_df)
assert df[output_name].equals(eval(expression))
assert df[output_name].array == expected_output
entity = EntityType(entity_name, db,
# following columns can be dynamically generated based on meters associated with each asset
Column('building',String(50)),
# Column('energy_value',Float()),
# Column('energy_unit',String(50)),
Column('temperature',Float()),
Column('temperature_unit',String(50)),
MaximoAssetHTTP( username = USERNAME,
password = PASSWORD,
request='GET',
url=URL,
output_item = 'http_preload_done'),
# bif.PythonExpression(expression='df["energy_value"]*df["energy_value"]',
bif.PythonExpression(expression='df["temperature"]',
output_name = 'F'),
**{
'_timestamp' : 'evt_timestamp',
'_db_schema' : db_schema
})
'''
When creating an EntityType object you will need to specify the name of the entity, the database
object that will contain entity data
After creating an EntityType you will need to register it so that it visible in the Add Data to Entity Function UI.
To also register the functions and constants associated with the entity type, specify
'publish_kpis' = True.
'''
entity.register(raise_error=False)
It also has a function EntityDataGenerator
The keyword args dict specifies extra properties. The database schema is only
needed if you are not using the default schema. You can also rename the timestamp.
'''
entity_name = 'test_http_preload'
db_schema = None # replace if you are not using the default schema
db.drop_table(entity_name, schema=db_schema)
entity = EntityType(
entity_name, db, Column('company_code', String(50)),
Column('temp', Float()), Column('pressure', Float()),
sample.HTTPPreload(request='GET',
url='internal_test',
output_item='http_preload_done'),
bif.PythonExpression(expression='df["temp"]*df["pressure"]',
output_name='volume'), **{
'_timestamp': 'evt_timestamp',
'_db_schema': db_schema
})
'''
When creating an EntityType object you will need to specify the name of the entity, the database
object that will contain entity data
After creating an EntityType you will need to register it so that it visible in the UI.
To also register the functions and constants associated with the entity type, specify
'publish_kpis' = True.
'''
entity.register(raise_error=False)
db.register_functions([sample.HTTPPreload])
'''
To test the execution of kpi calculations defined for the entity type locally
db.drop_table(entity_name, schema=db_schema)
entity = EntityType(
entity_name, db, Column('TURBINE_ID', String(50)),
Column('TEMPERATURE', Float()), Column('PRESSURE', Float()),
Column('STEP', Float()), Column('PRESS_X', Float()),
Column('PRESS_Y', Float()), Column('TEMP_X', Float()),
Column('TEMP_Y', Float()),
DemoHTTPPreload(
username=BI_USERNAME,
password=BI_PASSWORD,
request='GET',
url="https://turbine-simulator.mybluemix.net/v1/api/reading",
output_item='http_preload_done'),
bif.PythonExpression(expression='df["TEMPERATURE"]*df["PRESSURE"]',
output_name='VOLUME'), **{
'_timestamp': 'evt_timestamp',
'_db_schema': db_schema
})
'''
When creating an EntityType object you will need to specify the name of the entity, the database
object that will contain entity data
After creating an EntityType you will need to register it so that it visible in the Add Data to Entity Function UI.
To also register the functions and constants associated with the entity type, specify
'publish_kpis' = True.
'''
entity.register(raise_error=False)
# You must unregister_functions if you change the mehod signature or required inputs.
#db.unregister_functions(["DataHTTPPreload"])
#db.register_functions([DemoHTTPPreload])
The database schema is only needed if you are not using the default schema. You can also rename the timestamp.
'''
entity_name = 'Turbines'
# dash100462 Used in dev2
db_schema = 'dash100462'
# db_schema = None # replace if you are not using the default schema
#db.drop_table(entity_name, schema = db_schema)
entity = EntityType(
entity_name, db, Column('Turbine_ID', String(50)),
Column('Temperature', Float()), Column('drvn_p1', Float()),
Column('Volume', Float()),
TurbineHTTPPreload(request='GET',
url='internal_test',
output_item='http_preload_done'),
bif.PythonExpression(expression='df["Temperature"]*df["drvn_p1"]',
output_name='Volume'), **{
'_timestamp': 'evt_timestamp',
'_db_schema': db_schema
})
'''
When creating an EntityType object you will need to specify the name of the entity, the database
object that will contain entity data
After creating an EntityType you will need to register it so that it visible in the UI.
To also register the functions and constants associated with the entity type, specify
'publish_kpis' = True.
'''
entity.register(raise_error=False)
db.register_functions([TurbineHTTPPreload])
'''
To test the execution of kpi calculations defined for the entity type locally
use 'test_local_pipeline'.
With what you have seen so far, generate_data and the AS function EntityDataGenerator allow
for the simulation independent variables. Real world systems have a mix of independent and
dependent variables. You can use AS functions to simulate dependent variables.
Consider and extension to this example where operating temperature is dependent ambient
temperature (temp) and load. We can model the relationship between these variables using
an AS function. In this example the relationship is simple enough to be modeled using a
PythonExpression. You could PythonFunctions or custom functions to model more complex
relationships.
We will also add some random noise to the result of the expression. This will allow our
simulation to retain some of the random variation typically seen in the real world.
'''
temp_function = bif.PythonExpression(expression='df["temp"]+df["pressure"]/300*5',
output_name='operating_temperature_work')
entity = EntityType(entity_name, db, Column('temp', Float()), Column('pressure', Float()),
Column('company_code', String(50)), Column('category_code', String(5)),
bif.EntityDataGenerator(parameters=sim_parameters, data_item='is_generated'), temp_function,
bif.RandomNoise(input_items=['operating_temperature_work'], standard_deviation=1,
output_items=['operating_temperature']),
**{'_timestamp': 'evt_timestamp', '_db_schema': db_schema})
entity.exec_local_pipeline()
'''
Note:
entity.generate_data only writes simulated random data to the AS input table. It
does not retrieve this data and apply AS functions to it.
print(BI_USERNAME)
print(BI_PASSWORD)
print(BI_TENANT_ID)
entity = EntityType(
entity_name, db, Column('building', String(50)),
Column('energy_value', Float()), Column('energy_compare_percent', Float()),
Column('energy_unit', String(50)), Column('energy_trend', String(50)),
Column('energy_trend_status', String(50)),
BIAssetHTTPPreload(username=BI_USERNAME,
password=BI_PASSWORD,
request='GET',
url=BI_TENANT_ID,
output_item='http_preload_done'),
bif.PythonExpression(
expression='df["energy_value"]*df["energy_compare_percent"]',
output_name='volume'), **{
'_timestamp': 'evt_timestamp',
'_db_schema': db_schema
})
'''
When creating an EntityType object you will need to specify the name of the entity, the database
object that will contain entity data
After creating an EntityType you will need to register it so that it visible in the Add Data to Entity Function UI.
To also register the functions and constants associated with the entity type, specify
'publish_kpis' = True.
'''
entity.register(raise_error=False)
db.register_functions([BIAssetHTTPPreload])
'''
Column('building', String(50)),
Column('energy_value', Float()),
Column('energy_unit', String(50)),
Column('temperature', Float()),
Column('temperature_unit', String(50)),
# Column('energy_value',Float()),
# Column('energy_compare_percent', Float()),
# Column('energy_unit',String(50)),
# Column('energy_trend',String(50)),
# Column('energy_trend_status',String(50)),
MaximoAssetHTTPPreload(username=BI_USERNAME,
password=BI_PASSWORD,
request='GET',
url=BI_TENANT_ID,
output_item='http_preload_done'),
bif.PythonExpression(expression='df["energy_value"]*df["energy_value"]',
output_name='volume'),
**{
'_timestamp': 'evt_timestamp',
'_db_schema': db_schema
})
'''
When creating an EntityType object you will need to specify the name of the entity, the database
object that will contain entity data
After creating an EntityType you will need to register it so that it visible in the Add Data to Entity Function UI.
To also register the functions and constants associated with the entity type, specify
'publish_kpis' = True.
'''
entity.register(raise_error=False)
db.register_functions([MaximoAssetHTTPPreload])
'''
each child device.
'''
entity = EntityType(
consolidated, db,
bif.GetEntityData(source_entity_type_name=child1[0],
key_map_column='work_area',
input_items=child1[1],
output_items=child1[1]),
bif.GetEntityData(source_entity_type_name=child2[0],
key_map_column='work_area',
input_items=child2[1],
output_items=child2[1]),
bif.PythonExpression(expression='df["%s"]-21.5' % child1[1],
output_name='comfort_level'),
bif.PythonExpression(expression='df["%s"]' % child2[1],
output_name='is_occupied'), **{
'_timestamp': 'evt_timestamp',
'_db_schema': db_schema
})
'''
We also added two expression to the entity: comfort_level and is_occupied. These
are there to show how to work with child_entity_data.
Execute the local pipeline. This will read the child entity type data and compute
comfort_level and is_occupied.
'''
entity.exec_local_pipeline()
'pressure': 320
},
"data_item_sd": {
'temp': 2,
'pressure': 5
}
}
entity = EntityType(
entity_name, db, Column('temp', Float()), Column('pressure', Float()),
Column('company_code', String(50)),
bif.EntityDataGenerator(parameters=sim_parameters,
data_item='is_generated'),
TestMetadataProvider(108, 'custom_metadata_added'),
bif.PythonExpression(
expression='df["temp"] + self._entity_type.custom_metadata',
output_name='adjusted_temp1'),
bif.PythonExpression(expression='df["temp"] *2 + c["custom_metadata"]*2',
output_name='adjusted_temp2'), **{
'_timestamp': 'evt_timestamp',
'_db_schema': db_schema
})
entity.exec_local_pipeline(start_ts=dt.datetime.utcnow() -
dt.timedelta(days=30))
'''
Execution results
id evt_timestamp temp deviceid _timestamp entitydatagenerator custom_metadata_added adjusted_temp1 adjusted_temp2
73004 8/21/2019 20:50 24.46272725 73004 8/21/2019 20:50 TRUE TRUE 132.4627273 264.9254545
73003 8/21/2019 20:55 22.28387595 73003 8/21/2019 20:55 TRUE TRUE 130.283876 260.5677519
use 'test_local_pipeline'.
A local test will not update the server job log or write kpi data to the AS data
lake. Instead kpi data is written to the local filesystem in csv form.
Test missing data item dependency for function
'''
entity = EntityType(
entity_name, db, Column('company_code', String(50)),
Column('temp', Float()), Column('pressure', Float()),
bif.EntityDataGenerator(ids=['A01', 'A02', 'B01'],
data_item='is_generated'),
bif.PythonExpression('df["pressure"]*df["missing_item"]',
'unable_to_execute'), **{
'_timestamp': 'evt_timestamp',
'_db_schema': db_schema
})
entity.exec_local_pipeline(_abort_on_fail=False)
'''
Test broken expression
'''
entity = EntityType(
entity_name, db, Column('company_code', String(50)),
Column('temp', Float()), Column('pressure', Float()),
bif.EntityDataGenerator(ids=['A01', 'A02', 'B01'],
y1 is a direct linear function of x1 and x2 - it will be a breeze to predict
y2 is y1 with some added noise thrown in to make it more difficult to fit a model
y3 has a non linear relation to x1 and x2, but will be easy to predict with the right estimator
y4 is y3 with some noise
We will start by trying to predict the easy on: y1 using the SimpleRegressor function.
'''
entity_name = 'predict_test' # you can give your entity type a better nane
db = Database(credentials=credentials)
db_schema = None # set if you are not using the default
db.drop_table(entity_name)
fn_gen = bif.EntityDataGenerator(output_item='generator_ok')
fn_dep1 = bif.PythonExpression( # linear relatoionship
'5*df["x1"]-df["x2"]', 'y1')
fn_dep2 = bif.PythonExpression(
'df["x1"]*df["x1"]-df["x2"]', # non-linear relationship
'y3')
fn_noise = bif.RandomNoise( # add noise to y1 and y3 to produce y2 and y4
input_items=['y1', 'y3'],
standard_deviation=.5,
output_items=['y2', 'y4'])
job_settings = {
'delete_existing_models': True,
}
entity = EntityType(
entity_name, db, Column('x1', Float()), Column('x2', Float()),
Column('x3', Float()), Column('y0', Float()), fn_gen, fn_dep1, fn_dep2,
"schedule": {},
"backtrack": {},
"enabled": True
}
print(f"appending function {payload}")
rest_functions.append(payload)
# functions.append(f)
elif function_name == 'ratio':
# expression = "df['%s'].iloc[-1] / df['%s'].iloc[-1]" % (input_metrics[0], input_metrics[1])
function_name = "PythonExpression"
expression = "df['%s'] / df['%s']" % (input_metrics[0],
input_metrics[1])
input = {"expression": expression}
output_name = entity_type_name.lower() + "ratio"
f = bif.PythonExpression(expression=expression,
output_name=output_name)
functions.append(f)
continue
elif function_name == 'multiply':
# expression = "df['%s'].iloc[-1] / df['%s'].iloc[-1]" % (input_metrics[0], input_metrics[1])
function_name = "PythonExpression"
expression = "df['%s'] / df['%s']" % (input_metrics[0],
input_metrics[1])
input = {"expression": expression}
output_name = entity_type_name.lower() + "multiply"
f = bif.PythonExpression(expression=expression,
output_name=output_name)
functions.append(f)
continue
else:
function_name = "PythonExpression"