def main(argv):
sys.path.append(
os.path.realpath(os.path.join(os.path.dirname(__file__), '..')))
credPath = os.path.join(os.path.dirname(__file__),
f"credentials_as_{os.environ['USERNAME']}.json")
print(f"Loading credentials from {credPath}")
with io.open(credPath, encoding='utf-8') as F:
credentials = json.loads(F.read())
db_schema = None
db = Database(credentials=credentials)
from goodvibrations.predictStatus import PredictCondition
print(f"Registering function")
db.unregister_functions(["PredictCondition"])
try:
db.register_functions([PredictCondition])
except Exception as exc:
print(exc)
fn = PredictCondition(condition='predStatus')
df = fn.execute_local_test(db=db,
db_schema=db_schema,
generate_days=1,
to_csv=True)
print(df)
"""
def __init__(self,
dummy_input=None,
output_item='broken_preload',
**parameters):
super().__init__(dummy_items=[], output_item=output_item)
self.dummy_input = dummy_input
def execute(self, df, start_ts=None, end_ts=None, entities=None):
return True
try:
db.register_functions([NonPackage])
except BaseException as e:
print('Registration failed as expected: %s' % e)
else:
raise RuntimeError(
'Function is not in a package, function should have failed')
'''
Register a function with no build ui
'''
data_item='is_generated'), **{
'_timestamp': 'evt_timestamp',
'_db_schema': db_schema
})
# dimension columns
dimension_columns = []
dimension_columns.append(Column('business', String(50)))
dimension_columns.append(Column('site', String(50)))
dimension_columns.append(Column('equipment_type', String(50)))
dimension_columns.append(Column('train', String(50)))
dimension_columns.append(Column('service', String(50)))
dimension_columns.append(Column('asset_id', String(50)))
entity.register(raise_error=False)
db.register_functions([TurbineHTTPPreload])
df = db.read_table(table_name=entity_name, schema=db_schema)
print(df.head())
meta = db.get_entity_type(entityType)
jobsettings = {
'_production_mode': False,
'_start_ts_override': dt.datetime.utcnow() - dt.timedelta(days=10),
'_end_ts_override':
(dt.datetime.utcnow() -
dt.timedelta(days=1)), # .strftime('%Y-%m-%d %H:%M:%S'),
'_db_schema': 'BLUADMIN',
'save_trace_to_file': True
}
import datetime as dt
import json
import pandas as pd
import numpy as np
from sqlalchemy import Column, Integer, String, Float, DateTime, Boolean, func
from iotfunctions.base import BaseTransformer
from iotfunctions.metadata import EntityType
from iotfunctions.db import Database
from iotfunctions import ui
with open('credentials_as.json', encoding='utf-8') as F:
credentials = json.loads(F.read())
db_schema = None
db = Database(credentials=credentials)
from custom.multiply_by_factor import MultiplyByFactor
db.register_functions([MultiplyByFunction])
# if there is a 2nd argument do not register but exit
if (len(sys.argv) > 1):
sys.exit()
EngineLogging.configure_console_logging(logging.DEBUG)
#with open('credentials_as_dev.json', encoding='utf-8') as F:
# credentials = json.loads(F.read())
#fn = AggregateItemStats(
# input_item_1='x1',
# input_item_2='x2',
# output_item='y')
#df = fn.execute_local_test(generate_days=1,to_csv=True)
#print(df)
#cols = [
# Column('string_1', String(255))
# ]
#df = fn.execute_local_test(generate_days = 1,to_csv=True,
# columns = cols)
#db.register_functions([functions.AggregateItemStats])
db.register_module(functions)
db.register_functions([anomaly.SpectralAnomalyScore])
# fncsv.execute_local_test(db=db,db_schema=db_schema)
# db.register_functions([phg_iotfuncs.functions.CSVDataSource])
####################################################################################
fncsv = phg_iotfuncs.functions.CSVPreload(
# csv_file = 'extract_CaCO3_cont1.csv',
csv_file='*.csv',
rebaseTS=True,
output_item='loaded')
try:
fncsv.execute_local_test(db=db, db_schema=db_schema)
except Exception as exc:
print(f"Exception {exc}")
import traceback
traceback.print_exc()
pass
db.register_functions([phg_iotfuncs.functions.CSVPreload])
####################################################################################
# fnpre = phg_iotfuncs.functions.HTTPPreload(url='internal_test',request='GET',output_item='loaded')
# fnpre.execute_local_test(db=db,db_schema=db_schema)
# db.register_functions([phg_iotfuncs.functions.HTTPPreload])
####################################################################################
# fnts=phg_iotfuncs.functions.MergeSampleTimeSeries(input_items=['temp', 'pressure', 'velocity'],
# output_items=['temp', 'pressure', 'velocity'])
# db.register_functions([phg_iotfuncs.functions.MergeSampleTimeSeries])
# db.register_functions([HelloWorldPhG,CSVDataSource,HTTPPreload,MergeSampleTimeSeries])
return df
@classmethod
def build_ui(cls):
#define arguments that behave as function inputs
inputs = []
inputs.append(ui.UIMultiItem(
name = 'input_items',
datatype=float,
description = "Data items adjust",
output_item = 'output_items',
is_output_datatype_derived = True)
)
inputs.append(ui.UISingle(
name = 'factor',
datatype=float)
)
outputs = []
return (inputs,outputs)
'''
from iotfunctions.sample import MultiplyByFactor
adj = MultiplyByFactor(input_items=['speed', 'travel_time'],
factor=0.9,
output_items=['adjusted_speed', 'adjusted_travel_time'])
adj.execute_local_test()
db.register_functions(adj)
import datetime as dt
import json
import pandas as pd
import numpy as np
from sqlalchemy import Column, Integer, String, Float, DateTime, Boolean, func
from iotfunctions.base import BaseTransformer
from iotfunctions.metadata import EntityType
from iotfunctions.db import Database
from iotfunctions import ui
with open('credentials.json', encoding='utf-8') as F:
credentials = json.loads(F.read())
db_schema = None
db = Database(credentials=credentials)
from custom.functions import MultiplyByFator
db.register_functions([MultiplyByFator])
from custom.functions import MyCustomFunction
db.register_functions([MyCustomFunction])
# Paste contents in credentials_as.json file
# Save in scripts
'''
'''
1. Create a database object to access Watson IOT Platform Analytics DB.
'''
schema = 'bluadmin' # set if you are not using the default
with open('./scripts/credentials_as.json', encoding='utf-8') as F:
credentials = json.loads(F.read())
db = Database(credentials=credentials)
'''
2. Register custom function
You must unregister_functions if you change the method signature or required inputs
'''
db.unregister_functions(['SampleDimensionPreload_SS'])
db.register_functions([SampleDimensionPreload_SS])
'''
3. To do anything with IoT Platform Analytics, you will need one or more entity type.
This example assumes that the entity to which we are adding dimensions already exists
We add the custom function to this entity type to test it locally
'''
entity_name = 'issue_637_blank'
entity_type = db.get_entity_type(name=entity_name)
# get dimension table name - to add dimension values to
try:
dim_table_name = (
entity_type.get_attributes_dict()['_dimension_table_name']).lower()
except:
dim_table_name = entity_name + '_dimension'
bif.PythonExpression(
expression='df["energy_value"]*df["energy_compare_percent"]',
output_name='volume'), **{
'_timestamp': 'evt_timestamp',
'_db_schema': db_schema
})
'''
When creating an EntityType object you will need to specify the name of the entity, the database
object that will contain entity data
After creating an EntityType you will need to register it so that it visible in the Add Data to Entity Function UI.
To also register the functions and constants associated with the entity type, specify
'publish_kpis' = True.
'''
entity.register(raise_error=False)
db.register_functions([BIAssetHTTPPreload])
'''
To test the execution of kpi calculations defined for the entity type locally
use this function.
A local test will not update the server job log or write kpi data to the AS data
lake. Instead kpi data is written to the local filesystem in csv form.
'''
entity.exec_local_pipeline()
'''
view entity data
'''
df = db.read_table(table_name=entity_name, schema=db_schema)
print(df.head())
Column('deviceid', String(50)),
Column("torque", Integer()),
Column("acc", Integer()),
Column("load", Integer()),
Column("tool_type", Integer()),
Column("speed", Float()),
Column("travel_time", Float()),
**{
'_timestamp': 'evt_timestamp',
# '_production_mode': False,
'_db_schema': db_schema
})
# db.unregister_functions(["InvokeExternalModel"])
# exit()
db.register_functions([InvokeWMLModel])
# exit()
print("Function registered")
entity.register(raise_error=False)
print("Entity registered")
# generate data and set anomaly_score to zeros
print("Generating sample data")
entity.generate_data(days=2.0, drop_existing=True)
# df = db.read_table(table_name=entity_name, schema=db_schema)
# empty_columns = ["anomaly_score"]
# print("Setting " + empty_columns.join(" ") "columns to zeros ")
# df[empty_columns] = np.zeros(len(df))
# db.write_frame(df, table_name=entity_name, schema=db_schema, if_exists="replace")
import json
from iotfunctions.db import Database
with open('credentials.json', encoding='utf-8') as F:
credentials = json.loads(F.read())
db_schema = None
db = Database(credentials=credentials)
from functions.bad_functions import XXXXX
db.register_functions([XXXXX])
url='internal_test',
output_item='http_preload_done'),
bif.PythonExpression(expression='df["Temperature"]*df["Pressure"]',
output_name='Volume'), **{
'_timestamp': 'evt_timestamp',
'_db_schema': db_schema
})
'''
When creating an EntityType object you will need to specify the name of the entity, the database
object that will contain entity data
After creating an EntityType you will need to register it so that it visible in the UI.
To also register the functions and constants associated with the entity type, specify
'publish_kpis' = True.
'''
entity.register(raise_error=False)
db.register_functions([DemoHTTPPreload])
'''
To test the execution of kpi calculations defined for the entity type locally
use 'test_local_pipeline'.
A local test will not update the server job log or write kpi data to the AS data
lake. Instead kpi data is written to the local filesystem in csv form.
'''
entity.exec_local_pipeline()
'''
view entity data
'''
df = db.read_table(table_name=entity_name, schema=db_schema)
print(df.head())
bif.PythonExpression(expression='df["energy_value"]*df["energy_value"]',
output_name='volume'),
**{
'_timestamp': 'evt_timestamp',
'_db_schema': db_schema
})
'''
When creating an EntityType object you will need to specify the name of the entity, the database
object that will contain entity data
After creating an EntityType you will need to register it so that it visible in the Add Data to Entity Function UI.
To also register the functions and constants associated with the entity type, specify
'publish_kpis' = True.
'''
entity.register(raise_error=False)
db.register_functions([MaximoAssetHTTPPreload])
'''
To test the execution of kpi calculations defined for the entity type locally
use this function.
A local test will not update the server job log or write kpi data to the AS data
lake. Instead kpi data is written to the local filesystem in csv form.
'''
entity.exec_local_pipeline()
'''
view entity data
'''
# can get entity metadata with following
#
# TODO, for some reason this returns data but also throws exception.
# Paste contents in credentials_as.json file
# Save in scripts
'''
'''
1. Create a database object to access Watson IOT Platform Analytics DB.
'''
schema = 'bluadmin' # set if you are not using the default
with open('./scripts/credentials_as.json', encoding='utf-8') as F:
credentials = json.loads(F.read())
db = Database(credentials=credentials)
'''
2. Register custom function
You must unregister_functions if you change the method signature or required inputs
'''
db.unregister_functions(['SampleDimensionPreload_preset'])
db.register_functions([SampleDimensionPreload_preset])
'''
3. To do anything with IoT Platform Analytics, you will need one or more entity type.
This example assumes that the entity to which we are adding dimensions already exists
We add the custom function to this entity type to test it locally
'''
entity_name = 'issue_455_blank_preset'
entity_type = db.get_entity_type(name=entity_name)
# get dimension table name - to add dimension values to
try:
dim_table_name = (
entity_type.get_attributes_dict()['_dimension_table_name']).lower()
except:
dim_table_name = entity_name + '_dimension'
credentials = json.loads(F.read())
db_schema = None
db = Database(credentials=credentials)
'''
Import and instantiate the functions to be tested
The local test will generate data instead of using server data.
By default it will assume that the input data items are numeric.
Required data items will be inferred from the function inputs.
The function below executes an expression involving a column called x1
The local test function will generate data dataframe containing the column x1
By default test results are written to a file named df_test_entity_for_<function_name>
This file will be written to the working directory.
'''
from custom.functions import HelloWorldGG
fn = HelloWorldGG(name='Giri',
token='23537570-defd-4876-abde-68be94d9e26e',
output_col='greeting_output')
fn.execute_local_test(db=db, db_schema=db_schema)
'''
Register function so that you can see it in the UI
'''
db.register_functions([HelloWorldGG])
import datetime as dt
import json
import pandas as pd
import numpy as np
from sqlalchemy import Column, Integer, String, Float, DateTime, Boolean, func
from iotfunctions.base import BaseTransformer
from iotfunctions.metadata import EntityType
from iotfunctions.db import Database
from iotfunctions import ui
with open('credentials_as.json', encoding='utf-8') as F:
credentials = json.loads(F.read())
db_schema = None
db = Database(credentials=credentials)
from customSG.multiplybyfactorSG import MultiplyByFactorSG
db.register_functions([MultiplyByFactorSG])
from poc.functions import State_Timer
from iotfunctions.db import Database
from iotfunctions.enginelog import EngineLogging
EngineLogging.configure_console_logging(logging.DEBUG)
import datetime as dt
import sys
import pandas as pd
import numpy as np
logger = logging.getLogger(__name__)
with open('./credentials_Monitor-Demo2.json', encoding='utf-8') as F:
credentials = json.loads(F.read())
#with open('../Monitor-Demo-Credentials.json', encoding='utf-8') as F:
# credentials = json.loads(F.read())
db = Database(credentials=credentials)
db_schema = None # set if you are not using the default
#db.unregister_functions(["State_Timer"])
logger.debug("Function unregistered")
# exit()ßß
db.register_functions([State_Timer])
# exit()
#print("Function registered ")
if (len(sys.argv) <= 1) or (sys.argv[1] != 'test'):
db = Database(credentials=credentials)
print(db.cos_load)
# if in test mode call execute()
ais = anomaly.SpectralAnomalyScore('Val', windowsize=12, output_item='zscore')
kis = anomaly.KMeansAnomalyScore('Val', windowsize=4, output_item='kscore')
print("Instantiated")
# if there is a 2nd argument do not register but exit
if (len(sys.argv) > 1):
sys.exit()
EngineLogging.configure_console_logging(logging.DEBUG)
# with open('credentials_as_dev.json', encoding='utf-8') as F:
# credentials = json.loads(F.read())
#db.register_module(functions)
#db.register_functions([anomaly.SpectralAnomalyScore])
#db.register_functions([anomaly.KMeansAnomalyScore])
#db.register_functions([anomaly.NoDataAnomalyScoreNew])
#db.register_functions([anomaly.SimpleAnomaly])
#db.register_functions([anomaly.SimpleRegressor])
#db.register_functions([anomaly.GeneralizedAnomalyScore2])
#db.register_functions([anomaly.FFTbasedGeneralizedAnomalyScore2])
db.register_functions([anomaly.SaliencybasedGeneralizedAnomalyScore])
# Paste contents in credentials_as.json file
# Save in scripts
'''
'''
1. Create a database object to access Watson IOT Platform Analytics DB.
'''
schema = 'bluadmin' # set if you are not using the default
with open('./scripts/credentials_as.json', encoding='utf-8') as F:
credentials = json.loads(F.read())
db = Database(credentials=credentials)
'''
2. Register custom function
You must unregister_functions if you change the method signature or required inputs
'''
db.unregister_functions(['SampleDimensionPreload_SS'])
db.register_functions([SampleDimensionPreload_random])
'''
3. To do anything with IoT Platform Analytics, you will need one or more entity type.
This example assumes that the entity to which we are adding dimensions already exists
We add the custom function to this entity type to test it locally
'''
entity_name = 'issue_455_blank_random'
entity_type = db.get_entity_type(name=entity_name)
# get dimension table name - to add dimension values to
try:
dim_table_name = (
entity_type.get_attributes_dict()['_dimension_table_name']).lower()
except:
dim_table_name = entity_name + '_dimension'
To also register the functions and constants associated with the entity type, specify
'publish_kpis' = True.
'''
entity.register(raise_error=False)
print("Entity registered")
# entity.generate_data(days=0.5, drop_existing=True)
# print("Data generated")
df = db.read_table(table_name=entity_name, schema=db_schema)
print("Table head")
print(df.head())
print("Table columns")
print(list(df.columns))
db.register_functions([InvokeExternalModel])
print("Function registered")
'''
To test the execution of kpi calculations defined for the entity type locally
use this function.
A local test will not update the server job log or write kpi data to the AS data
lake. Instead kpi data is written to the local filesystem in csv form.
'''
# entity.exec_local_pipeline()
start = dt.datetime.utcnow() - dt.timedelta(days=7)
end = dt.datetime.utcnow() - dt.timedelta(days=1)
print("Starting pipeline")
entity.exec_local_pipeline() #start_ts=start, end_ts=end)
print("Completed pipeline")
import json
from iotfunctions.db import Database
with open('credentials_as.json', encoding='utf-8') as F:
credentials = json.loads(F.read())
db_schema = None
db = Database(credentials=credentials)
# from rawkintrevo_tutorial2.divbyfactor import RawkintrevosDivByFactor
from rawkintrevo_tutorial2.LegitRequest import ExternalModel
db.unregister_functions(["ExternalModel"])
db.register_functions([ExternalModel])
import rawkintrevo_tutorial2
db.register_module(rawkintrevo_tutorial2)
import rawkintrevo_tutorial2.LegitRequest
db.register_module(rawkintrevo_tutorial2.LegitRequest)
output_item='http_preload_done'),
bif.PythonExpression(expression='df["temp"]*df["pressure"]',
output_name='volume'), **{
'_timestamp': 'evt_timestamp',
'_db_schema': db_schema
})
'''
When creating an EntityType object you will need to specify the name of the entity, the database
object that will contain entity data
After creating an EntityType you will need to register it so that it visible in the UI.
To also register the functions and constants associated with the entity type, specify
'publish_kpis' = True.
'''
entity.register(raise_error=False)
db.register_functions([sample.HTTPPreload])
'''
To test the execution of kpi calculations defined for the entity type locally
use 'test_local_pipeline'.
A local test will not update the server job log or write kpi data to the AS data
lake. Instead kpi data is written to the local filesystem in csv form.
'''
entity.exec_local_pipeline()
'''
view entity data
'''
df = db.read_table(table_name=entity_name, schema=db_schema)
import json
from iotfunctions.db import Database
with open('credentials.json', encoding='utf-8') as F:
credentials = json.loads(F.read())
db_schema = None
db = Database(credentials=credentials)
from custom.functions import PrevDayHourlyAvgDiff, PrevDayHourlyAvgPercentDiff, PrevDayDailyAvgPercentDiff
db.register_functions([
PrevDayHourlyAvgDiff, PrevDayHourlyAvgPercentDiff,
PrevDayDailyAvgPercentDiff
],
url='git+https://github.com/rweidinger/functions@')
# fn = PrevDayHourlyAvgPercentDiff(
# input_item='GallonsRemaining',
# output_item='GallonsRemaining_PctChg')
#
# import datetime as dt
#
# today = dt.datetime.today()
# print('Right now it is:', today)
# yesterday = today - dt.timedelta(days=1)
# yesterday_date = yesterday.date()
# yesterday_hour = yesterday.hour
#
# df = fn.execute_local_test(generate_days=3, to_csv=True)
# print(df.loc[(df['deviceid'] == '73001') &
The automated test assumes that data items are numeric. You can also
specify datatypes by passing a list of SQL Alchemy column objects.
'''
cols = [Column('string_1', String(255))]
df = fn.execute_local_test(generate_days=1, to_csv=True, columns=cols)
'''
Custom functions must be registered in the AS function catalog before
you can use them. To register a function:
'''
db.register_functions([MultiplyTwoItems])
'''
After registration has completed successfully the function is available for
use in the AS UI.
The register_functions() method allows you to register more than one function
at a time. You can also register a whole module file with all of its functions.
'''
from iotfunctions import bif
db.register_module(bif)
'''
with open('credentials_as_dev.json', encoding='utf-8') as F:
credentials = json.loads(F.read())
db_schema = None
db = Database(credentials=credentials)
'''
Import and instantiate the functions to be tested
The local test will generate data instead of using server data.
By default it will assume that the input data items are numeric.
Required data items will be inferred from the function inputs.
The function below executes an expression involving a column called x1
The local test function will generate data dataframe containing the column x1
By default test results are written to a file named df_test_entity_for_<function_name>
This file will be written to the working directory.
'''
from AICustom.AIModelDesign import AIModelDesign
fn = AIModelDesign(name='AS_Tester', greeting_col='greeting')
fn.execute_local_test(db=db, db_schema=db_schema)
'''
Register function so that you can see it in the UI
'''
db.register_functions([AIModelDesign])
#!/user/bin/env python3
import json
import logging
from iotfunctions.db import Database
from iotfunctions.enginelog import EngineLogging
EngineLogging.configure_console_logging(logging.DEBUG)
with open('../dev_resources/credentials_as_dev.json', encoding='utf-8') as F:
#with open('../dev_resources/cognio.json', encoding='utf-8') as F:
credentials = json.loads(F.read())
db_schema = None
db = Database(credentials=credentials)
#from custom.forecast import Cognio_NeuralNetwork_Forecaster
from custom.functions import SS_HelloWorld
db.register_functions([SS_HelloWorld])
# bif.PythonExpression(expression='df["energy_value"]*df["energy_value"]',
bif.PythonExpression(expression='df["temperature"]', output_name='F'),
**{
'_timestamp': 'evt_timestamp',
'_db_schema': db_schema
})
'''
When creating an EntityType object you will need to specify the name of the entity, the database
object that will contain entity data
After creating an EntityType you will need to register it so that it visible in the Add Data to Entity Function UI.
To also register the functions and constants associated with the entity type, specify
'publish_kpis' = True.
'''
entity.register(raise_error=False)
db.register_functions([MaximoAssetHTTP])
'''
To test the execution of kpi calculations defined for the entity type locally
use this function.
A local test will not update the server job log or write kpi data to the AS data
lake. Instead kpi data is written to the local filesystem in csv form.
'''
entity.exec_local_pipeline()
'''
view entity data
'''
# can get entity metadata with following
#
# TODO, for some reason this returns data but also throws exception.
**{
'_timestamp' : 'evt_timestamp',
'_db_schema' : db_schema
})
'''
When creating an EntityType object you will need to specify the name of the entity, the database
object that will contain entity data
After creating an EntityType you will need to register it so that it visible in the Add Data to Entity Function UI.
To also register the functions and constants associated with the entity type, specify
'publish_kpis' = True.
'''
entity.register(raise_error=False)
db.register_functions([MaximoAssetHTTP])
'''
To test the execution of kpi calculations defined for the entity type locally
use this function.
A local test will not update the server job log or write kpi data to the AS data
lake. Instead kpi data is written to the local filesystem in csv form.
'''
entity.exec_local_pipeline()
'''
view entity data
'''
# can get entity metadata with following
Column('FUNCTION', String(50)), **{'schema': db_schema})
entity_dimension = entity.get_attributes_dict()['_dimension_table_name']
'''
When creating an EntityType object you will need to specify the name of the entity, the database
object that will contain entity data
After creating an EntityType you will need to register it so that it visible in the Add Data to Entity Function UI.
To also register the functions and constants associated with the entity type, specify
'publish_kpis' = True.
'''
entity.register(raise_error=False)
# When creating a custom preload function you can register it by uncommenting the following lines
# You must unregister_functions if you change the method signature or required inputs.
db.unregister_functions(['Issue455HTTPPreload'])
db.register_functions([Issue455HTTPPreload])
'''
To test the execution of kpi calculations defined for the entity type locally
use this function.
A local test will not update the server job log or write kpi data to the AS data
lake. Instead kpi data is written to the local filesystem in csv form.
'''
entity.exec_local_pipeline(**{'_production_mode': False})
'''
view entity data
'''
print("Read Table of new entity")
df = db.read_table(table_name=entity_name, schema=db_schema)
print(df.head())
