def __init__(self, config_file, end_point, function_url):
'''
Initialize the bootstrap object - this has the required methods
to parse the manifest and onboard the partner on to farmbeats
'''
self.fb_api = FarmbeatsApi(endpoint=end_point, function_url=function_url)
with open(config_file, "r") as conf:
file_content = conf.read()
self.bootstrap_manifest = json.loads(file_content)
def __init__(self):
self.adf_helper = ExtendedPropertiesReader()
if (FLAGS.get_access_token_url is None):
function_url = self.adf_helper.function_url
else:
function_url = FLAGS.get_access_token_url
self.fb_api = FarmbeatsApi(endpoint=FLAGS.end_point,
function_url=function_url)
if (FLAGS.eventhub_connection_string is None):
self.eventhub_connection_string = self.adf_helper.eventhub_connection_string
else:
self.eventhub_connection_string = FLAGS.eventhub_connection_string
def __init__(self, config_file, end_point, function_url, partner_id):
'''
Initialize the bootstrap object - this has the required methods
to parse the manifest and onboard the partner on to farmbeats
'''
self.partner_id = partner_id
# TODO: This needs to be initialized with function_url NOT aad creds
fb_config = BaseConfig(end_point=end_point,
tenant_id="",
app_id="",
app_secret="")
self.fb_api = FarmbeatsApi(config=fb_config)
with open(config_file, "r") as conf:
file_content = conf.read()
self.bootstrap_manifest = json.loads(file_content)
class GetWeatherDataJob:
'''
Class to fetch ISD weather data from Azure open datasets (NOAA ISD)
'''
#class constants
WEATHER_DATA_MODEL_NAME = "noaa_isd"
def __init__(self):
'''
Constructor for GetWeatherDataJob
'''
self.adf_helper = ExtendedPropertiesReader()
if (FLAGS.get_access_token_url is None):
function_url = self.adf_helper.function_url
else:
function_url = FLAGS.get_access_token_url
self.fb_api = FarmbeatsApi(endpoint=FLAGS.end_point, function_url=function_url)
if (FLAGS.eventhub_connection_string is None):
self.eventhub_connection_string = self.adf_helper.eventhub_connection_string
else:
self.eventhub_connection_string = FLAGS.eventhub_connection_string
def __get_weather_data_for_day(self, day, lat, lon):
'''
Gets weather data for a given day and pushes it to eventhub
'''
try:
# get data for given date range.
start_time = time.time()
LOG.info("Getting data for " + day.strftime("%m/%d/%Y, %H:%M:%S"))
weather_data = NoaaIsdWeather(day, day)
LOG.info("Successfully got data for " + day.strftime("%m/%d/%Y, %H:%M:%S"))
# get the data into a pandas data frame, so we can filter and process
weather_data_df = weather_data.to_pandas_dataframe()
LOG.info("Took {} seconds to get the data.".format(time.time() - start_time))
# out of the lat longs available get the nearest points
LOG.info("Finding the nearest latitude and longitude from the available data")
(nearest_lat, nearest_lon) = UtilFunctions.find_nearest_lat_longs_in_data(weather_data_df, lat, lon)
LOG.info("nearest lat, lon: [" + str(nearest_lat) + "," + str(nearest_lon) + "]")
# filter the data to this lat and lon
LOG.info("Filtering the data to nearest lat, lon")
filtered_weather_data = weather_data_df[(weather_data_df['latitude'] == nearest_lat) & (weather_data_df['longitude'] == nearest_lon)]
LOG.info(filtered_weather_data)
# push the data to eventhub
LOG.info("Pushing data to eventhub")
wdl_id = self.__push_weather_data_to_farmbeats(filtered_weather_data)
LOG.info("Successfully pushed data")
# Update the status for the job
if FLAGS.job_status_blob_sas_url:
msg = "Weather data pushed for start_date: {} to end_date: {}\n for nearest_lat: {}, nearest_lon: {}\n provided lat:{}, lon:{}".format(
FLAGS.start_date, FLAGS.end_date, nearest_lat, nearest_lon, FLAGS.latitude, FLAGS.longitude)
writer = JobStatusWriter(FLAGS.job_status_blob_sas_url)
output_writer = writer.get_output_writer()
output_writer.set_prop("WeatherDataLocationId: ", wdl_id)
output_writer.set_prop("Message: ", msg)
writer.set_success(True)
writer.flush()
except Exception as err:
# Update the status in failure
if FLAGS.job_status_blob_sas_url:
writer = JobStatusWriter(FLAGS.job_status_blob_sas_url)
writer.set_success(False)
writer.flush()
raise JobError(str(err), JobConstants.INTERNAL_ERROR, False)
def get_weather_data(self, start_date, end_date, lat, lon):
'''
Gets the closest proximity weather data available for the given date range,
'''
start_date = parser.parse(FLAGS.start_date)
end_date = parser.parse(FLAGS.end_date)
for day in UtilFunctions.daterange(start_date, end_date):
self.__get_weather_data_for_day(day, lat, lon)
def __get_eventhub_format(self, row):
'''
Convert the data to a format that can be pushed to eventhub, which can be subsequently read by TSI
'''
output = {}
# get the timestamp
output["timestamp"] = row["datetime"].isoformat()
output["Elevation"] = row["elevation"]
output["WindAngle"] = row["windAngle"]
output["AmbientTemperature"] = row["temperature"]
output["SeaLvlPressure"] = row["seaLvlPressure"]
output["PrecipTime"] = row["precipTime"]
output["PrecipDepth"] = row["precipDepth"]
output["SnowDepth"] = row["snowDepth"]
return output
def __process_weather_data_for_tsi(self, weather_data_location_id, weather_data):
'''
Converts the weather data from Pandas data frame to that expected by TSI
'''
msgs = []
msg = json.loads(JobConstants.WEATHER_TELEMETRY_FORMAT)
msg[JobConstants.WEATHER_DATA_LOCATIONS][0][JobConstants.ID] = weather_data_location_id
row_data = msg[JobConstants.WEATHER_DATA_LOCATIONS][0][JobConstants.WEATHER_DATA]
# iterrows gives (index, row) tuples rather than just rows.
# so, throwing away the index and just getting the row.
for _,row in weather_data.iterrows():
row_data.append(self.__get_eventhub_format(row))
msg[JobConstants.WEATHER_DATA_LOCATIONS][0][JobConstants.WEATHER_DATA] = row_data
msgs.append(json.dumps(msg))
LOG.info("Event hub msg: {}".format(json.dumps(msg)))
return msgs
async def __send_to_eventhub(self, weather_data_location_id, weather_data):
'''
Sends weather data to eventhub
'''
# Create a producer client to send messages to the event hub.
producer = EventHubProducerClient.from_connection_string(conn_str=self.eventhub_connection_string,
eventhub_name=FLAGS.eventhub_name)
async with producer:
event_data_batch = await producer.create_batch()
# process the weather data and create the msgs
msgs = self.__process_weather_data_for_tsi(weather_data_location_id, weather_data)
# Add events to the batch
for msg in msgs:
event_data_batch.add(EventData(msg))
await producer.send_batch(event_data_batch)
def __get_weather_data_model_id(self, name):
'''
returns the weather data model id, given the name
'''
wsms = self.fb_api.get_weather_data_model_api().weather_data_model_get_all(names=[name], includes=[JobConstants.WEATHER_MEASURES, JobConstants.PROPERTIES]).to_dict()
if (wsms):
return wsms[JobConstants.ITEMS][0][JobConstants.ID]
else:
raise JobError("weather data model {} not found!".format(wsms), JobConstants.INTERNAL_ERROR, False)
def __get_weather_data_location_id(self):
'''
checks if a weather data location already exists for the location,
if yes -> returns it's id.
Else, creates and returns the id.
'''
data_model_id = self.__get_weather_data_model_id(name=GetWeatherDataJob.WEATHER_DATA_MODEL_NAME)
weather_data_locations = self.fb_api.get_weather_data_location_api().weather_data_location_get_all().to_dict()
for loc in weather_data_locations[JobConstants.ITEMS]:
lat = loc[JobConstants.LOCATION][JobConstants.LATITUDE]
lon = loc[JobConstants.LOCATION][JobConstants.LONGITUDE]
if (lat == float(FLAGS.latitude) and lon == float(FLAGS.longitude) and data_model_id == loc[JobConstants.WEATHER_DATA_MODEL_ID]):
# Found! - weather data location for the given location already exists
return loc[JobConstants.ID]
# doesn't exist - create weather data_location
weather_data_location_payload = {}
weather_data_location_payload[JobConstants.NAME] = "NOAA_ISD_job_generated_location_[" + str(FLAGS.latitude) + "," + str(FLAGS.longitude) + "]"
weather_data_location_payload[JobConstants.WEATHER_DATA_MODEL_ID_PAYLOAD] = data_model_id
weather_data_location_payload[JobConstants.LOCATION] = { JobConstants.LATITUDE: FLAGS.latitude, JobConstants.LONGITUDE: FLAGS.longitude}
if (FLAGS.farm_id):
weather_data_location_payload[JobConstants.FARM_ID] = FLAGS.farm_id
res = self.fb_api.get_weather_data_location_api().weather_data_location_create(input=weather_data_location_payload).to_dict()
return res[JobConstants.ID]
def __push_weather_data_to_farmbeats(self, weather_data):
'''
Pushes weather data to farmbeats - ingests data
'''
weather_data_location_id = self.__get_weather_data_location_id()
LOG.info("Weather data location id: {}".format(weather_data_location_id))
loop = asyncio.get_event_loop()
loop.run_until_complete(self.__send_to_eventhub(weather_data_location_id, weather_data))
return weather_data_location_id
class Bootstrap:
# class constants
WEATHER_MEASURE_KEY = "WeatherMeasureType"
WEATHER_MEASURE_UNIT_KEY = "WeatherMeasureUnit"
ADD_MEASURE_TYPES = "add_extended_measure_types"
ADD_MEASURE_UNITS = "add_extended_measure_units"
ADD_WEATHER_DATA_MODELS = "add_data_models"
ADD_JOB_TYPES = "add_job_types"
def __init__(self, config_file, end_point, function_url):
'''
Initialize the bootstrap object - this has the required methods
to parse the manifest and onboard the partner on to farmbeats
'''
self.fb_api = FarmbeatsApi(endpoint=end_point, function_url=function_url)
with open(config_file, "r") as conf:
file_content = conf.read()
self.bootstrap_manifest = json.loads(file_content)
def add_new_extended_measure_types(self):
'''
Adds the extended measure types mentioned in the bootstrap_manifest
'''
# get the types that already exist
weather_types = self.fb_api.get_extended_type_api().extended_type_get_all(keys=[Bootstrap.WEATHER_MEASURE_KEY]).to_dict()
type_id = weather_types["items"][0]["id"]
weather_type_list = weather_types["items"][0]["value"]
# get the types that need to be added
types_to_be_added = self.bootstrap_manifest[Bootstrap.ADD_MEASURE_TYPES]
# merge them
merged_types_list = list(set(weather_type_list + types_to_be_added))
# update farmbeats with new list
inp = {}
inp["key"] = Bootstrap.WEATHER_MEASURE_KEY
inp["value"] = merged_types_list
self.fb_api.get_extended_type_api().extended_type_update(id=type_id, input=inp)
def add_new_extended_measure_units(self):
'''
Adds the extended measure units mentioned in the bootstrap_manifest
'''
# get the units that already exist
weather_units = self.fb_api.get_extended_type_api().extended_type_get_all(keys=[Bootstrap.WEATHER_MEASURE_UNIT_KEY]).to_dict()
unit_id = weather_units["items"][0]["id"]
weather_units_list = weather_units["items"][0]["value"]
# get the units that need to be added
units_to_be_added = self.bootstrap_manifest[Bootstrap.ADD_MEASURE_UNITS]
# merge them
merged_units_list = list(set(weather_units_list + units_to_be_added))
# update farmbeats with new list
inp = {}
inp["key"] = Bootstrap.WEATHER_MEASURE_UNIT_KEY
inp["Value"] = merged_units_list
self.fb_api.get_extended_type_api().extended_type_update(id=unit_id, input=inp)
def upsert_weather_data_models(self):
'''
Upserts the weather data models mentioned in the bootstrap_manifest
'''
# get the existing weather data models
existing_weather_data_models = self.fb_api.get_weather_data_model_api().weather_data_model_get_all(includes=["WeatherMeasures", "Properties"]).to_dict()
# get the weather data models to upsert
partner_wsm_list = self.bootstrap_manifest[Bootstrap.ADD_WEATHER_DATA_MODELS]
# for every weather station model in the manifest
for weather_data_model in partner_wsm_list:
existing_wsms = existing_weather_data_models["items"]
found = False
if (len(existing_wsms) > 0):
# if a matching name is found for the partner - update
for wsm in existing_wsms:
if (wsm["name"] == weather_data_model["name"]):
wsm_id = wsm["id"]
# update
found = True
self.fb_api.get_weather_data_model_api().weather_data_model_update(id=wsm_id, input=weather_data_model)
if (not found):
# else insert
self.fb_api.get_weather_data_model_api().weather_data_model_create(input=weather_data_model)
def upsert_job_types(self):
'''
Upserts the partner job types mentioned in the bootstrap_manifest
'''
# get the existing job types for this partner.
existing_partner_job_types = self.fb_api.get_job_type_api().job_type_get_all(includes=["PipelineDetails", "Properties"]).to_dict()
# get the job types to upsert
partner_job_types = self.bootstrap_manifest[Bootstrap.ADD_JOB_TYPES]
# for every job type in the manifest
for job_type in partner_job_types:
existing_job_types = existing_partner_job_types["items"]
found = False
if (len(existing_job_types) > 0):
# if a matching name is found for the job type - update
for existing_job_type in existing_job_types:
if (existing_job_type["name"] == job_type["name"]):
job_type_id = existing_job_type["id"]
# update
found = True
self.fb_api.get_job_type_api().job_type_update(id=job_type_id, input=job_type)
if (not found):
# else insert
self.fb_api.get_job_type_api().job_type_create(input=job_type)
class Bootstrap:
# class constants
WEATHER_MEASURE_KEY = "WeatherMeasureType"
WEATHER_MEASURE_UNIT_KEY = "WeatherMeasureUnit"
ADD_MEASURE_TYPES = "add_extended_measure_types"
ADD_MEASURE_UNITS = "add_extended_measure_units"
ADD_WEATHER_STATION_MODELS = "add_weather_station_models"
ADD_JOB_TYPES = "add_job_types"
def __init__(self, config_file, end_point, function_url, partner_id):
'''
Initialize the bootstrap object - this has the required methods
to parse the manifest and onboard the partner on to farmbeats
'''
self.partner_id = partner_id
# TODO: This needs to be initialized with function_url NOT aad creds
fb_config = BaseConfig(end_point=end_point,
tenant_id="",
app_id="",
app_secret="")
self.fb_api = FarmbeatsApi(config=fb_config)
with open(config_file, "r") as conf:
file_content = conf.read()
self.bootstrap_manifest = json.loads(file_content)
def add_new_extended_measure_types(self):
'''
Adds the extended measure types mentioned in the bootstrap_manifest
'''
# get the types that already exist
weather_types = self.fb_api.get_extended_type_api(
).extended_type_get_all(
keys=[Bootstrap.WEATHER_MEASURE_KEY]).to_dict()
type_id = weather_types["items"][0]["id"]
weather_type_list = weather_types["items"][0]["value"]
# get the types that need to be added
types_to_be_added = self.bootstrap_manifest[
Bootstrap.ADD_MEASURE_TYPES]
# merge them
merged_types_list = list(set(weather_type_list + types_to_be_added))
# update farmbeats with new list
inp = {}
inp["key"] = Bootstrap.WEATHER_MEASURE_KEY
inp["value"] = merged_types_list
self.fb_api.get_extended_type_api().extended_type_update(id=type_id,
input=inp)
def add_new_extended_measure_units(self):
'''
Adds the extended measure units mentioned in the bootstrap_manifest
'''
# get the units that already exist
weather_units = self.fb_api.get_extended_type_api(
).extended_type_get_all(
keys=[Bootstrap.WEATHER_MEASURE_UNIT_KEY]).to_dict()
unit_id = weather_units["items"][0]["id"]
weather_units_list = weather_units["items"][0]["value"]
# get the units that need to be added
units_to_be_added = self.bootstrap_manifest[
Bootstrap.ADD_MEASURE_UNITS]
# merge them
merged_units_list = list(set(weather_units_list + units_to_be_added))
# update farmbeats with new list
inp = {}
inp["key"] = Bootstrap.WEATHER_MEASURE_UNIT_KEY
inp["Value"] = merged_units_list
self.fb_api.get_extended_type_api().extended_type_update(id=unit_id,
input=inp)
def upsert_weather_station_models(self):
'''
Upserts the weather station models mentioned in the bootstrap_manifest
'''
# get the existing weather station models
# TODO - uncomment this; it's the right way to go.
# existing_weather_station_models = self.fb_api.get_weather_station_model_api().weather_station_model_get_all(partner_id=self.partner_id).to_dict()
existing_weather_station_models = self.fb_api.get_weather_station_model_api(
).weather_station_model_get_all().to_dict()
# get the weather station models to upsert
partner_wsm_list = self.bootstrap_manifest[
Bootstrap.ADD_WEATHER_STATION_MODELS]
# for every weather station model in the manifest
for weather_station_model in partner_wsm_list:
existing_wsms = existing_weather_station_models["items"]
if (len(existing_wsms) > 0):
# if a matching name is found for the partner - update
for wsm in existing_wsms:
if (wsm["name"] == weather_station_model["name"]):
wsm_id = wsm["id"]
self.fb_api.get_weather_station_model_api(
).weather_station_model_update(
id=wsm_id, input=weather_station_model)
return
else:
# else insert
self.fb_api.get_weather_station_model_api(
).weather_station_model_create(input=weather_station_model)
def upsert_job_types(self):
'''
Upserts the partner job types mentioned in the bootstrap_manifest
'''
# get the existing job types for this partner.
existing_partner_job_types = self.fb_api.get_job_type_api(
).job_type_get_all(partner_id=self.partner_id).to_dict()
# get the job types to upsert
partner_job_types = self.bootstrap_manifest[Bootstrap.ADD_JOB_TYPES]
# for every job type in the manifest
for job_type in partner_job_types:
existing_job_types = existing_partner_job_types["items"]
if (len(existing_job_types) > 0):
# if a matching name is found for the job type - update
for existing_job_type in existing_job_types:
if (existing_job_type["name"] == job_type["name"]):
job_type_id = existing_job_type["id"]
self.fb_api.get_job_type_api().job_type_update(
id=job_type_id, input=job_type)
return
else:
# else insert
self.fb_api.get_job_type_api().job_type_create(input=job_type)