def populate_model(self):
"""
Method to populate the model by pull all active parsing methods from the
ParsingRules table in the database
:return:
"""
self.clear()
Measurements = Lookups.alias()
Organizations = Lookups.alias()
rules = ParsingRules.select(ParsingRules, Equipment, Measurements, Organizations)\
.join(Measurements, JOIN.LEFT_OUTER, on=(Measurements.lookup == ParsingRules.measurement_lu).alias("measurement"))\
.switch(ParsingRules)\
.join(Equipment)\
.join(Organizations, on=(Equipment.organization == Organizations.lookup).alias("org"))\
.order_by(ParsingRules.priority.asc())
for rule in rules:
item = dict()
item["parsing_rules_id"] = rule.parsing_rules
item["equipment"] = "{0} {1}".format(rule.equipment.org.value,
rule.equipment.model)
item["line_starting"] = rule.line_starting
item["is_line_starting_substr"] = rule.is_line_starting_substr
item["field_position"] = rule.field_position
item["line_ending"] = rule.line_ending
item["measurement"] = "{0}, {1}".format(rule.measurement.value,
rule.measurement.subvalue)
item["priority"] = rule.priority
self.appendItem(item)
def process_data(self):
errors = list()
finclip_lu = Lookups.get(Lookups.type == 'Observation',
Lookups.value == "Finclip ID").lookup
results = SpecimenImport.select()
for i, result in enumerate(results):
# if i == 1:
# break
logging.info(f"row={i+1} = {model_to_dict(result)}")
try:
# Find the sibling specimen where finclip ID = result.finclip
sibling_specimen, sibling_specimen_created = Specimen.get_or_create(
alpha_value=result.finclip, action_type=finclip_lu)
# Determine the specimen lookup value
if "Age ID" in result.specimen_type:
subvalue = result.specimen_type.strip('Age ID').strip()
specimen_lu = Lookups.get(
Lookups.type == 'Observation',
Lookups.value == 'Age ID',
Lookups.subvalue == subvalue).lookup
else:
specimen_lu = Lookups.get(
Lookups.type == 'Observation',
Lookups.value == result.specimen_type).lookup
# Determine the species sampling plan
ssp_results = SpeciesSamplingPlanLu.select(SpeciesSamplingPlanLu, ProtocolLu) \
.join(ProtocolLu) \
.where(SpeciesSamplingPlanLu.plan_name==result.plan_name,
ProtocolLu.action_type==specimen_lu)
for ssp_result in ssp_results:
ssp_lu = ssp_result.species_sampling_plan
specimen, specimen_created = Specimen.get_or_create(
parent_specimen=sibling_specimen.parent_specimen,
species_sampling_plan=ssp_lu,
action_type=specimen_lu,
alpha_value=result.specimen_number,
defaults={'catch': sibling_specimen.catch})
logging.info(
f"Specimen created = {specimen_created}: {result.specimen_type} = {result.specimen_number}"
f" > {specimen.specimen}")
except Exception as ex:
errors.append(f"row={i+1}, set_id={result.set_id} > {ex}")
for err in errors:
logging.info(f"{err}")
def populate_model(self):
"""
Method to retrieve all of the equipment from the database equipment table
and use them to populate the lvEquipment Listiew on the AddComportDialog.qml
called from the SensorDataFeeds.qml screen
:return:
"""
self.clear()
Make = Lookups.alias()
Category = Lookups.alias()
records = Equipment.select(Equipment, Make, Category)\
.join(Make, on=(Make.lookup == Equipment.organization).alias('make'))\
.switch(Equipment)\
.join(Category, on=(Category.lookup == Equipment.equipment_category).alias('category'))\
.where(Equipment.is_active == "True")\
.order_by(Make.value.asc(), Equipment.name.asc(), Category.value.asc())
for record in records:
item = dict()
item["equipment_id"] = record.equipment
item["equipment"] = " ".join([record.make.value, record.model])
item["equipment_type"] = record.category.value
self.appendItem(item)
def populate_model(self):
""""
Method to initially populate the model on startup
"""
self.clear()
results = Lookups.select().where(Lookups.is_active == "True",
Lookups.type == "Vessel Number") \
.order_by(Lookups.value.asc())
for result in results:
item = dict()
item["id"] = result.lookup
item["text"] = result.description
self.appendItem(item)
def populate_model(self):
"""
Method to retrieve all of measurements + units of measurement from the database lookups table
and use them to populate the cbMeasurement combobox on the SensorDataFeeds.qml page
:return:
"""
self.clear()
records = Lookups.select(Lookups.value, Lookups.subvalue, Lookups.lookup)\
.where(Lookups.type == "Measurement", Lookups.is_active == "True")\
.order_by(Lookups.value.asc()) \
.group_by(Lookups.value, Lookups.subvalue)
for record in records:
item = dict()
item["lookup_id"] = record.lookup
item["text"] = record.value + ", " + record.subvalue \
if record.subvalue else record.value
self.appendItem(item)
def populate_model(self, set_id):
"""
Method to populate the model
:return:
"""
if set_id == "":
logging.info(f"Set ID is empty: {set_id}")
try:
self.clear()
results = CutterStationView.select()\
.where(CutterStationView.set_id==set_id)\
.order_by(CutterStationView.drop, CutterStationView.angler,
CutterStationView.hook)
for result in results:
self.appendItem({
'dropNumber': result.drop,
'angler': result.angler,
'hook': result.hook,
'species': result.species,
'length': result.length,
'weight': result.weight,
'sex': result.sex,
'finclip': result.finclip,
'otolith': result.otolith,
'disposition': result.disposition,
'tagNumber': result.tag_number,
'recordedBy': result.recorded_by
})
cutter_oa = Lookups.get(type='Cutter Attribute',
value='Recorder Name').lookup
site_op_id = Operations.get(operation_number=set_id).operation
cutter_recorder_results = OperationAttributes.select()\
.where(OperationAttributes.attribute_type_lu==cutter_oa,
OperationAttributes.operation==site_op_id)
if cutter_recorder_results.count() >= 1:
for result in cutter_recorder_results:
self.cutterRecorderChanged.emit(result.attribute_alpha)
break
else:
self.cutterRecorderChanged.emit("")
except Exception as ex:
logging.error(f"Error populating the CutterStation model: {ex}")
def add_new_measurement(self, measurement, unit_of_measurement):
"""
Method called by the NewMeasurementDialog.qml via SensorDataFeeds.qml when a
user wants to add a new measurement type. This consists of the measurement name
as well as the unit of measurement
:param measurement: str - should be in the form: Latitude - Vessel, Time - UTC, etc.
:param unit_of_measurement: str - abbreviated unit of measurements, e.g. m, kg, km, etc.
:return:
"""
if not isinstance(measurement, str) or not isinstance(unit_of_measurement, str) or \
measurement == "" or unit_of_measurement == "":
logging.error(
"Invalid measurement of unit of measurement provided: {0}, {1}"
.format(measurement, unit_of_measurement))
return
# Insert into the Lookups table in the database
try:
# Lookups.insert(type="Measurement", value=measurement, subvalue=unit_of_measurement,
# is_active="True").execute()
# lookup_id = Lookups.select().order_by(Lookups.lookup.desc()).get().lookup
lookup, created = Lookups.get_or_create(
type="Measurement",
value=measurement,
subvalue=unit_of_measurement,
is_active="True")
lookup_id = lookup.lookup
except Exception as ex:
logging.error(
"failed to insert new measurement type: {0}".format(ex))
# Add to the various models - measurements, uom, and measurementsUofm
self._measurements_model.add_row(lookup_id=lookup_id,
value=measurement)
self._units_of_measurement_model.add_row(lookup_id=lookup_id,
subvalue=unit_of_measurement)
self._measurements_uom_model.add_row(lookup_id=lookup_id,
value=measurement,
subvalue=unit_of_measurement)
# Emit a signal - this is used to auto-select this newly added measurement in SensorDataFeeds.qml
self.measurementsUomModelChanged.emit("{0}, {1}".format(
measurement, unit_of_measurement))
def populate_model(self):
"""
Method to retrieve all of the measurements form the database lookups table
and use them to populate the cbMeasurement combobox on the NewMeasurementDialog.qml
called from the SensorDataFeeds.qml screen
:return:
"""
self.clear()
records = Lookups.select(Lookups.value, Lookups.lookup) \
.where(Lookups.type == "Measurement", Lookups.is_active == "True")\
.order_by(Lookups.value.asc()) \
.group_by(Lookups.value)
for record in records:
item = dict()
item["lookup_id"] = record.lookup
item["text"] = record.value
self.appendItem(item)
def populate_model(self):
"""
Method to retrieve all of the units of measurement from the database lookups table
and use them to populate the cbUnitOfMeasurement combobox on the
NewMeasurementDialog.qml that is called from the SensorDataFeeds.qml page
:return:
"""
self.clear()
# TODO - Ensure that the units of measurement returned are unique
records = Lookups.select(Lookups.subvalue, Lookups.lookup) \
.where(Lookups.type == "Measurement", Lookups.is_active == "True") \
.order_by(Lookups.subvalue.asc()) \
.group_by(Lookups.subvalue)
for record in records:
item = dict()
item["lookup_id"] = record.lookup
item["text"] = record.subvalue
self.appendItem(item)
def populate_model(self):
"""
Method to retrieve all of the sensor configurations from the database
deployed_equipment table and use them to populate the tvSensorConfiguration
tableview on the SensorDataFeeds.qml screen
:return:
"""
self.clear()
Organization = Lookups.alias()
records = DeployedEquipment.select(DeployedEquipment, Equipment, Organization)\
.join(Equipment)\
.join(Organization, on=(Organization.lookup == Equipment.organization).alias('org'))\
.where(DeployedEquipment.deactivation_date.is_null())\
.order_by(DeployedEquipment.com_port.asc())
for record in records:
item = dict()
item[
"equipment"] = record.equipment.org.value + " " + record.equipment.model
# item["com_port"] = "COM{0}".format(record.com_port)
item["com_port"] = record.com_port
item["moxa_port"] = record.moxa_port
item["data_status"] = "red"
item["start_stop_status"] = "stopped"
item["delete_row"] = ""
item["baud_rate"] = record.baud_rate if record.baud_rate else 9600
item["data_bits"] = record.data_bits if record.data_bits else 8
item["parity"] = record.parity if record.parity else "None"
item["stop_bits"] = record.stop_bits if record.stop_bits else 1
item[
"flow_control"] = record.flow_control if record.flow_control else "None"
item["deployed_equipment_id"] = record.deployed_equipment
self.appendItem(item)
self._app.serial_port_manager.add_thread(com_port_dict=item)
self.setItems(
sorted(self.items, key=lambda x: int(x["com_port"].strip("COM"))))
def update_row(self, index, value):
"""
Method to update one of the settings rows
:param index:
:param value:
:return:
"""
if not isinstance(index, int) or index < 0 or index >= self.count:
logging.error("Error updating a settings row: {0}".format(value))
return
if isinstance(value, QJSValue):
value = value.toVariant()
# Convert all boolean objects to strings
value = str(value) if isinstance(value, bool) else value
# Set to empty string as need and then update the model
value = value if value else ""
# Update the model - two fields to update
self.setProperty(index=index, property="value", value=value)
self.setProperty(index=index, property="new_value", value=value)
# Database Storage Conversions
parameter = self.get(index=index)["parameter"]
# Set update_db flag
update_db = True
# Database Storage Conversions - Dates/Times
if parameter in [
"Leg 1 Start", "Leg 1 End", "Leg 2 Start", "Leg 2 End"
]:
if "end" in parameter.lower():
value += " 23:59:59"
value = self._dc.time_to_iso_format(value)
# Names to Foreign Keys in Personnel Table
elif any(substring in parameter for substring in [
"Leg 1 FPC", "Leg 2 FPC", "Scientist 1", "Scientist 2",
"Scientist 3", "Captain", "Second Captain", "Deckhand 1",
"Deckhand 2", "Deckhand 3"
]):
try:
if value:
logging.info(
f"Personnel found: {parameter} >>> value: {value}")
last_name, first_name = value.split(",")
person = Personnel.get(Personnel.first_name == first_name,
Personnel.last_name == last_name)
value = person.personnel
else:
update_db = False
except Exception as ex:
value = None
update_db = False
logging.error(
"Error getting the proper personnel name: {0}".format(ex))
# Names to Foreign Keys in Lookups Table
elif parameter in ["Vessel"]:
try:
if value:
vessel = Lookups.get(Lookups.description == value)
value = vessel.lookup
else:
update_db = False
except Exception as ex:
value = None
update_db = False
logging.error(
"Error getting the proper vessel name: {0}".format(ex))
# Update the database
try:
if update_db:
settings_id = self.get(index=index)["settings"]
SettingsTable.update(value=value).where(
SettingsTable.settings == settings_id).execute()
except Exception as ex:
logging.error(
"Exception updating the settings table: {0}".format(ex))
def populate_model(self):
""""
Method to initially populate the model on startup
"""
self.clear()
results = SettingsTable.select().where(SettingsTable.is_active == "True")\
.order_by(SettingsTable.parameter.asc())
for result in results:
# item = dict()
# item["settings_id"] = result.settings
# item["parameter"] = result.parameter
# item["value"] = result.value
item = model_to_dict(result)
item["delegate_type"] = "TextField"
if item["parameter"] in [
"Leg 1 Start", "Leg 1 End", "Leg 2 Start", "Leg 2 End"
]:
# Convert to mm/dd/yyyy format
item["value"] = self._dc.iso_to_common_date_format(
item["value"])
elif any(substring in item["parameter"] for substring in [
"Leg 1 FPC", "Leg 2 FPC", "Scientist 1", "Scientist 2",
"Scientist 3", "Captain", "Second Captain", "Cook",
"Deckhand 1", "Deckhand 2", "Deckhand 3"
]):
item["delegate_type"] = "ComboBox"
if any(substring in item["parameter"] for substring in
["FPC", "Scientist 1", "Scientist 2", "Scientist 3"]):
item["model_type"] = "Scientists"
is_science_team = "True"
else:
item["model_type"] = "Crew"
is_science_team = "False"
try:
person = Personnel.get(
Personnel.personnel == item["value"],
Personnel.is_science_team == is_science_team)
item["value"] = person.last_name + ", " + person.first_name
except Exception as ex:
item["value"] = ""
elif item["parameter"] in ["Vessel"]:
item["delegate_type"] = "ComboBox"
item["model_type"] = "Vessels"
try:
lookup = Lookups.get(Lookups.lookup == item["value"])
item["value"] = lookup.description
except Exception as ex:
item["value"] = ""
elif item["parameter"] in ["Backup Folder"]:
item["delegate_type"] = "FolderBrowser"
elif item["parameter"] == "Depth Output Serial Port":
item["model_type"] = "ComPorts"
item["delegate_type"] = "DialogListView"
elif item["parameter"] == "Depth Output Serial Port Baud Rate":
item["model_type"] = "BaudRates"
item["delegate_type"] = "DialogListView"
elif item["parameter"] == "Depth Output Serial Port Status":
item["delegate_type"] = "Switch"
item["new_value"] = item["value"]
self.appendItem(item)
def save_best_species_to_db(self, set_id, adh, species):
"""
Method to save the best species down to the database
:param set_id: str - the set id of the operation for saving to the database
:param adh:
:param species:
:return:
"""
if len(adh) != 3:
logging.error(f"ADH is not the right length for updating the best species: {adh}")
return
logging.info(f"save_best_species_to_db for set_id{set_id}, adh={adh}, species={species}")
try:
# Parse out the drop, angler, hook
drop = adh[1]
angler = adh[0]
hook = adh[2]
# Determine the best species catch content ID, it might be none if the best species is blank
best_catch_content = None
if species:
best_catch_content = CatchContentLu.get(CatchContentLu.display_name == species).catch_content
# Find the catch_id that matches this set_id and adh
set_op = Operations.get(Operations.operation_number==set_id).operation
drop_op = Operations.get(Operations.parent_operation==set_op, Operations.operation_number==drop).operation
angler_op = Operations.get(Operations.parent_operation==drop_op, Operations.operation_number==angler).operation
hook_type = Lookups.get(Lookups.type=="Receptacle Type", Lookups.value=="Hook").lookup
operation_type = Lookups.get(Lookups.type=="Operation", Lookups.value=="Angler").lookup
catch_record, created = Catch.get_or_create(operation=angler_op, receptacle_seq=hook,
receptacle_type=hook_type, operation_type=operation_type)
logging.info(f"\t\tupdating catch best species with catch id={catch_record.catch} "
f"to best species content = {best_catch_content}")
# NOT SURE WHY THE BELOW DOES NOT WORK, BUT IT DOESN'T
# Sets = Operations.alias()
# Drops = Operations.alias()
# Anglers = Operations.alias()
# results = Sets.select(Catch.catch) \
# .join(Drops, JOIN.LEFT_OUTER, on=(Drops.parent_operation == Sets.operation).alias('drop')) \
# .join(Anglers, JOIN.LEFT_OUTER, on=(Anglers.parent_operation == Drops.operation).alias('angler')) \
# .join(Catch, JOIN.LEFT_OUTER, on=(Catch.operation == Anglers.operation).alias('catch')) \
# .where(Sets.operation_number==set_id,
# Drops.operation_number==drop,
# Anglers.operation_number==angler,
# Catch.receptacle_seq==hook)
# logging.info(f"the query is: {results}")
#
# if results.count() == 1:
#
# logging.info(f"length is {results.count()}")
# THIS BREAKS, AS DOES TRYING TO ITERATE ON results
# logging.info(f"result = {results.get().catch}")
# Update the catch table with the new best species catch content ID
Catch.update(best_catch_content=best_catch_content).where(Catch.catch==catch_record.catch).execute()
except Exception as ex:
logging.error(f"Error updating the best species: {ex}")
def process_data(self):
errors = list()
set_op_type = Lookups.get(Lookups.type == 'Operation',
Lookups.value == 'Site').lookup
drop_op_type = Lookups.get(Lookups.type == 'Operation',
Lookups.value == 'Drop').lookup
angler_op_type = Lookups.get(Lookups.type == 'Operation',
Lookups.value == 'Angler').lookup
length_lu = Lookups.get(Lookups.type == 'Observation',
Lookups.value == "Length").lookup
weight_lu = Lookups.get(Lookups.type == 'Observation',
Lookups.value == "Weight").lookup
sex_lu = Lookups.get(Lookups.type == 'Observation',
Lookups.value == "Sex").lookup
finclip_lu = Lookups.get(Lookups.type == 'Observation',
Lookups.value == "Finclip ID").lookup
otolith_lu = Lookups.get(Lookups.type == 'Observation',
Lookups.value == "Age ID",
Lookups.subvalue == "Otolith").lookup
descended_lu = Lookups.get(Lookups.type == 'Observation',
Lookups.value == "Disposition",
Lookups.subvalue == "Descended").lookup
released_lu = Lookups.get(Lookups.type == 'Observation',
Lookups.value == "Disposition",
Lookups.subvalue == "Released").lookup
sacrificed_lu = Lookups.get(Lookups.type == 'Observation',
Lookups.value == "Disposition",
Lookups.subvalue == "Sacrificed").lookup
cutter_recorder = Lookups.get(Lookups.type == 'Cutter Attribute',
Lookups.value == 'Recorder Name').lookup
specimen_dict = {
"length": length_lu,
"weight": weight_lu,
"sex": sex_lu,
"finclip": finclip_lu,
"otolith": otolith_lu
}
disposition_dict = {
"Descended": descended_lu,
"Released": released_lu,
"Sacrificed": sacrificed_lu
}
receptacle_type = Lookups.get(Lookups.type == 'Receptacle Type',
Lookups.value == 'Hook').lookup
results = ImportCutterStation.select()
for i, result in enumerate(results):
# if i == 1:
# break
logging.info(f"row={i+1} = {model_to_dict(result)}")
try:
set_op, set_created = Operations.get_or_create(
operation_number=result.set_id,
defaults={'operation_type_lu': set_op_type})
drop_op, drop_created = Operations.get_or_create(
parent_operation=set_op.operation,
operation_number=result.drop,
defaults={'operation_type_lu': drop_op_type})
angler_op, angler_created = Operations.get_or_create(
parent_operation=drop_op.operation,
operation_number=result.angler,
defaults={'operation_type_lu': angler_op_type})
# Catch / Hook - Insert
species = result.species
if species is not None:
species = species.strip()
cs_catch_content = CatchContentLu.get(
CatchContentLu.display_name == species).catch_content
catch, catch_created = Catch.get_or_create(
receptacle_seq=result.hook,
receptacle_type=receptacle_type,
operation=angler_op.operation,
defaults={
'operation_type': angler_op_type,
'cs_catch_content': cs_catch_content
})
catch_id = catch.catch
logging.info(
f"Catch created = {catch_created}: Hook {result.hook} > {species}"
)
if not catch_created:
Catch.update(cs_catch_content=cs_catch_content).where(
Catch.catch == catch_id).execute()
logging.info(
f"CS Catch Content updated: {cs_catch_content}")
# Parent Specimen - Insert
parent_specimen, parent_specimen_created = Specimen.get_or_create(
catch=catch_id)
# Specimen / Observations - Insert
for k, v in specimen_dict.items():
alpha_value = numeric_value = None
if k in ["length", "weight"]:
numeric_value = getattr(result, k)
else:
alpha_value = getattr(result, k)
# logging.info(f"{k} > {alpha_value} / {numeric_value} >>> {v}")
if alpha_value is not None or numeric_value is not None:
specimen, specimen_created = Specimen.get_or_create(
parent_specimen=parent_specimen.specimen,
catch=catch_id,
action_type=v,
alpha_value=alpha_value,
numeric_value=numeric_value)
logging.info(
f"Specimeen created = {specimen_created}: {k} > {alpha_value} / {numeric_value} > {specimen.specimen}"
)
# Disposition - Insert
alpha_value = result.tag_number
numeric_value = None
disposition = result.disposition
if disposition:
disposition = disposition.strip()
specimen, specimen_created = Specimen.get_or_create(
parent_specimen=parent_specimen.specimen,
catch=catch_id,
action_type=disposition_dict[disposition],
alpha_value=alpha_value,
numeric_value=numeric_value)
logging.info(
f"Specimeen created = {specimen_created}: Disposition = {result.disposition} > {alpha_value} / {numeric_value} > {specimen.specimen}"
)
# Cutter Recorder - Insert
# This is a bogus TODO below, as I never added a recorded_by directly into the SPECIMENS table
# TODO Todd Hay - Okay for 2017 data, but will need to be updated for 2018 data as I'll add a recorded by in the SPECIMEN table
if result.recorded_by is not None:
op_att, op_att_created = OperationAttributes.get_or_create(
operation=angler_op,
attribute_alpha=result.recorded_by,
attribute_type_lu=v)
logging.info(
f"Cutter recorded by created = {op_att_created}: {result.recorded_by} > "
f"{op_att.operation_attribute}")
# Note - Insert
if result.notes is not None:
note, note_created = Notes.get_or_create(
note=result.notes,
operation=angler_op,
app_name="HookMatrix",
hl_drop=result.drop,
hl_angler=result.angler,
defaults={'date_time': arrow.now(tz="US/Pacific")})
logging.info(
f"Note created = {note_created} > {result.notes}")
except Exception as ex:
errors.append(
f"row={i+1}, set_id={result.set_id} > ERROR: {ex}")
for err in errors:
logging.info(f"{err}")
def process_data(self):
errors = list()
site_op_type = Lookups.get(Lookups.type == 'Operation',
Lookups.value == 'Site').lookup
drop_op_type = Lookups.get(Lookups.type == 'Operation',
Lookups.value == 'Drop').lookup
site_type_lu_id = Lookups.get(Lookups.type == 'Site Type',
Lookups.value == 'Fixed').lookup
drop_type_lu_id = Lookups.get(Lookups.type == "Drop Type",
Lookups.value == 'Fixed').lookup
results = ImportHookLogger.select()
current_set_id = None
for i, result in enumerate(results):
# if i == 6:
# break
logging.info(f"row={i+1} = {model_to_dict(result)}")
try:
vessel_lu_id = Lookups.get(
Lookups.type == 'Vessel Number',
Lookups.description == result.vessel)
area = Sites.get(Sites.name == result.sites_name)
fpc_name = result.recorded_by.split(" ")
first_name = fpc_name[0]
last_name = fpc_name[1]
fpc_id = Personnel.get(Personnel.first_name == first_name,
Personnel.last_name == last_name)
include_in_survey = "True" if result.include_in_survey == "Y" else "False"
op_date = f"{result.date}T00:00:00"
# Create or get the Site OPERATIONS record
defaults = {
'vessel_lu': vessel_lu_id.lookup,
'day_of_cruise': result.day_of_survey,
'area': area.area_description,
'fpc': fpc_id.personnel,
'date': op_date,
'site': area.site,
'recorder': fpc_id.personnel,
'site_type_lu': site_type_lu_id,
'include_in_survey': include_in_survey,
'operation_type_lu': site_op_type,
'is_rca': 'False',
'is_mpa': 'False'
}
set_op, set_created = Operations.get_or_create(
operation_number=result.set_id, defaults=defaults)
# Create or get the OPERATION_ATTRIBUTES records to include:
# TODO - Angler Start Time - need to get this from the ImportHookLogger Drop Time value, otherwise
# it is not captured anywhere else I don't believe. Do this in HookMatrixImport instance
# Create or get the OPERATION_DETAILS record - Site Comments
# Get Tide Station + Distance to Station
tide_station_id = area.tide_station.tide_station
distance_to_tide_station_nm = self.get_distance_to_tide_station(
site_id=area.site, tide_station_id=tide_station_id)
details_defaults = {
'tide_station': tide_station_id,
'distance_to_tide_station_nm': distance_to_tide_station_nm,
'habitat_comments': result.habitat_notes,
'fish_meter_comments': result.fishfinder_notes,
'ocean_weather_comments': result.ocean_weather_notes,
'general_comments': result.general_notes
}
op_details, op_details_created = OperationDetails.get_or_create(
operation=set_op.operation, defaults=details_defaults)
# Create or get the EVENT record
event_type_lu_id = Lookups.get(
Lookups.type == "Event Type",
Lookups.value == result.drop_number).lookup
start_date_time = f"{result.date}T{result.drop_time}"
start_latitude = self.get_lat_lon(input=result.start_latitude)
start_longitude = self.get_lat_lon(
input=result.start_longitude)
if start_longitude is not None:
start_longitude = -start_longitude
end_latitude = self.get_lat_lon(input=result.end_latitude)
end_longitude = self.get_lat_lon(input=result.end_longitude)
if end_longitude is not None:
end_longitude = -end_longitude
event_defaults = {
'start_date_time': start_date_time,
'start_latitude': start_latitude,
'end_latitude': end_latitude,
'start_longitude': start_longitude,
'end_longitude': end_longitude,
'start_depth_ftm': result.start_depth_ftm,
'end_depth_ftm': result.end_depth_ftm,
'drop_type_lu': drop_type_lu_id,
'include_in_results': 'True'
}
logging.info(f"\t\tevent_defaults={event_defaults}")
event_item, event_created = Events.get_or_create(
operation=set_op.operation,
event_type_lu=event_type_lu_id,
defaults=event_defaults)
except Exception as ex:
errors.append(f"row={i+1}, set_id={result.set_id} > {ex}")
for err in errors:
logging.info(f"{err}")
def process_data(self):
errors = list()
set_op_type = Lookups.get(Lookups.type == 'Operation',
Lookups.value == 'Site').lookup
drop_op_type = Lookups.get(Lookups.type == 'Operation',
Lookups.value == 'Drop').lookup
angler_op_type = Lookups.get(Lookups.type == 'Operation',
Lookups.value == 'Angler').lookup
angler_name = Lookups.get(Lookups.type == 'Angler Attribute',
Lookups.value == 'Angler Name').lookup
angler_start = Lookups.get(Lookups.type == 'Angler Time',
Lookups.value == 'Start').lookup
angler_begin_fishing = Lookups.get(
Lookups.type == 'Angler Time',
Lookups.value == 'Begin Fishing').lookup
angler_first_bite = Lookups.get(Lookups.type == 'Angler Time',
Lookups.value == 'First Bite').lookup
angler_retrieval = Lookups.get(Lookups.type == 'Angler Time',
Lookups.value == 'Retrieval').lookup
angler_at_surface = Lookups.get(Lookups.type == 'Angler Time',
Lookups.value == 'At Surface').lookup
drop_sinker_weight = Lookups.get(
Lookups.type == 'Drop Attribute',
Lookups.value == 'Sinker Weight').lookup
drop_recorder = Lookups.get(Lookups.type == 'Drop Attribute',
Lookups.value == 'Recorder Name').lookup
angler_dict = {
"angler_name": angler_name,
"start": angler_start,
"begin_fishing": angler_begin_fishing,
"first_bite": angler_first_bite,
"retrieval": angler_retrieval,
"at_surface": angler_at_surface
}
drop_dict = {
"sinker_weight": drop_sinker_weight,
"recorded_by": drop_recorder
}
receptacle_type = Lookups.get(Lookups.type == 'Receptacle Type',
Lookups.value == 'Hook').lookup
results = ImportHookMatrix.select()
for i, result in enumerate(results):
logging.info(f"row={i+1} = {model_to_dict(result)}")
try:
set_op, set_created = Operations.get_or_create(
operation_number=result.set_id,
defaults={'operation_type_lu': set_op_type})
drop_op, drop_created = Operations.get_or_create(
parent_operation=set_op.operation,
operation_number=result.drop,
defaults={'operation_type_lu': drop_op_type})
angler_op, angler_created = Operations.get_or_create(
parent_operation=drop_op.operation,
operation_number=result.angler,
defaults={'operation_type_lu': angler_op_type})
# Angler Attributes - Insert
for k, v in angler_dict.items():
alpha_value = getattr(result, k)
if alpha_value is not None:
if k == "start":
alpha_value = alpha_value[:-4]
elif k in [
'begin_fishing', 'first_bite', 'retrieval',
'at_surface'
]:
alpha_value = alpha_value[:-7]
op_att, op_att_created = OperationAttributes.get_or_create(
operation=angler_op,
attribute_alpha=alpha_value,
attribute_type_lu=v)
logging.info(
f"Angler attribute created = {op_att_created}: {k} = {alpha_value} >>> "
f"{op_att.operation_attribute}")
else:
if k == "start":
# Retrieve the start time from the HookLoggerImport table for this Angler
hook_logger_record = ImportHookLogger.get(
ImportHookLogger.set_id == result.set_id,
ImportHookLogger.drop_number ==
f"Drop {result.drop}")
if hook_logger_record:
alpha_value = hook_logger_record.drop_time[:-4]
logging.info(
f"Angler start time: {alpha_value}")
op_att, op_att_created = OperationAttributes.get_or_create(
operation=angler_op,
attribute_alpha=alpha_value,
attribute_type_lu=v)
logging.info(
f"Angler attribute created = {op_att_created}: {k} = {alpha_value} >>> "
f"{op_att.operation_attribute}")
# Angler Gear Performance - Insert
if result.gear_performance is not None:
for perf in result.gear_performance.split(","):
try:
perf = perf.strip().replace("(",
"").replace(")", "")
gp_id = Lookups.get(
Lookups.type == "Angler Gear Performance",
Lookups.value == perf).lookup
op_att, op_att_created = OperationAttributes.get_or_create(
operation=angler_op, attribute_type_lu=gp_id)
logging.info(
f"Angler attribute created = {op_att_created}: {perf} >>> {op_att.operation_attribute}"
)
except Exception as ex:
logging.info(
f"Error finding the gear performance lookup: {ex}"
)
# Drop Attributes - Insert
for k, v in drop_dict.items():
alpha_value = numeric_value = None
if k == "sinker_weight":
numeric_value = getattr(result, k)
elif k == "recorded_by":
alpha_value = getattr(result, k)
if alpha_value is not None or numeric_value is not None:
op_att, op_att_created = OperationAttributes.get_or_create(
operation=drop_op,
attribute_alpha=alpha_value,
attribute_numeric=numeric_value,
attribute_type_lu=v)
logging.info(
f"Drop attribute created = {op_att_created}: {k} = {alpha_value} / "
f"{numeric_value} >>> "
f"{op_att.operation_attribute}")
# Hooks - Insert
for j in range(1, 6):
species = getattr(result, f"hook{j}")
if species is not None:
species = species.strip()
hm_catch_content = CatchContentLu.get(
CatchContentLu.display_name ==
species).catch_content
catch, catch_created = Catch.get_or_create(
receptacle_seq=j,
receptacle_type=receptacle_type,
operation=angler_op.operation,
defaults={
'operation_type': angler_op_type,
'hm_catch_content': hm_catch_content
})
logging.info(
f"Catch created = {catch_created}: Hook {j} > {species}"
)
# Note - Insert
if result.notes is not None:
note, note_created = Notes.get_or_create(
note=result.notes,
operation=angler_op,
app_name="HookMatrix",
hl_drop=result.drop,
hl_angler=result.angler,
defaults={'date_time': arrow.now(tz="US/Pacific")})
logging.info(
f"Note created = {note_created} > {result.notes}")
except Exception as ex:
errors.append(f"row={i+1}, set_id={result.set_id} > {ex}")
for err in errors:
logging.info(f"{err}")