def populate_model(self): """ Method to populate the model by pull all active parsing methods from the ParsingRules table in the database :return: """ self.clear() Measurements = Lookups.alias() Organizations = Lookups.alias() rules = ParsingRules.select(ParsingRules, Equipment, Measurements, Organizations)\ .join(Measurements, JOIN.LEFT_OUTER, on=(Measurements.lookup == ParsingRules.measurement_lu).alias("measurement"))\ .switch(ParsingRules)\ .join(Equipment)\ .join(Organizations, on=(Equipment.organization == Organizations.lookup).alias("org"))\ .order_by(ParsingRules.priority.asc()) for rule in rules: item = dict() item["parsing_rules_id"] = rule.parsing_rules item["equipment"] = "{0} {1}".format(rule.equipment.org.value, rule.equipment.model) item["line_starting"] = rule.line_starting item["is_line_starting_substr"] = rule.is_line_starting_substr item["field_position"] = rule.field_position item["line_ending"] = rule.line_ending item["measurement"] = "{0}, {1}".format(rule.measurement.value, rule.measurement.subvalue) item["priority"] = rule.priority self.appendItem(item)
def process_data(self): errors = list() finclip_lu = Lookups.get(Lookups.type == 'Observation', Lookups.value == "Finclip ID").lookup results = SpecimenImport.select() for i, result in enumerate(results): # if i == 1: # break logging.info(f"row={i+1} = {model_to_dict(result)}") try: # Find the sibling specimen where finclip ID = result.finclip sibling_specimen, sibling_specimen_created = Specimen.get_or_create( alpha_value=result.finclip, action_type=finclip_lu) # Determine the specimen lookup value if "Age ID" in result.specimen_type: subvalue = result.specimen_type.strip('Age ID').strip() specimen_lu = Lookups.get( Lookups.type == 'Observation', Lookups.value == 'Age ID', Lookups.subvalue == subvalue).lookup else: specimen_lu = Lookups.get( Lookups.type == 'Observation', Lookups.value == result.specimen_type).lookup # Determine the species sampling plan ssp_results = SpeciesSamplingPlanLu.select(SpeciesSamplingPlanLu, ProtocolLu) \ .join(ProtocolLu) \ .where(SpeciesSamplingPlanLu.plan_name==result.plan_name, ProtocolLu.action_type==specimen_lu) for ssp_result in ssp_results: ssp_lu = ssp_result.species_sampling_plan specimen, specimen_created = Specimen.get_or_create( parent_specimen=sibling_specimen.parent_specimen, species_sampling_plan=ssp_lu, action_type=specimen_lu, alpha_value=result.specimen_number, defaults={'catch': sibling_specimen.catch}) logging.info( f"Specimen created = {specimen_created}: {result.specimen_type} = {result.specimen_number}" f" > {specimen.specimen}") except Exception as ex: errors.append(f"row={i+1}, set_id={result.set_id} > {ex}") for err in errors: logging.info(f"{err}")
def populate_model(self): """ Method to retrieve all of the equipment from the database equipment table and use them to populate the lvEquipment Listiew on the AddComportDialog.qml called from the SensorDataFeeds.qml screen :return: """ self.clear() Make = Lookups.alias() Category = Lookups.alias() records = Equipment.select(Equipment, Make, Category)\ .join(Make, on=(Make.lookup == Equipment.organization).alias('make'))\ .switch(Equipment)\ .join(Category, on=(Category.lookup == Equipment.equipment_category).alias('category'))\ .where(Equipment.is_active == "True")\ .order_by(Make.value.asc(), Equipment.name.asc(), Category.value.asc()) for record in records: item = dict() item["equipment_id"] = record.equipment item["equipment"] = " ".join([record.make.value, record.model]) item["equipment_type"] = record.category.value self.appendItem(item)
def populate_model(self): """" Method to initially populate the model on startup """ self.clear() results = Lookups.select().where(Lookups.is_active == "True", Lookups.type == "Vessel Number") \ .order_by(Lookups.value.asc()) for result in results: item = dict() item["id"] = result.lookup item["text"] = result.description self.appendItem(item)
def populate_model(self): """ Method to retrieve all of measurements + units of measurement from the database lookups table and use them to populate the cbMeasurement combobox on the SensorDataFeeds.qml page :return: """ self.clear() records = Lookups.select(Lookups.value, Lookups.subvalue, Lookups.lookup)\ .where(Lookups.type == "Measurement", Lookups.is_active == "True")\ .order_by(Lookups.value.asc()) \ .group_by(Lookups.value, Lookups.subvalue) for record in records: item = dict() item["lookup_id"] = record.lookup item["text"] = record.value + ", " + record.subvalue \ if record.subvalue else record.value self.appendItem(item)
def populate_model(self, set_id): """ Method to populate the model :return: """ if set_id == "": logging.info(f"Set ID is empty: {set_id}") try: self.clear() results = CutterStationView.select()\ .where(CutterStationView.set_id==set_id)\ .order_by(CutterStationView.drop, CutterStationView.angler, CutterStationView.hook) for result in results: self.appendItem({ 'dropNumber': result.drop, 'angler': result.angler, 'hook': result.hook, 'species': result.species, 'length': result.length, 'weight': result.weight, 'sex': result.sex, 'finclip': result.finclip, 'otolith': result.otolith, 'disposition': result.disposition, 'tagNumber': result.tag_number, 'recordedBy': result.recorded_by }) cutter_oa = Lookups.get(type='Cutter Attribute', value='Recorder Name').lookup site_op_id = Operations.get(operation_number=set_id).operation cutter_recorder_results = OperationAttributes.select()\ .where(OperationAttributes.attribute_type_lu==cutter_oa, OperationAttributes.operation==site_op_id) if cutter_recorder_results.count() >= 1: for result in cutter_recorder_results: self.cutterRecorderChanged.emit(result.attribute_alpha) break else: self.cutterRecorderChanged.emit("") except Exception as ex: logging.error(f"Error populating the CutterStation model: {ex}")
def add_new_measurement(self, measurement, unit_of_measurement): """ Method called by the NewMeasurementDialog.qml via SensorDataFeeds.qml when a user wants to add a new measurement type. This consists of the measurement name as well as the unit of measurement :param measurement: str - should be in the form: Latitude - Vessel, Time - UTC, etc. :param unit_of_measurement: str - abbreviated unit of measurements, e.g. m, kg, km, etc. :return: """ if not isinstance(measurement, str) or not isinstance(unit_of_measurement, str) or \ measurement == "" or unit_of_measurement == "": logging.error( "Invalid measurement of unit of measurement provided: {0}, {1}" .format(measurement, unit_of_measurement)) return # Insert into the Lookups table in the database try: # Lookups.insert(type="Measurement", value=measurement, subvalue=unit_of_measurement, # is_active="True").execute() # lookup_id = Lookups.select().order_by(Lookups.lookup.desc()).get().lookup lookup, created = Lookups.get_or_create( type="Measurement", value=measurement, subvalue=unit_of_measurement, is_active="True") lookup_id = lookup.lookup except Exception as ex: logging.error( "failed to insert new measurement type: {0}".format(ex)) # Add to the various models - measurements, uom, and measurementsUofm self._measurements_model.add_row(lookup_id=lookup_id, value=measurement) self._units_of_measurement_model.add_row(lookup_id=lookup_id, subvalue=unit_of_measurement) self._measurements_uom_model.add_row(lookup_id=lookup_id, value=measurement, subvalue=unit_of_measurement) # Emit a signal - this is used to auto-select this newly added measurement in SensorDataFeeds.qml self.measurementsUomModelChanged.emit("{0}, {1}".format( measurement, unit_of_measurement))
def populate_model(self): """ Method to retrieve all of the measurements form the database lookups table and use them to populate the cbMeasurement combobox on the NewMeasurementDialog.qml called from the SensorDataFeeds.qml screen :return: """ self.clear() records = Lookups.select(Lookups.value, Lookups.lookup) \ .where(Lookups.type == "Measurement", Lookups.is_active == "True")\ .order_by(Lookups.value.asc()) \ .group_by(Lookups.value) for record in records: item = dict() item["lookup_id"] = record.lookup item["text"] = record.value self.appendItem(item)
def populate_model(self): """ Method to retrieve all of the units of measurement from the database lookups table and use them to populate the cbUnitOfMeasurement combobox on the NewMeasurementDialog.qml that is called from the SensorDataFeeds.qml page :return: """ self.clear() # TODO - Ensure that the units of measurement returned are unique records = Lookups.select(Lookups.subvalue, Lookups.lookup) \ .where(Lookups.type == "Measurement", Lookups.is_active == "True") \ .order_by(Lookups.subvalue.asc()) \ .group_by(Lookups.subvalue) for record in records: item = dict() item["lookup_id"] = record.lookup item["text"] = record.subvalue self.appendItem(item)
def populate_model(self): """ Method to retrieve all of the sensor configurations from the database deployed_equipment table and use them to populate the tvSensorConfiguration tableview on the SensorDataFeeds.qml screen :return: """ self.clear() Organization = Lookups.alias() records = DeployedEquipment.select(DeployedEquipment, Equipment, Organization)\ .join(Equipment)\ .join(Organization, on=(Organization.lookup == Equipment.organization).alias('org'))\ .where(DeployedEquipment.deactivation_date.is_null())\ .order_by(DeployedEquipment.com_port.asc()) for record in records: item = dict() item[ "equipment"] = record.equipment.org.value + " " + record.equipment.model # item["com_port"] = "COM{0}".format(record.com_port) item["com_port"] = record.com_port item["moxa_port"] = record.moxa_port item["data_status"] = "red" item["start_stop_status"] = "stopped" item["delete_row"] = "" item["baud_rate"] = record.baud_rate if record.baud_rate else 9600 item["data_bits"] = record.data_bits if record.data_bits else 8 item["parity"] = record.parity if record.parity else "None" item["stop_bits"] = record.stop_bits if record.stop_bits else 1 item[ "flow_control"] = record.flow_control if record.flow_control else "None" item["deployed_equipment_id"] = record.deployed_equipment self.appendItem(item) self._app.serial_port_manager.add_thread(com_port_dict=item) self.setItems( sorted(self.items, key=lambda x: int(x["com_port"].strip("COM"))))
def update_row(self, index, value): """ Method to update one of the settings rows :param index: :param value: :return: """ if not isinstance(index, int) or index < 0 or index >= self.count: logging.error("Error updating a settings row: {0}".format(value)) return if isinstance(value, QJSValue): value = value.toVariant() # Convert all boolean objects to strings value = str(value) if isinstance(value, bool) else value # Set to empty string as need and then update the model value = value if value else "" # Update the model - two fields to update self.setProperty(index=index, property="value", value=value) self.setProperty(index=index, property="new_value", value=value) # Database Storage Conversions parameter = self.get(index=index)["parameter"] # Set update_db flag update_db = True # Database Storage Conversions - Dates/Times if parameter in [ "Leg 1 Start", "Leg 1 End", "Leg 2 Start", "Leg 2 End" ]: if "end" in parameter.lower(): value += " 23:59:59" value = self._dc.time_to_iso_format(value) # Names to Foreign Keys in Personnel Table elif any(substring in parameter for substring in [ "Leg 1 FPC", "Leg 2 FPC", "Scientist 1", "Scientist 2", "Scientist 3", "Captain", "Second Captain", "Deckhand 1", "Deckhand 2", "Deckhand 3" ]): try: if value: logging.info( f"Personnel found: {parameter} >>> value: {value}") last_name, first_name = value.split(",") person = Personnel.get(Personnel.first_name == first_name, Personnel.last_name == last_name) value = person.personnel else: update_db = False except Exception as ex: value = None update_db = False logging.error( "Error getting the proper personnel name: {0}".format(ex)) # Names to Foreign Keys in Lookups Table elif parameter in ["Vessel"]: try: if value: vessel = Lookups.get(Lookups.description == value) value = vessel.lookup else: update_db = False except Exception as ex: value = None update_db = False logging.error( "Error getting the proper vessel name: {0}".format(ex)) # Update the database try: if update_db: settings_id = self.get(index=index)["settings"] SettingsTable.update(value=value).where( SettingsTable.settings == settings_id).execute() except Exception as ex: logging.error( "Exception updating the settings table: {0}".format(ex))
def populate_model(self): """" Method to initially populate the model on startup """ self.clear() results = SettingsTable.select().where(SettingsTable.is_active == "True")\ .order_by(SettingsTable.parameter.asc()) for result in results: # item = dict() # item["settings_id"] = result.settings # item["parameter"] = result.parameter # item["value"] = result.value item = model_to_dict(result) item["delegate_type"] = "TextField" if item["parameter"] in [ "Leg 1 Start", "Leg 1 End", "Leg 2 Start", "Leg 2 End" ]: # Convert to mm/dd/yyyy format item["value"] = self._dc.iso_to_common_date_format( item["value"]) elif any(substring in item["parameter"] for substring in [ "Leg 1 FPC", "Leg 2 FPC", "Scientist 1", "Scientist 2", "Scientist 3", "Captain", "Second Captain", "Cook", "Deckhand 1", "Deckhand 2", "Deckhand 3" ]): item["delegate_type"] = "ComboBox" if any(substring in item["parameter"] for substring in ["FPC", "Scientist 1", "Scientist 2", "Scientist 3"]): item["model_type"] = "Scientists" is_science_team = "True" else: item["model_type"] = "Crew" is_science_team = "False" try: person = Personnel.get( Personnel.personnel == item["value"], Personnel.is_science_team == is_science_team) item["value"] = person.last_name + ", " + person.first_name except Exception as ex: item["value"] = "" elif item["parameter"] in ["Vessel"]: item["delegate_type"] = "ComboBox" item["model_type"] = "Vessels" try: lookup = Lookups.get(Lookups.lookup == item["value"]) item["value"] = lookup.description except Exception as ex: item["value"] = "" elif item["parameter"] in ["Backup Folder"]: item["delegate_type"] = "FolderBrowser" elif item["parameter"] == "Depth Output Serial Port": item["model_type"] = "ComPorts" item["delegate_type"] = "DialogListView" elif item["parameter"] == "Depth Output Serial Port Baud Rate": item["model_type"] = "BaudRates" item["delegate_type"] = "DialogListView" elif item["parameter"] == "Depth Output Serial Port Status": item["delegate_type"] = "Switch" item["new_value"] = item["value"] self.appendItem(item)
def save_best_species_to_db(self, set_id, adh, species): """ Method to save the best species down to the database :param set_id: str - the set id of the operation for saving to the database :param adh: :param species: :return: """ if len(adh) != 3: logging.error(f"ADH is not the right length for updating the best species: {adh}") return logging.info(f"save_best_species_to_db for set_id{set_id}, adh={adh}, species={species}") try: # Parse out the drop, angler, hook drop = adh[1] angler = adh[0] hook = adh[2] # Determine the best species catch content ID, it might be none if the best species is blank best_catch_content = None if species: best_catch_content = CatchContentLu.get(CatchContentLu.display_name == species).catch_content # Find the catch_id that matches this set_id and adh set_op = Operations.get(Operations.operation_number==set_id).operation drop_op = Operations.get(Operations.parent_operation==set_op, Operations.operation_number==drop).operation angler_op = Operations.get(Operations.parent_operation==drop_op, Operations.operation_number==angler).operation hook_type = Lookups.get(Lookups.type=="Receptacle Type", Lookups.value=="Hook").lookup operation_type = Lookups.get(Lookups.type=="Operation", Lookups.value=="Angler").lookup catch_record, created = Catch.get_or_create(operation=angler_op, receptacle_seq=hook, receptacle_type=hook_type, operation_type=operation_type) logging.info(f"\t\tupdating catch best species with catch id={catch_record.catch} " f"to best species content = {best_catch_content}") # NOT SURE WHY THE BELOW DOES NOT WORK, BUT IT DOESN'T # Sets = Operations.alias() # Drops = Operations.alias() # Anglers = Operations.alias() # results = Sets.select(Catch.catch) \ # .join(Drops, JOIN.LEFT_OUTER, on=(Drops.parent_operation == Sets.operation).alias('drop')) \ # .join(Anglers, JOIN.LEFT_OUTER, on=(Anglers.parent_operation == Drops.operation).alias('angler')) \ # .join(Catch, JOIN.LEFT_OUTER, on=(Catch.operation == Anglers.operation).alias('catch')) \ # .where(Sets.operation_number==set_id, # Drops.operation_number==drop, # Anglers.operation_number==angler, # Catch.receptacle_seq==hook) # logging.info(f"the query is: {results}") # # if results.count() == 1: # # logging.info(f"length is {results.count()}") # THIS BREAKS, AS DOES TRYING TO ITERATE ON results # logging.info(f"result = {results.get().catch}") # Update the catch table with the new best species catch content ID Catch.update(best_catch_content=best_catch_content).where(Catch.catch==catch_record.catch).execute() except Exception as ex: logging.error(f"Error updating the best species: {ex}")
def process_data(self): errors = list() set_op_type = Lookups.get(Lookups.type == 'Operation', Lookups.value == 'Site').lookup drop_op_type = Lookups.get(Lookups.type == 'Operation', Lookups.value == 'Drop').lookup angler_op_type = Lookups.get(Lookups.type == 'Operation', Lookups.value == 'Angler').lookup length_lu = Lookups.get(Lookups.type == 'Observation', Lookups.value == "Length").lookup weight_lu = Lookups.get(Lookups.type == 'Observation', Lookups.value == "Weight").lookup sex_lu = Lookups.get(Lookups.type == 'Observation', Lookups.value == "Sex").lookup finclip_lu = Lookups.get(Lookups.type == 'Observation', Lookups.value == "Finclip ID").lookup otolith_lu = Lookups.get(Lookups.type == 'Observation', Lookups.value == "Age ID", Lookups.subvalue == "Otolith").lookup descended_lu = Lookups.get(Lookups.type == 'Observation', Lookups.value == "Disposition", Lookups.subvalue == "Descended").lookup released_lu = Lookups.get(Lookups.type == 'Observation', Lookups.value == "Disposition", Lookups.subvalue == "Released").lookup sacrificed_lu = Lookups.get(Lookups.type == 'Observation', Lookups.value == "Disposition", Lookups.subvalue == "Sacrificed").lookup cutter_recorder = Lookups.get(Lookups.type == 'Cutter Attribute', Lookups.value == 'Recorder Name').lookup specimen_dict = { "length": length_lu, "weight": weight_lu, "sex": sex_lu, "finclip": finclip_lu, "otolith": otolith_lu } disposition_dict = { "Descended": descended_lu, "Released": released_lu, "Sacrificed": sacrificed_lu } receptacle_type = Lookups.get(Lookups.type == 'Receptacle Type', Lookups.value == 'Hook').lookup results = ImportCutterStation.select() for i, result in enumerate(results): # if i == 1: # break logging.info(f"row={i+1} = {model_to_dict(result)}") try: set_op, set_created = Operations.get_or_create( operation_number=result.set_id, defaults={'operation_type_lu': set_op_type}) drop_op, drop_created = Operations.get_or_create( parent_operation=set_op.operation, operation_number=result.drop, defaults={'operation_type_lu': drop_op_type}) angler_op, angler_created = Operations.get_or_create( parent_operation=drop_op.operation, operation_number=result.angler, defaults={'operation_type_lu': angler_op_type}) # Catch / Hook - Insert species = result.species if species is not None: species = species.strip() cs_catch_content = CatchContentLu.get( CatchContentLu.display_name == species).catch_content catch, catch_created = Catch.get_or_create( receptacle_seq=result.hook, receptacle_type=receptacle_type, operation=angler_op.operation, defaults={ 'operation_type': angler_op_type, 'cs_catch_content': cs_catch_content }) catch_id = catch.catch logging.info( f"Catch created = {catch_created}: Hook {result.hook} > {species}" ) if not catch_created: Catch.update(cs_catch_content=cs_catch_content).where( Catch.catch == catch_id).execute() logging.info( f"CS Catch Content updated: {cs_catch_content}") # Parent Specimen - Insert parent_specimen, parent_specimen_created = Specimen.get_or_create( catch=catch_id) # Specimen / Observations - Insert for k, v in specimen_dict.items(): alpha_value = numeric_value = None if k in ["length", "weight"]: numeric_value = getattr(result, k) else: alpha_value = getattr(result, k) # logging.info(f"{k} > {alpha_value} / {numeric_value} >>> {v}") if alpha_value is not None or numeric_value is not None: specimen, specimen_created = Specimen.get_or_create( parent_specimen=parent_specimen.specimen, catch=catch_id, action_type=v, alpha_value=alpha_value, numeric_value=numeric_value) logging.info( f"Specimeen created = {specimen_created}: {k} > {alpha_value} / {numeric_value} > {specimen.specimen}" ) # Disposition - Insert alpha_value = result.tag_number numeric_value = None disposition = result.disposition if disposition: disposition = disposition.strip() specimen, specimen_created = Specimen.get_or_create( parent_specimen=parent_specimen.specimen, catch=catch_id, action_type=disposition_dict[disposition], alpha_value=alpha_value, numeric_value=numeric_value) logging.info( f"Specimeen created = {specimen_created}: Disposition = {result.disposition} > {alpha_value} / {numeric_value} > {specimen.specimen}" ) # Cutter Recorder - Insert # This is a bogus TODO below, as I never added a recorded_by directly into the SPECIMENS table # TODO Todd Hay - Okay for 2017 data, but will need to be updated for 2018 data as I'll add a recorded by in the SPECIMEN table if result.recorded_by is not None: op_att, op_att_created = OperationAttributes.get_or_create( operation=angler_op, attribute_alpha=result.recorded_by, attribute_type_lu=v) logging.info( f"Cutter recorded by created = {op_att_created}: {result.recorded_by} > " f"{op_att.operation_attribute}") # Note - Insert if result.notes is not None: note, note_created = Notes.get_or_create( note=result.notes, operation=angler_op, app_name="HookMatrix", hl_drop=result.drop, hl_angler=result.angler, defaults={'date_time': arrow.now(tz="US/Pacific")}) logging.info( f"Note created = {note_created} > {result.notes}") except Exception as ex: errors.append( f"row={i+1}, set_id={result.set_id} > ERROR: {ex}") for err in errors: logging.info(f"{err}")
def process_data(self): errors = list() site_op_type = Lookups.get(Lookups.type == 'Operation', Lookups.value == 'Site').lookup drop_op_type = Lookups.get(Lookups.type == 'Operation', Lookups.value == 'Drop').lookup site_type_lu_id = Lookups.get(Lookups.type == 'Site Type', Lookups.value == 'Fixed').lookup drop_type_lu_id = Lookups.get(Lookups.type == "Drop Type", Lookups.value == 'Fixed').lookup results = ImportHookLogger.select() current_set_id = None for i, result in enumerate(results): # if i == 6: # break logging.info(f"row={i+1} = {model_to_dict(result)}") try: vessel_lu_id = Lookups.get( Lookups.type == 'Vessel Number', Lookups.description == result.vessel) area = Sites.get(Sites.name == result.sites_name) fpc_name = result.recorded_by.split(" ") first_name = fpc_name[0] last_name = fpc_name[1] fpc_id = Personnel.get(Personnel.first_name == first_name, Personnel.last_name == last_name) include_in_survey = "True" if result.include_in_survey == "Y" else "False" op_date = f"{result.date}T00:00:00" # Create or get the Site OPERATIONS record defaults = { 'vessel_lu': vessel_lu_id.lookup, 'day_of_cruise': result.day_of_survey, 'area': area.area_description, 'fpc': fpc_id.personnel, 'date': op_date, 'site': area.site, 'recorder': fpc_id.personnel, 'site_type_lu': site_type_lu_id, 'include_in_survey': include_in_survey, 'operation_type_lu': site_op_type, 'is_rca': 'False', 'is_mpa': 'False' } set_op, set_created = Operations.get_or_create( operation_number=result.set_id, defaults=defaults) # Create or get the OPERATION_ATTRIBUTES records to include: # TODO - Angler Start Time - need to get this from the ImportHookLogger Drop Time value, otherwise # it is not captured anywhere else I don't believe. Do this in HookMatrixImport instance # Create or get the OPERATION_DETAILS record - Site Comments # Get Tide Station + Distance to Station tide_station_id = area.tide_station.tide_station distance_to_tide_station_nm = self.get_distance_to_tide_station( site_id=area.site, tide_station_id=tide_station_id) details_defaults = { 'tide_station': tide_station_id, 'distance_to_tide_station_nm': distance_to_tide_station_nm, 'habitat_comments': result.habitat_notes, 'fish_meter_comments': result.fishfinder_notes, 'ocean_weather_comments': result.ocean_weather_notes, 'general_comments': result.general_notes } op_details, op_details_created = OperationDetails.get_or_create( operation=set_op.operation, defaults=details_defaults) # Create or get the EVENT record event_type_lu_id = Lookups.get( Lookups.type == "Event Type", Lookups.value == result.drop_number).lookup start_date_time = f"{result.date}T{result.drop_time}" start_latitude = self.get_lat_lon(input=result.start_latitude) start_longitude = self.get_lat_lon( input=result.start_longitude) if start_longitude is not None: start_longitude = -start_longitude end_latitude = self.get_lat_lon(input=result.end_latitude) end_longitude = self.get_lat_lon(input=result.end_longitude) if end_longitude is not None: end_longitude = -end_longitude event_defaults = { 'start_date_time': start_date_time, 'start_latitude': start_latitude, 'end_latitude': end_latitude, 'start_longitude': start_longitude, 'end_longitude': end_longitude, 'start_depth_ftm': result.start_depth_ftm, 'end_depth_ftm': result.end_depth_ftm, 'drop_type_lu': drop_type_lu_id, 'include_in_results': 'True' } logging.info(f"\t\tevent_defaults={event_defaults}") event_item, event_created = Events.get_or_create( operation=set_op.operation, event_type_lu=event_type_lu_id, defaults=event_defaults) except Exception as ex: errors.append(f"row={i+1}, set_id={result.set_id} > {ex}") for err in errors: logging.info(f"{err}")
def process_data(self): errors = list() set_op_type = Lookups.get(Lookups.type == 'Operation', Lookups.value == 'Site').lookup drop_op_type = Lookups.get(Lookups.type == 'Operation', Lookups.value == 'Drop').lookup angler_op_type = Lookups.get(Lookups.type == 'Operation', Lookups.value == 'Angler').lookup angler_name = Lookups.get(Lookups.type == 'Angler Attribute', Lookups.value == 'Angler Name').lookup angler_start = Lookups.get(Lookups.type == 'Angler Time', Lookups.value == 'Start').lookup angler_begin_fishing = Lookups.get( Lookups.type == 'Angler Time', Lookups.value == 'Begin Fishing').lookup angler_first_bite = Lookups.get(Lookups.type == 'Angler Time', Lookups.value == 'First Bite').lookup angler_retrieval = Lookups.get(Lookups.type == 'Angler Time', Lookups.value == 'Retrieval').lookup angler_at_surface = Lookups.get(Lookups.type == 'Angler Time', Lookups.value == 'At Surface').lookup drop_sinker_weight = Lookups.get( Lookups.type == 'Drop Attribute', Lookups.value == 'Sinker Weight').lookup drop_recorder = Lookups.get(Lookups.type == 'Drop Attribute', Lookups.value == 'Recorder Name').lookup angler_dict = { "angler_name": angler_name, "start": angler_start, "begin_fishing": angler_begin_fishing, "first_bite": angler_first_bite, "retrieval": angler_retrieval, "at_surface": angler_at_surface } drop_dict = { "sinker_weight": drop_sinker_weight, "recorded_by": drop_recorder } receptacle_type = Lookups.get(Lookups.type == 'Receptacle Type', Lookups.value == 'Hook').lookup results = ImportHookMatrix.select() for i, result in enumerate(results): logging.info(f"row={i+1} = {model_to_dict(result)}") try: set_op, set_created = Operations.get_or_create( operation_number=result.set_id, defaults={'operation_type_lu': set_op_type}) drop_op, drop_created = Operations.get_or_create( parent_operation=set_op.operation, operation_number=result.drop, defaults={'operation_type_lu': drop_op_type}) angler_op, angler_created = Operations.get_or_create( parent_operation=drop_op.operation, operation_number=result.angler, defaults={'operation_type_lu': angler_op_type}) # Angler Attributes - Insert for k, v in angler_dict.items(): alpha_value = getattr(result, k) if alpha_value is not None: if k == "start": alpha_value = alpha_value[:-4] elif k in [ 'begin_fishing', 'first_bite', 'retrieval', 'at_surface' ]: alpha_value = alpha_value[:-7] op_att, op_att_created = OperationAttributes.get_or_create( operation=angler_op, attribute_alpha=alpha_value, attribute_type_lu=v) logging.info( f"Angler attribute created = {op_att_created}: {k} = {alpha_value} >>> " f"{op_att.operation_attribute}") else: if k == "start": # Retrieve the start time from the HookLoggerImport table for this Angler hook_logger_record = ImportHookLogger.get( ImportHookLogger.set_id == result.set_id, ImportHookLogger.drop_number == f"Drop {result.drop}") if hook_logger_record: alpha_value = hook_logger_record.drop_time[:-4] logging.info( f"Angler start time: {alpha_value}") op_att, op_att_created = OperationAttributes.get_or_create( operation=angler_op, attribute_alpha=alpha_value, attribute_type_lu=v) logging.info( f"Angler attribute created = {op_att_created}: {k} = {alpha_value} >>> " f"{op_att.operation_attribute}") # Angler Gear Performance - Insert if result.gear_performance is not None: for perf in result.gear_performance.split(","): try: perf = perf.strip().replace("(", "").replace(")", "") gp_id = Lookups.get( Lookups.type == "Angler Gear Performance", Lookups.value == perf).lookup op_att, op_att_created = OperationAttributes.get_or_create( operation=angler_op, attribute_type_lu=gp_id) logging.info( f"Angler attribute created = {op_att_created}: {perf} >>> {op_att.operation_attribute}" ) except Exception as ex: logging.info( f"Error finding the gear performance lookup: {ex}" ) # Drop Attributes - Insert for k, v in drop_dict.items(): alpha_value = numeric_value = None if k == "sinker_weight": numeric_value = getattr(result, k) elif k == "recorded_by": alpha_value = getattr(result, k) if alpha_value is not None or numeric_value is not None: op_att, op_att_created = OperationAttributes.get_or_create( operation=drop_op, attribute_alpha=alpha_value, attribute_numeric=numeric_value, attribute_type_lu=v) logging.info( f"Drop attribute created = {op_att_created}: {k} = {alpha_value} / " f"{numeric_value} >>> " f"{op_att.operation_attribute}") # Hooks - Insert for j in range(1, 6): species = getattr(result, f"hook{j}") if species is not None: species = species.strip() hm_catch_content = CatchContentLu.get( CatchContentLu.display_name == species).catch_content catch, catch_created = Catch.get_or_create( receptacle_seq=j, receptacle_type=receptacle_type, operation=angler_op.operation, defaults={ 'operation_type': angler_op_type, 'hm_catch_content': hm_catch_content }) logging.info( f"Catch created = {catch_created}: Hook {j} > {species}" ) # Note - Insert if result.notes is not None: note, note_created = Notes.get_or_create( note=result.notes, operation=angler_op, app_name="HookMatrix", hl_drop=result.drop, hl_angler=result.angler, defaults={'date_time': arrow.now(tz="US/Pacific")}) logging.info( f"Note created = {note_created} > {result.notes}") except Exception as ex: errors.append(f"row={i+1}, set_id={result.set_id} > {ex}") for err in errors: logging.info(f"{err}")