def row_parser(self, row): super().row_parser(row) row_datetime = utils.get_row_date(row) row_date = row_datetime.date() if utils.nan_to_none(row.get(self.box_key)): self.row_entered += utils.enter_indvd(self.anix_indv.pk, self.cleaned_data, row_date, row[self.box_key], self.box_anidc_id.pk, None) if utils.nan_to_none(row.get(self.location_key)): self.row_entered += utils.enter_indvd(self.anix_indv.pk, self.cleaned_data, row_date, row[self.location_key], self.boxl_anidc_id.pk, None) if utils.nan_to_none(row.get(self.indt_key)) and utils.nan_to_none( row.get(self.indt_amt_key)): indvtc_id = models.IndTreatCode.objects.filter( name__icontains=row[self.indt_key]).get() unit_id = models.UnitCode.objects.filter( name__icontains="gram").order_by(Length('name').asc()).first() self.row_entered += utils.enter_indvt(self.anix_indv.pk, self.cleaned_data, row_datetime, row[self.indt_amt_key], indvtc_id.pk, unit_id=unit_id)
def row_parser(self, row): # need to: find the pair's group, link it to it's pairing, create a tray, and add the count. cleaned_data = self.cleaned_data row_date = utils.get_row_date(row) pair_list = utils.get_pair(row[self.cross_key], row["stok_id"], row[self.year_key], prog_grp=utils.nan_to_none( row.get(self.prog_key)), fail_on_not_found=True) if len(pair_list) == 1: pair_id = pair_list[0] else: raise Exception("Too many pairs found for row \n{}".format(row)) anix_id = models.AniDetailXref.objects.filter( pair_id=pair_id, grp_id__isnull=False).select_related('grp_id').first() grp_id = anix_id.grp_id self.row_entered += utils.enter_anix(cleaned_data, grp_pk=grp_id.pk, return_sucess=True) tray_id = utils.create_tray(row["trof_id"], row[self.tray_key], row_date, cleaned_data) contx, contx_entered = utils.enter_contx(tray_id, cleaned_data, True, grp_pk=grp_id.pk, return_contx=True) self.row_entered += contx_entered if utils.nan_to_none(row.get(self.fecu_key)): cnt, cnt_entered = utils.enter_cnt(cleaned_data, row[self.fecu_key], contx_pk=contx.pk, cnt_code="Photo Count") self.row_entered += cnt_entered self.row_entered += utils.enter_bulk_grpd(anix_id.pk, cleaned_data, row_date, comments=row.get( self.comment_key))
def data_preper(self): cleaned_data = self.cleaned_data self.prog_grp_anidc_id = models.AnimalDetCode.objects.filter(name="Program Group").get() self.sex_anidc_id = models.AnimalDetCode.objects.filter(name="Gender").get() self.len_anidc_id = models.AnimalDetCode.objects.filter(name="Length").get() self.weight_anidc_id = models.AnimalDetCode.objects.filter(name="Weight").get() self.vial_anidc_id = models.AnimalDetCode.objects.filter(name="Vial").get() self.ani_health_anidc_id = models.AnimalDetCode.objects.filter(name="Animal Health").get() self.envelope_anidc_id = models.AnimalDetCode.objects.filter(name="Scale Envelope").get() self.wr_adsc_id = models.AniDetSubjCode.objects.filter(name="Wild Return").get() self.locc_id = models.LocCode.objects.filter(name="Adult Collection Site").get() self.salmon_id = models.SpeciesCode.objects.filter(name="Salmon").get() self.sampc_id = models.SampleCode.objects.filter(name="Individual Sample").get() for site_name in self.data[self.site_key].unique(): if utils.nan_to_none(site_name): self.site_dict[site_name] = models.ReleaseSiteCode.objects.filter(name__icontains=site_name).select_related("rive_id").get() for tank_name in self.data[self.tank_key].unique(): if utils.nan_to_none(tank_name): self.tank_dict[tank_name] = models.Tank.objects.filter(name__iexact=tank_name, facic_id=cleaned_data["facic_id"]).get() utils.enter_contx(self.tank_dict[tank_name], cleaned_data)
def clean_data(self): if self.success: contx_df = DataFrame(self.data_dict) cnt_df = contx_df.groupby("start_contx_pk", as_index=False).sum() for row in cnt_df.to_dict('records'): if utils.nan_to_none(row["start_contx_pk"]): cnt, cnt_entered = utils.enter_cnt( self.cleaned_data, 0, int(row["start_contx_pk"]), cnt_code="Fish Removed from Container") cnt.cnt = row[self.nfish_key] cnt.save() super(GenericGrpParser, self).clean_data()
def row_parser(self, row): cleaned_data = self.cleaned_data rive_id = models.RiverCode.objects.filter( name__iexact=row.get(self.rive_key)).get() if not rive_id: self.log_data += "River Code not found on row: {}".format(row) self.success = False return subr_id = None if utils.nan_to_none(row.get(self.subr_key)): subr_id = models.SubRiverCode.objects.filter( name__iexact=row.get(self.subr_key), rive_id=rive_id).get() if not subr_id: self.log_data += "Subriver Code not found on row: {}".format( row) self.success = False return trib_id = None if utils.nan_to_none(row.get(self.trib_key)): trib_id = models.Tributary.objects.filter(name__iexact=row.get( self.trib_key), rive_id=rive_id).get() if not trib_id: self.log_data += "Tributary not found on row: {}".format(row) self.success = False return site_id = models.ReleaseSiteCode( name=row.get(self.name_key), description_en=row.get(self.desc_key), rive_id=rive_id, trib_id=trib_id, subr_id=subr_id, min_lat=utils.nan_to_none(row.get(self.lat_key)), max_lat=utils.nan_to_none(row.get(self.max_lat_key)), min_lon=utils.nan_to_none(row.get(self.lon_key)), max_lon=utils.nan_to_none(row.get(self.max_lon_key)), created_by=cleaned_data["created_by"], created_date=cleaned_data["created_date"]) try: site_id.clean() site_id.save() self.row_entered = True except (IntegrityError, ValidationError): self.log_data += "Row {} not entered. \n".format(self.row_count) self.row_count += 1
def row_parser(self, row): cleaned_data = self.cleaned_data # get tray, group, and row date row_date = utils.get_row_date(row) tray_qs = models.Tray.objects.filter(trof_id=row["trof_id"], name=row[self.tray_key]) tray_id = tray_qs.filter( Q(start_date__lte=row_date, end_date__gte=row_date) | Q(end_date__isnull=True)).get() pair_id = models.Pairing.objects.filter( cross=row[self.cross_key], end_date__isnull=True, indv_id__stok_id=row["stok_id"], start_date__year=row[self.year_key]).first() grp_id = utils.get_tray_group(pair_id, tray_id, row_date) # want to shift the hu move event, so that the counting math always works out. hu_move_date = row_date + timedelta(minutes=1) hu_cleaned_data = utils.create_new_evnt(cleaned_data, "Allocation", hu_move_date) hu_anix, data_entered = utils.enter_anix(hu_cleaned_data, grp_pk=grp_id.pk) self.row_entered += data_entered hu_contx, data_entered = utils.enter_contx(tray_id, hu_cleaned_data, None, grp_pk=grp_id.pk, return_contx=True) self.row_entered += data_entered # record development dev_at_hu_transfer = grp_id.get_development(hu_move_date) utils.enter_grpd(hu_anix.pk, hu_cleaned_data, hu_move_date, dev_at_hu_transfer, None, anidc_str="Development") self.row_entered += utils.enter_contx(row["trof_id"], cleaned_data) # HU Picks: self.row_entered += utils.enter_cnt(cleaned_data, row[self.loss_key], hu_contx.pk, cnt_code="HU Transfer Loss")[1] # generate new group, cup, and movement event: cont = None if utils.nan_to_none(row[self.end_tray_key]): trof_id = models.Trough.objects.filter( facic_id=cleaned_data["facic_id"], name=row[self.end_trof_key]).get() tray_qs = models.Tray.objects.filter(trof_id=trof_id, name=row[self.tray_key]) cont = tray_qs.filter( Q(start_date__lte=row_date, end_date__gte=row_date) | Q(end_date__isnull=True)).get() elif utils.nan_to_none(row[self.end_trof_key]): cont = models.Trough.objects.filter( facic_id=cleaned_data["facic_id"], name=row[self.end_trof_key]).get() elif utils.nan_to_none(row[self.heatl_key]): cont = utils.get_cont_from_dot(row[self.cont_key], cleaned_data, row_date) elif utils.nan_to_none(row[self.tank_key]): cont = models.Tank.objects.filter( facic_id=cleaned_data["facic_id"], name=row[self.tank_key]) self.row_entered += utils.enter_contx(cont, cleaned_data) if not utils.y_n_to_bool(row[self.final_key]): # NEW GROUPS TAKEN FROM INITIAL out_cnt = utils.enter_cnt(cleaned_data, 0, hu_contx.pk, cnt_code="Eggs Removed")[0] utils.enter_cnt_det(cleaned_data, out_cnt, row[self.cnt_key], "Program Group Split", row[self.prog_key]) indv, final_grp = cont.fish_in_cont(row_date) if not final_grp: final_grp = models.Group( spec_id=grp_id.spec_id, coll_id=grp_id.coll_id, grp_year=grp_id.grp_year, stok_id=grp_id.stok_id, grp_valid=True, created_by=cleaned_data["created_by"], created_date=cleaned_data["created_date"], ) try: final_grp.clean() final_grp.save() except (ValidationError, IntegrityError): return None else: # MAIN GROUP GETTING MOVED final_grp = final_grp[0] final_grp_anix = utils.enter_anix(cleaned_data, grp_pk=final_grp.pk, return_anix=True) self.row_entered += utils.enter_anix(hu_cleaned_data, grp_pk=final_grp.pk, return_sucess=True) self.row_entered += utils.enter_bulk_grpd( final_grp_anix, cleaned_data, row_date, prnt_grp=grp_id, prog_grp=row.get(self.prog_key), comments=row.get(self.comment_key)) self.row_entered += utils.enter_grpd(final_grp_anix.pk, cleaned_data, row_date, dev_at_hu_transfer, None, anidc_str="Development") # create movement for the new group, create 2 contx's and 3 anix's # cup contx is contx used to link the positive counts cont_contx = utils.create_egg_movement_evnt(tray_id, cont, cleaned_data, row_date, final_grp.pk, return_cup_contx=True) move_cleaned_data = cleaned_data.copy() move_cleaned_data["evnt_id"] = cont_contx.evnt_id cnt_contx = cont_contx cnt_contx.pk = None cnt_contx.tray_id = tray_id try: cnt_contx.save() except IntegrityError: cnt_contx = models.ContainerXRef.objects.filter( pk=cont_contx.pk).get() self.row_entered += utils.enter_anix(move_cleaned_data, grp_pk=final_grp.pk, contx_pk=cnt_contx.pk, return_sucess=True) # add the positive counts cnt = utils.enter_cnt( move_cleaned_data, row[self.cnt_key], cnt_contx.pk, cnt_code="Eggs Added", )[0] if utils.nan_to_none(self.weight_key): utils.enter_cnt_det(move_cleaned_data, cnt, row[self.weight_key], "Weight") utils.enter_cnt_det(move_cleaned_data, cnt, row[self.cnt_key], "Program Group Split", row[self.prog_key]) else: # Move main group to drawer, and add end date to tray: if cont: end_contx = utils.create_movement_evnt(tray_id, cont, cleaned_data, row_date, grp_pk=grp_id.pk, return_end_contx=True) tray_id.end_date = row_date tray_id.save() end_cnt = utils.enter_cnt(cleaned_data, row[self.cnt_key], end_contx.pk, cnt_code="Egg Count")[0] utils.enter_cnt_det(cleaned_data, end_cnt, row[self.weight_key], "Weight") else: self.log_data += "\n Draw {} from {} not found".format( cont, row[self.cont_key]) # link cup to egg development event utils.enter_contx(cont, cleaned_data, None)
def row_parser(self, row): cleaned_data = self.cleaned_data row_date = utils.get_row_date(row) self.row_entered += utils.enter_contx(row["trof_id"], cleaned_data) # find group from either cross or tray: if utils.nan_to_none(row.get(self.hu_key)): cont_id = utils.get_cont_from_dot(row[self.hu_key], cleaned_data, row_date) elif utils.nan_to_none(row.get(self.tray_key)): cont_id = models.Tray.objects.filter( trof_id=row["trof_id"], end_date__isnull=True, name=row[self.tray_key]).get() else: cont_id = row["trof_id"] if utils.nan_to_none(row.get(self.cross_key)): pair_id = models.Pairing.objects.filter( cross=row[self.cross_key], end_date__isnull=True, indv_id__stok_id=row["stok_id"], start_date__year=row[self.year_key]).first() grp_id = utils.get_tray_group(pair_id, cont_id, row_date) else: grp_id = cont_id.fish_in_cont(row_date, get_grp=True) grp_anix = None shock = False for pickc_id in cleaned_data["pickc_id"]: if utils.nan_to_none(row[pickc_id.name]): shock = utils.y_n_to_bool(row.get(self.shocking_key)) grp_anix, evnt_entered = utils.create_picks_evnt( cleaned_data, cont_id, grp_id.pk, row[pickc_id.name], row_date, pickc_id.name, cleaned_data["evnt_id"].perc_id, shocking=shock, return_anix=True, pick_comments=row.get(self.comment_key)) self.row_entered += evnt_entered for col_name in row.keys(): col_date = utils.get_col_date(col_name) if col_date: col_date_str = datetime.strftime(col_date, "%Y-%b-%d") self.date_dict[col_date_str] = True if utils.nan_to_none(row.get(col_name)): self.row_entered += utils.create_picks_evnt( cleaned_data, cont_id, grp_id.pk, row[col_name], col_date, self.default_pickc_id, cleaned_data["evnt_id"].perc_id, pick_comments=row.get(self.comment_key)) # record development if grp_anix and shock: pick_evnt_cleaned_data = cleaned_data.copy() pick_evnt_cleaned_data["evnt_id"] = grp_anix.evnt_id dev_at_pick = grp_id.get_development(row_date) utils.enter_grpd(grp_anix.pk, pick_evnt_cleaned_data, row_date, dev_at_pick, None, anidc_str="Development") self.row_entered += utils.enter_contx(row["trof_id"], cleaned_data)
def data_preper(self): cleaned_data = self.cleaned_data self.sampc_id = models.SampleCode.objects.filter( name="Individual Sample").get() self.prnt_grp_anidc_id = models.AnimalDetCode.objects.filter( name="Parent Group").get() self.prog_grp_anidc_id = models.AnimalDetCode.objects.filter( name="Program Group").get() self.sex_anidc_id = models.AnimalDetCode.objects.filter( name="Gender").get() self.len_anidc_id = models.AnimalDetCode.objects.filter( name="Length").get() self.weight_anidc_id = models.AnimalDetCode.objects.filter( name="Weight").get() self.vial_anidc_id = models.AnimalDetCode.objects.filter( name="Vial").get() self.envelope_anidc_id = models.AnimalDetCode.objects.filter( name="Scale Envelope").get() self.ani_health_anidc_id = models.AnimalDetCode.objects.filter( name="Animal Health").get() self.anidc_ufid_id = models.AnimalDetCode.objects.filter( name="UFID").get() self.vax_anidc_id = models.AnimalDetCode.objects.filter( name="Vaccination").get() self.mark_anidc_id = models.AnimalDetCode.objects.filter( name="Mark").get() self.lifestage_anidc_id = models.AnimalDetCode.objects.filter( name="Lifestage").get() self.comment_anidc_id = models.AnimalDetCode.objects.filter( name="Comment").get() # The following steps are to set additional columns on each row to facilitate parsing. # In particular, columns set will be: "datetime", "grp_year", "grp_coll", "start_tank_id", # "end_tank_id", "grp_key", "end_grp_key". # The two grp_keys will link to dictionaries of the groups, which are also set below # set date self.data = utils.set_row_datetime(self.data) # split year-coll self.data["grp_year"] = self.data.apply( lambda row: utils.year_coll_splitter(row[self.yr_coll_key])[0], axis=1) self.data["grp_coll"] = self.data.apply( lambda row: utils.year_coll_splitter(row[self.yr_coll_key])[1], axis=1) # set start and end tank columns: self.data = utils.set_row_tank(self.data, cleaned_data, self.start_tank_key, col_name="start_tank_id") self.data = utils.set_row_tank(self.data, cleaned_data, self.end_tank_key, col_name="end_tank_id") # set the dict keys for groups, use astype(str) to handle anything that might be a nan. self.data, self.start_grp_dict = utils.set_row_grp( self.data, self.rive_key, self.yr_coll_key, self.prio_key, "start_tank_id", "datetime", self.grp_mark_key, grp_col_name="start_grp_id", return_dict=True) for item, grp in self.start_grp_dict.items(): utils.enter_anix(cleaned_data, grp_pk=grp.pk) self.data["end_grp_key"] = self.data[self.rive_key] + self.data[self.yr_coll_key] + \ self.data[self.end_tank_key].astype(str) + self.data[self.prio_key].astype(str) + \ self.data["datetime"].astype(str) + self.data[self.grp_mark_key].astype(str) # create the end group dict and create, movement event, groups, counts, contxs, etc. necesarry end_grp_data = self.data.groupby([ self.rive_key, "grp_year", "grp_coll", "end_tank_id", "start_tank_id", self.prio_key, "datetime", self.grp_mark_key, "grp_key", "end_grp_key" ], dropna=False, sort=False).size().reset_index() for row in end_grp_data.to_dict('records'): # check if end tank is set, otherwise, skip this step if not utils.nan_to_none(row["end_tank_id"]): self.end_grp_dict[row["end_grp_key"]] = None continue grps = utils.get_grp(row[self.rive_key], row["grp_year"], row["grp_coll"], row["end_tank_id"], at_date=row["datetime"], prog_str=row[self.prio_key], mark_str=row[self.grp_mark_key]) start_grp_id = self.start_grp_dict[row["grp_key"]] start_contx, contx_entered = utils.enter_contx( row["start_tank_id"], cleaned_data, None, grp_pk=start_grp_id.pk, return_contx=True) self.row_entered += utils.enter_cnt( cleaned_data, sum(end_grp_data[end_grp_data["grp_key"] == row["grp_key"]] [0]), start_contx.pk, cnt_code="Fish Removed from Container")[1] if len(grps) > 0: end_grp_id = grps[0] self.end_grp_dict[row["end_grp_key"]] = grps[0] else: end_grp_id = copy.deepcopy(start_grp_id) end_grp_id.pk = None end_grp_id.save() self.end_grp_dict[row["end_grp_key"]] = end_grp_id if end_grp_id.pk != start_grp_id.pk: grp_anix = utils.enter_anix(cleaned_data, grp_pk=end_grp_id.pk, return_anix=True) utils.enter_grpd(grp_anix.pk, cleaned_data, row["datetime"], None, self.prnt_grp_anidc_id.pk, frm_grp_id=start_grp_id) if utils.nan_to_none(row[self.prio_key]): utils.enter_grpd(grp_anix.pk, cleaned_data, row["datetime"], row[self.prio_key], self.prog_grp_anidc_id.pk, row[self.prio_key]) if utils.nan_to_none(row[self.grp_mark_key]): utils.enter_grpd(grp_anix.pk, cleaned_data, row["datetime"], row[self.grp_mark_key], self.mark_anidc_id.pk, row[self.grp_mark_key]) end_contx = utils.create_movement_evnt(row["start_tank_id"], row["end_tank_id"], cleaned_data, row["datetime"], grp_pk=end_grp_id.pk, return_end_contx=True) if end_contx: self.row_entered += utils.enter_cnt( cleaned_data, row[0], end_contx.pk)[1] self.data_dict = self.data.to_dict("records")
def row_parser(self, row): cleaned_data = self.cleaned_data year, coll = utils.year_coll_splitter(row[self.coll_key]) row_datetime = utils.get_row_date(row) row_date = row_datetime.date() indv_ufid = utils.nan_to_none(row.get(self.ufid_key)) indv = models.Individual( grp_id=self.grp_id, spec_id=self.salmon_id, stok_id=self.stok_id, coll_id=self.coll_id, indv_year=year, pit_tag=row[self.pit_key], ufid=indv_ufid, indv_valid=True, comments=utils.nan_to_none(row.get(self.comment_key)), created_by=cleaned_data["created_by"], created_date=cleaned_data["created_date"], ) try: indv.clean() indv.save() self.row_entered = True except (ValidationError, IntegrityError): indv = models.Individual.objects.filter(pit_tag=indv.pit_tag).get() if utils.nan_to_none(row[self.from_tank_id_key]) or utils.nan_to_none( row[self.to_tank_id_key]): in_tank = row[self.from_tank_id_key] out_tank = row[self.to_tank_id_key] self.row_entered += utils.create_movement_evnt(in_tank, out_tank, cleaned_data, row_datetime, indv_pk=indv.pk) # if tagged fish goes back into same tank, still link fish to tank: if in_tank == out_tank: utils.enter_contx(in_tank, cleaned_data, True, indv_pk=indv.pk) anix_indv, anix_entered = utils.enter_anix(cleaned_data, indv_pk=indv.pk) self.row_entered += anix_entered self.anix_indv = anix_indv utils.enter_bulk_indvd( anix_indv.pk, self.cleaned_data, row_date, len_mm=row.get(self.len_key_mm), len_val=row.get(self.len_key), weight=row.get(self.weight_key), weight_kg=row.get(self.weight_key_kg), vial=row.get(self.vial_key), mark=row.get(self.mark_key), prog_grp=row.get(self.group_key), lifestage=row.get(self.lifestage_key), comments=row.get(self.comment_key), ) if utils.nan_to_none(row.get(self.precocity_key)): self.row_entered += utils.enter_indvd(anix_indv.pk, cleaned_data, row_date, None, self.ani_health_anidc_id.pk, "Precocity") if utils.nan_to_none(row.get(self.crew_key)): perc_list, inits_not_found = utils.team_list_splitter( row[self.crew_key]) for perc_id in perc_list: team_id, team_entered = utils.add_team_member( perc_id, cleaned_data["evnt_id"], role_id=self.tagger_code, return_team=True) self.row_entered += team_entered if team_id: self.row_entered += utils.enter_anix(cleaned_data, indv_pk=indv.pk, team_pk=team_id.pk, return_sucess=True) for inits in inits_not_found: self.log_data += "No valid personnel with initials ({}) for row with pit tag" \ " {}\n".format(inits, row[self.pit_key])
def row_parser(self, row): cleaned_data = self.cleaned_data indv_female, samp_female, new_log = utils.get_indv_or_samp( row, self.pit_key_f, self.samp_key_f, cleaned_data["evnt_id"]) if new_log: self.log_data += new_log return self.log_data, False indv_male, samp_male, new_log = utils.get_indv_or_samp( row, self.pit_key_m, self.samp_key_m, cleaned_data["evnt_id"]) if new_log: self.log_data += new_log return self.log_data, False if not (indv_female or samp_female) or not (indv_male or samp_male): raise Exception( "No Individual or Fish found for row {}".format(row)) if not utils.nan_to_none(row[self.choice_key]): raise Exception( "Choice column cannot be empty. Set Fecundity column to zero to indicate Duds." ) row_date = utils.get_row_date(row) if indv_female: anix_female, anix_entered = utils.enter_anix( cleaned_data, indv_pk=indv_female.pk) self.row_entered += anix_entered self.row_entered += utils.enter_bulk_indvd( anix_female.pk, cleaned_data, row_date, gender="F", len_mm=row.get(self.len_key_f_mm), len_val=row.get(self.len_key_f), weight=row.get(self.weight_key_f), weight_kg=row.get(self.weight_key_f_kg), status=row.get(self.status_key_f), comments=row.get(self.comment_key_f)) if indv_male: anix_male, anix_entered = utils.enter_anix(cleaned_data, indv_pk=indv_male.pk) self.row_entered += anix_entered self.row_entered += utils.enter_bulk_indvd( anix_male.pk, cleaned_data, row_date, gender="M", len_mm=row.get(self.len_key_m_mm), len_val=row.get(self.len_key_m), weight=row.get(self.weight_key_m), weight_kg=row.get(self.weight_key_m_kg), status=row.get(self.status_key_m), comments=row.get(self.comment_key_m)) if samp_female: self.row_entered += utils.enter_bulk_sampd( samp_female.pk, cleaned_data, row_date, gender="F", len_mm=row.get(self.len_key_f_mm), len_val=row.get(self.len_key_f), weight=row.get(self.weight_key_f), weight_kg=row.get(self.weight_key_f_kg), status=row.get(self.status_key_f), comments=row.get(self.comment_key_f)) if samp_male: self.row_entered += utils.enter_bulk_sampd( samp_male.pk, cleaned_data, row_date, gender="M", len_mm=row.get(self.len_key_m_mm), len_val=row.get(self.len_key_m), weight=row.get(self.weight_key_m), weight_kg=row.get(self.weight_key_m_kg), status=row.get(self.status_key_m), comments=row.get(self.comment_key_m)) if utils.nan_to_none(row.get(self.dest_key_f)) and indv_female: end_tank_id_f = models.Tank.objects.filter( name=row[self.dest_key_f], facic_id=cleaned_data["facic_id"]).get() self.row_entered += utils.create_movement_evnt( None, end_tank_id_f, cleaned_data, row_date, indv_female.pk) if utils.nan_to_none(row.get(self.dest_key_m)) and indv_male: end_tank_id_m = models.Tank.objects.filter( name=row[self.dest_key_m], facic_id=cleaned_data["facic_id"]).get() self.row_entered += utils.create_movement_evnt( None, end_tank_id_m, cleaned_data, row_date, indv_male.pk) # pair pair = models.Pairing( start_date=row_date, prio_id=models.PriorityCode.objects.filter( name__iexact=prio_dict[row[self.prio_key_f]]).get(), pair_prio_id=models.PriorityCode.objects.filter( name__iexact=prio_dict[row[self.prio_key_pair]]).get(), cross=row[self.cross_key], valid=True, indv_id=indv_female, samp_id=samp_female, comments=utils.nan_to_none(row[self.comment_key_pair]), created_by=cleaned_data["created_by"], created_date=cleaned_data["created_date"], ) try: pair.clean() pair.save() self.row_entered = True except (ValidationError, IntegrityError): pair = models.Pairing.objects.filter(start_date=row_date, indv_id=indv_female, samp_id=samp_female).get() # sire sire = models.Sire( prio_id=models.PriorityCode.objects.filter( name__iexact=prio_dict[row[self.prio_key_m]]).get(), pair_id=pair, indv_id=indv_male, samp_id=samp_male, choice=row[self.choice_key], comments=utils.nan_to_none(row[self.comment_key_m]), created_by=cleaned_data["created_by"], created_date=cleaned_data["created_date"], ) try: sire.clean() sire.save() self.row_entered = True except (ValidationError, IntegrityError): # don't use sire again anywhere pass self.row_entered += utils.enter_anix(cleaned_data, pair_pk=pair.pk, return_sucess=True) # pairing program: if utils.nan_to_none(row.get(self.prog_key)): self.row_entered += utils.enter_spwnd( pair.pk, cleaned_data, row[self.prog_key], self.prog_spwndc_id.pk, spwnsc_str=row[self.prog_key]) # fecu/dud/extra male if row[self.egg_est_key] > 0: self.row_entered += utils.enter_spwnd(pair.pk, cleaned_data, int(row[self.egg_est_key]), self.fecu_spwndc_id.pk, None, "Calculated") else: self.row_entered += utils.enter_spwnd(pair.pk, cleaned_data, row[self.choice_key], self.dud_spwndc_id.pk, None, "Good") # grp anix_grp_qs = models.AniDetailXref.objects.filter( evnt_id=cleaned_data["evnt_id"], grp_id__isnull=False, pair_id=pair, indv_id__isnull=True, contx_id__isnull=True, loc_id__isnull=True, ) anix_grp = False if anix_grp_qs.count() == 0: if indv_female: stok_id = indv_female.stok_id spec_id = indv_female.spec_id else: stok_id = samp_female.stok_id spec_id = samp_female.spec_id grp = models.Group( spec_id=spec_id, stok_id=stok_id, coll_id=models.Collection.objects.filter( name="Egg (F1)").get(), grp_year=row_date.year, grp_valid=False, created_by=cleaned_data["created_by"], created_date=cleaned_data["created_date"], ) try: grp.clean() grp.save() row_entered = True anix_grp, anix_entered = utils.enter_anix(cleaned_data, grp_pk=grp.pk) row_entered += utils.enter_anix(cleaned_data, grp_pk=grp.pk, pair_pk=pair.pk, return_sucess=True) grp.grp_valid = True grp.save() self.row_entered = True except ValidationError: # recovering the group is only doable through the anix with both grp and pair. # no way to find it here, so only make the group valid after anix's created. pass elif anix_grp_qs.count() == 1: anix_grp = anix_grp_qs.get() grp = anix_grp.grp_id if anix_grp: utils.enter_bulk_grpd(anix_grp, cleaned_data, row_date, prog_grp=row[self.prog_key])
def data_preper(self): cleaned_data = self.cleaned_data self.temp_envc_id = models.EnvCode.objects.filter(name="Temperature").get() self.leader_code = models.RoleCode.objects.filter(name__iexact="Crew Lead").get() self.settings_locdc_id = models.LocationDetCode.objects.filter(name__iexact="Electrofishing Settings").get() self.voltage_locdc_id = models.LocationDetCode.objects.filter(name__iexact="Voltage").get() self.fishing_time_locdc_id = models.LocationDetCode.objects.filter(name__iexact="Electrofishing Seconds").get() for river_name in self.data[self.rive_key].unique(): self.river_dict[river_name] = models.RiverCode.objects.filter(name__icontains=river_name).get() if self.cleaned_data["evntc_id"].__str__() == "Electrofishing": self.locc_id = models.LocCode.objects.filter(name__icontains="Electrofishing site").get() elif self.cleaned_data["evntc_id"].__str__() == "Smolt Wheel Collection": self.locc_id = models.LocCode.objects.filter(name__icontains="Smolt Wheel site").get() elif self.cleaned_data["evntc_id"].__str__() == "Bypass Collection": self.locc_id = models.LocCode.objects.filter(name__icontains="Bypass site").get() # assign groups to columns, add generic group data: self.data["grp_id"] = None river_group_data = self.data.groupby([self.rive_key, self.prio_key, self.year_key, self.coll_key, self.tank_key], dropna=False).size().reset_index() if not river_group_data[self.tank_key].is_unique: raise Exception("Too many different groups going into same tank. Create multiple events if needed") for index, row in river_group_data.iterrows(): if not utils.nan_to_none(row[self.tank_key]): # if fish are only observed, don't make a group data_rows = (self.data[self.tank_key].isnull()) self.data.loc[data_rows, "grp_id"] = None self.data.loc[data_rows, "contx_id"] = None break stok_id = models.StockCode.objects.filter(name__icontains=row[self.rive_key]).get() coll_str = row[self.coll_key] if len(coll_str.lstrip(' 0123456789')) == len(coll_str): # year taken from year coll: coll_id = utils.coll_getter(row[self.coll_key]) grp_year = row[self.year_key] else: grp_year, coll_str = utils.year_coll_splitter(row[self.coll_key]) coll_id = utils.coll_getter(coll_str) anix_grp_qs = models.AniDetailXref.objects.filter(evnt_id=cleaned_data["evnt_id"], grp_id__stok_id=stok_id, grp_id__coll_id=coll_id, grp_id__grp_year=grp_year, indv_id__isnull=True, contx_id__isnull=True, loc_id__isnull=True, pair_id__isnull=True) grp_found = False grp = None for anix in anix_grp_qs: anix_prog_grp_names = [adsc.name for adsc in anix.grp_id.prog_group()] if utils.nan_to_none(row[self.prio_key]) and row[self.prio_key] in anix_prog_grp_names: grp_found = True grp = anix.grp_id break elif not utils.nan_to_none(row[self.prio_key]) and not anix_prog_grp_names: grp_found = True grp = anix.grp_id break if not grp_found: grp = models.Group(spec_id=models.SpeciesCode.objects.filter(name__iexact="Salmon").get(), stok_id=stok_id, coll_id=coll_id, grp_year=grp_year, grp_valid=True, created_by=cleaned_data["created_by"], created_date=cleaned_data["created_date"], ) try: grp.clean() grp.save() except ValidationError: grp = models.Group.objects.filter(spec_id=grp.spec_id, stok_id=grp.stok_id, grp_year=grp.grp_year, coll_id=grp.coll_id).get() anix_grp = utils.enter_anix(cleaned_data, grp_pk=grp.pk, return_anix=True) if utils.nan_to_none(row.get(self.prio_key)): utils.enter_grpd(anix_grp.pk, cleaned_data, cleaned_data["evnt_id"].start_date, None, None, anidc_str="Program Group", adsc_str=row[self.prio_key]) # create index column matching all rows in data to this group-tank-prio combination if utils.nan_to_none(row.get(self.prio_key)): data_rows = (self.data[self.rive_key] == row[self.rive_key]) & \ (self.data[self.prio_key] == row[self.prio_key]) & \ (self.data[self.tank_key] == row[self.tank_key]) & \ (self.data[self.coll_key] == row[self.coll_key]) else: data_rows = (self.data[self.rive_key] == row[self.rive_key]) & \ (self.data[self.coll_key] == row[self.coll_key]) & \ (self.data[self.tank_key] == row[self.tank_key]) & \ (self.data[self.prio_key].isnull()) # grp found, assign to all rows: self.data.loc[data_rows, "grp_id"] = grp contx, data_entered = utils.enter_tank_contx(row[self.tank_key], cleaned_data, True, None, grp.pk, return_contx=True) self.data.loc[data_rows, "contx_id"] = contx self.data_dict = self.data.to_dict("records")
def row_parser(self, row): cleaned_data = self.cleaned_data row_datetime = utils.get_row_date(row) relc_id = self.site_dict[row[self.site_key]] year, coll = utils.year_coll_splitter(row[self.coll_key]) coll_id = utils.coll_getter(coll) stok_id = models.StockCode.objects.filter(name__iexact=relc_id.rive_id.name).get() indv_id = None if utils.nan_to_none(row[self.pit_key]): indv_id = models.Individual.objects.filter(pit_tag=row[self.pit_key]).first() if not indv_id: indv_id = models.Individual(spec_id=self.salmon_id, stok_id=stok_id, coll_id=coll_id, indv_year=year, pit_tag=row[self.pit_key], indv_valid=True, comments=utils.nan_to_none(row.get(self.comment_key)), created_by=cleaned_data["created_by"], created_date=cleaned_data["created_date"], ) try: indv_id.clean() indv_id.save() self.row_entered = True except (ValidationError, IntegrityError): indv_id = models.Individual.objects.filter(pit_tag=indv_id.pit_tag).get() indv_anix, data_entered = utils.enter_anix(cleaned_data, indv_pk=indv_id.pk) self.row_entered += data_entered # add program group to individual if needed: loc = models.Location(evnt_id_id=cleaned_data["evnt_id"].pk, locc_id=self.locc_id, rive_id=relc_id.rive_id, relc_id=relc_id, loc_date=row_datetime, created_by=cleaned_data["created_by"], created_date=cleaned_data["created_date"], ) try: loc.clean() loc.save() self.row_entered = True except ValidationError: loc = models.Location.objects.filter(evnt_id=loc.evnt_id, locc_id=loc.locc_id, rive_id=loc.rive_id, subr_id=loc.subr_id, relc_id=loc.relc_id, loc_lat=loc.loc_lat, loc_lon=loc.loc_lon, loc_date=loc.loc_date).get() self.loc = loc self.team_parser(row[self.crew_key], row, loc_id=loc) if indv_id: anix_loc_indv, anix_entered = utils.enter_anix(cleaned_data, loc_pk=loc.pk, indv_pk=indv_id.pk) self.row_entered += anix_entered self.row_entered += utils.enter_bulk_indvd(anix_loc_indv.pk, self.cleaned_data, row_datetime, gender=row.get(self.sex_key), len_mm=row.get(self.len_key_mm), len_val=row.get(self.len_key), weight=row.get(self.weight_key), weight_kg=row.get(self.weight_key_kg), vial=row.get(self.vial_key), scale_envelope=row.get(self.scale_key), prog_grp=row.get(self.grp_key), comments=row.get(self.comment_key) ) if utils.nan_to_none(row.get(self.mort_key)): if utils.y_n_to_bool(row[self.mort_key]): mort_anix, mort_entered = utils.enter_mortality(indv_id, self.cleaned_data, row_datetime) self.row_entered += mort_entered if utils.nan_to_none(row.get(self.wr_key)): if utils.y_n_to_bool(row[self.wr_key]): self.row_entered += utils.enter_indvd(anix_loc_indv.pk, cleaned_data, row_datetime, None, self.ani_health_anidc_id.pk, adsc_str=self.wr_adsc_id.name) if utils.nan_to_none(row.get(self.aquaculture_key)): if utils.y_n_to_bool(row[self.aquaculture_key]): self.row_entered += utils.enter_indvd(anix_loc_indv.pk, cleaned_data, row_datetime, None, self.ani_health_anidc_id.pk, adsc_str="Aquaculture") if utils.nan_to_none(row[self.tank_key]): self.row_entered += utils.enter_contx(self.tank_dict[row[self.tank_key]], cleaned_data, True, indv_id.pk) if self.loc.pk not in self.loc_caught_dict: self.loc_caught_dict[self.loc.pk] = 1 else: self.loc_caught_dict[self.loc.pk] += 1 else: if self.loc.pk not in self.loc_obs_dict: self.loc_obs_dict[self.loc.pk] = 1 else: self.loc_obs_dict[self.loc.pk] += 1 elif utils.nan_to_none(row.get(self.samp_key)): samp = models.Sample.objects.filter(anix_id__evnt_id=cleaned_data["evnt_id"], loc_id=loc, spec_id=self.salmon_id, samp_num=row[self.samp_key], sampc_id=self.sampc_id, ).get() if not samp: # create group for sample: grp_id = models.Group(spec_id=self.salmon_id, stok_id=stok_id, coll_id=coll_id, grp_year=year, grp_valid=False, created_by=cleaned_data["created_by"], created_date=cleaned_data["created_date"], ) grp_id.clean() grp_id.save() self.row_entered = True grp_anix, data_entered = utils.enter_anix(cleaned_data, grp_pk=grp_id.pk) self.row_entered += data_entered samp, samp_entered = utils.enter_samp(cleaned_data, row[self.samp_key], self.salmon_id.pk, self.sampc_id.pk, anix_pk=grp_anix.pk, loc_pk=loc.pk, comments=utils.nan_to_none(row.get(self.comment_key))) self.row_entered += samp_entered self.row_entered += utils.enter_bulk_sampd(samp.pk, self.cleaned_data, row_datetime, gender=row.get(self.sex_key), len_mm=row.get(self.len_key_mm), len_val=row.get(self.len_key), weight=row.get(self.weight_key), weight_kg=row.get(self.weight_key_kg), vial=row.get(self.vial_key), scale_envelope=row.get(self.scale_key), prog_grp=row.get(self.grp_key), comments=row.get(self.comment_key) ) if utils.nan_to_none(row.get(self.mort_key)): if utils.y_n_to_bool(row[self.mort_key]): self.row_entered += utils.enter_samp_mortality(samp, self.cleaned_data, row_datetime) if utils.nan_to_none(row.get(self.wr_key)): if utils.y_n_to_bool(row[self.wr_key]): self.row_entered += utils.enter_sampd(samp.pk, cleaned_data, row_datetime, None, self.ani_health_anidc_id.pk, adsc_str=self.wr_adsc_id.name) if utils.nan_to_none(row.get(self.aquaculture_key)): if utils.y_n_to_bool(row[self.aquaculture_key]): self.row_entered += utils.enter_sampd(samp.pk, cleaned_data, row_datetime, None, self.ani_health_anidc_id.pk, adsc_str="Aquaculture") else: raise Exception("Fish must either be assigned a sample number or a pit tag.")
def row_parser(self, row): cleaned_data = self.cleaned_data contx, data_entered = utils.enter_tank_contx(row[self.tank_key], cleaned_data, None, return_contx=True) self.row_entered += data_entered row_date = utils.get_row_date(row) if utils.nan_to_none(row[self.time_key]): row_time = row[self.time_key].replace(tzinfo=pytz.UTC) else: row_time = None if utils.nan_to_none(row.get(self.temp_key)): self.row_entered += utils.enter_env(row[self.temp_key], row_date, cleaned_data, self.temp_envc_id, contx=contx, env_time=row_time) if utils.nan_to_none(row.get(self.dox_key)): self.row_entered += utils.enter_env(row[self.dox_key], row_date, cleaned_data, self.oxlvl_envc_id, contx=contx, env_time=row_time) if utils.nan_to_none(row.get(self.ph_key)): self.row_entered += utils.enter_env(row[self.ph_key], row_date, cleaned_data, self.ph_envc_id, contx=contx, env_time=row_time) if utils.nan_to_none(row.get(self.dn_key)): self.row_entered += utils.enter_env(row[self.dn_key], row_date, cleaned_data, self.disn_envc_id, contx=contx, env_time=row_time) if utils.nan_to_none(row.get(self.source_key)): source_envsc_id = models.EnvSubjCode.objects.filter( name__icontains=row[self.source_key]).get() self.row_entered += utils.enter_env(row[self.source_key], row_date, cleaned_data, self.ws_envc_id, envsc_id=source_envsc_id, contx=contx, env_time=row_time) if utils.nan_to_none(row.get(self.crew_key)): perc_list, inits_not_found = utils.team_list_splitter( row[self.crew_key]) for perc_id in perc_list: team_id, team_entered = utils.add_team_member( perc_id, cleaned_data["evnt_id"], return_team=True) self.row_entered += team_entered if team_id: self.row_entered += utils.enter_tank_contx( row[self.tank_key], cleaned_data, team_pk=team_id.pk) for inits in inits_not_found: self.log_data += "No valid personnel with initials ({}) for row {} \n".format( inits, row)
def row_parser(self, row): super().row_parser(row) if utils.nan_to_none(row.get(self.crew_lead_key)): self.team_parser(row[self.crew_lead_key], row, loc_id=self.loc, role_id=self.leader_code)
def row_parser(self, row): cleaned_data = self.cleaned_data row_datetime = utils.get_row_date(row) relc_id = None rive_id = self.river_dict[row[self.rive_key]] if not utils.nan_to_none(row.get(self.tank_key)) and utils.nan_to_none(row.get(self.fish_caught_key)): # make sure if fish are caught they are assigned a tank: raise Exception("All caught fish must be assigned a tank") if utils.nan_to_none(row.get(self.site_key)): relc_qs = models.ReleaseSiteCode.objects.filter(name__iexact=row[self.site_key]) if len(relc_qs) == 1: relc_id = relc_qs.get() start_lat = utils.round_no_nan(row.get(self.lat_key), 6) start_lon = utils.round_no_nan(row.get(self.lon_key), 6) if not relc_id and not (start_lat and start_lon): raise Exception("Site code not found and lat-long not given for site on row") loc = models.Location(evnt_id_id=cleaned_data["evnt_id"].pk, locc_id=self.locc_id, rive_id=rive_id, relc_id=relc_id, loc_lat=start_lat, loc_lon=start_lon, end_lat=utils.round_no_nan(row.get(self.end_lat), 6), end_lon=utils.round_no_nan(row.get(self.end_lon), 6), loc_date=row_datetime, comments=utils.nan_to_none(row.get(self.comment_key)), created_by=cleaned_data["created_by"], created_date=cleaned_data["created_date"], ) try: loc.clean() loc.save() self.row_entered = True except ValidationError: loc = models.Location.objects.filter(evnt_id=loc.evnt_id, locc_id=loc.locc_id, rive_id=loc.rive_id, subr_id=loc.subr_id, relc_id=loc.relc_id, loc_lat=loc.loc_lat, loc_lon=loc.loc_lon, loc_date=loc.loc_date).get() self.loc = loc if row["grp_id"]: self.row_entered += utils.enter_anix(cleaned_data, loc_pk=loc.pk, grp_pk=row["grp_id"].pk, return_sucess=True) if self.loc.loc_lon and self.loc.loc_lat and not self.loc.relc_id: self.log_data += "\nNo site found in db for Lat-Long ({}, {}) given on row: \n{}\n\n"\ .format(self.loc.loc_lat, self.loc.loc_lon, row) if utils.nan_to_none(row["contx_id"]): self.row_entered += utils.enter_anix(cleaned_data, loc_pk=loc.pk, contx_pk=row["contx_id"].pk, return_sucess=True) self.team_parser(row[self.crew_key], row, loc_id=loc) if utils.nan_to_none(row.get(self.temp_key)): self.row_entered += utils.enter_env(row[self.temp_key], row_datetime, cleaned_data, self.temp_envc_id, loc_id=loc) cnt_caught, cnt_entered = utils.enter_cnt(cleaned_data, cnt_value=row[self.fish_caught_key], loc_pk=loc.pk, cnt_code="Fish Caught") self.row_entered += cnt_entered cnt_obs, cnt_entered = utils.enter_cnt(cleaned_data, cnt_value=row[self.fish_obs_key], loc_pk=loc.pk, cnt_code="Fish Observed") self.row_entered += cnt_entered if utils.nan_to_none(row.get(self.settings_key)): self.row_entered += utils.enter_locd(loc.pk, cleaned_data, row_datetime, row[self.settings_key], self.settings_locdc_id.pk) if utils.nan_to_none(row.get(self.fishing_time_key)): self.row_entered += utils.enter_locd(loc.pk, cleaned_data, row_datetime, row[self.fishing_time_key], self.fishing_time_locdc_id.pk) if utils.nan_to_none(row.get(self.voltage_key)): self.row_entered += utils.enter_locd(loc.pk, cleaned_data, row_datetime, row[self.voltage_key], self.voltage_locdc_id.pk)
def row_parser(self, row): row_datetime = utils.get_row_date(row) row_date = row_datetime.date() indv_qs = models.Individual.objects.filter(pit_tag=row[self.pit_key]) if len(indv_qs) == 1: indv = indv_qs.get() else: self.log_data += "Error parsing row: \n" self.log_data += str(row) self.log_data += "\nFish with PIT {} not found in db\n".format( row[self.pit_key]) self.success = False return anix, anix_entered = utils.enter_anix(self.cleaned_data, indv_pk=indv.pk) self.row_entered += anix_entered self.row_entered += utils.enter_bulk_indvd( anix.pk, self.cleaned_data, row_date, gender=row.get(self.sex_key), len_mm=row.get(self.len_key_mm), len_val=row.get(self.len_key), weight=row.get(self.weight_key), weight_kg=row.get(self.weight_key_kg), vial=row.get(self.vial_key), scale_envelope=row.get(self.envelope_key), tissue_yn=row.get(self.tissue_key), mark=row.get(self.mark_key), vaccinated=row.get(self.vax_key), status=row.get(self.status_key), lifestage=row.get(self.lifestage_key), comments=row.get(self.comment_key)) if utils.nan_to_none(row.get(self.precocity_key)): if utils.y_n_to_bool(row[self.precocity_key]): self.row_entered += utils.enter_indvd( anix.pk, self.cleaned_data, row_date, None, self.ani_health_anidc_id.pk, "Precocity") if utils.nan_to_none(row.get(self.mort_key)): if utils.y_n_to_bool(row[self.mort_key]): mort_anix, mort_entered = utils.enter_mortality( indv, self.cleaned_data, row_datetime) self.row_entered += mort_entered in_tank = None out_tank = None if utils.nan_to_none(row[self.start_tank_key]): in_tank = models.Tank.objects.filter( name=row[self.start_tank_key]).get() if utils.nan_to_none(row[self.end_tank_key]): out_tank = models.Tank.objects.filter( name=row[self.end_tank_key]).get() if in_tank or out_tank: self.row_entered += utils.create_movement_evnt(in_tank, out_tank, self.cleaned_data, row_datetime, indv_pk=indv.pk) self.row_entered += utils.parse_extra_cols(row, self.cleaned_data, anix, indv=True)
def row_parser(self, row): cleaned_data = self.cleaned_data row_date = row["datetime"].date() row_start_grp = utils.get_grp(row[self.rive_key], row["grp_year"], row["grp_coll"], row["start_tank_id"], row_date, prog_str=row.get(self.prio_key), mark_str=row.get(self.grp_mark_key), fail_on_not_found=True)[0] start_anix, self.row_entered = utils.enter_anix( cleaned_data, grp_pk=row_start_grp.pk) start_contx, contx_entered = utils.enter_contx(row["start_tank_id"], cleaned_data, None, return_contx=True) self.row_entered += contx_entered whole_grp = utils.y_n_to_bool(row[self.abs_key]) det_anix = start_anix row["start_contx_pk"] = None if not whole_grp: row["start_contx_pk"] = start_contx.pk if utils.nan_to_none(row["end_tank_id"]): # 4 possible cases here: group in tank or not and whole group move or not: row_end_grp_list = utils.get_grp(row[self.rive_key], row["grp_year"], row["grp_coll"], row["end_tank_id"], row_date, prog_str=row[self.prio_key], mark_str=row[self.grp_mark_key]) row_end_grp = None if not whole_grp and not row_end_grp_list: # splitting fish group, create end group: row_end_grp = copy.deepcopy(row_start_grp) row_end_grp.pk = None row_end_grp.id = None row_end_grp.save() end_grp_anix, anix_entered = utils.enter_anix( cleaned_data, grp_pk=row_end_grp.pk) self.row_entered = anix_entered self.row_entered += utils.enter_bulk_grpd( end_grp_anix.pk, cleaned_data, row_date, prog_grp=row.get(self.prio_key), mark=row.get(self.mark_key)) elif not whole_grp: # splitting fish group, merging to exsisting end group row_end_grp = row_end_grp_list[0] if row_end_grp: move_contx = utils.create_movement_evnt(row["start_tank_id"], row["end_tank_id"], cleaned_data, row_date, grp_pk=row_end_grp.pk, return_end_contx=True) end_grp_anix, anix_entered = utils.enter_anix( cleaned_data, grp_pk=row_end_grp.pk) self.row_entered += anix_entered self.row_entered += utils.enter_grpd(end_grp_anix.pk, cleaned_data, row_date, None, self.prnt_grp_anidc_id.pk, frm_grp_id=row_start_grp) cnt, cnt_entered = utils.enter_cnt(cleaned_data, row[self.nfish_key], move_contx.pk) self.row_entered = cnt_entered # record details on end tank group det_anix = end_grp_anix else: # move all the fish (whole group, merge to fish at destination if needed) move_contx = utils.create_movement_evnt( row["start_tank_id"], row["end_tank_id"], cleaned_data, row_date, grp_pk=row_start_grp.pk, return_end_contx=True) cnt, cnt_entered = utils.enter_cnt(cleaned_data, row[self.nfish_key], move_contx.pk, cnt_code="Fish Count") self.row_entered = cnt_entered else: if utils.nan_to_none(row[self.nfish_key]): cnt, cnt_entered = utils.enter_cnt(cleaned_data, row[self.nfish_key], start_contx.pk, cnt_code="Fish Count") self.row_entered = cnt_entered # add details to det_anix: self.row_entered += utils.enter_bulk_grpd( det_anix.pk, cleaned_data, row_date, vaccinated=row.get(self.vax_key), mark=row.get(self.mark_key), lifestage=row.get(self.lifestage_key), comments=row.get(self.comment_key)) self.row_entered += utils.parse_extra_cols(row, self.cleaned_data, det_anix, grp=True)
def row_parser(self, row): cleaned_data = self.cleaned_data row_date = row["datetime"].date() row_grp = row["start_grp_id"] row_end_grp = self.end_grp_dict[row["end_grp_key"]] if row_end_grp: row_grp = row_end_grp row_anix, data_entered = utils.enter_anix(cleaned_data, grp_pk=row_grp.pk) self.row_entered += data_entered row_contx, data_entered = utils.enter_contx(row.get("start_tank_id"), cleaned_data, None, grp_pk=row_grp.pk, return_contx=True) self.row_entered += data_entered samp_anix = row_contx.animal_details.filter( grp_id=row_grp, evnt_id=cleaned_data["evnt_id"], indv_id__isnull=True, loc_id__isnull=True, pair_id__isnull=True, final_contx_flag=None).get() row_samp, data_entered = utils.enter_samp(cleaned_data, row[self.samp_key], row_grp.spec_id.pk, self.sampc_id.pk, anix_pk=samp_anix.pk) self.row_entered += data_entered if utils.nan_to_none(row.get(self.mort_key)): if utils.y_n_to_bool(row[self.mort_key]): self.row_entered += utils.enter_samp_mortality( row_samp, cleaned_data, row_date) if row_samp: self.row_entered += utils.enter_bulk_sampd( row_samp.pk, self.cleaned_data, row_date, gender=row.get(self.sex_key), len_mm=row.get(self.len_key_mm), len_val=row.get(self.len_key), weight=row.get(self.weight_key), weight_kg=row.get(self.weight_key_kg), vial=row.get(self.vial_key), scale_envelope=row.get(self.envelope_key), tissue_yn=row.get(self.tissue_key), mark=row.get(self.mark_key), vaccinated=row.get(self.vax_key), lifestage=row.get(self.lifestage_key), comments=row.get(self.comment_key)) if utils.nan_to_none(row.get(self.precocity_key)): if utils.y_n_to_bool(row[self.precocity_key]): self.row_entered += utils.enter_sampd( row_samp.pk, cleaned_data, row_date, "Precocity", self.ani_health_anidc_id.pk, adsc_str="Precocity") if utils.nan_to_none(row.get(self.ufid_key)): self.row_entered += utils.enter_sampd(row_samp.pk, cleaned_data, row_date, row[self.ufid_key], self.anidc_ufid_id.pk) self.row_entered += utils.parse_extra_cols(row, self.cleaned_data, row_samp, samp=True) else: self.success = False