示例#1
0
 def test_two_cnts_one_grp(self):
     # add two counts in different containers and make sure group record proper count
     cnt_val = randint(0, 100)
     utils.enter_cnt(self.cleaned_data, cnt_val, self.contx.pk, cnt_code="Fish in Container")
     contx = utils.enter_contx(self.final_tank, self.cleaned_data, True, grp_pk=self.grp.pk, return_contx=True)
     utils.enter_cnt(self.cleaned_data, cnt_val, contx.pk, cnt_code="Fish in Container")
     self.assertEqual(self.grp.count_fish_in_group(), 2 * cnt_val)
示例#2
0
 def test_program_grp_cnt(self):
     #  take two types of eggs from group in same event.
     init_cnt = randint(300, 500)
     cnt_one_val = randint(0, 100)
     cnt_two_val = randint(0, 100)
     utils.enter_cnt(self.cleaned_data, init_cnt, self.contx.pk, cnt_code="Eggs Added")
     cnt = utils.enter_cnt(self.cleaned_data, 0, self.contx.pk, cnt_code="Eggs Removed")[0]
     utils.enter_cnt_det(self.cleaned_data, cnt, cnt_one_val, "Program Group Split", "EQU")
     utils.enter_cnt_det(self.cleaned_data, cnt, cnt_two_val, "Program Group Split", "PEQU")
     self.assertEqual(self.grp.count_fish_in_group(), init_cnt - cnt_one_val - cnt_two_val)
示例#3
0
 def test_two_cnts_one_grp(self):
     # add two counts in different containers and make sure group record proper count
     cnt_val = randint(0, 100)
     utils.enter_cnt(self.cleaned_data, cnt_val, self.contx.pk, cnt_code="Fish in Container")
     # sometimes factories will reuse an event/tank which will prevent new contx's and cnt's from being entered.
     # this loop ensures that new data does get added
     data_entered = False
     while not data_entered:
         contx, data_entered = utils.enter_contx(self.final_tank, self.cleaned_data, True, grp_pk=self.grp.pk,
                                                 return_contx=True)
     utils.enter_cnt(self.cleaned_data, cnt_val, contx.pk, cnt_code="Fish in Container")
     self.assertEqual(self.grp.count_fish_in_group(), 2 * cnt_val)
示例#4
0
 def data_cleaner(self):
     from_tanks = self.data[self.from_tank_key].value_counts()
     for tank_name in from_tanks.keys():
         fish_tagged_from_tank = int(from_tanks[tank_name])
         contx, data_entered = utils.enter_tank_contx(tank_name,
                                                      self.cleaned_data,
                                                      None,
                                                      grp_pk=self.grp_id.pk,
                                                      return_contx=True)
         if contx:
             utils.enter_cnt(self.cleaned_data,
                             fish_tagged_from_tank,
                             contx.pk,
                             cnt_code="Pit Tagged")
示例#5
0
    def test_aboslute_cnt(self):
        #  take eggs from a group and then record absolute count the following day.
        init_cnt = randint(300, 500)
        cnt_one_val = randint(5, 100)
        cnt_final_val = randint(0, 5)
        next_day_evnt = BioFactoryFloor.EvntFactory()
        next_day_evnt.facic_id = self.evnt.facic_id
        next_day_evnt.start_datetime = self.evnt.start_datetime + datetime.timedelta(days=1)
        next_day_evnt.save()
        new_cleaned_data = self.cleaned_data.copy()
        new_cleaned_data["evnt_id"] = next_day_evnt
        end_contx = utils.enter_contx(self.tank, new_cleaned_data, None, grp_pk=self.grp.pk, return_contx=True)

        utils.enter_cnt(self.cleaned_data, init_cnt, self.contx.pk, cnt_code="Eggs Added")
        cnt = utils.enter_cnt(self.cleaned_data, 0, self.contx.pk, cnt_code="Eggs Removed")
        utils.enter_cnt_det(self.cleaned_data, cnt, cnt_one_val, "Program Group", "EQU")
        utils.enter_cnt(new_cleaned_data, cnt_final_val, end_contx.pk, cnt_code="Egg Count")
        self.assertEqual(self.grp.count_fish_in_group(), cnt_final_val)
示例#6
0
    def clean(self):
        cleaned_data = super().clean()
        cleaned_data["move_date"] = utils.naive_to_aware(
            cleaned_data["move_date"])
        cleaned_data["facic_id"] = cleaned_data["evnt_id"].facic_id

        if cleaned_data["grp_id"] and not cleaned_data["new_grp_id"]:
            grp_id = cleaned_data["grp_id"]
        elif not cleaned_data["grp_id"] and cleaned_data["new_grp_id"]:
            grp_id = cleaned_data["new_grp_id"]
        if cleaned_data["grp_id"] and cleaned_data["new_grp_id"]:
            self.add_error('grp_id',
                           gettext("Can only add to one group at a time"))
            self.add_error('new_grp_id',
                           gettext("Can only add to one group at a time"))

        if not self.is_valid():
            return cleaned_data

        # fish into tank contx
        contx, entered = utils.enter_contx(self.cont,
                                           cleaned_data,
                                           True,
                                           grp_pk=grp_id.pk,
                                           return_contx=True)

        #fish to event
        utils.enter_anix(cleaned_data, grp_pk=grp_id.pk)

        # perc contx:
        if cleaned_data["perc_id"]:
            team_id, entered = utils.add_team_member(cleaned_data["perc_id"],
                                                     cleaned_data["evnt_id"],
                                                     return_team=True)
            utils.enter_anix(cleaned_data,
                             grp_pk=grp_id.pk,
                             team_pk=team_id.pk)

        # cnt:
        utils.enter_cnt(cleaned_data,
                        cleaned_data["num_fish"],
                        contx_pk=contx.pk)
示例#7
0
 def clean_data(self):
     if self.success:
         contx_df = DataFrame(self.data_dict)
         cnt_df = contx_df.groupby("start_contx_pk", as_index=False).sum()
         for row in cnt_df.to_dict('records'):
             if utils.nan_to_none(row["start_contx_pk"]):
                 cnt, cnt_entered = utils.enter_cnt(
                     self.cleaned_data,
                     0,
                     int(row["start_contx_pk"]),
                     cnt_code="Fish Removed from Container")
                 cnt.cnt = row[self.nfish_key]
                 cnt.save()
     super(GenericGrpParser, self).clean_data()
示例#8
0
    def row_parser(self, row):
        # need to: find the pair's group, link it to it's pairing, create a tray, and add the count.
        cleaned_data = self.cleaned_data
        row_date = utils.get_row_date(row)
        pair_list = utils.get_pair(row[self.cross_key],
                                   row["stok_id"],
                                   row[self.year_key],
                                   prog_grp=utils.nan_to_none(
                                       row.get(self.prog_key)),
                                   fail_on_not_found=True)
        if len(pair_list) == 1:
            pair_id = pair_list[0]
        else:
            raise Exception("Too many pairs found for row \n{}".format(row))

        anix_id = models.AniDetailXref.objects.filter(
            pair_id=pair_id,
            grp_id__isnull=False).select_related('grp_id').first()
        grp_id = anix_id.grp_id
        self.row_entered += utils.enter_anix(cleaned_data,
                                             grp_pk=grp_id.pk,
                                             return_sucess=True)

        tray_id = utils.create_tray(row["trof_id"], row[self.tray_key],
                                    row_date, cleaned_data)
        contx, contx_entered = utils.enter_contx(tray_id,
                                                 cleaned_data,
                                                 True,
                                                 grp_pk=grp_id.pk,
                                                 return_contx=True)
        self.row_entered += contx_entered

        if utils.nan_to_none(row.get(self.fecu_key)):
            cnt, cnt_entered = utils.enter_cnt(cleaned_data,
                                               row[self.fecu_key],
                                               contx_pk=contx.pk,
                                               cnt_code="Photo Count")
            self.row_entered += cnt_entered

        self.row_entered += utils.enter_bulk_grpd(anix_id.pk,
                                                  cleaned_data,
                                                  row_date,
                                                  comments=row.get(
                                                      self.comment_key))
示例#9
0
    def row_parser(self, row):
        cleaned_data = self.cleaned_data
        # get tray, group, and row date
        row_date = utils.get_row_date(row)

        tray_qs = models.Tray.objects.filter(trof_id=row["trof_id"],
                                             name=row[self.tray_key])
        tray_id = tray_qs.filter(
            Q(start_date__lte=row_date, end_date__gte=row_date)
            | Q(end_date__isnull=True)).get()
        pair_id = models.Pairing.objects.filter(
            cross=row[self.cross_key],
            end_date__isnull=True,
            indv_id__stok_id=row["stok_id"],
            start_date__year=row[self.year_key]).first()

        grp_id = utils.get_tray_group(pair_id, tray_id, row_date)

        # want to shift the hu move event, so that the counting math always works out.
        hu_move_date = row_date + timedelta(minutes=1)
        hu_cleaned_data = utils.create_new_evnt(cleaned_data, "Allocation",
                                                hu_move_date)
        hu_anix, data_entered = utils.enter_anix(hu_cleaned_data,
                                                 grp_pk=grp_id.pk)
        self.row_entered += data_entered
        hu_contx, data_entered = utils.enter_contx(tray_id,
                                                   hu_cleaned_data,
                                                   None,
                                                   grp_pk=grp_id.pk,
                                                   return_contx=True)
        self.row_entered += data_entered
        # record development
        dev_at_hu_transfer = grp_id.get_development(hu_move_date)
        utils.enter_grpd(hu_anix.pk,
                         hu_cleaned_data,
                         hu_move_date,
                         dev_at_hu_transfer,
                         None,
                         anidc_str="Development")
        self.row_entered += utils.enter_contx(row["trof_id"], cleaned_data)

        # HU Picks:
        self.row_entered += utils.enter_cnt(cleaned_data,
                                            row[self.loss_key],
                                            hu_contx.pk,
                                            cnt_code="HU Transfer Loss")[1]

        # generate new group, cup, and movement event:
        cont = None
        if utils.nan_to_none(row[self.end_tray_key]):
            trof_id = models.Trough.objects.filter(
                facic_id=cleaned_data["facic_id"],
                name=row[self.end_trof_key]).get()
            tray_qs = models.Tray.objects.filter(trof_id=trof_id,
                                                 name=row[self.tray_key])
            cont = tray_qs.filter(
                Q(start_date__lte=row_date, end_date__gte=row_date)
                | Q(end_date__isnull=True)).get()
        elif utils.nan_to_none(row[self.end_trof_key]):
            cont = models.Trough.objects.filter(
                facic_id=cleaned_data["facic_id"],
                name=row[self.end_trof_key]).get()
        elif utils.nan_to_none(row[self.heatl_key]):
            cont = utils.get_cont_from_dot(row[self.cont_key], cleaned_data,
                                           row_date)
        elif utils.nan_to_none(row[self.tank_key]):
            cont = models.Tank.objects.filter(
                facic_id=cleaned_data["facic_id"], name=row[self.tank_key])

        self.row_entered += utils.enter_contx(cont, cleaned_data)
        if not utils.y_n_to_bool(row[self.final_key]):
            # NEW GROUPS TAKEN FROM INITIAL
            out_cnt = utils.enter_cnt(cleaned_data,
                                      0,
                                      hu_contx.pk,
                                      cnt_code="Eggs Removed")[0]
            utils.enter_cnt_det(cleaned_data, out_cnt, row[self.cnt_key],
                                "Program Group Split", row[self.prog_key])

            indv, final_grp = cont.fish_in_cont(row_date)
            if not final_grp:
                final_grp = models.Group(
                    spec_id=grp_id.spec_id,
                    coll_id=grp_id.coll_id,
                    grp_year=grp_id.grp_year,
                    stok_id=grp_id.stok_id,
                    grp_valid=True,
                    created_by=cleaned_data["created_by"],
                    created_date=cleaned_data["created_date"],
                )
                try:
                    final_grp.clean()
                    final_grp.save()
                except (ValidationError, IntegrityError):
                    return None
            else:
                # MAIN GROUP GETTING MOVED
                final_grp = final_grp[0]
            final_grp_anix = utils.enter_anix(cleaned_data,
                                              grp_pk=final_grp.pk,
                                              return_anix=True)
            self.row_entered += utils.enter_anix(hu_cleaned_data,
                                                 grp_pk=final_grp.pk,
                                                 return_sucess=True)
            self.row_entered += utils.enter_bulk_grpd(
                final_grp_anix,
                cleaned_data,
                row_date,
                prnt_grp=grp_id,
                prog_grp=row.get(self.prog_key),
                comments=row.get(self.comment_key))
            self.row_entered += utils.enter_grpd(final_grp_anix.pk,
                                                 cleaned_data,
                                                 row_date,
                                                 dev_at_hu_transfer,
                                                 None,
                                                 anidc_str="Development")

            # create movement for the new group, create 2 contx's and 3 anix's
            # cup contx is contx used to link the positive counts
            cont_contx = utils.create_egg_movement_evnt(tray_id,
                                                        cont,
                                                        cleaned_data,
                                                        row_date,
                                                        final_grp.pk,
                                                        return_cup_contx=True)

            move_cleaned_data = cleaned_data.copy()
            move_cleaned_data["evnt_id"] = cont_contx.evnt_id
            cnt_contx = cont_contx
            cnt_contx.pk = None
            cnt_contx.tray_id = tray_id
            try:
                cnt_contx.save()
            except IntegrityError:
                cnt_contx = models.ContainerXRef.objects.filter(
                    pk=cont_contx.pk).get()
            self.row_entered += utils.enter_anix(move_cleaned_data,
                                                 grp_pk=final_grp.pk,
                                                 contx_pk=cnt_contx.pk,
                                                 return_sucess=True)
            # add the positive counts
            cnt = utils.enter_cnt(
                move_cleaned_data,
                row[self.cnt_key],
                cnt_contx.pk,
                cnt_code="Eggs Added",
            )[0]
            if utils.nan_to_none(self.weight_key):
                utils.enter_cnt_det(move_cleaned_data, cnt,
                                    row[self.weight_key], "Weight")
            utils.enter_cnt_det(move_cleaned_data, cnt, row[self.cnt_key],
                                "Program Group Split", row[self.prog_key])
        else:
            # Move main group to drawer, and add end date to tray:
            if cont:
                end_contx = utils.create_movement_evnt(tray_id,
                                                       cont,
                                                       cleaned_data,
                                                       row_date,
                                                       grp_pk=grp_id.pk,
                                                       return_end_contx=True)
                tray_id.end_date = row_date
                tray_id.save()
                end_cnt = utils.enter_cnt(cleaned_data,
                                          row[self.cnt_key],
                                          end_contx.pk,
                                          cnt_code="Egg Count")[0]
                utils.enter_cnt_det(cleaned_data, end_cnt,
                                    row[self.weight_key], "Weight")
            else:
                self.log_data += "\n Draw {} from {} not found".format(
                    cont, row[self.cont_key])

            # link cup to egg development event
            utils.enter_contx(cont, cleaned_data, None)
示例#10
0
 def test_simple_cnt(self):
     # test groups record a single count correctly
     cnt_val = randint(0, 100)
     utils.enter_cnt(self.cleaned_data, cnt_val, self.contx.pk, cnt_code="Fish in Container")
     self.assertEqual(self.grp.count_fish_in_group(), cnt_val)
示例#11
0
 def data_cleaner(self):
     for loc_pk, cnt_caught in self.loc_caught_dict.items():
         utils.enter_cnt(self.cleaned_data, cnt_caught, loc_pk=loc_pk, cnt_code="Fish Caught")
     for loc_pk, cnt_obs in self.loc_obs_dict.items():
         utils.enter_cnt(self.cleaned_data, cnt_obs, loc_pk=loc_pk, cnt_code="Fish Observed")
示例#12
0
    def row_parser(self, row):
        cleaned_data = self.cleaned_data
        row_datetime = utils.get_row_date(row)
        relc_id = None
        rive_id = self.river_dict[row[self.rive_key]]

        if not utils.nan_to_none(row.get(self.tank_key)) and utils.nan_to_none(row.get(self.fish_caught_key)):
            # make sure if fish are caught they are assigned a tank:
            raise Exception("All caught fish must be assigned a tank")

        if utils.nan_to_none(row.get(self.site_key)):
            relc_qs = models.ReleaseSiteCode.objects.filter(name__iexact=row[self.site_key])
            if len(relc_qs) == 1:
                relc_id = relc_qs.get()

        start_lat = utils.round_no_nan(row.get(self.lat_key), 6)
        start_lon = utils.round_no_nan(row.get(self.lon_key), 6)
        if not relc_id and not (start_lat and start_lon):
            raise Exception("Site code not found and lat-long not given for site on row")
        loc = models.Location(evnt_id_id=cleaned_data["evnt_id"].pk,
                              locc_id=self.locc_id,
                              rive_id=rive_id,
                              relc_id=relc_id,
                              loc_lat=start_lat,
                              loc_lon=start_lon,
                              end_lat=utils.round_no_nan(row.get(self.end_lat), 6),
                              end_lon=utils.round_no_nan(row.get(self.end_lon), 6),
                              loc_date=row_datetime,
                              comments=utils.nan_to_none(row.get(self.comment_key)),
                              created_by=cleaned_data["created_by"],
                              created_date=cleaned_data["created_date"],
                              )
        try:
            loc.clean()
            loc.save()
            self.row_entered = True
        except ValidationError:
            loc = models.Location.objects.filter(evnt_id=loc.evnt_id, locc_id=loc.locc_id,
                                                 rive_id=loc.rive_id, subr_id=loc.subr_id,
                                                 relc_id=loc.relc_id, loc_lat=loc.loc_lat,
                                                 loc_lon=loc.loc_lon, loc_date=loc.loc_date).get()
        self.loc = loc
        if row["grp_id"]:
            self.row_entered += utils.enter_anix(cleaned_data, loc_pk=loc.pk, grp_pk=row["grp_id"].pk,
                                                 return_sucess=True)
        if self.loc.loc_lon and self.loc.loc_lat and not self.loc.relc_id:
            self.log_data += "\nNo site found in db for Lat-Long ({}, {}) given on row: \n{}\n\n"\
                .format(self.loc.loc_lat, self.loc.loc_lon, row)

        if utils.nan_to_none(row["contx_id"]):
            self.row_entered += utils.enter_anix(cleaned_data, loc_pk=loc.pk, contx_pk=row["contx_id"].pk,
                                                 return_sucess=True)

        self.team_parser(row[self.crew_key], row, loc_id=loc)

        if utils.nan_to_none(row.get(self.temp_key)):
            self.row_entered += utils.enter_env(row[self.temp_key], row_datetime, cleaned_data, self.temp_envc_id,
                                                loc_id=loc)

        cnt_caught, cnt_entered = utils.enter_cnt(cleaned_data, cnt_value=row[self.fish_caught_key], loc_pk=loc.pk,
                                                  cnt_code="Fish Caught")
        self.row_entered += cnt_entered
        cnt_obs, cnt_entered = utils.enter_cnt(cleaned_data, cnt_value=row[self.fish_obs_key], loc_pk=loc.pk,
                                               cnt_code="Fish Observed")
        self.row_entered += cnt_entered

        if utils.nan_to_none(row.get(self.settings_key)):
            self.row_entered += utils.enter_locd(loc.pk, cleaned_data, row_datetime, row[self.settings_key],
                                                 self.settings_locdc_id.pk)
        if utils.nan_to_none(row.get(self.fishing_time_key)):
            self.row_entered += utils.enter_locd(loc.pk, cleaned_data, row_datetime, row[self.fishing_time_key],
                                                 self.fishing_time_locdc_id.pk)
        if utils.nan_to_none(row.get(self.voltage_key)):
            self.row_entered += utils.enter_locd(loc.pk, cleaned_data, row_datetime, row[self.voltage_key],
                                                 self.voltage_locdc_id.pk)
示例#13
0
    def row_parser(self, row):
        cleaned_data = self.cleaned_data
        row_date = row["datetime"].date()
        row_start_grp = utils.get_grp(row[self.rive_key],
                                      row["grp_year"],
                                      row["grp_coll"],
                                      row["start_tank_id"],
                                      row_date,
                                      prog_str=row.get(self.prio_key),
                                      mark_str=row.get(self.grp_mark_key),
                                      fail_on_not_found=True)[0]
        start_anix, self.row_entered = utils.enter_anix(
            cleaned_data, grp_pk=row_start_grp.pk)
        start_contx, contx_entered = utils.enter_contx(row["start_tank_id"],
                                                       cleaned_data,
                                                       None,
                                                       return_contx=True)
        self.row_entered += contx_entered

        whole_grp = utils.y_n_to_bool(row[self.abs_key])
        det_anix = start_anix
        row["start_contx_pk"] = None
        if not whole_grp:
            row["start_contx_pk"] = start_contx.pk

        if utils.nan_to_none(row["end_tank_id"]):
            # 4 possible cases here: group in tank or not and whole group move or not:
            row_end_grp_list = utils.get_grp(row[self.rive_key],
                                             row["grp_year"],
                                             row["grp_coll"],
                                             row["end_tank_id"],
                                             row_date,
                                             prog_str=row[self.prio_key],
                                             mark_str=row[self.grp_mark_key])
            row_end_grp = None
            if not whole_grp and not row_end_grp_list:
                # splitting fish group, create end group:
                row_end_grp = copy.deepcopy(row_start_grp)
                row_end_grp.pk = None
                row_end_grp.id = None
                row_end_grp.save()
                end_grp_anix, anix_entered = utils.enter_anix(
                    cleaned_data, grp_pk=row_end_grp.pk)
                self.row_entered = anix_entered

                self.row_entered += utils.enter_bulk_grpd(
                    end_grp_anix.pk,
                    cleaned_data,
                    row_date,
                    prog_grp=row.get(self.prio_key),
                    mark=row.get(self.mark_key))
            elif not whole_grp:
                # splitting fish group, merging to exsisting end group
                row_end_grp = row_end_grp_list[0]

            if row_end_grp:
                move_contx = utils.create_movement_evnt(row["start_tank_id"],
                                                        row["end_tank_id"],
                                                        cleaned_data,
                                                        row_date,
                                                        grp_pk=row_end_grp.pk,
                                                        return_end_contx=True)
                end_grp_anix, anix_entered = utils.enter_anix(
                    cleaned_data, grp_pk=row_end_grp.pk)
                self.row_entered += anix_entered
                self.row_entered += utils.enter_grpd(end_grp_anix.pk,
                                                     cleaned_data,
                                                     row_date,
                                                     None,
                                                     self.prnt_grp_anidc_id.pk,
                                                     frm_grp_id=row_start_grp)
                cnt, cnt_entered = utils.enter_cnt(cleaned_data,
                                                   row[self.nfish_key],
                                                   move_contx.pk)
                self.row_entered = cnt_entered

                # record details on end tank group
                det_anix = end_grp_anix

            else:
                # move all the fish (whole group, merge to fish at destination if needed)
                move_contx = utils.create_movement_evnt(
                    row["start_tank_id"],
                    row["end_tank_id"],
                    cleaned_data,
                    row_date,
                    grp_pk=row_start_grp.pk,
                    return_end_contx=True)
                cnt, cnt_entered = utils.enter_cnt(cleaned_data,
                                                   row[self.nfish_key],
                                                   move_contx.pk,
                                                   cnt_code="Fish Count")
                self.row_entered = cnt_entered
        else:
            if utils.nan_to_none(row[self.nfish_key]):
                cnt, cnt_entered = utils.enter_cnt(cleaned_data,
                                                   row[self.nfish_key],
                                                   start_contx.pk,
                                                   cnt_code="Fish Count")
                self.row_entered = cnt_entered

        # add details to det_anix:
        self.row_entered += utils.enter_bulk_grpd(
            det_anix.pk,
            cleaned_data,
            row_date,
            vaccinated=row.get(self.vax_key),
            mark=row.get(self.mark_key),
            lifestage=row.get(self.lifestage_key),
            comments=row.get(self.comment_key))

        self.row_entered += utils.parse_extra_cols(row,
                                                   self.cleaned_data,
                                                   det_anix,
                                                   grp=True)
示例#14
0
    def data_preper(self):
        cleaned_data = self.cleaned_data
        self.sampc_id = models.SampleCode.objects.filter(
            name="Individual Sample").get()
        self.prnt_grp_anidc_id = models.AnimalDetCode.objects.filter(
            name="Parent Group").get()
        self.prog_grp_anidc_id = models.AnimalDetCode.objects.filter(
            name="Program Group").get()
        self.sex_anidc_id = models.AnimalDetCode.objects.filter(
            name="Gender").get()
        self.len_anidc_id = models.AnimalDetCode.objects.filter(
            name="Length").get()
        self.weight_anidc_id = models.AnimalDetCode.objects.filter(
            name="Weight").get()
        self.vial_anidc_id = models.AnimalDetCode.objects.filter(
            name="Vial").get()
        self.envelope_anidc_id = models.AnimalDetCode.objects.filter(
            name="Scale Envelope").get()
        self.ani_health_anidc_id = models.AnimalDetCode.objects.filter(
            name="Animal Health").get()
        self.anidc_ufid_id = models.AnimalDetCode.objects.filter(
            name="UFID").get()
        self.vax_anidc_id = models.AnimalDetCode.objects.filter(
            name="Vaccination").get()
        self.mark_anidc_id = models.AnimalDetCode.objects.filter(
            name="Mark").get()
        self.lifestage_anidc_id = models.AnimalDetCode.objects.filter(
            name="Lifestage").get()
        self.comment_anidc_id = models.AnimalDetCode.objects.filter(
            name="Comment").get()

        # The following steps are to set additional columns on each row to facilitate parsing.
        # In particular,  columns set will be: "datetime", "grp_year", "grp_coll", "start_tank_id",
        # "end_tank_id", "grp_key", "end_grp_key".
        # The two grp_keys will link to dictionaries of the groups, which are also set below

        # set date
        self.data = utils.set_row_datetime(self.data)
        # split year-coll
        self.data["grp_year"] = self.data.apply(
            lambda row: utils.year_coll_splitter(row[self.yr_coll_key])[0],
            axis=1)
        self.data["grp_coll"] = self.data.apply(
            lambda row: utils.year_coll_splitter(row[self.yr_coll_key])[1],
            axis=1)

        # set start and end tank columns:
        self.data = utils.set_row_tank(self.data,
                                       cleaned_data,
                                       self.start_tank_key,
                                       col_name="start_tank_id")
        self.data = utils.set_row_tank(self.data,
                                       cleaned_data,
                                       self.end_tank_key,
                                       col_name="end_tank_id")

        # set the dict keys for groups, use astype(str) to handle anything that might be a nan.
        self.data, self.start_grp_dict = utils.set_row_grp(
            self.data,
            self.rive_key,
            self.yr_coll_key,
            self.prio_key,
            "start_tank_id",
            "datetime",
            self.grp_mark_key,
            grp_col_name="start_grp_id",
            return_dict=True)
        for item, grp in self.start_grp_dict.items():
            utils.enter_anix(cleaned_data, grp_pk=grp.pk)

        self.data["end_grp_key"] = self.data[self.rive_key] + self.data[self.yr_coll_key] + \
                                   self.data[self.end_tank_key].astype(str) + self.data[self.prio_key].astype(str) + \
                                   self.data["datetime"].astype(str) + self.data[self.grp_mark_key].astype(str)

        # create the end group dict and create, movement event, groups, counts, contxs, etc. necesarry
        end_grp_data = self.data.groupby([
            self.rive_key, "grp_year", "grp_coll", "end_tank_id",
            "start_tank_id", self.prio_key, "datetime", self.grp_mark_key,
            "grp_key", "end_grp_key"
        ],
                                         dropna=False,
                                         sort=False).size().reset_index()
        for row in end_grp_data.to_dict('records'):
            # check if end tank is set, otherwise, skip this step
            if not utils.nan_to_none(row["end_tank_id"]):
                self.end_grp_dict[row["end_grp_key"]] = None
                continue
            grps = utils.get_grp(row[self.rive_key],
                                 row["grp_year"],
                                 row["grp_coll"],
                                 row["end_tank_id"],
                                 at_date=row["datetime"],
                                 prog_str=row[self.prio_key],
                                 mark_str=row[self.grp_mark_key])
            start_grp_id = self.start_grp_dict[row["grp_key"]]
            start_contx, contx_entered = utils.enter_contx(
                row["start_tank_id"],
                cleaned_data,
                None,
                grp_pk=start_grp_id.pk,
                return_contx=True)
            self.row_entered += utils.enter_cnt(
                cleaned_data,
                sum(end_grp_data[end_grp_data["grp_key"] == row["grp_key"]]
                    [0]),
                start_contx.pk,
                cnt_code="Fish Removed from Container")[1]

            if len(grps) > 0:
                end_grp_id = grps[0]
                self.end_grp_dict[row["end_grp_key"]] = grps[0]
            else:
                end_grp_id = copy.deepcopy(start_grp_id)
                end_grp_id.pk = None
                end_grp_id.save()
                self.end_grp_dict[row["end_grp_key"]] = end_grp_id

            if end_grp_id.pk != start_grp_id.pk:
                grp_anix = utils.enter_anix(cleaned_data,
                                            grp_pk=end_grp_id.pk,
                                            return_anix=True)
                utils.enter_grpd(grp_anix.pk,
                                 cleaned_data,
                                 row["datetime"],
                                 None,
                                 self.prnt_grp_anidc_id.pk,
                                 frm_grp_id=start_grp_id)
                if utils.nan_to_none(row[self.prio_key]):
                    utils.enter_grpd(grp_anix.pk, cleaned_data,
                                     row["datetime"], row[self.prio_key],
                                     self.prog_grp_anidc_id.pk,
                                     row[self.prio_key])
                if utils.nan_to_none(row[self.grp_mark_key]):
                    utils.enter_grpd(grp_anix.pk, cleaned_data,
                                     row["datetime"], row[self.grp_mark_key],
                                     self.mark_anidc_id.pk,
                                     row[self.grp_mark_key])
                end_contx = utils.create_movement_evnt(row["start_tank_id"],
                                                       row["end_tank_id"],
                                                       cleaned_data,
                                                       row["datetime"],
                                                       grp_pk=end_grp_id.pk,
                                                       return_end_contx=True)
                if end_contx:
                    self.row_entered += utils.enter_cnt(
                        cleaned_data, row[0], end_contx.pk)[1]
        self.data_dict = self.data.to_dict("records")