def test_fishbelt_ingest(
db_setup,
fishbelt_file,
project1,
profile1,
base_project,
all_test_fish_attributes,
belt_transect_width_5m,
fish_size_bin_1,
tide2,
current2,
site1,
management1,
fish_species3,
reef_slope1,
relative_depth1,
visibility1,
):
new_records, output = utils.ingest(
protocol=FISHBELT_PROTOCOL,
datafile=fishbelt_file,
project_id=project1.pk,
profile_id=profile1.pk,
request=None,
dry_run=False,
clear_existing=False,
bulk_validation=False,
bulk_submission=False,
validation_suppressants=None,
serializer_class=None,
)
assert new_records is not None and len(new_records) == 1
new_record = new_records[0]
sample_event = new_record.data.get("sample_event")
fishbelt_transect = new_record.data.get("fishbelt_transect")
observations = new_record.data["obs_belt_fishes"]
assert new_record.project == project1
assert new_record.profile == profile1
print(fishbelt_transect)
print(new_record)
assert fishbelt_transect.get("current") == str(current2.id)
assert fishbelt_transect.get("tide") == str(tide2.id)
assert fishbelt_transect.get("width") == str(belt_transect_width_5m.id)
assert fishbelt_transect.get("depth") == 10
assert fishbelt_transect.get("reef_slope") == str(reef_slope1.id)
assert fishbelt_transect.get("visibility") == str(visibility1.id)
assert fishbelt_transect.get("relative_depth") == str(relative_depth1.id)
assert sample_event.get("site") == str(site1.id)
assert sample_event.get("management") == str(management1.id)
assert str(sample_event.get("sample_date")) == "2015-12-03"
assert len(observations) == 6
assert observations[2].get("size") == 20
assert observations[2].get("count") == 1
assert observations[2].get("fish_attribute") == str(fish_species3.id)
def test_benthicpit_ingest(
db_setup,
benthicpit_file,
project1,
profile1,
base_project,
all_test_benthic_attributes,
tide1,
relative_depth1,
benthic_attribute_2a1,
growth_form1,
site1,
management1,
):
new_records, output = utils.ingest(
protocol=BENTHICPIT_PROTOCOL,
datafile=benthicpit_file,
project_id=project1.pk,
profile_id=profile1.pk,
request=None,
dry_run=False,
clear_existing=False,
bulk_validation=False,
bulk_submission=False,
validation_suppressants=None,
serializer_class=None,
)
assert new_records is not None and len(new_records) == 2
new_record = new_records[1]
sample_event = new_record.data.get("sample_event")
benthic_transect = new_record.data.get("benthic_transect")
observations = new_record.data["obs_benthic_pits"]
assert new_record.project == project1
assert new_record.profile == profile1
assert benthic_transect.get("depth") == 8.0
assert benthic_transect.get("tide") == str(tide1.id)
assert benthic_transect.get("relative_depth") == str(relative_depth1.id)
assert sample_event.get("site") == str(site1.id)
assert sample_event.get("management") == str(management1.id)
assert str(sample_event.get("sample_date")) == "2011-03-31"
assert len(observations) == 3
assert observations[0].get("attribute") == str(benthic_attribute_2a1.id)
assert observations[0].get("growth_form") == str(growth_form1.id)
def test_habitatcomplexity_ingest(
db_setup,
habitatcomplexity_file,
project1,
profile1,
base_project,
all_test_benthic_attributes,
site1,
site2,
management1,
management2,
relative_depth1,
relative_depth2,
reef_slope2,
reef_slope3,
visibility1,
visibility2,
current1,
current2,
tide1,
tide2,
habitat_complexity_score1,
habitat_complexity_score2,
habitat_complexity_score3,
):
new_records, output = utils.ingest(
protocol=HABITATCOMPLEXITY_PROTOCOL,
datafile=habitatcomplexity_file,
project_id=project1.pk,
profile_id=profile1.pk,
request=None,
dry_run=False,
clear_existing=False,
bulk_validation=False,
bulk_submission=False,
validation_suppressants=None,
serializer_class=None,
)
assert new_records is not None and len(new_records) == 2
new_record = new_records[0]
sample_event = new_record.data.get("sample_event")
benthic_transect = new_record.data.get("benthic_transect")
observations = new_record.data["obs_habitat_complexities"]
assert new_record.project == project1
assert new_record.profile == profile1
assert str(new_record.data.get("observers")[0]["profile"]) == str(
profile1.id)
assert benthic_transect.get("depth") == 5.0
assert benthic_transect.get("tide") == str(tide2.id)
assert benthic_transect.get("relative_depth") == str(relative_depth2.id)
assert benthic_transect.get("sample_time") == datetime.time(8, 0)
assert benthic_transect.get("number") == 2
assert benthic_transect.get("reef_slope") == str(reef_slope3.id)
assert sample_event.get("site") == str(site2.id)
assert sample_event.get("management") == str(management2.id)
assert str(sample_event.get("sample_date")) == "2020-02-22"
assert len(observations) == 5
assert observations[0].get("interval") == 0.5
assert observations[0].get("score") == str(habitat_complexity_score3.id)
def test_benthiclit_ingest(
db_setup,
benthiclit_file,
project1,
profile1,
base_project,
all_test_benthic_attributes,
site1,
site2,
management1,
management2,
relative_depth1,
relative_depth2,
reef_slope2,
reef_slope3,
visibility1,
visibility2,
current1,
current2,
tide1,
tide2,
benthic_attribute_1a,
benthic_attribute_2a1,
benthic_attribute_2b1,
benthic_attribute_3,
benthic_attribute_4,
benthic_attribute_2b,
growth_form3,
growth_form4,
):
new_records, output = utils.ingest(
protocol=BENTHICLIT_PROTOCOL,
datafile=benthiclit_file,
project_id=project1.pk,
profile_id=profile1.pk,
request=None,
dry_run=False,
clear_existing=False,
bulk_validation=False,
bulk_submission=False,
validation_suppressants=None,
serializer_class=None,
)
assert new_records is not None and len(new_records) == 2
new_record = new_records[1]
sample_event = new_record.data.get("sample_event")
benthic_transect = new_record.data.get("benthic_transect")
observations = new_record.data["obs_benthic_lits"]
assert new_record.project == project1
assert new_record.profile == profile1
assert str(new_record.data.get("observers")[0]["profile"]) == str(
profile1.id)
assert benthic_transect.get("depth") == 5.0
assert benthic_transect.get("tide") == str(tide2.id)
assert benthic_transect.get("relative_depth") == str(relative_depth2.id)
assert benthic_transect.get("sample_time") == datetime.time(8, 0)
assert benthic_transect.get("number") == 2
assert benthic_transect.get("reef_slope") == str(reef_slope3.id)
assert sample_event.get("site") == str(site2.id)
assert sample_event.get("management") == str(management2.id)
assert str(sample_event.get("sample_date")) == "2020-02-22"
assert len(observations) == 5
assert observations[0].get("attribute") == str(benthic_attribute_3.id)
assert observations[0].get("growth_form") is None
assert observations[0].get("length") == 322
def test_bleaching_ingest(
db_setup,
bleaching_file,
project1,
profile1,
base_project,
all_test_benthic_attributes,
tide1,
visibility1,
current3,
relative_depth1,
benthic_attribute_2,
benthic_attribute_2a1,
growth_form4,
site1,
management1,
):
new_records, output = utils.ingest(
protocol=BLEACHINGQC_PROTOCOL,
datafile=bleaching_file,
project_id=project1.pk,
profile_id=profile1.pk,
request=None,
dry_run=False,
clear_existing=False,
bulk_validation=False,
bulk_submission=False,
validation_suppressants=None,
serializer_class=None,
)
assert new_records is not None and len(new_records) == 1
new_record = new_records[0]
sample_event = new_record.data.get("sample_event")
quadrat_collection = new_record.data.get("quadrat_collection")
obs_colonies_bleached = new_record.data["obs_colonies_bleached"]
print(new_record.data)
assert new_record.project == project1
assert new_record.profile == profile1
assert quadrat_collection.get("tide") == str(tide1.id)
assert quadrat_collection.get("depth") == 1.0
assert quadrat_collection.get("quadrat_size") == 1
assert quadrat_collection.get("visibility") == str(visibility1.id)
assert quadrat_collection.get("current") == str(current3.id)
assert quadrat_collection.get("relative_depth") == str(relative_depth1.id)
assert sample_event.get("site") == str(site1.id)
assert sample_event.get("management") == str(management1.id)
assert str(sample_event.get("sample_date")) == "2020-05-20"
assert len(obs_colonies_bleached) == 2
assert obs_colonies_bleached[0]["attribute"] == str(
benthic_attribute_2a1.id)
assert obs_colonies_bleached[0]["growth_form"] == str(growth_form4.id)
assert obs_colonies_bleached[1]["attribute"] == str(benthic_attribute_2.id)
obs_quadrat_benthic_percent = new_record.data[
"obs_quadrat_benthic_percent"]
assert len(obs_quadrat_benthic_percent) == 4
assert obs_quadrat_benthic_percent[3]["quadrat_number"] == 4
assert obs_quadrat_benthic_percent[3]["percent_hard"] == 87
assert obs_quadrat_benthic_percent[3]["percent_soft"] == 0
assert obs_quadrat_benthic_percent[3]["percent_algae"] == 13
def handle(
self,
datafile,
project,
profile,
protocol,
dry_run,
clear_existing,
*args,
**options
):
datafile = datafile[0]
project = project[0]
profile = profile[0]
verbosity = options["verbosity"]
is_validate = options.get("validate")
validate_config = None
try:
config = options.get("validate_config")
if config:
validate_config = json.loads(config)
except (ValueError, TypeError):
self.stderr.write("validate_config is invalid")
sys.exit(1)
is_submit = options.get("submit")
if protocol is None:
raise NotImplementedError()
try:
with transaction.atomic():
sid = transaction.savepoint()
try:
records, ingest_output = ingest(
protocol=protocol,
datafile=datafile,
project_id=project,
profile_id=profile,
request=None,
dry_run=dry_run,
clear_existing=clear_existing,
bulk_validation=is_validate,
bulk_submission=is_submit,
validation_suppressants=validate_config,
serializer_class=CollectRecordSerializer
)
except InvalidSchema as schema_error:
missing_required_fields = schema_error.errors
transaction.savepoint_rollback(sid)
self.stderr.write(f"Missing required fields: {', '.join(missing_required_fields)}")
sys.exit(1)
transaction.savepoint_commit(sid)
except Exception as err:
transaction.savepoint_rollback(sid)
self.stderr.write(str(err))
sys.exit(1)
if "errors" in ingest_output:
errors = ingest_output["errors"]
if verbosity > 0:
for err in errors:
self.stdout.write(json.dumps(err))
if verbosity > 1:
num_errors = len(errors)
msg = "{} {} errors".format(
num_errors, pluralize(num_errors, "record has,records have")
)
self.stderr.write(msg)
sys.exit(1)
if dry_run:
msg = "[DRYRUN] {} records would have been created.".format(len(records))
else:
msg = "{} records created.".format(len(records))
if verbosity > 1:
self.stdout.write(self.style.SUCCESS(msg))
if "validate" in ingest_output:
validation_results = ingest_output["validate"]
validation_oks, validation_warns, validation_errors = self._validation_summary(validation_results.values())
if verbosity > 0:
pass
if verbosity > 1:
self.stdout.write("\nCollect record validation results:")
self.stdout.write(self.style.SUCCESS(f"Valid: {validation_oks}"))
self.stdout.write(self.style.WARNING(f"Warnings: {validation_warns}"))
self.stdout.write(self.style.ERROR(f"Errors: {validation_errors}"))
if "submit" in ingest_output:
submission_results = ingest_output["submit"]
submission_oks, submission_warns, submission_errors = self._validation_summary(submission_results.values())
if verbosity > 0:
pass
if verbosity > 1:
self.stdout.write("\nCollect record submission results:")
self.stdout.write(self.style.SUCCESS(f"Submitted: {validation_oks}"))
self.stdout.write(self.style.WARNING(f"Warnings: {validation_warns}"))
self.stdout.write(self.style.ERROR(f"Errors: {validation_errors}"))
sys.exit(0)