def pull(self):
"""
Pull all observations from iNaturalist.
If we don't have an observation with that iNaturalist ID, create one (check the vespawatch-evidence field to
determine whether we should create a Nest or an Individual)
If we do have an observation with that iNaturalist ID, update it.
"""
self.w("\n3. Pull all observations from iNaturalist")
observations = get_all_observations(
params={'project_id': settings.VESPAWATCH_PROJECT_ID})
pulled_inat_ids = []
for inat_observation_data in observations:
pulled_inat_ids.append(inat_observation_data['id'])
local_obs = get_local_observation_with_inaturalist_id(
inat_observation_data['id'])
if local_obs is None:
# This is a new one. Check vespawatch-evidence and create a nest or individual
self.w(
f"iNaturalist observation with ID #{inat_observation_data['id']} is not yet known, we'll create it locally...",
ending='')
create_observation_from_inat_data(
inat_observation_data
) #TODO: check all the required fields are set
self.w("OK")
else:
# We already have an observation for this id. Update it
self.w(
f'updating observation {type(local_obs).__name__} {local_obs.pk}'
)
local_obs.update_from_inat_data(inat_observation_data)
return pulled_inat_ids
def pull(self):
"""
Pull all observations from iNaturalist.
If we don't have an observation with that iNaturalist ID, create one (check the vespawatch-evidence field to
determine whether we should create a Nest or an Individual)
If we do have an observation with that iNaturalist ID, update it.
"""
self.w(
"\n3. Pull all observations from iNaturalist (based on the project)"
)
observations = get_all_observations(
params={'project_id': settings.VESPAWATCH_PROJECT_ID})
# The most recent observations are the more important one.
# Let's start with those in case the sync process is interrupted (that unfortunately happens)
observations = sorted(observations,
key=lambda x: x['id'],
reverse=True)
pulled_inat_ids = []
for inat_observation_data in observations:
pulled_inat_ids.append(inat_observation_data['id'])
local_obs = get_local_observation_with_inaturalist_id(
inat_observation_data['id'])
if local_obs is None:
# This is a new one. Check vespawatch-evidence and create a nest or individual
self.w(
f"iNaturalist observation with ID #{inat_observation_data['id']} is not yet known, we'll create it locally...",
ending='')
create_observation_from_inat_data(
inat_observation_data
) #TODO: check all the required fields are set
self.w("OK")
else:
# We already have an observation for this id. Update it
self.w(
f'updating observation {type(local_obs).__name__} {local_obs.pk}'
)
local_obs.update_from_inat_data(inat_observation_data)
return pulled_inat_ids
def test_get_all_observations(sleep, requests_mock):
# Make response appear as though there are more pages
page_1 = load_sample_data("get_observations_node_page1.json")
page_1["total_results"] = PER_PAGE_RESULTS + 1
page_2 = load_sample_data("get_observations_node_page2.json")
requests_mock.get(
urljoin(INAT_NODE_API_BASE_URL, "observations"),
[
{
"json": page_1,
"status_code": 200
},
{
"json": page_2,
"status_code": 200
},
],
)
observations = get_all_observations(id=[57754375, 57707611])
assert type(observations) is list
assert len(observations) == 2
def handle(self, *args, **options):
self.w(
f"Will import observations from project #{settings.VESPAWATCH_PROJECT_ID}..."
)
observations = get_all_observations(params=PULL_CRITERIA)
total_count = len(observations)
success_count, from_us_count, from_inat_total_count, from_inat_nest_count = 0, 0, 0, 0
self.w(
"Step 1: Will remove from our database all observations that originates from iNaturalist (they'll be recreated right after)...",
ending="")
Individual.from_inat_objects.all().delete()
Nest.from_inat_objects.all().delete()
self.w(self.style.SUCCESS("OK"))
self.w(
f"Step 2: Will parse the {total_count} matching observations received from iNaturalist."
)
for inat_observation_data in observations:
inat_id = inat_observation_data['id']
self.w(f"Now processing iNaturalist observation #{inat_id}...",
ending="")
# TODO: it happened once during developpement that inat search (via Node API) returned...
# TODO: ...a recently deleted occurrence, hence an ObservationNotFound raised in...
# TODO: ...inat_observation_comes_from_vespawatch()
if inat_observation_comes_from_vespawatch(
inat_observation_data['id']):
from_us_count = from_us_count + 1
self.w("Observation initially comes from Vespa-Watch. ",
ending="")
# The only thing we do is updating the identification, if needed.
try:
r = update_loc_obs_taxon_according_to_inat(
inat_observation_data)
if r == 'no_community_id':
self.w(
"There's no community ID, so we keep what we have. ",
ending="")
elif r == 'matching_community_id':
self.w(
"The community ID agree with us, so we keep what we have. ",
ending="")
elif r == 'updated':
self.w("Database updated to match the community ID! ",
ending="")
success_count = success_count + 1
self.w(" ")
except ObjectDoesNotExist:
self.w(
self.style.ERROR(
"Error: can't find a local observation!!! "))
except TaxonMatchError:
self.w(
self.style.WARNING(
"Error: We don't understand the community taxon id, so we ignore it "
))
else:
from_inat_total_count = from_inat_total_count + 1
self.w(
"Observation initially comes from a regular iNaturalist user. ",
ending="")
# All those observations have been dropped before: recreate
self.w("Creating a local observation for it. ", ending="")
try:
r = create_observation_from_inat_data(
inat_observation_data)
if r.__class__ == Nest:
from_inat_nest_count = from_inat_nest_count + 1
self.w(self.style.SUCCESS("OK"))
success_count = success_count + 1
except TaxonMatchError:
if inat_observation_data['taxon']['rank'] == 'genus':
self.w(
self.style.WARNING(
"Observation at the Genus level, skipping."))
else:
self.w(
self.style.ERROR(
"Error: cannot match taxon, skipping: " +
str(inat_observation_data)))
except ParseDateError:
self.w(
self.style.ERROR(
"Error: cannot parse date, skipping: " +
str(inat_observation_data)))
self.w("DONE. Stats:")
self.w(f"{total_count} observations processed (total).")
self.w(
f"{success_count} were successful, {total_count - success_count} had errors."
)
self.w(
f"{from_us_count} were from the Vespa-Watch app, {from_inat_total_count} were regular iNaturalist "
f"observations, including {from_inat_nest_count} nest(s).")