def handle_uploaded_file(f, mode, using):
import csv
more_results = []
assert using in ['using-raw-assets', 'using-assets']
if f.size > 25000000:
raise ValueError(
"handle_uploaded_file hasn't implemented saving the file for reading/parsing yet."
)
#for chunk in f.chunks(): # "Looping over chunks() instead of using read()
# # ensures that large files don't overwhelm your system's memory.
# destination.write(chunk)
else:
decoded_file = f.read().decode('utf-8').splitlines()
# Validate the file
if using == 'using-assets':
reader = csv.DictReader(decoded_file)
for row in reader:
if 'asset_id' in row and row['asset_id'] not in ['']:
try:
assert row['asset_id'] == ''
except AssertionError:
more_results.append(
f"id should be blank but is actually {row['asset_id']}. ASSET UPDATER FAILURE."
)
return more_results
if 'id' in row:
# Verify that this matches an Asset in the database.
raw_id = row['id']
if raw_id not in ['']:
try:
primary_asset_iterator = Asset.objects.filter(
id=raw_id)
assert len(
primary_asset_iterator
) == 1 # To ensure it exists in the database.
except AssertionError:
more_results.append(
f"Failed to find Asset with id == {raw_id}. ASSET UPDATER FAILURE."
)
return more_results
if 'ids_to_merge' in row:
# Verify that these match Assets in the database.
ids_to_merge = row['ids_to_merge']
if ids_to_merge not in ['']:
try:
asset_ids = [
int(i) for i in ids_to_merge.split('+')
]
assets_iterator = Asset.objects.filter(
id__in=asset_ids)
assert len(assets_iterator) == len(
asset_ids
) # To ensure they all exist in the database.
except AssertionError:
more_results.append(
f"Failed to find Assets with ids == {asset_ids}. ASSET UPDATER FAILURE."
)
return more_results
if 'location_id' in row and row['location_id'] not in [
'', None
]:
try:
location = Location.objects.get(pk=row['location_id'])
except Location.DoesNotExist:
more_results.append(
f"Failed to find Location with id == {row['location_id']}. ASSET UPDATER FAILURE."
)
return more_results
if 'organization_id' in row and row['organization_id'] not in [
'', None
]:
try:
organization = Organization.objects.get(
pk=row['organization_id'])
except Location.DoesNotExist:
more_results.append(
f"Failed to find Organization with id == {row['location_id']}. ASSET UPDATER FAILURE."
)
return more_results
asset_ids_to_sync_to_carto = []
reader = csv.DictReader(decoded_file)
for row in reader:
created_new_asset = False
# Process the 'id' field
raw_id = row['id']
if using == 'using-raw-assets':
primary_raw_asset_iterator = RawAsset.objects.filter(id=raw_id)
assert len(primary_raw_asset_iterator
) == 1 # To ensure it exists in the database.
primary_raw_asset = primary_raw_asset_iterator[0]
elif using == 'using-assets':
primary_asset_iterator = Asset.objects.filter(id=raw_id)
assert len(primary_asset_iterator
) == 1 # To ensure it exists in the database.
destination_asset = primary_asset_iterator[0] # Note that here
# the primary asset is also the destination asset.
# Process the 'asset_id' field
if using == 'using-raw-assets':
asset_id = row['asset_id']
if asset_id in ['', None]:
created_new_asset = True
destination_asset = Asset()
more_results.append(
f"A new Asset {'would' if mode == 'validate' else 'will'} be created."
)
else:
destination_asset_iterator = Asset.objects.filter(
id=asset_id)
assert len(
destination_asset_iterator
) == 1 # To ensure there is exactly one in the database.
destination_asset = destination_asset_iterator[0]
# Process the 'ids_to_merge' field
ids_to_merge = row['ids_to_merge']
if using == 'using-raw-assets':
if ids_to_merge == '':
continue # Skip rows with no ids to merge.
raw_ids = [int(i) for i in ids_to_merge.split('+')]
raw_assets_iterator = RawAsset.objects.filter(id__in=raw_ids)
assert len(raw_assets_iterator) == len(
raw_ids) # To ensure they all exist in the database.
raw_assets = list(raw_assets_iterator)
for raw_asset in raw_assets:
raw_asset.asset = destination_asset
if len(raw_assets) == 1:
if created_new_asset:
summary = f"{'Validating this process: ' if mode == 'validate' else ''}Creating a new Asset, "
else:
summary = f"{'Validating this process: ' if mode == 'validate' else ''}Editing the Asset with id = {asset_id}, previously named {destination_asset.name}, "
summary += f"and linking it to RawAsset with id = {raw_assets[0].id} and name = {raw_assets[0].name}."
else:
summary = f"{'Validating this process: ' if mode == 'validate' else ''}Merging RawAssets with ids = {', '.join([str(r.id) for r in raw_assets])} and names = {', '.join([r.name for r in raw_assets])} "
if created_new_asset:
summary += f" to a new Asset with name {row.get('name', '(No name given)')}."
else:
summary += f" to Asset with id = {asset_id}, previously named {destination_asset.name}."
more_results.append(summary)
elif using == 'using-assets':
# When merging Assets, the Asset that is not the destination
# asset should be delisted.
if mode == 'update':
if ids_to_merge == '':
destination_asset.do_not_display = True
destination_asset._change_reason = f'Asset Updater: Delisting Asset'
destination_asset.save(override_carto_sync=True)
asset_ids_to_sync_to_carto.append(destination_asset.id)
s = f"Delisting {destination_asset.name}."
more_results.append(s)
continue # Skip rows with no ids to merge.
asset_ids = [int(i) for i in ids_to_merge.split('+')]
assert destination_asset.id in asset_ids
assets_iterator = Asset.objects.filter(id__in=asset_ids)
assert len(assets_iterator) == len(
asset_ids) # To ensure they all exist in the database.
s = f"{'Validating this process: ' if mode == 'validate' else ''}Editing the Asset with id = {destination_asset.id}, previously named {destination_asset.name}."
more_results.append(s)
if len(assets_iterator) > 1:
s = f"Delisting extra Assets (from the list {ids_to_merge}) and assigning corresponding RawAssets to the destination Asset."
more_results.append(s)
for asset in assets_iterator:
if destination_asset.id is not None and asset.id != destination_asset.id:
asset.do_not_display = True # These Assets could be deleted (rather than delisted)
# AFTER reassinging their RawAssets.
asset._change_reason = f'Asset Updater: Delisting Asset'
asset.save()
asset_ids_to_sync_to_carto.append(asset.id)
# Iterate over raw assets of this asset and point them to destination_asset.
for raw_asset in asset.rawasset_set.all():
raw_asset.asset = destination_asset
raw_asset._change_reason = f'Asset Updater: Linking RawAsset to different Asset because of Asset merge'
raw_asset.save(
) # This saving couldn't be done below
# because there can be multiple sets of RawAssets. They'd
# all have to be collected into raw_assets to do it below.
else:
if '+' in ids_to_merge:
s = f"Extra Assets (from the list {ids_to_merge}) would be delisted and corresponding RawAssets would be assigned to the destination Asset."
more_results.append(s)
elif ids_to_merge == '':
s = f"{destination_asset.name} would be delisted."
more_results.append(s)
### At this point the fields that differentiate Asset-based Asset updates from
### RawAsset-based Asset updates have been processed.
### What comes out of this stage is destination_asset and raw_assets.
destination_asset, location, organization, more_results, error = modify_destination_asset(
mode, row, destination_asset, created_new_asset, more_results)
if error:
return more_results
if mode == 'update':
more_results.append(
f"Updating associated Asset, RawAsset, Location, and Organization instances. (This may leave some orphaned.)\n"
)
more_results.append(
f' <a href="https://assets.wprdc.org/api/dev/assets/assets/{destination_asset.id}/" target="_blank">Updated Asset</a>\n'
)
change_reason = f'Asset Updater: {"Creating new " if created_new_asset else "Updating "}Asset'
destination_asset._change_reason = change_reason
destination_asset.save(
override_carto_sync=True
) # Is this save actually necessary, given that there's another below?
if using == 'using-raw-assets':
for raw_asset in raw_assets: # RawAssets must be saved first because an Asset needs at least one
# linked RawAsset or else it will automatically have do_not_display set to True.
raw_asset._change_reason = f'Asset Updater: Linking to {"new " if created_new_asset else ""}Asset'
raw_asset.save()
if organization is not None:
organization._change_reason = change_reason
organization.save()
if location is not None:
location._change_reason = change_reason
location.save()
more_results.append(
f' <a href="https://assets.wprdc.org/api/dev/assets/locations/{location.id}/" target="_blank">Linked Location</a>\n<hr>'
)
destination_asset.location = location
destination_asset.organization = organization
destination_asset._change_reason = change_reason
destination_asset.save(override_carto_sync=True)
asset_ids_to_sync_to_carto.append(destination_asset.id)
else:
more_results.append(f"\n<hr>")
if mode == 'update' and len(asset_ids_to_sync_to_carto) > 0:
sync_assets_to_carto_eventually(asset_ids_to_sync_to_carto)
more_results.append(
f"\nasset_ids_to_sync_to_carto = {asset_ids_to_sync_to_carto}")
return more_results
