def redownload_images_view(request):
request.current_app = "admin"
context = {"title": "Redownload Images"}
if request.method == "POST":
form = AdminRedownloadImagesForm(request.POST, request.FILES)
if form.is_valid():
context["assets_to_download"] = assets_to_download = []
rows = slurp_excel(request.FILES["spreadsheet_file"])
required_fields = [
"download_url",
]
for idx, row in enumerate(rows):
missing_fields = [i for i in required_fields if i not in row]
if missing_fields:
messages.warning(
request, f"Skipping row {idx}: missing fields {missing_fields}"
)
continue
download_url = row["download_url"]
# optional real_file_url data
real_file_url = row["real_file_url"]
if not download_url:
if not any(row.values()):
# No messages for completely blank rows
continue
warning_message = (
f"Skipping row {idx}: the required field "
"download_url is empty"
)
messages.warning(request, warning_message)
continue
if not download_url.startswith("http"):
messages.warning(
request, f"Skipping unrecognized URL value: {download_url}"
)
continue
try:
# Use the download_url to look up the related asset.
# Then queue the task to redownload the image file.
assets = Asset.objects.filter(download_url=download_url)
for asset in assets:
redownload_image_task.delay(asset.pk)
if real_file_url:
correct_assets = Asset.objects.filter(
download_url=real_file_url
)
for correct_asset in correct_assets:
asset.correct_asset_pk = correct_asset.pk
asset.correct_asset_slug = correct_asset.slug
assets_to_download.append(asset)
if not assets:
messages.warning(
request,
f"No matching asset for download URL {download_url}",
)
else:
messages.info(
request, f"Queued download for {download_url}",
)
except Exception as exc:
messages.error(
request,
f"Unhandled error attempting to import {download_url}: {exc}",
)
else:
form = AdminRedownloadImagesForm()
context["form"] = form
return render(request, "admin/redownload_images.html", context)
def admin_bulk_import_view(request):
request.current_app = "admin"
url_regex = r"[-\w+]+"
pattern = re.compile(url_regex)
context = {"title": "Bulk Import"}
if request.method == "POST":
form = AdminProjectBulkImportForm(request.POST, request.FILES)
if form.is_valid():
context["import_jobs"] = import_jobs = []
rows = slurp_excel(request.FILES["spreadsheet_file"])
required_fields = [
"Campaign",
"Campaign Short Description",
"Campaign Long Description",
"Campaign Slug",
"Project Slug",
"Project",
"Project Description",
"Import URLs",
]
for idx, row in enumerate(rows):
missing_fields = [i for i in required_fields if i not in row]
if missing_fields:
messages.warning(
request, f"Skipping row {idx}: missing fields {missing_fields}"
)
continue
campaign_title = row["Campaign"]
project_title = row["Project"]
import_url_blob = row["Import URLs"]
if not all((campaign_title, project_title, import_url_blob)):
if not any(row.values()):
# No messages for completely blank rows
continue
warning_message = (
f"Skipping row {idx}: at least one required field "
"(Campaign, Project, Import URLs) is empty"
)
messages.warning(request, warning_message)
continue
try:
# Read Campaign slug value from excel
campaign_slug = row["Campaign Slug"]
if campaign_slug and not pattern.match(campaign_slug):
messages.warning(
request, "Campaign slug doesn't match pattern."
)
campaign, created = validated_get_or_create(
Campaign,
title=campaign_title,
defaults={
"slug": row["Campaign Slug"]
or slugify(campaign_title, allow_unicode=True),
"description": row["Campaign Long Description"] or "",
"short_description": row["Campaign Short Description"]
or "",
},
)
except ValidationError as exc:
messages.error(
request, f"Unable to create campaign {campaign_title}: {exc}"
)
continue
if created:
messages.info(request, f"Created new campaign {campaign_title}")
else:
messages.info(
request,
f"Reusing campaign {campaign_title} without modification",
)
try:
# Read Project slug value from excel
project_slug = row["Project Slug"]
if project_slug and not pattern.match(project_slug):
messages.warning(request, "Project slug doesn't match pattern.")
project, created = validated_get_or_create(
Project,
title=project_title,
campaign=campaign,
defaults={
"slug": row["Project Slug"]
or slugify(project_title, allow_unicode=True),
"description": row["Project Description"] or "",
"campaign": campaign,
},
)
except ValidationError as exc:
messages.error(
request, f"Unable to create project {project_title}: {exc}"
)
continue
if created:
messages.info(request, f"Created new project {project_title}")
else:
messages.info(
request, f"Reusing project {project_title} without modification"
)
potential_urls = filter(None, re.split(r"[\s]+", import_url_blob))
for url in potential_urls:
if not url.startswith("http"):
messages.warning(
request, f"Skipping unrecognized URL value: {url}"
)
continue
try:
import_jobs.append(
import_items_into_project_from_url(
request.user, project, url
)
)
messages.info(
request,
f"Queued {campaign_title} {project_title} import for {url}",
)
except Exception as exc:
messages.error(
request,
f"Unhandled error attempting to import {url}: {exc}",
)
else:
form = AdminProjectBulkImportForm()
context["form"] = form
return render(request, "admin/bulk_import.html", context)
def admin_bulk_import_review(request):
request.current_app = "admin"
url_regex = r"[-\w+]+"
pattern = re.compile(url_regex)
context = {"title": "Bulk Import Review"}
urls = []
all_urls = []
url_counter = 0
sum_count = 0
if request.method == "POST":
form = AdminProjectBulkImportForm(request.POST, request.FILES)
if form.is_valid():
rows = slurp_excel(request.FILES["spreadsheet_file"])
required_fields = [
"Campaign",
"Campaign Short Description",
"Campaign Long Description",
"Campaign Slug",
"Project Slug",
"Project",
"Project Description",
"Import URLs",
]
try:
for idx, row in enumerate(rows):
missing_fields = [
i for i in required_fields if i not in row
]
if missing_fields:
messages.warning(
request,
f"Skipping row {idx}: missing fields {missing_fields}",
)
continue
campaign_title = row["Campaign"]
project_title = row["Project"]
import_url_blob = row["Import URLs"]
if not all(
(campaign_title, project_title, import_url_blob)):
if not any(row.values()):
# No messages for completely blank rows
continue
warning_message = (
f"Skipping row {idx}: at least one required field "
"(Campaign, Project, Import URLs) is empty")
messages.warning(request, warning_message)
continue
# Read Campaign slug value from excel
campaign_slug = row["Campaign Slug"]
if campaign_slug and not pattern.match(campaign_slug):
messages.warning(
request, "Campaign slug doesn't match pattern.")
# Read Project slug value from excel
project_slug = row["Project Slug"]
if project_slug and not pattern.match(project_slug):
messages.warning(
request, "Project slug doesn't match pattern.")
potential_urls = filter(
None, re.split(r"[\s]+", import_url_blob))
for url in potential_urls:
if not url.startswith("http"):
messages.warning(
request,
f"Skipping unrecognized URL value: {url}")
continue
try:
urls.append(url)
url_counter = url_counter + 1
if url_counter == 50:
all_urls.append(urls)
url_counter = 0
urls = []
except Exception as exc:
messages.error(
request,
f"Unhandled error attempting to count {url}: {exc}",
)
all_urls.append(urls)
for i, val in enumerate(all_urls):
return_result = fetch_all_urls(val)
for res in return_result[0]:
messages.info(request, f"{res}")
sum_count = sum_count + return_result[1]
time.sleep(7)
messages.info(request, f"Total Asset Count:{sum_count}")
finally:
messages.info(request, "All Processes Completed")
else:
form = AdminProjectBulkImportForm()
context["form"] = form
return render(request, "admin/bulk_review.html", context)
def admin_bulk_import_view(request):
# TODO: when we upgrade to Django 2.1 we can use the admin site override
# mechanism (the old one is broken in 2.0): see
# https://code.djangoproject.com/ticket/27887 in the meantime, this will
# simply be a regular Django view using forms and just enough context to
# reuse the Django admin template
request.current_app = "admin"
context = {"title": "Bulk Import"}
if request.method == "POST":
form = AdminProjectBulkImportForm(request.POST, request.FILES)
if form.is_valid():
context["import_jobs"] = import_jobs = []
rows = slurp_excel(request.FILES["spreadsheet_file"])
required_fields = [
"Campaign",
"Campaign Short Description",
"Campaign Long Description",
"Project",
"Project Description",
"Import URLs",
]
for idx, row in enumerate(rows):
missing_fields = [i for i in required_fields if i not in row]
if missing_fields:
messages.warning(
request,
f"Skipping row {idx}: missing fields {missing_fields}")
continue
campaign_title = row["Campaign"]
project_title = row["Project"]
import_url_blob = row["Import URLs"]
if not all((campaign_title, project_title, import_url_blob)):
if not any(row.values()):
# No messages for completely blank rows
continue
warning_message = (
f"Skipping row {idx}: at least one required field "
"(Campaign, Project, Import URLs) is empty")
messages.warning(request, warning_message)
continue
try:
campaign, created = validated_get_or_create(
Campaign,
title=campaign_title,
defaults={
"slug":
slugify(campaign_title),
"description":
row["Campaign Long Description"] or "",
"short_description":
row["Campaign Short Description"] or "",
},
)
except ValidationError as exc:
messages.error(
request,
f"Unable to create campaign {campaign_title}: {exc}")
continue
if created:
messages.info(request,
f"Created new campaign {campaign_title}")
else:
messages.info(
request,
f"Reusing campaign {campaign_title} without modification",
)
try:
project, created = validated_get_or_create(
Project,
title=project_title,
campaign=campaign,
defaults={
"slug": slugify(project_title),
"description": row["Project Description"] or "",
"campaign": campaign,
},
)
except ValidationError as exc:
messages.error(
request,
request,
f"Unable to create project {project_title}: {exc}",
)
continue
if created:
messages.info(request,
f"Created new project {project_title}")
else:
messages.info(
request,
f"Reusing project {project_title} without modification"
)
potential_urls = filter(None,
re.split(r"[\s]+", import_url_blob))
for url in potential_urls:
if not url.startswith("http"):
messages.warning(
request, f"Skipping unrecognized URL value: {url}")
continue
try:
import_jobs.append(
import_items_into_project_from_url(
request.user, project, url))
messages.info(
request,
f"Queued {campaign_title} {project_title} import for {url}",
)
except Exception as exc:
messages.error(
request,
f"Unhandled error attempting to import {url}: {exc}",
)
else:
form = AdminProjectBulkImportForm()
context["form"] = form
return render(request, "admin/bulk_import.html", context)
def admin_bulk_import_view(request):
# TODO: when we upgrade to Django 2.1 we can use the admin site override
# mechanism (the old one is broken in 2.0): see
# https://code.djangoproject.com/ticket/27887 in the meantime, this will
# simply be a regular Django view using forms and just enough context to
# reuse the Django admin template
request.current_app = "admin"
context = {"title": "Bulk Import"}
if request.method == "POST":
form = AdminProjectBulkImportForm(request.POST, request.FILES)
if form.is_valid():
context["import_jobs"] = import_jobs = []
rows = slurp_excel(request.FILES["spreadsheet_file"])
required_fields = [
"Campaign",
"Campaign Short Description",
"Campaign Long Description",
"Project",
"Project Description",
"Import URLs",
]
for idx, row in enumerate(rows):
missing_fields = [i for i in required_fields if i not in row]
if missing_fields:
messages.add_message(
request,
messages.WARNING,
f"Skipping row {idx}: missing fields {missing_fields}",
)
continue
campaign_title = row["Campaign"]
project_title = row["Project"]
import_url_blob = row["Import URLs"]
if not all((campaign_title, project_title, import_url_blob)):
messages.add_message(
request,
messages.WARNING,
f"Skipping row {idx}: at least one required field (Campaign, Project, Import URLs) is empty",
)
continue
campaign, created = Campaign.objects.get_or_create(
title=campaign_title,
defaults={
"slug":
slugify(campaign_title),
"description":
row["Campaign Long Description"] or "",
"short_description":
row["Campaign Short Description"] or "",
},
)
if created:
messages.add_message(
request, messages.INFO,
f"Created new campaign {campaign_title}")
else:
messages.add_message(
request,
messages.INFO,
f"Reusing campaign {campaign_title} without modification",
)
project, created = campaign.project_set.get_or_create(
title=project_title,
defaults={"slug": slugify(project_title)})
if created:
messages.add_message(
request, messages.INFO,
f"Created new project {project_title}")
else:
messages.add_message(
request,
messages.INFO,
f"Reusing project {project_title} without modification",
)
potential_urls = filter(None,
re.split(r"[\s]+", import_url_blob))
for url in potential_urls:
if not url.startswith("http"):
continue
import_jobs.append(
import_items_into_project_from_url(
request.user, project, url))
messages.add_message(
request,
messages.INFO,
f"Queued {campaign_title} {project_title} import for {url}",
)
else:
form = AdminProjectBulkImportForm()
context["form"] = form
return render(request, "admin/bulk_import.html", context)
