def gallery_frequency(request: HttpRequest) -> HttpResponse:
if not crawler_settings.urls.enable_gallery_frequency:
if not request.user.is_staff:
raise Http404("Page not found")
else:
return render_error(
request,
"Page disabled by settings (urls: enable_gallery_frequency).")
title = request.GET.get("title", '')
tags = request.GET.get("tags", '')
if 'clear' in request.GET:
request.GET = QueryDict('')
form = GallerySearchForm()
form_simple = GallerySearchSimpleForm(request.GET,
error_class=SpanErrorList)
else:
form = GallerySearchForm(initial={'title': title, 'tags': tags})
form_simple = GallerySearchSimpleForm(request.GET,
error_class=SpanErrorList)
form_simple.is_valid()
for field_name, errors in form_simple.errors.items():
messages.error(request,
field_name + ": " + ", ".join(errors),
extra_tags='danger')
d = {'form': form, 'form_simple': form_simple}
return render(request, "viewer/graph_gallery_posted.html", d)
def missing_archives_for_galleries(request: HttpRequest) -> HttpResponse:
p = request.POST
get = request.GET
title = get.get("title", '')
tags = get.get("tags", '')
try:
page = int(get.get("page", '1'))
except ValueError:
page = 1
if 'clear' in get:
form = GallerySearchForm()
else:
form = GallerySearchForm(initial={'title': title, 'tags': tags})
if p and request.user.is_staff:
pks = []
for k, v in p.items():
if k.startswith("sel-"):
# k, pk = k.split('-')
# results[pk][k] = v
pks.append(v)
results = Gallery.objects.filter(id__in=pks).order_by('-create_date')
if 'delete_galleries' in p:
for gallery in results:
message = 'Removing gallery: {}, link: {}'.format(
gallery.title, gallery.get_link())
frontend_logger.info(message)
messages.success(request, message)
gallery.mark_as_deleted()
elif 'download_galleries' in p:
for gallery in results:
message = 'Queueing gallery: {}, link: {}'.format(
gallery.title, gallery.get_link())
frontend_logger.info(message)
messages.success(request, message)
# Force replace_metadata when queueing from this list, since it's mostly used to download non used.
current_settings = Settings(
load_from_config=crawler_settings.config)
if current_settings.workers.web_queue:
current_settings.replace_metadata = True
current_settings.retry_failed = True
if 'reason' in p and p['reason'] != '':
reason = p['reason']
# Force limit string length (reason field max_length)
current_settings.archive_reason = reason[:200]
current_settings.archive_details = gallery.reason
current_settings.gallery_reason = reason[:200]
elif gallery.reason:
current_settings.archive_reason = gallery.reason
current_settings.workers.web_queue.enqueue_args_list(
(gallery.get_link(), ),
override_options=current_settings)
elif 'recall_api' in p:
message = 'Recalling API for {} galleries'.format(results.count())
frontend_logger.info(message)
messages.success(request, message)
gallery_links = [x.get_link() for x in results]
gallery_providers = list(
results.values_list('provider', flat=True).distinct())
current_settings = Settings(
load_from_config=crawler_settings.config)
if current_settings.workers.web_queue:
current_settings.set_update_metadata_options(
providers=gallery_providers)
current_settings.workers.web_queue.enqueue_args_list(
gallery_links, override_options=current_settings)
if 'force_public' in request.GET:
force_public = True
else:
force_public = False
if request.user.is_staff and not force_public:
providers = Gallery.objects.all().values_list('provider',
flat=True).distinct()
params = {}
for k, v in get.items():
params[k] = v
for k in gallery_filter_keys:
if k not in params:
params[k] = ''
results = filter_galleries_simple(params)
results = results.non_used_galleries().prefetch_related(
'foundgallery_set')
paginator = Paginator(results, 50)
try:
results = paginator.page(page)
except (InvalidPage, EmptyPage):
results = paginator.page(paginator.num_pages)
d = {
'results': results,
'providers': providers,
'force_public': force_public,
'form': form
}
else:
params = {}
for k, v in get.items():
params[k] = v
for k in gallery_filter_keys:
if k not in params:
params[k] = ''
results = filter_galleries_simple(params)
results = results.non_used_galleries(public=True,
provider__in=['panda', 'fakku'])
d = {'results': results}
return render(request, "viewer/archives_missing_for_galleries.html", d)
def archive_filesize_different_from_gallery(
request: HttpRequest) -> HttpResponse:
providers = Gallery.objects.all().values_list('provider',
flat=True).distinct()
p = request.POST
get = request.GET
title = get.get("title", '')
tags = get.get("tags", '')
try:
page = int(get.get("page", '1'))
except ValueError:
page = 1
if 'clear' in get:
form = GallerySearchForm()
else:
form = GallerySearchForm(initial={'title': title, 'tags': tags})
if p:
pks: List[str] = []
for k, v in p.items():
if k.startswith("del-"):
# k, pk = k.split('-')
# results[pk][k] = v
pks.extend(p.getlist(k))
results = Archive.objects.filter(id__in=pks).order_by('-pk')
for archive in results:
if 'delete_archives' in p:
message = "Removing archive: {} and keeping its file: {}".format(
archive.title, archive.zipped.name)
frontend_logger.info(message)
messages.success(request, message)
archive.delete()
if 'delete_archives_and_files' in p:
message = "Removing archive: {} and deleting its file: {}".format(
archive.title, archive.zipped.name)
frontend_logger.info(message)
messages.success(request, message)
archive.delete_all_files()
archive.delete()
params = {}
for k, v in get.items():
params[k] = v
for k in gallery_filter_keys:
if k not in params:
params[k] = ''
results = filter_galleries_simple(params)
results = results.different_filesize_archive()
paginator = Paginator(results, 50)
try:
results = paginator.page(page)
except (InvalidPage, EmptyPage):
results = paginator.page(paginator.num_pages)
d = {'results': results, 'providers': providers, 'form': form}
return render(request, "viewer/archives_different_filesize.html", d)
def repeated_archives_for_galleries(request: HttpRequest) -> HttpResponse:
p = request.POST
get = request.GET
title = get.get("title", '')
tags = get.get("tags", '')
try:
page = int(get.get("page", '1'))
except ValueError:
page = 1
if 'clear' in get:
form = GallerySearchForm()
else:
form = GallerySearchForm(initial={'title': title, 'tags': tags})
if p:
pks: List[str] = []
for k, v in p.items():
if k.startswith("del-"):
# k, pk = k.split('-')
# results[pk][k] = v
pks.extend(p.getlist(k))
results = Archive.objects.filter(id__in=pks).order_by('-pk')
for archive in results:
if 'delete_archives_and_files' in p:
message = 'Removing archive and deleting file: {}'.format(
archive.zipped.name)
frontend_logger.info(message)
messages.success(request, message)
archive.delete_all_files()
else:
message = 'Removing archive: {}'.format(archive.zipped.name)
frontend_logger.info(message)
messages.success(request, message)
archive.delete()
params = {
'sort': 'create_date',
'asc_desc': 'desc',
}
for k, v in get.items():
params[k] = v
for k in gallery_filter_keys:
if k not in params:
params[k] = ''
results = filter_galleries_simple(params)
results = results.several_archives()
paginator = Paginator(results, 50)
try:
results = paginator.page(page)
except (InvalidPage, EmptyPage):
results = paginator.page(paginator.num_pages)
d = {'results': results, 'form': form}
return render(request, "viewer/archives_repeated.html", d)
def repeated_galleries_by_field(request: HttpRequest) -> HttpResponse:
p = request.POST
get = request.GET
title = get.get("title", '')
tags = get.get("tags", '')
if 'clear' in get:
form = GallerySearchForm()
else:
form = GallerySearchForm(initial={'title': title, 'tags': tags})
if p:
pks = []
for k, v in p.items():
if k.startswith("del-"):
# k, pk = k.split('-')
# results[pk][k] = v
pks.append(v)
results = Gallery.objects.filter(id__in=pks).order_by('-create_date')
if 'delete_galleries' in p:
user_reason = p.get('reason', '')
for gallery in results:
message = 'Removing gallery: {}, link: {}'.format(
gallery.title, gallery.get_link())
frontend_logger.info(message)
messages.success(request, message)
gallery.mark_as_deleted()
event_log(request.user,
'DELETE_GALLERY',
reason=user_reason,
content_object=gallery,
result='deleted')
params = {
'sort': 'create_date',
'asc_desc': 'desc',
}
for k, v in get.items():
params[k] = v
for k in gallery_filter_keys:
if k not in params:
params[k] = ''
results = filter_galleries_simple(params)
results = results.eligible_for_use().exclude(title__exact='')
if 'has-archives' in get:
results = results.annotate(archives=Count('archive')).filter(
archives__gt=0)
by_title = dict()
if 'same-uploader' in get:
for k, v in groupby(results.order_by('title', 'uploader'), lambda x:
(x.title, x.uploader)):
objects = list(v)
if len(objects) > 1:
by_title[k] = objects
else:
for k, v in groupby(results.order_by('title'), lambda x: x.title):
objects = list(v)
if len(objects) > 1:
by_title[k] = objects
providers = Gallery.objects.all().values_list('provider',
flat=True).distinct()
d = {'by_title': by_title, 'form': form, 'providers': providers}
return render(request, "viewer/galleries_repeated_by_fields.html", d)
def submit_queue(request: HttpRequest) -> HttpResponse:
p = request.POST
get = request.GET
title = get.get("title", '')
tags = get.get("tags", '')
user_reason = p.get('reason', '')
try:
page = int(get.get("page", '1'))
except ValueError:
page = 1
if 'clear' in get:
form = GallerySearchForm()
else:
form = GallerySearchForm(initial={'title': title, 'tags': tags})
if p:
pks = []
for k, v in p.items():
if k.startswith("sel-"):
# k, pk = k.split('-')
# results[pk][k] = v
pks.append(v)
preserved = Case(
*[When(pk=pk, then=pos) for pos, pk in enumerate(pks)])
if 'denied' in get:
results = Gallery.objects.submitted_galleries(
id__in=pks).order_by(preserved)
else:
results = Gallery.objects.submitted_galleries(
~Q(status=Gallery.DENIED), id__in=pks).order_by(preserved)
if 'deny_galleries' in p:
for gallery in results:
message = 'Denying gallery: {}, link: {}, source link: {}'.format(
gallery.title, gallery.get_absolute_url(),
gallery.get_link())
if 'reason' in p and p['reason'] != '':
message += ', reason: {}'.format(p['reason'])
frontend_logger.info("User {}: {}".format(
request.user.username, message))
messages.success(request, message)
gallery.mark_as_denied()
event_log(request.user,
'DENY_GALLERY',
reason=user_reason,
content_object=gallery,
result='denied')
elif 'download_galleries' in p:
for gallery in results:
message = 'Queueing gallery: {}, link: {}, source link: {}'.format(
gallery.title, gallery.get_absolute_url(),
gallery.get_link())
if 'reason' in p and p['reason'] != '':
message += ', reason: {}'.format(p['reason'])
frontend_logger.info("User {}: {}".format(
request.user.username, message))
messages.success(request, message)
event_log(request.user,
'ACCEPT_GALLERY',
reason=user_reason,
content_object=gallery,
result='accepted')
# Force replace_metadata when queueing from this list, since it's mostly used to download non used.
current_settings = Settings(
load_from_config=crawler_settings.config)
if current_settings.workers.web_queue:
current_settings.replace_metadata = True
current_settings.retry_failed = True
if 'reason' in p and p['reason'] != '':
reason = p['reason']
# Force limit string length (reason field max_length)
current_settings.archive_reason = reason[:200]
current_settings.archive_details = gallery.reason
current_settings.gallery_reason = reason[:200]
elif gallery.reason:
current_settings.archive_reason = gallery.reason
def archive_callback(x: Optional['Archive'],
crawled_url: Optional[str],
result: str) -> None:
event_log(request.user,
'ADD_ARCHIVE',
reason=user_reason,
content_object=x,
result=result,
data=crawled_url)
def gallery_callback(x: Optional['Gallery'],
crawled_url: Optional[str],
result: str) -> None:
event_log(request.user,
'ADD_GALLERY',
reason=user_reason,
content_object=x,
result=result,
data=crawled_url)
current_settings.workers.web_queue.enqueue_args_list(
(gallery.get_link(), ),
override_options=current_settings,
archive_callback=archive_callback,
gallery_callback=gallery_callback,
)
providers = Gallery.objects.all().values_list('provider',
flat=True).distinct()
params = {}
for k, v in get.items():
params[k] = v
for k in gallery_filter_keys:
if k not in params:
params[k] = ''
results = filter_galleries_simple(params)
if 'denied' in get:
results = results.submitted_galleries().prefetch_related(
'foundgallery_set')
else:
results = results.submitted_galleries(~Q(
status=Gallery.DENIED)).prefetch_related('foundgallery_set')
paginator = Paginator(results, 50)
try:
results = paginator.page(page)
except (InvalidPage, EmptyPage):
results = paginator.page(paginator.num_pages)
d = {'results': results, 'providers': providers, 'form': form}
return render(request, "viewer/collaborators/submit_queue.html", d)