def upload_thumb(img, screenshot, basename): # always reduced redundancy
thumb, thumb_size, thumb_format = img.create_thumbnail((200, 150))
screenshot.thumbnail_url = upload_to_s3(thumb,
'screens/t/' + basename +
thumb_format,
thumb_format,
reduced_redundancy=True)
screenshot.thumbnail_width, screenshot.thumbnail_height = thumb_size
def upload_standard(img, screenshot, basename): # always reduced redundancy
standard, standard_size, standard_format = img.create_thumbnail((400, 300))
screenshot.standard_url = upload_to_s3(standard,
'screens/s/' + basename +
standard_format,
standard_format,
reduced_redundancy=True)
screenshot.standard_width, screenshot.standard_height = standard_size
def upload_original(img, screenshot, basename, reduced_redundancy=False):
orig, orig_size, orig_format = img.create_original()
screenshot.original_url = upload_to_s3(
orig,
'screens/o/' + basename + orig_format,
orig_format,
reduced_redundancy=reduced_redundancy)
screenshot.original_width, screenshot.original_height = orig_size
def create_ansi_from_production_link(production_link_id): try: prod_link = ProductionLink.objects.get(id=production_link_id) except ProductionLink.DoesNotExist: # guess it was deleted in the meantime, then. return if prod_link.production.ansis.count(): return # don't create an ANSI if there's one already blob = fetch_link(prod_link) sha1 = blob.sha1 file_ext = prod_link.download_file_extension() filename = 'ansi/' + sha1[0:2] + '/' + sha1[2:4] + '/' + sha1[4:8] + '.pl' + str(production_link_id) + '.' + file_ext url = upload_to_s3(blob.as_io_buffer(), filename, file_ext, reduced_redundancy=True) Ansi.objects.create(production_id=prod_link.production_id, url=url)
def handle_noargs(self, **options):
filetype_filter = Q(parameter__iendswith='.sap') | Q(
parameter__iendswith='.sid') | Q(parameter__iendswith='.mod') | Q(
parameter__iendswith='.s3m') | Q(
parameter__iendswith='.xm') | Q(parameter__iendswith='.it')
links = ProductionLink.objects.filter(
is_download_link=True,
link_class__in=[
'BaseUrl', 'AmigascneFile', 'SceneOrgFile', 'FujiologyFile',
'UntergrundFile', 'PaduaOrgFile'
],
production__supertype='music',
).filter(filetype_filter).exclude(
parameter__istartswith='https://media.demozoo.org/'
).select_related('production')
for prod_link in links:
# see if this prod already has a playable link
tracks, media = get_playable_track_data(prod_link.production)
if tracks:
# already playable
continue
print("prod %s: downloading from %s" %
(prod_link.production_id, prod_link.url))
try:
download = fetch_origin_url(prod_link.download_url)
sha1 = download.sha1
(basename, file_ext) = splitext(download.filename)
filename = 'music/' + sha1[0:2] + '/' + sha1[
2:4] + '/' + slugify(basename) + file_ext
new_url = upload_to_s3(download.as_io_buffer(),
filename,
file_ext,
reduced_redundancy=True)
ProductionLink.objects.create(production=prod_link.production,
link_class='BaseUrl',
parameter=new_url,
is_download_link=True)
except (urllib2.URLError, FileTooBig, timeout) as ex:
pass
time.sleep(5)
def create_ansi_from_production_link(production_link_id):
try:
prod_link = ProductionLink.objects.get(id=production_link_id)
except ProductionLink.DoesNotExist:
# guess it was deleted in the meantime, then.
return
if prod_link.production.ansis.count():
return # don't create an ANSI if there's one already
blob = fetch_link(prod_link)
sha1 = blob.sha1
file_ext = prod_link.download_file_extension()
filename = 'ansi/' + sha1[0:2] + '/' + sha1[2:4] + '/' + sha1[
4:8] + '.pl' + str(production_link_id) + '.' + file_ext
url = upload_to_s3(blob.as_io_buffer(),
filename,
file_ext,
reduced_redundancy=True)
Ansi.objects.create(production_id=prod_link.production_id, url=url)
def handle_noargs(self, **options):
filetype_filter = Q(parameter__iendswith='.sap') | Q(parameter__iendswith='.sid') | Q(parameter__iendswith='.mod') | Q(parameter__iendswith='.s3m') | Q(parameter__iendswith='.xm') | Q(parameter__iendswith='.it')
links = ProductionLink.objects.filter(
is_download_link=True,
link_class__in=['BaseUrl', 'AmigascneFile', 'SceneOrgFile', 'FujiologyFile', 'UntergrundFile', 'PaduaOrgFile'],
production__supertype='music',
).filter(
filetype_filter
).exclude(
parameter__istartswith='https://media.demozoo.org/'
).select_related('production')
for prod_link in links:
# see if this prod already has a playable link
tracks, media = get_playable_track_data(prod_link.production)
if tracks:
# already playable
continue
print("prod %s: downloading from %s" % (prod_link.production_id, prod_link.url))
try:
download = fetch_origin_url(prod_link.download_url)
sha1 = download.sha1
(basename, file_ext) = splitext(download.filename)
filename = 'music/' + sha1[0:2] + '/' + sha1[2:4] + '/' + slugify(basename) + file_ext
new_url = upload_to_s3(download.as_io_buffer(), filename, file_ext, reduced_redundancy=True)
ProductionLink.objects.create(
production=prod_link.production,
link_class='BaseUrl', parameter=new_url,
is_download_link=True
)
except (urllib2.URLError, FileTooBig, timeout) as ex:
pass
time.sleep(5)
def upload_thumb(img, screenshot, basename): # always reduced redundancy thumb, thumb_size, thumb_format = img.create_thumbnail((200, 150)) screenshot.thumbnail_url = upload_to_s3(thumb, 'screens/t/' + basename + thumb_format, thumb_format, reduced_redundancy=True) screenshot.thumbnail_width, screenshot.thumbnail_height = thumb_size
def upload_standard(img, screenshot, basename): # always reduced redundancy standard, standard_size, standard_format = img.create_thumbnail((400, 300)) screenshot.standard_url = upload_to_s3(standard, 'screens/s/' + basename + standard_format, standard_format, reduced_redundancy=True) screenshot.standard_width, screenshot.standard_height = standard_size
def upload_original(img, screenshot, basename, reduced_redundancy=False): orig, orig_size, orig_format = img.create_original() screenshot.original_url = upload_to_s3(orig, 'screens/o/' + basename + orig_format, orig_format, reduced_redundancy=reduced_redundancy) screenshot.original_width, screenshot.original_height = orig_size
def upload_thumb(img, screenshot, basename):
thumb, thumb_size, thumb_format = img.create_thumbnail((200, 150))
screenshot.thumbnail_url = upload_to_s3(
thumb, 'screens/t/' + basename + thumb_format)
screenshot.thumbnail_width, screenshot.thumbnail_height = thumb_size
def upload_standard(img, screenshot, basename):
standard, standard_size, standard_format = img.create_thumbnail((400, 300))
screenshot.standard_url = upload_to_s3(
standard, 'screens/s/' + basename + standard_format)
screenshot.standard_width, screenshot.standard_height = standard_size
def upload_original(img, screenshot, basename):
orig, orig_size, orig_format = img.create_original()
screenshot.original_url = upload_to_s3(
orig, 'screens/o/' + basename + orig_format)
screenshot.original_width, screenshot.original_height = orig_size
def handle(self, *args, **kwargs):
prods = Production.objects.filter(
platforms__name='ZX Spectrum', supertype='production').exclude(
id__in=EmulatorConfig.objects.values_list('production_id',
flat=True)
).exclude(id__in=ProductionLink.objects.filter(
link_class='BaseUrl',
parameter__startswith='https://files.zxdemo.org/').values_list(
'production_id', flat=True)).prefetch_related('links')
for prod in prods:
success = False
for prod_link in prod.links.all():
if not prod_link.is_download_link:
continue
url = urlparse(prod_link.download_url)
basename, ext = splitext(url.path)
ext = ext.lower()
if ext in ('.sna', '.tzx', '.tap', '.z80', '.szx'):
# yay, we can use this directly
print("direct link for %s: %s" %
(prod.title, prod_link.download_url))
try:
download = fetch_link(prod_link)
except (URLError, FileTooBig, timeout, BadZipFile):
print("- broken link :-(")
else:
sha1 = download.sha1
basename, file_ext = splitext(download.filename)
filename = 'emulation/' + sha1[0:2] + '/' + sha1[
2:4] + '/' + slugify(basename) + file_ext
new_url = upload_to_s3(download.as_io_buffer(),
filename)
EmulatorConfig.objects.create(production_id=prod.id,
launch_url=new_url,
emulator='jsspeccy',
configuration='{}')
print("- successfully mirrored at %s" % new_url)
success = True
sleep(1)
elif ext == '.zip':
print("zip file for %s: %s" %
(prod.title, prod_link.download_url))
try:
download = fetch_link(prod_link)
except (URLError, FileTooBig, timeout, BadZipFile):
print("- broken link :-(")
else:
try:
zip = download.as_zipfile()
except BadZipFile: # pragma: no cover
print("- bad zip :-(")
else:
loadable_file_count = 0
for filename in zip.namelist():
if filename.startswith('__MACOSX'):
continue
ext = filename.split('.')[-1].lower()
if ext in ('tap', 'tzx', 'sna', 'z80', 'szx'):
loadable_file_count += 1
if loadable_file_count == 1:
sha1 = download.sha1
basename, file_ext = splitext(
download.filename)
filename = ('emulation/' + sha1[0:2] + '/' +
sha1[2:4] + '/' +
slugify(basename) + file_ext)
new_url = upload_to_s3(download.as_io_buffer(),
filename)
EmulatorConfig.objects.create(
production_id=prod.id,
launch_url=new_url,
emulator='jsspeccy',
configuration='{}')
print("- successfully mirrored at %s" %
new_url)
success = True
elif loadable_file_count == 0:
print("- no loadable files :-(")
else:
print("- multiple loadable files :-/")
sleep(1)
if success:
break
