def upload_thumb(img, screenshot, basename): # always reduced redundancy thumb, thumb_size, thumb_format = img.create_thumbnail((200, 150)) screenshot.thumbnail_url = upload_to_s3(thumb, 'screens/t/' + basename + thumb_format, thumb_format, reduced_redundancy=True) screenshot.thumbnail_width, screenshot.thumbnail_height = thumb_size
def upload_standard(img, screenshot, basename): # always reduced redundancy standard, standard_size, standard_format = img.create_thumbnail((400, 300)) screenshot.standard_url = upload_to_s3(standard, 'screens/s/' + basename + standard_format, standard_format, reduced_redundancy=True) screenshot.standard_width, screenshot.standard_height = standard_size
def upload_original(img, screenshot, basename, reduced_redundancy=False): orig, orig_size, orig_format = img.create_original() screenshot.original_url = upload_to_s3( orig, 'screens/o/' + basename + orig_format, orig_format, reduced_redundancy=reduced_redundancy) screenshot.original_width, screenshot.original_height = orig_size
def create_ansi_from_production_link(production_link_id): try: prod_link = ProductionLink.objects.get(id=production_link_id) except ProductionLink.DoesNotExist: # guess it was deleted in the meantime, then. return if prod_link.production.ansis.count(): return # don't create an ANSI if there's one already blob = fetch_link(prod_link) sha1 = blob.sha1 file_ext = prod_link.download_file_extension() filename = 'ansi/' + sha1[0:2] + '/' + sha1[2:4] + '/' + sha1[4:8] + '.pl' + str(production_link_id) + '.' + file_ext url = upload_to_s3(blob.as_io_buffer(), filename, file_ext, reduced_redundancy=True) Ansi.objects.create(production_id=prod_link.production_id, url=url)
def handle_noargs(self, **options): filetype_filter = Q(parameter__iendswith='.sap') | Q( parameter__iendswith='.sid') | Q(parameter__iendswith='.mod') | Q( parameter__iendswith='.s3m') | Q( parameter__iendswith='.xm') | Q(parameter__iendswith='.it') links = ProductionLink.objects.filter( is_download_link=True, link_class__in=[ 'BaseUrl', 'AmigascneFile', 'SceneOrgFile', 'FujiologyFile', 'UntergrundFile', 'PaduaOrgFile' ], production__supertype='music', ).filter(filetype_filter).exclude( parameter__istartswith='https://media.demozoo.org/' ).select_related('production') for prod_link in links: # see if this prod already has a playable link tracks, media = get_playable_track_data(prod_link.production) if tracks: # already playable continue print("prod %s: downloading from %s" % (prod_link.production_id, prod_link.url)) try: download = fetch_origin_url(prod_link.download_url) sha1 = download.sha1 (basename, file_ext) = splitext(download.filename) filename = 'music/' + sha1[0:2] + '/' + sha1[ 2:4] + '/' + slugify(basename) + file_ext new_url = upload_to_s3(download.as_io_buffer(), filename, file_ext, reduced_redundancy=True) ProductionLink.objects.create(production=prod_link.production, link_class='BaseUrl', parameter=new_url, is_download_link=True) except (urllib2.URLError, FileTooBig, timeout) as ex: pass time.sleep(5)
def create_ansi_from_production_link(production_link_id): try: prod_link = ProductionLink.objects.get(id=production_link_id) except ProductionLink.DoesNotExist: # guess it was deleted in the meantime, then. return if prod_link.production.ansis.count(): return # don't create an ANSI if there's one already blob = fetch_link(prod_link) sha1 = blob.sha1 file_ext = prod_link.download_file_extension() filename = 'ansi/' + sha1[0:2] + '/' + sha1[2:4] + '/' + sha1[ 4:8] + '.pl' + str(production_link_id) + '.' + file_ext url = upload_to_s3(blob.as_io_buffer(), filename, file_ext, reduced_redundancy=True) Ansi.objects.create(production_id=prod_link.production_id, url=url)
def handle_noargs(self, **options): filetype_filter = Q(parameter__iendswith='.sap') | Q(parameter__iendswith='.sid') | Q(parameter__iendswith='.mod') | Q(parameter__iendswith='.s3m') | Q(parameter__iendswith='.xm') | Q(parameter__iendswith='.it') links = ProductionLink.objects.filter( is_download_link=True, link_class__in=['BaseUrl', 'AmigascneFile', 'SceneOrgFile', 'FujiologyFile', 'UntergrundFile', 'PaduaOrgFile'], production__supertype='music', ).filter( filetype_filter ).exclude( parameter__istartswith='https://media.demozoo.org/' ).select_related('production') for prod_link in links: # see if this prod already has a playable link tracks, media = get_playable_track_data(prod_link.production) if tracks: # already playable continue print("prod %s: downloading from %s" % (prod_link.production_id, prod_link.url)) try: download = fetch_origin_url(prod_link.download_url) sha1 = download.sha1 (basename, file_ext) = splitext(download.filename) filename = 'music/' + sha1[0:2] + '/' + sha1[2:4] + '/' + slugify(basename) + file_ext new_url = upload_to_s3(download.as_io_buffer(), filename, file_ext, reduced_redundancy=True) ProductionLink.objects.create( production=prod_link.production, link_class='BaseUrl', parameter=new_url, is_download_link=True ) except (urllib2.URLError, FileTooBig, timeout) as ex: pass time.sleep(5)
def upload_original(img, screenshot, basename, reduced_redundancy=False): orig, orig_size, orig_format = img.create_original() screenshot.original_url = upload_to_s3(orig, 'screens/o/' + basename + orig_format, orig_format, reduced_redundancy=reduced_redundancy) screenshot.original_width, screenshot.original_height = orig_size
def upload_thumb(img, screenshot, basename): thumb, thumb_size, thumb_format = img.create_thumbnail((200, 150)) screenshot.thumbnail_url = upload_to_s3( thumb, 'screens/t/' + basename + thumb_format) screenshot.thumbnail_width, screenshot.thumbnail_height = thumb_size
def upload_standard(img, screenshot, basename): standard, standard_size, standard_format = img.create_thumbnail((400, 300)) screenshot.standard_url = upload_to_s3( standard, 'screens/s/' + basename + standard_format) screenshot.standard_width, screenshot.standard_height = standard_size
def upload_original(img, screenshot, basename): orig, orig_size, orig_format = img.create_original() screenshot.original_url = upload_to_s3( orig, 'screens/o/' + basename + orig_format) screenshot.original_width, screenshot.original_height = orig_size
def handle(self, *args, **kwargs): prods = Production.objects.filter( platforms__name='ZX Spectrum', supertype='production').exclude( id__in=EmulatorConfig.objects.values_list('production_id', flat=True) ).exclude(id__in=ProductionLink.objects.filter( link_class='BaseUrl', parameter__startswith='https://files.zxdemo.org/').values_list( 'production_id', flat=True)).prefetch_related('links') for prod in prods: success = False for prod_link in prod.links.all(): if not prod_link.is_download_link: continue url = urlparse(prod_link.download_url) basename, ext = splitext(url.path) ext = ext.lower() if ext in ('.sna', '.tzx', '.tap', '.z80', '.szx'): # yay, we can use this directly print("direct link for %s: %s" % (prod.title, prod_link.download_url)) try: download = fetch_link(prod_link) except (URLError, FileTooBig, timeout, BadZipFile): print("- broken link :-(") else: sha1 = download.sha1 basename, file_ext = splitext(download.filename) filename = 'emulation/' + sha1[0:2] + '/' + sha1[ 2:4] + '/' + slugify(basename) + file_ext new_url = upload_to_s3(download.as_io_buffer(), filename) EmulatorConfig.objects.create(production_id=prod.id, launch_url=new_url, emulator='jsspeccy', configuration='{}') print("- successfully mirrored at %s" % new_url) success = True sleep(1) elif ext == '.zip': print("zip file for %s: %s" % (prod.title, prod_link.download_url)) try: download = fetch_link(prod_link) except (URLError, FileTooBig, timeout, BadZipFile): print("- broken link :-(") else: try: zip = download.as_zipfile() except BadZipFile: # pragma: no cover print("- bad zip :-(") else: loadable_file_count = 0 for filename in zip.namelist(): if filename.startswith('__MACOSX'): continue ext = filename.split('.')[-1].lower() if ext in ('tap', 'tzx', 'sna', 'z80', 'szx'): loadable_file_count += 1 if loadable_file_count == 1: sha1 = download.sha1 basename, file_ext = splitext( download.filename) filename = ('emulation/' + sha1[0:2] + '/' + sha1[2:4] + '/' + slugify(basename) + file_ext) new_url = upload_to_s3(download.as_io_buffer(), filename) EmulatorConfig.objects.create( production_id=prod.id, launch_url=new_url, emulator='jsspeccy', configuration='{}') print("- successfully mirrored at %s" % new_url) success = True elif loadable_file_count == 0: print("- no loadable files :-(") else: print("- multiple loadable files :-/") sleep(1) if success: break