def joerd_enqueue_renders(cfg): """ Sends each output tile configured for the regions in the config file to the queue for processing by workers. Note that downloading to the store should have happened before this is run, as the render process has no way to download files. """ logger = logging.getLogger('enqueuer') j = Server(cfg) logger.info("Streaming jobs to the queue") queue = _make_queue(j, cfg.queue_config) max_batch_len = 1000 # size limit is 256KB for SQS, but we'll leave a little bit of space # just in case there's some small overhead for encoding it as an array. size_limit = 256 * 1024 - 100 dispatcher = GroupingDispatcher(queue, max_batch_len, logger, size_limit) idx = 0 next_idx = 0 logger.info("Starting loop") for output in j.outputs.itervalues(): logger.info("Starting output %r" % output.__class__.__name__) for tile in output.generate_tiles(): if idx >= next_idx: next_idx += 10000 logger.info("[%d] At job %r" % (idx, tile.__class__.__name__)) idx += 1 sources = [] for name, s in j.sources: v = s.vrts_for(tile) if v: vrts = [] for rasters in v: files = [r.output_file() for r in rasters] if files: vrts.append(files) if vrts: sources.append(dict(source=name, vrts=vrts)) assert sources, "Was expecting at least one source for tile %r, " \ "but it has none." % tile.tile_name() job = dict(job='render', data=tile.freeze_dry(), sources=sources) dispatcher.append(job) dispatcher.flush() logger.info("Done.")
def joerd_enqueue_single_renders(cfg): """ Renders single tiles without a queue. The list of tiles is read from a file called `tiles_to_enqueue.txt`. Can be useful for testing. """ logger = logging.getLogger('enqueuer') j = Server(cfg) logger.info("Fetching download information") j.list_downloads() logger.info("Streaming jobs to the queue") queue = _make_queue(j, cfg.queue_config) max_batch_len = 1000 # size limit is 256KB for SQS, but we'll leave a little bit of space # just in case there's some small overhead for encoding it as an array. size_limit = 256 * 1024 - 100 dispatcher = GroupingDispatcher(queue, max_batch_len, logger, size_limit) idx = 0 next_idx = 0 tiff_output = j.outputs['tiff'] normal_output = j.outputs['normal'] terrarium_output = j.outputs['terrarium'] import joerd.output as output logger.info("Starting loop") with open('tiles_to_enqueue.txt', 'r') as fh: for line in fh: location = line.split(".")[0] tile = None if location.startswith('skadi/'): output_name, y, tile_name = location.split("/") pos = output.skadi._parse_tile(tile_name) if pos is None: raise Exception, "Couldn't parse skadi tile name %r" \ % tile_name tile = output.skadi.SkadiTile('skadi', *pos) else: output_name, z, x, y = location.split("/") z = int(z) x = int(x) y = int(y) if output_name == 'geotiff': tile = output.tiff.TiffTile(tiff_output, z, x, y) elif output_name == 'normal': tile = output.normal.NormalTile(normal_output, z, x, y) elif output_name == 'terrarium': tile = output.terrarium.TerrariumTile(terrarium_output, z, x, y) else: raise Exception, "Couldn't make a tile from line %r" \ % line sources = [] for name, s in j.sources: v = s.vrts_for(tile) if v: vrts = [] for rasters in v: files = [r.output_file() for r in rasters] if files: vrts.append(files) if vrts: sources.append(dict(source=name, vrts=vrts)) assert sources, "Was expecting at least one source for tile %r, " \ "but it has none." % tile.tile_name() job = dict(job='render', data=tile.freeze_dry(), sources=sources) dispatcher.append(job) dispatcher.flush() logger.info("Done.")