def _initiate_subbasin_mapshed_job_chain(mapshed_input, job_id): errback = save_job_error.s(job_id) area_of_interest, wkaoi = _parse_input(mapshed_input) if not wkaoi: raise ValidationError('You must provide the `wkaoi` key: ' + 'a HUC id is currently required for ' + 'subbasin modeling.') [layer_code, shape_id] = wkaoi.split('__') if layer_code not in ['huc8', 'huc10']: raise ValidationError('Only HUC-08s and HUC-10s are valid for ' + 'subbasin modeling.') huc12s = split_into_huc12s(layer_code, shape_id) if not huc12s: raise EmptyResultSet('No subbasins found') huc12_job_chains = [] for (huc12_id, huc12, huc12_aoi) in huc12s: huc12_wkaoi = 'huc12__{id}'.format(id=huc12_id) huc12_job_chains.append( chain( (group(geoprocessing_chains(huc12_aoi, huc12_wkaoi, errback)) | collect_data.s(huc12_aoi, huc12).set(link_error=errback)))) return chain( group(huc12_job_chains) | tasks.subbasin_results_to_dict.s() | save_job_result.s(job_id, mapshed_input)).apply_async()
def _initiate_mapshed_job_chain(mapshed_input, job_id): errback = save_job_error.s(job_id) area_of_interest, wkaoi = _parse_input(mapshed_input) job_chain = (group(geoprocessing_chains(area_of_interest, wkaoi, errback)) | collect_data.s(area_of_interest).set(link_error=errback) | save_job_result.s(job_id, mapshed_input)) return chain(job_chain).apply_async()
def _initiate_mapshed_job_chain(mapshed_input, job_id): errback = save_job_error.s(job_id) area_of_interest, wkaoi = _parse_input(mapshed_input) job_chain = (multi_mapshed(area_of_interest, wkaoi) | convert_data.s(wkaoi) | collect_data.s(area_of_interest) | save_job_result.s(job_id, mapshed_input)) return chain(job_chain).apply_async(link_error=errback)
def _initiate_mapshed_job_chain(mapshed_input, job_id): workers = get_living_workers() get_worker = lambda: random.choice(workers) errback = save_job_error.s(job_id).set(exchange=MAGIC_EXCHANGE, routing_key=get_worker()) area_of_interest, wkaoi = parse_input(mapshed_input) job_chain = ( group( geoprocessing_chains(area_of_interest, wkaoi, MAGIC_EXCHANGE, errback, choose_worker)) | combine.s().set(exchange=MAGIC_EXCHANGE, routing_key=get_worker()) | collect_data.s(area_of_interest).set(link_error=errback, exchange=MAGIC_EXCHANGE, routing_key=get_worker()) | save_job_result.s(job_id, mapshed_input).set( exchange=MAGIC_EXCHANGE, routing_key=get_worker())) return chain(job_chain).apply_async(link_error=errback)