def __init__(self, user, scenarios, indicators): LogEvents("Start scenario", "Starting all scenarios proccesing", -1, 1) self.user = user self.indicator = Indicator(self.user) if scenarios != "": self.scenarios = [ int(scenario.strip()) for scenario in scenarios.split('_') ] self.base_scenario = self.__get_base_scenario(self.scenarios[0]) if self.base_scenario in self.scenarios: if len(self.scenarios) > 1: self.scenarios.append( self.scenarios.pop( self.scenarios.index( self.scenarios.index(self.base_scenario)))) if indicators != "": self.indicators = indicators.split('_') indicator_param = [] for indicator_set in self.indicators: indicator_param.append(indicator_set.strip()) order = Modules.objects.filter( module__in=indicator_param).values("module").order_by('order') self.indicators = [] for module_in in order: self.indicators.append(module_in["module"]) self.copiados = dict(buffers=True, policies=True, amenities=[], transit=[], cycle=[], footprint=[], roads=[])
def run(self): try: error = True count = 0 while error and count < 3: self.__Indicator = Indicator(self.__user) db = self.__Indicator.get_up_calculator_connection() try: query = """ select urbper_indicator_potable_water_network({scenario}) """.format(scenario=self.__scenario) LogEvents("potable water network", "potable water network module started: " + query, self.__scenario, self.__user) with transaction.atomic(): db.execute(query) except Exception as e: error = True count += 1 time.sleep(randint(1, 3)) db.close() LogEvents( "potable water network", "potable water network module failed " + str(count) + ": " + str(e), self.__scenario, self.__user) else: error = False db.close() LogEvents("potable water network", "potable water network module finished", self.__scenario, self.__user) except Exception as e: LogEvents("potable water network", "unknown error " + str(e), self.__scenario, self.__user)
def __ModulejobDensity(self, scenario_id, offset=0, limit=0): try: error = True count = 0 while error and count < 3: self.__Indicator = Indicator(self.__user) db = self.__Indicator.get_up_calculator_connection() try: with transaction.atomic(): query = """select urbper_indicator_job_density({scenario},{offset},{limit})""".format( scenario=scenario_id, offset=offset, limit=limit) LogEvents("job density", "job density module started: " + query, scenario_id, self.__user) db.execute(query) except Exception as e: error = True count += 1 LogEvents( "job density ", "job density module failed " + str(count) + ": " + str(e), scenario_id, self.__user, True) db.close() else: error = False LogEvents("job density ", "job density module finished", scenario_id, self.__user) db.close() except Exception as e: LogEvents("Running scenarios", "Unknown error " + str(e), scenario_id, self.__user, True)
def run(self): query = """ delete from mmu_info where mmu_id in ( select mmu_id from mmu where scenario_id={scenario} and not st_contains( ( select location from footprint inner join classification on classification.name=footprint.name where classification.category='footprint' and classification.fclass in ('political_boundary','study_area') and scenario_id={scenario} ),mmu.location ) ); delete from mmu where scenario_id={scenario} and not st_contains( ( select location from footprint inner join classification on classification.name=footprint.name where classification.category='footprint' and classification.fclass in ('political_boundary','study_area') and scenario_id={scenario} ),mmu.location ); """.format(scenario=self.__scenario) self.__Indicator = Indicator(self.__user) db = self.__Indicator.get_up_calculator_connection() db.execute(query) db.close() self.__evaluateAmenityBuffers()
def run(self): try: self.__Indicator = Indicator(self.__user) road_classes=self.__getClassess("road_infrastructure","roads") road_classes_array="'{"+",".join(road_classes)+"}'" risk_classes=self.__getClassess("risk","risk") risk_classes_array="'{"+",".join(risk_classes)+"}'" error = True count = 0 while error and count < 3: print("Entrando\n") self.__Indicator = Indicator(self.__user) db = self.__Indicator.get_up_calculator_connection() try: query = """ select urbper_indicator_road_infrastructure_in_risk({scenario},'road_risk'::varchar(10),{road_class},{risk_classes}) """.format( scenario=self.__scenario, road_class=road_classes_array, risk_classes=risk_classes_array ) LogEvents( "road infrastructure in risk", "road infrastructure in risk module started: " + query, self.__scenario, self.__user ) with transaction.atomic(): db.execute(query) except Exception as e: error = True count += 1 time.sleep(randint(1, 3)) db.close() LogEvents( "road infrastructure in risk", "road infrastructure in risk module failed " + str(count) + ": " + str(e), self.__scenario, self.__user ) else: error = False db.close() LogEvents( "road infrastructure in risk", "road infrastructure in risk module finished", self.__scenario, self.__user ) except Exception as e: LogEvents( "road infrastructure in risk", "unknown error " + str(e), self.__scenario, self.__user )
def run(self): try: self.__Indicator = Indicator(self.__user) inadequate_classes=self.__getClassess("inadequate_hu") inadequate_array="'{"+",".join(inadequate_classes)+"}'" error = True count = 0 while error and count < 3: self.__Indicator = Indicator(self.__user) db = self.__Indicator.get_up_calculator_connection() try: query = """ select urbper_indicator_population_inadequate_housing({scenario},{inadequate}) """.format( scenario=self.__scenario, inadequate=inadequate_array ) LogEvents( "urbper_indicator population in inadequate housing", "population in inadequate housing started: " + query, self.__scenario, self.__user ) with transaction.atomic(): db.execute(query) except Exception as e: error = True count += 1 time.sleep(randint(1, 3)) db.close() LogEvents( "population in inadequate housing", "population in inadequate housing module failed " + str(count) + ": " + str(e), self.__scenario, self.__user ) else: error = False db.close() LogEvents( "population in inadequate housing", "population in inadequate housing module finished", self.__scenario, self.__user ) except Exception as e: LogEvents( "population in inadequate housing", "unknown error " + str(e), self.__scenario, self.__user )
def run(self): try: self.__Indicator = Indicator(self.__user) greenland_classes=green_classes=self.__getClassess("green_land") greenland_array="'{"+",".join(greenland_classes)+"}'" error = True count = 0 while error and count < 3: self.__Indicator = Indicator(self.__user) db = self.__Indicator.get_up_calculator_connection() try: query = """ select urbper_indicator_green_land_consumption({scenario},'greenland_consumption',{greenland}) """.format( scenario=self.__scenario, greenland=greenland_array, ) LogEvents( "green land consumption", "green land consumption module started: " + query, self.__scenario, self.__user ) with transaction.atomic(): db.execute(query) except Exception as e: error = True count += 1 time.sleep(randint(1, 3)) db.close() LogEvents( "green land consumption", "green land consumption module failed " + str(count) + ": " + str(e), self.__scenario, self.__user ) else: error = False db.close() LogEvents( "green land consumption", "green land consumption module finished", self.__scenario, self.__user ) except Exception as e: LogEvents( "green land consumption", "unknown error " + str(e), self.__scenario, self.__user )
def run(self): try: error = True count = 0 dwells_classes= self.__getAmenityClassess() dwells_array="'{"+",".join(dwells_classes)+"}'" while error and count < 3: self.__Indicator = Indicator(self.__user) db = self.__Indicator.get_up_calculator_connection() try: query = """ select urbper_indicator_potable_water_wells({scenario},{fclass_array}) """.format( scenario=self.__scenario,fclass_array=dwells_array ) LogEvents( "potable water wells", "potable water wells module started: " + query, self.__scenario, self.__user ) with transaction.atomic(): db.execute(query) except Exception as e: error = True count += 1 time.sleep(randint(1, 3)) db.close() LogEvents( "potable water wells", "potable water wells module failed " + str(count) + ": " + str(e), self.__scenario, self.__user ) else: error = False db.close() LogEvents( "potable water wells", "potable water wells module finished", self.__scenario, self.__user ) except Exception as e: LogEvents( "potable water wells", "unknown error " + str(e), self.__scenario, self.__user )
def run(self): try: error = True count = 0 while error and count < 3: self.__Indicator = Indicator(self.__user) db = self.__Indicator.get_up_calculator_connection() try: amenity_classes = [] amenity_classes.append("elementary_school") amenity_classes_array="'{"+",".join(amenity_classes)+"}'" query = """ select urbper_indicator_elementary_school_capacity({scenario},'elementary_capacity'::varchar(30),{fclass_array}) """.format(scenario=self.__scenario, fclass_array=amenity_classes_array) LogEvents( "elementary school capacity", "elementary school capacity module started: " + query, self.__scenario, self.__user ) with transaction.atomic(): db.execute(query) except Exception as e: error = True count += 1 time.sleep(randint(1, 3)) db.close() LogEvents( "elementary school capacity", "elementary school capacity module failed " + str(count) + ": " + str(e) , self.__scenario, self.__user ) else: error = False db.close() LogEvents( "elementary school capacity", "elementary school capacity module finished", self.__scenario, self.__user ) except Exception as e: LogEvents( "elementary school capacity", "unknown error " + str(e), self.__scenario, self.__user )
def run(self): try: # get agricultural land agricultural_sus = self.__getClassess("agricultural_sus") # get sustainable agricultural land agricultural_sus_array = "'{" + ",".join(agricultural_sus) + "}'" agricultural = self.__getClassess("agricultural") agricultural_array = "'{" + ",".join(agricultural) + "}'" error = True count = 0 while error and count < 3: self.__Indicator = Indicator(self.__user) db = self.__Indicator.get_up_calculator_connection() try: query = """ select urbper_indicator_sustainable_agricultural_land({scenario},{agricultural_sus_array},{agricultural_array}) """.format( scenario=self.__scenario, agricultural_sus_array=agricultural_sus_array, agricultural_array=agricultural_array) LogEvents( "sustainable agricultural land", "sustainable agricultural land started: " + query, self.__scenario, self.__user) with transaction.atomic(): db.execute(query) except Exception as e: error = True count += 1 time.sleep(randint(1, 3)) db.close() LogEvents( "sustainable agricultural land", "sustainable agricultural land module failed " + str(count) + ": " + str(e), self.__scenario, self.__user) else: error = False db.close() LogEvents("sustainable agricultural land", "sustainable agricultural land module finished", self.__scenario, self.__user) except Exception as e: LogEvents("sustainable agricultural land", "unknown error " + str(e), self.__scenario, self.__user)
def run(self): try: self.__Indicator = Indicator(self.__user) heritage_area_classes_set = assumptions.objects.filter( category='heritage').values("name") heritage_area_classes = [] for heritage_area in list(heritage_area_classes_set): heritage_area_classes.append(heritage_area["name"]) heritage_area_array = "'{" + ",".join(heritage_area_classes) + "}'" error = True count = 0 while error and count < 3: self.__Indicator = Indicator(self.__user) db = self.__Indicator.get_up_calculator_connection() try: query = """ select urbper_indicator_solid_waste_management_coverage({scenario},'solidw_coverage') """.format( scenario=self.__scenario, heritage_area=heritage_area_array, ) LogEvents( "solid waste management coverage", "solid waste management coverage module started: " + query, self.__scenario, self.__user) with transaction.atomic(): db.execute(query) except Exception as e: error = True count += 1 time.sleep(randint(1, 3)) db.close() LogEvents( "solid waste management coverage", "solid waste management coverage module failed " + str(count) + ": " + str(e), self.__scenario, self.__user) else: error = False db.close() LogEvents( "solid waste management coverage", "solid waste management coverage module finished", self.__scenario, self.__user) except Exception as e: LogEvents("solid waste management coverage", "unknown error " + str(e), self.__scenario, self.__user)
def run(self): try: error = True count = 0 while error and count < 3: self.__Indicator = Indicator(self.__user) db = self.__Indicator.get_up_calculator_connection() try: query = """ select urbper_indicator_elementary_school_ratio({scenario},'elemax_perc'::varchar(30)) """.format(scenario=self.__scenario) LogEvents( "elementary school ratio", "elementary school ratio module started: " + query, self.__scenario, self.__user ) with transaction.atomic(): db.execute(query) except Exception as e: error = True count += 1 time.sleep(randint(1, 3)) db.close() LogEvents( "elementary school ratio", "elementary school ratio module failed " + str(count) + ": " + str(e) , self.__scenario, self.__user ) else: error = False db.close() LogEvents( "elementary school ratio", "elementary school ratio module finished", self.__scenario, self.__user ) except Exception as e: LogEvents( "elementary school ratio", "unknown error " + str(e), self.__scenario, self.__user )
class Module: def __init__(self, user, scenario, extra_dict_arguments=None): self.__user = user self.__scenario = scenario self.__base_scenario = extra_dict_arguments["base_scenario"] def run(self): self.__Indicator = Indicator(self.__user) vacuum(self.__Indicator.get_uri(), "mmu") self.__urbper_base_total_population() def __urbper_base_total_population(self): try: error = True count = 0 while error and count < 3: self.__Indicator = Indicator(self.__user) db = self.__Indicator.get_up_calculator_connection() try: with transaction.atomic(): self.__Indicator = Indicator(self.__user) db = self.__Indicator.get_up_calculator_connection() query = """select urbper_indicator_land_consumption({scenario})""".format( scenario=self.__scenario) LogEvents("land consumption", "land consumption module started: " + query, self.__scenario, self.__user) db.execute(query) except Exception as e: db.close() error = True count += 1 time.sleep(randint(1, 3)) LogEvents( "land consumption", "land consumption module failed " + str(count) + ": " + str(e), self.__scenario, self.__user, True) else: error = False db.close() LogEvents("land consumption", "land consumption module finished", self.__scenario, self.__user) except Exception as e: LogEvents("land consumption", "unknown error " + str(e), self.__scenario, self.__user, True)
def run(self): try: self.__Indicator = Indicator(self.__user) self.__db = self.__Indicator.get_up_calculator_connection() vacuum(self.__Indicator.get_uri(), "mmu") self.__db.close() amenity_classes_set = self.__getAmentityClassess() error = True count = 0 while error and count < 3: self.__Indicator = Indicator(self.__user) db = self.__Indicator.get_up_calculator_connection() try: for amenity in amenity_classes_set: amenity_classes = [] amenity_classes.append(amenity) amenity_classes_array = "'{" + ",".join( amenity_classes) + "}'" query = """ select urbper_indicator_pop_amenity_prox({scenario},'pop_prox_{fclass}'::varchar(30),'{fclass}_proximity'::varchar(30),{fclass_array}) """.format(scenario=self.__scenario, fclass=amenity, fclass_array=amenity_classes_array) LogEvents( amenity + " proximity", amenity + " proximity module started: " + query, self.__scenario, self.__user) with transaction.atomic(): db.execute(query) except Exception as e: error = True count += 1 time.sleep(randint(1, 3)) db.close() LogEvents( "amenity proximity", "amenity proximity module failed " + str(count) + ": " + str(e), self.__scenario, self.__user) else: error = False db.close() LogEvents("amenity proximity", "amenity proximity module finished", self.__scenario, self.__user) except Exception as e: LogEvents("amenity proximity", "unknown error " + str(e), self.__scenario, self.__user)
def run(self): try: self.__Indicator = Indicator(self.__user) amenity_classes = self.__getAmenitiesClassess() amenity_classes_array = "'{" + ",".join(amenity_classes) + "}'" risk_classes = self.__getRiskClassess() risk_classes_array = "'{" + ",".join(risk_classes) + "}'" error = True count = 0 while error and count < 3: self.__Indicator = Indicator(self.__user) db = self.__Indicator.get_up_calculator_connection() try: query = """ select urbper_indicator_health_infrastructure_in_risk({scenario},'health_risk',{amenity_class},{risk_classes}) """.format(scenario=self.__scenario, amenity_class=amenity_classes_array, risk_classes=risk_classes_array) LogEvents( "health infrastructure in risk", "health infrastructure in risk module started: " + query, self.__scenario, self.__user) with transaction.atomic(): db.execute(query) except Exception as e: error = True count += 1 time.sleep(randint(1, 3)) db.close() LogEvents( "health infrastructure in risk", "health infrastructure in risk module failed " + str(count) + ": " + str(e), self.__scenario, self.__user) else: error = False db.close() LogEvents("health infrastructure in risk", "health infrastructure in risk module finished", self.__scenario, self.__user) except Exception as e: LogEvents("health infrastructure in risk", "unknown error " + str(e), self.__scenario, self.__user)
def __mmu_limit_offset(self): try: self.__Indicator = Indicator(self.__user) db = self.__Indicator.get_up_calculator_connection() try: # get the max an min of pk self.__limits["inferior"] = mmu.objects.filter( scenario_id=self.__scenario).aggregate(Min('mmu_id'))["mmu_id__min"] self.__limits["superior"] = mmu.objects.filter( scenario_id=self.__scenario).aggregate(Max('mmu_id'))["mmu_id__max"] except Exception as e: LogEvents("squares max min", "unknown error " + str(e), self.__scenario, self.__user, True) db.close() except Exception as e: LogEvents("squares max min", "unknown error " + str(e), self.__scenario, self.__user, True)
class Module: def __init__(self, user, scenario, extra_dict_arguments=None): self.__user = user self.__scenario = scenario def run(self): try: self.__Indicator = Indicator(self.__user) self.__db = self.__Indicator.get_up_calculator_connection() vacuum(self.__Indicator.get_uri(), "mmu") self.__db.close() error = True count = 0 while error and count < 3: self.__Indicator = Indicator(self.__user) db = self.__Indicator.get_up_calculator_connection() try: query = """ select urbper_indicator_pop_transit_prox({scenario},'pop_prox_{fclass}'::varchar(30),'{fclass}_proximity'::varchar(30)) """.format(scenario=self.__scenario, fclass='transit') LogEvents("transit proximity", "transit proximity module started: " + query, self.__scenario, self.__user) with transaction.atomic(): db.execute(query) except Exception as e: error = True count += 1 time.sleep(randint(1, 3)) db.close() LogEvents( "transit proximity", "transit proximity module failed " + str(count) + ": " + str(e), self.__scenario, self.__user) else: error = False db.close() LogEvents("transit proximity", "transit proximity module finished", self.__scenario, self.__user) except Exception as e: LogEvents("transit proximity", "unknown error " + str(e), self.__scenario, self.__user)
def __ModuleAreaDensity(self, scenario_id, offset=0, limit=0): try: error = True count = 0 while error and count < 3: self.__Indicator = Indicator(self.__user) db = self.__Indicator.get_up_calculator_connection() try: query = """select urbper_indicator_area_den_avg({scenario},{offset},{limit})""".format( scenario=scenario_id, offset=offset, limit=limit) LogEvents("area density avg", "area density avg module started: " + query, scenario_id, self.__user) conn = psycopg2.connect(self.__Indicator.get_uri()) cursor = conn.cursor( cursor_factory=psycopg2.extras.DictCursor) old_isolation_level = conn.isolation_level conn.set_isolation_level(0) cursor.execute(query) conn.commit() conn.set_isolation_level(old_isolation_level) except Exception as e: error = True count += 1 time.sleep(randint(1, 3)) LogEvents( "area density avg", "area density avg module failed " + str(count) + ": " + str(e), scenario_id, self.__user, True) conn.close() else: error = False LogEvents("area density avg", "area density avg module finished", scenario_id, self.__user) conn.close() except Exception as e: LogEvents("Running scenarios", "Unknown error " + str(e), scenario_id, self.__user, True)
class Module: def __init__(self, user, scenario, extra_dict_arguments=None): self.__user = user self.__scenario = scenario def run(self): try: error = True count = 0 while error and count < 3: self.__Indicator = Indicator(self.__user) db = self.__Indicator.get_up_calculator_connection() try: query = """ select urbper_indicator_connected_to_electricity({scenario}) """.format( scenario=self.__scenario ) LogEvents( "connected to electricity", "connected to electricity module started: " + query, self.__scenario, self.__user ) with transaction.atomic(): db.execute(query) except Exception as e: error = True count += 1 time.sleep(randint(1, 3)) db.close() LogEvents( "connected to electricity risk", "connected to electricity module failed " + str(count) + ": " + str(e), self.__scenario, self.__user ) else: error = False db.close() LogEvents( "connected to electricity", "connected to electricity module finished", self.__scenario, self.__user ) except Exception as e: LogEvents( "connected to electricity", "unknown error " + str(e), self.__scenario, self.__user )
def __urbper_base_footprint_area(self): try: error = True count = 0 while error and count < 3: try: self.__Indicator = Indicator(self.__user) query = """select urbper_indicator_footprints_area({scenario})""".format( scenario=self.__scenario) LogEvents("footprints area", "footprints area module started: " + query, self.__scenario, self.__user) conn = psycopg2.connect(self.__Indicator.get_uri()) cursor = conn.cursor( cursor_factory=psycopg2.extras.DictCursor) old_isolation_level = conn.isolation_level conn.set_isolation_level(0) cursor.execute(query) conn.commit() conn.set_isolation_level(old_isolation_level) except Exception as e: conn.rollback() conn.close() error = True count += 1 time.sleep(randint(1, 3)) LogEvents( "footprints area", "footprints area module failed " + str(count) + ": " + str(e), self.__scenario, self.__user, True) else: error = False conn.close() LogEvents("footprints area", "footprints area module finished", self.__scenario, self.__user) except Exception as e: LogEvents("footprints area", "unknown error+ " + str(e), self.__scenario, self.__user, True)
def run(self): try: self.__Indicator = Indicator(self.__user) road_classes = self.__getClassess("allseason") road_classes_array = "'{" + ",".join(road_classes) + "}'" error = True count = 0 while error and count < 3: self.__Indicator = Indicator(self.__user) db = self.__Indicator.get_up_calculator_connection() try: query = """ select urbper_indicator_road_all_season_proximity({scenario},'pop_prox_allseason','allseason_prox',{road_class}) """.format(scenario=self.__scenario, road_class=road_classes_array) LogEvents("all season roads", "all season roads module started: " + query, self.__scenario, self.__user) with transaction.atomic(): db.execute(query) except Exception as e: error = True count += 1 time.sleep(randint(1, 3)) db.close() LogEvents( "all season roads", "all season roads module failed " + str(count) + ": " + str(e), self.__scenario, self.__user) else: error = False db.close() LogEvents("all season roads", "all season roads module finished", self.__scenario, self.__user) except Exception as e: LogEvents("all season roads", "unknown error " + str(e), self.__scenario, self.__user)
def __urbper_base_total_population(self): try: error = True count = 0 while error and count < 3: self.__Indicator = Indicator(self.__user) db = self.__Indicator.get_up_calculator_connection() try: with transaction.atomic(): self.__Indicator = Indicator(self.__user) db = self.__Indicator.get_up_calculator_connection() query = """select urbper_indicator_base_calculus_total_population({scenario},{base_scenario})""".format( scenario=self.__scenario, base_scenario=self.__base_scenario) LogEvents("total population", "total population module started: " + query, self.__scenario, self.__user) db.execute(query) except Exception as e: db.close() error = True count += 1 time.sleep(randint(1, 3)) LogEvents( "total population", "total population module failed " + str(count) + ": " + str(e), self.__scenario, self.__user, True) else: error = False db.close() LogEvents("total population", "total population module finished", self.__scenario, self.__user) except Exception as e: LogEvents("total population", "unknown error " + str(e), self.__scenario, self.__user, True)
class Module: def __init__(self, user, scenario, extra_dict_arguments=None): self.__user = user self.__scenario = scenario self.__base_scenario = extra_dict_arguments["base_scenario"] def run(self): query = """ delete from mmu_info where mmu_id in ( select mmu_id from mmu where scenario_id={scenario} and not st_contains( ( select location from footprint inner join classification on classification.name=footprint.name where classification.category='footprint' and classification.fclass in ('political_boundary','study_area') and scenario_id={scenario} ),mmu.location ) ); delete from mmu where scenario_id={scenario} and not st_contains( ( select location from footprint inner join classification on classification.name=footprint.name where classification.category='footprint' and classification.fclass in ('political_boundary','study_area') and scenario_id={scenario} ),mmu.location ); """.format(scenario=self.__scenario) self.__Indicator = Indicator(self.__user) db = self.__Indicator.get_up_calculator_connection() db.execute(query) db.close() self.__EvaluateJobsBuffer() def __EvaluateJobsBuffer(self): self.__limits = {"inferior": 0, "superior": 0} self.__jobs_limit_offset() self.__jobs_preprocessing_threads() def __jobs_limit_offset(self): try: try: # get the max an min of pk self.__limits["inferior"] = jobs.objects.filter( scenario_id=self.__scenario).aggregate(Min('jobs_id'))["jobs_id__min"] self.__limits["superior"] = jobs.objects.filter( scenario_id=self.__scenario).aggregate(Max('jobs_id'))["jobs_id__max"] except Exception as e: LogEvents("jobs max min", "unknown error " + str(e), self.__scenario, self.__user, True) except Exception as e: LogEvents("jobs max min", "unknown error " + str(e), self.__scenario, self.__user, True) def __jobs_preprocessing_threads(self): self.__scenario_t = {} self.__scenario_t["limit"] = 0 self.__scenario_t["offset"] = 0 inferior = self.__limits["inferior"] superior = self.__limits["superior"] _threads = {} self.max_threads = min(self.__Indicator.get_max_threads(), int(math.ceil( (superior - inferior) / self.__Indicator.get_max_rows()))) num_partitions = self.max_threads partition_size = (int)( math.ceil((superior - inferior) / self.max_threads)) for h in range(0, num_partitions): self.__scenario_t["offset"] = inferior self.__scenario_t["limit"] = self.__scenario_t["offset"] + \ partition_size inferior = self.__scenario_t["limit"] + 1 _threads[h] = threading.Thread(target=self.__ModuleJobsDensity, args=( self.__scenario, self.__scenario_t["offset"], self.__scenario_t["limit"])) for process in _threads: _threads[process].start() for process in _threads: if _threads[process].is_alive(): _threads[process].join() def __ModuleJobsDensity(self, scenario_id, offset=0, limit=0): import psycopg2 import psycopg2.extras try: error = True count = 0 while error and count < 3: try: query = """select urbper_buffer_job_and_job_density({scenario},{offset},{limit})""".format( scenario=scenario_id, offset=offset, limit=limit) LogEvents("job buffers and density", "job buffers and density module started: " + query, scenario_id, self.__user) conn = psycopg2.connect(self.__Indicator.get_uri()) cursor = conn.cursor( cursor_factory=psycopg2.extras.DictCursor) old_isolation_level = conn.isolation_level conn.set_isolation_level(0) cursor.execute(query) conn.commit() conn.set_isolation_level(old_isolation_level) except Exception as e: error = True count += 1 LogEvents("job buffers and density ", "job buffers and density module failed " + str(count) + ": " + str(e), scenario_id, self.__user, True) else: error = False # db.commit() LogEvents("job buffers and density ", "job buffers and density module finished", scenario_id, self.__user) except Exception as e: LogEvents("Running scenarios", "Unknown error " + str(e), scenario_id, self.__user, True)
class Module: def __init__(self, user, scenario, extra_dict_arguments=None): self.__user = user self.__scenario = scenario def run(self): try: # get agricultural land agricultural_sus = self.__getClassess("agricultural_sus") # get sustainable agricultural land agricultural_sus_array = "'{" + ",".join(agricultural_sus) + "}'" agricultural = self.__getClassess("agricultural") agricultural_array = "'{" + ",".join(agricultural) + "}'" error = True count = 0 while error and count < 3: self.__Indicator = Indicator(self.__user) db = self.__Indicator.get_up_calculator_connection() try: query = """ select urbper_indicator_sustainable_agricultural_land({scenario},{agricultural_sus_array},{agricultural_array}) """.format( scenario=self.__scenario, agricultural_sus_array=agricultural_sus_array, agricultural_array=agricultural_array) LogEvents( "sustainable agricultural land", "sustainable agricultural land started: " + query, self.__scenario, self.__user) with transaction.atomic(): db.execute(query) except Exception as e: error = True count += 1 time.sleep(randint(1, 3)) db.close() LogEvents( "sustainable agricultural land", "sustainable agricultural land module failed " + str(count) + ": " + str(e), self.__scenario, self.__user) else: error = False db.close() LogEvents("sustainable agricultural land", "sustainable agricultural land module finished", self.__scenario, self.__user) except Exception as e: LogEvents("sustainable agricultural land", "unknown error " + str(e), self.__scenario, self.__user) def __getClassess(self, fclass): try: query = """select distinct classification.name from classification where classification.category='footprint' and classification.fclass='{fclass}' """.format(fclass=fclass) results = classification.objects.filter( category='footprint', fclass=fclass).distinct().values_list('name', flat=True) LogEvents("classes", "classes finished: " + query, self.__scenario, self.__user) results_set = [row for row in results] results = results_set except Exception as e: error = True time.sleep(randint(1, 3)) LogEvents("classes", "classes failed: " + str(e), self.__scenario, self.__user) return [] else: error = False LogEvents("classes", "classes finished", self.__scenario, self.__user) return results
class Module: def __init__(self, user, scenario, extra_dict_arguments=None): self.__user = user self.__scenario = scenario def run(self): try: self.__Indicator = Indicator(self.__user) biodiversity_classes = self.__getBiodiversityClassess() biodiversity_array = "'{" + ",".join(biodiversity_classes) + "}'" error = True count = 0 while error and count < 3: self.__Indicator = Indicator(self.__user) db = self.__Indicator.get_up_calculator_connection() try: query = """ select urbper_indicator_biodiversity_land_consumption({scenario},'biodiversity_consumption',{biodiversity}) """.format( scenario=self.__scenario, biodiversity=biodiversity_array, ) LogEvents( "biodiversity land consumption", "biodiversity land consumption module started: " + query, self.__scenario, self.__user) with transaction.atomic(): db.execute(query) except Exception as e: error = True count += 1 time.sleep(randint(1, 3)) db.close() LogEvents( "biodiversity land consumption", "biodiversity land consumption module failed " + str(count) + ": " + str(e), self.__scenario, self.__user) else: error = False db.close() LogEvents("biodiversity land consumption", "biodiversity land consumption module finished", self.__scenario, self.__user) except Exception as e: LogEvents("biodiversity land consumption", "unknown error " + str(e), self.__scenario, self.__user) def __getBiodiversityClassess(self): try: error = True count = 0 while error and count < 3: try: query = """select distinct classification.name from classification where classification.category='footprint' and classification.fclass='biodiversity' """ results = classification.objects.filter( category='footprint', fclass='biodiversity').distinct().values_list( 'name', flat=True) LogEvents("biodiversity", "biodiversity classes finished: " + query, self.__scenario, self.__user) results_set = [row for row in results] results = results_set except Exception as e: error = True count += 1 time.sleep(randint(1, 3)) LogEvents( "biodiversity", "biodiversity classes failed " + str(count) + ": " + str(e), self.__scenario, self.__user) else: error = False LogEvents("biodiversity", "biodiversity classes finished", self.__scenario, self.__user) return results except Exception as e: LogEvents("biodiversity", "unknown error " + str(e), self.__scenario, self.__user)
class Module: def __init__(self, user, scenario, extra_dict_arguments=None): self.__user = user self.__scenario = scenario self.__base_scenario = extra_dict_arguments["base_scenario"] def run(self): self.__limits = {"inferior": 0, "superior": 0} self.__mmu_limit_offset() self.__job_density_threads() self.__energy_transport() def __mmu_limit_offset(self): try: self.__Indicator = Indicator(self.__user) try: # get the max an min of pk self.__limits["inferior"] = mmu.objects.filter( scenario_id=self.__scenario).aggregate( Min('mmu_id'))["mmu_id__min"] self.__limits["superior"] = mmu.objects.filter( scenario_id=self.__scenario).aggregate( Max('mmu_id'))["mmu_id__max"] except Exception as e: LogEvents("squares max min", "unknown error " + str(e), self.__scenario, self.__user, True) except Exception as e: LogEvents("squares max min", "unknown error " + str(e), self.__scenario, self.__user, True) def __job_density_threads(self): self.__scenario_t = {} self.__scenario_t["limit"] = 0 self.__scenario_t["offset"] = 0 inferior = self.__limits["inferior"] superior = self.__limits["superior"] _threads = {} self.max_threads = min( self.__Indicator.get_max_threads(), int( math.ceil( (superior - inferior) / self.__Indicator.get_max_rows()))) num_partitions = self.max_threads partition_size = (int)(math.ceil( (superior - inferior) / self.max_threads)) for h in range(0, num_partitions): self.__scenario_t["offset"] = inferior self.__scenario_t["limit"] = self.__scenario_t["offset"] + \ partition_size inferior = self.__scenario_t["limit"] + 1 _threads[h] = threading.Thread(target=self.__ModulejobDensity, args=(self.__scenario, self.__scenario_t["offset"], self.__scenario_t["limit"])) for process in _threads: _threads[process].start() for process in _threads: if _threads[process].is_alive(): _threads[process].join() def __ModulejobDensity(self, scenario_id, offset=0, limit=0): try: error = True count = 0 while error and count < 3: self.__Indicator = Indicator(self.__user) db = self.__Indicator.get_up_calculator_connection() try: with transaction.atomic(): query = """select urbper_indicator_job_density({scenario},{offset},{limit})""".format( scenario=scenario_id, offset=offset, limit=limit) LogEvents("job density", "job density module started: " + query, scenario_id, self.__user) db.execute(query) except Exception as e: error = True count += 1 LogEvents( "job density ", "job density module failed " + str(count) + ": " + str(e), scenario_id, self.__user, True) db.close() else: error = False LogEvents("job density ", "job density module finished", scenario_id, self.__user) db.close() except Exception as e: LogEvents("Running scenarios", "Unknown error " + str(e), scenario_id, self.__user, True)
def __general_calculus(self): self.__Indicator = Indicator(self.__user) vacuum(self.__Indicator.get_uri(), "mmu") self.__urbper_base_footprint_area() self.__urbper_base_total_population() self.__urbper_base_footprint_km2()
class Module: def __init__(self, user, scenario, extra_dict_arguments=None): self.__user = user self.__scenario = scenario self.__base_scenario = extra_dict_arguments["base_scenario"] def run(self): query = """ delete from mmu_info where mmu_id in ( select mmu_id from mmu where scenario_id={scenario} and not st_contains( ( select location from footprint inner join classification on classification.name=footprint.name where classification.category='footprint' and classification.fclass in ('political_boundary','study_area') and scenario_id={scenario} ),mmu.location ) ); delete from mmu where scenario_id={scenario} and not st_contains( ( select location from footprint inner join classification on classification.name=footprint.name where classification.category='footprint' and classification.fclass in ('political_boundary','study_area') and scenario_id={scenario} ),mmu.location ); """.format(scenario=self.__scenario) self.__Indicator = Indicator(self.__user) db = self.__Indicator.get_up_calculator_connection() db.execute(query) db.close() self.__evaluateAmenityBuffers() def __evaluateAmenityBuffers(self): import psycopg2 import psycopg2.extras try: error = True count = 0 while error and count < 3: try: query = """ select urbper_buffer_amenities({scenario}) """.format(scenario=self.__scenario) LogEvents( "amenity buffers", "amenity buffer module started: " + query, self.__scenario, self.__user ) conn = psycopg2.connect(self.__Indicator.get_uri()) cursor = conn.cursor( cursor_factory=psycopg2.extras.DictCursor) old_isolation_level = conn.isolation_level conn.set_isolation_level(0) cursor.execute(query) conn.commit() conn.set_isolation_level(old_isolation_level) except Exception as e: error = True count += 1 time.sleep(randint(1, 3)) LogEvents( "amenity buffers", "amenity buffers module failed " + str(count) + ": " + str(e), self.__scenario, self.__user ) else: error = False LogEvents( "amenity buffers", "amenity buffers module finished", self.__scenario, self.__user ) except Exception as e: LogEvents( "amenity buffers", "unknown error " + str(e), self.__scenario, self.__user )
class Module: def __init__(self, user, scenario, extra_dict_arguments=None): self.__user = user self.__scenario = scenario def run(self): try: self.__Indicator = Indicator(self.__user) green_classes = self.__getClassess("green_land") green_array = "'{" + ",".join(green_classes) + "}'" protected_classes = self.__getClassess("protected") protected_array = "'{" + ",".join(protected_classes) + "}'" error = True count = 0 while error and count < 3: self.__Indicator = Indicator(self.__user) db = self.__Indicator.get_up_calculator_connection() try: query = """ select urbper_indicator_greenland_availability({scenario},'green_availability',{green},{protected}) """.format( scenario=self.__scenario, green=green_array, protected=protected_array, ) LogEvents( "green land availability", "green land availability module started: " + query, self.__scenario, self.__user) with transaction.atomic(): db.execute(query) except Exception as e: error = True count += 1 time.sleep(randint(1, 3)) db.close() LogEvents( "green land availability", "green land availability module failed " + str(count) + ": " + str(e), self.__scenario, self.__user) else: error = False db.close() LogEvents("green land availability", "green land availability module finished", self.__scenario, self.__user) except Exception as e: LogEvents("green land availability", "unknown error " + str(e), self.__scenario, self.__user) def __getClassess(self, fclass): try: query = """select distinct classification.name from classification where classification.category='footprint' and classification.fclass='{fclass}' """.format(fclass=fclass) results = classification.objects.filter( category='footprint', fclass=fclass).distinct().values_list('name', flat=True) LogEvents("classes", "classes finished: " + query, self.__scenario, self.__user) # cursor.execute(query) results_set = [row for row in results] results = results_set except Exception as e: time.sleep(randint(1, 3)) LogEvents("classes", "classes failed :" + str(e), self.__scenario, self.__user) else: error = False LogEvents("classes", "classes finished", self.__scenario, self.__user) return results
class Module: def __init__(self, user, scenario, extra_dict_arguments=None): self.__user = user self.__scenario = scenario def run(self): try: self.__Indicator = Indicator(self.__user) road_classes = self.__getClassess("road_season") road_classes_array = "'{" + ",".join(road_classes) + "}'" risk_classes = self.__getClassess("risk") risk_classes_array = "'{" + ",".join(risk_classes) + "}'" error = True count = 0 while error and count < 3: print("Entrando\n") self.__Indicator = Indicator(self.__user) db = self.__Indicator.get_up_calculator_connection() try: query = """ select urbper_indicator_road_all_season_proximity({scenario},'allseason_prox'::varchar(10),'pop_prox_allseason'::varchar(10),{road_class}) """.format(scenario=self.__scenario, road_class=road_classes_array) LogEvents("all season roads", "all season roads module started: " + query, self.__scenario, self.__user) with transaction.atomic(): db.execute(query) except Exception as e: error = True count += 1 time.sleep(randint(1, 3)) db.close() LogEvents( "all season roads", "all season roads module failed " + str(count) + ": " + str(e), self.__scenario, self.__user) else: error = False db.close() LogEvents("all season roads", "all season roads module finished", self.__scenario, self.__user) except Exception as e: LogEvents("all season roads", "unknown error " + str(e), self.__scenario, self.__user) def __getClassess(self, fclass): try: query = """select distinct classification.name from classification where classification.category='footprint' and classification.fclass='{fclass}' """.format(fclass=fclass) results = classification.objects.filter( category='footprint', fclass=fclass).distinct().values_list('name', flat=True) LogEvents("classes", "classes finished: " + query, self.__scenario, self.__user) results_set = [row for row in results] results = results_set except Exception as e: error = True time.sleep(randint(1, 3)) LogEvents("classes", "classes failed: " + str(e), self.__scenario, self.__user) else: error = False LogEvents("classes", "classes finished", self.__scenario, self.__user) return results