class Module:
def __init__(self, user, scenario, extra_dict_arguments=None):
self.__user = user
self.__scenario = scenario
self.__base_scenario = extra_dict_arguments["base_scenario"]
def run(self):
self.__Indicator = Indicator(self.__user)
vacuum(self.__Indicator.get_uri(), "mmu")
self.__urbper_base_total_population()
def __urbper_base_total_population(self):
try:
error = True
count = 0
while error and count < 3:
self.__Indicator = Indicator(self.__user)
db = self.__Indicator.get_up_calculator_connection()
try:
with transaction.atomic():
self.__Indicator = Indicator(self.__user)
db = self.__Indicator.get_up_calculator_connection()
query = """select urbper_indicator_land_consumption({scenario})""".format(
scenario=self.__scenario)
LogEvents("land consumption",
"land consumption module started: " + query,
self.__scenario, self.__user)
db.execute(query)
except Exception as e:
db.close()
error = True
count += 1
time.sleep(randint(1, 3))
LogEvents(
"land consumption", "land consumption module failed " +
str(count) + ": " + str(e), self.__scenario,
self.__user, True)
else:
error = False
db.close()
LogEvents("land consumption",
"land consumption module finished",
self.__scenario, self.__user)
except Exception as e:
LogEvents("land consumption", "unknown error " + str(e),
self.__scenario, self.__user, True)
class Module:
def __init__(self, user, scenario, extra_dict_arguments=None):
self.__user = user
self.__scenario = scenario
def run(self):
try:
error = True
count = 0
while error and count < 3:
self.__Indicator = Indicator(self.__user)
db = self.__Indicator.get_up_calculator_connection()
try:
amenity_classes = []
amenity_classes.append("elementary_school")
amenity_classes_array="'{"+",".join(amenity_classes)+"}'"
query = """
select urbper_indicator_elementary_school_capacity({scenario},'elementary_capacity'::varchar(30),{fclass_array})
""".format(scenario=self.__scenario, fclass_array=amenity_classes_array)
LogEvents(
"elementary school capacity",
"elementary school capacity module started: " + query,
self.__scenario,
self.__user
)
with transaction.atomic():
db.execute(query)
except Exception as e:
error = True
count += 1
time.sleep(randint(1, 3))
db.close()
LogEvents(
"elementary school capacity",
"elementary school capacity module failed " +
str(count) + ": " + str(e) ,
self.__scenario,
self.__user
)
else:
error = False
db.close()
LogEvents(
"elementary school capacity",
"elementary school capacity module finished",
self.__scenario,
self.__user
)
except Exception as e:
LogEvents(
"elementary school capacity",
"unknown error " +
str(e),
self.__scenario,
self.__user
)
class Module:
def __init__(self, user, scenario, extra_dict_arguments=None):
self.__user = user
self.__scenario = scenario
def run(self):
try:
self.__Indicator = Indicator(self.__user)
self.__db = self.__Indicator.get_up_calculator_connection()
vacuum(self.__Indicator.get_uri(), "mmu")
self.__db.close()
error = True
count = 0
while error and count < 3:
self.__Indicator = Indicator(self.__user)
db = self.__Indicator.get_up_calculator_connection()
try:
query = """
select urbper_indicator_pop_transit_prox({scenario},'pop_prox_{fclass}'::varchar(30),'{fclass}_proximity'::varchar(30))
""".format(scenario=self.__scenario, fclass='transit')
LogEvents("transit proximity",
"transit proximity module started: " + query,
self.__scenario, self.__user)
with transaction.atomic():
db.execute(query)
except Exception as e:
error = True
count += 1
time.sleep(randint(1, 3))
db.close()
LogEvents(
"transit proximity",
"transit proximity module failed " + str(count) +
": " + str(e), self.__scenario, self.__user)
else:
error = False
db.close()
LogEvents("transit proximity",
"transit proximity module finished",
self.__scenario, self.__user)
except Exception as e:
LogEvents("transit proximity", "unknown error " + str(e),
self.__scenario, self.__user)
class Module:
def __init__(self, user, scenario, extra_dict_arguments=None):
self.__user = user
self.__scenario = scenario
def run(self):
try:
error = True
count = 0
while error and count < 3:
self.__Indicator = Indicator(self.__user)
db = self.__Indicator.get_up_calculator_connection()
try:
query = """
select urbper_indicator_connected_to_electricity({scenario})
""".format(
scenario=self.__scenario
)
LogEvents(
"connected to electricity",
"connected to electricity module started: " + query,
self.__scenario,
self.__user
)
with transaction.atomic():
db.execute(query)
except Exception as e:
error = True
count += 1
time.sleep(randint(1, 3))
db.close()
LogEvents(
"connected to electricity risk",
"connected to electricity module failed " +
str(count) + ": " + str(e),
self.__scenario,
self.__user
)
else:
error = False
db.close()
LogEvents(
"connected to electricity",
"connected to electricity module finished",
self.__scenario,
self.__user
)
except Exception as e:
LogEvents(
"connected to electricity",
"unknown error " +
str(e),
self.__scenario,
self.__user
)
class Module:
def __init__(self, user, scenario, extra_dict_arguments=None):
self.__user = user
self.__scenario = scenario
def run(self):
try:
self.__Indicator = Indicator(self.__user)
heritage_area_classes_set = assumptions.objects.filter(
category='heritage').values("name")
heritage_area_classes = []
for heritage_area in list(heritage_area_classes_set):
heritage_area_classes.append(heritage_area["name"])
heritage_area_array = "'{" + ",".join(heritage_area_classes) + "}'"
error = True
count = 0
while error and count < 3:
self.__Indicator = Indicator(self.__user)
db = self.__Indicator.get_up_calculator_connection()
try:
query = """
select urbper_indicator_solid_waste_management_coverage({scenario},'solidw_coverage')
""".format(
scenario=self.__scenario,
heritage_area=heritage_area_array,
)
LogEvents(
"solid waste management coverage",
"solid waste management coverage module started: " +
query, self.__scenario, self.__user)
with transaction.atomic():
db.execute(query)
except Exception as e:
error = True
count += 1
time.sleep(randint(1, 3))
db.close()
LogEvents(
"solid waste management coverage",
"solid waste management coverage module failed " +
str(count) + ": " + str(e), self.__scenario,
self.__user)
else:
error = False
db.close()
LogEvents(
"solid waste management coverage",
"solid waste management coverage module finished",
self.__scenario, self.__user)
except Exception as e:
LogEvents("solid waste management coverage",
"unknown error " + str(e), self.__scenario, self.__user)
class Module:
def __init__(self, user, scenario, extra_dict_arguments=None):
self.__user = user
self.__scenario = scenario
def run(self):
try:
# get agricultural land
agricultural_sus = self.__getClassess("agricultural_sus")
# get sustainable agricultural land
agricultural_sus_array = "'{" + ",".join(agricultural_sus) + "}'"
agricultural = self.__getClassess("agricultural")
agricultural_array = "'{" + ",".join(agricultural) + "}'"
error = True
count = 0
while error and count < 3:
self.__Indicator = Indicator(self.__user)
db = self.__Indicator.get_up_calculator_connection()
try:
query = """
select urbper_indicator_sustainable_agricultural_land({scenario},{agricultural_sus_array},{agricultural_array})
""".format(
scenario=self.__scenario,
agricultural_sus_array=agricultural_sus_array,
agricultural_array=agricultural_array)
LogEvents(
"sustainable agricultural land",
"sustainable agricultural land started: " + query,
self.__scenario, self.__user)
with transaction.atomic():
db.execute(query)
except Exception as e:
error = True
count += 1
time.sleep(randint(1, 3))
db.close()
LogEvents(
"sustainable agricultural land",
"sustainable agricultural land module failed " +
str(count) + ": " + str(e), self.__scenario,
self.__user)
else:
error = False
db.close()
LogEvents("sustainable agricultural land",
"sustainable agricultural land module finished",
self.__scenario, self.__user)
except Exception as e:
LogEvents("sustainable agricultural land",
"unknown error " + str(e), self.__scenario, self.__user)
def __getClassess(self, fclass):
try:
query = """select distinct classification.name
from classification
where classification.category='footprint'
and classification.fclass='{fclass}'
""".format(fclass=fclass)
results = classification.objects.filter(
category='footprint',
fclass=fclass).distinct().values_list('name', flat=True)
LogEvents("classes", "classes finished: " + query, self.__scenario,
self.__user)
results_set = [row for row in results]
results = results_set
except Exception as e:
error = True
time.sleep(randint(1, 3))
LogEvents("classes", "classes failed: " + str(e), self.__scenario,
self.__user)
return []
else:
error = False
LogEvents("classes", "classes finished", self.__scenario,
self.__user)
return results
class Module:
def __init__(self, user, scenario, extra_dict_arguments=None):
self.__user = user
self.__scenario = scenario
def run(self):
try:
self.__Indicator = Indicator(self.__user)
biodiversity_classes = self.__getBiodiversityClassess()
biodiversity_array = "'{" + ",".join(biodiversity_classes) + "}'"
error = True
count = 0
while error and count < 3:
self.__Indicator = Indicator(self.__user)
db = self.__Indicator.get_up_calculator_connection()
try:
query = """
select urbper_indicator_biodiversity_land_consumption({scenario},'biodiversity_consumption',{biodiversity})
""".format(
scenario=self.__scenario,
biodiversity=biodiversity_array,
)
LogEvents(
"biodiversity land consumption",
"biodiversity land consumption module started: " +
query, self.__scenario, self.__user)
with transaction.atomic():
db.execute(query)
except Exception as e:
error = True
count += 1
time.sleep(randint(1, 3))
db.close()
LogEvents(
"biodiversity land consumption",
"biodiversity land consumption module failed " +
str(count) + ": " + str(e), self.__scenario,
self.__user)
else:
error = False
db.close()
LogEvents("biodiversity land consumption",
"biodiversity land consumption module finished",
self.__scenario, self.__user)
except Exception as e:
LogEvents("biodiversity land consumption",
"unknown error " + str(e), self.__scenario, self.__user)
def __getBiodiversityClassess(self):
try:
error = True
count = 0
while error and count < 3:
try:
query = """select distinct classification.name
from classification
where classification.category='footprint'
and classification.fclass='biodiversity'
"""
results = classification.objects.filter(
category='footprint',
fclass='biodiversity').distinct().values_list(
'name', flat=True)
LogEvents("biodiversity",
"biodiversity classes finished: " + query,
self.__scenario, self.__user)
results_set = [row for row in results]
results = results_set
except Exception as e:
error = True
count += 1
time.sleep(randint(1, 3))
LogEvents(
"biodiversity", "biodiversity classes failed " +
str(count) + ": " + str(e), self.__scenario,
self.__user)
else:
error = False
LogEvents("biodiversity", "biodiversity classes finished",
self.__scenario, self.__user)
return results
except Exception as e:
LogEvents("biodiversity", "unknown error " + str(e),
self.__scenario, self.__user)
class Module:
def __init__(self, user, scenario, extra_dict_arguments=None):
self.__user = user
self.__scenario = scenario
self.__base_scenario = extra_dict_arguments["base_scenario"]
def run(self):
self.__limits = {"inferior": 0, "superior": 0}
self.__mmu_limit_offset()
self.__job_density_threads()
self.__energy_transport()
def __mmu_limit_offset(self):
try:
self.__Indicator = Indicator(self.__user)
try:
# get the max an min of pk
self.__limits["inferior"] = mmu.objects.filter(
scenario_id=self.__scenario).aggregate(
Min('mmu_id'))["mmu_id__min"]
self.__limits["superior"] = mmu.objects.filter(
scenario_id=self.__scenario).aggregate(
Max('mmu_id'))["mmu_id__max"]
except Exception as e:
LogEvents("squares max min", "unknown error " + str(e),
self.__scenario, self.__user, True)
except Exception as e:
LogEvents("squares max min", "unknown error " + str(e),
self.__scenario, self.__user, True)
def __job_density_threads(self):
self.__scenario_t = {}
self.__scenario_t["limit"] = 0
self.__scenario_t["offset"] = 0
inferior = self.__limits["inferior"]
superior = self.__limits["superior"]
_threads = {}
self.max_threads = min(
self.__Indicator.get_max_threads(),
int(
math.ceil(
(superior - inferior) / self.__Indicator.get_max_rows())))
num_partitions = self.max_threads
partition_size = (int)(math.ceil(
(superior - inferior) / self.max_threads))
for h in range(0, num_partitions):
self.__scenario_t["offset"] = inferior
self.__scenario_t["limit"] = self.__scenario_t["offset"] + \
partition_size
inferior = self.__scenario_t["limit"] + 1
_threads[h] = threading.Thread(target=self.__ModulejobDensity,
args=(self.__scenario,
self.__scenario_t["offset"],
self.__scenario_t["limit"]))
for process in _threads:
_threads[process].start()
for process in _threads:
if _threads[process].is_alive():
_threads[process].join()
def __ModulejobDensity(self, scenario_id, offset=0, limit=0):
try:
error = True
count = 0
while error and count < 3:
self.__Indicator = Indicator(self.__user)
db = self.__Indicator.get_up_calculator_connection()
try:
with transaction.atomic():
query = """select urbper_indicator_job_density({scenario},{offset},{limit})""".format(
scenario=scenario_id, offset=offset, limit=limit)
LogEvents("job density",
"job density module started: " + query,
scenario_id, self.__user)
db.execute(query)
except Exception as e:
error = True
count += 1
LogEvents(
"job density ", "job density module failed " +
str(count) + ": " + str(e), scenario_id, self.__user,
True)
db.close()
else:
error = False
LogEvents("job density ", "job density module finished",
scenario_id, self.__user)
db.close()
except Exception as e:
LogEvents("Running scenarios", "Unknown error " + str(e),
scenario_id, self.__user, True)
class Module:
def __init__(self, user, scenario, extra_dict_arguments=None):
self.__user = user
self.__scenario = scenario
self.__base_scenario = extra_dict_arguments["base_scenario"]
def run(self):
query = """
delete from mmu_info where mmu_id in (
select mmu_id
from mmu
where scenario_id={scenario} and not st_contains(
(
select location
from footprint
inner join classification on classification.name=footprint.name
where
classification.category='footprint'
and classification.fclass in ('political_boundary','study_area')
and scenario_id={scenario}
),mmu.location
)
);
delete from mmu
where scenario_id={scenario} and not st_contains(
(
select location
from footprint
inner join classification on classification.name=footprint.name
where
classification.category='footprint'
and classification.fclass in ('political_boundary','study_area')
and scenario_id={scenario}
),mmu.location
);
""".format(scenario=self.__scenario)
self.__Indicator = Indicator(self.__user)
db = self.__Indicator.get_up_calculator_connection()
db.execute(query)
db.close()
self.__evaluateAmenityBuffers()
def __evaluateAmenityBuffers(self):
import psycopg2
import psycopg2.extras
try:
error = True
count = 0
while error and count < 3:
try:
query = """
select urbper_buffer_amenities({scenario})
""".format(scenario=self.__scenario)
LogEvents(
"amenity buffers",
"amenity buffer module started: " + query,
self.__scenario,
self.__user
)
conn = psycopg2.connect(self.__Indicator.get_uri())
cursor = conn.cursor(
cursor_factory=psycopg2.extras.DictCursor)
old_isolation_level = conn.isolation_level
conn.set_isolation_level(0)
cursor.execute(query)
conn.commit()
conn.set_isolation_level(old_isolation_level)
except Exception as e:
error = True
count += 1
time.sleep(randint(1, 3))
LogEvents(
"amenity buffers",
"amenity buffers module failed " +
str(count) + ": " + str(e),
self.__scenario,
self.__user
)
else:
error = False
LogEvents(
"amenity buffers",
"amenity buffers module finished",
self.__scenario,
self.__user
)
except Exception as e:
LogEvents(
"amenity buffers",
"unknown error " +
str(e),
self.__scenario,
self.__user
)
class Module:
def __init__(self, user, scenario, extra_dict_arguments=None):
self.__user = user
self.__scenario = scenario
def run(self):
try:
self.__Indicator = Indicator(self.__user)
green_classes = self.__getClassess("green_land")
green_array = "'{" + ",".join(green_classes) + "}'"
protected_classes = self.__getClassess("protected")
protected_array = "'{" + ",".join(protected_classes) + "}'"
error = True
count = 0
while error and count < 3:
self.__Indicator = Indicator(self.__user)
db = self.__Indicator.get_up_calculator_connection()
try:
query = """
select urbper_indicator_greenland_availability({scenario},'green_availability',{green},{protected})
""".format(
scenario=self.__scenario,
green=green_array,
protected=protected_array,
)
LogEvents(
"green land availability",
"green land availability module started: " + query,
self.__scenario, self.__user)
with transaction.atomic():
db.execute(query)
except Exception as e:
error = True
count += 1
time.sleep(randint(1, 3))
db.close()
LogEvents(
"green land availability",
"green land availability module failed " + str(count) +
": " + str(e), self.__scenario, self.__user)
else:
error = False
db.close()
LogEvents("green land availability",
"green land availability module finished",
self.__scenario, self.__user)
except Exception as e:
LogEvents("green land availability", "unknown error " + str(e),
self.__scenario, self.__user)
def __getClassess(self, fclass):
try:
query = """select distinct classification.name
from classification
where classification.category='footprint'
and classification.fclass='{fclass}'
""".format(fclass=fclass)
results = classification.objects.filter(
category='footprint',
fclass=fclass).distinct().values_list('name', flat=True)
LogEvents("classes", "classes finished: " + query, self.__scenario,
self.__user)
# cursor.execute(query)
results_set = [row for row in results]
results = results_set
except Exception as e:
time.sleep(randint(1, 3))
LogEvents("classes", "classes failed :" + str(e), self.__scenario,
self.__user)
else:
error = False
LogEvents("classes", "classes finished", self.__scenario,
self.__user)
return results
class Module:
def __init__(self, user, scenario, extra_dict_arguments=None):
self.__user = user
self.__scenario = scenario
def run(self):
try:
self.__Indicator = Indicator(self.__user)
road_classes = self.__getClassess("road_season")
road_classes_array = "'{" + ",".join(road_classes) + "}'"
risk_classes = self.__getClassess("risk")
risk_classes_array = "'{" + ",".join(risk_classes) + "}'"
error = True
count = 0
while error and count < 3:
print("Entrando\n")
self.__Indicator = Indicator(self.__user)
db = self.__Indicator.get_up_calculator_connection()
try:
query = """
select urbper_indicator_road_all_season_proximity({scenario},'allseason_prox'::varchar(10),'pop_prox_allseason'::varchar(10),{road_class})
""".format(scenario=self.__scenario,
road_class=road_classes_array)
LogEvents("all season roads",
"all season roads module started: " + query,
self.__scenario, self.__user)
with transaction.atomic():
db.execute(query)
except Exception as e:
error = True
count += 1
time.sleep(randint(1, 3))
db.close()
LogEvents(
"all season roads", "all season roads module failed " +
str(count) + ": " + str(e), self.__scenario,
self.__user)
else:
error = False
db.close()
LogEvents("all season roads",
"all season roads module finished",
self.__scenario, self.__user)
except Exception as e:
LogEvents("all season roads", "unknown error " + str(e),
self.__scenario, self.__user)
def __getClassess(self, fclass):
try:
query = """select distinct classification.name
from classification
where classification.category='footprint'
and classification.fclass='{fclass}'
""".format(fclass=fclass)
results = classification.objects.filter(
category='footprint',
fclass=fclass).distinct().values_list('name', flat=True)
LogEvents("classes", "classes finished: " + query, self.__scenario,
self.__user)
results_set = [row for row in results]
results = results_set
except Exception as e:
error = True
time.sleep(randint(1, 3))
LogEvents("classes", "classes failed: " + str(e), self.__scenario,
self.__user)
else:
error = False
LogEvents("classes", "classes finished", self.__scenario,
self.__user)
return results
class Module:
def __init__(self, user, scenario, extra_dict_arguments=None):
self.__user = user
self.__scenario = scenario
def run(self):
try:
error = True
count = 0
dwells_classes= self.__getAmenityClassess()
dwells_array="'{"+",".join(dwells_classes)+"}'"
while error and count < 3:
self.__Indicator = Indicator(self.__user)
db = self.__Indicator.get_up_calculator_connection()
try:
query = """
select urbper_indicator_potable_water_wells({scenario},{fclass_array})
""".format(
scenario=self.__scenario,fclass_array=dwells_array
)
LogEvents(
"potable water wells",
"potable water wells module started: " + query,
self.__scenario,
self.__user
)
with transaction.atomic():
db.execute(query)
except Exception as e:
error = True
count += 1
time.sleep(randint(1, 3))
db.close()
LogEvents(
"potable water wells",
"potable water wells module failed " +
str(count) + ": " + str(e),
self.__scenario,
self.__user
)
else:
error = False
db.close()
LogEvents(
"potable water wells",
"potable water wells module finished",
self.__scenario,
self.__user
)
except Exception as e:
LogEvents(
"potable water wells",
"unknown error " +
str(e),
self.__scenario,
self.__user
)
def __getAmenityClassess(self):
try:
error = True
count = 0
while error and count < 3:
try:
query="""select distinct classification.name
from classification
where classification.category='amenities'
and classification.fclass='dwells'
"""
results = classification.objects.filter(category='amenities',fclass='dwells').distinct().values_list('name',flat=True)
LogEvents(
"dwells",
"dwells classes finished: " + query,
self.__scenario,
self.__user
)
# cursor.execute(query)
results_set=[row for row in results]
results=results_set
except Exception as e:
error = True
count += 1
time.sleep(randint(1, 3))
LogEvents(
"dwells",
"dwells classes failed " +
str(count) + ": " + str(e),
self.__scenario,
self.__user
)
else:
error = False
LogEvents(
"dwells",
"dwells classes finished",
self.__scenario,
self.__user
)
return results
except Exception as e:
LogEvents(
"dwells",
"unknown error " + str(e),
self.__scenario,
self.__user
)
class Module:
def __init__(self, user, scenario, extra_dict_arguments=None):
self.__user = user
self.__scenario = scenario
def run(self):
try:
self.__Indicator = Indicator(self.__user)
self.__db = self.__Indicator.get_up_calculator_connection()
vacuum(self.__Indicator.get_uri(), "mmu")
self.__db.close()
amenity_classes_set = self.__getAmentityClassess()
error = True
count = 0
while error and count < 3:
self.__Indicator = Indicator(self.__user)
db = self.__Indicator.get_up_calculator_connection()
try:
for amenity in amenity_classes_set:
amenity_classes = []
amenity_classes.append(amenity)
amenity_classes_array = "'{" + ",".join(
amenity_classes) + "}'"
query = """
select urbper_indicator_pop_amenity_prox({scenario},'pop_prox_{fclass}'::varchar(30),'{fclass}_proximity'::varchar(30),{fclass_array})
""".format(scenario=self.__scenario,
fclass=amenity,
fclass_array=amenity_classes_array)
LogEvents(
amenity + " proximity",
amenity + " proximity module started: " + query,
self.__scenario, self.__user)
with transaction.atomic():
db.execute(query)
except Exception as e:
error = True
count += 1
time.sleep(randint(1, 3))
db.close()
LogEvents(
"amenity proximity",
"amenity proximity module failed " + str(count) +
": " + str(e), self.__scenario, self.__user)
else:
error = False
db.close()
LogEvents("amenity proximity",
"amenity proximity module finished",
self.__scenario, self.__user)
except Exception as e:
LogEvents("amenity proximity", "unknown error " + str(e),
self.__scenario, self.__user)
def __getAmentityClassess(self):
try:
error = True
count = 0
while error and count < 3:
try:
with connection.cursor() as cursor:
query = """select distinct assumptions.name from amenities
inner join classification on classification."name" = amenities.fclass
inner join assumptions on assumptions.name = classification.fclass
where classification.category='amenities'
and amenities.scenario_id=assumptions.scenario_id
and assumptions.scenario_id={}""".format(
self.__scenario)
LogEvents(
"amenity proximity",
"amenity proximity classes started: " + query,
self.__scenario, self.__user)
cursor.execute(query)
results_set = [
list(row)[0] for row in cursor.fetchall()
]
print(results_set)
results = results_set
except Exception as e:
error = True
count += 1
time.sleep(randint(1, 3))
LogEvents(
"amenity proximity",
"amenity proximity classes failed " + str(count) +
": " + str(e), self.__scenario, self.__user)
else:
error = False
LogEvents("amenity proximity",
"amenity proximity classes finished",
self.__scenario, self.__user)
return results
except Exception as e:
LogEvents("amenity proximity", "unknown error " + str(e),
self.__scenario, self.__user)
class Module:
def __init__(self, user, scenario, extra_dict_arguments=None):
self.__user = user
self.__scenario = scenario
def run(self):
try:
self.__Indicator = Indicator(self.__user)
# get educational clasification
amenity_classes=self.__getAmenitiesClassess()
amenity_classes_array="'{"+",".join(amenity_classes)+"}'"
risk_classes=self.__getRiskClassess()
risk_classes_array="'{"+",".join(risk_classes)+"}'"
error = True
count = 0
while error and count < 3:
print("Entrando\n")
self.__Indicator = Indicator(self.__user)
db = self.__Indicator.get_up_calculator_connection()
try:
query = """
select urbper_indicator_educational_infrastructure_in_risk({scenario},'edu_risk'::varchar(10),{amenity_class},{risk_classes})
""".format(
scenario=self.__scenario,
amenity_class=amenity_classes_array,
risk_classes=risk_classes_array
)
LogEvents(
"educational infrastructure in risk",
"educational infrastructure in risk module started: " + query,
self.__scenario,
self.__user
)
with transaction.atomic():
db.execute(query)
except Exception as e:
error = True
count += 1
time.sleep(randint(1, 3))
db.close()
LogEvents(
"educational infrastructure in risk",
"educational infrastructure in risk module failed " +
str(count) + ": " + str(e),
self.__scenario,
self.__user
)
else:
error = False
db.close()
LogEvents(
"educational infrastructure in risk",
"educational infrastructure in risk module finished",
self.__scenario,
self.__user
)
except Exception as e:
LogEvents(
"educational infrastructure in risk",
"unknown error " +
str(e),
self.__scenario,
self.__user
)
def __getRiskClassess(self):
try:
error = True
count = 0
while error and count < 3:
try:
with connection.cursor() as cursor:
query="""select distinct risk.fclass from risk
inner join classification on classification.name=risk.fclass
where classification.category='risk'
and classification.fclass='risk'
and risk.scenario_id={}""".format(self.__scenario)
LogEvents(
"risk fclasses",
"risk fclasses started: " + query,
self.__scenario,
self.__user
)
cursor.execute(query)
results_set=[list(row)[0] for row in cursor.fetchall()]
results=results_set
except Exception as e:
error = True
count += 1
time.sleep(randint(1, 3))
LogEvents(
"risk fclasses",
"risk fclasses failed " +
str(count) + ": " + str(e),
self.__scenario,
self.__user
)
else:
error = False
LogEvents(
"risk fclasses",
"risk fclasses finished",
self.__scenario,
self.__user
)
return results
except Exception as e:
LogEvents(
"risk fclasses",
"unknown error " + str(e),
self.__scenario,
self.__user
)
def __getAmenitiesClassess(self):
try:
error = True
count = 0
while error and count < 3:
try:
with connection.cursor() as cursor:
query="""
select distinct amenities.fclass from amenities
inner join classification on classification.name=amenities.fclass
where classification.category='amenities'
and classification.fclass='educational_infrastructure'
and amenities.scenario_id={}""".format(self.__scenario)
LogEvents(
"amenities fclasses",
"amenities fclasses started: " + query,
self.__scenario,
self.__user
)
cursor.execute(query)
results_set=[list(row)[0] for row in cursor.fetchall()]
results=results_set
except Exception as e:
error = True
count += 1
time.sleep(randint(1, 3))
LogEvents(
"amenities fclasses",
"amenities fclasses failed " +
str(count) + ": " + str(e),
self.__scenario,
self.__user
)
else:
error = False
LogEvents(
"amenities fclasses",
"amenities fclasses finished",
self.__scenario,
self.__user
)
return results
except Exception as e:
LogEvents(
"amenities fclasses",
"unknown error " + str(e),
self.__scenario,
self.__user
)
class Module:
def __init__(self, user, scenario, extra_dict_arguments=None):
self.__user = user
self.__scenario = scenario
def run(self):
try:
self.__Indicator = Indicator(self.__user)
informal_classes = self.__getClassess("informal_set")
informal_array = "'{" + ",".join(informal_classes) + "}'"
error = True
count = 0
while error and count < 3:
self.__Indicator = Indicator(self.__user)
db = self.__Indicator.get_up_calculator_connection()
try:
query = """
select urbper_indicator_population_informal_settlements({scenario},{informal})
""".format(scenario=self.__scenario,
informal=informal_array)
LogEvents(
"urbper_indicator population in informal settlements",
"population in informal settlements started: " + query,
self.__scenario, self.__user)
with transaction.atomic():
db.execute(query)
except Exception as e:
error = True
count += 1
time.sleep(randint(1, 3))
db.close()
LogEvents(
"population in informal settlements",
"population in informal settlements module failed " +
str(count) + ": " + str(e), self.__scenario,
self.__user)
else:
error = False
db.close()
LogEvents(
"population in informal settlements",
"population in informal settlements module finished",
self.__scenario, self.__user)
except Exception as e:
LogEvents("population in informal settlements",
"unknown error " + str(e), self.__scenario, self.__user)
def __getClassess(self, fclass):
try:
query = """select distinct classification.name
from classification
where classification.category='risk'
and classification.fclass='{fclass}'
""".format(fclass=fclass)
results = classification.objects.filter(
category='risk',
fclass=fclass).distinct().values_list('name', flat=True)
LogEvents("classes", "classes finished: " + query, self.__scenario,
self.__user)
results_set = [row for row in results]
results = results_set
except Exception as e:
error = True
time.sleep(randint(1, 3))
LogEvents("classes", "classes failed: " + str(e), self.__scenario,
self.__user)
else:
error = False
LogEvents("classes", "classes finished", self.__scenario,
self.__user)
return results
class Module:
def __init__(self, user, scenario, extra_dict_arguments=None):
self.__user = user
self.__scenario = scenario
self.__base_scenario = extra_dict_arguments["base_scenario"]
def run(self):
query = """
delete from mmu_info where mmu_id in (
select mmu_id
from mmu
where scenario_id={scenario} and not st_contains(
(
select location
from footprint
inner join classification on classification.name=footprint.name
where
classification.category='footprint'
and classification.fclass in ('political_boundary','study_area')
and scenario_id={scenario}
),mmu.location
)
);
delete from mmu
where scenario_id={scenario} and not st_contains(
(
select location
from footprint
inner join classification on classification.name=footprint.name
where
classification.category='footprint'
and classification.fclass in ('political_boundary','study_area')
and scenario_id={scenario}
),mmu.location
);
""".format(scenario=self.__scenario)
self.__Indicator = Indicator(self.__user)
db = self.__Indicator.get_up_calculator_connection()
db.execute(query)
db.close()
self.__EvaluateJobsBuffer()
def __EvaluateJobsBuffer(self):
self.__limits = {"inferior": 0, "superior": 0}
self.__jobs_limit_offset()
self.__jobs_preprocessing_threads()
def __jobs_limit_offset(self):
try:
try:
# get the max an min of pk
self.__limits["inferior"] = jobs.objects.filter(
scenario_id=self.__scenario).aggregate(Min('jobs_id'))["jobs_id__min"]
self.__limits["superior"] = jobs.objects.filter(
scenario_id=self.__scenario).aggregate(Max('jobs_id'))["jobs_id__max"]
except Exception as e:
LogEvents("jobs max min", "unknown error " +
str(e), self.__scenario, self.__user, True)
except Exception as e:
LogEvents("jobs max min", "unknown error " +
str(e), self.__scenario, self.__user, True)
def __jobs_preprocessing_threads(self):
self.__scenario_t = {}
self.__scenario_t["limit"] = 0
self.__scenario_t["offset"] = 0
inferior = self.__limits["inferior"]
superior = self.__limits["superior"]
_threads = {}
self.max_threads = min(self.__Indicator.get_max_threads(), int(math.ceil(
(superior - inferior) / self.__Indicator.get_max_rows())))
num_partitions = self.max_threads
partition_size = (int)(
math.ceil((superior - inferior) / self.max_threads))
for h in range(0, num_partitions):
self.__scenario_t["offset"] = inferior
self.__scenario_t["limit"] = self.__scenario_t["offset"] + \
partition_size
inferior = self.__scenario_t["limit"] + 1
_threads[h] = threading.Thread(target=self.__ModuleJobsDensity, args=(
self.__scenario, self.__scenario_t["offset"], self.__scenario_t["limit"]))
for process in _threads:
_threads[process].start()
for process in _threads:
if _threads[process].is_alive():
_threads[process].join()
def __ModuleJobsDensity(self, scenario_id, offset=0, limit=0):
import psycopg2
import psycopg2.extras
try:
error = True
count = 0
while error and count < 3:
try:
query = """select urbper_buffer_job_and_job_density({scenario},{offset},{limit})""".format(
scenario=scenario_id, offset=offset, limit=limit)
LogEvents("job buffers and density", "job buffers and density module started: " + query,
scenario_id, self.__user)
conn = psycopg2.connect(self.__Indicator.get_uri())
cursor = conn.cursor(
cursor_factory=psycopg2.extras.DictCursor)
old_isolation_level = conn.isolation_level
conn.set_isolation_level(0)
cursor.execute(query)
conn.commit()
conn.set_isolation_level(old_isolation_level)
except Exception as e:
error = True
count += 1
LogEvents("job buffers and density ", "job buffers and density module failed " +
str(count) + ": " + str(e), scenario_id, self.__user, True)
else:
error = False
# db.commit()
LogEvents("job buffers and density ", "job buffers and density module finished",
scenario_id, self.__user)
except Exception as e:
LogEvents("Running scenarios", "Unknown error " +
str(e), scenario_id, self.__user, True)
class Module:
def __init__(self, user, scenario, extra_dict_arguments=None):
self.__user = user
self.__scenario = scenario
self.__base_scenario = extra_dict_arguments["base_scenario"]
def run(self):
self.__general_calculus()
self.__limits = {"inferior": 0, "superior": 0}
self.__mmu_limit_offset()
self.__pop_density_threads()
self.__area_density_threads()
def __general_calculus(self):
self.__Indicator = Indicator(self.__user)
vacuum(self.__Indicator.get_uri(), "mmu")
self.__urbper_base_footprint_area()
self.__urbper_base_total_population()
self.__urbper_base_footprint_km2()
def __urbper_base_footprint_area(self):
try:
error = True
count = 0
while error and count < 3:
try:
self.__Indicator = Indicator(self.__user)
query = """select urbper_indicator_footprints_area({scenario})""".format(
scenario=self.__scenario)
LogEvents("footprints area",
"footprints area module started: " + query,
self.__scenario, self.__user)
conn = psycopg2.connect(self.__Indicator.get_uri())
cursor = conn.cursor(
cursor_factory=psycopg2.extras.DictCursor)
old_isolation_level = conn.isolation_level
conn.set_isolation_level(0)
cursor.execute(query)
conn.commit()
conn.set_isolation_level(old_isolation_level)
except Exception as e:
conn.rollback()
conn.close()
error = True
count += 1
time.sleep(randint(1, 3))
LogEvents(
"footprints area", "footprints area module failed " +
str(count) + ": " + str(e), self.__scenario,
self.__user, True)
else:
error = False
conn.close()
LogEvents("footprints area",
"footprints area module finished",
self.__scenario, self.__user)
except Exception as e:
LogEvents("footprints area", "unknown error+ " + str(e),
self.__scenario, self.__user, True)
def __urbper_base_footprint_km2(self):
try:
error = True
count = 0
while error and count < 3:
try:
self.__Indicator = Indicator(self.__user)
query = """select urbper_indicator_footprint_km2({scenario})""".format(
scenario=self.__scenario)
LogEvents("footprints area",
"footprints area module started: " + query,
self.__scenario, self.__user)
conn = psycopg2.connect(self.__Indicator.get_uri())
cursor = conn.cursor(
cursor_factory=psycopg2.extras.DictCursor)
old_isolation_level = conn.isolation_level
conn.set_isolation_level(0)
cursor.execute(query)
conn.commit()
conn.set_isolation_level(old_isolation_level)
except Exception as e:
conn.rollback()
conn.close()
error = True
count += 1
time.sleep(randint(1, 3))
LogEvents(
"footprint_km2", "footprint_km2 module failed " +
str(count) + ": " + str(e), self.__scenario,
self.__user, True)
else:
error = False
conn.close()
LogEvents("footprint_km2", "footprint_km2 module finished",
self.__scenario, self.__user)
except Exception as e:
LogEvents("footprint_km2", "unknown error+ " + str(e),
self.__scenario, self.__user, True)
def __urbper_base_total_population(self):
try:
error = True
count = 0
while error and count < 3:
self.__Indicator = Indicator(self.__user)
db = self.__Indicator.get_up_calculator_connection()
try:
with transaction.atomic():
self.__Indicator = Indicator(self.__user)
db = self.__Indicator.get_up_calculator_connection()
query = """select urbper_indicator_base_calculus_total_population({scenario},{base_scenario})""".format(
scenario=self.__scenario,
base_scenario=self.__base_scenario)
LogEvents("total population",
"total population module started: " + query,
self.__scenario, self.__user)
db.execute(query)
except Exception as e:
db.close()
error = True
count += 1
time.sleep(randint(1, 3))
LogEvents(
"total population", "total population module failed " +
str(count) + ": " + str(e), self.__scenario,
self.__user, True)
else:
error = False
db.close()
LogEvents("total population",
"total population module finished",
self.__scenario, self.__user)
except Exception as e:
LogEvents("total population", "unknown error " + str(e),
self.__scenario, self.__user, True)
def __mmu_limit_offset(self):
try:
self.__Indicator = Indicator(self.__user)
db = self.__Indicator.get_up_calculator_connection()
try:
self.__limits["inferior"] = mmu.objects.filter(
scenario_id=self.__scenario).aggregate(
Min('mmu_id'))["mmu_id__min"]
self.__limits["superior"] = mmu.objects.filter(
scenario_id=self.__scenario).aggregate(
Max('mmu_id'))["mmu_id__max"]
except Exception as e:
LogEvents("squares max min", "unknown error " + str(e),
self.__scenario, self.__user, True)
db.close()
except Exception as e:
LogEvents("squares max min", "unknown error " + str(e),
self.__scenario, self.__user, True)
def __pop_density_threads(self):
self.__scenario_t = {}
self.__scenario_t["limit"] = 0
self.__scenario_t["offset"] = 0
inferior = self.__limits["inferior"]
superior = self.__limits["superior"]
_threads = {}
self.max_threads = min(
self.__Indicator.get_max_threads(),
int(
math.ceil(
(superior - inferior) / self.__Indicator.get_max_rows())))
num_partitions = self.max_threads
partition_size = (int)(math.ceil(
(superior - inferior) / self.max_threads))
for h in range(0, num_partitions):
self.__scenario_t["offset"] = inferior
self.__scenario_t["limit"] = self.__scenario_t["offset"] + \
partition_size
inferior = self.__scenario_t["limit"] + 1
_threads[h] = threading.Thread(
target=self.__ModulePopulationDensity,
args=(self.__scenario, self.__scenario_t["offset"],
self.__scenario_t["limit"]))
for process in _threads:
_threads[process].start()
for process in _threads:
if _threads[process].is_alive():
_threads[process].join()
def __area_density_threads(self):
self.__scenario_t = {}
self.__scenario_t["limit"] = 0
self.__scenario_t["offset"] = 0
inferior = self.__limits["inferior"]
superior = self.__limits["superior"]
_threads = {}
self.max_threads = min(
self.__Indicator.get_max_threads(),
int(
math.ceil(
(superior - inferior) / self.__Indicator.get_max_rows())))
num_partitions = self.max_threads
partition_size = (int)(math.ceil(
(superior - inferior) / self.max_threads))
for h in range(0, num_partitions):
self.__scenario_t["offset"] = inferior
self.__scenario_t["limit"] = self.__scenario_t["offset"] + \
partition_size
inferior = self.__scenario_t["limit"] + 1
_threads[h] = threading.Thread(target=self.__ModuleAreaDensity,
args=(self.__scenario,
self.__scenario_t["offset"],
self.__scenario_t["limit"]))
for process in _threads:
_threads[process].start()
for process in _threads:
if _threads[process].is_alive():
_threads[process].join()
def __ModuleAreaDensity(self, scenario_id, offset=0, limit=0):
try:
error = True
count = 0
while error and count < 3:
self.__Indicator = Indicator(self.__user)
db = self.__Indicator.get_up_calculator_connection()
try:
query = """select urbper_indicator_area_den_avg({scenario},{offset},{limit})""".format(
scenario=scenario_id, offset=offset, limit=limit)
LogEvents("area density avg",
"area density avg module started: " + query,
scenario_id, self.__user)
conn = psycopg2.connect(self.__Indicator.get_uri())
cursor = conn.cursor(
cursor_factory=psycopg2.extras.DictCursor)
old_isolation_level = conn.isolation_level
conn.set_isolation_level(0)
cursor.execute(query)
conn.commit()
conn.set_isolation_level(old_isolation_level)
except Exception as e:
error = True
count += 1
time.sleep(randint(1, 3))
LogEvents(
"area density avg", "area density avg module failed " +
str(count) + ": " + str(e), scenario_id, self.__user,
True)
conn.close()
else:
error = False
LogEvents("area density avg",
"area density avg module finished", scenario_id,
self.__user)
conn.close()
except Exception as e:
LogEvents("Running scenarios", "Unknown error " + str(e),
scenario_id, self.__user, True)
def __ModulePopulationDensity(self, scenario_id, offset=0, limit=0):
try:
error = True
count = 0
while error and count < 3:
self.__Indicator = Indicator(self.__user)
db = self.__Indicator.get_up_calculator_connection()
try:
query = """select urbper_indicator_pop_den_avg({scenario},{offset},{limit})""".format(
scenario=scenario_id, offset=offset, limit=limit)
LogEvents(
"population density avg",
"population density avg module started: " + query,
scenario_id, self.__user)
conn = psycopg2.connect(self.__Indicator.get_uri())
cursor = conn.cursor(
cursor_factory=psycopg2.extras.DictCursor)
old_isolation_level = conn.isolation_level
conn.set_isolation_level(0)
cursor.execute(query)
conn.commit()
conn.set_isolation_level(old_isolation_level)
except Exception as e:
error = True
count += 1
LogEvents(
"population density avg",
"population density avg module failed " + str(count) +
": " + str(e), scenario_id, self.__user, True)
conn.close()
else:
error = False
LogEvents("population density avg",
"population density avg module finished",
scenario_id, self.__user)
conn.close()
except Exception as e:
LogEvents("Running scenarios", "Unknown error " + str(e),
scenario_id, self.__user, True)
