class Module:
def __init__(self, user, scenario, extra_dict_arguments=None):
self.__user = user
self.__scenario = scenario
self.__base_scenario = extra_dict_arguments["base_scenario"]
def run(self):
self.__limits = {"inferior": 0, "superior": 0}
self.__mmu_limit_offset()
self.__job_density_threads()
self.__energy_transport()
def __mmu_limit_offset(self):
try:
self.__Indicator = Indicator(self.__user)
try:
# get the max an min of pk
self.__limits["inferior"] = mmu.objects.filter(
scenario_id=self.__scenario).aggregate(
Min('mmu_id'))["mmu_id__min"]
self.__limits["superior"] = mmu.objects.filter(
scenario_id=self.__scenario).aggregate(
Max('mmu_id'))["mmu_id__max"]
except Exception as e:
LogEvents("squares max min", "unknown error " + str(e),
self.__scenario, self.__user, True)
except Exception as e:
LogEvents("squares max min", "unknown error " + str(e),
self.__scenario, self.__user, True)
def __job_density_threads(self):
self.__scenario_t = {}
self.__scenario_t["limit"] = 0
self.__scenario_t["offset"] = 0
inferior = self.__limits["inferior"]
superior = self.__limits["superior"]
_threads = {}
self.max_threads = min(
self.__Indicator.get_max_threads(),
int(
math.ceil(
(superior - inferior) / self.__Indicator.get_max_rows())))
num_partitions = self.max_threads
partition_size = (int)(math.ceil(
(superior - inferior) / self.max_threads))
for h in range(0, num_partitions):
self.__scenario_t["offset"] = inferior
self.__scenario_t["limit"] = self.__scenario_t["offset"] + \
partition_size
inferior = self.__scenario_t["limit"] + 1
_threads[h] = threading.Thread(target=self.__ModulejobDensity,
args=(self.__scenario,
self.__scenario_t["offset"],
self.__scenario_t["limit"]))
for process in _threads:
_threads[process].start()
for process in _threads:
if _threads[process].is_alive():
_threads[process].join()
def __ModulejobDensity(self, scenario_id, offset=0, limit=0):
try:
error = True
count = 0
while error and count < 3:
self.__Indicator = Indicator(self.__user)
db = self.__Indicator.get_up_calculator_connection()
try:
with transaction.atomic():
query = """select urbper_indicator_job_density({scenario},{offset},{limit})""".format(
scenario=scenario_id, offset=offset, limit=limit)
LogEvents("job density",
"job density module started: " + query,
scenario_id, self.__user)
db.execute(query)
except Exception as e:
error = True
count += 1
LogEvents(
"job density ", "job density module failed " +
str(count) + ": " + str(e), scenario_id, self.__user,
True)
db.close()
else:
error = False
LogEvents("job density ", "job density module finished",
scenario_id, self.__user)
db.close()
except Exception as e:
LogEvents("Running scenarios", "Unknown error " + str(e),
scenario_id, self.__user, True)
class Module:
def __init__(self, user, scenario, extra_dict_arguments=None):
self.__user = user
self.__scenario = scenario
self.__base_scenario = extra_dict_arguments["base_scenario"]
def run(self):
query = """
delete from mmu_info where mmu_id in (
select mmu_id
from mmu
where scenario_id={scenario} and not st_contains(
(
select location
from footprint
inner join classification on classification.name=footprint.name
where
classification.category='footprint'
and classification.fclass in ('political_boundary','study_area')
and scenario_id={scenario}
),mmu.location
)
);
delete from mmu
where scenario_id={scenario} and not st_contains(
(
select location
from footprint
inner join classification on classification.name=footprint.name
where
classification.category='footprint'
and classification.fclass in ('political_boundary','study_area')
and scenario_id={scenario}
),mmu.location
);
""".format(scenario=self.__scenario)
self.__Indicator = Indicator(self.__user)
db = self.__Indicator.get_up_calculator_connection()
db.execute(query)
db.close()
self.__EvaluateJobsBuffer()
def __EvaluateJobsBuffer(self):
self.__limits = {"inferior": 0, "superior": 0}
self.__jobs_limit_offset()
self.__jobs_preprocessing_threads()
def __jobs_limit_offset(self):
try:
try:
# get the max an min of pk
self.__limits["inferior"] = jobs.objects.filter(
scenario_id=self.__scenario).aggregate(Min('jobs_id'))["jobs_id__min"]
self.__limits["superior"] = jobs.objects.filter(
scenario_id=self.__scenario).aggregate(Max('jobs_id'))["jobs_id__max"]
except Exception as e:
LogEvents("jobs max min", "unknown error " +
str(e), self.__scenario, self.__user, True)
except Exception as e:
LogEvents("jobs max min", "unknown error " +
str(e), self.__scenario, self.__user, True)
def __jobs_preprocessing_threads(self):
self.__scenario_t = {}
self.__scenario_t["limit"] = 0
self.__scenario_t["offset"] = 0
inferior = self.__limits["inferior"]
superior = self.__limits["superior"]
_threads = {}
self.max_threads = min(self.__Indicator.get_max_threads(), int(math.ceil(
(superior - inferior) / self.__Indicator.get_max_rows())))
num_partitions = self.max_threads
partition_size = (int)(
math.ceil((superior - inferior) / self.max_threads))
for h in range(0, num_partitions):
self.__scenario_t["offset"] = inferior
self.__scenario_t["limit"] = self.__scenario_t["offset"] + \
partition_size
inferior = self.__scenario_t["limit"] + 1
_threads[h] = threading.Thread(target=self.__ModuleJobsDensity, args=(
self.__scenario, self.__scenario_t["offset"], self.__scenario_t["limit"]))
for process in _threads:
_threads[process].start()
for process in _threads:
if _threads[process].is_alive():
_threads[process].join()
def __ModuleJobsDensity(self, scenario_id, offset=0, limit=0):
import psycopg2
import psycopg2.extras
try:
error = True
count = 0
while error and count < 3:
try:
query = """select urbper_buffer_job_and_job_density({scenario},{offset},{limit})""".format(
scenario=scenario_id, offset=offset, limit=limit)
LogEvents("job buffers and density", "job buffers and density module started: " + query,
scenario_id, self.__user)
conn = psycopg2.connect(self.__Indicator.get_uri())
cursor = conn.cursor(
cursor_factory=psycopg2.extras.DictCursor)
old_isolation_level = conn.isolation_level
conn.set_isolation_level(0)
cursor.execute(query)
conn.commit()
conn.set_isolation_level(old_isolation_level)
except Exception as e:
error = True
count += 1
LogEvents("job buffers and density ", "job buffers and density module failed " +
str(count) + ": " + str(e), scenario_id, self.__user, True)
else:
error = False
# db.commit()
LogEvents("job buffers and density ", "job buffers and density module finished",
scenario_id, self.__user)
except Exception as e:
LogEvents("Running scenarios", "Unknown error " +
str(e), scenario_id, self.__user, True)
class Module:
def __init__(self, user, scenario, extra_dict_arguments=None):
self.__user = user
self.__scenario = scenario
self.__base_scenario = extra_dict_arguments["base_scenario"]
def run(self):
self.__general_calculus()
self.__limits = {"inferior": 0, "superior": 0}
self.__mmu_limit_offset()
self.__pop_density_threads()
self.__area_density_threads()
def __general_calculus(self):
self.__Indicator = Indicator(self.__user)
vacuum(self.__Indicator.get_uri(), "mmu")
self.__urbper_base_footprint_area()
self.__urbper_base_total_population()
self.__urbper_base_footprint_km2()
def __urbper_base_footprint_area(self):
try:
error = True
count = 0
while error and count < 3:
try:
self.__Indicator = Indicator(self.__user)
query = """select urbper_indicator_footprints_area({scenario})""".format(
scenario=self.__scenario)
LogEvents("footprints area",
"footprints area module started: " + query,
self.__scenario, self.__user)
conn = psycopg2.connect(self.__Indicator.get_uri())
cursor = conn.cursor(
cursor_factory=psycopg2.extras.DictCursor)
old_isolation_level = conn.isolation_level
conn.set_isolation_level(0)
cursor.execute(query)
conn.commit()
conn.set_isolation_level(old_isolation_level)
except Exception as e:
conn.rollback()
conn.close()
error = True
count += 1
time.sleep(randint(1, 3))
LogEvents(
"footprints area", "footprints area module failed " +
str(count) + ": " + str(e), self.__scenario,
self.__user, True)
else:
error = False
conn.close()
LogEvents("footprints area",
"footprints area module finished",
self.__scenario, self.__user)
except Exception as e:
LogEvents("footprints area", "unknown error+ " + str(e),
self.__scenario, self.__user, True)
def __urbper_base_footprint_km2(self):
try:
error = True
count = 0
while error and count < 3:
try:
self.__Indicator = Indicator(self.__user)
query = """select urbper_indicator_footprint_km2({scenario})""".format(
scenario=self.__scenario)
LogEvents("footprints area",
"footprints area module started: " + query,
self.__scenario, self.__user)
conn = psycopg2.connect(self.__Indicator.get_uri())
cursor = conn.cursor(
cursor_factory=psycopg2.extras.DictCursor)
old_isolation_level = conn.isolation_level
conn.set_isolation_level(0)
cursor.execute(query)
conn.commit()
conn.set_isolation_level(old_isolation_level)
except Exception as e:
conn.rollback()
conn.close()
error = True
count += 1
time.sleep(randint(1, 3))
LogEvents(
"footprint_km2", "footprint_km2 module failed " +
str(count) + ": " + str(e), self.__scenario,
self.__user, True)
else:
error = False
conn.close()
LogEvents("footprint_km2", "footprint_km2 module finished",
self.__scenario, self.__user)
except Exception as e:
LogEvents("footprint_km2", "unknown error+ " + str(e),
self.__scenario, self.__user, True)
def __urbper_base_total_population(self):
try:
error = True
count = 0
while error and count < 3:
self.__Indicator = Indicator(self.__user)
db = self.__Indicator.get_up_calculator_connection()
try:
with transaction.atomic():
self.__Indicator = Indicator(self.__user)
db = self.__Indicator.get_up_calculator_connection()
query = """select urbper_indicator_base_calculus_total_population({scenario},{base_scenario})""".format(
scenario=self.__scenario,
base_scenario=self.__base_scenario)
LogEvents("total population",
"total population module started: " + query,
self.__scenario, self.__user)
db.execute(query)
except Exception as e:
db.close()
error = True
count += 1
time.sleep(randint(1, 3))
LogEvents(
"total population", "total population module failed " +
str(count) + ": " + str(e), self.__scenario,
self.__user, True)
else:
error = False
db.close()
LogEvents("total population",
"total population module finished",
self.__scenario, self.__user)
except Exception as e:
LogEvents("total population", "unknown error " + str(e),
self.__scenario, self.__user, True)
def __mmu_limit_offset(self):
try:
self.__Indicator = Indicator(self.__user)
db = self.__Indicator.get_up_calculator_connection()
try:
self.__limits["inferior"] = mmu.objects.filter(
scenario_id=self.__scenario).aggregate(
Min('mmu_id'))["mmu_id__min"]
self.__limits["superior"] = mmu.objects.filter(
scenario_id=self.__scenario).aggregate(
Max('mmu_id'))["mmu_id__max"]
except Exception as e:
LogEvents("squares max min", "unknown error " + str(e),
self.__scenario, self.__user, True)
db.close()
except Exception as e:
LogEvents("squares max min", "unknown error " + str(e),
self.__scenario, self.__user, True)
def __pop_density_threads(self):
self.__scenario_t = {}
self.__scenario_t["limit"] = 0
self.__scenario_t["offset"] = 0
inferior = self.__limits["inferior"]
superior = self.__limits["superior"]
_threads = {}
self.max_threads = min(
self.__Indicator.get_max_threads(),
int(
math.ceil(
(superior - inferior) / self.__Indicator.get_max_rows())))
num_partitions = self.max_threads
partition_size = (int)(math.ceil(
(superior - inferior) / self.max_threads))
for h in range(0, num_partitions):
self.__scenario_t["offset"] = inferior
self.__scenario_t["limit"] = self.__scenario_t["offset"] + \
partition_size
inferior = self.__scenario_t["limit"] + 1
_threads[h] = threading.Thread(
target=self.__ModulePopulationDensity,
args=(self.__scenario, self.__scenario_t["offset"],
self.__scenario_t["limit"]))
for process in _threads:
_threads[process].start()
for process in _threads:
if _threads[process].is_alive():
_threads[process].join()
def __area_density_threads(self):
self.__scenario_t = {}
self.__scenario_t["limit"] = 0
self.__scenario_t["offset"] = 0
inferior = self.__limits["inferior"]
superior = self.__limits["superior"]
_threads = {}
self.max_threads = min(
self.__Indicator.get_max_threads(),
int(
math.ceil(
(superior - inferior) / self.__Indicator.get_max_rows())))
num_partitions = self.max_threads
partition_size = (int)(math.ceil(
(superior - inferior) / self.max_threads))
for h in range(0, num_partitions):
self.__scenario_t["offset"] = inferior
self.__scenario_t["limit"] = self.__scenario_t["offset"] + \
partition_size
inferior = self.__scenario_t["limit"] + 1
_threads[h] = threading.Thread(target=self.__ModuleAreaDensity,
args=(self.__scenario,
self.__scenario_t["offset"],
self.__scenario_t["limit"]))
for process in _threads:
_threads[process].start()
for process in _threads:
if _threads[process].is_alive():
_threads[process].join()
def __ModuleAreaDensity(self, scenario_id, offset=0, limit=0):
try:
error = True
count = 0
while error and count < 3:
self.__Indicator = Indicator(self.__user)
db = self.__Indicator.get_up_calculator_connection()
try:
query = """select urbper_indicator_area_den_avg({scenario},{offset},{limit})""".format(
scenario=scenario_id, offset=offset, limit=limit)
LogEvents("area density avg",
"area density avg module started: " + query,
scenario_id, self.__user)
conn = psycopg2.connect(self.__Indicator.get_uri())
cursor = conn.cursor(
cursor_factory=psycopg2.extras.DictCursor)
old_isolation_level = conn.isolation_level
conn.set_isolation_level(0)
cursor.execute(query)
conn.commit()
conn.set_isolation_level(old_isolation_level)
except Exception as e:
error = True
count += 1
time.sleep(randint(1, 3))
LogEvents(
"area density avg", "area density avg module failed " +
str(count) + ": " + str(e), scenario_id, self.__user,
True)
conn.close()
else:
error = False
LogEvents("area density avg",
"area density avg module finished", scenario_id,
self.__user)
conn.close()
except Exception as e:
LogEvents("Running scenarios", "Unknown error " + str(e),
scenario_id, self.__user, True)
def __ModulePopulationDensity(self, scenario_id, offset=0, limit=0):
try:
error = True
count = 0
while error and count < 3:
self.__Indicator = Indicator(self.__user)
db = self.__Indicator.get_up_calculator_connection()
try:
query = """select urbper_indicator_pop_den_avg({scenario},{offset},{limit})""".format(
scenario=scenario_id, offset=offset, limit=limit)
LogEvents(
"population density avg",
"population density avg module started: " + query,
scenario_id, self.__user)
conn = psycopg2.connect(self.__Indicator.get_uri())
cursor = conn.cursor(
cursor_factory=psycopg2.extras.DictCursor)
old_isolation_level = conn.isolation_level
conn.set_isolation_level(0)
cursor.execute(query)
conn.commit()
conn.set_isolation_level(old_isolation_level)
except Exception as e:
error = True
count += 1
LogEvents(
"population density avg",
"population density avg module failed " + str(count) +
": " + str(e), scenario_id, self.__user, True)
conn.close()
else:
error = False
LogEvents("population density avg",
"population density avg module finished",
scenario_id, self.__user)
conn.close()
except Exception as e:
LogEvents("Running scenarios", "Unknown error " + str(e),
scenario_id, self.__user, True)
