def progress(self): """ Returns the progress status of the calculation, from 0 to 100. A value of 100 sets the internal status to 2 (completed or stopped). After that, any subsequent call to progress will always return 100. If start has not yet been started, returns 0 """ with self.__lock: status = self.status() if status < 2: conn = glb.connection() s_id = self.__session_id progrez = conn.fetchall( "(SELECT COUNT(*) FROM processing.ground_motion WHERE session_id=%s)\ UNION ALL (SELECT num_targets_failed from processing.sessions where gid=%s)\ UNION ALL (SELECT num_targets from processing.sessions where gid=%s)", ( s_id, s_id, s_id, )) total = 1 done = 0 failed = progrez[1][0] lp = len(progrez) #PROGRES SHOULD BE SOMETHING LIKE: [(0L,), (5210L,), (5210L,)] (done_ok, done_failed, total) total = progrez[2][0] if lp == 3 else 1 done_ok = progrez[0][0] if lp == 3 else 0 done_failed = progrez[1][0] if lp == 3 else 0 done = done_ok + done_failed conn.close() if done >= total: if failed > total: self.stop( "No target succesfully written (internal server error)" ) elif failed == total: self.stop("No target succesfully written") else: mzg = "{:d} of {:d} ground motion distributions succesfully calculated".format( done_ok, total) if done_ok < total: self.warning(mzg) else: self.msg(mzg) self.__status = 2 return 100.0 else: return (100.0 * done) / total else: return 100.0 if status >= 2 else 0.0
def progress(self): """ Returns the progress status of the calculation, from 0 to 100. A value of 100 sets the internal status to 2 (completed or stopped). After that, any subsequent call to progress will always return 100. If start has not yet been started, returns 0 """ with self.__lock: status = self.status() if status < 2: conn = glb.connection() s_id = self.__session_id progrez = conn.fetchall("(SELECT COUNT(*) FROM processing.ground_motion WHERE session_id=%s)\ UNION ALL (SELECT num_targets_failed from processing.sessions where gid=%s)\ UNION ALL (SELECT num_targets from processing.sessions where gid=%s)",(s_id,s_id,s_id,)) total=1 done=0 failed = progrez[1][0] lp = len(progrez) #PROGRES SHOULD BE SOMETHING LIKE: [(0L,), (5210L,), (5210L,)] (done_ok, done_failed, total) total = progrez[2][0] if lp==3 else 1 done_ok = progrez[0][0] if lp==3 else 0 done_failed = progrez[1][0] if lp==3 else 0 done = done_ok + done_failed conn.close() if done >= total: if failed > total: self.stop("No target succesfully written (internal server error)") elif failed == total: self.stop("No target succesfully written") else: mzg = "{:d} of {:d} ground motion distributions succesfully calculated" .format(done_ok, total) if done_ok < total: self.warning(mzg) else: self.msg(mzg) self.__status = 2 return 100.0 else: return (100.0 * done) / total else: return 100.0 if status>=2 else 0.0
def calculaterisk(gm, percentiles, session_id, scenario_id, target_id, geocell_id): db_conn = glb.connection() #FIXME: THIS CAN BE PASSED FROM THE MAIN METHOD? # get exposure informations for the given location exp = exposure_module.exposure(db_conn,geocell_id) #Get DPMs vul = vulnerability_module.vulnerability(gm,exp.bldg_dist,exp.bt_prop) #dg_pdfs=vul.dg_pdf #FIXME: UNUSED #get bt damage bt_dmg=vul.damage_bts() ###Calculate loss loss = loss_module.loss(session_id, scenario_id, target_id, geocell_id, exp.bldg_dist, exp.bt_prop, exp.target_prop, bt_dmg) loss.calculate() loss.write2db(db_conn, percentiles)
def geocell_run(intensity, lat_sta, lon_sta, percentiles, target_id, geocell_id, ground_motion_only, scenario_id, session_id, logdir = None ): """ Performs a ground motion calculation given the above arguments. Writes to database the percentiles """ def val(value): #value should be either a scalar number or a numpy.ndarray element. Check below is WEAK, because it checks only if #len(value) can be called, but it avoids importing numpy etcetera return mcerp.UncertainFunction(value) if hasattr(value, '__len__') else float(value) try: #intensity = gmpe_func(val(M), val(lat_hyp), val(lon_hyp), val(depth_hyp), lat_sta, lon_sta) #intensity = gmpe_func(lat_sta, lon_sta) #INTENSITY IS EITHER A SCALAR OR A UNCERTAINFUNCTION.MCERP ARRAY (NUMPTY ARRAY) intensity = val(intensity) # _p = intensity.percentile(percentiles) if isinstance(intensity, mcerp.UncertainFunction) else len(percentiles)*[intensity] _dist = [v for v in globals.discretepdf(intensity,_intensity_labels)] _dist.append(intensity.percentile(0.5) if isinstance(intensity, mcerp.UncertainFunction) else intensity) #write to dbase arg1 = """INSERT INTO processing.ground_motion (target_id, geocell_id, scenario_id, session_id, ground_motion) VALUES (%s, %s, %s, %s, %s);""" arg2 = (target_id, geocell_id, scenario_id, session_id, _dist) # arg1 = """INSERT INTO processing.ground_motion (target_id, geocell_id, scenario_id, session_id, percentiles, ground_motion) VALUES (%s, %s, %s, %s, %s, %s);""" # arg2 = (target_id, geocell_id, scenario_id, session_id, _p, _dist) conn = globals.connection() conn.execute(arg1, arg2) conn.close() #do risk calculation (risk is Michael source, modified by me) if not ground_motion_only: risk_calc.calculaterisk(intensity, percentiles, session_id, scenario_id, target_id, geocell_id) except Exception as run_exc: if _DEBUG_: import traceback traceback.print_exc() #connect to the database and write the failed number: conn = globals.connection() conn.execute("UPDATE processing.sessions SET num_targets_failed = num_targets_failed + 1 where gid=%s",(session_id,)) conn.close() # log dir must be passed as argument problems when declaring global var (maybe multiprocess?) if _DEBUG_: if logdir is not None: try: folder = logdir if os.path.exists(folder): name = ''.join(random.choice('ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789') for _ in range(10)) if geocell_id is None else str(geocell_id) file = os.path.join(folder, name+".log") output = open(file,'w') traceback.print_exc(file=output) output.close() except: traceback.print_exc()
def caravan_run(input_event): """ Performs a gorund motion calculation of the Caravan application """ conn = None exception = None try: #Note: we pass a runinfo object actually, does this is necessary? how to handle if runinfo is already stopped? #should not be the case, however... runinfo = input_event.start() if isinstance(input_event, RunInfo) else RunInfo(input_event) #.start() harmless if already started scenario = runinfo.scenario() #NOTE: MCERP NPTS (params.DNP) CANNOT BE SET HERE CAUSE IT NEEDS TO BE SET PRIOR TO ANY CALCULATION, #EVEN THE SCENARIO CREATION (WHICH). MOVED TO CARAVAN_WSGI RUN_SIMULATION gmpe_func = scenario.gmpe #Connect from within the process: conn = globals.connection(async=False) scenario_id , isnew = scenario.writetodb(conn) if not isnew: #arg1 = """DELETE FROM processing.ground_motion WHERE scenario_id=%d """ % scenario_id arg1 = """DELETE FROM processing.ground_motion WHERE scenario_id=%s""" #From see http://www.postgresql.org/docs/9.1/static/sql-delete.html: #we could write "DELETE FROM processing.ground_motion WHERE scenario_id=%d RETURNING COUNT(*)""" % scenario_id #and then: conn.execute(arg1) #but that gives me (first command, postgres error): "aggregate functions are not allowed in RETURNING" #googling there are solutions (BUT starting from postgres9.1): #http://stackoverflow.com/questions/6577823/how-can-i-insert-the-return-of-delete-into-insert-in-postgresql #Therefore, skip that solution #ANOTHER NICER EXAMPLE found here: http://stackoverflow.com/questions/2251567/how-to-get-the-number-of-deleted-rows-in-postgresql #AS LONG AS THE CONNECTION IS NOT IN ASYNC MODE (we can switch it to sync here. It must be async inside each apply_async call below only) #USES psycopg2 cursor class and is preferable. Therefore: cursor = conn.cursor(arg1, (scenario_id,)) conn.commit() #NOTE: in async mode does nothing rows_deleted = cursor.rowcount cursor.close() #this works because (psycopg2 cursor doc) a QUERY was executed in execute(). Now, delete seems not a query to me, #however it seems to work runinfo.msg("Using already stored Scenario (hash={0:d}), deleted {1:d} previously calculated cells" .format(scenario.dbhash(),rows_deleted)) else: runinfo.msg("Written new scenario to databse (hash={0:d}):<br>Scenario = {1}".format(scenario.dbhash(),scenario.dbstr())) runinfo.msg("Scenario id: {:d}".format(scenario_id)) #define function to get value from scenario def scalar(key): val = scenario[key] if isinstance(val, mcerp.UncertainFunction): return val.mean #NOTE: not mean() !! it is a @property!! else: return float(val) #parse to float to be sure #CALCULATE RADIUS OF INTEREST key_a_ref = gk.AOI a_ref = scenario[key_a_ref] if key_a_ref in scenario else None ref_d = None #GET TESSELLATIONS: key_tess_ids = gk.TES tess_ids = globals.tess_ids if not key_tess_ids in scenario else scenario[key_tess_ids] tess_id_str = " or ".join([("t.tess_id={:d}".format(t)) for t in tess_ids]) runinfo.msg("Tessellation id(s): {}".format(', '.join(["{:d}".format(tid) for tid in tess_ids]))) if a_ref is not None: lon1, lat1, lon2, lat2 = a_ref targets = conn.fetchall("""select t.gid as target_id, t.geocell_id as geocell_id, st_X(t.the_geom) as lon, st_Y(t.the_geom) as lat from exposure.targets as t where ("""+tess_id_str+""") and st_within(t.the_geom, ST_MakeEnvelope(%s, %s, %s, %s, 4326));""", (lon1, lat1, lon2, lat2)) runinfo.msg("Area: map rectangle ("+key_a_ref+" parameter [lon1, lat1, lon2, lat2], see above)") #str should place dot or not automatically else: key_i_ref = gk.AIR key_i_ref_km_step = gk.AKS I_ref = scenario[key_i_ref] if key_i_ref in scenario else globals.aoi_i_ref ref_dist_km_step = scenario[key_i_ref_km_step] if key_i_ref_km_step in scenario else globals.aoi_km_step #ref_dist_km_step ref_d = ref_dist(gmpe_func, I_ref, ref_dist_km_step) runinfo.msg("Area(I ≥ {:.2f}) radius: {:.1f} Km" .format(I_ref, ref_d)) #str should place dot or not automatically if ref_d < ref_dist_km_step: msg = "Epicentral intensity smaller than intensity reference = {:.2f} mw".format(I_ref) if _DEBUG_: print(msg) raise Exception(msg) #GET TESSELLATION POINTS (GEOCELLS): key_lat = gk.LAT #"latitude" key_lon = gk.LON #"longitude" targets = conn.fetchall("""select t.gid as target_id, t.geocell_id as geocell_id, st_X(t.the_geom) as lon, st_Y(t.the_geom) as lat from exposure.targets as t where ("""+tess_id_str+""") and st_dwithin(geography(st_point(%s, %s)), geography(t.the_geom), %s);""" , (scalar(key_lon), scalar(key_lat), ref_d * 1000)) #conn.commit() subprocesses = len(targets) num_malformed = 0 if subprocesses: #targets is a table of columns: #target_id, geocell_id, lat, lon #with length the number of geocells #Note that geocell_id might be missing #Do a check Now? YES! def formwell(t): try: #t is a tuple, we cannot assign to it. #Simply do a check and preserve old values #Note that the check below is fine also if elements are numeric strings #the drawback of instantiating a new list is however too much effort #as data should be numeric _,_,_,_ = int(t[0]), int(t[1]), float(t[2]), float(t[3]) except: # import traceback # traceback.print_exc() return None return t for i in range(subprocesses): t = targets[i] twf = formwell(t) if twf is None: num_malformed+=1 targets[i] = twf else: msg = "No target cells found (zero cells)" raise Exception(msg) if num_malformed >= subprocesses: msg = "All target cells are malformed (missing or non-numeric values)" raise Exception(msg) elif num_malformed > 0: runinfo.warning("{:d} of {:d} target cells found (skipping {:d} malfromed cells, containing missing or NaN values)".\ format(subprocesses - num_malformed, subprocesses, num_malformed)) else: runinfo.msg("{:d} target cells found".format(subprocesses)) #WRITE session_id in processing.sessions #see https://docs.python.org/2/library/datetime.html#datetime.datetime.now (we use utcnow instead of now) #and http://initd.org/psycopg/docs/usage.html#adaptation-of-python-values-to-sql-types session_id = None while True: session_timestamp = datetime.utcnow() #returns a datetime object, which is converted to timestamp in psycopg2 session_rows = conn.fetchall("SELECT COUNT(*) FROM processing.sessions WHERE session_timestamp=%s;", (session_timestamp,)) if session_rows[0][0] == 0: #NOTE: do NOT run immediately num_malformed, otherwise the progressbar and time counter have invalid data #just set subprocesses-num_malformed as the cells to be done: cursor = conn.cursor("INSERT INTO processing.sessions (scenario_id, session_timestamp, num_targets, num_targets_failed) \ VALUES(%s, %s, %s, %s) RETURNING gid;", (scenario_id, session_timestamp, subprocesses-num_malformed, 0,)) #Note returns works because of RETURNING string in dbase execute command. execute() can return elements without return #explicitly written as long as (psycopg2 cursor doc) a QUERY was executed in execute(). Now, insert is not a query, #but for instance (see few lines above) DELETE it is conn.commit() #note: in async mode does nothing ret = cursor.fetchall() session_id = ret[0][0] #gid (serial number) of the newly added row cursor.close() break if session_id is None: raise Exception("Unable to get session_id. Internal server error") runinfo.msg("Session id: {:d}".format(session_id), "Starting main process (might take a while...)") #CREATE A PROCESS and a Pool of processes. For info see: #For info: http://stackoverflow.com/questions/25071910/multiprocessing-pool-calling-helper-functions-when-using-apply-asyncs-callback P = multiprocessing.Pool() runinfo.setprocess(P, session_id) #NOTES: #ARGUMENTS TO APPLY_ASYNC MUST BE PICKABLE, AS WELL AS THE FUNCTION (FIRST ARGUMENT). #SEE https://docs.python.org/2/library/pickle.html#what-can-be-pickled-and-unpickled #THIS MEANS THAT DISTRIBUTIONS MUST BE CREATED EACH TIME INSIDE OUR TARGET FUNCTION #NOTE ALSO THAT FOR DEBUG WE SHOULD USE APPLY OR MAP, the COUNTERPART OF APPLY_ASYNC AND MAP_ASYNC, cause the latter #do not raise exceptions #See also https://github.com/tisimst/mcerp/blob/master/mcerp/__init__.py logdir = None DO_WRITE_DIR = False #secondary debug flag if _DEBUG_ and DO_WRITE_DIR: folder = os.path.join(os.path.dirname(os.path.realpath(__file__)),"../_tmp/runerrors/{:d}".format(session_id)) if os.path.exists(folder): print("EMPTYING DIR {0}".format(str(folder))) for the_file in os.listdir(folder): file_path = os.path.join(folder, the_file) try: if os.path.isfile(file_path): os.unlink(file_path) except Exception: pass elif os.path.exists(os.path.dirname(folder)): try:os.mkdir(folder) except: pass if os.path.exists(folder): logdir = folder #'opposite' to scalar defined above: returns a numpy.ndarray if the value represents an mcerp distribution, #otherwise the float value. Note that numpy.ndarray seems to be PICKABLE ('passable' as multiprocess.Pool argument) def val(value): #value = scenario[key] if isinstance(value, mcerp.UncertainFunction): return value._mcpts #this is a numpy.array element WHICH IS PICKABLE, SO IT CAN BE PASSED TO multiprocess.Pool else: return float(value) #parse to float to be sure #close the connection NOW? conn.commit() conn.close() conn = None #set percentiles. Note that any mcerp distribution raises error if percentiles are outside #the number of points. Thus a distribution with 10 pts cannot calculate percentiles at, e.g., 0.05 and 0.95 #we therefore need to set the mcepr npts here percentiles = globals.percentiles key_gm_only = gk.GMO gm_only = scenario[key_gm_only] if key_gm_only in scenario else globals.gm_only #gmpe's might contain uncertain functions defined in mcerp WHICH ARE NOT PICKABLE! #Given the fact that making them pickable seems to be a mess for our case #(examples found in the internet deal with easy to use cases), we FIRST #calculate the intensity distribution, then we pass ITS ARRAY (a numpy array, which IS pickable) #to our geocell_run function #Drawback: we are not calculating intensity in a separate process, which means #we therefore do not take 100% advantages of the multiprocess pool for t in targets: if t is None: continue intensity = gmpe_func(t[3], t[2]) #geocell_run(gmpe_func, t[3], t[2], percentiles, t[0], t[1], gm_only, scenario_id, session_id, logdir) P.apply_async(geocell_run, [val(intensity), t[3], t[2], percentiles, t[0], t[1], gm_only, scenario_id, session_id, logdir]) P.close() except Exception as e: exception = e if _DEBUG_: import traceback print(traceback.format_exc()) finally: if conn is not None: conn.close() if exception is not None: runinfo.stop(e) return runinfo
def geocell_run(intensity, lat_sta, lon_sta, percentiles, target_id, geocell_id, ground_motion_only, scenario_id, session_id, logdir=None): """ Performs a ground motion calculation given the above arguments. Writes to database the percentiles """ def val(value): #value should be either a scalar number or a numpy.ndarray element. Check below is WEAK, because it checks only if #len(value) can be called, but it avoids importing numpy etcetera return mcerp.UncertainFunction(value) if hasattr( value, '__len__') else float(value) try: #intensity = gmpe_func(val(M), val(lat_hyp), val(lon_hyp), val(depth_hyp), lat_sta, lon_sta) #intensity = gmpe_func(lat_sta, lon_sta) #INTENSITY IS EITHER A SCALAR OR A UNCERTAINFUNCTION.MCERP ARRAY (NUMPTY ARRAY) intensity = val(intensity) # _p = intensity.percentile(percentiles) if isinstance(intensity, mcerp.UncertainFunction) else len(percentiles)*[intensity] _dist = [v for v in globals.discretepdf(intensity, _intensity_labels)] _dist.append( intensity.percentile(0.5) if isinstance(intensity, mcerp.UncertainFunction) else intensity) #write to dbase arg1 = """INSERT INTO processing.ground_motion (target_id, geocell_id, scenario_id, session_id, ground_motion) VALUES (%s, %s, %s, %s, %s);""" arg2 = (target_id, geocell_id, scenario_id, session_id, _dist) # arg1 = """INSERT INTO processing.ground_motion (target_id, geocell_id, scenario_id, session_id, percentiles, ground_motion) VALUES (%s, %s, %s, %s, %s, %s);""" # arg2 = (target_id, geocell_id, scenario_id, session_id, _p, _dist) conn = globals.connection() conn.execute(arg1, arg2) conn.close() #do risk calculation (risk is Michael source, modified by me) if not ground_motion_only: risk_calc.calculaterisk(intensity, percentiles, session_id, scenario_id, target_id, geocell_id) except Exception as run_exc: if _DEBUG_: import traceback traceback.print_exc() #connect to the database and write the failed number: conn = globals.connection() conn.execute( "UPDATE processing.sessions SET num_targets_failed = num_targets_failed + 1 where gid=%s", (session_id, )) conn.close() # log dir must be passed as argument problems when declaring global var (maybe multiprocess?) if _DEBUG_: if logdir is not None: try: folder = logdir if os.path.exists(folder): name = ''.join( random.choice( 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789') for _ in range(10)) if geocell_id is None else str( geocell_id) file = os.path.join(folder, name + ".log") output = open(file, 'w') traceback.print_exc(file=output) output.close() except: traceback.print_exc()
def query_simulation_data(request, response): event = request.json session_id = event['session_id'] #print("session id"+str(session_id)) conn = glb.connection(async=True) #query: #note: ::json casts as json, not as jason-formatted string #::double precision is NECESSARY as it returns a json convertible value, otherwise array alone returns python decimals #which need to be converted to double prior to json dumps data = conn.fetchall("""SELECT ST_AsGeoJSON(ST_Transform(G.the_geom,4326))::json AS geometry, GM.geocell_id, GM.ground_motion, risk.social_conseq.fatalities_prob_dist, risk.econ_conseq.total_loss FROM processing.ground_motion as GM LEFT JOIN risk.social_conseq ON (risk.social_conseq.geocell_id = GM.geocell_id and risk.social_conseq.session_id = GM.session_id) LEFT JOIN risk.econ_conseq ON (risk.econ_conseq.geocell_id = GM.geocell_id and risk.econ_conseq.session_id = GM.session_id) LEFT JOIN exposure.geocells as G ON (G.gid = GM.geocell_id) WHERE GM.session_id=%s""",(session_id,)) #conn.conn.commit() conn.close() #HYPOTHESES: #1) The query above returns a table T whose header (columns) are: # | geometry | geocell_id | ground_motion | fatalities_prob_dist | total_loss # #2) a single T row (R) corresponds to a geojson feature F: #{ # type: 'Feture', //geojson standard string (see doc) # geometry : dict, //associated to geometry column # id: number_or_string, //associated to geocell_id column # properties:{ # gk.MSI: {data: usually_array, value:numeric_scalar}, //associated to ground_motion column # gk.FAT: {data: usually_array, value:numeric_scalar}, //associated to fatalities_prob_dist column # gk.ECL: {data: usually_array, value:numeric_scalar} //associated to total_loss column # } #} # gk.MSI, gk.FAT and gk.MSI.ECL refer to globalkeys global variables. They are just strings # but defined globally for multi-purpose usage # 3) EACH PROPERTIES FIELD HAS TWO VALUES, DATA AND VALUE. WHICH WILL BE CALCULATED FROM ANY DATABASE ROW IN # THE FUNTION process DEFINED BELOW. DATA IS MEANT TO BE AN ARRAY OF DATA TO BE VISUALIZED WHEN MOUSE IS # OVER THE RELATIVE GEOCELL ON THE MAP, WHEREAS VALUE IS THE VALUE TO BE VISUALIZED BY MEANS OF E.G. A COLOR # FOR A PARTICULAR GEOCELL. Example: given a set of data bins representing the distribution at some values, e.g. [0.2, 0.5, 0.3], # then properties.data is that array, and value might be e.g., the median, or the max, or the index of max, or whatever # (the important thing is that JavaScript side one knows what are data and value in order to display them on the map) #NOW we define the columns to be set as properties (excluding geometry): #Each column here corresponds to a leaflet Layer in JavaScript #associating each of them to a table column index (see 1) in comment above): captions = {gk.MSI:2, gk.FAT:3, gk.ECL: 4} #THIS IS THE MAIN FUNCTION def process(name, row, row_index): try: data = row[row_index] if name == gk.MSI: #return the median #pop last element, which is the median according to core.py m = data.pop() #removes and returns the last index return data, m elif name == gk.FAT: #return INDEX OF max value remVal = 1 max = 0 ind_of_max = 0 i = 0 for n in data: if n > max: max = n ind_of_max = i remVal -= n if remVal < max: break #< not <=: priority to higher value, if two or more are equal i += 1 return data, ind_of_max elif name == gk.ECL: #economic losses, to be implemented pass except: pass #exception: return None, None below #elif ... here implement new values for newly added names return None, None dataret = {"type": "FeatureCollection", "features": None, "captions": {k:"" for k in captions}} features = [] #pre-allocation doesn't seem to matter. See e.g. http://stackoverflow.com/questions/311775/python-create-a-list-with-initial-capacity #set set of empty layers. As soon as we have a valid data #for a geocell g and a layer name N in the loop below, remove N from the #set defined below. This might be used JavaScript side to know immediately if a layer is empty #or not, avoiding consuming memory empty_layers = {k for k in captions} #do a cpoy for row in data: cell = {'geometry': row[0], 'id':row[1], 'type':'Feature', 'properties':{}} for name in captions: index = captions[name] data, value = process(name, row, index) #remove the empty layers key if data is valid: if name in empty_layers and not (data is None and value is None): empty_layers.remove(name) property = {'data': data, 'value': value} cell['properties'][name] = property features.append(cell) dataret['features'] = features dataret['emptyLayers'] = {k:True for k in empty_layers} return response.tojson(dataret)