sys.path.insert(0, utils_dir) from config_utility import BranchConfig config = BranchConfig(branch=branch) config.test_connection() # ----------------------------------------------------------------------------- import geo_rasterstats as rs # ------------------------------------- # initialize mpi import mpi_utility job = mpi_utility.NewParallel(capture=True) # ------------------------------------- # verify good mongo connection on all processors connect_status = job.comm.gather( (job.rank, config.connection_status, config.connection_error), root=0) if job.rank == 0: connection_error = False for i in connect_status: if i[1] != 0: print( "mongodb connection error ({0} - {1}) " "on processor rank {2})").format(i[1], i[2], [3]) connection_error = True
utils_dir = os.path.join( os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'utils') sys.path.insert(0, utils_dir) from config_utility import BranchConfig config = BranchConfig(branch=branch) config.test_connection() # ----------------------------------------------------------------------------- # ------------------------------------- # initialize mpi import mpi_utility job = mpi_utility.NewParallel(capture=True, print_worker_log=False) # ------------------------------------- # verify good mongo connection on all processors connect_status = job.comm.gather( (job.rank, config.connection_status, config.connection_error), root=0) if job.rank == 0: connection_error = False for i in connect_status: if i[1] != 0: print( "mongodb connection error ({0} - {1}) " "on processor rank {2})").format(i[1], i[2], [3]) connection_error = True
update = sys.argv[4] else: update = False if len(sys.argv) >= 6: dry_run = sys.argv[5] else: dry_run = False qlist = [ os.path.join(data_dir, i) for i in os.listdir(data_dir) if os.path.isdir(os.path.join(data_dir, i)) ] qlist.sort() job = mpi_utility.NewParallel(parallel=method) if job.rank == 0: print 'GeoBoundaries found: {}'.format(len(qlist)) def tmp_master_init(self): # start job timer self.Ts = int(time.time()) self.T_start = time.localtime() print 'Start: ' + time.strftime('%Y-%m-%d %H:%M:%S', self.T_start) def tmp_master_process(self, worker_data): pass
config_dir = os.path.join(branch_dir, 'asdf', 'src', 'utils') sys.path.insert(0, config_dir) from config_utility import BranchConfig config_attempts = 0 while True: config = BranchConfig(branch=branch) config_attempts += 1 if config.connection_status == 0 or config_attempts > 5: break # ------------------------------------------------------------------------- import mpi_utility job = mpi_utility.NewParallel() connect_status = job.comm.gather( (job.rank, config.connection_status, config.connection_error), root=0) if job.rank == 0: connection_error = False for i in connect_status: if i[1] != 0: print "mongodb connection error ({0} - {1}) on processor rank {2})".format( i[1], i[2], [3]) connection_error = True if connection_error: sys.exit()
self.state.at[ix, 'valid_extract'] = valid_extract self.state.at[ix, 'valid_files'] = valid_files self.state.at[ix, 'shapefile'] = valid_shapefile_path def s1_master_final(self): # stop job timer T_run = int(time.time() - self.Ts) T_end = time.localtime() print '\n\n' print 'Start: ' + time.strftime('%Y-%m-%d %H:%M:%S', self.T_start) print 'End: ' + time.strftime('%Y-%m-%d %H:%M:%S', T_end) print 'Runtime: ' + str(T_run // 60) + 'm ' + str(int( T_run % 60)) + 's' print '\n\n' s1_job = mpi_utility.NewParallel(parallel=parallel) s1_job.state = state.copy(deep=True) s1_job.shapefile_extenions = ["shp", "shx", "dbf"] s1_qlist = list(state.loc[state['valid_init'] == True].iterrows()) s1_job.set_task_list(s1_qlist) s1_job.set_general_init(s1_general_init) s1_job.set_master_init(s1_master_init) s1_job.set_worker_job(s1_worker_job) s1_job.set_master_process(s1_master_process) s1_job.set_master_final(s1_master_final) s1_job.run()