sys.path.insert(0, utils_dir)
from config_utility import BranchConfig
config = BranchConfig(branch=branch)
config.test_connection()
# -----------------------------------------------------------------------------
import geo_rasterstats as rs
# -------------------------------------
# initialize mpi
import mpi_utility
job = mpi_utility.NewParallel(capture=True)
# -------------------------------------
# verify good mongo connection on all processors
connect_status = job.comm.gather(
(job.rank, config.connection_status, config.connection_error), root=0)
if job.rank == 0:
connection_error = False
for i in connect_status:
if i[1] != 0:
print(
"mongodb connection error ({0} - {1}) "
"on processor rank {2})").format(i[1], i[2], [3])
connection_error = True
utils_dir = os.path.join(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'utils')
sys.path.insert(0, utils_dir)
from config_utility import BranchConfig
config = BranchConfig(branch=branch)
config.test_connection()
# -----------------------------------------------------------------------------
# -------------------------------------
# initialize mpi
import mpi_utility
job = mpi_utility.NewParallel(capture=True, print_worker_log=False)
# -------------------------------------
# verify good mongo connection on all processors
connect_status = job.comm.gather(
(job.rank, config.connection_status, config.connection_error), root=0)
if job.rank == 0:
connection_error = False
for i in connect_status:
if i[1] != 0:
print(
"mongodb connection error ({0} - {1}) "
"on processor rank {2})").format(i[1], i[2], [3])
connection_error = True
update = sys.argv[4]
else:
update = False
if len(sys.argv) >= 6:
dry_run = sys.argv[5]
else:
dry_run = False
qlist = [
os.path.join(data_dir, i) for i in os.listdir(data_dir)
if os.path.isdir(os.path.join(data_dir, i))
]
qlist.sort()
job = mpi_utility.NewParallel(parallel=method)
if job.rank == 0:
print 'GeoBoundaries found: {}'.format(len(qlist))
def tmp_master_init(self):
# start job timer
self.Ts = int(time.time())
self.T_start = time.localtime()
print 'Start: ' + time.strftime('%Y-%m-%d %H:%M:%S', self.T_start)
def tmp_master_process(self, worker_data):
pass
config_dir = os.path.join(branch_dir, 'asdf', 'src', 'utils')
sys.path.insert(0, config_dir)
from config_utility import BranchConfig
config_attempts = 0
while True:
config = BranchConfig(branch=branch)
config_attempts += 1
if config.connection_status == 0 or config_attempts > 5:
break
# -------------------------------------------------------------------------
import mpi_utility
job = mpi_utility.NewParallel()
connect_status = job.comm.gather(
(job.rank, config.connection_status, config.connection_error), root=0)
if job.rank == 0:
connection_error = False
for i in connect_status:
if i[1] != 0:
print "mongodb connection error ({0} - {1}) on processor rank {2})".format(
i[1], i[2], [3])
connection_error = True
if connection_error:
sys.exit()
self.state.at[ix, 'valid_extract'] = valid_extract
self.state.at[ix, 'valid_files'] = valid_files
self.state.at[ix, 'shapefile'] = valid_shapefile_path
def s1_master_final(self):
# stop job timer
T_run = int(time.time() - self.Ts)
T_end = time.localtime()
print '\n\n'
print 'Start: ' + time.strftime('%Y-%m-%d %H:%M:%S', self.T_start)
print 'End: ' + time.strftime('%Y-%m-%d %H:%M:%S', T_end)
print 'Runtime: ' + str(T_run // 60) + 'm ' + str(int(
T_run % 60)) + 's'
print '\n\n'
s1_job = mpi_utility.NewParallel(parallel=parallel)
s1_job.state = state.copy(deep=True)
s1_job.shapefile_extenions = ["shp", "shx", "dbf"]
s1_qlist = list(state.loc[state['valid_init'] == True].iterrows())
s1_job.set_task_list(s1_qlist)
s1_job.set_general_init(s1_general_init)
s1_job.set_master_init(s1_master_init)
s1_job.set_worker_job(s1_worker_job)
s1_job.set_master_process(s1_master_process)
s1_job.set_master_final(s1_master_final)
s1_job.run()
