def do_point_query(points, raster, csv_pth, num, log_out):
try:
u.verify_dir(csv_pth)
dataset = raster.split('\\')[-2].replace(' ', '_')
raster_name_to_csv = os.path.basename(raster).replace('.tif', '.csv')
csv_name = '__'.join([dataset, raster_name_to_csv])
csv_out = os.path.join(csv_pth, csv_name)
start = timer()
u.write_to_log(' {}) Raster: {}'.format(num, os.path.basename(raster)), log_out)
stats = point_query(points, raster, interpolate='nearest', geojson_out=True)
print(' point_query... ({} sec.)'.format(round(timer()-start, 2)))
start = timer()
attributes = []
for item in stats:
#print ('{}'.format(item['properties']))
attributes.append(item['properties'])
print(' append dicts... ({} sec.)'.format(round(timer()-start, 2)))
start = timer()
with open(csv_out, 'w', newline='') as outfile:
fp = csv.DictWriter(outfile, attributes[0].keys())
fp.writeheader()
fp.writerows(attributes)
print(' write to csv... ({} sec.)'.format(round(timer()-start, 2)))
u.write_to_log(' CSV file: {}'.format(csv_out), log_out)
u.write_to_log(' Log file: {}'.format(log_out), log_out)
except Exception as e:
u.write_to_log(str(e), log_out)
u.write_to_log('FINISH POINTS: {}'.format(time.strftime("%Y-%m-%d %H:%M:%S")), log_out)
def save(self, path: str):
self.conns[0].send(("save", path))
assert self.conns[0].recv(2400, "Save main ckpt")
for i, conn in enumerate(self.conns[1:]):
conn.send(("save_state", path + ".state{:d}".format(i + 1)))
assert conn.recv(600, "Save state {:d}".format(i))
verify_dir(path)
torch.save(self.historical_train_loss, path + ".state-dist-controller")
def save_state(self, path: str):
verify_dir(path)
torch.save(dict(
data=self.data,
last_loss=self.last_loss,
historical_train_loss=self.historical_train_loss,
historical_valid_loss=self.historical_valid_loss,
state_repr=self._state,
historical_last_epoch_valid_loss=self.historical_last_epoch_valid_loss,
k_coverage_num=self.k_coverage_num,
k_coverage_num_current_epoch=self.k_coverage_num_current_epoch,
), path)
def save(self, path: str):
verify_dir(path)
extra_state = dict(
train_iterator=self.epoch_itr.state_dict(),
data=self.data,
last_loss=self.last_loss,
historical_train_loss=self.historical_train_loss,
historical_valid_loss=self.historical_valid_loss,
state_repr=self._state,
historical_last_epoch_valid_loss=self.historical_last_epoch_valid_loss,
k_coverage_num=self.k_coverage_num,
k_coverage_num_current_epoch=self.k_coverage_num_current_epoch,
)
self.trainer.save_checkpoint(path, extra_state)
def download_shock_handle(self, handle, stdout=False, outdir=None, prefix=''):
shock_url = handle.get('shock_url') or handle.get('url')
shock_id = handle.get('shock_id') or handle.get('id')
if not shock_url or not shock_id:
raise Error("Invalid shock handle: {}".format(handle))
url = "{}/node/{}?download".format(shock_url, shock_id)
if stdout:
filename = None
else:
outdir = utils.verify_dir(outdir) if outdir else None
filename = handle.get('filename') or handle.get('local_file') or shock_id
filename = prefix + filename.split('/')[-1]
filename = os.path.join(outdir, filename) if outdir else filename
headers = {'Authorization': 'OAuth {}'.format(self.token)}
r = requests.get(url, stream=True, headers=headers)
with self.smart_open(filename) as f:
for chunk in r.iter_content(chunk_size=1024):
if chunk: # filter out keep-alive new chunks
f.write(chunk)
f.flush()
if filename:
if not os.path.exists(filename):
raise Error('Data exists but file not properly saved')
else:
sys.stderr.write("File downloaded: {}\n".format(filename))
return filename
def download_shock_handle(self, handle, stdout=False, outdir=None, prefix=''):
shock_url = handle.get('shock_url') or handle.get('url')
shock_id = handle.get('shock_id') or handle.get('id')
if not shock_url or not shock_id:
raise Error("Invalid shock handle: {}".format(handle))
url = "{}/node/{}?download".format(shock_url, shock_id)
if stdout:
filename = None
else:
outdir = utils.verify_dir(outdir) if outdir else None
filename = handle.get('filename') or handle.get('local_file') or shock_id
filename = prefix + filename.split('/')[-1]
filename = os.path.join(outdir, filename) if outdir else filename
headers = {'Authorization': 'OAuth {}'.format(self.token)}
r = requests.get(url, stream=True, headers=headers)
with self.smart_open(filename) as f:
for chunk in r.iter_content(chunk_size=1024):
if chunk: # filter out keep-alive new chunks
f.write(chunk)
f.flush()
if filename:
if not os.path.exists(filename):
raise Error('Data exists but file not properly saved')
else:
sys.stderr.write("File downloaded: {}\n".format(filename))
return filename
def save(self, student_metric):
self.clear_print_buffer()
# pdb.set_trace()
teacher, student, trainer = self.checkpoint_name(self.step)
self.agent.save(teacher)
self.env.save(student)
verify_dir(trainer)
torch.save(dict(best_student_metric=self.best_student_metric), trainer)
self.write_ckpt_info(
os.path.join(self.args.rl_search_save_dir, "checkpoint_last.txt"),
self.step)
if student_metric < self.best_student_metric:
print("= New best student! step %d, %r -> %r" %
(self.step, self.best_student_metric, student_metric))
self.best_student_metric = student_metric
self.write_ckpt_info(
os.path.join(self.args.rl_search_save_dir, "best.txt"),
self.step)
def main_code(buff_dir, pt_dir, combine_dir, log):
u.verify_dir(combine_dir)
start_all = timer()
u.write_to_log(
'\nSTART COMBINE: {}'.format(time.strftime("%Y-%m-%d %H:%M:%S")), log)
buff_list = []
for filename in glob.iglob(buff_dir + '/*.csv'):
buff_list.append(filename)
pt_list = []
for filename in glob.iglob(pt_dir + '/*.csv'):
pt_list.append(filename)
match_count = 0
for buff_csv in buff_list:
buff_name = os.path.basename(buff_csv)
process_buff = buff_csv
for pt_csv in pt_list:
pt_name = os.path.basename(pt_csv)
if buff_name == pt_name:
process_pt = pt_csv
match_count += 1
u.write_to_log(
' {}) buffers: {} points: {}'.format(
match_count, buff_name, pt_name), log)
#csv_name = os.path.basename(os.path.basename(buff_csv).replace('.csv', '_pandas.csv'))
out_csv = os.path.join(combine_dir, buff_name)
buff_df = pd.read_csv(buff_csv, dtype='object')
pt_df = pd.read_csv(pt_csv, dtype='object')
print(' buff shape: {}'.format(buff_df.shape))
print(' pt shape: {}'.format(pt_df.shape))
merged = pd.merge(left=buff_df, right=pt_df, on='DHSID')
print(' merge shape: {}'.format(merged.shape))
for col in list(merged):
if col.endswith("_y"):
merged = merged.drop(str(col), 1)
# These select columns by name and merge point values into them if
# the buffer analysis resulted in a blank value.
try:
merged.loc[merged['mean'].isnull(),
'mean'] = merged['value']
except:
print(' no MEAN column')
try:
merged.loc[merged['sum'].isnull(), 'sum'] = merged['value']
except:
print(' no SUM column')
try:
merged.loc[merged['majority'].isnull(),
'majority'] = merged['value']
except:
print(' no MAJORITY column')
try:
merged = merged.drop('nodata', 1)
except:
print(' no NODATA column')
merged = merged.drop('value', 1)
merged.to_csv(out_csv, sep=',')
u.write_to_log(
'FINISH COMBINE: {}\nELAPSED TIME: {} sec.'.format(
time.strftime("%Y-%m-%d %H:%M:%S"), round(timer() - start_all,
3)), log)
help="specify the rabbitmq port",
action="store")
parser.add_argument("--logfile", help="specify the logfile", action="store")
parser.add_argument("--deploy-config",
help="specify the deployment-specific config",
action="store")
parser.add_argument("-c",
"--config",
help="specify the config file",
action="store",
required=True)
args = parser.parse_args()
logfile = args.logfile or 'ar_server.log'
utils.verify_dir(os.path.dirname(logfile))
logging.basicConfig(format="[%(asctime)s %(levelname)s %(name)s] %(message)s",
level=logging.INFO,
filename=logfile)
if args.verbose:
logging.root.setLevel(logging.DEBUG)
start(args.config,
shock_url=args.shock_url,
mongo_host=args.mongo_host,
mongo_port=args.mongo_port,
rabbit_host=args.rabbit_host,
rabbit_port=args.rabbit_port,
deploy_config=args.deploy_config)
action="store")
parser.add_argument("--mongo-host", help="specify the mongodb url",
action="store")
parser.add_argument("--mongo-port", help="specify the mongodb port",
action="store")
parser.add_argument("--rabbit-host", help="specify the rabbitmq url",
action="store")
parser.add_argument("--rabbit-port", help="specify the rabbitmq port",
action="store")
parser.add_argument("--logfile", help="specify the logfile",
action="store")
parser.add_argument("--deploy-config", help="specify the deployment-specific config",
action="store")
parser.add_argument("-c", "--config", help="specify the config file",
action="store", required=True)
args = parser.parse_args()
logfile = args.logfile or 'ar_server.log'
utils.verify_dir(os.path.dirname(logfile))
logging.basicConfig(format="[%(asctime)s %(levelname)s %(name)s] %(message)s",
level=logging.INFO, filename=logfile)
if args.verbose:
logging.root.setLevel(logging.DEBUG)
start(args.config, shock_url=args.shock_url, mongo_host=args.mongo_host, mongo_port=args.mongo_port,
rabbit_host=args.rabbit_host, rabbit_port=args.rabbit_port,
deploy_config=args.deploy_config)
import sys
import time
import config as c
import spatial_covariates_buffer_extraction as run_buff
import spatial_covariates_point_extraction as run_pts
import combine_with_pandas as combine
import utils as u
try:
# User input - should correspond to a value in config points, buffers, and rasters
# to run, in command prompt, for example, python run_all.py clark_1866
projection_group = sys.argv[1]
u.assert_valid_user_input(projection_group)
# Initialize log file
u.verify_dir(c.log_pth)
log_name = 'log_{}.txt'.format(time.strftime("%Y-%m-%d_%H-%M-%S"))
log_out = os.path.join(c.log_pth, log_name)
print('START')
# Do zonal stats with buffers
run_buff.main_code(c.rasters[projection_group],
c.buffers[projection_group], c.csv_pth['buffers'],
log_out, c.rasters['regex'], c.stats)
# Assign raster value to points
run_pts.main_code(c.rasters[projection_group], c.points[projection_group],
c.csv_pth['points'], log_out, c.rasters['regex'])
# Combine csv files. Where zonal stats is blank, assign point value.
combine.main_code(c.csv_pth['buffers'], c.csv_pth['points'],
