def do_point_query(points, raster, csv_pth, num, log_out): try: u.verify_dir(csv_pth) dataset = raster.split('\\')[-2].replace(' ', '_') raster_name_to_csv = os.path.basename(raster).replace('.tif', '.csv') csv_name = '__'.join([dataset, raster_name_to_csv]) csv_out = os.path.join(csv_pth, csv_name) start = timer() u.write_to_log(' {}) Raster: {}'.format(num, os.path.basename(raster)), log_out) stats = point_query(points, raster, interpolate='nearest', geojson_out=True) print(' point_query... ({} sec.)'.format(round(timer()-start, 2))) start = timer() attributes = [] for item in stats: #print ('{}'.format(item['properties'])) attributes.append(item['properties']) print(' append dicts... ({} sec.)'.format(round(timer()-start, 2))) start = timer() with open(csv_out, 'w', newline='') as outfile: fp = csv.DictWriter(outfile, attributes[0].keys()) fp.writeheader() fp.writerows(attributes) print(' write to csv... ({} sec.)'.format(round(timer()-start, 2))) u.write_to_log(' CSV file: {}'.format(csv_out), log_out) u.write_to_log(' Log file: {}'.format(log_out), log_out) except Exception as e: u.write_to_log(str(e), log_out) u.write_to_log('FINISH POINTS: {}'.format(time.strftime("%Y-%m-%d %H:%M:%S")), log_out)
def save(self, path: str): self.conns[0].send(("save", path)) assert self.conns[0].recv(2400, "Save main ckpt") for i, conn in enumerate(self.conns[1:]): conn.send(("save_state", path + ".state{:d}".format(i + 1))) assert conn.recv(600, "Save state {:d}".format(i)) verify_dir(path) torch.save(self.historical_train_loss, path + ".state-dist-controller")
def save_state(self, path: str): verify_dir(path) torch.save(dict( data=self.data, last_loss=self.last_loss, historical_train_loss=self.historical_train_loss, historical_valid_loss=self.historical_valid_loss, state_repr=self._state, historical_last_epoch_valid_loss=self.historical_last_epoch_valid_loss, k_coverage_num=self.k_coverage_num, k_coverage_num_current_epoch=self.k_coverage_num_current_epoch, ), path)
def save(self, path: str): verify_dir(path) extra_state = dict( train_iterator=self.epoch_itr.state_dict(), data=self.data, last_loss=self.last_loss, historical_train_loss=self.historical_train_loss, historical_valid_loss=self.historical_valid_loss, state_repr=self._state, historical_last_epoch_valid_loss=self.historical_last_epoch_valid_loss, k_coverage_num=self.k_coverage_num, k_coverage_num_current_epoch=self.k_coverage_num_current_epoch, ) self.trainer.save_checkpoint(path, extra_state)
def download_shock_handle(self, handle, stdout=False, outdir=None, prefix=''): shock_url = handle.get('shock_url') or handle.get('url') shock_id = handle.get('shock_id') or handle.get('id') if not shock_url or not shock_id: raise Error("Invalid shock handle: {}".format(handle)) url = "{}/node/{}?download".format(shock_url, shock_id) if stdout: filename = None else: outdir = utils.verify_dir(outdir) if outdir else None filename = handle.get('filename') or handle.get('local_file') or shock_id filename = prefix + filename.split('/')[-1] filename = os.path.join(outdir, filename) if outdir else filename headers = {'Authorization': 'OAuth {}'.format(self.token)} r = requests.get(url, stream=True, headers=headers) with self.smart_open(filename) as f: for chunk in r.iter_content(chunk_size=1024): if chunk: # filter out keep-alive new chunks f.write(chunk) f.flush() if filename: if not os.path.exists(filename): raise Error('Data exists but file not properly saved') else: sys.stderr.write("File downloaded: {}\n".format(filename)) return filename
def save(self, student_metric): self.clear_print_buffer() # pdb.set_trace() teacher, student, trainer = self.checkpoint_name(self.step) self.agent.save(teacher) self.env.save(student) verify_dir(trainer) torch.save(dict(best_student_metric=self.best_student_metric), trainer) self.write_ckpt_info( os.path.join(self.args.rl_search_save_dir, "checkpoint_last.txt"), self.step) if student_metric < self.best_student_metric: print("= New best student! step %d, %r -> %r" % (self.step, self.best_student_metric, student_metric)) self.best_student_metric = student_metric self.write_ckpt_info( os.path.join(self.args.rl_search_save_dir, "best.txt"), self.step)
def main_code(buff_dir, pt_dir, combine_dir, log): u.verify_dir(combine_dir) start_all = timer() u.write_to_log( '\nSTART COMBINE: {}'.format(time.strftime("%Y-%m-%d %H:%M:%S")), log) buff_list = [] for filename in glob.iglob(buff_dir + '/*.csv'): buff_list.append(filename) pt_list = [] for filename in glob.iglob(pt_dir + '/*.csv'): pt_list.append(filename) match_count = 0 for buff_csv in buff_list: buff_name = os.path.basename(buff_csv) process_buff = buff_csv for pt_csv in pt_list: pt_name = os.path.basename(pt_csv) if buff_name == pt_name: process_pt = pt_csv match_count += 1 u.write_to_log( ' {}) buffers: {} points: {}'.format( match_count, buff_name, pt_name), log) #csv_name = os.path.basename(os.path.basename(buff_csv).replace('.csv', '_pandas.csv')) out_csv = os.path.join(combine_dir, buff_name) buff_df = pd.read_csv(buff_csv, dtype='object') pt_df = pd.read_csv(pt_csv, dtype='object') print(' buff shape: {}'.format(buff_df.shape)) print(' pt shape: {}'.format(pt_df.shape)) merged = pd.merge(left=buff_df, right=pt_df, on='DHSID') print(' merge shape: {}'.format(merged.shape)) for col in list(merged): if col.endswith("_y"): merged = merged.drop(str(col), 1) # These select columns by name and merge point values into them if # the buffer analysis resulted in a blank value. try: merged.loc[merged['mean'].isnull(), 'mean'] = merged['value'] except: print(' no MEAN column') try: merged.loc[merged['sum'].isnull(), 'sum'] = merged['value'] except: print(' no SUM column') try: merged.loc[merged['majority'].isnull(), 'majority'] = merged['value'] except: print(' no MAJORITY column') try: merged = merged.drop('nodata', 1) except: print(' no NODATA column') merged = merged.drop('value', 1) merged.to_csv(out_csv, sep=',') u.write_to_log( 'FINISH COMBINE: {}\nELAPSED TIME: {} sec.'.format( time.strftime("%Y-%m-%d %H:%M:%S"), round(timer() - start_all, 3)), log)
help="specify the rabbitmq port", action="store") parser.add_argument("--logfile", help="specify the logfile", action="store") parser.add_argument("--deploy-config", help="specify the deployment-specific config", action="store") parser.add_argument("-c", "--config", help="specify the config file", action="store", required=True) args = parser.parse_args() logfile = args.logfile or 'ar_server.log' utils.verify_dir(os.path.dirname(logfile)) logging.basicConfig(format="[%(asctime)s %(levelname)s %(name)s] %(message)s", level=logging.INFO, filename=logfile) if args.verbose: logging.root.setLevel(logging.DEBUG) start(args.config, shock_url=args.shock_url, mongo_host=args.mongo_host, mongo_port=args.mongo_port, rabbit_host=args.rabbit_host, rabbit_port=args.rabbit_port, deploy_config=args.deploy_config)
action="store") parser.add_argument("--mongo-host", help="specify the mongodb url", action="store") parser.add_argument("--mongo-port", help="specify the mongodb port", action="store") parser.add_argument("--rabbit-host", help="specify the rabbitmq url", action="store") parser.add_argument("--rabbit-port", help="specify the rabbitmq port", action="store") parser.add_argument("--logfile", help="specify the logfile", action="store") parser.add_argument("--deploy-config", help="specify the deployment-specific config", action="store") parser.add_argument("-c", "--config", help="specify the config file", action="store", required=True) args = parser.parse_args() logfile = args.logfile or 'ar_server.log' utils.verify_dir(os.path.dirname(logfile)) logging.basicConfig(format="[%(asctime)s %(levelname)s %(name)s] %(message)s", level=logging.INFO, filename=logfile) if args.verbose: logging.root.setLevel(logging.DEBUG) start(args.config, shock_url=args.shock_url, mongo_host=args.mongo_host, mongo_port=args.mongo_port, rabbit_host=args.rabbit_host, rabbit_port=args.rabbit_port, deploy_config=args.deploy_config)
import sys import time import config as c import spatial_covariates_buffer_extraction as run_buff import spatial_covariates_point_extraction as run_pts import combine_with_pandas as combine import utils as u try: # User input - should correspond to a value in config points, buffers, and rasters # to run, in command prompt, for example, python run_all.py clark_1866 projection_group = sys.argv[1] u.assert_valid_user_input(projection_group) # Initialize log file u.verify_dir(c.log_pth) log_name = 'log_{}.txt'.format(time.strftime("%Y-%m-%d_%H-%M-%S")) log_out = os.path.join(c.log_pth, log_name) print('START') # Do zonal stats with buffers run_buff.main_code(c.rasters[projection_group], c.buffers[projection_group], c.csv_pth['buffers'], log_out, c.rasters['regex'], c.stats) # Assign raster value to points run_pts.main_code(c.rasters[projection_group], c.points[projection_group], c.csv_pth['points'], log_out, c.rasters['regex']) # Combine csv files. Where zonal stats is blank, assign point value. combine.main_code(c.csv_pth['buffers'], c.csv_pth['points'],