# before running, type:
# cd /cygdrive/r/working/bin/
# source bash_profile
# then, type:
# cd /cygdrive/r/working/education/
# grad
'''
# this script will use this flag to call itself for the second version
v_2 = False
try:
v_2 = True if sys.argv[1] == "2" else False
if not v_2:
if sys.argv[1] != "1":
err("argument must be 1 or 2")
except:
pass
if len(sys.argv) < 3:
print "Important: delete the *.p files before running, if you change the cohort file!!!"
err("usage:\n\tgrad [version 1 or 2: start with 1] [cohort file.csv]\nhuman should enter 1 for first parameter."
)
if not os.path.exists("dd") or not os.path.isdir("dd"):
# find, extract and clean data dictionary files
run("dd_list")
run("dd_clean")
# data dictionary file for registry:
dd_reg = "dd/2018-09-27_data_dictionary_consolidation-file-january-1-1986-onwards.xlsx_registry.C.csv2"
# stack raster files
import os
import sys
from misc import args, sep, exists, pd, run, err
if len(args) < 4:
err("python3 raster_stack.py [input raster 1] [input raster 2] .." +
" [input raster n] [output raster name]")
rasters = args[1:-1] # raster files to cat together
outf = args[-1] # output filename
ofhn = args[-1][:-4] + '.hdr' # output header file name
cmd = ['cat'] + rasters + ['>', outf] # cat command
cmd = ' '.join(cmd)
print(cmd)
print('')
if not exists(outf):
a = os.system(cmd)
for r in rasters:
print(" ", r)
cmd = [
'python3', # envi_header_cat.py is almost like a reverse-polish notation. Have to put the "first thing" on the back..
pd + 'envi_header_cat.py',
rasters[1][:-4] + '.hdr',
rasters[0][:-4] + '.hdr',
ofhn
] # 'raster.hdr']
cmd = ' '.join(cmd)
run(cmd)
'''demo method to map an effective fire boundary'''
POINT_THRES = 50 # don't get hulls for shapes with less than 50 points
import os
import sys
from misc import err, run, pd, sep, exists, args
cd = pd + '..' + sep + 'cpp' + sep
if len(args) < 2:
err('sentinel2_trace_active.py [input file, binary mask envi type 4] [optional arg: class index]'
)
class_select = None
if len(args) > 2:
class_select = int(args[2])
print(class_select)
fn = args[1]
if not exists(fn):
err('please check input file')
for i in [150]: # [10, 20, 60]: # 90
if not exists(fn + '_flood4.bin'):
run('ulimit -s 1000000')
run(cd + 'flood.exe ' + fn)
if not exists(fn + '_flood4.bin_link.bin'):
run(cd + 'class_link.exe ' + fn + '_flood4.bin ' + str(i)) # 40')
if not exists(fn + '_flood4.bin_link.bin_recode.bin'):
run(cd + 'class_recode.exe ' + fn + '_flood4.bin_link.bin 1')
if not exists(fn + '_flood4.bin_link.bin_recode.bin_wheel.bin'):
run(cd + 'class_wheel.exe ' + fn + '_flood4.bin_link.bin_recode.bin')
'''raster_band_select.py:
band-subselection by matching on a string (such as by date)
'''
from misc import args, sep, exists, pd, run, hdr_fn, band_names, err, read_hdr
if len(args) < 3:
err('raster_band_select.py [input binary file] ' +
'[string to match band names e.g. 20210721 for jul 21 2021')
fn, s = args[1], args[2]
cd = pd + '..' + sep + 'cpp' + sep
hfn = hdr_fn(fn)
samples, lines, bands = read_hdr(hfn)
nrow, ncol = lines, samples
bi = []
bn = band_names(hfn)
for i in range(len(bn)):
if len(bn[i].split(s)) > 1:
bi.append(i)
print(' ', str(i + 1), ' ', bn[i])
if not exists(fn + '_001.hdr'):
run(cd + 'unstack.exe ' + fn)
for i in bi:
f = fn + '_' + str(i + 1).zfill(3) + '.bin'
print(f)
fni = [fn + '_' + str(i + 1).zfill(3) + '.bin' for i in bi]
bands = str(len(bi)) # number of bands to write
err('python3 alphashape.py ' +
'# please check ../cpp/binary_hull.cpp for parameters')
'''
ci, x = 0, None
for line in fileinput.input():
if ci == 0:
X = copy.deepcopy(line)
ci += 1
X_bak = copy.deepcopy(X)
X = X.strip().split()
N = int(X[1])
X = X[2:]
if len(X) != 2 * N:
err("bad data count")
X = [float(x) for x in X]
ci, points = 0, []
for i in range(N):
#points.append((X[ci + 1], -X[ci]))
points.append((X[ci], X[ci + 1]))
ci += 2
points = np.array(points)
print("optimizing alpha..")
alpha = 1. / 50. # 0.95 * alphashape.optimizealpha(points) # optimal alpha
patch_alpha = .2
print("finding alpha shape..")
alpha_shape = alphashape.alphashape(points, alpha)
count = {}
for row in reader:
N = len(row)
if N not in count:
count[N] = 0
count[N] += 1
lines.append(row)
print(count)
hdr = lines[0]
lines = lines[1:]
print(args)
to_slice = args[2:]
if len(to_slice) == 1:
to_slice = to_slice[0].strip('"').strip().split(',')
to_slice = [x.strip() for x in to_slice]
for field in to_slice:
if field not in hdr:
err("field not found in header: " + field)
slice_i = [hdr.index(field) for field in to_slice]
f = wopen(args[1] + "_slice.csv")
f.write((','.join([hdr[i] for i in slice_i])).encode())
for i in range(len(lines)):
f.write(('\n' + (','.join([lines[i][j] for j in slice_i]))).encode())
f.close()
...spectra with the same label are given the same color and same key in legend'''
import os
import sys
import csv
import matplotlib
import numpy as np
import matplotlib.pyplot as plt
from misc import read_csv
from misc import markers
from misc import colors
from misc import exist
from misc import err
args = sys.argv
if len(args) < 3:
err('python3 csv_raster_class_stats.py [csv file] [label of col to aggregate over]'
)
'''read the csv and locate the spectra'''
fields, data = read_csv(args[1])
nf = len(fields) # number of fields
f_i = {fields[i]: i for i in range(nf)}
if len(args) < 3: # call the program on all fields!
for f in fields:
if (f[-2:] != 'nm') and \
(f not in ['ObjectID', 'GlobalID', 'x', 'y',
'ctr_lat', 'ctr_lon', 'image']):
cmd = 'python3 ' + __file__ + ' ' + args[1] + ' ' + f
print(cmd)
a = os.system(cmd)
sys.exit(1)
'''horizontally concatenate two csv files'''
import os
import sys
from misc import read_csv
from misc import exist
from misc import err
args = sys.argv
if len(args) < 3:
err('python3 csv_hcat.py [csv file 1] [csv file 2]')
'''read the csv and locate the spectra'''
lines1 = [x.strip() for x in open(args[1]).read().strip().split('\n')]
lines2 = [x.strip() for x in open(args[2]).read().strip().split('\n')]
if len(lines1) != len(lines2):
print(len(lines1), len(lines2))
err('different number of lines per input file')
ofn = args[1] + '_hcat.csv'
lines = [lines1[i] + ',' + lines2[i] for i in range(len(lines1))]
print('+w', ofn)
open(ofn, 'wb').write(('\n'.join(lines)).encode())
'''determine pixel location (lat lon) for point within a raster.
Uses map info from the raster, but not the raster data itself'''
import os
import sys
import utm
import json
import struct
import pyproj
from osgeo import ogr
from osgeo import osr
from osgeo import gdal
import matplotlib.pyplot as plt
from misc import args, err, run
if len(args) < 4:
err("python3 raster pixel location [input image name] [row] [col].. " +
"..plus additional [row] [col] pairs as needed")
# open image and get geotransform
img = args[1]
if not img: err('pls check input file')
src = gdal.Open(img)
GT = src.GetGeoTransform()
n_coords = int((len(args) - 2) / 2) # number of coordinate pairs provided
print('X_pixel,Y_line,X_geo,Y_geo,Lat,Lon')
from pyproj import Proj
d = os.popen('gdalsrsinfo -o proj4 ' + img).read().strip()
print(d)
# myProj = Proj(os.popen('gdalsrsinfo -o proj4 ' + img).read().strip())
w = d.split()
w = [x.strip('+').split('=') for x in w]
srs_info = {x[0]: x[1] if len(x) > 1 else '' for x in w}
'''20211128 averaging over a window, where the windowed data are from:
raster_extract_spectra.py'''
import os
import sys
import csv
from misc import read_csv
from misc import exist
from misc import err
args = sys.argv
in_f = args[1]
if not exist(in_f):
err('could not find input file: ' + in_f)
'''read the csv and locate the spectra'''
fields, data = read_csv(in_f)
fields = [x.strip().replace(',', '_') for x in fields] # forbid comma in header
nf = len(fields) # number of fields
f_i = {fields[i]:i for i in range(nf)}
'''insist on fields xoff and yoff'''
if (not 'xoff' in fields) or (not 'yoff' in fields):
err("missing req'd fields: xoff, yoff")
spec_fi, nonspec_fi = [], [] # list col-idx for all spectral data columns
for i in range(nf):
if fields[i][-2:] == 'nm':
spec_fi += [i]
else: # list non-spec fields except: offset-index coding analysis-window pos'n
if fields[i] not in ['xoff', 'yoff', 'row', 'lin']:
nonspec_fi += [i]
from misc import err
from misc import parfor
from misc import read_csv
from misc import exist
from misc import hdr_fn
from misc import read_binary
from misc import write_hdr
from misc import write_binary
args = sys.argv
from multiprocessing import Lock
lock = Lock()
n_processed = 0
if len(args) < 5:
err('python3 csv_spectra_distance_simple.py [csv spectra file (one spectrum)] ' +
' [field to select from] [field value to select]' +
' [raster file]')
csv_fn, dfn = args[1], args[4]
select_field = args[2]
select_value = args[3]
'''read the csv and locate the spectra'''
fields, csv_data = read_csv(csv_fn)
nf = len(fields) # number of fields
f_i = {fields[i]:i for i in range(nf)}
spec_fi = []
for i in range(nf):
if fields[i][-2:] == 'nm':
spec_fi += [i]
import os
import sys
from misc import pd
from misc import err
from misc import run
from misc import args
from misc import hdr_fn
from misc import read_hdr
if len(args) < 2:
err('python3 raster_video.py [input ENVI format file]')
samples, lines, bands = [int(x) for x in read_hdr(hdr_fn(args[1]))]
if bands % 12 != 0:
err('expected bands to be multiple of 12')
frames, bi = int(bands / 12), [4, 3, 2]
for i in range(frames):
cmd = ' '.join([pd + 'raster_plot.py', args[1]] + [str(j)
for j in bi] + ['1'])
bi = [j + 12 for j in bi]
run(cmd)
from misc import load_fields, assert_exists, err, run
if not os.path.exists("dd") or not os.path.isdir("dd"):
# find, extract and clean data dictionary files
run("dd_list")
run("dd_clean")
# data dictionary file
dd_reg = "dd/2019-01-09_data_dictionary_consolidation-file-january-1-1986-onwards.xlsx_registry.C.csv2"
dd_schlstud, dd_studcrd = "dd/2019-01-24_data_dictionary_education.xlsx_schlstud.csv2", "dd/2019-01-24_data_dictionary_education.xlsx_studcrd.csv2"
for f in [dd_reg, dd_schlstud, dd_studcrd]:
assert_exists(f)
if len(sys.argv) < 3:
print "Important: delete the *.p files before running, if you change the cohort file!!!"
err("Usage: grad_gr8_cohort [cohort_file.csv] [number of years=6?]")
cohort_file = sys.argv[1] #'youth_cohort.csv' # delete *.p files and other intermediary files if cohort changes
number_of_years = None
try:
number_of_years = int(sys.argv[2])
except:
err("failed to parse second parameter, year interval e.g. 4, 6, or 8")
schlstud_file = 'idomed1991-2017.ft_schlstud.A.dat_dd_sliceapply.csv'
studcrd_file = 'idomed1991-2017.ft_studcrd.A.dat_dd_sliceapply.csv'
registry_file = 'registry1991-2016_dd_sliceapply.csv_cat.csv'
files = [cohort_file, schlstud_file, studcrd_file, registry_file]
# prepare school student file, if not yet
if not os.path.exists(schlstud_file):