def extract_data_sub(subfolder, transform_y = lambda y: np.log10(y)):
base = osio.cwd()
osio.cd(subfolder)
X = np.load('xnkl.array.npy')
y = float(open('jeff2_h.txt', 'r').readline())
y = transform_y(y)
osio.cd(base)
return X, y
def write_gdma_infile(fchkfile):
assert fchkfile[-5:] == '.fchk'
tag = fchkfile[:-5]
osio << "File 'gdma.in' in" << osio.cwd() << ":" << fchkfile << endl
ofs = open('gdma.in', 'w')
ofs.write('''\
Title "{tag:s}"
File {fchk:s}
Multipoles
Limit 2
Limit 1 H
Radius H 0.35
Punch {tag:s}.mps
Start
Finish
'''.format(fchk=fchkfile, tag=tag))
ofs.close()
def write_gdma_infile(fchkfile):
assert fchkfile[-5:] == ".fchk"
tag = fchkfile[:-5]
osio << "File 'gdma.in' in" << osio.cwd() << ":" << fchkfile << endl
ofs = open("gdma.in", "w")
ofs.write(
"""\
Title "{tag:s}"
File {fchk:s}
Multipoles
Limit 2
Limit 1 H
Radius H 0.35
Punch {tag:s}.mps
Start
Finish
""".format(
fchk=fchkfile, tag=tag
)
)
ofs.close()
def extract_data(folder, N_train, N_test, regex):
def extract_data_sub(subfolder, transform_y = lambda y: np.log10(y)):
base = osio.cwd()
osio.cd(subfolder)
X = np.load('xnkl.array.npy')
y = float(open('jeff2_h.txt', 'r').readline())
y = transform_y(y)
osio.cd(base)
return X, y
def sort_subfolders(folders):
return sorted(folders, key = lambda f: int(f.split('_')[1])*len(folders)+int(f.split('_')[2]))
base = osio.cwd()
osio << "Base =" << base << endl
osio.cd(folder)
# GET SUBFOLDERS
subfolders = sort_subfolders(get_dirs('./', regex))
# LIMIT RANGE
N = N_train + N_test
assert N < len(subfolders)
subfolders = subfolders[:N]
# FIND DESCRIPTOR DIMENSIONS
dim_x = extract_data_sub(subfolders[0])[0].shape[0]
osio << "Descriptor dimension =" << dim_x << endl
# EXTRACT FEATURES & TARGETS
X_list = np.zeros((N, dim_x))
y_list = np.zeros((N))
idx = -1
for subfolder in subfolders:
idx += 1
osio << osio.back << subfolder << flush
X, y = extract_data_sub(subfolder)
X_list[idx] = X
y_list[idx] = y
osio << endl
# SPLIT ONTO TRAINING & TEST SET
X_train = np.zeros((N_train, dim_x))
y_train = np.zeros((N_train))
X_test = np.zeros((N_test, dim_x))
y_test = np.zeros((N_test))
X_train[0] = X_list[0]
y_train[0] = y_list[0]
count_train = 1
count_test = 0
for idx in range(1,N):
if float(count_train)/(count_train+count_test) < float(N_train)/(N):
X_train[count_train] = X_list[idx]
y_train[count_train] = y_list[idx]
count_train += 1
else:
X_test[count_test] = X_list[idx]
y_test[count_test] = y_list[idx]
count_test += 1
assert count_train == N_train
assert count_test == N_test
osio.cd(base)
return X_train, y_train, X_test, y_test
ifs = open(f, 'r')
for ln in ifs.readlines():
if 'Timing' in ln:
#print ln[:-1]
time_lns.append(ln)
print "=>", len(time_lns)
ifs.close()
"""
if 'ctp.log' in files:
os.system('cat ctp.log | grep started')
return
base = osio.cwd()
# EXECUTE QSTAT & GENERATE JOBS
qstat = cmds.getoutput('qstat')
qstat = qstat.split('\n')
# Pop header
for i in range(2): qstat.pop(0)
# Load job data
jobs = []
for q in qstat:
job = Job(q)
jobs.append(job)
# CHECKS AND BALANCES
for job in jobs:
#do = Do()
time_lns = []
ifs = open(f, 'r')
for ln in ifs.readlines():
if 'Timing' in ln:
#print ln[:-1]
time_lns.append(ln)
print "=>", len(time_lns)
ifs.close()
"""
if 'ctp.log' in files:
os.system('cat ctp.log | grep started')
return
base = osio.cwd()
# EXECUTE QSTAT & GENERATE JOBS
qstat = cmds.getoutput('qstat')
qstat = qstat.split('\n')
# Pop header
for i in range(2):
qstat.pop(0)
# Load job data
jobs = []
for q in qstat:
job = Job(q)
jobs.append(job)
# CHECKS AND BALANCES
for job in jobs:
par.add_argument('--cln', dest='cln', action='store_const', const=1, default=0)
par.add_argument('--loc', dest='loc', action='store_const', const=1, default=0)
par.add_argument('--sub', dest='sub', action='store_const', const=1, default=0)
par.add_argument('--chk', dest='chk', action='store_const', const=1, default=0)
opts = par.parse_args()
z_xx = get_dirs('./', '^\d*K_confout')
#z_xx = ['COR_DPBIC']
z_xx.sort()
queue = 'PE_16'
n_procs = 16
n_threads = 16
workground = 'EWDBGPOL'
root = osio.cwd()
pres = ['Q' + z[4:] for z in z_xx]
for folder, pre in zip(z_xx, pres):
print "=" * 40, folder, "=" * 40
os.chdir(folder)
# CLEAN IF APPLICABLE
if opts.cln:
os.system('rm -rf %s' % workground)
# SUPPLY FROM SOURCE
if opts.gen:
os.system('mkdir -p %s' % workground)
par.add_argument('--cln', dest='cln', action='store_const', const=1, default=0)
par.add_argument('--loc', dest='loc', action='store_const', const=1, default=0)
par.add_argument('--sub', dest='sub', action='store_const', const=1, default=0)
par.add_argument('--chk', dest='chk', action='store_const', const=1, default=0)
opts = par.parse_args()
z_xx = get_dirs('./', '^\d*K_confout')
#z_xx = ['COR_DPBIC']
z_xx.sort()
queue = 'PE_16'
n_procs = 16
n_threads = 16
workground = 'EWDBGPOL'
root = osio.cwd()
pres = [ 'Q'+z[4:] for z in z_xx ]
for folder,pre in zip(z_xx, pres):
print "="*40, folder, "="*40
os.chdir(folder)
# CLEAN IF APPLICABLE
if opts.cln:
os.system('rm -rf %s' % workground)
# SUPPLY FROM SOURCE
if opts.gen:
os.system('mkdir -p %s' % workground)