def start(self):
self.stop()
cmd = '%s -1 %s' % (misc.find_program_in_path('dhclient'),
self.interface_name)
self.dhclient = misc.run(cmd,
include_stderr=True,
return_fileobject=True)
def start(self, network):
self.stop()
profile = network.profile
if not 'encryption_type' in network.profile:
raise WicdError('Network profile does not specify encryption type')
encryption_type = network.profile['encryption_type']
template_class = self.template_manager.get_template(encryption_type)
if not template_class:
raise WicdError('Template type %s does not exist' % encryption_type)
values = {}
for requirement in template_class.require:
if requirement.name in profile:
values[requirement.name] = str(profile[requirement.name])
else:
raise WicdError('Missing required value %s' % requirement.name)
values['essid'] = network.essid
values['scan'] = 0
log('wpa_supplicant values', values)
wpa_conf_name = network.bssid.replace(':', '').lower()
wpa_conf_name += 'vpb'
self.create_configuration_file(wpa_conf_name, template_class, values)
pathname = os.path.join(wpath.networks, wpa_conf_name)
self.wpa_supplicant = misc.run([
misc.find_program_in_path('wpa_supplicant'), '-i',
self.interface_name, '-D', 'wext', '-c', pathname
], return_fileobject=True)
def get_ip(self):
""" Get the IP address of the interface.
Returns:
The IP address of the interface in dotted quad form.
"""
cmd = "%s %s" % (misc.find_program_in_path("ifconfig"), self.interface_name)
output = misc.run(cmd)
return misc.run_regex(InterfaceRegexPatterns.ip, output)
def get_rvm_hostname():
ret_code = misc.run("grep 'option dhcp-server-identifier' /var/lib/dhcp3/dhclient.eth0.leases | tail -1 | awk '{print $NF}' | tr '\;' ' ' | tr -d ' ' > /tmp/hostname")
if ret_code:
f = open('/tmp/hostname','r')
rvm_host = f.readline()
f.close()
return rvm_host
else:
log.error("Error retrieving host name")
return None
def _slow_is_up(self, ifconfig=None):
""" Determine if an interface is up using ifconfig. """
cmd = [misc.find_program_in_path("ifconfig"), self.interface_name]
output = misc.run(cmd)
lines = output.split('\n')
if len(lines) < 5:
return False
for line in lines[1:4]:
if line.strip().startswith('UP'):
return True
return False
def _do_scan(self):
self.up()
cmd = [misc.find_program_in_path("iwlist"), self.interface_name, "scan"]
results = misc.run(cmd)
networks = results.split(" Cell ")
access_points = []
for cell in networks:
if "ESSID:" in cell:
entry = self._parse_access_point(cell)
if entry is not None:
access_points.append(entry)
return access_points
def convert_wf():
print("Convert old WF ===>>> new WF....",end='')
program =str(yambo)
options =""
failure=run(program=program,options=options,logfile="conversion_wf.log")
program =str(ypp)
options ="-w c"
failure=run(program=program,options=options,logfile="conversion_wf.log")
if failure:
print("KO!")
exit(0)
else:
print("OK")
print("Rename FixSAVE,SAVE ===>>> SAVE, oldSAVE....",end='')
try:
oldSAVE=Path('SAVE')
oldSAVE.rename('oldSAVE')
FixSAVE=Path('FixSAVE/SAVE')
FixSAVE.rename('SAVE')
except:
print("KO!")
exit(1)
print("OK")
def read_queue():
sqs_conn = boto.connect_sqs()
request_queue = sqs_conn.create_queue(worker_queue_name)
message = request_queue.read(VISIBILITY_TIMEOUT)
if message is not None:
#handle_message
get_message = message.get_body()
msg = get_message.split('|')[1]
log.debug("Get Message: %s" % (msg))
if get_message.startswith("setup_hosts"):
print get_message
misc.run("echo '%s' > /etc/hosts" % (msg))
elif get_message.startswith("setup_cert"):
print get_message
misc.run("echo '%s' > ~/.ssh/authorized_keys" % (msg))
misc.run("chmod 700 ~/.ssh")
misc.run("chmod 644 ~/.ssh/authorized_keys")
request_queue.delete_message(message)
log.debug("Delete Message from SQS")
else:
log.debug("SQS Server - Slave Message 0 :)")
pass
def down(self):
''' Put the interface down. '''
cmd = [misc.find_program_in_path('ifconfig'),
self.interface_name, 'down']
misc.run(cmd)
flag_file = Path("INPUTS/"+test.name+".flags")
if flag_file.is_file():
flag_file=open(str(flag_file),"r")
flag=flag_file.read().strip()
flag_file.close()
previous_test_dir=Path(flag)
new_test_dir =Path(test.name)
Path(new_test_dir).mkdir(parents=True,exist_ok=True)
copy_all_files(previous_test_dir,new_test_dir)
options=" -J "+test.name
# ****** run yambo or ypp *****************
failure=run(program=program,options=options,inputfile=inputfile,logfile=test.name+".log")
if(failure):
print("KO!")
exit(1)
else:
print("OK")
if not args.skipcomp:
print("\n\n ********** COMPARE wiht references ***********\n")
reference_list=read_files_list(reference_dir,begin='o-')
test_total =0
test_ok =0
test_failed =0
test_nan =0
"tau": 1e-3,
"lr_actor": 1e-4,
"lr_critic": 1e-4,
"weight_decay": 0,
"update_every": 20,
"update_steps": 10,
"double": False,
"fc1": 400,
"fc2": 300,
"batchnorm": True,
"weight_init": "uniform",
"seed": 2,
"scores_window": 100,
"progress_every": 2,
"save_every": 60,
"n_episodes": int(1e5),
"max_steps": int(5000),
"verbose": True,
"max_time": 200 * 60,
"score_solved": 0.5,
"stop_on_solve": False,
"folder": "trained/test",
"overwrite": True,
"type": "MADDPG"
}
torch.autograd.set_detect_anomaly(True)
run(env, params)
env.close()
def filter_table_for_cohort(cohort_id_file, table_file):
select_file = table_file + "_select.csv"
if not os.path.exists(select_file):
run("csv_select " + cohort_id_file + " studyid " + table_file)
assert_exists(select_file)
return select_file
try:
v_2 = True if sys.argv[1] == "2" else False
if not v_2:
if sys.argv[1] != "1":
err("argument must be 1 or 2")
except:
pass
if len(sys.argv) < 3:
print "Important: delete the *.p files before running, if you change the cohort file!!!"
err("usage:\n\tgrad [version 1 or 2: start with 1] [cohort file.csv]\nhuman should enter 1 for first parameter."
)
if not os.path.exists("dd") or not os.path.isdir("dd"):
# find, extract and clean data dictionary files
run("dd_list")
run("dd_clean")
# data dictionary file for registry:
dd_reg = "dd/2018-09-27_data_dictionary_consolidation-file-january-1-1986-onwards.xlsx_registry.C.csv2"
assert_exists(dd_reg)
dd_schlstud = "dd/2019-01-24_data_dictionary_education.xlsx_schlstud.csv2"
assert_exists(dd_schlstud)
dd_studcrd = "dd/2019-01-24_data_dictionary_education.xlsx_studcrd.csv2"
assert_exists(dd_studcrd)
cohort_file = sys.argv[2]
#'youth_cohort.csv' # delete *.p files and other intermediary files if cohort changes
schlstud_file = 'idomed1991-2017.ft_schlstud.A.dat_dd_sliceapply.csv'
studcrd_file = 'idomed1991-2017.ft_studcrd.A.dat_dd_sliceapply.csv'
registry_file = 'registry1991-2016_dd_sliceapply.csv_cat.csv'
if len(args) < 2:
err('sentinel2_trace_active.py [input file, binary mask envi type 4] [optional arg: class index]'
)
class_select = None
if len(args) > 2:
class_select = int(args[2])
print(class_select)
fn = args[1]
if not exists(fn):
err('please check input file')
for i in [150]: # [10, 20, 60]: # 90
if not exists(fn + '_flood4.bin'):
run('ulimit -s 1000000')
run(cd + 'flood.exe ' + fn)
if not exists(fn + '_flood4.bin_link.bin'):
run(cd + 'class_link.exe ' + fn + '_flood4.bin ' + str(i)) # 40')
if not exists(fn + '_flood4.bin_link.bin_recode.bin'):
run(cd + 'class_recode.exe ' + fn + '_flood4.bin_link.bin 1')
if not exists(fn + '_flood4.bin_link.bin_recode.bin_wheel.bin'):
run(cd + 'class_wheel.exe ' + fn + '_flood4.bin_link.bin_recode.bin')
run('python3 ' + pd + 'raster_plot.py ' + fn +
'_flood4.bin_link.bin_recode.bin_wheel.bin 1 2 3 1 1')
print('class onehot..')
cmd = cd + 'class_onehot.exe ' + fn + '_flood4.bin_link.bin_recode.bin ' + str(
POINT_THRES)
print(cmd)
lines = None
if not exists('class_onehot.dat'):
def remove_cert():
misc.run('rm -rf /tmp/id_dsa*')
return 1
def build_cert():
misc.run('ssh-keygen -t dsa -N "" -f /tmp/id_dsa')
misc.run('cp /tmp/id_dsa* ~/.ssh/')
return 1
def build_hosts(newhost):
misc.run("echo '%s' >> /etc/hosts" % (newhost))
misc.run("cat /etc/hosts > /tmp/hosts")
return 1
def set_netmask(self, new_ip):
''' Sets the netmask of the current network interface. '''
cmd = '%s %s netmask %s' % (misc.find_program_in_path('ifconfig'),
self.interface_name, new_ip)
misc.run(cmd)
def set_gateway(self, new_ip):
''' Sets the netmask of the current network interface. '''
cmd = '%s add default gateway %s %s' % (misc.find_program_in_path('route'),
new_ip, self.interface_name)
misc.run(cmd)
def set_netmask(self, new_ip):
""" Sets the netmask of the current network interface. """
cmd = "%s %s netmask %s" % (misc.find_program_in_path("ifconfig"), self.interface_name, new_ip)
misc.run(cmd)
def up(self):
''' Put the interface up. '''
cmd = [misc.find_program_in_path('ifconfig'), self.interface_name, 'up']
misc.run(cmd)
def set_gateway(self, new_ip):
""" Sets the netmask of the current network interface. """
cmd = "%s add default gateway %s %s" % (misc.find_program_in_path("route"), new_ip, self.interface_name)
misc.run(cmd)
fn, s = args[1], args[2]
cd = pd + '..' + sep + 'cpp' + sep
hfn = hdr_fn(fn)
samples, lines, bands = read_hdr(hfn)
nrow, ncol = lines, samples
bi = []
bn = band_names(hfn)
for i in range(len(bn)):
if len(bn[i].split(s)) > 1:
bi.append(i)
print(' ', str(i + 1), ' ', bn[i])
if not exists(fn + '_001.hdr'):
run(cd + 'unstack.exe ' + fn)
for i in bi:
f = fn + '_' + str(i + 1).zfill(3) + '.bin'
print(f)
fni = [fn + '_' + str(i + 1).zfill(3) + '.bin' for i in bi]
bands = str(len(bi)) # number of bands to write
ofn = fn + '_' + s + '.bin'
ohn = fn + '_' + s + '.hdr'
run('cp ' + hfn + ' ' + ohn)
c = ['python3', pd + 'envi_header_modify.py', ohn, nrow, ncol, bands]
c += [('"' + bn[bi[i]] + '"') for i in range(len(bi))]
c = ' '.join(c)
run(c)
def up(self):
""" Put the interface up. """
cmd = [misc.find_program_in_path("ifconfig"), self.interface_name, "up"]
misc.run(cmd)
print('')
if not exists(outf):
a = os.system(cmd)
for r in rasters:
print(" ", r)
cmd = [
'python3', # envi_header_cat.py is almost like a reverse-polish notation. Have to put the "first thing" on the back..
pd + 'envi_header_cat.py',
rasters[1][:-4] + '.hdr',
rasters[0][:-4] + '.hdr',
ofhn
] # 'raster.hdr']
cmd = ' '.join(cmd)
run(cmd)
print('')
# envi_header_cat.py almost like a RPN. Put "first thing" on the back
if len(rasters) > 2:
for i in range(2, len(rasters)):
cmd = [
'python3',
pd + 'envi_header_cat.py',
rasters[i][:-4] + '.hdr',
ofhn, # 'raster.hdr',
ofhn
] # 'raster.hdr']
cmd = ' '.join(cmd)
run(cmd)
# run by opening C:/cygwin64/Cygwin.bat
# before running, type:
# cd /cygdrive/r/working/bin/
# source bash_profile
# then, type:
# cd /cygdrive/r/working/education/
# grad_gr8_cohort youth_cohort.csv 4
# youth_cohort.csv has studyid, dob (yyyy-mm)'''
import os, sys, math, pickle, datetime
from misc import load_fields, assert_exists, err, run
if not os.path.exists("dd") or not os.path.isdir("dd"):
# find, extract and clean data dictionary files
run("dd_list")
run("dd_clean")
# data dictionary file
dd_reg = "dd/2019-01-09_data_dictionary_consolidation-file-january-1-1986-onwards.xlsx_registry.C.csv2"
dd_schlstud, dd_studcrd = "dd/2019-01-24_data_dictionary_education.xlsx_schlstud.csv2", "dd/2019-01-24_data_dictionary_education.xlsx_studcrd.csv2"
for f in [dd_reg, dd_schlstud, dd_studcrd]:
assert_exists(f)
if len(sys.argv) < 3:
print "Important: delete the *.p files before running, if you change the cohort file!!!"
err("Usage: grad_gr8_cohort [cohort_file.csv] [number of years=6?]")
cohort_file = sys.argv[1] #'youth_cohort.csv' # delete *.p files and other intermediary files if cohort changes
number_of_years = None
try:
def down(self):
""" Put the interface down. """
cmd = [misc.find_program_in_path("ifconfig"), self.interface_name, "down"]
misc.run(cmd)
for i in ix:
dist_row(i)
fake_parfor(dist_row, [i for i in range(nrow)]) # REMOVE THIS
sys.exit(1) # REMOVE THIS
x = parfor(dist_row, [i for i in range(nrow)])
print("assembling")
for i in range(nrow):
print("assemble", i, x[i])
ix = range(i * ncol, (i + 1) * ncol)
out[ix] = x[i]
if DEBUG:
print(i, x[i])
numbers = []
w = csv_fn.split(os.path.sep)[-1][:-4].split('_')
for i in w:
try:
numbers.append(int(i))
except Exception:
pass
ofn = ('_'.join([dfn] +
[str(x) for x in numbers]) + '_' +
select_field.replace(' ','_') + '_eq_' +
select_value.replace(' ', '_') + '_distance.bin')
ohn = ofn[:-4] + '.hdr'
write_binary(out, ofn)
write_hdr(ohn, ncol, nrow, 1)
run('python3 ' + pd + 'envi_header_cleanup.py ' + ohn)
print("spec_avg", spec_avg)
