def _check_skat(self):
skat_dir = None
# Check for R/SKAT installation
try:
runCommand(["R", "-e", "library('SKAT')"])
except:
skat_dir = os.path.join(env.local_resource, 'Rlib')
try:
runCommand(["R", "-e", "library('SKAT', lib.loc='{0}')".format(skat_dir)])
except:
self._install_skat(skat_dir)
return skat_dir
def _install_skat(self, skat_dir):
try:
skat_url = 'SKAT_{0}.tar.gz'.format(env.skat_version)
env.logger.info('Downloading SKAT (V{0}) ...'.format(env.skat_version))
skat_tar = downloadFile(skat_url, env.temp_dir)
mkdir_p(skat_dir)
env.logger.info('Installing SKAT (V{0}) to {1} ...'.format(env.skat_version, skat_dir))
runCommand(['R', '-e', 'install.packages("{0}", repos = NULL, type = "source", lib="{1}")'.\
format(skat_tar, skat_dir)])
runCommand(["R", "-e", "library('SKAT', lib.loc='{0}')".format(skat_dir)])
except Exception as e:
raise ValueError("Cannot auto-install / load R library SKAT: {0}".format(e))
def _check_skat(self):
skat_dir = None
# Check for R/SKAT installation
try:
runCommand(["R", "-e", "library('SKAT')"])
except:
skat_dir = os.path.join(env.local_resource, 'Rlib')
try:
runCommand([
"R", "-e",
"library('SKAT', lib.loc='{0}')".format(skat_dir)
])
except:
self._install_skat(skat_dir)
return skat_dir
def _install_skat(self, skat_dir):
try:
skat_url = 'SKAT_{0}.tar.gz'.format(env.skat_version)
env.logger.info('Downloading SKAT (V{0}) ...'.format(
env.skat_version))
skat_tar = downloadFile(skat_url, env.temp_dir)
mkdir_p(skat_dir)
env.logger.info('Installing SKAT (V{0}) to {1} ...'.format(
env.skat_version, skat_dir))
runCommand(['R', '-e', 'install.packages("{0}", repos = NULL, type = "source", lib="{1}")'.\
format(skat_tar, skat_dir)])
runCommand(
["R", "-e", "library('SKAT', lib.loc='{0}')".format(skat_dir)])
except Exception as e:
raise ValueError(
"Cannot auto-install / load R library SKAT: {0}".format(e))
def lineplot(self):
def _cvt(x):
if type(x) is str:
x = eval(x)
out = []
for item in x:
out.append('"{}"={}'.format(item, x[item]))
return ','.join(out)
#
r = self.__prepare_input('line')
# finalize input
r += '''spowerlineplot(dat, "{0}", "{1}", "{2}", "{3}", range = {4},
cvalues = {5}, svalues = {6}, lvalues = {7}, xaxis.fontsize = {8},
yaxis.fontsize = {9}, xlab.fontsize = {10}, ylab.fontsize = {11},
width = {12}, height = {13}, legend = {14})'''.\
format(self.opts['plot_title'], self.opts['xlab'], self.opts['ylab'], self.pdf,
'c({})'.format(self.opts['y_axis_range']) if self.opts['y_axis_range'] else 'NULL',
'c({})'.format(_cvt(self.opts['object_color'])) if self.opts['object_color'] else 'NULL',
'c({})'.format(_cvt(self.opts['object_shape'])) if self.opts['object_shape'] else 'NULL',
'c({})'.format(_cvt(self.opts['object_line'])) if self.opts['object_line'] else 'NULL',
'{}'.format(self.opts['x_axis_fontsize']) if self.opts['x_axis_fontsize'] else 'NULL',
'{}'.format(self.opts['y_axis_fontsize']) if self.opts['y_axis_fontsize'] else 'NULL',
'{}'.format(self.opts['xlab_fontsize']) if self.opts['xlab_fontsize'] else 'NULL',
'{}'.format(self.opts['ylab_fontsize']) if self.opts['ylab_fontsize'] else 'NULL',
self.opts['plot_width'] if self.opts['plot_width'] else 8,
self.opts['plot_height'] if self.opts['plot_height'] else 8,
'F' if self.opts['remove_legend'] else 'T')
# make plot
sys.stderr.write("Generating graph(s) ...\n")
out = runCommand("R --slave --no-save --no-restore", r)
sys.stderr.write("Complete!\n")
return r
def lineplot(self):
def _cvt(x):
if type(x) is str:
x = eval(x)
out = []
for item in x:
out.append('"{}"={}'.format(item, x[item]))
return ','.join(out)
#
r = self.__prepare_input('line')
# finalize input
r += '''spowerlineplot(dat, "{0}", "{1}", "{2}", "{3}", range = {4},
cvalues = {5}, svalues = {6}, lvalues = {7}, xaxis.fontsize = {8},
yaxis.fontsize = {9}, xlab.fontsize = {10}, ylab.fontsize = {11},
width = {12}, height = {13}, legend = {14})'''.\
format(self.opts['plot_title'], self.opts['xlab'], self.opts['ylab'], self.pdf,
'c({})'.format(self.opts['y_axis_range']) if self.opts['y_axis_range'] else 'NULL',
'c({})'.format(_cvt(self.opts['object_color'])) if self.opts['object_color'] else 'NULL',
'c({})'.format(_cvt(self.opts['object_shape'])) if self.opts['object_shape'] else 'NULL',
'c({})'.format(_cvt(self.opts['object_line'])) if self.opts['object_line'] else 'NULL',
'{}'.format(self.opts['x_axis_fontsize']) if self.opts['x_axis_fontsize'] else 'NULL',
'{}'.format(self.opts['y_axis_fontsize']) if self.opts['y_axis_fontsize'] else 'NULL',
'{}'.format(self.opts['xlab_fontsize']) if self.opts['xlab_fontsize'] else 'NULL',
'{}'.format(self.opts['ylab_fontsize']) if self.opts['ylab_fontsize'] else 'NULL',
self.opts['plot_width'] if self.opts['plot_width'] else 8,
self.opts['plot_height'] if self.opts['plot_height'] else 8,
'F' if self.opts['remove_legend'] else 'T')
# make plot
sys.stderr.write("Generating graph(s) ...\n")
out = runCommand("R --slave --no-save --no-restore", r)
sys.stderr.write("Complete!\n")
return r
def barplot(self):
def _cvt(x):
if type(x) is str:
x = eval(x)
out = []
for item in x:
out.append('"{}"={}'.format(item, x[item]))
return ",".join(out)
#
r = self.__prepare_input("bar")
r += """spowerbarplot(dat, "{0}", "{1}", "{2}", "{3}", range = {4},
xaxis.fontsize = {5}, yaxis.fontsize = {6}, xlab.fontsize = {7}, ylab.fontsize = {8},
width = {9}, height = {10}, legend = {11}, cvalues = {12})""".format(
self.opts["plot_title"],
self.opts["xlab"],
self.opts["ylab"],
self.pdf,
"c({})".format(self.opts["y_axis_range"]) if self.opts["y_axis_range"] else "NULL",
"{}".format(self.opts["x_axis_fontsize"]) if self.opts["x_axis_fontsize"] else "NULL",
"{}".format(self.opts["y_axis_fontsize"]) if self.opts["y_axis_fontsize"] else "NULL",
"{}".format(self.opts["xlab_fontsize"]) if self.opts["xlab_fontsize"] else "NULL",
"{}".format(self.opts["ylab_fontsize"]) if self.opts["ylab_fontsize"] else "NULL",
self.opts["plot_width"] if self.opts["plot_width"] else 8,
self.opts["plot_height"] if self.opts["plot_height"] else 8,
"F" if self.opts["remove_legend"] else "T",
"c({})".format(_cvt(self.opts["object_color"])) if self.opts["object_color"] else "NULL",
)
# make plot
sys.stderr.write("Generating graph(s) ...\n")
out = runCommand("R --slave --no-save --no-restore", r)
sys.stderr.write("Complete!\n")
return r
def show_gdat():
if '-v2' in unknown_args:
try:
print(runCommand('ptdump {}'.format(fn)))
except:
raise ValueError('Cannot display summary information. Make sure "{}" exists and "ptdump" is installed'.format(fn))
else:
try:
gf = GFile(fn)
names = gf.getnames()
gf.close()
except:
names = []
for idx, name in enumerate(names):
print('/%s' % name)
if idx >= 50 and '-v0' in unknown_args:
remaining = len(names) - 50
if remaining:
printinfo('%s more items not displayed. Use "-v1/-v2" switch to see more.' % remaining)
break
def execute(args, unknown_args):
c = CommandGenerator(args.file, args.sliding, args.fixed)
if not args.plot:
# run commands
cmds = ['{} {} {}'.format("echo spower", item, ' '.join(unknown_args)) for item in c.generate()]
for idx, item in enumerate(cmds):
out = runCommand(item)
if args.dry_run:
print out
else:
sys.stderr.write("\033[1;40;32m{}\033[0m\n".format("Running command {}/{} ...".format(idx + 1, len(cmds))))
os.system(out)
else:
# generate plot
p = Plotter(c.plot())
r = p.lineplot()
# save script
if '--debug' in unknown_args:
with open(os.path.join(env.cache_dir, os.path.split(args.file)[-1] + '.R'), 'w') as f:
f.write(r)
def execute(args, unknown_args):
c = CommandGenerator(args.file, args.sliding, args.fixed)
if not args.plot:
# run commands
cmds = ['{} {} {}'.format("echo spower", item, ' '.join(unknown_args)) for item in c.generate()]
for idx, item in enumerate(cmds):
out = runCommand(item)
if args.dry_run:
print out
else:
sys.stderr.write("\033[1;40;32m{}\033[0m\n".format("Running command {}/{} ...".format(idx + 1, len(cmds))))
os.system(out)
else:
# generate plot
p = Plotter(c.plot())
r = p.lineplot()
# save script
if '--debug' in unknown_args:
with open(os.path.join(env.cache_dir, os.path.split(args.file)[-1] + '.R'), 'w') as f:
f.write(r)
def show_gdat():
if '-v2' in unknown_args:
try:
print(runCommand('ptdump {}'.format(fn)))
except:
raise ValueError(
'Cannot display summary information. Make sure "{}" exists and "ptdump" is installed'
.format(fn))
else:
try:
gf = GFile(fn)
names = gf.getnames()
gf.close()
except:
names = []
for idx, name in enumerate(names):
print('/%s' % name)
if idx >= 50 and '-v0' in unknown_args:
remaining = len(names) - 50
if remaining:
printinfo(
'%s more items not displayed. Use "-v1/-v2" switch to see more.'
% remaining)
break
def timed_command(cmd, instream=None, msg='', upon_succ=None):
return runCommand(cmd, instream, msg, upon_succ)
def showFields(fn, border, unknown_args):
def set_style(pt):
if border == 'less':
pt.set_style(MSWORD_FRIENDLY)
if border == 'no':
pt.set_style(PLAIN_COLUMNS)
#
if fn.endswith('.csv'):
pt = from_csv(openFile(fn), delimiter = ',')
set_style(pt)
header = [x for x in pt.field_names if not x.startswith('_')]
if len(unknown_args) == 0:
print('\n'.join(header))
return
fields = [re.compile(item.replace('*', '(.*?)')) if '*' in item else item for item in unknown_args]
output = []
for item in fields:
if type(item) is str:
item = [x for x in header if x == item]
else:
item = [x for x in header if re.match(item, x)]
output.extend(item)
print pt.get_string(fields=output)
elif fn.endswith('.SEQPowerDB'):
if not os.path.isfile(fn):
raise OSError('Cannot find {}'.format(fn))
rs = ResultManager(fn)
pt = PrettyTable()
set_style(pt)
# show tables
if len(unknown_args) == 0:
pt.add_column('TABLES', rs.get_tables())
print pt
return
table = unknown_args[0]
if table not in rs.get_tables():
raise ValueError("Cannot find table '{}'".format(table))
if '--debug' in unknown_args:
debug = True
unknown_args.pop(unknown_args.index('--debug'))
else:
debug = False
if '--condition' in unknown_args:
fields = unknown_args[1:unknown_args.index('--condition')]
condition = ' '.join(unknown_args[(unknown_args.index('--condition') + 1):])
else:
fields = unknown_args[1:]
condition = None
# show fields
header = sorted(rs.get_fields(table),
key = lambda x: x.replace("_", "|").replace('method', 'AAA').replace('power', 'AAB'))
if len(fields) == 0:
pt.add_column(table,header)
pt.align[table] = "l"
print pt
else:
names = [x for x in fields if x in header]
select_query = "SELECT {} from {} {}".format(','.join(names),
table, condition if condition else '')
if debug:
sys.stderr.write(select_query + '\n')
pt.field_names = names
for item in rs.cur.execute(select_query).fetchall():
pt.add_row(item)
print pt
elif fn.split('.')[-1] in ['gdat', 'h5', 'hdf5']:
def show_gdat():
if '-v2' in unknown_args:
try:
print(runCommand('ptdump {}'.format(fn)))
except:
raise ValueError('Cannot display summary information. Make sure "{}" exists and "ptdump" is installed'.format(fn))
else:
try:
gf = GFile(fn)
names = gf.getnames()
gf.close()
except:
names = []
for idx, name in enumerate(names):
print('/%s' % name)
if idx >= 50 and '-v0' in unknown_args:
remaining = len(names) - 50
if remaining:
printinfo('%s more items not displayed. Use "-v1/-v2" switch to see more.' % remaining)
break
#
if '--from' in unknown_args:
for item in unknown_args[(unknown_args.index('--from') + 1):]:
prefix, surfix = os.path.splitext(os.path.basename(item))
if surfix in ['.gdat', '.h5', '.hdf5']:
runCommand('h5copy -v -i {0} -o {1} -s "/{2}" -d "/{2}"'.format(item, fn, re.sub(r'[^a-zA-Z0-9_]', '_', prefix)), accepted_rc = [0,1])
if not '-v0' in unknown_args:
printinfo('File {} processed!'.format(item))
if '--to' in unknown_args:
target = unknown_args[(unknown_args.index('--to') + 1):]
target = target[0] if target else os.path.splitext(fn)[0]
runCommand('mkdir -p {}'.format(target))
gf = GFile(fn)
names = gf.getnames()
gf.close()
for name in gf.getnames():
dat = GData(fn, name)
if not '-v0' in unknown_args:
printinfo('Saving files {}'.format(os.path.join(target, '{}.*.txt'.format(name))))
dat.decompress()
for key in dat:
np.savetxt(os.path.join(target, '{}.{}.txt'.format(name, key)), dat[key], fmt = '%s', delimiter = '\t')
show_gdat()
else:
raise ValueError('Unsupported file type {}'.format(fn))
return
def timed_command(cmd, instream = None, msg = '', upon_succ=None):
return runCommand(cmd, instream, msg, upon_succ)
def showFields(fn, border, unknown_args):
def set_style(pt):
if border == 'less':
pt.set_style(MSWORD_FRIENDLY)
if border == 'no':
pt.set_style(PLAIN_COLUMNS)
#
if fn.endswith('.csv'):
pt = from_csv(openFile(fn), delimiter=',')
set_style(pt)
header = [x for x in pt.field_names if not x.startswith('_')]
if len(unknown_args) == 0:
print('\n'.join(header))
return
fields = [
re.compile(item.replace('*', '(.*?)')) if '*' in item else item
for item in unknown_args
]
output = []
for item in fields:
if type(item) is str:
item = [x for x in header if x == item]
else:
item = [x for x in header if re.match(item, x)]
output.extend(item)
print pt.get_string(fields=output)
elif fn.endswith('.SEQPowerDB'):
if not os.path.isfile(fn):
raise OSError('Cannot find {}'.format(fn))
rs = ResultManager(fn)
pt = PrettyTable()
set_style(pt)
# show tables
if len(unknown_args) == 0:
pt.add_column('TABLES', rs.get_tables())
print pt
return
table = unknown_args[0]
if table not in rs.get_tables():
raise ValueError("Cannot find table '{}'".format(table))
if '--debug' in unknown_args:
debug = True
unknown_args.pop(unknown_args.index('--debug'))
else:
debug = False
if '--condition' in unknown_args:
fields = unknown_args[1:unknown_args.index('--condition')]
condition = ' '.join(
unknown_args[(unknown_args.index('--condition') + 1):])
else:
fields = unknown_args[1:]
condition = None
# show fields
header = sorted(rs.get_fields(table),
key=lambda x: x.replace("_", "|").replace(
'method', 'AAA').replace('power', 'AAB'))
if len(fields) == 0:
pt.add_column(table, header)
pt.align[table] = "l"
print pt
else:
names = [x for x in fields if x in header]
select_query = "SELECT {} from {} {}".format(
','.join(names), table, condition if condition else '')
if debug:
sys.stderr.write(select_query + '\n')
pt.field_names = names
for item in rs.cur.execute(select_query).fetchall():
pt.add_row(item)
print pt
elif fn.split('.')[-1] in ['gdat', 'h5', 'hdf5']:
def show_gdat():
if '-v2' in unknown_args:
try:
print(runCommand('ptdump {}'.format(fn)))
except:
raise ValueError(
'Cannot display summary information. Make sure "{}" exists and "ptdump" is installed'
.format(fn))
else:
try:
gf = GFile(fn)
names = gf.getnames()
gf.close()
except:
names = []
for idx, name in enumerate(names):
print('/%s' % name)
if idx >= 50 and '-v0' in unknown_args:
remaining = len(names) - 50
if remaining:
printinfo(
'%s more items not displayed. Use "-v1/-v2" switch to see more.'
% remaining)
break
#
if '--from' in unknown_args:
for item in unknown_args[(unknown_args.index('--from') + 1):]:
prefix, surfix = os.path.splitext(os.path.basename(item))
if surfix in ['.gdat', '.h5', '.hdf5']:
runCommand(
'h5copy -v -i {0} -o {1} -s "/{2}" -d "/{2}"'.format(
item, fn, re.sub(r'[^a-zA-Z0-9_]', '_', prefix)),
accepted_rc=[0, 1])
if not '-v0' in unknown_args:
printinfo('File {} processed!'.format(item))
if '--to' in unknown_args:
target = unknown_args[(unknown_args.index('--to') + 1):]
target = target[0] if target else os.path.splitext(fn)[0]
runCommand('mkdir -p {}'.format(target))
gf = GFile(fn)
names = gf.getnames()
gf.close()
for name in gf.getnames():
dat = GData(fn, name)
if not '-v0' in unknown_args:
printinfo('Saving files {}'.format(
os.path.join(target, '{}.*.txt'.format(name))))
dat.decompress()
for key in dat:
np.savetxt(os.path.join(target,
'{}.{}.txt'.format(name, key)),
dat[key],
fmt='%s',
delimiter='\t')
show_gdat()
else:
raise ValueError('Unsupported file type {}'.format(fn))
return
def showFields(fn, border, unknown_args):
def set_style(pt):
if border == "less":
pt.set_style(MSWORD_FRIENDLY)
if border == "no":
pt.set_style(PLAIN_COLUMNS)
#
if fn.endswith(".csv"):
pt = from_csv(openFile(fn), delimiter=",")
set_style(pt)
header = [x for x in pt.field_names if not x.startswith("_")]
if len(unknown_args) == 0:
print ("\n".join(header))
return
fields = [re.compile(item.replace("*", "(.*?)")) if "*" in item else item for item in unknown_args]
output = []
for item in fields:
if type(item) is str:
item = [x for x in header if x == item]
else:
item = [x for x in header if re.match(item, x)]
output.extend(item)
print pt.get_string(fields=output)
elif fn.endswith(".SEQPowerDB"):
if not os.path.isfile(fn):
raise OSError("Cannot find {}".format(fn))
rs = ResultManager(fn)
pt = PrettyTable()
set_style(pt)
# show tables
if len(unknown_args) == 0:
pt.add_column("TABLES", rs.get_tables())
print pt
return
table = unknown_args[0]
if table not in rs.get_tables():
raise ValueError("Cannot find table '{}'".format(table))
if "--debug" in unknown_args:
debug = True
unknown_args.pop(unknown_args.index("--debug"))
else:
debug = False
if "--condition" in unknown_args:
fields = unknown_args[1 : unknown_args.index("--condition")]
condition = " ".join(unknown_args[(unknown_args.index("--condition") + 1) :])
else:
fields = unknown_args[1:]
condition = None
# show fields
header = sorted(
rs.get_fields(table), key=lambda x: x.replace("_", "|").replace("method", "AAA").replace("power", "AAB")
)
if len(fields) == 0:
pt.add_column(table, header)
pt.align[table] = "l"
print pt
else:
names = [x for x in fields if x in header]
select_query = "SELECT {} from {} {}".format(",".join(names), table, condition if condition else "")
if debug:
sys.stderr.write(select_query + "\n")
pt.field_names = names
for item in rs.cur.execute(select_query).fetchall():
pt.add_row(item)
print pt
elif fn.split(".")[-1] in ["gdat", "h5", "hdf5"]:
def show_gdat():
if "-v2" in unknown_args:
try:
print (runCommand("ptdump {}".format(fn)))
except:
raise ValueError(
'Cannot display summary information. Make sure "{}" exists and "ptdump" is installed'.format(fn)
)
else:
try:
gf = GFile(fn)
names = gf.getnames()
gf.close()
except:
names = []
for idx, name in enumerate(names):
print ("/%s" % name)
if idx >= 50 and "-v0" in unknown_args:
remaining = len(names) - 50
if remaining:
printinfo('%s more items not displayed. Use "-v1/-v2" switch to see more.' % remaining)
break
#
if "--from" in unknown_args:
for item in unknown_args[(unknown_args.index("--from") + 1) :]:
prefix, surfix = os.path.splitext(os.path.basename(item))
if surfix in [".gdat", ".h5", ".hdf5"]:
runCommand(
'h5copy -v -i {0} -o {1} -s "/{2}" -d "/{2}"'.format(
item, fn, re.sub(r"[^a-zA-Z0-9_]", "_", prefix)
),
accepted_rc=[0, 1],
)
if not "-v0" in unknown_args:
printinfo("File {} processed!".format(item))
if "--to" in unknown_args:
target = unknown_args[(unknown_args.index("--to") + 1) :]
target = target[0] if target else os.path.splitext(fn)[0]
runCommand("mkdir -p {}".format(target))
gf = GFile(fn)
names = gf.getnames()
gf.close()
for name in gf.getnames():
dat = GData(fn, name)
if not "-v0" in unknown_args:
printinfo("Saving files {}".format(os.path.join(target, "{}.*.txt".format(name))))
dat.decompress()
for key in dat:
np.savetxt(os.path.join(target, "{}.{}.txt".format(name, key)), dat[key], fmt="%s", delimiter="\t")
show_gdat()
else:
raise ValueError("Unsupported file type {}".format(fn))
return