def install(self):
debug(self._label + "install files")
src_install = pjoin(Settings.CONTROL_PATH, self._package["package"] + ".install")
if not pexists(src_install):
warning("Missing: " + src_install)
return
try:
lineno = 0
with open(src_install) as fin:
for line in fin:
lineno += 1
src, dst = line.strip().split(" ", 1)
dst = dst.strip()
if dst.startswith("/"):
dst = "." + dst
dst = pjoin(self._env["QPKG_DEST_DATA"], dst)
if not pexists(dst):
makedirs(dst)
src_files = glob(src)
if not src_files:
raise FileSyntaxError(src_install, lineno, "`{}` not found".format(src))
for fn in glob(src):
try:
sp.check_call(["cp", "-a", fn, dst])
except sp.CalledProcessError as e:
warning("Error in copy files: {}".format(e))
return -1
except ValueError:
raise FileSyntaxError(src_install, lineno, line)
def links(self):
debug(self._label + 'create additional symlinks')
src_install = pjoin(Settings.CONTROL_PATH,
self._package['package'] + '.links')
if not pexists(src_install):
return
try:
lineno = 0
with open(src_install) as fin:
for line in fin:
lineno += 1
src, dst = line.strip().split(' ', 1)
dst = dst.strip()
if dst.startswith('/'):
dst = '.' + dst
dst = pjoin(self._env['QPKG_DEST_DATA'], dst)
if dst.endswith('/'):
if not pexists(dst):
makedirs(dst)
dst = pjoin(dst, pbasename(src))
else:
if not pexists(pdirname(dst)):
makedirs(pdirname(dst))
symlink(src, dst)
except ValueError:
raise FileSyntaxError(src_install, lineno, line)
def links(self):
debug(self._label + "create additional symlinks")
src_install = pjoin(Settings.CONTROL_PATH, self._package["package"] + ".links")
if not pexists(src_install):
return
try:
lineno = 0
with open(src_install) as fin:
for line in fin:
lineno += 1
src, dst = line.strip().split(" ", 1)
dst = dst.strip()
if dst.startswith("/"):
dst = "." + dst
dst = pjoin(self._env["QPKG_DEST_DATA"], dst)
if dst.endswith("/"):
if not pexists(dst):
makedirs(dst)
dst = pjoin(dst, pbasename(src))
else:
if not pexists(pdirname(dst)):
makedirs(pdirname(dst))
symlink(src, dst)
except ValueError:
raise FileSyntaxError(src_install, lineno, line)
def load_best_model_from_trained_dir(the_dir):
''' Follow the model architecture in main.py '''
if not pexists(pjoin(the_dir, 'MY_IS_FINISHED')):
with Timer('copying from v'):
cmd = 'rsync -avzL v:/h/kingsley/node/%s/ %s' % (the_dir, the_dir)
print(cmd)
os.system(cmd)
hparams = load_hparams(the_dir)
from . import arch
model, step_callbacks = getattr(arch, hparams['arch'] +
'Block').load_model_by_hparams(
hparams, ret_step_callback=True)
# Set the step!
if step_callbacks is not None and len(step_callbacks) > 0:
bstep = json.load(open(pjoin(the_dir,
'recorder.json')))['best_step_err']
for c in step_callbacks:
c(bstep)
best_ckpt = pjoin(the_dir, 'checkpoint_{}.pth'.format('best'))
if not pexists(best_ckpt):
print('NO BEST CHECKPT EXISTS in {}!'.format(best_ckpt))
return None
tmp = torch.load(best_ckpt, map_location='cpu')
model.load_state_dict(tmp['model'])
model.train(False)
return model
def install(self):
debug(self._label + 'install files')
src_install = pjoin(Settings.CONTROL_PATH,
self._package['package'] + '.install')
if not pexists(src_install):
warning('Missing: ' + src_install)
return
try:
lineno = 0
with open(src_install) as fin:
for line in fin:
lineno += 1
src, dst = line.strip().split(' ', 1)
dst = dst.strip()
if dst.startswith('/'):
dst = '.' + dst
dst = pjoin(self._env['QPKG_DEST_DATA'], dst)
if not pexists(dst):
makedirs(dst)
src_files = glob(src)
if not src_files:
raise FileSyntaxError(src_install, lineno,
'`{}` not found'.format(src))
for fn in glob(src):
try:
sp.check_call(['cp', '-a', fn, dst])
except sp.CalledProcessError as e:
warning('Error in copy files: {}'.format(e))
return -1
except ValueError:
raise FileSyntaxError(src_install, lineno, line)
def on_test_start(self, trainer, pl_module):
result_f = pjoin(pl_module.hparams.logdir, pl_module.hparams.name,
'results.tsv')
if not pexists(result_f):
raise Exception('WIERD!!!! No results.tsv found in the model directory.')
bpath = pjoin(pl_module.hparams.logdir, pl_module.hparams.name, 'best.ckpt')
if not pexists(bpath):
df = pd.read_csv(result_f, delimiter='\t')
func = {'min': np.argmin, 'max': np.argmax}[
trainer.checkpoint_callback.mode]
best_idx = int(func(df[trainer.checkpoint_callback.monitor].values))
best_record = df.iloc[best_idx].to_dict()
best_epoch = best_record['epoch']
best_filename = trainer.checkpoint_callback.format_checkpoint_name(
best_epoch, best_record)
os.symlink(best_filename, bpath)
assert pexists(bpath), 'Best model %s does not exist!' % bpath
best_pl_module = pl_module.load_from_checkpoint(bpath)
pl_module.model.load_state_dict(best_pl_module.model.state_dict())
pl_module.current_epoch = best_pl_module.current_epoch
pl_module.global_step = best_pl_module.global_step
print('Load best model from %s' % bpath)
def make_vis_pial_surface(in_dir, subject_id, out_dir, annot_file='aparc.annot'):
"""Generate screenshot for the pial surface in different views"""
out_vis_dir = pjoin(out_dir, cfg.annot_vis_dir_name)
makedirs(out_vis_dir, exist_ok=True)
hemis = ('lh', 'rh')
hemis_long = ('left', 'right')
vis_list = dict()
print('Processing {}'.format(subject_id))
for hemi, hemi_l in zip(hemis, hemis_long):
vis_list[hemi_l] = dict()
script_file, vis_files = make_tcl_script_vis_annot(subject_id, hemi_l, out_vis_dir, annot_file)
try:
# run the script only if all the visualizations were not generated before
all_vis_exist = all([pexists(vis_path) for vis_path in vis_files.values()])
if not all_vis_exist:
_, _ = run_tksurfer_script(in_dir, subject_id, hemi, script_file)
vis_list[hemi_l].update(vis_files)
except:
traceback.print_exc()
print('unable to generate tksurfer visualizations for {} hemi - skipping'.format(hemi))
# flattening it for easier use later on
out_vis_list = dict()
pref_order = [ ('right', 'lateral'), ('left', 'lateral'),
('right', 'medial'), ('left', 'medial'),
('right', 'transverse'), ('left', 'transverse')]
for hemi_l, view in pref_order:
if pexists(vis_list[hemi_l][view]):
out_vis_list[(hemi_l, view)] = vis_list[hemi_l][view]
return out_vis_list
def load(self):
"""
Loads the assignment (checks all paths are proper and
loads all assignment questions)
:raises AssertionError: invalid assignment
"""
# checking that assignment has correct paths
n = self.full_name
assert pexists(self.log_path), \
f'started assignment "{n}" with no log directory'
assert pexists(self.rubric_path), \
f'started assignment "{n}" with no rubric directory'
assert pexists(self.grade_path), \
f'started assignment "{n}" with no grade directory'
assert pexists(self.blocklist_path), \
f'started assignment "{n}" with no blocklist file'
assert pexists(self.files_path), \
f'started assignment "{n}" with no student code directory'
if not self.anonymous:
with locked_file(self.anon_path) as f:
data: Dict[str, int] = json.load(f)
self._login_to_id_map: Dict[str, int] = data
self._id_to_login_map: Dict[int, str] = {data[k]: k for k in data}
self._load_questions()
self.loaded = True
return self
def on_test_start(self, trainer, pl_module):
# Check if it already runs
dr_exp = ''
if pl_module.is_data_ratio_exp():
dr_exp += 'dr_'
if pl_module.is_bbox_noise_exp():
dr_exp += 'bn_'
ood_fname = pjoin(pl_module.hparams.result_dir,
f'{pl_module.__class__.__name__}_{dr_exp}ood_results.tsv')
if pexists(ood_fname):
ood_df = pd.read_csv(ood_fname, delimiter='\t')
if pl_module.hparams.name in set(ood_df['name']):
print('Already ood test for %s. Exit!' % pl_module.hparams.name)
sys.exit()
# For OOD detections
bpath = pjoin(pl_module.hparams.logdir, pl_module.hparams.name, 'best.ckpt')
assert pexists(bpath), 'Best path %s not exists!' % bpath
best_pl_module = pl_module.load_from_checkpoint(bpath)
pl_module.model.load_state_dict(best_pl_module.model.state_dict())
pl_module.current_epoch = best_pl_module.current_epoch
pl_module.global_step = best_pl_module.global_step
print('Load best model from %s' % bpath)
trainer.callback_metrics = {}
print('Clean up the callback metrics before test; Remove last val metrics')
def handle_args(self, arguments):
"""
@return a list of command_bundles
"""
content = []
for a in arguments:
if pexists(a):
content.append(a.decode('utf-8'))
else:
paths = myglob(a)
content.extend([p.decode('utf-8') for p in paths if pexists(p)])
content.sort()
if not content:
print 'No content found'
return []
self.commands = []
for c in content:
# TODO: Could really do with removing this ...
self.reset_env()
try:
commandset = self.handle_content(c)
if commandset:
self.commands.append(commandset)
except Exception as e:
# parsing one arg failed -- proceed with the rest
print e
print traceback.format_exc()
log.error(e)
return self.commands
def test_openbis(self):
self._send_file("äää \t({QJFDC066BIblub.RAw")
expected_name = 'QJFDC066BI_blub.RAw'
print(os.listdir(self.paths['openbis_raw']))
assert pexists(pjoin(self.paths['openbis_raw'], expected_name))
marker = '.MARKER_is_finished_' + expected_name
assert pexists(pjoin(self.paths['openbis_raw'], marker))
origname_file = pjoin(
self.paths['openbis_raw'],
expected_name,
expected_name + '.origlabfilename',
)
assert pexists(origname_file)
with open(origname_file, 'r') as f:
assert f.read() == "äää \t({QJFDC066BIblub.RAw"
checksum_file = pjoin(self.paths['openbis_raw'], expected_name,
expected_name + '.sha256sum')
assert pexists(checksum_file)
source_file = pjoin(self.paths['openbis_raw'], expected_name,
"source_dropbox.txt")
with open(source_file) as f:
assert f.read() == "incoming1"
def test_umask(self):
self._send_dir('testdir', 'file1')
outpath = pjoin(self.paths['manual'], 'incoming1', 'testdir')
assert pexists(outpath)
assert os.path.isdir(outpath)
assert pexists(pjoin(outpath, 'testdir', 'file1'))
assert os.stat(outpath).st_mode & self.umask == 0
def read_features_and_groups(features_path, groups_path):
"Reader for data and groups"
try:
if not pexists(features_path):
raise ValueError('non-existent features file')
if not pexists(groups_path):
raise ValueError('non-existent groups file')
if isinstance(features_path, str):
features = np.genfromtxt(features_path, dtype=float)
else:
raise ValueError('features input must be a file path ')
if isinstance(groups_path, str):
groups = np.genfromtxt(groups_path, dtype=str)
else:
raise ValueError('groups input must be a file path ')
except:
raise IOError('error reading the specified features and/or groups.')
if len(features) != len(groups):
raise ValueError("lengths of features and groups do not match!")
return features, groups
def run(self, args, infile):
print("hmm_mapper: run")
##
# Step 1. Sequence search
self.search(args, infile)
##
# If running in scratch, move files to real output dir and clean up
if self.scratch_dir:
scratch_path = pjoin(self.scratch_dir, self.hmm_hits_file)
if pexists(scratch_path):
print(" Copying result file %s from scratch to %s" %
(scratch_path, self.output_dir))
shutil.copy(scratch_path, self.output_dir)
print(
colorify(
f"Data in {self.scratch_dir} will be not removed. Please, clear it manually.",
'red'))
##
# Finalize and exit
print(colorify('Done', 'green'))
colorify('Result files:', 'yellow')
output_path = pjoin(self.output_dir, self.hmm_hits_file)
if pexists(output_path):
print(" %s" % (output_path))
return
def raw_clean_bootstrap(pyver):
bdir = bootstrap_dir(pyver)
for d in ["build", "dist"]:
if pexists(pjoin(bdir, d)):
rmtree(pjoin(bdir, d))
if pexists(pjoin(bdir, "site.cfg")):
os.remove(pjoin(bdir, "site.cfg"))
def raw_clean_bootstrap(pyver):
bdir = bootstrap_dir(pyver)
for d in ["build", "dist"]:
if pexists(pjoin(bdir, d)):
rmtree(pjoin(bdir, d))
if pexists(pjoin(bdir, "site.cfg")):
os.remove(pjoin(bdir, "site.cfg"))
def fetch_EPSILON(path='./data/', train_size=None, valid_size=None, test_size=None, fold=0):
path = pjoin(path, 'EPSILON')
train_path = pjoin(path, 'epsilon_normalized')
test_path = pjoin(path, 'epsilon_normalized.t')
if not all(pexists(fname) for fname in (train_path, test_path)):
os.makedirs(path, exist_ok=True)
train_archive_path = pjoin(path, 'epsilon_normalized.bz2')
test_archive_path = pjoin(path, 'epsilon_normalized.t.bz2')
if not all(pexists(fname) for fname in (train_archive_path, test_archive_path)):
download("https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/binary/epsilon_normalized.bz2", train_archive_path)
download("https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/binary/epsilon_normalized.t.bz2", test_archive_path)
print("unpacking dataset")
for file_name, archive_name in zip((train_path, test_path), (train_archive_path, test_archive_path)):
zipfile = bz2.BZ2File(archive_name)
with open(file_name, 'wb') as f:
f.write(zipfile.read())
with Timer("reading dataset (it may take a long time)"):
X_train, y_train = load_svmlight_file(train_path, dtype=np.float32, n_features=2000)
X_test, y_test = load_svmlight_file(test_path, dtype=np.float32, n_features=2000)
X_train, X_test = X_train.toarray(), X_test.toarray()
y_train, y_test = y_train.astype(np.int), y_test.astype(np.int)
y_train[y_train == -1] = 0
y_test[y_test == -1] = 0
if all(sizes is None for sizes in (train_size, valid_size, test_size)):
train_idx_path = pjoin(path, 'stratified_train_idx.txt')
valid_idx_path = pjoin(path, 'stratified_valid_idx.txt')
if not all(pexists(fname) for fname in (train_idx_path, valid_idx_path)):
download("https://www.dropbox.com/s/wxgm94gvm6d3xn5/stratified_train_idx.txt?dl=1", train_idx_path)
download("https://www.dropbox.com/s/fm4llo5uucdglti/stratified_valid_idx.txt?dl=1", valid_idx_path)
train_idx = pd.read_csv(train_idx_path, header=None)[0].values
valid_idx = pd.read_csv(valid_idx_path, header=None)[0].values
else:
assert train_size, "please provide either train_size or none of sizes"
if valid_size is None:
valid_size = len(X_train) - train_size
assert valid_size > 0
if train_size + valid_size > len(X_train):
warnings.warn('train_size + valid_size = {} exceeds dataset size: {}.'.format(
train_size + valid_size, len(X_train)), Warning)
if test_size is not None:
warnings.warn('Test set is fixed for this dataset.', Warning)
shuffled_indices = np.random.permutation(np.arange(len(X_train)))
train_idx = shuffled_indices[:train_size]
valid_idx = shuffled_indices[train_size: train_size + valid_size]
X_train = pd.DataFrame(X_train)
X_test = pd.DataFrame(X_test)
return dict(
X_train=X_train.iloc[train_idx], y_train=y_train[train_idx],
X_valid=X_train.iloc[valid_idx], y_valid=y_train[valid_idx],
X_test=X_test, y_test=y_test,
problem='classification',
)
def analyze():
''' analyze the data in the data file. '''
if pexists(pjoin('/root/output', 'box_samples.txt')):
if pexists(pjoin('/root/output', 'analysis.txt')):
os.remove(pjoin('/root/output', 'analysis.txt'))
call(['/root/bin/analyzer.out','/root/output/box_samples.txt','/root/output/analysis.txt'])
return flask.send_from_directory('/root/output', 'analysis.txt')
else:
return flask.Response(404,'box_samples.txt not found or path doesn\'t exist.')
def fetch_PROTEIN(path='./data/', train_size=None, valid_size=None, test_size=None, fold=0):
"""
https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/multiclass.html#protein
"""
path = pjoin(path, 'PROTEIN')
train_path = pjoin(path, 'protein')
test_path = pjoin(path, 'protein.t')
if not all(pexists(fname) for fname in (train_path, test_path)):
os.makedirs(path, exist_ok=True)
download("https://www.dropbox.com/s/pflp4vftdj3qzbj/protein.tr?dl=1", train_path)
download("https://www.dropbox.com/s/z7i5n0xdcw57weh/protein.t?dl=1", test_path)
for fname in (train_path, test_path):
raw = open(fname).read().replace(' .', '0.')
with open(fname, 'w') as f:
f.write(raw)
X_train, y_train = load_svmlight_file(train_path, dtype=np.float32, n_features=357)
X_test, y_test = load_svmlight_file(test_path, dtype=np.float32, n_features=357)
X_train, X_test = X_train.toarray(), X_test.toarray()
y_train, y_test = y_train.astype(np.int), y_test.astype(np.int)
if all(sizes is None for sizes in (train_size, valid_size, test_size)):
train_idx_path = pjoin(path, 'stratified_train_idx.txt')
valid_idx_path = pjoin(path, 'stratified_valid_idx.txt')
if not all(pexists(fname) for fname in (train_idx_path, valid_idx_path)):
download("https://www.dropbox.com/s/wq2v9hl1wxfufs3/small_stratified_train_idx.txt?dl=1", train_idx_path)
download("https://www.dropbox.com/s/7o9el8pp1bvyy22/small_stratified_valid_idx.txt?dl=1", valid_idx_path)
train_idx = pd.read_csv(train_idx_path, header=None)[0].values
valid_idx = pd.read_csv(valid_idx_path, header=None)[0].values
else:
assert train_size, "please provide either train_size or none of sizes"
if valid_size is None:
valid_size = len(X_train) - train_size
assert valid_size > 0
if train_size + valid_size > len(X_train):
warnings.warn('train_size + valid_size = {} exceeds dataset size: {}.'.format(
train_size + valid_size, len(X_train)), Warning)
if test_size is not None:
warnings.warn('Test set is fixed for this dataset.', Warning)
shuffled_indices = np.random.permutation(np.arange(len(X_train)))
train_idx = shuffled_indices[:train_size]
valid_idx = shuffled_indices[train_size: train_size + valid_size]
X_train = pd.DataFrame(X_train)
X_test = pd.DataFrame(X_test)
return dict(
X_train=X_train.iloc[train_idx], y_train=y_train[train_idx],
X_valid=X_train.iloc[valid_idx], y_valid=y_train[valid_idx],
X_test=X_test, y_test=y_test
)
def __init__(self, remote_config_path, ctx):
# Required for providers registration :
from pcs_api.providers import (dropbox, googledrive)
#
from pcs_api.credentials.app_info_file_repo \
import AppInfoFileRepository
from pcs_api.credentials.user_creds_file_repo \
import UserCredentialsFileRepository
from pcs_api.credentials.user_credentials import UserCredentials
from pcs_api.storage import StorageFacade
ctx.logger.info("Setting up cloud storage app")
app_info_path = pjoin(remote_config_path, "app_info_data.txt")
user_credentials_path = pjoin(remote_config_path,
"user_credentials_data.txt")
if not pexists(remote_config_path):
msg = 'No remote named ' + os.path.split(remote_config_path)[1]
ctx.logger.critical(msg)
msg = "Run 'hit remote add ...'"
ctx.logger.critical(msg)
exit(1)
if not pexists(app_info_path):
msg = 'No remote application information: ' \
+ repr(app_info_path)
ctx.logger.critical(msg)
msg = "Run 'hit remote add ...'"
ctx.logger.critical(msg)
exit(1)
apps_repo = AppInfoFileRepository(app_info_path)
if not pexists(user_credentials_path):
msg = 'No user credentials found: ' \
+ repr(user_credentials_path)
ctx.logger.critical(msg)
msg = "Run 'hit remote add ...'"
ctx.logger.critical(msg)
exit(1)
user_credentials_repo = UserCredentialsFileRepository(
user_credentials_path)
provider_name = apps_repo._app_info.keys()[0].split(".")[0]
app_info = apps_repo.get(provider_name)
user_info = user_credentials_repo.get(app_info)
self.storage = StorageFacade \
.for_provider(provider_name) \
.app_info_repository(apps_repo, app_info.app_name) \
.user_credentials_repository(user_credentials_repo,
user_info.user_id) \
.build()
msg = "Cloud storage user_id = " + repr(self.storage.get_user_id())
ctx.logger.info(msg)
msg = "Cloud storage quota = " + repr(self.storage.get_quota())
ctx.logger.info(msg)
ctx.logger.info("Cloud storage is ready")
self.ctx = ctx
def link_contents(content_path, linkdir_path):
content_realpath = realpath(content_path)
for fn in os.listdir(content_realpath):
f_realpath = pjoin(content_realpath, fn)
f_sympath = pjoin(linkdir_path, fn)
if not pexists(f_sympath):
os.symlink(f_realpath, f_sympath)
elif pexists(f_sympath) and not sym_sametarget(f_sympath, f_realpath):
print_advice_on_duplicates([realpath(f_sympath), realpath(f_realpath)])
else:
# symlink exists, but points to same location.
pass
def raw_clean(src_dir, pyver):
# Clean sdist
sdir = pjoin(src_dir, "dist")
if pexists(sdir):
rmtree(sdir)
mani = pjoin(src_dir, "MANIFEST")
if pexists(mani):
os.remove(mani)
# Clean bootstrap directory
bdir = bootstrap_dir(pyver)
if pexists(bdir):
rmtree(bdir)
def raw_clean(src_dir, pyver):
# Clean sdist
sdir = pjoin(src_dir, "dist")
if pexists(sdir):
rmtree(sdir)
mani = pjoin(src_dir, "MANIFEST")
if pexists(mani):
os.remove(mani)
# Clean bootstrap directory
bdir = bootstrap_dir(pyver)
if pexists(bdir):
rmtree(bdir)
def run(self, args, infile, annotate_hits_table=None, cache_dir=None):
##
# Step 0. Gene prediction
predictor = self.gene_prediction(args, infile)
##
# Step 1. Sequence search
searcher = self.search(args, infile, predictor)
##
# Step 2. Annotation
annotator = self.annotate(args, searcher, annotate_hits_table,
cache_dir)
##
# Decorate GFF
self.decorate_gff_f(predictor, searcher, annotator)
##
# Clear things
if predictor is not None:
shutil.move(predictor.outprots,
pjoin(self._current_dir, self.genepred_fasta_file))
predictor.clear() # removes gene predictor output directory
##
# If running in scratch, move files to real output dir and clean up
if self.scratch_dir:
for fname in self._output_files:
pathname = pjoin(self.scratch_dir, fname)
if pexists(pathname):
print(" Copying result file %s from scratch to %s" %
(pathname, self.output_dir))
shutil.copy(pathname, self.output_dir)
print(
colorify(
f"Data in {self.scratch_dir} will be not removed. Please, clear it manually.",
'red'))
##
# Finalize and exit
print(colorify('Done', 'green'))
for fname in self._output_files:
pathname = pjoin(self.output_dir, fname)
colorify('Result files:', 'yellow')
if pexists(pathname):
print(" %s" % (pathname))
return
def on_train_end(self, trainer, pl_module):
result_f = pjoin(pl_module.hparams.logdir, pl_module.hparams.name,
'results.tsv')
assert pexists(result_f), 'Result %s should exists!' % result_f
df = pd.read_csv(result_f, delimiter='\t')
func = {'min': np.argmin, 'max': np.argmax}[
trainer.checkpoint_callback.mode]
best_idx = int(func(df[trainer.checkpoint_callback.monitor].values))
best_metric = df.iloc[best_idx].to_dict()
csv_dict = OrderedDict()
csv_dict['name'] = pl_module.hparams.name
csv_dict.update(best_metric)
csv_dict.update(
vars(pl_module.hparams) if isinstance(pl_module.hparams, Namespace)
else pl_module.hparams)
postfix = '_test' if pl_module.hparams.test_run else ''
dr_exp = ''
if pl_module.is_data_ratio_exp():
dr_exp += 'dr_'
if pl_module.is_bbox_noise_exp():
dr_exp += 'bn_'
fname = pjoin(pl_module.hparams.result_dir,
f'{pl_module.__class__.__name__}_{dr_exp}results{postfix}.tsv')
# Check if already exists
if not pexists(fname):
output_csv(fname, csv_dict, delimiter='\t')
else:
try:
tmp_df = pd.read_csv(fname, delimiter='\t')
if pl_module.hparams.name not in set(tmp_df['name']):
output_csv(fname, csv_dict, delimiter='\t')
except:
# Avoid reading CSV error
output_csv(fname, csv_dict, delimiter='\t')
bpath = pjoin(pl_module.hparams.logdir, pl_module.hparams.name, 'best.ckpt')
if pexists(bpath):
return
if os.path.islink(bpath):
os.unlink(bpath)
best_filename = trainer.checkpoint_callback.format_checkpoint_name(
best_metric['epoch'], dict(gstep=best_metric['global_step']))
os.symlink(best_filename, bpath)
def annotate(self, args, hits, annotate_hits_table, queries_file,
cache_file):
annotated_hits = None
if self.annot == True or self.report_orthologs:
if cache_file is not None:
if not pexists(cache_file):
raise EmaperException(
f"Could not find cache file: {cache_file}")
annotator = get_cache_annotator(args)
if annotator is not None:
annotated_hits = annotator.annotate(
cache_file, pjoin(self._current_dir, self.annot_file),
pjoin(self._current_dir, self.no_annot_file))
else:
annot_in = None # a generator of hits to annotate
if annotate_hits_table is not None:
if not pexists(annotate_hits_table):
raise EmapperException(
f"Could not find the file with the hits "
f"table to annotate: {annotate_hits_table}")
# function which parses the file and yields hits
annot_in = parse_seeds(annotate_hits_table)
elif hits is not None:
annot_in = hits
else:
raise EmapperException("Could not find hits to annotate.")
annotator = get_annotator(args, self.annot, self.excel,
self.report_orthologs)
if annot_in is not None and annotator is not None:
annotated_hits = annotator.annotate(
annot_in, pjoin(self._current_dir, self.annot_file),
pjoin(self._current_dir, self.excel_file),
pjoin(self._current_dir, self.orthologs_file),
pjoin(self._current_dir, self.pfam_file), queries_file)
else:
annotated_hits = ((hit, None) for hit in hits
) # hits generator without annotations
return annotated_hits
def test_vis():
" ensures the CLI works. "
res_path = pjoin(out_dir, 'rhst_results.pkl')
if pexists(res_path):
with raises(SystemExit):
sys.argv = shlex.split('neuropredict --make_vis {}'.format(out_dir))
cli()
expected_results = ['balanced_accuracy.pdf', 'compare_misclf_rates.pdf', 'feature_importance.pdf']
for rpath in expected_results:
if not pexists(rpath):
raise ValueError('an expected result {} not produced'.format(rpath))
else:
print('previously computed results not found in \n {}'.format(out_dir))
def test_manual(self):
self._send_file('dataaä .txt')
assert pexists(pjoin(self.paths['manual'], 'incoming1', 'dataa.txt'))
assert pexists(
pjoin(self.paths['manual'], 'incoming1', 'dataa.txt',
'dataa.txt.sha256sum'))
with open(
pjoin(self.paths['manual'], 'incoming1', 'dataa.txt',
'dataa.txt.sha256sum')) as f:
assert 'dataa.txt' in f.read()
origfile = pjoin(self.paths['manual'], 'incoming1', 'dataa.txt',
'dataa.txt.origlabfilename')
with open(origfile) as f:
assert f.read() == 'dataaä .txt'
def fetch_HIGGS(path='./data/', train_size=None, valid_size=None, test_size=5 * 10 ** 5, fold=0):
path = pjoin(path, 'HIGGS')
data_path = pjoin(path, 'higgs.csv')
if not pexists(data_path):
os.makedirs(path, exist_ok=True)
archive_path = pjoin(path, 'HIGGS.csv.gz')
download('https://archive.ics.uci.edu/ml/machine-learning-databases/00280/HIGGS.csv.gz', archive_path)
with gzip.open(archive_path, 'rb') as f_in:
with open(data_path, 'wb') as f_out:
shutil.copyfileobj(f_in, f_out)
n_features = 29
types = {i: (np.float32 if i != 0 else np.int) for i in range(n_features)}
data = pd.read_csv(data_path, header=None, dtype=types)
data_train, data_test = data.iloc[:-test_size], data.iloc[-test_size:]
X_train, y_train = data_train.iloc[:, 1:].values, data_train.iloc[:, 0].values
X_test, y_test = data_test.iloc[:, 1:].values, data_test.iloc[:, 0].values
if all(sizes is None for sizes in (train_size, valid_size)):
train_idx_path = pjoin(path, 'stratified_train_idx.txt')
valid_idx_path = pjoin(path, 'stratified_valid_idx.txt')
if not all(pexists(fname) for fname in (train_idx_path, valid_idx_path)):
download("https://www.dropbox.com/s/i2uekmwqnp9r4ix/stratified_train_idx.txt?dl=1", train_idx_path)
download("https://www.dropbox.com/s/wkbk74orytmb2su/stratified_valid_idx.txt?dl=1", valid_idx_path)
train_idx = pd.read_csv(train_idx_path, header=None)[0].values
valid_idx = pd.read_csv(valid_idx_path, header=None)[0].values
else:
assert train_size, "please provide either train_size or none of sizes"
if valid_size is None:
valid_size = len(X_train) - train_size
assert valid_size > 0
if train_size + valid_size > len(X_train):
warnings.warn('train_size + valid_size = {} exceeds dataset size: {}.'.format(
train_size + valid_size, len(X_train)), Warning)
shuffled_indices = np.random.permutation(np.arange(len(X_train)))
train_idx = shuffled_indices[:train_size]
valid_idx = shuffled_indices[train_size: train_size + valid_size]
X_train = pd.DataFrame(X_train)
X_test = pd.DataFrame(X_test)
return dict(
X_train=X_train.iloc[train_idx], y_train=y_train[train_idx],
X_valid=X_train.iloc[valid_idx], y_valid=y_train[valid_idx],
X_test=X_test, y_test=y_test,
problem='classification',
)
def import_features(fs_dir,
subject_list,
base_feature= 'freesurfer_thickness',
fwhm=10,
atlas='fsaverage'):
"Ensure subjects are provided and their data exist."
if isinstance(subject_list, collections.Iterable):
if len(subject_list) < 1:
raise ValueError('Empty subject list.')
subjects_list = subject_list
elif isinstance(subject_list, str):
if not pexists(subject_list):
raise IOError('path to subject list does not exist: {}'.format(subject_list))
subjects_list = np.atleast_1d(np.genfromtxt(subject_list, dtype=str).astype(str))
else:
raise ValueError('Invalid value provided for subject list. \n '
'Must be a list of paths, or path to file containing list of paths, one for each subject.')
features= dict()
for subject_id in subjects_list:
try:
print('Reading {} for {} ... '.format(base_feature, subject_id), end='')
features[subject_id] = get_data(fs_dir, subject_id, base_feature, fwhm, atlas)
print(' Done.')
except:
traceback.print_exc()
raise ValueError('{} data for {} could not be read!'.format(base_feature, subject_id))
return features
def controls(self):
debug(self._label + "control files")
package = self._package
suffix_normal = [".conf", ".conffiles", ".mime", ".service"]
suffix_script = [".init", ".preinst", ".postinst", ".prerm", ".postrm"]
dest_base = pjoin(self._env["QPKG_DEST_DATA"], ".qdk2")
makedirs(dest_base)
for suffix in suffix_normal + suffix_script:
src = pjoin(Settings.CONTROL_PATH, package["package"] + suffix)
dest = pjoin(dest_base, package["package"] + suffix)
if not pexists(src):
continue
if suffix in (".init",):
dest = pjoin(self._env["QPKG_DEST_DATA"], package["package"] + suffix)
# copy to destination and replace template variables
tpl_vars = [("%" + k + "%", v) for k, v in self._env.iteritems() if k.startswith("QPKG_")]
with open(src, "r") as fin, open(dest, "w+") as fout:
for line in fin:
for k, v in tpl_vars:
line = line.replace(k, v)
fout.write(line)
if suffix in suffix_script:
chmod(dest, 0755)
else:
chmod(dest, 0644)
def make_predictions(model_name, X):
saved_dir = pjoin('logs', model_name)
hparams = json.load(open(pjoin(saved_dir, 'hparams.json')))
assert pexists(pjoin(saved_dir,
'checkpoint_best.pth')), 'No best ckpt exists!'
model = load_best_model_from_trained_dir(saved_dir)
device = 'cuda' if torch.cuda.is_available() else 'cpu'
model.to(device)
pp = pickle.load(open(pjoin(saved_dir, 'preprocessor.pkl'), 'rb'))
X = pp.transform(X)
X = torch.as_tensor(X, device=device)
with torch.no_grad():
logits = process_in_chunks(model,
X,
batch_size=2 * hparams['batch_size'])
logits = check_numpy(logits)
ret = logits
if len(logits.shape) == 2 and logits.shape[1] == 2:
ret = logits[:, 1] - logits[:, 0]
elif len(logits.shape) == 1: # regression or binary cls
if pp.y_mu is not None and pp.y_std is not None:
ret = (ret * pp.y_std) + pp.y_mu
return ret
def restore_ratings(self):
"""Method to restore ratings from previous sessions, if any."""
print('Restoring ratings from previous session(s), if they exist ..')
# making a copy
self.incomplete_list = list(self.id_list)
prev_done = [] # empty list
ratings_file, backup_name_ratings = get_ratings_path_info(self)
if pexists(ratings_file):
self.ratings, self.notes = load_ratings_csv(ratings_file)
# finding the remaining
prev_done = set(self.ratings.keys())
self.incomplete_list = list(set(self.id_list) - prev_done)
else:
self.ratings = dict()
self.notes = dict()
if len(prev_done) > 0:
print('\nRatings for {}/{} subjects were restored.'
''.format(len(prev_done), len(self.id_list)))
if len(self.incomplete_list) < 1:
print('No subjects to review/rate - exiting.')
sys.exit(0)
else:
self.num_units_to_review = len(self.incomplete_list)
print('To be reviewed : {}\n'.format(self.num_units_to_review))
def save_ratings(self):
"""Saves ratings to disk """
print('Saving ratings .. \n')
ratings_file, prev_ratings_backup = get_ratings_path_info(self)
if pexists(ratings_file):
copyfile(ratings_file, prev_ratings_backup)
# add column names: subject_id,issue1:issue2:issue3,...,notes etc
# TODO add path(s) to data (images etc) that produced the review
lines = '\n'.join([
'{},{},{}'.format(sid, self._join_ratings(rating_set),
self.notes[sid])
for sid, rating_set in self.ratings.items()
])
try:
with open(ratings_file, 'w') as cf:
cf.write(lines)
except:
raise IOError('Error in saving ratings to file!!\n'
'Backup might be helpful at:\n\t{}'.format(
prev_ratings_backup))
# summarize ratings to stdout and id lists
summarize_ratings(ratings_file)
def load_hparams(the_dir):
if pexists(pjoin(the_dir, 'hparams.json')):
hparams = json.load(open(pjoin(the_dir, 'hparams.json')))
else:
name = os.path.basename(the_dir)
if not pexists(pjoin('logs', 'hparams', name)):
cmd = 'rsync -avzL v:/h/kingsley/node/logs/hparams/%s ./logs/hparams/' % (
name)
print(cmd)
os.system(cmd)
if pexists(pjoin('logs', 'hparams', name)):
hparams = json.load(open(pjoin('logs', 'hparams', name)))
else:
raise RuntimeError('No hparams exist: %s' % the_dir)
return hparams
def run(self):
if self.qpkg_dir is None:
error('Cannot find QNAP/control anywhere!')
error('Are you in the source code tree?')
return -1
if self._args.filename is None:
self._args.filename = 'control'
cfiles = self._get_support_control_files()
if self._args.filename not in cfiles:
error('Support control files: {}'.format(', '.join(cfiles)))
return -1
filename = pjoin(self.qpkg_dir, Settings.CONTROL_PATH,
self._args.filename)
editor = Editor()
if not pexists(filename):
editor.set_template_file(
pjoin(Settings.TEMPLATE_PATH, Settings.CONTROL_PATH,
'{}{}'.format(Settings.DEFAULT_PACKAGE,
filename[filename.rfind('.'):])))
editor.open(filename)
return 0
def _setup_all(self, control):
cwd = getcwd()
dest = prealpath(pjoin(self.build_dir, control.source['source']))
if pexists(dest):
rmtree(dest)
if not pexists(self.build_dir):
makedirs(self.build_dir)
# if dest in qpkg_dir
copytree(self.qpkg_dir, dest, True)
self.source = control.source
chdir(dest)
yield None
chdir(cwd)
del self.source
def extract_GAM_from_baselines(saved_dir, max_n_bins=256, **kwargs):
from .data import DATASETS
model = pickle.load(open(pjoin(saved_dir, 'model.pkl'), 'rb'))
hparams = load_hparams(saved_dir)
pp = None
if pexists(pjoin(saved_dir, 'preprocessor.pkl')):
pp = pickle.load(open(pjoin(saved_dir, 'preprocessor.pkl'), 'rb'))
dataset = DATASETS[hparams['dataset'].upper()](path='./data/')
all_X = pd.concat([dataset['X_train'], dataset['X_test']], axis=0)
def predict_fn(X):
if isinstance(X, np.ndarray):
X = pd.DataFrame(X, columns=all_X.columns)
if pp is not None:
X = pp.transform(X)
if dataset['problem'] == 'classification':
prob = model.predict_proba(X)
return prob[:, 1]
return model.predict(X)
predict_type = 'binary_prob' \
if dataset['problem'] == 'classification' else 'regression'
df = extract_GAM(all_X,
predict_fn,
max_n_bins=max_n_bins,
predict_type=predict_type)
return df
def load_results(results_file_path):
"Loads the results serialized by RHsT."
# TODO need to standardize what needs to saved/read back
if not pexists(results_file_path):
raise IOError("Results file to be loaded doesn't exist!")
try:
with open(results_file_path, 'rb') as rf:
results_dict = pickle.load(rf)
# # below is possible, but not explicit and a bad practice
# # importing the keys and their values into the workspace
# locals().update(results_dict)
dataset_paths, method_names, train_perc, num_repetitions, num_classes, \
pred_prob_per_class, pred_labels_per_rep_fs, test_labels_per_rep, \
best_params, feature_importances_rf, \
feature_names, num_times_misclfd, num_times_tested, \
confusion_matrix, class_set, target_sizes, accuracy_balanced, \
auc_weighted, positive_class, classifier_name, feat_select_method = \
[results_dict.get(var_name)
for var_name in cfg.rhst_data_variables_to_persist]
except:
raise IOError('Error loading the saved results from \n{}'
''.format(results_file_path))
# TODO need a consolidated way to identify the variables saved their order
return dataset_paths, method_names, train_perc, num_repetitions, num_classes, \
pred_prob_per_class, pred_labels_per_rep_fs, test_labels_per_rep, \
best_params, feature_importances_rf, feature_names, \
num_times_misclfd, num_times_tested, \
confusion_matrix, class_set, target_sizes, \
accuracy_balanced, auc_weighted, positive_class, \
classifier_name, feat_select_method
def run(self):
if not pexists(self.directory):
makedirs(self.directory)
else:
if isdir(self.directory):
if listdir(self.directory):
error('Directory is not empty: ' + self.directory)
return -1
else:
error('This is not directory: ' + self.directory)
return -1
try:
src_type, src_value = self.probe_source_type()
if self.import_source(src_type, src_value) != 0:
error('Import failed')
if pexists(pjoin(self._directory, Settings.CONTROL_PATH)):
info('{} folder exists!'.format(Settings.CONTROL_PATH))
return 0
# copy template control files
info('Copy default CONTROL files to {}'.format(
pjoin(self.directory, Settings.CONTROL_PATH)))
copytree(pjoin(Settings.TEMPLATE_PATH, Settings.CONTROL_PATH),
pjoin(self.directory, Settings.CONTROL_PATH), True)
info('Modify package name to ' + self._args.project)
# sed control, rules, *.init, *.conf
files_check = ('control', 'rules', 'foobar.init', 'foobar.conf')
for fn in files_check:
fp = pjoin(self.directory, Settings.CONTROL_PATH, fn)
for line in fileinput.input(fp, inplace=True):
print line.replace(Settings.DEFAULT_PACKAGE,
self._args.project),
# mv foobar.* to self._args.project.*
for fn in glob(pjoin(self.directory, Settings.CONTROL_PATH,
Settings.DEFAULT_PACKAGE + '.*')):
dest = pjoin(pdirname(fn),
self._args.project + fn[fn.index('.'):])
move(fn, dest)
except Exception as e:
error(e)
return -1
return 0
def write_default_config():
confdir = expanduser('~/.retorrent')
skelfile = '/usr/share/retorrent/' + config_filename + '_skel'
if pexists(skelfile):
if not pexists(pjoin(confdir, config_filename + '_skel')):
print('Creating a skeleton $HOME/.retorrent/retorrent.conf, ' +
'please configure it to your system')
mkdir_p(confdir)
shutil.copyfile('/usr/share/retorrent/retorrent.conf_skel',
expanduser('~/.retorrent/retorrent.conf_skel'))
else:
print('Please configure the retorrent.conf_skel in ${HOME}/.retorrent ' +
'and rename it to ' + config_filename)
else:
print('Cannot find ' + config_filename + ' or a valid skeleton ' + config_filename + '.')
print('Please create and configure %s or check your installation. ' %
(pjoin(confdir, config_filename)))
def metatranscriptomics(opts):
"""Performs analyse of metagenomic data.
For more information please refer to
run_fastq_to_fasta(), run_rapsearch() run_ko_map(), run_SARTools(),
run_pre_ko_remap(), run_ko_remap(), run_new_ko_remap(), run_ko_csv().
"""
assert opts.out_dir!='in_situ'
before_cwd = getcwd()
tmp_dir = pjoin(opts.out_dir, '.meta_tmp_results')
if not pexists(tmp_dir) or not opts.e:
system('mkdir ' + tmp_dir)
chdir(tmp_dir)
for i in ('template_script_DESeq2.r','template_script_edgeR.r'):
system('cp '+pjoin(dirname(realpath(__file__)),i)+' .')
ref_cond, all_conds, fastqs = config_from_file(opts.metatr_config)
progress('configuration file reading', 15)
if (len(fastqs) > len(glob('*_R[12]_*.fasta'))) or not opts.e:
run_fastq_to_fasta(fastqs)
progress('fastq_to_fasta', 25)
if (len(fastqs)/2 > len(glob('*.tmp.fasta'))) or not opts.e:
run_cat_pairing()
progress('cat', 35)
if (len(fastqs)/2 > len(glob('*.txt.m8'))) or not opts.e:
run_rapsearch(opts.threads)
progress('rapsearch', 45)
if (len(fastqs)/2 > len(glob('*.count'))) or not opts.e:
run_ko_map()
progress('KO mapping', 55)
if ('edger' not in glob('*')) or not opts.e:
system('mkdir deseq edger')
run_SARTools()
progress('SARTools', 65)
path_names, kopath_keys, kopath_values, edger_files, deseq_files = \
run_pre_ko_remap()
if opts.metatr_output_type != 'new':
if (len(all_conds)*(len(all_conds)-1) > len(glob('*.path_counts.csv'))\
or not opts.e):
run_ko_remap(deseq_files, edger_files, kopath_values, path_names)
if opts.metatr_output_type != 'old':
if ('edger.csv' not in glob('*')) or not opts.e:
ko_dict_deseq, ko_dict_edger = run_new_ko_remap(
deseq_files, edger_files, kopath_values, all_conds, ref_cond)
progress('pathway mapping', 75)
run_ko_csv(ko_dict_deseq, ko_dict_edger, all_conds, kopath_keys,
path_names, ref_cond)
progress('generating summative CSV', 85)
if opts.metatr_output_type != 'new':
old_path = opts.out_dir + '/old'
system('mkdir ' + old_path)
system('cp *path_counts.csv ' + old_path)
if opts.metatr_output_type != 'old':
system('cp deseq.csv edger.csv ' + opts.out_dir)
system('cp edger/_report.html ' + opts.out_dir + '/edger_report.html')
system('cp deseq/_report.html ' + opts.out_dir + '/deseq_report.html')
chdir(before_cwd)
progress('METATRANSCRIPTOMIC WORKFLOW', 95)
def prepare_dest():
dest = prealpath(pjoin(
'.', Settings.CONTROL_PATH,
package['package'] + '_' + package['architecture']))
if pexists(dest):
rmtree(dest)
self._env['QPKG_DEST_CONTROL'] = dest
self._env['QPKG_DEST_DATA'] = pjoin(dest, 'shared')
makedirs(self._env['QPKG_DEST_DATA'])
def dirs(self):
debug(self._label + "create directories")
src_install = pjoin(Settings.CONTROL_PATH, self._package["package"] + ".dirs")
if not pexists(src_install):
return
try:
lineno = 0
with open(src_install) as fin:
for line in fin:
lineno += 1
line = line.strip()
if line.startswith("/"):
raise FileSyntaxError(src_install, lineno, line)
dst = pjoin(self._env["QPKG_DEST_DATA"], line)
if not pexists(dst):
makedirs(dst)
except ValueError:
raise FileSyntaxError(src_install, lineno, line)
def test_tar_out(self):
image = pjoin(self.tmpdir, 'image.img')
data = self._prepare_data()
prepare_data_image(image, data, size="20M")
outdir = pjoin(self.tmpdir, 'outdir')
os.mkdir(outdir)
outfile = pjoin(outdir, "out.tar")
extract_from_image(image, outfile, path='/', use_tar=True)
assert pexists(outfile)
def qpkg_dir(self):
if not hasattr(self, '_qpkg_dir'):
cwd = os.getcwd()
while cwd != '/':
if pexists(pjoin(cwd, Settings.CONTROL_PATH, 'control')):
break
cwd = pabspath(pjoin(cwd, os.pardir))
self._qpkg_dir = cwd if cwd != '/' else None
return self._qpkg_dir
def format_qdk2(self):
info('Create CONTROL at {}'.format(self.directory))
if pexists(pjoin(self.directory, Settings.CONTROL_PATH)):
yn = raw_input('{} folder exists! Delete? [Y/n] '.format(
pjoin(self.directory, Settings.CONTROL_PATH)))
if len(yn) and yn[0].lower() == 'n':
raise UserExit()
rmtree(pjoin(self.directory, Settings.CONTROL_PATH))
if not pexists(self.directory):
makedirs(self.directory)
# copy template control files
info('Copy default CONTROL files to {}'.format(
pjoin(self.directory, Settings.CONTROL_PATH)))
copytree(pjoin(Settings.TEMPLATE_PATH, Settings.CONTROL_PATH),
pjoin(self.directory, Settings.CONTROL_PATH), True)
self.rename_ctrl_files()
def parse(self):
TITLE, MSG, AUTHOR = range(3)
if self._parsed:
return
self._parsed = True
if not pexists(self._filename):
return
try:
fp = open(self._filename, 'r')
block_type = TITLE
log = {}
for line in fp:
self._lineno += 1
line = line.rstrip()
# empty line
if len(line.strip()) == 0:
continue
# message line
if line.startswith(' '):
if block_type != MSG:
raise ChangelogFileSyntaxError(
self._filename, self._lineno,
'This line expects as log message. Message lines '
'must start with 2 spaces')
log = self._append_message(log, line)
continue
# author
if line.startswith(' '):
if block_type != MSG:
raise ChangelogFileSyntaxError(
self._filename, self._lineno,
'It must have any log message')
self._append_author(log, line)
block_type = TITLE
log.clear()
continue
# new log
if line[0] != ' ':
if block_type != TITLE:
raise ChangelogFileSyntaxError(
self._filename, self._lineno,
'This line must start with package name')
log = self._new_log(line)
block_type = MSG
continue
raise ChangelogFileSyntaxError(
self._filename, self._lineno,
'This line must start with package name')
if len(log) != 0:
raise ChangelogFileSyntaxError(
self._filename, self._lineno,
'Unfinished block')
except OSError:
pass
finally:
fp.close()
def extract_qpkg(self, package, to):
extractor = Settings.QBUILD
check_call([extractor, '--extract', package, to])
if not pexists(pjoin(to, 'shared')):
makedirs(pjoin(to, 'shared'))
data = glob(pjoin(to, 'data.*'))[0]
try:
check_call(['tar', 'xvf', data, '-C', pjoin(to, 'shared')])
except CalledProcessError as e:
warning('Error in extracting: {}'.format(e))
def write_site_cfg(arch, cwd=None):
if not cwd:
cwd = os.getcwd()
scfg = pjoin(cwd, "site.cfg")
if pexists(scfg):
os.remove(scfg)
f = open(scfg, 'w')
f.writelines(SITECFG[arch])
f.close()
def run(ctx, args):
for path in [DEFAULT_STORE_DIR, DEFAULT_CONFIG_FILENAME]:
if pexists(path):
ctx.logger.error('%s already exists, aborting\n' % path)
return 2
for path in DEFAULT_CONFIG_DIRS:
os.makedirs(pjoin(DEFAULT_STORE_DIR, path))
sys.stdout.write('Directory %s created.\n' % path)
shutil.copyfile(get_config_example_filename(), DEFAULT_CONFIG_FILENAME)
sys.stdout.write('Default configuration file %s written.\n' % DEFAULT_CONFIG_FILENAME)
def prepare_env():
myenv = os.environ.copy()
for k, v in self.package.iteritems():
myenv['QPKG_' + k.upper().replace('-', '_')] = v
for k, v in self.source.iteritems():
myenv['QPKG_' + k.upper().replace('-', '_')] = v
myenv['QPKG_VERSION'] = ChangelogFile().version
if pexists(pjoin(Settings.CONTROL_PATH,
package['package'] + '.init')):
myenv['QPKG_INIT'] = package['package'] + '.init'
self._env = myenv
def _prepare(self):
self._klen = 0
for k in self.cfile.source.iterkeys():
_klen = len('QPKG_') + len(k)
self._klen = self._klen if self._klen > _klen else _klen
for k in self.cfile.packages.iterkeys():
self._klen = self._klen if self._klen > len(k) else len(k)
initfile = self.cfile.packages[k]['package'] + '.init'
if pexists(pjoin(self.qpkg_dir, Settings.CONTROL_PATH, initfile)):
self.cfile.packages[k]['init'] = initfile
config = self.cfile.packages[k]['package'] + '.conf'
if pexists(pjoin(self.qpkg_dir, Settings.CONTROL_PATH, config)):
self.cfile.packages[k]['config'] = config
if self._args.show_env:
for k in sorted(os.environ):
self._klen = self._klen if self._klen > len(k) else len(k)
def run(ctx, args):
store_dir = os.path.expanduser('~/.hashdist')
for x in [DEFAULT_CONFIG_FILENAME, store_dir]:
if pexists(x):
ctx.logger.error('%s already exists, aborting\n' % x)
return 2
for x in ['ba', 'bld', 'src', 'db', 'cache', 'gcroots']:
os.makedirs(pjoin(store_dir, x))
sys.stdout.write('Directory %s created.\n' % store_dir)
shutil.copyfile(get_config_example_filename(), DEFAULT_CONFIG_FILENAME)
sys.stdout.write('Default configuration file %s written.\n' % DEFAULT_CONFIG_FILENAME)
def same_volume(orig_paths, proposed_path):
pp = proposed_path
# Should only be one loop
# (eg. video/tv/foo -> video/tv)
while not pexists(pp):
pp = dirname(pp)
prop_dev = device_number(pp)
for p in orig_paths:
if not prop_dev == device_number(p):
return False
return True
def decompress(self, filename, directory, ftype='tarball', strip=0):
if not pexists(directory):
makedirs(directory)
if ftype == 'tarball':
cmd = ['tar', 'xvf', filename, '-C', directory]
if strip != 0:
cmd.append('--strip-components=' + str(strip))
elif ftype == 'zip':
cmd = ['unzip', '-o', filename, '-d', directory]
elif ftype == '7z':
cmd = ['7z', '-y', 'x', filename, '-o' + directory]
self.__exec(cmd)
def find_configfile(filename, extra_configdir=''):
default_path = pjoin(expanduser('~/.retorrent'), filename)
if extra_configdir:
config_paths.insert(0, extra_configdir)
for path in config_paths:
filepath = pjoin(path, filename)
if pexists(filepath):
# check permissions?
return filepath
# nothing found - touch an empty config return that
touch_default(default_path)
return default_path
def run(self):
directory = pjoin(self._args.directory)
if pexists(directory) and not pisdir(directory):
error('{} is not directory'.format(directory))
return -1
if not pexists(self._args.file) and pisdir(self._args.file):
error('{} is not file'.format(self._args.file))
return -1
if not pexists(directory):
makedirs(directory)
if self._args.as_qpkg:
self.extract_qpkg(self._args.file, directory)
elif self._args.as_image:
self.extract_image(self._args.file, directory)
elif self._args.file.endswith('.img'):
self.extract_image(self._args.file, directory)
elif self._args.file.endswith('.qpkg'):
self.extract_qpkg(self._args.file, directory)
else:
error('Unknown file suffix. Speicify --qpkg or --image')
return -1
return 0
def icons(self):
debug(self._label + "icon files")
dest_base = pjoin(self._env["QPKG_DEST_CONTROL"], "icons")
makedirs(dest_base)
package_name = self._package["package"]
icons = ((".icon.64", ".gif"), (".icon.80", "_80.gif"), (".icon.gray", "_gray.gif"))
for suffix, rsuffix in icons:
src = pjoin(Settings.CONTROL_PATH, package_name + suffix)
dest = pjoin(dest_base, package_name + rsuffix)
if not pexists(src):
warning("Missing: " + src)
copy(pjoin(Settings.TEMPLATE_PATH, Settings.CONTROL_PATH, Settings.DEFAULT_PACKAGE + suffix), dest)
continue
copy(src, dest)
chmod(dest, 0644)
def __prepare_file(self, filename, action):
if self._template is not None:
filename = self._template
fid, tmpfile = tempfile.mkstemp()
os.close(fid)
fd = open(tmpfile, "w")
if self._content:
fd.write(self._content)
if pexists(filename):
with open(filename, 'r') as fread:
fd.writelines(fread)
fd.close()
return tmpfile
