def read_video(video_path, show=False, log=True, intercept=None, strides=1, title='Face Detection', seek=None):
if strides <= 0:
strides = 1
utils.check_file(video_path)
cap = cv2.VideoCapture(video_path)
if seek is not None:
cap.set(0, seek)
frame_index = 0
while cap.isOpened():
ret, frame = cap.read()
if frame_index % strides == 0:
if log:
print('Progress %3.3f%%, Frame %s, %s' % (cap.get(2) * 100, frame_index, cap.get(0)))
if intercept is not None:
if isinstance(intercept, list):
for it in intercept:
frame = it(frame)
else:
frame = intercept(frame)
try:
if show:
cv2.imshow(title, frame)
# cv2.waitKey(1000 / int(fps)) # 延迟
except Exception as e:
print(e)
break
frame_index += 1
if cv2.waitKey(1) & 0xFF == ord('q'):
break
cap.release()
if show:
cv2.destroyAllWindows()
def dar_par(mode="Testing", cmd=None, output=sys.stdout):
'''Entry point console script for creating/testing par2 files.
Warning: dar_par assumes that the las directory in path is the jobname.
'''
logger = logging.getLogger(__name__)
parser = argparse.ArgumentParser()
for arg in ['path', 'basename', 'slice', 'extension', 'context']:
parser.add_argument(arg)
if mode == "Creating":
parser.add_argument('redundancy')
#Check if we've been called as an entry point console-script.
if cmd is not None:
opts = parser.parse_args(cmd)
else:
opts = parser.parse_args()
jobname = opts.path.strip("/").split("/")[-1]
cf = Config("~/.dardrive/jobs.cfg", DARDRIVE_DEFAULTS)
cs = getattr(cf, jobname)
par_bin = cs.par_bin
try:
check_file(par_bin)
except IOError, e:
sys.stderr.write('Disabling parity data-recovery archive:\n %s\n' %
e.message)
sys.exit(1)
def convert_vocab_to_text(vocabs):
vocab_str = ""
length = len(vocabs)
i = 0
vocab_idx = dict()
vocab_lst = list()
idx_file = '%s/%s' % (folder, 'vocabs_idx.pkl')
if u.check_file(idx_file):
vocab_idx = u.load_file(idx_file)
else:
for key, value in vocabs.iteritems():
vocab_idx[value] = key
u.save_file(idx_file, vocab_idx)
lst_file = '%s/%s' % (folder, 'vocabs_list.pkl')
if u.check_file(lst_file):
vocab_lst = u.load_file(lst_file)
else:
for key in sorted(vocab_idx.iterkeys()):
vocab_lst.append(vocab_idx[key])
u.save_file(lst_file, vocab_lst)
regex = RegexpTokenizer(r'\w+')
for w in vocab_lst:
words = regex.tokenize(w)
if len(words) != 0:
w_ = '_'.join(words)
i += 1
if i % 10000 == 0:
print('Processed %i' % i)
# break
if i == length:
vocab_str += '%s' % w_
else:
vocab_str += '%s\n' % w_
return vocab_str
def build_dataset(self, pos, neg):
# Check if files exist
if not (utils.check_file(pos) and utils.check_file(neg)):
raise Exception("Files ", pos, " and ", neg, " do not exist")
pos = list(open(pos, "r").readlines())
neg = list(open(neg, "r").readlines())
# TODO: Deprecate full clean
if self.full_clean:
pos = utils.full_clean(dataset=pos)
neg = utils.full_clean(dataset=neg)
# Create labels
pos_labels = [[0, 1] for _ in pos]
neg_labels = [[1, 0] for _ in neg]
# Combine sets
text = pos + neg
labels = pos_labels + neg_labels
# Build set
for i, line in enumerate(text):
text[i] = []
line = line.strip().split(" ")
for w in line:
text[i].append(self.word_id(w))
text[i] = [text[i], labels[i]]
return np.array(text)
def plot(directory, onebest=False):
"""Plot training and validation cross entropy."""
plt.clf()
plt.rcParams["font.family"] = "Times New Roman"
plt.rcParams["font.size"] = 16
cols = [2] if onebest else [1, 2]
train_file = os.path.join(directory, 'train.log')
val_file = os.path.join(directory, 'val.log')
utils.check_file(train_file)
utils.check_file(val_file)
train_history = np.loadtxt(train_file, usecols=cols)
val_history = np.loadtxt(val_file, usecols=cols)
assert train_history.size == val_history.size
x_axis = list(range(1, train_history.shape[0] + 1))
axis = plt.gca()
if onebest:
plt.plot(x_axis, train_history, 'r-', label='train')
plt.plot(x_axis, val_history, 'g-', label='val')
else:
plt.plot(x_axis, train_history[:, 0], 'r-', label='train_allarc')
plt.plot(x_axis, val_history[:, 0], 'g-', label='val_allarc')
plt.plot(x_axis, train_history[:, 1], 'r--', label='train_onebest')
plt.plot(x_axis, val_history[:, 1], 'g--', label='val_onebest')
plt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)
plt.xticks(x_axis)
axis.spines['top'].set_visible(False)
axis.spines['right'].set_visible(False)
axis.set_xlabel('Epoch')
axis.set_ylabel('Cross Entropy')
plt.savefig(os.path.join(directory, 'plot.pdf'), bbox_inches='tight')
plt.close()
def _check_files(self):
input_status = check_file(self.input_filename)
is_pattern = self.get_plugin(Pattern)
if is_pattern is None:
if input_status == Exist.NOT_EXISTS:
print_error('input file "{0}" not exists'.format(
self.input_filename))
if input_status == Exist.DIR:
print_error('input file "{0}" is directory'.format(
self.input_filename))
output_status = check_file(self.output_filename)
if output_status is Exist.DIR:
print_error('output file "{0}" is directory'.format(
self.output_filename))
if output_status is Exist.EXISTS:
ans = print_confirm(
'output file "{0}" already exists. Overwrite? '.format(
self.output_filename))
if ans is False:
print('Cancelled')
exit(0)
def __init__(self,
pdb,
name,
charge=0,
working_directory="antechamber",
create_frcmod=True):
amberhome = get_amberhome()
utils.set_working_directory(working_directory)
pdb.to_filename('ligand.pdb')
antechamber_command = (
amberhome + "/bin/antechamber " +
"-i ligand.pdb -fi pdb -o {name}.prepc "
"-fo prepc -rn {name} -c bcc -nc {charge}".format(name=name,
charge=charge))
utils.run_in_shell(antechamber_command, 'antechamber.out')
utils.check_file(
name + '.prepc',
"Antechamber failed to generate {name}.prepc file".format(
name=name))
self.working_directory = os.getcwd()
if create_frcmod:
parmck_command = (
amberhome + "/bin/parmchk2 " +
"-i {name}.prepc -f prepc -o {name}.frcmod".format(name=name))
utils.run_in_shell(parmck_command, 'parmchk2.out')
# TODO: check for ATTN warnings
os.chdir('..')
def get_data(data_dir, name2index, file_num=None, im_width=128, im_height=128):
utils.check_file(data_dir)
depth = len(name2index)
a = 0
images, labels = [], []
for subdir in os.listdir(data_dir):
subdir_path = os.path.join(data_dir, subdir)
role_type = name2index[subdir]
for file in os.listdir(subdir_path):
file_path = os.path.join(subdir_path, file)
images.append(file_path)
labels.append(role_type)
package = np.vstack([images, labels]).transpose()
np.random.shuffle(package)
xs, ys = [], []
for i, (x, y) in enumerate(package):
if file_num is not None and i >= file_num:
break
im = Image.open(x).resize((im_width, im_height))
im = np.array(im) / 255.
xs.append(im)
ys.append([(1 if j == int(y) else 0) for j in range(depth)])
xs = np.array(xs)
ys = np.array(ys)
return xs, ys
def cal_sample_info(config):
data_ssr = os.path.join(config['data'], 'ssr_structure')
utils.check_file(data_ssr) # check if file data_ssr exists
ploidy = int(config['ploidy'])
samples = []
# A namedtuple to contain parameters
project_info = namedtuple('project_info', ['samples', 'loci', 'filestr'])
count = 0
for line in file(data_ssr):
count += 1
if count == 1:
header = line.strip().split()
else:
samp_name = line.strip().split()[0]
# samples.add(line.strip().split()[0])
if samp_name not in samples:
samples.append(samp_name)
header = filter(lambda x: x.strip(), header)
loci_num = len(header)
samp_num = len(samples)
INFO = project_info(samples=samp_num, loci=loci_num, filestr=data_ssr)
write_names(config, samples)
if (count - 1) % ploidy != 0:
#logging...
logging.warning('sample num incorrect in %s', data_ssr)
print '[WARNING] sample num incorrect in %s' % data_ssr
return INFO
def main():
global LOGGER
from logger import get_logger
from utils import check_file
# Init argparser
parser = argparse.ArgumentParser(prog="prep_CIRIquant")
parser.add_argument('--lib',
dest='lib',
metavar='FILE',
required=True,
help='library information')
parser.add_argument(
'--bsj',
dest='bsj',
metavar='FILE',
required=True,
help='circRNA expression matrix',
)
parser.add_argument(
'--gene',
dest='gene',
metavar='FILE',
required=True,
help='gene expression matrix',
)
parser.add_argument(
'--out',
dest='out',
metavar='FILE',
required=True,
help='output result of differential expression analysis',
)
# Optional parameters
args = parser.parse_args()
LOGGER = get_logger('CIRI_DE', None, False)
lib_path = os.path.dirname(os.path.split(
os.path.realpath(__file__))[0]) + '/libs'
os.environ['PATH'] = lib_path + ':' + os.environ['PATH']
os.chmod(lib_path + '/CIRI_DE.R', 0o755)
# Load GTF
lib_file = check_file(args.lib)
bsj_file = check_file(args.bsj)
gene_file = check_file(args.gene)
out_file = os.path.abspath(args.out)
LOGGER.info('Library information: {}'.format(lib_file))
LOGGER.info('circRNA expression matrix: {}'.format(bsj_file))
LOGGER.info('gene expression matrix: {}'.format(gene_file))
LOGGER.info('Output DE results: {}'.format(out_file))
de_cmd = 'CIRI_DE.R --lib={} --bsj={} --gene={} --out={}'.format(
lib_file, bsj_file, gene_file, out_file)
subprocess.call(de_cmd, shell=True)
LOGGER.info('Finished!')
def read_div_res(self):
divfile = os.path.join(self.out, 'diversity.txt')
sumfile = os.path.join(self.out, 'summ_stats.locus')
utils.check_file(divfile)
utils.check_file(sumfile)
param1 = self.file_to_dic(divfile, self.dic)
param2 = self.file_to_dic(sumfile, self.dic)
return param1 + param2
def testAnnotate(self):
'''
Check file
'''
print('#%s: Test annotate_fusion' % __name__)
test_file = 'data/test_annotated_junction.txt'
result_file = 'data/annotated_junction.txt'
check_file(test_file, result_file)
def testAnnotate(self):
'''
Check file
'''
print('#%s: Test annotate_fusion' % __name__)
test_file = 'data/test_annotated_junction.txt'
result_file = 'data/annotated_junction.txt'
check_file(test_file, result_file)
def testFix(self):
'''
Check file
'''
print('#%s: Test fix_fusion' % __name__)
test_file = 'data/test_circular_RNA.txt'
result_file = 'data/circular_RNA.txt'
check_file(test_file, result_file)
def testConvert(self):
'''
Check file
'''
print('#%s: Test convert_fusion' % __name__)
test_file = 'data/test_fusion_junction.txt'
result_file = 'data/fusion_junction.txt'
check_file(test_file, result_file)
def testConvert(self):
'''
Check file
'''
print('#%s: Test convert_fusion' % __name__)
test_file = 'data/test_fusion_junction.txt'
result_file = 'data/fusion_junction.txt'
check_file(test_file, result_file)
def testFix(self):
'''
Check file
'''
print('#%s: Test fix_fusion' % __name__)
test_file = 'data/test_circular_RNA.txt'
result_file = 'data/circular_RNA.txt'
check_file(test_file, result_file)
def load_model(load_states=True):
if utils.check_file(params):
model.load_params(params, ctx=ctxs)
logging.info("Loading parameters from : {}".format(params))
if load_states and utils.check_file(trainingfile):
trainer.set_learning_rate(float(
utils.read_kvstore(trainingfile)['lr']))
logging.info("Loading lr from : {}".format(trainingfile))
def test_star_parse():
'''
Test STAR parser
'''
print('#%s: Test parse STAR' % __name__)
options = {'-t': 'STAR', '--output': 'circ_out',
'<fusion>': 'Chimeric.out.junction'}
parse(options, command='CIRCexplorer2 parse (STAR)', name='parse')
assert os.path.isdir('circ_out'), 'No circ_out directory'
check_file('fusion_junction.bed', 'circ_out', 'STAR_out')
def test_segemehl_parse():
'''
Test segemehl parser
'''
print('#%s: Test parse segemehl' % __name__)
options = {'-t': 'segemehl', '--output': 'circ_out',
'<fusion>': 'splicesites.bed'}
parse(options, command='CIRCexplorer2 parse (segemehl)', name='parse')
assert os.path.isdir('circ_out'), 'No circ_out directory'
check_file('fusion_junction.bed', 'circ_out', 'segemehl_out')
def test_mapsplice_parse():
'''
Test MapSplice parser
'''
print('#%s: Test parse MapSplice' % __name__)
options = {'-t': 'MapSplice', '--output': 'circ_out',
'<fusion>': 'fusions_raw.txt'}
parse(options, command='CIRCexplorer2 parse (Mapsplice)', name='parse')
assert os.path.isdir('circ_out'), 'No circ_out directory'
check_file('fusion_junction.bed', 'circ_out', 'MapSplice_out')
def test_mapsplice_parse():
'''
Test MapSplice parser
'''
print('#%s: Test parse MapSplice' % __name__)
options = {'-t': 'MapSplice', '--output': 'circ_out',
'<fusion>': 'fusions_raw.txt', '--pe': False}
parse(options, command='CIRCexplorer2 parse (Mapsplice)', name='parse')
assert os.path.isdir('circ_out'), 'No circ_out directory'
check_file('fusion_junction.bed', 'circ_out', 'MapSplice_out')
def test_segemehl_parse():
'''
Test segemehl parser
'''
print('#%s: Test parse segemehl' % __name__)
options = {'-t': 'segemehl', '--output': 'circ_out',
'<fusion>': 'splicesites.bed', '--pe': False}
parse(options, command='CIRCexplorer2 parse (segemehl)', name='parse')
assert os.path.isdir('circ_out'), 'No circ_out directory'
check_file('fusion_junction.bed', 'circ_out', 'segemehl_out')
def test_tophat_fusion_parse():
'''
Test TopHat-Fusion parser
'''
print('#%s: Test parse TopHat-Fusion' % __name__)
options = {'-t': 'TopHat-Fusion', '--output': 'circ_out',
'<fusion>': 'tophat_fusion.bam', '--pe': False}
parse(options, command='CIRCexplorer2 parse (TopHat-Fusion)', name='parse')
assert os.path.isdir('circ_out'), 'No circ_out directory'
check_file('fusion_junction.bed', 'circ_out', 'TopHat_Fusion_out')
def test_bwa_parse():
'''
Test BWA parser
'''
print('#%s: Test parse BWA' % __name__)
options = {'-t': 'BWA', '--output': 'circ_out',
'<fusion>': 'RNA_seq_bwa.sam', '--pe': False}
parse(options, command='CIRCexplorer2 parse (BWA)', name='parse')
assert os.path.isdir('circ_out'), 'No circ_out directory'
check_file('fusion_junction.bed', 'circ_out', 'BWA_out')
def test_star_parse():
'''
Test STAR parser
'''
print('#%s: Test parse STAR' % __name__)
options = {'-t': 'STAR', '--output': 'circ_out',
'<fusion>': 'Chimeric.out.junction', '--pe': False}
parse(options, command='CIRCexplorer2 parse (STAR)', name='parse')
assert os.path.isdir('circ_out'), 'No circ_out directory'
check_file('fusion_junction.bed', 'circ_out', 'STAR_out')
def get_model(params):
if not params.resume:
params.weights = check_file(params.weights)
ckpt = None
model = EfficientNet.from_pretrained('efficientnet-b{}'.format(params.model_level), weights_path=params.weights, num_classes=params.num_classes, image_size=params.img_size)
else:
params.resume = check_file(params.resume)
ckpt = torch.load(params.resume)
model = EfficientNet.from_name('efficientnet-b{}'.format(params.model_level), num_classes=params.num_classes, image_size=params.img_size)
model.load_state_dict(ckpt['network'], strict=False)
return model, ckpt
def testAnnotate(self):
'''
Check files in annotate directory
'''
print('#%s: Test annotate' % __name__)
result_path = 'data/annotate'
assert os.path.isdir(result_path), 'No annotate directory'
# check annotated_fusion.txt file
check_file('annotated_fusion.txt', result_path, 'data/annotate_result')
# check circ_fusion.txt file
check_file('circ_fusion.txt', result_path, 'data/annotate_result')
def parse_args(cls):
if len(sys.argv) != 3:
cls.usage()
infile = sys.argv[1]
regexp = sys.argv[2]
utils.check_file(infile)
utils.check_regexp(regexp)
cls.print_script_params(infile, regexp)
return (infile, regexp)
def testAnnotate(self):
'''
Check files in annotate directory
'''
print('#%s: Test annotate' % __name__)
result_path = 'data/annotate'
assert os.path.isdir(result_path), 'No annotate directory'
# check annotated_fusion.txt file
check_file('annotated_fusion.txt', result_path, 'data/annotate_result')
# check circ_fusion.txt file
check_file('circ_fusion.txt', result_path, 'data/annotate_result')
def __init__(self, path, ignore_graphemes=False):
"""Confusion network constructor."""
self.path = path
self.cn_arcs = []
self.num_sets = None
self.num_arcs = []
self.has_graphemes = False
self.ignore_graphemes = ignore_graphemes
utils.check_file(self.path)
self.name = self.path.split('/')[-1].strip('.scf.gz')
self.load()
def tree_anno_ggtree(treefile):
"""
Tree ornamentation with ggtree
"""
utils.check_file(treefile)
grdevices = importr('grDevices')
Gtree = importr('ggtree')
ape = importr('ape')
raw_tree = ape.read_tree(treefile)
grdevices.pdf(file="output.pdf")
Gtree.ggtree(raw_tree)
grdevices.dev_off()
return None
def testDenovo(self):
'''
Check files in denovo directory
'''
print('#%s: Test denovo' % __name__)
result_path = 'data/denovo'
assert os.path.isdir(result_path), 'No denovo directory'
# check files in denovo directory
file_list = ['annotated_fusion.txt', 'circ_fusion.txt',
'annotated_circ.txt', 'novel_circ.txt',
'all_exon_info.txt']
for f in file_list:
check_file(f, result_path, 'data/denovo_result')
def setup(opt, checkpoint):
"""Setup criterion."""
# Load criterion if checkpoint is provided, create one otherwise
if checkpoint is not None:
criterion_path = os.path.join(opt.resume, checkpoint['criterion_file'])
utils.check_file(criterion_path)
print("".ljust(4) + "=> Resuming criterion from %s" % criterion_path)
criterion = torch.load(criterion_path)
else:
print("".ljust(4) + "=> Creating new criterion")
criterion = create_criterion()
return criterion
def parse_args(cls):
if len(sys.argv) != 4:
cls.usage()
infile = sys.argv[1]
resdir = sys.argv[2]
regexp = sys.argv[3]
utils.check_file(infile)
utils.create_dir(resdir)
utils.check_regexp(regexp)
cls.print_script_params(infile, resdir, regexp)
return (infile, resdir, regexp)
def resample_dataset(opt, split):
"""Resampling from the entire dataset."""
data_file = os.path.join(opt.data, opt.dataset, '%s.txt' %split)
utils.check_file(data_file)
stats_file = os.path.join(opt.data, opt.dataset, 'stats.npz')
utils.check_file(stats_file)
tgt_dir = os.path.join(opt.data, opt.dataset, 'target')
utils.check_dir(tgt_dir)
dataset = LatticeDataset(data_file, stats_file, tgt_dir, opt.trainPctg)
loader = DataLoader(dataset=dataset, batch_size=opt.batchSize,
shuffle=opt.shuffle, collate_fn=collate_fn,
num_workers=opt.nThreads)
return loader
def testDenovo(self):
'''
Check files in denovo directory
'''
print('#%s: Test denovo' % __name__)
result_path = 'data/denovo'
assert os.path.isdir(result_path), 'No denovo directory'
# check files in denovo directory
file_list = [
'annotated_fusion.txt', 'circ_fusion.txt', 'annotated_circ.txt',
'novel_circ.txt', 'all_exon_info.txt'
]
for f in file_list:
check_file(f, result_path, 'data/denovo_result')
def build_user_embeddings_store(self, filename):
user_embeddings = {'ue_store_filename': filename}
check_file(self.path_data, filename)
print("User embeddings process began:", str(datetime.now()))
start_time = time.monotonic()
with open(self.path_data + filename) as f:
for line in f:
user = list(json.loads(line).keys())[0]
user_vec = self.get_user_embedding(user)
user_embeddings[user] = user_vec
end_time = time.monotonic()
print("User embeddings process ended:",
str(datetime.now()) + ".", "Time taken:",
timedelta(seconds=end_time - start_time))
return user_embeddings
def export_csv(self):
info = {}
# Check if template exist before
if not check_file(self.template, "xml"):
print "Template doesnot exist, or invalid template. Check if " "it has a .xml extenion"
return 0
try:
data = csv.reader(open(self.file_name, "rb"))
except:
print "Data doesnt exist"
try:
DOMAIN_URL = CONST.COMMCARE_URL
SUBMIT_CASEXML = CONST.SUBMIT_TO_COMMCARE
except:
SUBMIT_CASEXML = False
DOMAIN_URL = u""
# Remove Header
header = data.next()
info = {}
c = 1
# Loop through each row and get data
for x in data:
for label, value in zip(header, x):
info[label] = value.strip()
print "Processing %d case: %s" % (c, info["hid"])
form = CaseXMLInterface(info, self.template)
save_casexmlform(form)
if SUBMIT_CASEXML:
transmit_form(form, DOMAIN_URL)
c += 1
print "Processed %d cases" % c
def install_agent(self, package_path):
checksum = utils.get_checksum(package_path, self.get_checksum_dir())
if not utils.check_file(package_path, checksum):
LOG.debug("check downloaded package failed, removing %s", package_path)
os.remove(package_path)
utils.remove_checksum(package_path, self.get_checksum_dir())
return None
dirname = utils.get_file_name_without_extension(package_path)
agent_dir = '%s\%s' % (self.get_install_dir(), dirname)
tmp_dir = '%s\%s' % (self.get_tmp_dir(), dirname)
try:
if utils.unzip(package_path, tmp_dir):
tmp_name = os.listdir(tmp_dir)[0]
tmp_path = '%s\%s' % (tmp_dir, tmp_name)
ren_path = '%s\%s' % (tmp_dir, dirname)
if os.system('ren "%s" "%s"' % (tmp_path, dirname)) != 0:
return None
if os.system('move "%s" "%s"' % (ren_path, self.get_install_dir())) != 0:
return None
else:
return None
finally:
if os.path.isdir(tmp_dir):
os.system('rd /s /q "%s"' % tmp_dir)
installer = Installer(agent_dir, self.recycle_queue)
installer.install()
return agent_dir
def __init__(self, config, section, root=None, engine=None, dmd=True):
#a mock-friendly datetime
self.dt = DateShift()
if section not in config.sections():
raise ConfigException('There is not such job')
self.logger = logging.getLogger(__name__)
if engine is None:
engine = setts.engine
self.cf = getattr(config, section)
self.section = section
self.args['jobname'] = section
check_file(self.cf.dar_bin, ConfigException)
if root is not None:
self.args['path'] = root
else:
self.args['path'] = self.cf.root
for opt in self.cf.options():
self.args[opt] = getattr(self.cf, opt)
create_all(engine)
Sess = sessionmaker(bind=engine)
self.sess = Sess(bind=engine)
#init dardrive backup types
for btype, ext in dardrive_types:
setattr(self, btype, self.get_or_create(BackupType, name=btype))
for each_section in config.sections():
self.get_or_create(Job, name=each_section)
self.sess.commit()
self.Job = self.get_or_create(Job, name=section)
self.dmd = os.path.expanduser('~/.dardrive/dmd/%s.dmd' % section)
if not os.path.exists(self.dmd) and dmd:
args = {'dar_manager': self.cf.dar_manager_bin,
'dmd_file': self.dmd}
run = self.run_command('%(dar_manager)s -C %(dmd_file)s', args)
if run[0].returncode > 0:
raise BackupException
def add_seeds_from_file(self, list_file):
# line format in file should be: name, file_location
log.info('Importing seed volumes from file: {}'.format(list_file))
f = open(check_file(list_file), 'r')
try:
for entry in f:
fields = entry.strip().split(',')
if len(fields) > 1:
name,filename = fields
seed = FCSeed(self.dir_seeds, name, filename)
self.add_seed(seed)
except:
log.error('Problem importing seed volumes from list file {}'.format(list_file))
self.exit()
finally:
f.close()
def __init__(self, file_name, defaults={}):
#SafeConfigParser defalts will apply to all sections, which actually,
#sucks. We're going to manually set the defaults for the global_section
#is unset.
self.logger = logging.getLogger(__name__)
file_name = check_file(file_name, InitException)
self.cp = SafeConfigParser()
if not self.cp.has_section(self.global_section):
self.cp.add_section(self.global_section)
for key in defaults.keys():
if not self.cp.has_option(self.global_section, key):
self.cp.set(self.global_section, key, defaults[key][0])
self.cp.read(os.path.expanduser(file_name))
self.file_name = file_name
self._defaults = defaults
def __init__(self):
""" Get arguments """
args = sys.argv
if len(args) >= 3:
try:
self.file_name = args[1]
self.template = args[2]
except:
self.file_name = ""
self.template = ""
if not check_file(self.file_name, "csv"):
print "Invalid file location or file does not exist." "Check if it has a .csv extenion"
else:
self.export_csv()
else:
self.help()
def create_seeds_from_file(output_dir, list_file, radius=None):
# creates spherical ROIs from x,y,z coordinates in file
# * each line should be: name, x, y, z, radius
# * radius is optional
log.info('Creating seeds from MNI coords specified in list file: {}'.format(list_file))
# open the file up and loop through each row
f = open(check_file(list_file), 'r')
try:
for i, entry in enumerate(f):
fields = entry.strip().split(',')
if len(fields) < 4:
log.warning('line #{} in seed file does not contain enough information, skipping.'.format(i+1))
continue
# name is first field
n = fields[0]
# replace spaces with underscore
n = re.sub(' ','_',n).lower()
# following 3 fields are coords, convert to float
x,y,z = [float(j.strip()) for j in fields[1:4]]
# radius is optional, so we need to check that it was specified somewhere
if radius is None:
if len(fields) == 5:
seed_radius = float(fields[4].strip())
else:
log.error('When creating seed from a file, you must specify the radius in the file itself or on the command line')
sys.exit()
else:
seed_radius = radius
# create seed
seed = FCSeed(output_dir, n)
seed.create(x,y,z,seed_radius)
yield seed
except:
log.error('Problem importing seed coordinates from file {}'.format(list_file))
sys.exit()
finally:
f.close()
def add_to_dmd(self, catalog_id):
catalog_path = os.path.join(self.cf.catalog_store, self.section,
catalog_id)
archive_path = os.path.join(self.cf.archive_store, self.section,
catalog_id)
#pass on non dar bkps..
try:
ck = check_file(catalog_path + ".1.dar")
self.logger.debug('Updating the recovery catalog.')
ctpl = "%(dar_manager)s -B %(dmd_file)s -A %(catalog)s"
cargs = {'dar_manager': self.cf.dar_manager_bin,
'dmd_file': self.dmd,
'catalog': catalog_path}
crun = self.run_command(ctpl, cargs)
dmds = self.load_dmd()
#change the dar path so that the dmd points to the real backup
dmd = dmds[catalog_id]
ntpl = "%(dar_manager)s -B %(dmd_file)s -p %(id)s %(path)s"
archivestore = os.path.join(self.cf.archive_store, self.section)
cargs.update({'id': dmd.num, 'path': archivestore})
nrun = self.run_command(ntpl, cargs)
except IOError:
self.logger.debug("Extracting catalog %s from %s.." %
(catalog_path, archive_path))
_c_dir = os.path.dirname(catalog_path)
if not os.path.exists(_c_dir):
os.makedirs(_c_dir)
ntpl = "%(dar)s -C %(catalog_path)s -A %(archive_path)s"
args = {'dar': self.cf.dar_bin,
'catalog_path': catalog_path,
'archive_path': archive_path}
run = self.run_command(ntpl, args)
if run[0].returncode == 0:
self.add_to_dmd(catalog_id)
else:
raise CatalogException(
'Couldn\'t extract catalog from %s.' % archive_path)
def export_csv(self):
info = {}
#Check if template exist before
if not check_file(self.template, "xml"):
print "Template doesnot exist, or invalid template. Check if " \
"it has a .xml extenion"
return 0
try:
data = csv.reader(open(self.file_name, 'rb'))
except:
print "Data doesnt exist"
#Remove Header
header = data.next()
info = {}
#Loop through each row and get data
for x in data:
for label, value in zip(header, x):
info[label] = value
form = CaseXMLInterface(info, self.template)
save_casexmlform(form)
def parse_cmd(argv, flags, params, optdir):
get_opt(argv, flags, params, optdir)
if optdir['list']:
if not optdir['dbname']:
raise getopt.GetoptError("--dbname required")
return 0
filename = optdir.get('filename', None)
date = optdir.get('date', None)
ftype = optdir.get('ftype', None)
# check required options
if not filename and not ftype:
raise getopt.GetoptError("--file or --type required")
if filename and ftype:
raise getopt.GetoptError("--file and --type are mutually exclusive")
if ftype and not date:
if ftype.startswith('ST_Z'):
date_list = [yesterday()]
else:
date_list = [last_day_of_month()]
elif ftype and date and ':' in date:
if ftype.startswith('ST_Z'):
date_list = get_date_interval(date)
else:
raise getopt.GetoptError("Date interval is valid only for '--type ST_ZXXX'")
else:
date_list = [date]
if optdir['overwrite'] and optdir.get('append', False):
raise getopt.GetoptError("--append and --overwrite are mutually exclusive")
if optdir['layer']:
optdir['layer'] = optdir['layer'].split(',')
if filename:
# is file a valid VFR file
filename = check_file(filename)
else: # --date & --type
flist = []
base_url = "http://vdp.cuzk.cz/vymenny_format/"
if ftype != 'ST_UVOH':
base_url += "soucasna/"
else:
base_url += "specialni/"
for d in date_list:
fname = "%s_%s.xml.gz" % (d, ftype)
url = base_url + fname
if os.path.exists(fname):
# use existing file
flist.append(fname)
else:
# download file
flist.append(download_vfr(url))
if not flist:
raise getopt.GetoptError("Empty date range")
filename = os.linesep.join(flist)
if not filename:
raise getopt.GetoptError("Invalid input file")
return filename
def write_settings_cfg(self, pobj):
'''Creates the main system settings configuration file for BDS'''
self.settings_cfg = os.path.join(pobj.files['config_path'], '{0.name}.settings.cfg'.format(pobj))
with open(self.settings_cfg, 'wb') as o:
# General
o.write('# GENERAL SETTINGS\n')
o.write('min_map_q = {0}\n'.format(self.min_map_q))
o.write('min_clip_length = {0.min_length}\n'.format(self))
if self.system['module_source']: o.write('module_source = {0}\n'.format(self.system['module_source']))
o.write('\n')
# Reference
o.write('# REFERENCE SETTINGS\n')
o.write('reference = {0}\n'.format(pobj.assembly['referenceseq']))
o.write('reference_name = {0}\n'.format(pobj.assembly['refname']))
o.write('exons_bed = {0}\n'.format(pobj.assembly['exons_bed']))
o.write('reference_dict = {0}\n'.format(pobj.assembly['sequence_dictionary']))
o.write('known_indel = {0}\n'.format(pobj.assembly['knownindel']))
o.write('knowndb = {0}\n'.format(pobj.assembly['knowndb']))
o.write('cosmic = {0}\n'.format(pobj.assembly['cosmic']))
if pobj.assembly['novoalign_index']:
utils.check_file(pobj.assembly['novoalign_index'], "Novoalign Index")
o.write('novoalign_index = {0}\n'.format(pobj.assembly['novoalign_index']))
if self.target_bed:
utils.check_file(self.target_bed, "target bed file")
o.write('target_bed = {0.target_bed}\n'.format(self))
o.write('\n')
# Preprocessing
self.__fastqc_settings(o)
self.__cutadapt_settings(o)
self.__seqprep_settings(o)
# Sam and bedtools
self.__sam_bed_tools_settings(o)
# Alignments
if 'bwa_aln' in self.aln_list: self.__bwa_aln_settings(o)
if 'bwa_mem' in self.aln_list: self.__bwa_mem_settings(o)
if 'novoalign' in self.aln_list: self.__novoalign_settings(o)
self.__picard_settings(o)
# GATK
self.__gatk_settings(o)
# Mutation calling
if 'mutect' in self.smd_list: self.__mutect_settings(o)
if 'shimmer' in self.smd_list: self.__shimmer_settings(o)
if 'sniper' in self.smd_list: self.__sniper_settings(o)
if 'strelka' in self.smd_list: self.__strelka_settings(o)
if 'varscan' in self.smd_list: self.__varscan_settings(o)
if 'virmid' in self.smd_list: self.__virmid_settings(o)
# Annovar
self.__annovar_settings(o)
# Others
o.write('# OTHER SETTINGS\n')
if self.system['python']['use_module']:
o.write('python_modname = {0}\n'.format(self.system['python']['module']['name']))
o.write('python_exe = {0}\n'.format(self.system['python']['module']['exe']))
else:
o.write('python_exe = {0}\n'.format(self.system['python']['exe']))
if self.system['java']['use_module']:
o.write('java_modname = {0}\n'.format(self.system['java']['module']['name']))
o.write('somatic_filter_exe = {0}\n'.format(os.path.join(self.PATH, 'ExScaliburSMD-filter.py')))
utils.check_file(os.path.join(self.PATH, 'ExScaliburSMD-filter.py'), 'ExScaliburSMD-filter.py')
o.write('build_reporter_exe = {0}\n'.format(os.path.join(self.PATH, 'scripts/BuildReporterFiles.py')))
utils.check_file(os.path.join(self.PATH, 'scripts/BuildReporterFiles.py'), 'BuildReporterFiles.py')
o.write('fix_intersect_exe = {0}\n'.format(os.path.join(self.PATH, 'scripts/FixBedOutput.sh')))
utils.check_file(os.path.join(self.PATH, 'scripts/FixBedOutput.sh'), 'FixBedOutput.sh')
o.write('split_bed_exe = {0}\n\n'.format(os.path.join(self.PATH, 'scripts/CollapseSplitBed.py')))
utils.check_file(os.path.join(self.PATH, 'scripts/CollapseSplitBed.py'), 'CollapseSplitBed.py')
IP = '127.0.0.1' PORT = 6677 DGRAM_FORMAT = '50s50s50s200s' CMD_FORMAT = '50s50s50s' BASE_DIR = './demo' DEBUG = True PID_FILE = os.path.join(BASE_DIR, 'pychat_server.pid') LOG_FILE = os.path.join(BASE_DIR, 'pychat_server.log') # const USER_PATH = os.path.join(BASE_DIR, 'user') MSG_PATH = os.path.join(BASE_DIR, 'msg') FILE_PATH = os.path.join(BASE_DIR, 'file') history_msg_file = os.path.join(MSG_PATH, 'history.pk') offline_msg_file = os.path.join(MSG_PATH, 'offline.pk') user_file = os.path.join(USER_PATH, 'user.pk') friend_file = os.path.join(USER_PATH, 'friends.pk') file_info = os.path.join(FILE_PATH, 'file_info.txt') check_dir(USER_PATH) check_dir(MSG_PATH) check_dir(FILE_PATH) check_file(file_info)