def parse_args(args):
parser = ArgumentParser(prog="derl",
formatter_class=lambda prog: HelpFormatter(
prog, max_help_position=35, width=90),
description="Dead URL searching utility")
parser.add_argument(
action="store", # default behavior
dest="directory",
help="directory for looking for dead URLs",
metavar="directory")
parser.add_argument("-c",
"--context",
action="store_true",
dest="context",
help="showing up to 3 lines of context")
parser.add_argument("-d",
"--dispatch",
action="store_true",
dest="dispatch",
help="dispatching HTTP requests for every found URL")
parser.add_argument("-r",
"--retry",
action="store",
default=_DEFAULT_RETRY,
dest="retry",
help="set how often to retry a request (default is 3)",
type=int)
parser.add_argument(
"-t",
"--timeout",
action="store",
default=_DEFAULT_TIMEOUT,
dest="timeout",
help="set timeout for requests in seconds (default is 10)",
type=int)
parser.add_argument("--version",
action="version",
version="derl {ver}".format(ver=__version__))
parser.add_argument(
"-v",
"--verbose",
action="store_const",
const=INFO,
dest="loglevel",
help="set loglevel to INFO",
)
parser.add_argument("-vv",
"--very-verbose",
action="store_const",
const=DEBUG,
dest="loglevel",
help="set loglevel to DEBUG")
return parser.parse_args(args)
def populate_args(parser):
"""
parse option from call
"""
parser.formatter_class = lambda prog: HelpFormatter(
prog, width=95, max_help_position=27)
oblopt = parser.add_argument_group('Required options')
glopts = parser.add_argument_group('General options')
rfiltr = parser.add_argument_group('Read filtering options')
normpt = parser.add_argument_group('Normalization options')
outopt = parser.add_argument_group('Output options')
oblopt.add_argument('-w',
'--workdir',
dest='workdir',
metavar="PATH",
action='store',
default=None,
type=str,
required=True,
help='''path to working directory (generated with the
tool tadbit mapper)''')
oblopt.add_argument('-r',
'--resolution',
dest='reso',
metavar="INT",
action='store',
default=None,
type=int,
required=True,
help='''resolution at which to output matrices''')
oblopt.add_argument('--format',
dest='format',
action='store',
default='text',
choices=['text', 'matrix', 'cooler', 'hic'],
help='''[%(default)s] can be any of [%(choices)s]''')
oblopt.add_argument('-o',
'--output',
dest='out',
metavar="STR",
action='store',
default=None,
type=str,
required=True,
help='''path to output file''')
glopts.add_argument('--bam',
dest='bam',
metavar="PATH",
action='store',
default=None,
type=str,
help='''path to a TADbit-generated BAM file with
all reads (other wise the tool will guess from the
working directory database)''')
glopts.add_argument(
'-j',
'--jobid',
dest='jobid',
metavar="INT",
action='store',
default=None,
type=int,
help='''Use as input data generated by a job with a given
jobid. Use tadbit describe to find out which.''')
glopts.add_argument('--force',
dest='force',
action='store_true',
default=False,
help='overwrite previously run job')
glopts.add_argument('-q',
'--quiet',
dest='quiet',
action='store_true',
default=False,
help='remove all messages')
glopts.add_argument(
'--tmpdb',
dest='tmpdb',
action='store',
default=None,
metavar='PATH',
type=str,
help='''if provided uses this directory to manipulate the
database''')
glopts.add_argument('--nchunks',
dest='nchunks',
action='store',
default=100,
type=int,
help='''maximum number of chunks into which to
cut the BAM''')
glopts.add_argument("-C",
"--cpus",
dest="cpus",
type=int,
default=cpu_count(),
help='''[%(default)s] Maximum
number of CPU cores available in the execution host.
If higher than 1, tasks with multi-threading
capabilities will enabled (if 0 all available)
cores will be used''')
glopts.add_argument(
'--chr_name',
dest='chr_name',
metavar="STR",
nargs='+',
default=None,
type=str,
help='''[fasta header] chromosome name(s). Order of chromosomes
in the output matrices.''')
glopts.add_argument(
'--juicerjar',
dest='juicerjar',
metavar="PATH",
action='store',
default=None,
type=str,
help='''path to the juicer tools jar file needed to export
matrices to hic format (check https://github.com/aidenlab/juicer/wiki/Download).
Note that you also need java available in the path.''')
outopt.add_argument('--rownames',
dest='row_names',
action='store_true',
default=False,
help='''To store row names in the output text matrix.
WARNING: when non-matrix, results in two extra columns'''
)
outopt.add_argument('-c',
'--coord',
dest='coord1',
metavar='',
default=None,
help='''Coordinate of the region to
retrieve. By default all genome, arguments can be
either one chromosome name, or the coordinate in
the form: "-c chr3:110000000-120000000"''')
outopt.add_argument('-c2',
'--coord2',
dest='coord2',
metavar='',
default=None,
help='''Coordinate of a second region to
retrieve the matrix in the intersection with the first
region.''')
normpt.add_argument('--biases',
dest='biases',
metavar="PATH",
action='store',
default=None,
type=str,
help='''path to file with pre-calculated biases by
columns''')
normpt.add_argument('--norm',
dest='norm',
action='store_true',
default=False,
help='export normalized matrix')
rfiltr.add_argument(
'-F',
'--filter',
dest='filter',
nargs='+',
type=int,
metavar='INT',
default=[1, 2, 3, 4, 6, 7, 9, 10],
choices=list(range(0, 11)),
help=(
"""[%(default)s] Use filters to define a set of
valid pair of reads e.g.:
'--filter 1 2 3 4 8 9 10'. Where these numbers """ +
"correspond to: 0: nothing, %s" %
(', '.join(['%2d: %15s' % (k, MASKED[k]['name'])
for k in MASKED]))))
def populate_args(parser):
"""
parse option from call
"""
parser.formatter_class=lambda prog: HelpFormatter(prog, width=95,
max_help_position=27)
glopts = parser.add_argument_group('General options')
glopts.add_argument('-w', '--workdir', dest='workdir', metavar="PATH",
action='store', default=None, type=str,
help='''path to a new output folder''')
glopts.add_argument('-w1', '--workdir1', dest='workdir1', metavar="PATH",
action='store', default=None, type=str,
help='''path to working directory of the first HiC data
sample to merge''')
glopts.add_argument('-w2', '--workdir2', dest='workdir2', metavar="PATH",
action='store', default=None, type=str,
help='''path to working directory of the second HiC data
sample to merge''')
glopts.add_argument('--bam1', dest='bam1', metavar="PATH",
action='store', default=None, type=str,
help='''path to the first TADbit-generated BAM file with
all reads (other wise the tool will guess from the
working directory database)''')
glopts.add_argument('--noX', action='store_true', help='no display server (X screen)')
glopts.add_argument('--bam2', dest='bam2', metavar="PATH",
action='store', default=None, type=str,
help='''path to the second TADbit-generated BAM file with
all reads (other wise the tool will guess from the
working directory database)''')
glopts.add_argument("-C", "--cpus", dest="cpus", type=int,
default=cpu_count(), help='''[%(default)s] Maximum number of CPU
cores available in the execution host. If higher
than 1, tasks with multi-threading
capabilities will enabled (if 0 all available)
cores will be used''')
glopts.add_argument('-r', '--resolution', dest='reso', metavar="INT",
action='store', default=None, type=int,
help='''resolution at which to do the comparison,
and generate the matrices.''')
glopts.add_argument('--skip_comparison', dest='skip_comparison',
action='store_true', default=False,
help='''skip the comparison between replicates (faster).
Comparisons are performed at 3 levels 1- comparing first
diagonals of each experiment (and generating SCC score
and standard deviation see
https://doi.org/10.1101/gr.220640.117) 2- Comparing the
first eigenvectors of input experiments 3- Generates
reproducibility score using function from
https://doi.org/10.1093/bioinformatics/btx152''')
glopts.add_argument('--skip_merge', dest='skip_merge',
action='store_true', default=False,
help='''skip the merge of replicates (faster).''')
glopts.add_argument('--save', dest='save', metavar="STR",
action='store', default='genome', nargs='+', type=str,
choices=['genome', 'chromosomes'],
help='''[%(default)s] save genomic or chromosomic matrix.''')
glopts.add_argument('--jobid1', dest='jobid1', metavar="INT",
action='store', default=None, type=int,
help='''Use as input data generated by a job with a given
jobid. Use tadbit describe to find out which.''')
glopts.add_argument('--jobid2', dest='jobid2', metavar="INT",
action='store', default=None, type=int,
help='''Use as input data generated by a job with a given
jobid. Use tadbit describe to find out which.''')
glopts.add_argument('--force', dest='force', action='store_true',
default=False,
help='overwrite previously run job')
glopts.add_argument('--norm', dest='norm', action='store_true',
default=False,
help='compare normalized matrices')
glopts.add_argument('--biases1', dest='biases1', metavar="PATH",
action='store', default=None, type=str,
help='''path to file with precalculated biases by
columns''')
glopts.add_argument('--biases2', dest='biases2', metavar="PATH",
action='store', default=None, type=str,
help='''path to file with precalculated biases by
columns''')
glopts.add_argument('--filter', dest='filter', nargs='+',
type=int, metavar='INT', default=[1, 2, 3, 4, 6, 7, 9, 10],
choices = list(range(1, 11)),
help=("""[%(default)s] Use filters to define a set os
valid pair of reads e.g.:
'--apply 1 2 3 4 8 9 10'. Where these numbers""" +
"correspond to: %s" % (', '.join(
['%2d: %15s' % (k, MASKED[k]['name'])
for k in MASKED]))))
glopts.add_argument('--samtools', dest='samtools', metavar="PATH",
action='store', default='samtools', type=str,
help='''path samtools binary''')
glopts.add_argument('--tmpdb', dest='tmpdb', action='store', default=None,
metavar='PATH', type=str,
help='''if provided uses this directory to manipulate the
database''')
parser.add_argument_group(glopts)
def populate_args(parser):
"""
parse option from call
"""
parser.formatter_class = lambda prog: HelpFormatter(
prog, width=95, max_help_position=27)
glopts = parser.add_argument_group('General options')
cmopts = parser.add_argument_group('Compartment calling options')
tdopts = parser.add_argument_group('TAD calling options')
glopts.add_argument('-w',
'--workdir',
dest='workdir',
metavar="PATH",
action='store',
default=None,
type=str,
required=True,
help='''path to working directory (generated with the
tool tadbit mapper)''')
glopts.add_argument(
'--tmpdb',
dest='tmpdb',
action='store',
default=None,
metavar='PATH',
type=str,
help='''if provided uses this directory to manipulate the
database''')
glopts.add_argument('--nosql',
dest='nosql',
action='store_true',
default=False,
help='do not load/store data from/in sqlite database')
glopts.add_argument(
'--all_bins',
dest='all_bins',
action='store_true',
default=False,
help='skip the filtering of bins for detection of TADs')
glopts.add_argument('--mreads',
dest='mreads',
metavar="PATH",
action='store',
default=None,
type=str,
help='''path valid-pairs file (TADbit BAM format)''')
glopts.add_argument('--biases',
dest='biases',
metavar="PATH",
action='store',
default=None,
type=str,
help='''path to file with precalculated biases by
columns''')
glopts.add_argument('-r',
'--resolution',
dest='reso',
metavar="INT",
action='store',
default=None,
type=int,
required=True,
help='''resolution at which to output matrices''')
glopts.add_argument('--norm_matrix',
dest='norm_matrix',
metavar="PATH",
action='store',
default=None,
type=str,
help='''path to a matrix file with normalized read
counts''')
glopts.add_argument('--raw_matrix',
dest='raw_matrix',
metavar="PATH",
action='store',
default=None,
type=str,
help='''path to a matrix file with raw read
counts''')
glopts.add_argument(
'-F',
'--filter',
dest='filter',
nargs='+',
type=int,
metavar='INT',
default=[1, 2, 3, 4, 6, 7, 9, 10],
choices=range(1, 11),
help=(
"""[%(default)s] Use filters to define a set os
valid pair of reads e.g.:
'--apply 1 2 3 4 8 9 10'. Where these numbers""" +
"correspond to: %s" %
(', '.join(['%2d: %15s' % (k, MASKED[k]['name'])
for k in MASKED]))))
cmopts.add_argument('--rich_in_A',
dest='rich_in_A',
metavar="PATH",
action='store',
default=None,
type=str,
help='''path to a BAD or bedGraph file with list of
protein coding gene or other active epigenetic mark,
to be used to label compartments instead of using
the relative interaction count.''')
cmopts.add_argument(
'--fasta',
dest='fasta',
action='store',
default=None,
metavar='PATH',
type=str,
help='''Path to fasta file with genome sequence, to compute
GC content and use it to label compartments''')
cmopts.add_argument(
'--savecorr',
dest='savecorr',
action='store_true',
default=False,
help='''Save correlation matrix used to predict compartments.''')
cmopts.add_argument(
'--fix_corr_scale',
dest='fix_corr_scale',
action='store_true',
default=False,
help='''Correlation matrix plot scaled between correlation 1 and -1
instead of maximum observed values.''')
cmopts.add_argument('--format',
dest='format',
action='store',
default='png',
type=str,
help='[%(default)s] file format for figures')
cmopts.add_argument('--n_evs',
dest='n_evs',
metavar="INT",
action='store',
default=3,
type=int,
help='''[%(default)s] Number of eigenvectors to
store. if "-1" all eigenvectors will be calculated''')
cmopts.add_argument('--ev_index',
dest='ev_index',
nargs='+',
type=int,
metavar='INT',
default=None,
help="""list of indexes of eigenvectors capturing
compartments signal (one index per chromosome, in the
same order as chromosomes in fasta file). Example
picking the first eigenvector for all chromosomes but
for chromosome 3:
'--ev_index 1 1 2 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1"""
)
glopts.add_argument(
'--only_compartments',
dest='only_compartments',
action='store_true',
default=False,
help='''search A/B compartments using first eigen vector
of the correlation matrix''')
glopts.add_argument('--only_tads',
dest='only_tads',
action='store_true',
default=False,
help='''search TAD boundaries break-point detection
algorithm''')
glopts.add_argument('-v',
'--verbose',
dest='verbose',
action='store_true',
default=False,
help='''print more messages''')
glopts.add_argument(
'-j',
'--jobid',
dest='jobid',
metavar="INT",
action='store',
default=None,
type=int,
help='''Use as input data generated by a job with a given
jobid. Use tadbit describe to find out which.''')
glopts.add_argument('-c',
'--chromosomes',
dest='crms',
metavar="STR",
action='store',
default=None,
type=str,
nargs='+',
help='''Name of the chromosomes on which to search
for TADs or compartments.''')
tdopts.add_argument(
'--max_tad_size',
dest='max_tad_size',
metavar="INT",
action='store',
default=None,
type=int,
help='''an integer defining the maximum size of TAD. Default
defines it as the number of rows/columns''')
glopts.add_argument("-C",
"--cpu",
dest="cpus",
type=int,
default=cpu_count(),
help='''[%(default)s] Maximum number of CPU
cores available in the execution host. If higher
than 1, tasks with multi-threading
capabilities will enabled (if 0 all available)
cores will be used''')
glopts.add_argument('--force',
dest='force',
action='store_true',
default=False,
help='overwrite previously run job')
parser.add_argument_group(glopts)
def populate_args(parser):
"""
parse option from call
"""
parser.formatter_class = lambda prog: HelpFormatter(
prog, width=95, max_help_position=27)
glopts = parser.add_argument_group('General options')
mapper = parser.add_argument_group('Mapping options')
descro = parser.add_argument_group('Descriptive, optional arguments')
# glopts.add_argument('--cfg', dest='cfg', metavar="PATH", action='store',
# default=None, type=str,
# help='path to a configuration file with predefined ' +
# 'parameters')
glopts.add_argument(
'--skip_mapping',
dest='skip_mapping',
action='store_true',
default=False,
help='generate a Hi-C specific quality plot from FASTQ and exits')
glopts.add_argument('-w',
'--workdir',
dest='workdir',
metavar="PATH",
action='store',
default=None,
type=str,
required=True,
help='path to an output folder.')
glopts.add_argument('--fastq',
dest='fastq',
metavar="PATH",
action='store',
default=None,
type=str,
required=True,
help='path to a FASTQ files (can be compressed files)')
glopts.add_argument(
'--fastq2',
dest='fastq2',
metavar="PATH",
action='store',
default=None,
type=str,
required=False,
help='''(beta) path to a FASTQ file of read 2 (can be compressed
files). Needed for fast_fragment''')
glopts.add_argument(
'--index',
dest='index',
metavar="PATH",
type=str,
required=True,
help='''paths to file(s) with indexed FASTA files of the
reference genome.''')
glopts.add_argument('--genome',
dest='genome',
metavar="PATH",
nargs='+',
type=str,
help='''paths to file(s) with FASTA files of the
reference genome. Needed for fast_fragment mapping.
If many, files will be concatenated.
I.e.: --genome chr_1.fa chr_2.fa
In this last case, order is important or the rest of the
analysis. Note: it can also be the path to a previously
parsed genome in pickle format.''')
glopts.add_argument('--read',
dest='read',
metavar="INT",
type=int,
required=True,
help='read number')
glopts.add_argument('--renz',
dest='renz',
metavar="STR",
type=str,
required=True,
nargs='+',
help='''restriction enzyme name(s). Use "--renz CHECK"
to search for most probable and exit; and use
"--renz NONE" to avoid using RE site information.''')
glopts.add_argument('--chr_name',
dest='chr_name',
metavar="STR",
nargs='+',
default=[],
type=str,
help='''[fasta header] chromosome name(s). Used in the
same order as data.''')
glopts.add_argument('--tmp',
dest='tmp',
metavar="PATH",
action='store',
default=None,
type=str,
help='''path to a temporary directory (default next to
"workdir" directory)''')
glopts.add_argument(
'--tmpdb',
dest='tmpdb',
action='store',
default=None,
metavar='PATH',
type=str,
help='''if provided uses this directory to manipulate the
database''')
glopts.add_argument('--noX',
action='store_true',
help='no display server (X screen)')
mapper.add_argument('--iterative',
dest='iterative',
default=False,
action='store_true',
help='''default mapping strategy is fragment based
use this flag for iterative mapping''')
mapper.add_argument(
'--fast_fragment',
dest='fast_fragment',
default=False,
action='store_true',
help=
'''(beta) use fast fragment mapping. Both fastq files are mapped using
fragment based mapping in GEM v3. The output file is an intersected
read file than can be used directly in tadbit filter
(no tadbit parse needed). Access to samtools is needed for
fast_fragment to work.
--fastq2 and --genome needs to be
specified and --read value should be 0.''')
mapper.add_argument('--windows',
dest='windows',
default=None,
nargs='+',
help='''defines windows to be used to trim the input
FASTQ reads, for example an iterative mapping can be defined
as: "--windows 1:20 1:25 1:30 1:35 1:40 1:45 1:50". But
this parameter can also be used for fragment based mapping
if for example pair-end reads are both in the same FASTQ,
for example: "--windows 1:50" (if the length of the reads
is 100). Note: that the numbers are both inclusive.''')
descro.add_argument('--species',
dest='species',
metavar="STR",
type=str,
help='species name')
descro.add_argument(
'--descr',
dest='description',
metavar="LIST",
nargs='+',
type=str,
help='''extra descriptive fields each filed separated by
coma, and inside each, name and value separated by column:
--descr=cell:lymphoblast,flowcell:C68AEACXX,index:24nf'''
)
glopts.add_argument('--skip',
dest='skip',
action='store_true',
default=False,
help='[DEBUG] in case already mapped.')
glopts.add_argument('--keep_tmp',
dest='keep_tmp',
action='store_true',
default=False,
help='[DEBUG] keep temporary files.')
mapper.add_argument("-C",
"--cpus",
dest="cpus",
type=int,
default=cpu_count(),
help='''[%(default)s] Maximum
number of CPU cores available in the execution host.
If higher than 1, tasks with multi-threading
capabilities will enabled (if 0 all available)
cores will be used''')
mapper.add_argument(
'--mapper',
dest='mapper',
metavar="STR",
type=str,
default='gem',
help='[%(default)s] mapper used, options are gem, bowtie2 or hisat2')
mapper.add_argument('--mapper_binary',
dest='mapper_binary',
metavar="STR",
type=str,
default=None,
help='[%(default)s] path to mapper binary')
mapper.add_argument(
'--mapper_param',
dest="mapper_param",
type=str,
default=0,
nargs='+',
help='''any parameter that could be passed to the GEM, BOWTIE2 or HISAT2
mapper. e.g. if we want to set the proportion of
mismatches to 0.05 and the maximum indel length to 10,
(in GEM it would be: -e 0.05 --max-big-indel-length 10),
here we could write: "--gem_param e:0.05
max-big-indel-length:10". IMPORTANT: some options are
incompatible with 3C-derived experiments.''')
def populate_args(parser):
"""
parse option from call
"""
parser.formatter_class = lambda prog: HelpFormatter(
prog, width=95, max_help_position=27)
oblopt = parser.add_argument_group('Required options')
glopts = parser.add_argument_group('General options')
rfiltr = parser.add_argument_group('Read filtering options')
normpt = parser.add_argument_group('Normalization options')
outopt = parser.add_argument_group('Output options')
pltopt = parser.add_argument_group('Plotting options')
oblopt.add_argument('-w',
'--workdir',
dest='workdir',
metavar="PATH",
action='store',
default=None,
type=str,
required=True,
help='''path to working directory (generated with the
tool tadbit mapper)''')
oblopt.add_argument('-r',
'--resolution',
dest='reso',
metavar="INT",
action='store',
default=None,
type=int,
required=True,
help='''resolution at which to output matrices''')
glopts.add_argument('--bam',
dest='bam',
metavar="PATH",
action='store',
default=None,
type=str,
help='''path to a TADbit-generated BAM file with
all reads (other wise the tool will guess from the
working directory database)''')
glopts.add_argument(
'-j',
'--jobid',
dest='jobid',
metavar="INT",
action='store',
default=None,
type=int,
help='''Use as input data generated by a job with a given
jobid. Use tadbit describe to find out which.''')
glopts.add_argument('--force',
dest='force',
action='store_true',
default=False,
help='overwrite previously run job')
glopts.add_argument('-q',
'--quiet',
dest='quiet',
action='store_true',
default=False,
help='remove all messages')
glopts.add_argument(
'--tmpdb',
dest='tmpdb',
action='store',
default=None,
metavar='PATH',
type=str,
help='''if provided uses this directory to manipulate the
database''')
glopts.add_argument('--nchunks',
dest='nchunks',
action='store',
default=None,
type=int,
help='''maximum number of chunks into which to
cut the BAM''')
glopts.add_argument("-C",
"--cpus",
dest="cpus",
type=int,
default=cpu_count(),
help='''[%(default)s] Maximum
number of CPU cores available in the execution host.
If higher than 1, tasks with multi-threading
capabilities will enabled (if 0 all available)
cores will be used''')
outopt.add_argument('--matrix',
dest='matrix',
action='store_true',
default=False,
help='''Write text matrix in multiple columns. By
defaults matrices are written in BED-like format (also
only way to get a raw matrix with all values including
the ones in masked columns).''')
outopt.add_argument('--rownames',
dest='row_names',
action='store_true',
default=False,
help='To store row names in the output text matrix.')
pltopt.add_argument('--plot',
dest='plot',
action='store_true',
default=False,
help='[%(default)s] Plot matrix in desired format.')
outopt.add_argument(
'--only_plot',
dest='only_plot',
action='store_true',
default=False,
help='[%(default)s] Skip writing matrix in text format.')
outopt.add_argument('-i',
'--interactive',
dest='interactive',
action='store_true',
default=False,
help='''[%(default)s] Open matplotlib interactive plot
(nothing will be saved).''')
pltopt.add_argument('--cmap',
dest='cmap',
action='store',
default='viridis',
help='[%(default)s] Matplotlib color map to use.')
pltopt.add_argument('--format',
dest='format',
action='store',
default='png',
help='[%(default)s] plot file format.')
pltopt.add_argument('--zrange',
dest='zrange',
action='store',
default=None,
help='''Range, in log2 scale of the color scale.
i.e.: --zrange 2,-2''')
outopt.add_argument('-c',
'--coord',
dest='coord1',
metavar='',
default=None,
help='''Coordinate of the region to
retrieve. By default all genome, arguments can be
either one chromosome name, or the coordinate in
the form: "-c chr3:110000000-120000000"''')
outopt.add_argument('-c2',
'--coord2',
dest='coord2',
metavar='',
default=None,
help='''Coordinate of a second region to
retrieve the matrix in the intersection with the first
region.''')
normpt.add_argument('--biases',
dest='biases',
metavar="PATH",
action='store',
default=None,
type=str,
help='''path to file with pre-calculated biases by
columns''')
normpt.add_argument('--norm',
dest='normalizations',
metavar="STR",
action='store',
default=['raw'],
type=str,
nargs='+',
choices=['norm', 'decay', 'raw', 'raw&decay'],
help='''[%(default)s] normalization(s) to apply.
Choices are: [%(choices)s]''')
rfiltr.add_argument(
'-F',
'--filter',
dest='filter',
nargs='+',
type=int,
metavar='INT',
default=[1, 2, 3, 4, 6, 7, 9, 10],
choices=range(0, 11),
help=(
"""[%(default)s] Use filters to define a set os
valid pair of reads e.g.:
'--apply 1 2 3 4 8 9 10'. Where these numbers""" +
"correspond to: 0: nothing, %s" %
(', '.join(['%2d: %15s' % (k, MASKED[k]['name'])
for k in MASKED]))))
outopt.add_argument('--only_txt',
dest='only_txt',
action='store_true',
default=False,
help='Save only text file for matrices, not images')
rfiltr.add_argument(
'--valid',
dest='only_valid',
action='store_true',
default=False,
help='input BAM file contains only valid pairs (already filtered).')
def default_formatter(prog):
return HelpFormatter(prog)
def __parse_args(self):
""" Parse command-line arguments with the 'argparse' module. To learn
more about each command, invoke the simulator in one of the following
ways:
(venv) $ python tsalis.py --help
(venv) $ python tsalis.py new --help
(venv) $ python tsalis.py load --help
"""
# Custom formatter helps keep all help data aligned.
formatter = lambda prog: HelpFormatter(prog, max_help_position=30)
# Initialize the main parser with our custom formatter.
parser = ArgumentParser(
description=("Viewer/controller for the Salis simulator."),
formatter_class=formatter)
parser.add_argument("-v",
"--version",
action="version",
version="Salis: A-Life Simulator (" + __version__ +
")")
parser.add_argument("-d",
"--debug",
action="store_true",
help="run debug build of Salis library")
parser.add_argument("-m",
"--minimal",
action="store_true",
help="start up Salis in minimal mode")
parser.add_argument("-r",
"--running",
action="store_true",
help="start up Salis in running state")
# Initialize the 'new/load' action subparsers.
subparsers = parser.add_subparsers(
dest="action",
help=("call 'salis.py new --help' or 'salis.py load --help' for "
"sublist of commands"))
subparsers.required = True
# Set up subparser for the create 'new' action.
new_parser = subparsers.add_parser("new", formatter_class=formatter)
new_parser.add_argument("-o",
"--order",
required=True,
type=lambda x: int(x, 0),
metavar="[1-31]",
help="create new simulation of given ORDER")
new_parser.add_argument(
"-f",
"--file",
required=True,
type=str,
metavar="FILE",
help="name of FILE to save simulation to on exit")
new_parser.add_argument(
"-a",
"--auto",
required=False,
type=lambda x: int(x, 0),
metavar="INT",
help="auto-save interval for the new simulation")
# Set up subparser for the 'load' existing action.
load_parser = subparsers.add_parser("load", formatter_class=formatter)
load_parser.add_argument(
"-f",
"--file",
required=True,
type=str,
metavar="FILE",
help="load previously saved simulation from FILE")
load_parser.add_argument(
"-a",
"--auto",
required=False,
type=lambda x: int(x, 0),
metavar="INT",
help="auto-save interval for the loaded simulation")
# Finally, parse all arguments.
args = parser.parse_args()
# Revise that parsed CL arguments are valid.
if args.action == "new":
if args.order not in range(1, 32):
parser.error("Order must be an integer between 1 and 31")
else:
savefile = os.path.join(self.path, "sims", args.file)
# No save-file with given name has been detected.
if not os.path.isfile(savefile):
parser.error(
"Save file provided '{}' does not exist".format(savefile))
# Set autosave interval, if given.
#if args.auto:
if args.auto:
self.set_autosave(args.auto)
return args
def populate_args(parser):
"""
parse option from call
"""
parser.formatter_class = lambda prog: HelpFormatter(
prog, width=95, max_help_position=27)
glopts = parser.add_argument_group('General options')
glopts.add_argument('-w',
'--workdir',
dest='workdir',
metavar="PATH",
action='store',
default=None,
type=str,
help='''path to working directory (generated with the
tool tadbit mapper)''')
glopts.add_argument('-j',
'--jobids',
dest='jobids',
metavar="INT",
action='store',
default=None,
nargs='+',
type=int,
help='jobids of the files and entries to be removed')
glopts.add_argument('--delete',
dest='delete',
action="store_true",
default=False,
help='delete files, otherwise only DB entries.')
glopts.add_argument('--compress',
dest='compress',
action="store_true",
default=False,
help='compress files and update paths accordingly')
glopts.add_argument('--noX',
action='store_true',
help='no display server (X screen)')
parser.add_argument(
'--change_workdir',
dest='new_workdir',
metavar="PATH",
action='store',
default=None,
type=str,
help='''In case folder was moved, input the new path''')
glopts.add_argument(
'--tmpdb',
dest='tmpdb',
action='store',
default=None,
metavar='PATH',
type=str,
help='''if provided uses this directory to manipulate the
database''')
parser.add_argument_group(glopts)
def get_options():
"""
parse option from call
"""
parser = ArgumentParser(usage="%(prog)s [options] [--cfg CONFIG_PATH]",
formatter_class=lambda prog: HelpFormatter(
prog, width=95, max_help_position=27))
glopts = parser.add_argument_group('General arguments')
taddet = parser.add_argument_group('TAD detection arguments')
optimo = parser.add_argument_group('Optimization of IMP arguments')
modelo = parser.add_argument_group('Modeling with optimal IMP arguments')
descro = parser.add_argument_group('Descriptive, optional arguments')
analyz = parser.add_argument_group('Output arguments')
parser.add_argument('--cfg',
dest='cfg',
metavar="PATH",
action='store',
default=None,
type=str,
help='path to a configuration file with predefined ' +
'parameters')
parser.add_argument('--analyze_only',
dest='analyze_only',
action='store_true',
default=False,
help=('load precomputed models in outdir, ' +
'skip optimization, modeling'))
parser.add_argument('--optimize_only',
dest='optimize_only',
default=False,
action='store_true',
help='do the optimization of the region and exit')
#########################################
# GENERAL
glopts.add_argument(
'--root_path',
dest='root_path',
metavar="PATH",
default='',
type=str,
help=('path to search for data files (just pass file name' +
'in "data")'))
glopts.add_argument('--data',
dest='data',
metavar="PATH",
nargs='+',
type=str,
help='''paths to file(s) with FASTQ files. If many,
files will be summed up. I.e.: --data
replicate_1.txt replicate_2.txt''')
glopts.add_argument('--fasta',
dest='fasta',
metavar="PATH",
nargs='+',
type=str,
help='''paths to file(s) with FASTA files of the
reference genome. If many, files will be concatenated.
I.e.: --fasta chr_1.fa chr_2.fa
In this last case, order is important or the rest of the
analysis.''')
glopts.add_argument('--chr_name',
dest='chr_name',
metavar="STR",
nargs='+',
default=[],
type=str,
help='''[fasta header] chromsome name(s). Used in the
same order as data.''')
# update fasta paths adding root directory
if opts.root_path and opts.fasta[0]:
for i in xrange(len(opts.fasta)):
logging.info(os.path.join(opts.root_path, opts.fasta[i]))
opts.data[i] = os.path.join(opts.root_path, opts.fasta[i])
return opts
def get_formatter(*args, **kwargs):
kwargs['max_help_position'] = width or 45
return HelpFormatter(*args, **kwargs)
def get_options():
"""
parse option from call
"""
parser = ArgumentParser(
usage="%(prog)s [options] [--cfg CONFIG_PATH]",
formatter_class=lambda prog: HelpFormatter(prog, width=95,
max_help_position=27))
glopts = parser.add_argument_group('General arguments')
taddet = parser.add_argument_group('TAD detection arguments')
optimo = parser.add_argument_group('Optimization of IMP arguments')
modelo = parser.add_argument_group('Modeling with optimal IMP arguments')
descro = parser.add_argument_group('Descriptive, optional arguments')
analyz = parser.add_argument_group('Output arguments')
## Define analysis actions:
actions = {0 : "do nothing",
1 : "column filtering",
2 : "TAD borders",
3 : "TAD alignment",
4 : "optimization plot",
5 : "correlation real/models",
6 : "z-score plot",
7 : "constraints",
8 : "objective function",
9 : "centroid",
10 : "consistency",
11 : "density",
12 : "contact map",
13 : "walking angle",
14 : "persistence length",
15 : "accessibility",
16 : "interaction"}
parser.add_argument('--usage', dest='usage', action="store_true",
default=False,
help='''show detailed usage documentation, with examples
and exit''')
parser.add_argument('--cfg', dest='cfg', metavar="PATH", action='store',
default=None, type=str,
help='path to a configuration file with predefined ' +
'parameters')
parser.add_argument('--analyze_only', dest='analyze_only',
action='store_true', default=False,
help=('load precomputed models in outdir, ' +
'skip optimization, modeling'))
parser.add_argument('--optimize_only', dest='optimize_only', default=False,
action='store_true',
help='do the optimization of the region and exit')
parser.add_argument('--tad_only', dest='tad_only', action="store_true",
default=False,
help='[%(default)s] exit after searching for TADs')
parser.add_argument('--ncpus', dest='ncpus', metavar="INT", default=1,
type=int, help='[%(default)s] Number of CPUs to use')
#########################################
# GENERAL
glopts.add_argument(
'--root_path', dest='root_path', metavar="PATH",
default='', type=str,
help=('path to search for data files (just pass file name' +
'in "data")'))
glopts.add_argument('--data', dest='data', metavar="PATH", nargs='+',
type=str,
help='''path to file(s) with Hi-C data matrix. If many,
experiments will be summed up. I.e.: --data
replicate_1.txt replicate_2.txt''')
glopts.add_argument('--xname', dest='xname', metavar="STR", nargs='+',
default=[], type=str,
help='''[file name] experiment name(s). Use same order
as data.''')
glopts.add_argument('--norm', dest='norm', metavar="PATH", nargs='+',
type=str,
help='path to file(s) with normalizedHi-C data matrix.')
glopts.add_argument('--nodiag', dest='nodiag', action='store_true',
help='''If the matrix does not contain self interacting
bins (only zeroes in the diagonal)''')
glopts.add_argument('--filt', dest='filt', metavar='INT', default=90,
help='''Filter out column with more than a given
percentage of zeroes''')
glopts.add_argument('--crm', dest='crm', metavar="NAME",
help='chromosome name')
glopts.add_argument('--beg', dest='beg', metavar="INT", type=float,
default=None,
help='genomic coordinate from which to start modeling')
glopts.add_argument('--end', dest='end', metavar="INT", type=float,
help='genomic coordinate where to end modeling')
glopts.add_argument('--res', dest='res', metavar="INT", type=int,
help='resolution of the Hi-C experiment')
glopts.add_argument('--outdir', dest='outdir', metavar="PATH",
default=None,
help='out directory for results')
#########################################
# TADs
taddet.add_argument('--tad', dest='tad', action="store_true", default=False,
help='[%(default)s] search for TADs in experiments')
taddet.add_argument('--centromere', dest='centromere', action="store_true",
default=False,
help='[%(default)s] search for centromeric region')
taddet.add_argument('--group', dest='group', nargs='+', type=int,
default=0, metavar='INT',
help='''[all together] How to group Hi-C experiments for
the detection of TAD borders. I.e.: "--exp_group 2 2 1"
first 2 experiments used together, next 2 also, and last
alone (batch_mode option used)''')
#########################################
# MODELING
modelo.add_argument('--nmodels_mod', dest='nmodels_mod', metavar="INT",
default='5000', type=int,
help=('[%(default)s] number of models to generate for' +
' modeling'))
modelo.add_argument('--nkeep_mod', dest='nkeep_mod', metavar="INT",
default='1000', type=int,
help=('[%(default)s] number of models to keep for ' +
'modeling'))
#########################################
# OPTIMIZATION
optimo.add_argument('--maxdist', action='store', metavar="LIST",
default='400', dest='maxdist',
help='range of numbers for maxdist' +
', i.e. 400:1000:100 -- or just a number')
optimo.add_argument('--upfreq', dest='upfreq', metavar="LIST",
default='0',
help='range of numbers for upfreq' +
', i.e. 0:1.2:0.3 -- or just a number')
optimo.add_argument('--lowfreq', dest='lowfreq', metavar="LIST",
default='0',
help='range of numbers for lowfreq' +
', i.e. -1.2:0:0.3 -- or just a number')
optimo.add_argument('--scale', dest='scale', metavar="LIST",
default="0.01",
help='[%(default)s] range of numbers to be test as ' +
'optimal scale value, i.e. 0.005:0.01:0.001 -- Can ' +
'also pass only one number')
optimo.add_argument('--dcutoff', dest='dcutoff', metavar="LIST",
default="2",
help='[%(default)s] range of numbers to be test as ' +
'optimal distance cutoff parameter (distance, in ' +
'number of beads, from which to consider 2 beads as ' +
'being close), i.e. 1:5:0.5 -- Can also pass only one' +
' number')
optimo.add_argument('--nmodels_opt', dest='nmodels_opt', metavar="INT",
default='500', type=int,
help='[%(default)s] number of models to generate for ' +
'optimization')
optimo.add_argument('--nkeep_opt', dest='nkeep_opt', metavar="INT",
default='100', type=int,
help='[%(default)s] number of models to keep for ' +
'optimization')
optimo.add_argument('--force_opt', dest='optimize_from_scratch',
action="store_true", default=False,
help='''[%(default)s] do not take into account previous
optimizations. Usefull for running in parallel in a
cluster for example.''')
#########################################
# DESCRIPTION
descro.add_argument('--species', dest='species', metavar="STRING",
default='UNKNOWN',
help='species name, with no spaces, i.e.: homo_sapiens')
descro.add_argument('--cell', dest='cell', metavar="STRING",
help='cell type name')
descro.add_argument('--exp_type', dest='exp_type', metavar="STRING",
help='experiment type name (i.e.: Hi-C)')
descro.add_argument('--assembly', dest='assembly', metavar="STRING",
default=None,
help='''NCBI ID of the original assembly
(i.e.: NCBI36 for human)''')
descro.add_argument('--enzyme', dest='enzyme', metavar="STRING",
default=None,
help='''name of the enzyme used to digest
chromatin (i.e. HindIII)''')
descro.add_argument('--identifier', dest='identifier', metavar="STRING",
default=None,
help='''NCBI identifier of the experiment''')
descro.add_argument('--project', dest='project', metavar="STRING",
default=None,
help='''project name''')
#########################################
# OUTPUT
analyz.add_argument('--analyze', dest='analyze', nargs='+',
choices=range(len(actions)), type=int,
default=range(2, len(actions)), metavar='INT',
help=('''[%s] list of numbers representing the
analysis to be done. Choose between:
%s''' % (' '.join([str(i) for i in range(
2, len(actions))]),
'\n'.join(['%s) %s' % (k, actions[k])
for k in actions]))))
analyz.add_argument('--not_write_cmm', dest='not_write_cmm',
default=False, action='store_true',
help='''[%(default)s] do not generate cmm files for each
model (Chimera input)''')
analyz.add_argument('--not_write_xyz', dest='not_write_xyz',
default=False, action='store_true',
help='''[%(default)s] do not generate xyz files for each
model (3D coordinates)''')
analyz.add_argument('--not_write_json', dest='not_write_json',
default=False, action='store_true',
help='''[%(default)s] do not generate json file.''')
parser.add_argument_group(optimo)
parser.add_argument_group(modelo)
parser.add_argument_group(descro)
parser.add_argument_group(analyz)
opts = parser.parse_args()
if opts.usage:
print __doc__
exit()
log = '\tSummary of arguments:\n'
# merger opts with CFG file and write summary
args = reduce(lambda x, y: x + y, [i.strip('-').split('=')
for i in sys.argv])
new_opts = {}
if opts.cfg:
for line in open(opts.cfg):
if not '=' in line:
continue
if line.startswith('#'):
continue
key, value = line.split('#')[0].strip().split('=')
key = key.strip()
value = value.strip()
if value == 'True':
value = True
elif value == 'False':
value = False
elif key in ['data', 'norm', 'xname', 'group', 'analyze']:
new_opts.setdefault(key, []).extend(value.split())
continue
new_opts[key] = value
# bad key in configuration file
opts.__dict__['description'] = {}
for bad_k in set(new_opts.keys()) - set(opts.__dict__.keys()):
sys.stderr.write('WARNING: parameter "%s" not recognized (used as description)\n' % (bad_k))
try:
opts.__dict__['description'][bad_k] = int(new_opts[bad_k])
except ValueError:
opts.__dict__['description'][bad_k] = new_opts[bad_k]
for key in sorted(opts.__dict__.keys()):
if key in args:
log += ' * Command setting %13s to %s\n' % (
key, opts.__dict__[key])
elif key in new_opts:
opts.__dict__[key] = new_opts[key]
log += ' - Config. setting %13s to %s\n' % (
key, new_opts[key])
else:
log += ' o Default setting %13s to %s\n' % (
key, opts.__dict__[key])
# rename analysis actions
for i, j in enumerate(opts.analyze):
opts.analyze[i] = actions[int(j)]
if not opts.data and not opts.norm:
sys.stderr.write('MISSING data')
exit(parser.print_help())
if not opts.outdir:
sys.stderr.write('MISSING outdir')
exit(parser.print_help())
if not opts.crm:
sys.stderr.write('MISSING crm NAME')
exit(parser.print_help())
if not opts.res:
sys.stderr.write('MISSING resolution')
exit(parser.print_help())
if not opts.analyze_only:
if not opts.maxdist:
sys.stderr.write('MISSING maxdist')
exit(parser.print_help())
if not opts.lowfreq:
sys.stderr.write('MISSING lowfreq')
exit(parser.print_help())
if not opts.upfreq:
sys.stderr.write('MISSING upfreq')
exit(parser.print_help())
if not opts.beg and not opts.tad_only:
sys.stderr.write('WARNING: no begin coordinate given all')
if not opts.end and not opts.tad_only:
sys.stderr.write('WARNING: no begin coordinate given all')
# groups for TAD detection
if not opts.data:
opts.data = [None] * len(opts.norm)
else:
opts.norm = [None] * len(opts.data)
if not opts.group:
opts.group = [len(opts.data)]
else:
opts.group = [int(i) for i in opts.group]
if sum(opts.group) > len(opts.data):
logging.info('ERROR: Number of experiments in groups larger than ' +
'the number of Hi-C data files given.')
exit()
# this options should stay as this now
# opts.scale = '0.01'
# switch to number
opts.nmodels_mod = int(opts.nmodels_mod)
opts.nkeep_mod = int(opts.nkeep_mod )
opts.nmodels_opt = int(opts.nmodels_opt)
opts.nkeep_opt = int(opts.nkeep_opt )
opts.ncpus = int(opts.ncpus )
opts.res = int(opts.res )
# TODO: UNDER TEST
opts.container = None #['cylinder', 1000, 5000, 100]
# do the division to bins
if not opts.tad_only:
try:
opts.beg = int(float(opts.beg) / opts.res)
opts.end = int(float(opts.end) / opts.res)
if opts.end - opts.beg <= 2:
raise Exception('"beg" and "end" parameter should be given in ' +
'genomic coordinates, not bin')
except TypeError:
pass
# Create out-directory
name = '{0}_{1}_{2}'.format(opts.crm, opts.beg, opts.end)
if not os.path.exists(os.path.join(opts.outdir, name)):
os.makedirs(os.path.join(opts.outdir, name))
# write version log
if not os.path.exists(os.path.join(opts.outdir,
'TADbit_and_dependencies_versions.log')):
vlog = os.path.join(opts.outdir, 'TADbit_and_dependencies_versions.log')
vlog = open(vlog, 'w')
vlog.write(get_dependencies_version())
vlog.close()
# write log
if opts.optimize_only:
log_format = '[OPTIMIZATION {}_{}_{}_{}_{}] %(message)s'.format(
opts.maxdist, opts.upfreq, opts.lowfreq, opts.scale, opts.dcutoff)
elif opts.analyze_only:
log_format = '[ANALYZE] %(message)s'
elif opts.tad_only:
log_format = '[TAD] %(message)s'
else:
log_format = '[DEFAULT] %(message)s'
try:
logging.basicConfig(filename=os.path.join(opts.outdir, name, name + '.log'),
level=logging.INFO, format=log_format)
except IOError:
logging.basicConfig(filename=os.path.join(opts.outdir, name, name + '.log2'),
level=logging.INFO, format=log_format)
logging.getLogger().addHandler(logging.StreamHandler())
logging.info(('\n' + log_format.replace(' %(message)s', '')
).join(log.split('\n')))
# update path to Hi-C data adding root directory
if opts.root_path and opts.data[0]:
for i in xrange(len(opts.data)):
logging.info(os.path.join(opts.root_path, opts.data[i]))
opts.data[i] = os.path.join(opts.root_path, opts.data[i])
# update path to Hi-C norm adding root directory
if opts.root_path and opts.norm[0]:
for i in xrange(len(opts.norm)):
logging.info(os.path.join(opts.root_path, opts.norm[i]))
opts.norm[i] = os.path.join(opts.root_path, opts.norm[i])
return opts
def populate_args(parser):
"""
parse option from call
"""
parser.formatter_class=lambda prog: HelpFormatter(prog, width=95,
max_help_position=27)
glopts = parser.add_argument_group('General options')
glopts.add_argument('-w', '--workdir', dest='workdir', metavar="PATH",
action='store', default=None, type=str, required=True,
help='''path to working directory (generated with the
tool tadbit mapper)''')
glopts.add_argument('--tmp', dest='tmpdb', action='store', default=None,
metavar='PATH', type=str,
help='''if provided uses this directory to manipulate the
database''')
glopts.add_argument('--nosql', dest='nosql',
action='store_true', default=False,
help='do not load/store data from/in sqlite database')
glopts.add_argument('--mreads', dest='mreads', metavar="PATH",
action='store', default=None, type=str,
help='''path valid-pairs file''')
glopts.add_argument('--bad_cols', dest='bad_co', metavar="PATH",
action='store', default=None, type=str,
help='''path to file with bad columns''')
glopts.add_argument('--biases', dest='biases', metavar="PATH",
action='store', default=None, type=str,
help='''path to file with precalculated biases by
columns''')
glopts.add_argument('-r', '--resolution', dest='reso', metavar="INT",
action='store', default=None, type=int,
help='''resolution at which to output matrices''')
glopts.add_argument('--norm_matrix', dest='norm_matrix', metavar="PATH",
action='store', default=None, type=str,
help='''path to a matrix file with normalized read
counts''')
glopts.add_argument('--raw_matrix', dest='raw_matrix', metavar="PATH",
action='store', default=None, type=str,
help='''path to a matrix file with raw read
counts''')
glopts.add_argument('--rich_in_A', dest='rich_in_A', metavar="PATH",
action='store', default=None, type=str,
help='''path to a BAD or bedGraph file with list of
protein coding gene or other active epigenetic mark,
to be used to label compartments instead of using
the relative interaction count.''')
glopts.add_argument('--only_compartments', dest='only_compartments',
action='store_true', default=False,
help='''search A/B compartments using first eigen vector
of the correlation matrix''')
glopts.add_argument('--only_tads', dest='only_tads',
action='store_true', default=False,
help='''search TAD boundaries break-point detection
algorithm''')
glopts.add_argument('--perc_zeros', dest='perc_zeros', metavar="FLOAT",
action='store', default=95, type=float,
help='maximum percentage of zeroes allowed per column')
glopts.add_argument('-j', '--jobid', dest='jobid', metavar="INT",
action='store', default=None, type=int,
help='''Use as input data generated by a job with a given
jobid. Use tadbit describe to find out which.''')
glopts.add_argument('-c', '--chromosomes', dest='crms', metavar="STR",
action='store', default=None, type=str, nargs='+',
help='''Name of the chromosomes on which to search
for TADs or compartments.''')
glopts.add_argument('--max_tad_size', dest='max_tad_size', metavar="INT",
action='store', default=None, type=int,
help='''an integer defining the maximum size of TAD. Default
defines it as the number of rows/columns''')
glopts.add_argument("-C", "--cpu", dest="cpus", type=int,
default=0, help='''[%(default)s] Maximum number of CPU
cores available in the execution host. If higher
than 1, tasks with multi-threading
capabilities will enabled (if 0 all available)
cores will be used''')
glopts.add_argument('--force', dest='force', action='store_true',
default=False,
help='overwrite previously run job')
parser.add_argument_group(glopts)
def parse_args():
'''
Parse command-line options found in 'args' (default: sys.argv[1:]).
:returns: On success, a namedtuple of Values instances.
'''
parser = ArgumentParser(
prog='worker',
description=
'Streaming Worker reads from Kafka cluster and writes to Hive '
'through the Spark2 streaming module.',
epilog='END',
formatter_class=lambda prog: HelpFormatter(
prog, max_help_position=40, width=80),
usage='worker [OPTIONS]... -t <pipeline> --topic <topic> -p <partition>'
)
# .................................set optional arguments
parser._optionals.title = 'Optional Arguments'
parser.add_argument(
'-a',
'--app-name',
metavar='STRING',
help='name of the Spark Job to display on the cluster web UI')
parser.add_argument(
'-b',
'--batch-duration',
default=30,
type=int,
metavar='INTEGER',
help=
'time interval (in seconds) at which streaming data will be divided into batches'
)
parser.add_argument('-c',
'--config-file',
metavar='FILE',
type=file,
help='path of configuration file',
default=os.path.expanduser('~/.worker.json'))
parser.add_argument(
'-g',
'--group-id',
metavar='STRING',
help=
'name of the consumer group to join for dynamic partition assignment')
parser.add_argument('-l',
'--log-level',
default='INFO',
metavar='STRING',
help='determine the level of the logger')
parser.add_argument('-v',
'--version',
action='version',
version='%(prog)s {0}'.format(__version__))
# .................................set required arguments
required = parser.add_argument_group('Required Arguments')
required.add_argument('-p',
'--partition',
required=True,
metavar='INTEGER',
help='partition number to consume')
required.add_argument('--topic',
required=True,
metavar='STRING',
help='topic to listen for new messages')
required.add_argument('-t',
'--type',
choices=pipelines.__all__,
required=True,
help='type of the data that will be ingested',
metavar='STRING')
return parser.parse_args()
def populate_args(parser):
"""
parse option from call
"""
parser.formatter_class = lambda prog: HelpFormatter(
prog, width=95, max_help_position=27)
oblopt = parser.add_argument_group('Required options')
glopts = parser.add_argument_group('General options')
oblopt.add_argument('-w',
'--workdir',
dest='workdir',
metavar="PATH",
action='store',
default=None,
type=str,
required=True,
help='''path to working directory (generated with the
tool tadbit mapper)''')
oblopt.add_argument('-r',
'--resolution',
dest='reso',
metavar="INT",
action='store',
default=None,
type=int,
required=True,
help='''resolution at which to output matrices''')
oblopt.add_argument('--format',
dest='format',
action='store',
default='text',
choices=['text', 'matrix', 'cooler'],
help='''[%(default)s] can be any of [%(choices)s]''')
oblopt.add_argument('-i',
'--input',
dest='input',
metavar="STR",
action='store',
default=None,
type=str,
required=True,
help='''path to input file''')
glopts.add_argument('-c',
'--coord',
dest='coord1',
metavar='',
default=None,
help='''Coordinate of the region to
import. By default all genome, arguments can be
either one chromosome name, or the coordinate in
the form: "-c chr3:110000000-120000000"''')
glopts.add_argument(
'--tmpdb',
dest='tmpdb',
action='store',
default=None,
metavar='PATH',
type=str,
help='''if provided uses this directory to manipulate the
database''')
glopts.add_argument("-C",
"--cpus",
dest="cpus",
type=int,
default=cpu_count(),
help='''[%(default)s] Maximum
number of CPU cores available in the execution host.
If higher than 1, tasks with multi-threading
capabilities will enabled (if 0 all available)
cores will be used''')
glopts.add_argument('--samtools',
dest='samtools',
metavar="PATH",
action='store',
default='samtools',
type=str,
help='''path samtools binary''')
def setup_argparser():
"""Set up the command-line argument parser."""
# https://stackoverflow.com/questions/5462873/control-formatting-of-the-argparse-help-argument-list
# setting the width voids whatever os.environ['COLUMNS'] would be set to
parser = ArgumentParser(
prog=os.path.basename(__file__),
description=
"Generate a smoothed elevation plot (PNG) using a colour gradient \
and stepped grades to show general climbs and downhills.",
formatter_class=lambda prog: HelpFormatter(
prog, max_help_position=40, width=160))
# positional
parser.add_argument("filename",
help="the GPX file to generate the plot for",
metavar="filename")
# optional
parser.add_argument(
"-a",
"--all",
dest="plot_all",
action="store_true",
help=
"generate plots for all available and supported data and show the stats summary"
)
parser.add_argument("-hr",
"--heart-rate",
dest="plot_heart_rate",
action="store_true",
help="generate a heart rate plot too")
parser.add_argument("-s",
"--speed",
dest="plot_speed",
action="store_true",
help="generate a speed plot too")
parser.add_argument("-c",
"--cadence",
dest="plot_cadence",
action="store_true",
help="generate a cadence plot too")
parser.add_argument(
"-i",
"--interactive",
dest="interactive_plot",
action="store_true",
help="open an interactive plot window besides saving the plot to file")
parser.add_argument("-ss",
"--summary",
dest="show_summary",
action="store_true",
help="print some summary statistics for the workout")
parser.add_argument("-f",
"--format",
dest="output_format",
choices=["png", "html"],
default="png",
help="the format of the generated plot")
parser.add_argument("-q",
"--quiet",
dest="quiet",
action="store_true",
help="run in quiet mode")
return parser
def populate_args(parser):
"""
parse option from call
"""
parser.formatter_class = lambda prog: HelpFormatter(
prog, width=95, max_help_position=27)
glopts = parser.add_argument_group('General options')
mapper = parser.add_argument_group('Mapping options')
descro = parser.add_argument_group('Descriptive, optional arguments')
glopts.add_argument('--cfg',
dest='cfg',
metavar="PATH",
action='store',
default=None,
type=str,
help='path to a configuration file with predefined ' +
'parameters')
glopts.add_argument(
'--skip_mapping',
dest='skip_mapping',
action='store_true',
default=False,
help='generate a Hi-C specific quality plot from FASTQ and exits')
glopts.add_argument('-w',
'--workdir',
dest='workdir',
metavar="PATH",
action='store',
default=None,
type=str,
required=True,
help='path to an output folder.')
glopts.add_argument('--fastq',
dest='fastq',
metavar="PATH",
action='store',
default=None,
type=str,
required=True,
help='path to a FASTQ files (can be compressed files)')
glopts.add_argument(
'--index',
dest='index',
metavar="PATH",
type=str,
required=True,
help='''paths to file(s) with indexed FASTA files of the
reference genome.''')
glopts.add_argument('--read',
dest='read',
metavar="INT",
type=int,
required=True,
help='read number')
glopts.add_argument('--renz',
dest='renz',
metavar="STR",
type=str,
required=True,
nargs='+',
help='restriction enzyme name(s)')
glopts.add_argument('--chr_name',
dest='chr_name',
metavar="STR",
nargs='+',
default=[],
type=str,
help='''[fasta header] chromosome name(s). Used in the
same order as data.''')
glopts.add_argument('--tmp',
dest='tmp',
metavar="PATH",
action='store',
default=None,
type=str,
help='''path to a temporary directory (default next to
"workdir" directory)''')
glopts.add_argument(
'--tmpdb',
dest='tmpdb',
action='store',
default=None,
metavar='PATH',
type=str,
help='''if provided uses this directory to manipulate the
database''')
mapper.add_argument('--iterative',
dest='iterative',
default=False,
action='store_true',
help='''default mapping strategy is fragment based
use this flag for iterative mapping''')
mapper.add_argument('--windows',
dest='windows',
default=None,
nargs='+',
help='''defines windows to be used to trim the input
FASTQ reads, for example an iterative mapping can be defined
as: "--windows 1:20 1:25 1:30 1:35 1:40 1:45 1:50". But
this parameter can also be used for fragment based mapping
if for example pair-end reads are both in the same FASTQ,
for example: "--windows 1:50" (if the length of the reads
is 100). Note: that the numbers are both inclusive.''')
descro.add_argument('--species',
dest='species',
metavar="STR",
type=str,
help='species name')
descro.add_argument(
'--descr',
dest='description',
metavar="LIST",
nargs='+',
type=str,
help='''extra descriptive fields each filed separated by
coma, and inside each, name and value separated by column:
--descr=cell:lymphoblast,flowcell:C68AEACXX,index:24nf'''
)
glopts.add_argument('--skip',
dest='skip',
action='store_true',
default=False,
help='[DEBUG] in case already mapped.')
glopts.add_argument('--keep_tmp',
dest='keep_tmp',
action='store_true',
default=False,
help='[DEBUG] keep temporary files.')
mapper.add_argument("-C",
"--cpus",
dest="cpus",
type=int,
default=cpu_count(),
help='''[%(default)s] Maximum
number of CPU cores available in the execution host.
If higher than 1, tasks with multi-threading
capabilities will enabled (if 0 all available)
cores will be used''')
mapper.add_argument('--gem_binary',
dest='gem_binary',
metavar="STR",
type=str,
default='gem-mapper',
help='[%(default)s] path to GEM mapper binary')
mapper.add_argument('--gem_param',
dest="gem_param",
type=str,
default=0,
nargs='+',
help='''any parameter that could be passed to the GEM
mapper. e.g. if we want to set the proportion of
mismatches to 0.05 and the maximum indel length to 10,
(in GEM it would be: -e 0.05 --max-big-indel-length 10),
here we could write: "--gem_param e:0.05
max-big-indel-length:10". IMPORTANT: some options are
incompatible with 3C-derived experiments.''')
continue
svg_fh.write(__tail__)
svg_fh.write(polygons)
svg_fh.write("</svg>")
svg_fh.close()
printif(f"\033[92mSuccessfully created SVG\033[0m", ARGS.verbose)
if __name__ == "__main__":
PARSER = ArgumentParser(prog="tagore",
add_help=True,
description='''
tagore: a utility for illustrating human chromosomes
https://github.com/jordanlab/tagore
''',
formatter_class=lambda prog: HelpFormatter(
prog, width=120, max_help_position=120))
PARSER.add_argument('--version',
action='version',
help='Print the software version',
version='tagore (version {})'.format(VERSION))
# Input arguments
PARSER.add_argument('-i',
'--input',
required=True,
default=None,
metavar='<input.bed>',
help='Input BED-like file')
PARSER.add_argument('-p',
'--prefix',
def populate_args(parser):
"""
parse option from call
"""
parser.formatter_class = lambda prog: HelpFormatter(
prog, width=95, max_help_position=27)
glopts = parser.add_argument_group('General options')
reopts = parser.add_argument_group('Modeling preparation')
opopts = parser.add_argument_group('Parameter optimization')
anopts = parser.add_argument_group('Analysis')
ruopts = parser.add_argument_group('Running jobs')
glopts.add_argument('-w',
'--workdir',
dest='workdir',
metavar="PATH",
action='store',
default=None,
type=str,
required=True,
help='''path to working directory (generated with the
tool TADbit mapper)''')
glopts.add_argument('--input_matrix',
dest='matrix',
metavar="PATH",
type=str,
help='''In case input was not generated with the TADbit
tools''')
glopts.add_argument('--rand',
dest='rand',
metavar="INT",
type=str,
default='1',
help='''[%(default)s] random initial number. NOTE:
when running single model at the time, should be
different for each run''')
glopts.add_argument(
'--nmodels',
dest='nmodels',
metavar="INT",
default=5000,
type=int,
help=('[%(default)s] number of models to generate for' + ' modeling'))
glopts.add_argument('--nkeep',
dest='nkeep',
metavar="INT",
default=1000,
type=int,
help=('[%(default)s] number of models to keep for ' +
'modeling'))
glopts.add_argument(
'--optimization_id',
dest='optimization_id',
metavar="INT",
type=float,
default=None,
help="[%(default)s] ID of a pre-run optimization batch job")
reopts.add_argument('--crm',
dest='crm',
metavar="NAME",
help='chromosome name')
reopts.add_argument('--beg',
dest='beg',
metavar="INT",
type=float,
required=True,
help='genomic coordinate from which to start modeling')
reopts.add_argument('--end',
dest='end',
metavar="INT",
type=float,
required=True,
help='genomic coordinate where to end modeling')
reopts.add_argument('-r',
'--reso',
dest='reso',
metavar="INT",
type=int,
help='resolution of the Hi-C experiment')
reopts.add_argument('--perc_zero',
dest='perc_zero',
metavar="FLOAT",
type=float,
default=90.0)
opopts.add_argument(
'--optimize',
dest='optimize',
default=False,
action="store_true",
help='''optimization run, store less info about models''')
opopts.add_argument('--maxdist',
action='store',
metavar="LIST",
default=[400],
dest='maxdist',
nargs='+',
help='range of numbers for maxdist' +
', i.e. 400:1000:100 -- or just a number -- or a ' +
'list of numbers')
opopts.add_argument('--upfreq',
dest='upfreq',
metavar="LIST",
default=[0],
nargs='+',
help='range of numbers for upfreq' +
', i.e. 0:1.2:0.3 -- or just a number -- or a ' +
'list of numbers')
opopts.add_argument('--lowfreq',
dest='lowfreq',
metavar="LIST",
default=[0],
nargs='+',
help='range of numbers for lowfreq' +
', i.e. -1.2:0:0.3 -- or just a number -- or a ' +
'list of numbers')
opopts.add_argument('--scale',
dest='scale',
metavar="LIST",
default=[0.01],
nargs='+',
help='%(default)s range of numbers to be test as ' +
'optimal scale value, i.e. 0.005:0.01:0.001 -- Can ' +
'also pass only one number -- or a ' +
'list of numbers')
opopts.add_argument(
'--dcutoff',
dest='dcutoff',
metavar="LIST",
default=[2],
nargs='+',
help='%(default)s range of numbers to be test as ' +
'optimal distance cutoff parameter (distance, in ' +
'number of beads, from which to consider 2 beads as ' +
'being close), i.e. 1:5:0.5 -- Can also pass only one' +
' number -- or a list of numbers')
anopts.add_argument('--analyze',
dest='analyze',
default=False,
action="store_true",
help='''analyze models.''')
ruopts.add_argument('--nmodels_run',
dest='nmodels_run',
metavar="INT",
default=None,
type=int,
help='[ALL] number of models to run with this call')
ruopts.add_argument("-C",
"--cpu",
dest="cpus",
type=int,
default=1,
help='''[%(default)s] Maximum number of CPU
cores available in the execution host. If higher
than 1, tasks with multi-threading
capabilities will enabled (if 0 all available)
cores will be used''')
ruopts.add_argument(
'--job_list',
dest='job_list',
action='store_true',
default=False,
help=('generate a list of commands stored in a '
'file named joblist_HASH.q (where HASH is '
'replaced by a string specific to the parameters '
'used). note that dcutoff will nwver be split as '
'it does not require to re-run models.'))
# ruopts.add_argument('--job_list', dest='job_list', metavar='LIST/nothing', nargs='*',
# choices=['maxdist', 'upfreq', 'lowfreq', 'scale', 'dcutoff'],
# default=None,
# help=('[False] generate a list of commands stored in a '
# 'file named joblist_HASH.q (where HASH is '
# 'replaced by a string specific to the parameters '
# 'used). With no extra argument any combination of'
# ' optimized parameters will generate a new job. '
# 'Otherwise the parameters to be splitted can be '
# 'specified (e.g. "--job_list maxdist lowfreq '
# 'upfreq scale" will run in a single job all '
# 'combinations of dcutoff -- which makes sense as '
# 'dcutoff does not require to re-run models). '
# 'Choices are: %(choices)s'))
ruopts.add_argument(
'--tmpdb',
dest='tmpdb',
action='store',
default=None,
metavar='PATH',
type=str,
help='''if provided uses this directory to manipulate the
database''')
parser.add_argument_group(glopts)
def main():
parser = ArgumentParser(description="YTFS - YouTube Filesystem: search and play materials from YouTube using filesystem operations.", epilog="Streaming may not work if your player will read whole file into its buffer.", prog="ytfs", formatter_class=lambda prog: HelpFormatter(prog, max_help_position=50))
parser.add_argument('MOUNTPOINT', type=str, nargs=1, help="Mountpoint")
avgrp = parser.add_mutually_exclusive_group()
avgrp.add_argument('-a', action='store_true', default=False, help="Download only audio")
avgrp.add_argument('-v', action='store_true', default=False, help="Download only video")
parser.add_argument('-f', default=False, help="Preferred video format as video height (e.g. 720). Ignored if -a specified.", metavar="FORMAT")
parser.add_argument('-r', action='store_true', default=False, help="RickRoll flag")
s_grp = parser.add_mutually_exclusive_group()
s_grp.add_argument('-P', action='store_true', default=False, help="Always download whole data before reading. Useful for obtaining heighest video quality.")
parser.add_argument('-d', action='store_true', default=False, help="debug: run in foreground")
parser.add_argument('-m', default="", help="Metadata to fetch. Values: `desc` for descriptions, `thumb` for thumbnails. Use comma (,) for separating multiple values.", metavar="META1[,META2[,...]]")
avgrp.add_argument('-o', choices=['date', 'rating', 'relevance', 'title', 'viewCount'], default='relevance',
help='Specify the method that will be used to order resources. Values: `date`, `rating`, `relevance`, `title` and `viewCount`. Default is relevance.')
parser.add_argument('--allow-other', action='store_true', default=False, help="Allow other users to access the filesystem. For this to work, you'll need to set 'user_allow_other' in /etc/fuse.conf. You will need this setting if you want to share YTFS over network (SMB, NFS, etc).")
x = parser.parse_args()
if x.a:
YTStor.preferences['audio'] = True
YTStor.preferences['video'] = False
elif x.v:
YTStor.preferences['video'] = True
YTStor.preferences['audio'] = False
if x.r: YTStor.rickastley = True
if x.f:
YTStor.preferences['format'] = x.f
if x.P:
YTStor.preferences['stream'] = False
if x.m:
for m in x.m.split(','):
YTActions.preferences['metadata'][m] = True
YTActions.preferences['order'] = x.o
print("Mounting YTFS ver. " + __version__ + ".\nIf you encounter any bugs, please open an issue on GitHub: https://github.com/rasguanabana/ytfs")
FUSE(YTFS(), x.MOUNTPOINT[0], foreground=x.d, allow_other=x.allow_other)
def main():
fmt = lambda prog: HelpFormatter(prog, max_help_position=28)
desc = u"A lightweight Firefox Sync server, that just works™. If it " \
u"doesn't just work for you, please file a bug: " \
u"https://github.com/posativ/weave-minimal/issues"
parser = ArgumentParser(description=desc, formatter_class=fmt)
option = parser.add_argument
option("--host",
dest="host",
default="127.0.0.1",
type=str,
metavar="127.0.0.1",
help="host interface")
option("--port",
dest="port",
default=8080,
type=int,
metavar="8080",
help="port to listen on")
option("--log-file",
dest="logfile",
default=None,
type=str,
metavar="FILE",
help="log to a file")
option("--data-dir",
dest="data_dir",
default=".data/",
metavar="/var/...",
help="directory to store sync data, defaults to .data/")
option("--enable-registration",
dest="registration",
action="store_true",
help="enable public registration"),
option("--base-url",
dest="base_url",
default=None,
metavar="URL",
help="public URL, e.g. https://example.org/weave/")
option("--register",
dest="creds",
default=None,
metavar="user:pass",
help="register a new user and exit")
option("--use-reloader",
action="store_true",
dest="reloader",
help=SUPPRESS,
default=False)
option("--version",
action="store_true",
dest="version",
help=SUPPRESS,
default=False)
options = parser.parse_args()
if options.version:
print('weave-minimal', dist.version, end=' ')
print('(Storage API 1.1, User API 1.0)')
sys.exit(0)
if options.logfile:
handler = logging.FileHandler(options.logfile)
logger.addHandler(handler)
logging.getLogger('werkzeug').addHandler(handler)
logger.propagate = False
logging.getLogger('werkzeug').propagate = False
app = make_app(options.data_dir, options.base_url, options.registration)
if options.creds:
try:
username, passwd = options.creds.split(':', 1)
except ValueError:
logger.error("provide credentials as `user:pass`!")
sys.exit(os.EX_DATAERR)
if len(passwd) < 8:
logger.error("password too short, minimum length is 8")
sys.exit(os.EX_DATAERR)
app.initialize(encode(username), passwd)
sys.exit(os.EX_OK)
try:
from gevent.pywsgi import WSGIServer
WSGIServer((options.host, options.port), app).serve_forever()
except ImportError:
run_simple(options.host,
options.port,
app,
use_reloader=options.reloader,
threaded=True)
def populate_args(parser):
"""
parse option from call
"""
parser.formatter_class = lambda prog: HelpFormatter(
prog, width=95, max_help_position=27)
glopts = parser.add_argument_group('General options')
# glopts.add_argument('--qc_plot', dest='quality_plot', action='store_true',
# default=False,
# help='generate a quality plot of FASTQ and exits')
glopts.add_argument('-w',
'--workdir',
dest='workdir',
metavar="PATH",
action='store',
default=None,
type=str,
help='''path to working directory (generated with the
tool tadbit mapper)''')
glopts.add_argument('--type',
dest='type',
metavar="STR",
type=str,
default='map',
choices=['map', 'sam', 'bam'],
help='''[%(default)s]file type to be parser, MAP
(GEM-mapper), SAM or BAM''')
glopts.add_argument(
'--read',
dest='read',
metavar="INT",
type=int,
default=None,
help='In case only one of the reads needs to be parsed')
glopts.add_argument(
'--filter_chrom',
dest='filter_chrom',
default=
"^(chr)?[A-Za-z]?[0-9]{0,3}[XVI]{0,3}(?:ito)?[A-Z-a-z]?(_dna)?$",
help='''default: --filter_chrom "%(default)s", regexp
to consider only chromosome names passing''')
glopts.add_argument('--skip',
dest='skip',
action='store_true',
default=False,
help='[DEBUG] in case already mapped.')
glopts.add_argument('--compress_input',
dest='compress_input',
action='store_true',
default=False,
help='''Compress input mapped files when parsing is
done. This is done in background, while next MAP file is
processed, or while reads are sorted.''')
glopts.add_argument(
'--tmpdb',
dest='tmpdb',
action='store',
default=None,
metavar='PATH',
type=str,
help='''if provided uses this directory to manipulate the
database''')
glopts.add_argument('--genome',
dest='genome',
metavar="PATH",
nargs='+',
type=str,
help='''paths to file(s) with FASTA files of the
reference genome. If many, files will be concatenated.
I.e.: --genome chr_1.fa chr_2.fa
In this last case, order is important or the rest of the
analysis. Note: it can also be the path to a previously
parsed genome in pickle format.''')
glopts.add_argument(
'--jobids',
dest='jobids',
metavar="INT",
action='store',
default=None,
nargs='+',
type=int,
help='''Use as input data generated by a job with a given
jobid(s). Use tadbit describe to find out which.
In this case one jobid can be passed per read.''')
glopts.add_argument('--noX',
action='store_true',
help='no display server (X screen)')
parser.add_argument_group(glopts)
def populate_args(parser):
"""
parse option from call
"""
parser.formatter_class = lambda prog: HelpFormatter(
prog, width=95, max_help_position=27)
glopts = parser.add_argument_group('General options')
glopts.add_argument('-w',
'--workdir',
dest='workdir',
metavar="PATH",
action='store',
default=None,
type=str,
help='''path to working directory (generated with the
tool tadbit map)''')
glopts.add_argument(
'-t',
'--tables',
dest='tables',
metavar='',
action='store',
nargs='+',
type=str,
default=[str(t) for t in range(1,
len(TABLE_IDX) + 1)],
help=('[%(default)s] what tables to show, write either '
'the sequence of names or indexes, according to '
'this list: {}').format(', '.join(
['%s: %s' % (k, v) for k, v in TABLE_IDX.iteritems()])))
glopts.add_argument(
'-T',
'--skip_tables',
dest='skip_tables',
metavar='',
action='store',
nargs='+',
type=str,
default=[],
help=('[%(default)s] what tables NOT to show, write either '
'the sequence of names or indexes, according to '
'this list: {}').format(', '.join(
['%s: %s' % (k, v) for k, v in TABLE_IDX.iteritems()])))
glopts.add_argument('-j',
'--jobids',
dest='jobids',
metavar="INT",
nargs='+',
action='store',
default=None,
type=int,
help='''Display only items matching these jobids.''')
glopts.add_argument('-W',
'--where',
dest='where',
metavar="STR",
action='store',
default={},
type=str,
nargs='+',
help='''ensemble of keyword and value to filter results
in the form: `-s Chromosome,18`''')
glopts.add_argument('-s',
'--select',
dest='select',
metavar="STR",
action='store',
default={},
type=str,
nargs='+',
help='''ensemble of keyword to show results. E.g.
to show only JobIds: `-s Jobids`''')
glopts.add_argument(
'--tmpdb',
dest='tmpdb',
action='store',
default=None,
metavar='PATH',
type=str,
help='''if provided uses this directory to manipulate the
database''')
glopts.add_argument('--tsv',
dest='tsv',
default=False,
action='store_true',
help='''Print output in tab separated format''')
glopts.add_argument('-o',
'--output',
dest='output',
default=None,
action='store',
help='''Writes output in specified file.''')
parser.add_argument_group(glopts)
def main():
"""
The main command-line argument parser for the Marmoset submission
command-line interface tool. Parses the arguments and determines
the appropriate call to take.
@param arguments: List of command line arguments.
@return: None
"""
parser = ArgumentParser(
prog=PROG,
description=DESCRIPTION,
version="%s %s" % (PROG, VERSION),
epilog=EPILOG,
formatter_class=lambda prog: HelpFormatter(prog, max_help_position=30))
parser_a = ArgumentParser(add_help=False)
parser_a.add_argument('course', help="specify a course")
parser_a.add_argument('assignment', help="specify an assignment")
parser_a.add_argument(
'-u',
'--username',
dest='username',
metavar='username',
help="user to login as, defaults to the default user")
parser_a.add_argument('-n',
'--nosave',
action='store_true',
help="don't save the user being logged in as")
parser_b = ArgumentParser(add_help=False)
parser_b.add_argument('submission',
nargs='?',
type=int,
help="specific submission or number of submissions")
parser_c = ArgumentParser(add_help=False)
group = parser_c.add_argument_group(title="keyring utilities")
group.add_argument('-a',
'--adduser',
action='store_true',
help="add a user to the marmoset keyring")
group.add_argument('-c',
'--changeuser',
action='store_true',
help="change the default user in the marmoset keyring")
group.add_argument('-r',
'--removeuser',
action='store_true',
help="remove a user from the marmoset keyring")
subparsers = parser.add_subparsers(dest='method', title="commands")
commands = sorted(
['submit', 'download', 'fetch', 'long', 'release', 'keyring'])
for cmd in commands:
if cmd == 'submit':
sub = subparsers.add_parser(
cmd,
parents=[parser_a],
help="submit file(s) to the marmoset submission server")
sub.add_argument('files',
nargs='+',
help="one or more files to submit")
elif cmd == 'keyring':
sub = subparsers.add_parser(cmd,
parents=[parser_c],
help="modify the marmoset keyring")
sub.add_argument('username', help="marmoset username")
else:
sub = subparsers.add_parser(
cmd,
parents=[parser_a, parser_b],
help=
"{0} the most recent submission(s). Defaults to last/latest".
format(cmd))
if cmd in ['submit', 'download']:
sub.add_argument('-z',
'--zipname',
type=str,
metavar='zipname',
help="name of the zipfile to submit/download")
args = vars(parser.parse_args())
if len(args) == 0:
parser.print_help()
return
MarmosetSession().call(**args)
def populate_args(parser):
"""
parse option from call
"""
masked = {
1: {
'name': 'self-circle'
},
2: {
'name': 'dangling-end'
},
3: {
'name': 'error'
},
4: {
'name': 'extra dangling-end'
},
5: {
'name': 'too close from RES'
},
6: {
'name': 'too short'
},
7: {
'name': 'too large'
},
8: {
'name': 'over-represented'
},
9: {
'name': 'duplicated'
},
10: {
'name': 'random breaks'
}
}
parser.formatter_class = lambda prog: HelpFormatter(
prog, width=95, max_help_position=27)
glopts = parser.add_argument_group('General options')
output = parser.add_argument_group('Storage options')
filter_ = parser.add_argument_group('Filtering options')
glopts.add_argument('--force',
dest='force',
action='store_true',
default=False,
help='overwrite previously run job')
glopts.add_argument('--resume',
dest='resume',
action='store_true',
default=False,
help='use filters of previously run job')
filter_.add_argument(
'--apply',
dest='apply',
nargs='+',
type=int,
metavar='INT',
default=[1, 2, 3, 4, 6, 7, 9, 10],
choices=list(range(1, 11)),
help=(
"""[%(default)s] Use filters to define a set os valid pair of reads
e.g.: '--apply 1 2 3 4 6 7 8 9'. Where these numbers"""
+ "correspond to: %s" %
(', '.join(['%2d: %15s' % (k, masked[k]['name'])
for k in masked]))))
glopts.add_argument('-w',
'--workdir',
dest='workdir',
metavar="PATH",
action='store',
default=None,
type=str,
help='''path to working directory (generated with the
tool tadbit mapper)''')
glopts.add_argument("-C",
"--cpus",
dest="cpus",
type=int,
default=cpu_count(),
help='''[%(default)s] Maximum
number of CPU cores available in the execution host.
If higher than 1, tasks with multi-threading
capabilities will enabled (if 0 all available)
cores will be used''')
glopts.add_argument('--noX',
action='store_true',
help='no display server (X screen)')
filter_.add_argument(
'--over_represented',
dest='over_represented',
metavar="NUM",
action='store',
default=0.001,
type=float,
help='''[%(default)s%%] percentage of restriction-enzyme
(RE) genomic fragments with more coverage to exclude
(possible PCR artifact).''')
filter_.add_argument('--max_frag_size',
dest='max_frag_size',
metavar="NUM",
action='store',
default=100000,
type=int,
help='''[%(default)s] to exclude large genomic RE
fragments (probably resulting from gaps in the reference
genome)''')
filter_.add_argument('--min_frag_size',
dest='min_frag_size',
metavar="NUM",
action='store',
default=50,
type=int,
help='''[%(default)s] to exclude small genomic RE
fragments (smaller than sequenced reads)''')
filter_.add_argument('--re_proximity',
dest='re_proximity',
metavar="NUM",
action='store',
default=5,
type=int,
help='''[%(default)s] to exclude read-ends falling too
close from RE site (pseudo-dangling-ends)''')
glopts.add_argument(
'--tmpdb',
dest='tmpdb',
action='store',
default=None,
metavar='PATH',
type=str,
help='''if provided uses this directory to manipulate the
database''')
glopts.add_argument(
'--pathids',
dest='pathids',
metavar="INT",
action='store',
default=None,
nargs='+',
type=int,
help='''Use as input data generated by a job under a given
pathids. Use tadbit describe to find out which.
To filter an intersected file produced with tadbit map
--fast_fragment only one PATHid is needed otherwise one
per read is needed, first for read 1, second for read 2.'''
)
glopts.add_argument('--compress_input',
dest='compress_input',
action='store_true',
default=False,
help='''Compress input mapped files when parsing is
done. This is done in background, while next MAP file is
processed, or while reads are sorted.''')
output.add_argument('--format',
dest='format',
default='mid',
choices=['short', 'mid', 'long'],
help='''[%(default)s] for compression into pseudo-BAM
format. Short contains only positions of reads mapped,
mid everything but restriction sites.''')
output.add_argument('--valid',
dest='valid',
default=False,
action='store_true',
help='''stores only valid-pairs discards filtered out
reads.''')
output.add_argument('--clean',
dest='clean',
default=False,
action='store_true',
help='''remove intermediate files. WARNING: together
with format "short" or valid options, this may results
in losing data''')
glopts.add_argument('--samtools',
dest='samtools',
metavar="PATH",
action='store',
default='samtools',
type=str,
help='''path samtools binary''')
parser.add_argument_group(glopts)
def populate_args(parser):
"""
parse option from call
"""
parser.formatter_class = lambda prog: HelpFormatter(
prog, width=95, max_help_position=27)
oblopt = parser.add_argument_group('Required options')
glopts = parser.add_argument_group('General options')
rfiltr = parser.add_argument_group('Read filtering options')
normpt = parser.add_argument_group('Normalization options')
outopt = parser.add_argument_group('Output options')
pltopt = parser.add_argument_group('Plotting options')
oblopt.add_argument('-w',
'--workdir',
dest='workdir',
metavar="PATH",
action='store',
default=None,
type=str,
required=True,
help='''path to working directory (generated with the
tool tadbit mapper)''')
glopts.add_argument('--noX',
help='no display server (X screen)',
dest='nox',
action='store_true')
oblopt.add_argument('-r',
'--resolution',
dest='reso',
metavar="INT",
action='store',
default=None,
type=int,
required=True,
help='''resolution at which to output matrices''')
glopts.add_argument('--bam',
dest='bam',
metavar="PATH",
action='store',
default=None,
type=str,
help='''path to a TADbit-generated BAM file with
all reads (other wise the tool will guess from the
working directory database)''')
glopts.add_argument(
'-j',
'--jobid',
dest='jobid',
metavar="INT",
action='store',
default=None,
type=int,
help='''Use as input data generated by a job with a given
jobid. Use tadbit describe to find out which.''')
glopts.add_argument('--force',
dest='force',
action='store_true',
default=False,
help='overwrite previously run job')
glopts.add_argument('-q',
'--quiet',
dest='quiet',
action='store_true',
default=False,
help='remove all messages')
glopts.add_argument(
'--tmpdb',
dest='tmpdb',
action='store',
default=None,
metavar='PATH',
type=str,
help='''if provided uses this directory to manipulate the
database''')
glopts.add_argument('--nchunks',
dest='nchunks',
action='store',
default=100,
type=int,
help='''maximum number of chunks into which to
cut the BAM''')
glopts.add_argument("-C",
"--cpus",
dest="cpus",
type=int,
default=cpu_count(),
help='''[%(default)s] Maximum
number of CPU cores available in the execution host.
If higher than 1, tasks with multi-threading
capabilities will enabled (if 0 all available)
cores will be used''')
glopts.add_argument(
'--chr_name',
dest='chr_name',
metavar="STR",
nargs='+',
default=None,
type=str,
help='''[fasta header] chromosome name(s). Order of chromosomes
in the output matrices.''')
outopt.add_argument('--matrix',
dest='matrix',
action='store_true',
default=False,
help='''Write text matrix in multiple columns (square).
By defaults matrices are written in BED-like format (also
only way to get a raw matrix with all values including
the ones in masked columns).''')
outopt.add_argument(
'--cooler',
dest='cooler',
action='store_true',
default=False,
help='''Write i,j,v matrix in cooler format instead of text.
''')
outopt.add_argument('--rownames',
dest='row_names',
action='store_true',
default=False,
help='''To store row names in the output text matrix.
WARNING: when non-matrix, results in two extra columns'''
)
pltopt.add_argument('--plot',
dest='plot',
action='store_true',
default=False,
help='Plot matrix in desired format.')
pltopt.add_argument('--force_plot',
dest='force_plot',
action='store_true',
default=False,
help=('Force plotting even with demoniacally big '
'matrices (more than 5000x5000, or 1500x1500'
'with interactive option).'))
outopt.add_argument(
'--only_plot',
dest='only_plot',
action='store_true',
default=False,
help='[%(default)s] Skip writing matrix in text format.')
outopt.add_argument('-i',
'--interactive',
dest='interactive',
action='store_true',
default=False,
help='''[%(default)s] Open matplotlib interactive plot
(nothing will be saved).''')
pltopt.add_argument('--triangular',
dest='triangular',
action='store_true',
default=False,
help='''[%(default)s] represents only half matrix. Note
that this also results in truly vectorial images of
matrix.''')
pltopt.add_argument('--xtick_rotation',
dest='xtick_rotation',
default=-25,
type=int,
help='''[%(default)s] x-tick rotation''')
pltopt.add_argument('--cmap',
dest='cmap',
action='store',
default='viridis',
help='[%(default)s] Matplotlib color map to use.')
pltopt.add_argument('--bad_color',
dest='bad_color',
action='store',
default='white',
help='''[%(default)s] Matplotlib color to use on bins
filtered out (only used with normalized matrices, not
raw).''')
pltopt.add_argument('--format',
dest='format',
action='store',
default='png',
help='[%(default)s] plot file format.')
pltopt.add_argument('--zrange',
dest='zrange',
action='store',
default=None,
help='''Range, in log2 scale of the color scale.
i.e.: --zrange=-2,2''')
pltopt.add_argument('--transform',
dest='transform',
action='store',
default='log2',
choices=['log2', 'log', 'none'],
help='''[%(default)s] can be any of [%(choices)s]''')
pltopt.add_argument('--figsize',
dest='figsize',
action='store',
default=None,
help='''Range, in log2 scale of the color scale.
default for triangular matrices: --figsize=16,10
and for square matrices: --figsize=16,14''')
pltopt.add_argument('--tad_def',
dest='tad_def',
action='store',
default=None,
help='''tsv file with tad definition, columns:
# start end score density''')
outopt.add_argument('-c',
'--coord',
dest='coord1',
metavar='',
default=None,
help='''Coordinate of the region to
retrieve. By default all genome, arguments can be
either one chromosome name, or the coordinate in
the form: "-c chr3:110000000-120000000"''')
outopt.add_argument('-c2',
'--coord2',
dest='coord2',
metavar='',
default=None,
help='''Coordinate of a second region to
retrieve the matrix in the intersection with the first
region.''')
normpt.add_argument('--biases',
dest='biases',
metavar="PATH",
action='store',
default=None,
type=str,
help='''path to file with pre-calculated biases by
columns''')
normpt.add_argument('--norm',
dest='normalizations',
metavar="STR",
action='store',
default=['raw'],
type=str,
nargs='+',
choices=['norm', 'decay', 'raw', 'raw&decay'],
help='''[%(default)s] normalization(s) to apply.
Choices are: [%(choices)s]''')
rfiltr.add_argument(
'-F',
'--filter',
dest='filter',
nargs='+',
type=int,
metavar='INT',
default=[1, 2, 3, 4, 6, 7, 9, 10],
choices=list(range(0, 11)),
help=(
"""[%(default)s] Use filters to define a set os
valid pair of reads e.g.:
'--apply 1 2 3 4 8 9 10'. Where these numbers""" +
"correspond to: 0: nothing, %s" %
(', '.join(['%2d: %15s' % (k, MASKED[k]['name'])
for k in MASKED]))))
outopt.add_argument('--only_txt',
dest='only_txt',
action='store_true',
default=False,
help='Save only text file for matrices, not images')
def main(argv):
parser = ArgumentParser(argv[0], description=__doc__,
formatter_class=lambda prog: HelpFormatter(prog, max_help_position=10, width=120))
parser.add_argument('dataset', type=str, nargs='+',
help='Dataset(s) used for training.')
parser.add_argument('output', type=str,
help='Directory or file where trained models will be stored.')
parser.add_argument('--num_components', '-c', type=int, default=3,
help='Number of components used in STM model (default: %(default)d).')
parser.add_argument('--num_features', '-f', type=int, default=2,
help='Number of quadratic features used in STM model (default: %(default)d).')
parser.add_argument('--num_models', '-m', type=int, default=4,
help='Number of models trained (predictions will be averaged across models, default: %(default)d).')
parser.add_argument('--keep_all', '-k', type=int, default=1,
help='If set to 0, only the best model of all trained models is kept (default: %(default)d).')
parser.add_argument('--finetune', '-n', type=int, default=0,
help='If set to 1, enables another finetuning step which is performed after training (default: %(default)d).')
parser.add_argument('--num_train', '-t', type=int, default=0,
help='If specified, a (random) subset of cells is used for training.')
parser.add_argument('--num_valid', '-s', type=int, default=0,
help='If specified, a (random) subset of cells will be used for early stopping based on validation error.')
parser.add_argument('--var_explained', '-e', type=float, default=95.,
help='Controls the degree of dimensionality reduction of fluorescence windows (default: %(default).0f).')
parser.add_argument('--window_length', '-w', type=float, default=1000.,
help='Length of windows extracted from calcium signal for prediction (in milliseconds, default: %(default).0f).')
parser.add_argument('--regularize', '-r', type=float, default=0.,
help='Amount of parameter regularization (filters are regularized for smoothness, default: %(default).1f).')
parser.add_argument('--preprocess', '-p', type=int, default=0,
help='If the data is not already preprocessed, this can be used to do it.')
parser.add_argument('--verbosity', '-v', type=int, default=1)
args, _ = parser.parse_known_args(argv[1:])
experiment = Experiment()
if not args.dataset:
print 'You have to specify at least 1 dataset.'
return 0
data = []
for dataset in args.dataset:
with open(dataset) as handle:
data = data + load(handle)
if args.preprocess:
data = preprocess(data, args.verbosity)
if 'cell_num' not in data[0]:
# no cell number is given, assume traces correspond to cells
for k, entry in enumerate(data):
entry['cell_num'] = k
# collect cell ids
cell_ids = unique([entry['cell_num'] for entry in data])
# pick cells for training
if args.num_train > 0:
training_cells = random_select(args.num_train, len(cell_ids))
else:
# use all cells for training
training_cells = range(len(cell_ids))
models = train([entry for entry in data if entry['cell_num'] in training_cells],
num_valid=args.num_valid,
num_models=args.num_models,
var_explained=args.var_explained,
window_length=args.window_length,
keep_all=args.keep_all,
finetune=args.finetune,
model_parameters={
'num_components': args.num_components,
'num_features': args.num_features},
training_parameters={
'verbosity': 1},
regularize=args.regularize,
verbosity=args.verbosity)
experiment['args'] = args
experiment['training_cells'] = training_cells
experiment['models'] = models
if os.path.isdir(args.output):
experiment.save(os.path.join(args.output, 'model.xpck'))
else:
experiment.save(args.output)
return 0
def parse_args():
'''
Parse command-line options found in 'args' (default: sys.argv[1:]).
:returns: On success, a namedtuple of Values instances.
'''
parser = ArgumentParser(
prog='s-worker',
description=
'Simple Worker listens to a partition/topic of the Kafka cluster and '
'stores incoming records to the HDFS.',
epilog='END',
formatter_class=lambda prog: HelpFormatter(
prog, max_help_position=42, width=90),
usage=
's-worker [OPTIONS]... --topic <topic> -p <partition> -d <HDFS folder>'
)
# .................................set optional arguments
parser._optionals.title = 'Optional Arguments'
parser.add_argument('-c',
'--config-file',
metavar='FILE',
type=file,
help='path of configuration file',
default=os.path.expanduser('~/.worker.json'))
parser.add_argument(
'-i',
'--interval',
default=10000,
metavar='INTEGER',
type=int,
help=
'milliseconds spent waiting in poll if data is not available in the buffer'
)
parser.add_argument('-l',
'--log-level',
default='INFO',
metavar='STRING',
help='determine the level of the logger')
parser.add_argument('--parallel-processes',
default=cpu_count(),
type=int,
help='number of the parallel processes',
metavar='INTEGER')
parser.add_argument('-v',
'--version',
action='version',
version='%(prog)s {0}'.format(__version__))
# .................................set required arguments
required = parser.add_argument_group('Required Arguments')
required.add_argument('-d',
'--hdfs-directory',
required=True,
metavar='STRING',
help='destination folder in HDFS')
required.add_argument('-p',
'--partition',
required=True,
metavar='INTEGER',
help='partition number to consume')
required.add_argument('--topic',
required=True,
metavar='STRING',
help='topic to listen for new messages')
return parser.parse_args()
def construct_parser(description, epilog=''):
# construct main module
module_args = {
'description': description,
'usage': __command__,
'formatter_class':
SubcommandHelpFormatter # remove sub-command braces in help
}
if epilog:
module_args['epilog'] = epilog
parser = ArgumentParser(**module_args)
current_version = '%s %s' % (__module__, __version__)
parser.add_argument('-v',
'--version',
action='version',
version=current_version)
# construct command parsing method
help_details = {
'title': 'list of commands' # title for sub-commands list in help
}
subparsers = parser.add_subparsers(**help_details)
# retrieve module path
module_path = find_spec(__module__).submodule_search_locations[0]
# compile command models
compile_kwargs = {
'folder_path': '%s/commands/' % module_path,
'module_name': __module__,
'fields_model': fields_model,
'preferred_order': __order__
}
command_list = compile_commands(**compile_kwargs)
# construct each command
for command_details in command_list:
# add command details to parser
try:
details = {
'description':
command_details['description'],
'help':
command_details['help'],
'formatter_class':
lambda prog:
HelpFormatter(prog, max_help_position=30, width=80
) # adjusts column width to options
}
if command_details['epilog']:
details['epilog'] = command_details['epilog']
cmd_args = subparsers.add_parser(command_details['name'],
**details)
# construct default arguments
cmd_args.set_defaults(**command_details['default_args'])
# construct positional arguments
for argument in command_details['positional_args']:
cmd_args.add_argument(argument['args'], **argument['kwargs'])
# construct optional arguments
for argument in command_details['optional_args']:
cmd_args.add_argument(*argument['args'], **argument['kwargs'])
# construct mutually exclusive arguments
for key in command_details['exclusive_args'].keys():
exclusive_options = cmd_args.add_mutually_exclusive_group()
for argument in command_details['exclusive_args'][key]:
exclusive_options.add_argument(*argument['args'],
**argument['kwargs'])
# remove command from module if there is model parsing error
except Exception as err:
# print(err)
pass
return parser
