Example #1
0
 def __init__(self, **kwargs):
     OptionParser.__init__(self, **kwargs)
     self.add_option('-E', '--exclude',
                     help='exclude match')
     self.add_option('-I', '--include',
                     help='include match')
     self.add_option('-i', '--ignore-missing',
                     action='store_true',
                     help='ignore missing tags')
     self.add_option('-n', '--replicas',
                     help='number of replicas to create when pushing')
     self.add_option('-p', '--prefix',
                     action='store_true',
                     help='prefix mode for commands that take it.')
     self.add_option('-r', '--recursive',
                     action='store_true',
                     help='recursively perform operations')
     self.add_option('-w', '--warn-missing',
                     action='store_true',
                     help='warn about missing tags')
     self.add_option('-x', '--tarballs',
                     action='store_true',
                     help='extract files as tarballs when pushing')
     self.add_option('-z', '--compress',
                     action='store_true',
                     help='compress tar blobs when pushing')
Example #2
0
 def __init__(self, **kwargs):
     OptionParser.__init__(self, **kwargs)
     self.add_option('-H', '--host',
                     help='host that the client should connect to')
     self.add_option('-P', '--port',
                     help='port that the client should connect to')
     self.add_option('-n', '--nr-ichunks',
                     help='the number of ichunks to create')
     self.add_option('-p', '--profile',
                     action='store_true',
                     help='turn on job profiling')
     self.add_option('--parser',
                     help='parser object to use for indexing')
     self.add_option('--demuxer',
                     help='demuxer object to user for indexing')
     self.add_option('--balancer',
                     help='balancer to use for indexing')
     self.add_option('--metakeyer',
                     help='balancer to use for indexing')
     self.add_option('--no-sort',
                     action='store_true',
                     help='whether or not to sort before writing discodbs')
     self.add_option('--k-viter',
                     action='store_true',
                     help='treat the dataset as a precomputed k-viter list')
Example #3
0
 def __init__(self, **kwargs):
     OptionParser.__init__(self, **kwargs)
     self.add_option('-H',
                     '--host',
                     help='host that the client should connect to')
     self.add_option('-P',
                     '--port',
                     help='port that the client should connect to')
Example #4
0
 def __init__(self, **kwargs):
     OptionParser.__init__(self, **kwargs)
     self.add_option('-k', '--sort-stats',
                     action='append',
                     default=[],
                     help='keys to use for sorting profiling statistics')
     self.add_option('-o', '--offset',
                     default=0,
                     help='offset to use in requests to disco master')
     self.add_option('-S', '--status',
                     action='store_true',
                     help='show job status when printing jobs')
Example #5
0
 def __init__(self, **kwargs):
     OptionParser.__init__(self, **kwargs)
     self.add_option('-k',
                     '--sort-stats',
                     action='append',
                     default=[],
                     help='keys to use for sorting profiling statistics')
     self.add_option('-o',
                     '--offset',
                     default=0,
                     help='offset to use in requests to disco master')
     self.add_option('-S',
                     '--status',
                     action='store_true',
                     help='show job status when printing jobs')
     self.add_option('-n',
                     '--name',
                     help='prefix to use for submitting a job')
     self.add_option('--save',
                     action='callback',
                     callback=self.update_jobdict,
                     help='save results to DDFS')
     self.add_option('--sort',
                     action='callback',
                     callback=self.update_jobdict,
                     help='sort input to reduce')
     self.add_option('--profile',
                     action='callback',
                     callback=self.update_jobdict,
                     help='enable job profiling')
     self.add_option('--param',
                     action='append',
                     default=[],
                     dest='params',
                     nargs=2,
                     help='add a job parameter')
     self.add_option('--partitions',
                     action='callback',
                     callback=self.update_jobdict,
                     type='int',
                     help='enable job profiling')
     self.add_option('--sched_max_cores',
                     action='callback',
                     callback=self.update_jobdict,
                     type='int',
                     help='enable job profiling')
     self.add_option('--status_interval',
                     action='callback',
                     callback=self.update_jobdict,
                     type='int',
                     help='how often to report status in job')
     self.jobdict = {}
Example #6
0
 def __init__(self, **kwargs):
     OptionParser.__init__(self, **kwargs)
     self.add_option('-k', '--sort-stats',
                     action='append',
                     default=[],
                     help='keys to use for sorting profiling statistics')
     self.add_option('-o', '--offset',
                     default=0,
                     help='offset to use in requests to disco master')
     self.add_option('-S', '--status',
                     action='store_true',
                     help='show job status when printing jobs')
     self.add_option('-n', '--name',
                     help='prefix to use for submitting a job')
     self.add_option('--save',
                     action='callback',
                     callback=self.update_jobdict,
                     help='save results to DDFS')
     self.add_option('--sort',
                     action='callback',
                     callback=self.update_jobdict,
                     help='sort input to reduce')
     self.add_option('--profile',
                     action='callback',
                     callback=self.update_jobdict,
                     help='enable job profiling')
     self.add_option('--param',
                     action='append',
                     default=[],
                     dest='params',
                     nargs=2,
                     help='add a job parameter')
     self.add_option('--partitions',
                     action='callback',
                     callback=self.update_jobdict,
                     type='int',
                     help='enable job profiling')
     self.add_option('--sched_max_cores',
                     action='callback',
                     callback=self.update_jobdict,
                     type='int',
                     help='enable job profiling')
     self.add_option('--status_interval',
                     action='callback',
                     callback=self.update_jobdict,
                     type='int',
                     help='how often to report status in job')
     self.jobdict = {}
Example #7
0
 def __init__(self, **kwargs):
     OptionParser.__init__(self, **kwargs)
     self.add_option('-f',
                     '--files',
                     action='store_true',
                     help='file mode for commands that take it.')
     self.add_option('-H',
                     '--host',
                     help='host that the client should connect to')
     self.add_option('-P',
                     '--port',
                     help='port that the client should connect to')
     self.add_option('-n',
                     '--nr-ichunks',
                     help='the number of ichunks to create')
     self.add_option('-u',
                     '--unique-items',
                     action='store_true',
                     help='ensure unique items in discodbs')
     self.add_option('-p',
                     '--profile',
                     action='store_true',
                     help='turn on job profiling')
     self.add_option('--parser', help='parser object to use for indexing')
     self.add_option('--demuxer',
                     help='demuxer object to user for indexing')
     self.add_option('--balancer', help='balancer to use for indexing')
     self.add_option('--param',
                     action='append',
                     default=[],
                     dest='params',
                     nargs=2,
                     help='add a param to the inquiry job params')
     self.add_option(
         '--stream',
         action='append',
         default=[],
         dest='streams',
         help='add a stream for indexing or filtering inquiries')
     self.add_option('--reduce', help='reduce used for filtering inquiries')
Example #8
0
 def __init__(self, **kwargs):
     OptionParser.__init__(self, **kwargs)
     self.add_option('-E', '--exclude',
                     help='exclude match')
     self.add_option('-I', '--include',
                     help='include match')
     self.add_option('-f', '--files',
                     action='store_true',
                     help='file mode for commands that take it')
     self.add_option('-i', '--ignore-missing',
                     action='store_true',
                     help='ignore missing tags')
     self.add_option('-n', '--replicas',
                     help='number of replicas to create when pushing')
     self.add_option('-p', '--prefix',
                     action='store_true',
                     help='prefix mode for commands that take it')
     self.add_option('-R', '--reader',
                     help='input reader to import and use')
     self.add_option('-r', '--recursive',
                     action='store_true',
                     help='recursively perform operations')
     self.add_option('-T', '--stream',
                     default='disco.func.default_stream',
                     help='input stream to import and use')
     self.add_option('-t', '--token',
                     help='authorization token for the command')
     self.add_option('-u', '--update',
                     action='store_true',
                     help='whether to perform an update or an append')
     self.add_option('-w', '--warn-missing',
                     action='store_true',
                     help='warn about missing tags')
     self.add_option('-x', '--tarballs',
                     action='store_true',
                     help='extract files as tarballs when pushing')
     self.add_option('-z', '--compress',
                     action='store_true',
                     help='compress tar blobs when pushing')
Example #9
0
 def __init__(self, **kwargs):
     OptionParser.__init__(self, **kwargs)
     self.add_option('-f', '--files',
                     action='store_true',
                     help='file mode for commands that take it.')
     self.add_option('-H', '--host',
                     help='host that the client should connect to')
     self.add_option('-P', '--port',
                     help='port that the client should connect to')
     self.add_option('-n', '--nr-ichunks',
                     help='the number of ichunks to create')
     self.add_option('-u', '--unique-items',
                     action='store_true',
                     help='ensure unique items in discodbs')
     self.add_option('-p', '--profile',
                     action='store_true',
                     help='turn on job profiling')
     self.add_option('--parser',
                     help='parser object to use for indexing')
     self.add_option('--demuxer',
                     help='demuxer object to user for indexing')
     self.add_option('--balancer',
                     help='balancer to use for indexing')
     self.add_option('--param',
                     action='append',
                     default=[],
                     dest='params',
                     nargs=2,
                     help='add a param to the inquiry job params')
     self.add_option('--stream',
                     action='append',
                     default=[],
                     dest='streams',
                     help='add a stream for indexing or filtering inquiries')
     self.add_option('--reduce',
                     help='reduce used for filtering inquiries')
Example #10
0
 def __init__(self, **kwargs):
     OptionParser.__init__(self, **kwargs)
     self.add_option('-H', '--host',
                     help='host that the client should connect to')
     self.add_option('-P', '--port',
                     help='port that the client should connect to')
Example #11
0
File: fq.py Project: sqs/freequery
 def __init__(self, **kwargs):
     OptionParser.__init__(self, **kwargs)
     self.add_option("-u", "--unranked", action="store_true", help="return unranked query results")
     self.add_option("-p", "--profile", action="store_true", help="show profiler stats for Disco jobs")
Example #12
0
 def __init__(self, **kwargs):
     OptionParser.__init__(self, **kwargs)