def get_query_slices(total_size, ideal_chunk_size=None, min_chunk_size=None):
from ctk.config import get_config
conf = get_config()
if not ideal_chunk_size:
ideal_chunk_size = conf.sqlalchemy_ideal_chunk_size
if not min_chunk_size:
min_chunk_size = conf.sqlalchemy_min_chunk_size
if ideal_chunk_size >= total_size:
yield (0, total_size)
raise StopIteration
start = 0 - ideal_chunk_size
while True:
start += ideal_chunk_size
end = start + ideal_chunk_size - 1
if end > total_size:
yield (start, total_size)
raise StopIteration
next_start = start + ideal_chunk_size
next_end = min(next_start + ideal_chunk_size - 1, total_size)
if next_end - next_start < min_chunk_size:
yield (start, total_size)
raise StopIteration
yield (start, end)
def get_query_slices(total_size, ideal_chunk_size=None, min_chunk_size=None):
from ctk.config import get_config
conf = get_config()
if not ideal_chunk_size:
ideal_chunk_size = conf.sqlalchemy_ideal_chunk_size
if not min_chunk_size:
min_chunk_size = conf.sqlalchemy_min_chunk_size
if ideal_chunk_size >= total_size:
yield (0, total_size)
raise StopIteration
start = 0 - ideal_chunk_size
while True:
start += ideal_chunk_size
end = start + ideal_chunk_size - 1
if end > total_size:
yield (start, total_size)
raise StopIteration
next_start = start + ideal_chunk_size
next_end = min(next_start + ideal_chunk_size - 1, total_size)
if next_end - next_start < min_chunk_size:
yield (start, total_size)
raise StopIteration
yield (start, end)
def run(self, args):
k = Dict()
k.prog = self.prog
if self._usage_:
k.usage = self._usage_
if self._description_:
k.description = self._description_
else:
docstring = self.command.__doc__
if docstring:
k.description = textwrap.dedent(docstring)
self.parser = optparse.OptionParser(**k)
if self.command._verbose_:
assert self.command._quiet_ is None
self.parser.add_option(
'-v',
'--verbose',
dest='verbose',
action='store_true',
default=False,
help="run in verbose mode [default: %default]")
if self.command._quiet_:
assert self.command._verbose_ is None
self.parser.add_option(
'-q',
'--quiet',
dest='quiet',
action='store_true',
default=False,
help="run in quiet mode [default: %default]")
if self.command._conf_:
self.parser.add_option(
'-c',
'--conf',
metavar='FILE',
help="use alternate configuration file FILE")
self._add_parser_options()
(opts, self.args) = self.parser.parse_args(args)
# Ignore variable argument commands altogether.
# xxx: todo
if 0 and self._vargc_ is not True:
arglen = len(self.args)
if arglen == 0 and self._argc_ != 0:
self.parser.print_help()
self.parser.exit(status=1)
if len(self.args) != self._argc_ and self._argc_ != 0:
self.usage_error("invalid number of arguments")
self.options = Options(opts.__dict__)
if self.mandatory_opts:
d = opts.__dict__
for (opt, name) in self.mandatory_opts.items():
if d.get(name) is None:
self.usage_error("%s is mandatory" % '/'.join(opt))
#self._pre_process_parser_results()
f = None
if self._conf_:
f = self.options.conf
if f and not os.path.exists(f):
self.usage_error("configuration file '%s' does not exist" % f)
try:
self.conf = self.config_class(options=self.options)
self.conf.load(filename=f)
except ConfigObjectAlreadyCreated:
self.conf = get_config()
self.command.conf = self.conf
self.command.args = self.args
self.command.options = self.options
self.command.start()
def run(self, args):
k = Dict()
k.prog = self.prog
if self._usage_:
k.usage = self._usage_
if self._description_:
k.description = self._description_
else:
docstring = self.command.__doc__
if docstring:
k.description = textwrap.dedent(docstring)
self.parser = optparse.OptionParser(**k)
if self.command._verbose_:
assert self.command._quiet_ is None
self.parser.add_option(
'-v', '--verbose',
dest='verbose',
action='store_true',
default=False,
help="run in verbose mode [default: %default]"
)
if self.command._quiet_:
assert self.command._verbose_ is None
self.parser.add_option(
'-q', '--quiet',
dest='quiet',
action='store_true',
default=False,
help="run in quiet mode [default: %default]"
)
if self.command._conf_:
self.parser.add_option(
'-c', '--conf',
metavar='FILE',
help="use alternate configuration file FILE"
)
self._add_parser_options()
(opts, self.args) = self.parser.parse_args(args)
# Ignore variable argument commands altogether.
# xxx: todo
if 0 and self._vargc_ is not True:
arglen = len(self.args)
if arglen == 0 and self._argc_ != 0:
self.parser.print_help()
self.parser.exit(status=1)
if len(self.args) != self._argc_ and self._argc_ != 0:
self.usage_error("invalid number of arguments")
self.options = Options(opts.__dict__)
if self.mandatory_opts:
d = opts.__dict__
for (opt, name) in self.mandatory_opts.items():
if d.get(name) is None:
self.usage_error("%s is mandatory" % '/'.join(opt))
#self._pre_process_parser_results()
f = None
if self._conf_:
f = self.options.conf
if f and not os.path.exists(f):
self.usage_error("configuration file '%s' does not exist" % f)
try:
self.conf = self.config_class(options=self.options)
self.conf.load(filename=f)
except ConfigObjectAlreadyCreated:
self.conf = get_config()
self.command.conf = self.conf
self.command.args = self.args
self.command.options = self.options
self.command.start()