def setprogressoptions(self):
"""Sets the progress options."""
self.progresstypes = OrderedDict([
("dots", progressbar.DotsProgressBar),
("none", progressbar.NoProgressBar),
("bar", progressbar.HashProgressBar),
("names", progressbar.MessageProgressBar),
("verbose", progressbar.VerboseProgressBar),
])
progressoption = optparse.Option(None, "--progress", dest="progress",
default="bar",
choices=list(self.progresstypes.keys()), metavar="PROGRESS",
help="show progress as: %s" % (", ".join(self.progresstypes)))
self.define_option(progressoption)
def setprogressoptions(self):
"""Sets the progress options."""
self.progresstypes = OrderedDict([
("dots", progressbar.DotsProgressBar),
("none", progressbar.NoProgressBar),
("bar", progressbar.HashProgressBar),
("names", progressbar.MessageProgressBar),
("verbose", progressbar.VerboseProgressBar),
])
progressoption = optparse.Option(
None,
"--progress",
dest="progress",
default="bar",
choices=list(self.progresstypes.keys()),
metavar="PROGRESS",
help="show progress as: %s" % (", ".join(self.progresstypes)))
self.define_option(progressoption)
class RecursiveOptionParser(optparse.OptionParser, object):
"""A specialized Option Parser for recursing through directories."""
def __init__(self, formats, usetemplates=False, allowmissingtemplate=False,
description=None):
"""Construct the specialized Option Parser.
:type formats: Dictionary
:param formats: See :meth:`~.RecursiveOptionParser.setformats`
for an explanation of the formats parameter.
"""
optparse.OptionParser.__init__(self, version="%prog " + __version__.sver,
description=description)
self.setmanpageoption()
self.setprogressoptions()
self.seterrorleveloptions()
self.setformats(formats, usetemplates)
self.passthrough = []
self.allowmissingtemplate = allowmissingtemplate
logging.basicConfig(format="%(name)s: %(levelname)s: %(message)s")
def get_prog_name(self):
return os.path.basename(sys.argv[0])
def setmanpageoption(self):
"""creates a manpage option that allows the optionparser to generate a
manpage"""
manpageoption = ManPageOption(None, "--manpage", dest="manpage",
default=False, action="manpage",
help="output a manpage based on the help")
self.define_option(manpageoption)
def format_manpage(self):
"""returns a formatted manpage"""
result = []
prog = self.get_prog_name()
formatprog = lambda x: x.replace("%prog", prog)
formatToolkit = lambda x: x.replace("%prog", "Translate Toolkit")
result.append('.\\" Autogenerated manpage\n')
result.append('.TH %s 1 "%s" "" "%s"\n' % (prog,
formatToolkit(self.version),
formatToolkit(self.version)))
result.append('.SH NAME\n')
result.append('%s \\- %s\n' % (self.get_prog_name(),
self.description.split('\n\n')[0]))
result.append('.SH SYNOPSIS\n')
result.append('.PP\n')
usage = "\\fB%prog "
usage += " ".join([self.getusageman(option) for option in self.option_list])
usage += "\\fP"
result.append('%s\n' % formatprog(usage))
description_lines = self.description.split('\n\n')[1:]
if description_lines:
result.append('.SH DESCRIPTION\n')
result.append('\n\n'.join([re.sub('\.\. note::', 'Note:', l)
for l in description_lines]))
result.append('.SH OPTIONS\n')
ManHelpFormatter().store_option_strings(self)
result.append('.PP\n')
for option in self.option_list:
result.append('.TP\n')
result.append('%s\n' % str(option).replace('-', '\-'))
result.append('%s\n' % option.help.replace('-', '\-'))
return "".join(result)
def print_manpage(self, file=None):
"""outputs a manpage for the program using the help information"""
if file is None:
file = sys.stdout
file.write(self.format_manpage())
def set_usage(self, usage=None):
"""sets the usage string - if usage not given, uses getusagestring for
each option"""
if usage is None:
self.usage = "%prog " + " ".join([self.getusagestring(option) for option in self.option_list])
else:
super(RecursiveOptionParser, self).set_usage(usage)
def warning(self, msg, options=None, exc_info=None):
"""Print a warning message incorporating 'msg' to stderr and exit."""
if options:
if options.errorlevel == "traceback":
errorinfo = "\n".join(traceback.format_exception(exc_info[0],
exc_info[1], exc_info[2]))
elif options.errorlevel == "exception":
errorinfo = "\n".join(traceback.format_exception_only(exc_info[0], exc_info[1]))
elif options.errorlevel == "message":
errorinfo = str(exc_info[1])
else:
errorinfo = ""
if errorinfo:
msg += ": " + errorinfo
logging.getLogger(self.get_prog_name()).warning(msg)
def getusagestring(self, option):
"""returns the usage string for the given option"""
optionstring = "|".join(option._short_opts + option._long_opts)
if getattr(option, "optionalswitch", False):
optionstring = "[%s]" % optionstring
if option.metavar:
optionstring += " " + option.metavar
if getattr(option, "required", False):
return optionstring
else:
return "[%s]" % optionstring
def getusageman(self, option):
"""returns the usage string for the given option"""
optionstring = "\\fR|\\fP".join(option._short_opts + option._long_opts)
if getattr(option, "optionalswitch", False):
optionstring = "\\fR[\\fP%s\\fR]\\fP" % optionstring
if option.metavar:
optionstring += " \\fI%s\\fP" % option.metavar
if getattr(option, "required", False):
return optionstring
else:
return "\\fR[\\fP%s\\fR]\\fP" % optionstring
def define_option(self, option):
"""Defines the given option, replacing an existing one of the same short
name if neccessary..."""
for short_opt in option._short_opts:
if self.has_option(short_opt):
self.remove_option(short_opt)
for long_opt in option._long_opts:
if self.has_option(long_opt):
self.remove_option(long_opt)
self.add_option(option)
def setformats(self, formats, usetemplates):
"""Sets the format options using the given format dictionary.
:type formats: Dictionary
:param formats: The dictionary *keys* should be:
- Single strings (or 1-tuples) containing an
input format (if not *usetemplates*)
- Tuples containing an input format and
template format (if *usetemplates*)
- Formats can be *None* to indicate what to do
with standard input
The dictionary *values* should be tuples of
outputformat (string) and processor method.
"""
inputformats = []
outputformats = []
templateformats = []
self.outputoptions = {}
self.usetemplates = usetemplates
for formatgroup, outputoptions in six.iteritems(formats):
if isinstance(formatgroup, six.string_types) or formatgroup is None:
formatgroup = (formatgroup, )
if not isinstance(formatgroup, tuple):
raise ValueError("formatgroups must be tuples or None/str/unicode")
if len(formatgroup) < 1 or len(formatgroup) > 2:
raise ValueError("formatgroups must be tuples of length 1 or 2")
if len(formatgroup) == 1:
formatgroup += (None, )
inputformat, templateformat = formatgroup
if not isinstance(outputoptions, tuple) or len(outputoptions) != 2:
raise ValueError("output options must be tuples of length 2")
outputformat, processor = outputoptions
if not inputformat in inputformats:
inputformats.append(inputformat)
if not outputformat in outputformats:
outputformats.append(outputformat)
if not templateformat in templateformats:
templateformats.append(templateformat)
self.outputoptions[(inputformat, templateformat)] = (outputformat, processor)
self.inputformats = inputformats
inputformathelp = self.getformathelp(inputformats)
inputoption = optparse.Option("-i", "--input", dest="input",
default=None, metavar="INPUT",
help="read from INPUT in %s" % (inputformathelp))
inputoption.optionalswitch = True
inputoption.required = True
self.define_option(inputoption)
excludeoption = optparse.Option("-x", "--exclude", dest="exclude",
action="append", type="string", metavar="EXCLUDE",
default=["CVS", ".svn", "_darcs", ".git", ".hg", ".bzr"],
help="exclude names matching EXCLUDE from input paths")
self.define_option(excludeoption)
outputformathelp = self.getformathelp(outputformats)
outputoption = optparse.Option("-o", "--output", dest="output",
default=None, metavar="OUTPUT",
help="write to OUTPUT in %s" % (outputformathelp))
outputoption.optionalswitch = True
outputoption.required = True
self.define_option(outputoption)
if self.usetemplates:
self.templateformats = templateformats
templateformathelp = self.getformathelp(self.templateformats)
templateoption = optparse.Option("-t", "--template",
dest="template", default=None, metavar="TEMPLATE",
help="read from TEMPLATE in %s" % (templateformathelp))
self.define_option(templateoption)
def setprogressoptions(self):
"""Sets the progress options."""
self.progresstypes = OrderedDict([
("dots", progressbar.DotsProgressBar),
("none", progressbar.NoProgressBar),
("bar", progressbar.HashProgressBar),
("names", progressbar.MessageProgressBar),
("verbose", progressbar.VerboseProgressBar),
])
progressoption = optparse.Option(None, "--progress", dest="progress",
default="bar",
choices=list(self.progresstypes.keys()), metavar="PROGRESS",
help="show progress as: %s" % (", ".join(self.progresstypes)))
self.define_option(progressoption)
def seterrorleveloptions(self):
"""Sets the errorlevel options."""
self.errorleveltypes = ["none", "message", "exception", "traceback"]
errorleveloption = optparse.Option(None, "--errorlevel",
dest="errorlevel", default="message",
choices=self.errorleveltypes, metavar="ERRORLEVEL",
help="show errorlevel as: %s" %
(", ".join(self.errorleveltypes)))
self.define_option(errorleveloption)
def getformathelp(self, formats):
"""Make a nice help string for describing formats..."""
formats = sorted([f for f in formats if f is not None])
if len(formats) == 0:
return ""
elif len(formats) == 1:
return "%s format" % (", ".join(formats))
else:
return "%s formats" % (", ".join(formats))
def isrecursive(self, fileoption, filepurpose='input'):
"""Checks if fileoption is a recursive file."""
if fileoption is None:
return False
elif isinstance(fileoption, list):
return True
else:
return os.path.isdir(fileoption)
def parse_args(self, args=None, values=None):
"""Parses the command line options, handling implicit input/output
args."""
(options, args) = super(RecursiveOptionParser, self).parse_args(args, values)
# some intelligent as to what reasonable people might give on the
# command line
if args and not options.input:
if len(args) > 1:
options.input = args[:-1]
args = args[-1:]
else:
options.input = args[0]
args = []
if args and not options.output:
options.output = args[-1]
args = args[:-1]
if args:
self.error("You have used an invalid combination of --input, --output and freestanding args")
if isinstance(options.input, list) and len(options.input) == 1:
options.input = options.input[0]
if options.input is None:
self.error("You need to give an inputfile or use - for stdin ; use --help for full usage instructions")
elif options.input == '-':
options.input = None
return (options, args)
def getpassthroughoptions(self, options):
"""Get the options required to pass to the filtermethod..."""
passthroughoptions = {}
for optionname in dir(options):
if optionname in self.passthrough:
passthroughoptions[optionname] = getattr(options, optionname)
return passthroughoptions
def getoutputoptions(self, options, inputpath, templatepath):
"""Works out which output format and processor method to use..."""
if inputpath:
inputbase, inputext = self.splitinputext(inputpath)
else:
inputext = None
if templatepath:
templatebase, templateext = self.splittemplateext(templatepath)
else:
templateext = None
if (inputext, templateext) in options.outputoptions:
return options.outputoptions[inputext, templateext]
elif (inputext, "*") in options.outputoptions:
outputformat, fileprocessor = options.outputoptions[inputext, "*"]
elif ("*", templateext) in options.outputoptions:
outputformat, fileprocessor = options.outputoptions["*", templateext]
elif ("*", "*") in options.outputoptions:
outputformat, fileprocessor = options.outputoptions["*", "*"]
elif (inputext, None) in options.outputoptions:
return options.outputoptions[inputext, None]
elif (None, templateext) in options.outputoptions:
return options.outputoptions[None, templateext]
elif ("*", None) in options.outputoptions:
outputformat, fileprocessor = options.outputoptions["*", None]
elif (None, "*") in options.outputoptions:
outputformat, fileprocessor = options.outputoptions[None, "*"]
else:
if self.usetemplates:
if inputext is None:
raise ValueError("don't know what to do with input format (no file extension), no template file")
elif templateext is None:
raise ValueError("don't know what to do with input format %s, no template file" %
(os.extsep + inputext))
else:
raise ValueError("don't know what to do with input format %s, template format %s" %
(os.extsep + inputext, os.extsep + templateext))
else:
raise ValueError("don't know what to do with input format %s" %
(os.extsep + inputext))
if outputformat == "*":
if inputext:
outputformat = inputext
elif templateext:
outputformat = templateext
elif ("*", "*") in options.outputoptions:
outputformat = None
else:
if self.usetemplates:
raise ValueError("don't know what to do with input format (no file extension), no template file")
else:
raise ValueError("don't know what to do with input format (no file extension)")
return outputformat, fileprocessor
def initprogressbar(self, allfiles, options):
"""Sets up a progress bar appropriate to the options and files."""
if options.progress in ('bar', 'verbose'):
self.progressbar = \
self.progresstypes[options.progress](0, len(allfiles))
# should use .getChild("progress") but that is only in 2.7
logger = logging.getLogger(self.get_prog_name() + ".progress")
logger.setLevel(logging.INFO)
logger.propagate = False
handler = logging.StreamHandler()
handler.setLevel(logging.INFO)
handler.setFormatter(logging.Formatter())
logger.addHandler(handler)
logger.info("processing %d files...", len(allfiles))
else:
self.progressbar = self.progresstypes[options.progress]()
def getfullinputpath(self, options, inputpath):
"""Gets the absolute path to an input file."""
if options.input:
return os.path.join(options.input, inputpath)
else:
return inputpath
def getfulloutputpath(self, options, outputpath):
"""Gets the absolute path to an output file."""
if options.recursiveoutput and options.output:
return os.path.join(options.output, outputpath)
else:
return outputpath
def getfulltemplatepath(self, options, templatepath):
"""Gets the absolute path to a template file."""
if not options.recursivetemplate:
return templatepath
elif (templatepath is not None and
self.usetemplates and options.template):
return os.path.join(options.template, templatepath)
else:
return None
def run(self):
"""Parses the arguments, and runs recursiveprocess with the resulting
options..."""
(options, args) = self.parse_args()
# this is so derived classes can modify the inputformats etc based on
# the options
options.inputformats = self.inputformats
options.outputoptions = self.outputoptions
self.recursiveprocess(options)
def recursiveprocess(self, options):
"""Recurse through directories and process files."""
if self.isrecursive(options.input, 'input') and getattr(options, "allowrecursiveinput", True):
if not self.isrecursive(options.output, 'output'):
if not options.output:
self.error(optparse.OptionValueError("No output directory given"))
try:
self.warning("Output directory does not exist. Attempting to create")
os.mkdir(options.output)
except IOError as e:
self.error(optparse.OptionValueError("Output directory does not exist, attempt to create failed"))
if isinstance(options.input, list):
inputfiles = self.recurseinputfilelist(options)
else:
inputfiles = self.recurseinputfiles(options)
else:
if options.input:
inputfiles = [os.path.basename(options.input)]
options.input = os.path.dirname(options.input)
else:
inputfiles = [options.input]
options.recursiveoutput = (self.isrecursive(options.output, 'output') and
getattr(options, "allowrecursiveoutput", True))
options.recursivetemplate = (self.usetemplates and
self.isrecursive(options.template, 'template') and
getattr(options, "allowrecursivetemplate", True))
self.initprogressbar(inputfiles, options)
for inputpath in inputfiles:
try:
templatepath = self.gettemplatename(options, inputpath)
# If we have a recursive template, but the template doesn't
# have this input file, let's drop it.
if (options.recursivetemplate and templatepath is None and
not self.allowmissingtemplate):
self.warning("No template at %s. Skipping %s." %
(templatepath, inputpath))
continue
outputformat, fileprocessor = self.getoutputoptions(options, inputpath, templatepath)
fullinputpath = self.getfullinputpath(options, inputpath)
fulltemplatepath = self.getfulltemplatepath(options,
templatepath)
outputpath = self.getoutputname(options, inputpath, outputformat)
fulloutputpath = self.getfulloutputpath(options, outputpath)
if options.recursiveoutput and outputpath:
self.checkoutputsubdir(options, os.path.dirname(outputpath))
except Exception:
self.warning("Couldn't handle input file %s" %
inputpath, options, sys.exc_info())
continue
try:
success = self.processfile(fileprocessor, options,
fullinputpath, fulloutputpath,
fulltemplatepath)
except Exception:
self.warning("Error processing: input %s, output %s, template %s" %
(fullinputpath, fulloutputpath,
fulltemplatepath), options, sys.exc_info())
success = False
self.reportprogress(inputpath, success)
del self.progressbar
def openinputfile(self, options, fullinputpath):
"""Opens the input file."""
if fullinputpath is None:
return sys.stdin
return open(fullinputpath, 'rb')
def openoutputfile(self, options, fulloutputpath):
"""Opens the output file."""
if fulloutputpath is None:
return sys.stdout
return open(fulloutputpath, 'wb')
def opentempoutputfile(self, options, fulloutputpath):
"""Opens a temporary output file."""
return BytesIO()
def finalizetempoutputfile(self, options, outputfile, fulloutputpath):
"""Write the temp outputfile to its final destination."""
outputfile.reset()
outputstring = outputfile.read()
outputfile = self.openoutputfile(options, fulloutputpath)
outputfile.write(outputstring)
outputfile.close()
def opentemplatefile(self, options, fulltemplatepath):
"""Opens the template file (if required)."""
if fulltemplatepath is not None:
if os.path.isfile(fulltemplatepath):
return open(fulltemplatepath, 'rb')
else:
self.warning("missing template file %s" % fulltemplatepath)
return None
def processfile(self, fileprocessor, options, fullinputpath,
fulloutputpath, fulltemplatepath):
"""Process an individual file."""
inputfile = self.openinputfile(options, fullinputpath)
if (fulloutputpath and
fulloutputpath in (fullinputpath, fulltemplatepath)):
outputfile = self.opentempoutputfile(options, fulloutputpath)
tempoutput = True
else:
outputfile = self.openoutputfile(options, fulloutputpath)
tempoutput = False
templatefile = self.opentemplatefile(options, fulltemplatepath)
passthroughoptions = self.getpassthroughoptions(options)
if fileprocessor(inputfile, outputfile, templatefile,
**passthroughoptions):
if tempoutput:
self.warning("writing to temporary output...")
self.finalizetempoutputfile(options, outputfile,
fulloutputpath)
return True
else:
# remove the file if it is a file (could be stdout etc)
if fulloutputpath and os.path.isfile(fulloutputpath):
outputfile.close()
os.unlink(fulloutputpath)
return False
def reportprogress(self, filename, success):
"""Shows that we are progressing..."""
self.progressbar.amount += 1
self.progressbar.show(filename)
def mkdir(self, parent, subdir):
"""Makes a subdirectory (recursively if neccessary)."""
if not os.path.isdir(parent):
raise ValueError("cannot make child directory %r if parent %r does not exist" %
(subdir, parent))
currentpath = parent
subparts = subdir.split(os.sep)
for part in subparts:
currentpath = os.path.join(currentpath, part)
if not os.path.isdir(currentpath):
os.mkdir(currentpath)
def checkoutputsubdir(self, options, subdir):
"""Checks to see if subdir under options.output needs to be created,
creates if neccessary."""
fullpath = os.path.join(options.output, subdir)
if not os.path.isdir(fullpath):
self.mkdir(options.output, subdir)
def isexcluded(self, options, inputpath):
"""Checks if this path has been excluded."""
basename = os.path.basename(inputpath)
for excludename in options.exclude:
if fnmatch.fnmatch(basename, excludename):
return True
return False
def recurseinputfilelist(self, options):
"""Use a list of files, and find a common base directory for them."""
# find a common base directory for the files to do everything
# relative to
commondir = os.path.dirname(os.path.commonprefix(options.input))
inputfiles = []
for inputfile in options.input:
if self.isexcluded(options, inputfile):
continue
if inputfile.startswith(commondir + os.sep):
inputfiles.append(inputfile.replace(commondir + os.sep, "", 1))
else:
inputfiles.append(inputfile.replace(commondir, "", 1))
options.input = commondir
return inputfiles
def recurseinputfiles(self, options):
"""Recurse through directories and return files to be processed."""
dirstack = ['']
join = os.path.join
inputfiles = []
while dirstack:
top = dirstack.pop(-1)
names = os.listdir(join(options.input, top))
dirs = []
for name in names:
inputpath = join(top, name)
if self.isexcluded(options, inputpath):
continue
fullinputpath = self.getfullinputpath(options, inputpath)
# handle directories...
if os.path.isdir(fullinputpath):
dirs.append(inputpath)
elif os.path.isfile(fullinputpath):
if not self.isvalidinputname(options, name):
# only handle names that match recognized input
# file extensions
continue
inputfiles.append(inputpath)
# make sure the directories are processed next time round.
dirs.reverse()
dirstack.extend(dirs)
return inputfiles
def splitext(self, pathname):
"""Splits *pathname* into name and ext, and removes the extsep.
:param pathname: A file path
:type pathname: string
:return: root, ext
:rtype: tuple
"""
root, ext = os.path.splitext(pathname)
ext = ext.replace(os.extsep, "", 1)
return (root, ext)
def splitinputext(self, inputpath):
"""Splits an *inputpath* into name and extension."""
return self.splitext(inputpath)
def splittemplateext(self, templatepath):
"""Splits a *templatepath* into name and extension."""
return self.splitext(templatepath)
def templateexists(self, options, templatepath):
"""Returns whether the given template exists..."""
fulltemplatepath = self.getfulltemplatepath(options, templatepath)
return os.path.isfile(fulltemplatepath)
def gettemplatename(self, options, inputname):
"""Gets an output filename based on the input filename."""
if not self.usetemplates:
return None
if not inputname or not options.recursivetemplate:
return options.template
inputbase, inputext = self.splitinputext(inputname)
if options.template:
for inputext1, templateext1 in options.outputoptions:
if inputext == inputext1:
if templateext1:
templatepath = inputbase + os.extsep + templateext1
if self.templateexists(options, templatepath):
return templatepath
if "*" in options.inputformats:
for inputext1, templateext1 in options.outputoptions:
if (inputext == inputext1) or (inputext1 == "*"):
if templateext1 == "*":
templatepath = inputname
if self.templateexists(options, templatepath):
return templatepath
elif templateext1:
templatepath = inputbase + os.extsep + templateext1
if self.templateexists(options, templatepath):
return templatepath
return None
def getoutputname(self, options, inputname, outputformat):
"""Gets an output filename based on the input filename."""
if not inputname or not options.recursiveoutput:
return options.output
inputbase, inputext = self.splitinputext(inputname)
outputname = inputbase
if outputformat:
outputname += os.extsep + outputformat
return outputname
def isvalidinputname(self, options, inputname):
"""Checks if this is a valid input filename."""
inputbase, inputext = self.splitinputext(inputname)
return ((inputext in options.inputformats) or
("*" in options.inputformats))
class RecursiveOptionParser(optparse.OptionParser, object):
"""A specialized Option Parser for recursing through directories."""
def __init__(self,
formats,
usetemplates=False,
allowmissingtemplate=False,
description=None):
"""Construct the specialized Option Parser.
:type formats: Dictionary
:param formats: See :meth:`~.RecursiveOptionParser.setformats`
for an explanation of the formats parameter.
"""
optparse.OptionParser.__init__(self,
version="%prog " + __version__.sver,
description=description)
self.setmanpageoption()
self.setprogressoptions()
self.seterrorleveloptions()
self.setformats(formats, usetemplates)
self.passthrough = []
self.allowmissingtemplate = allowmissingtemplate
logging.basicConfig(format="%(name)s: %(levelname)s: %(message)s")
def get_prog_name(self):
return os.path.basename(sys.argv[0])
def setmanpageoption(self):
"""creates a manpage option that allows the optionparser to generate a
manpage"""
manpageoption = ManPageOption(
None,
"--manpage",
dest="manpage",
default=False,
action="manpage",
help="output a manpage based on the help")
self.define_option(manpageoption)
def format_manpage(self):
"""returns a formatted manpage"""
result = []
prog = self.get_prog_name()
formatprog = lambda x: x.replace("%prog", prog)
formatToolkit = lambda x: x.replace("%prog", "Translate Toolkit")
result.append('.\\" Autogenerated manpage\n')
result.append(
'.TH %s 1 "%s" "" "%s"\n' %
(prog, formatToolkit(self.version), formatToolkit(self.version)))
result.append('.SH NAME\n')
result.append(
'%s \\- %s\n' %
(self.get_prog_name(), self.description.split('\n\n')[0]))
result.append('.SH SYNOPSIS\n')
result.append('.PP\n')
usage = "\\fB%prog "
usage += " ".join(
[self.getusageman(option) for option in self.option_list])
usage += "\\fP"
result.append('%s\n' % formatprog(usage))
description_lines = self.description.split('\n\n')[1:]
if description_lines:
result.append('.SH DESCRIPTION\n')
result.append('\n\n'.join([
re.sub('\.\. note::', 'Note:', l) for l in description_lines
]))
result.append('.SH OPTIONS\n')
ManHelpFormatter().store_option_strings(self)
result.append('.PP\n')
for option in self.option_list:
result.append('.TP\n')
result.append('%s\n' % str(option).replace('-', '\-'))
result.append('%s\n' % option.help.replace('-', '\-'))
return "".join(result)
def print_manpage(self, file=None):
"""outputs a manpage for the program using the help information"""
if file is None:
file = sys.stdout
file.write(self.format_manpage())
def set_usage(self, usage=None):
"""sets the usage string - if usage not given, uses getusagestring for
each option"""
if usage is None:
self.usage = "%prog " + " ".join(
[self.getusagestring(option) for option in self.option_list])
else:
super(RecursiveOptionParser, self).set_usage(usage)
def warning(self, msg, options=None, exc_info=None):
"""Print a warning message incorporating 'msg' to stderr and exit."""
if options:
if options.errorlevel == "traceback":
errorinfo = "\n".join(
traceback.format_exception(exc_info[0], exc_info[1],
exc_info[2]))
elif options.errorlevel == "exception":
errorinfo = "\n".join(
traceback.format_exception_only(exc_info[0], exc_info[1]))
elif options.errorlevel == "message":
errorinfo = str(exc_info[1])
else:
errorinfo = ""
if errorinfo:
msg += ": " + errorinfo
logging.getLogger(self.get_prog_name()).warning(msg)
def getusagestring(self, option):
"""returns the usage string for the given option"""
optionstring = "|".join(option._short_opts + option._long_opts)
if getattr(option, "optionalswitch", False):
optionstring = "[%s]" % optionstring
if option.metavar:
optionstring += " " + option.metavar
if getattr(option, "required", False):
return optionstring
else:
return "[%s]" % optionstring
def getusageman(self, option):
"""returns the usage string for the given option"""
optionstring = "\\fR|\\fP".join(option._short_opts + option._long_opts)
if getattr(option, "optionalswitch", False):
optionstring = "\\fR[\\fP%s\\fR]\\fP" % optionstring
if option.metavar:
optionstring += " \\fI%s\\fP" % option.metavar
if getattr(option, "required", False):
return optionstring
else:
return "\\fR[\\fP%s\\fR]\\fP" % optionstring
def define_option(self, option):
"""Defines the given option, replacing an existing one of the same short
name if neccessary..."""
for short_opt in option._short_opts:
if self.has_option(short_opt):
self.remove_option(short_opt)
for long_opt in option._long_opts:
if self.has_option(long_opt):
self.remove_option(long_opt)
self.add_option(option)
def setformats(self, formats, usetemplates):
"""Sets the format options using the given format dictionary.
:type formats: Dictionary
:param formats: The dictionary *keys* should be:
- Single strings (or 1-tuples) containing an
input format (if not *usetemplates*)
- Tuples containing an input format and
template format (if *usetemplates*)
- Formats can be *None* to indicate what to do
with standard input
The dictionary *values* should be tuples of
outputformat (string) and processor method.
"""
inputformats = []
outputformats = []
templateformats = []
self.outputoptions = {}
self.usetemplates = usetemplates
for formatgroup, outputoptions in six.iteritems(formats):
if isinstance(formatgroup,
six.string_types) or formatgroup is None:
formatgroup = (formatgroup, )
if not isinstance(formatgroup, tuple):
raise ValueError(
"formatgroups must be tuples or None/str/unicode")
if len(formatgroup) < 1 or len(formatgroup) > 2:
raise ValueError(
"formatgroups must be tuples of length 1 or 2")
if len(formatgroup) == 1:
formatgroup += (None, )
inputformat, templateformat = formatgroup
if not isinstance(outputoptions, tuple) or len(outputoptions) != 2:
raise ValueError("output options must be tuples of length 2")
outputformat, processor = outputoptions
if not inputformat in inputformats:
inputformats.append(inputformat)
if not outputformat in outputformats:
outputformats.append(outputformat)
if not templateformat in templateformats:
templateformats.append(templateformat)
self.outputoptions[(inputformat, templateformat)] = (outputformat,
processor)
self.inputformats = inputformats
inputformathelp = self.getformathelp(inputformats)
inputoption = optparse.Option("-i",
"--input",
dest="input",
default=None,
metavar="INPUT",
help="read from INPUT in %s" %
(inputformathelp))
inputoption.optionalswitch = True
inputoption.required = True
self.define_option(inputoption)
excludeoption = optparse.Option(
"-x",
"--exclude",
dest="exclude",
action="append",
type="string",
metavar="EXCLUDE",
default=["CVS", ".svn", "_darcs", ".git", ".hg", ".bzr"],
help="exclude names matching EXCLUDE from input paths")
self.define_option(excludeoption)
outputformathelp = self.getformathelp(outputformats)
outputoption = optparse.Option("-o",
"--output",
dest="output",
default=None,
metavar="OUTPUT",
help="write to OUTPUT in %s" %
(outputformathelp))
outputoption.optionalswitch = True
outputoption.required = True
self.define_option(outputoption)
if self.usetemplates:
self.templateformats = templateformats
templateformathelp = self.getformathelp(self.templateformats)
templateoption = optparse.Option("-t",
"--template",
dest="template",
default=None,
metavar="TEMPLATE",
help="read from TEMPLATE in %s" %
(templateformathelp))
self.define_option(templateoption)
def setprogressoptions(self):
"""Sets the progress options."""
self.progresstypes = OrderedDict([
("dots", progressbar.DotsProgressBar),
("none", progressbar.NoProgressBar),
("bar", progressbar.HashProgressBar),
("names", progressbar.MessageProgressBar),
("verbose", progressbar.VerboseProgressBar),
])
progressoption = optparse.Option(
None,
"--progress",
dest="progress",
default="bar",
choices=list(self.progresstypes.keys()),
metavar="PROGRESS",
help="show progress as: %s" % (", ".join(self.progresstypes)))
self.define_option(progressoption)
def seterrorleveloptions(self):
"""Sets the errorlevel options."""
self.errorleveltypes = ["none", "message", "exception", "traceback"]
errorleveloption = optparse.Option(None,
"--errorlevel",
dest="errorlevel",
default="message",
choices=self.errorleveltypes,
metavar="ERRORLEVEL",
help="show errorlevel as: %s" %
(", ".join(self.errorleveltypes)))
self.define_option(errorleveloption)
def getformathelp(self, formats):
"""Make a nice help string for describing formats..."""
formats = sorted([f for f in formats if f is not None])
if len(formats) == 0:
return ""
elif len(formats) == 1:
return "%s format" % (", ".join(formats))
else:
return "%s formats" % (", ".join(formats))
def isrecursive(self, fileoption, filepurpose='input'):
"""Checks if fileoption is a recursive file."""
if fileoption is None:
return False
elif isinstance(fileoption, list):
return True
else:
return os.path.isdir(fileoption)
def parse_args(self, args=None, values=None):
"""Parses the command line options, handling implicit input/output
args."""
(options, args) = super(RecursiveOptionParser,
self).parse_args(args, values)
# some intelligent as to what reasonable people might give on the
# command line
if args and not options.input:
if len(args) > 1:
options.input = args[:-1]
args = args[-1:]
else:
options.input = args[0]
args = []
if args and not options.output:
options.output = args[-1]
args = args[:-1]
if args:
self.error(
"You have used an invalid combination of --input, --output and freestanding args"
)
if isinstance(options.input, list) and len(options.input) == 1:
options.input = options.input[0]
if options.input is None:
self.error(
"You need to give an inputfile or use - for stdin ; use --help for full usage instructions"
)
elif options.input == '-':
options.input = None
return (options, args)
def getpassthroughoptions(self, options):
"""Get the options required to pass to the filtermethod..."""
passthroughoptions = {}
for optionname in dir(options):
if optionname in self.passthrough:
passthroughoptions[optionname] = getattr(options, optionname)
return passthroughoptions
def getoutputoptions(self, options, inputpath, templatepath):
"""Works out which output format and processor method to use..."""
if inputpath:
inputbase, inputext = self.splitinputext(inputpath)
else:
inputext = None
if templatepath:
templatebase, templateext = self.splittemplateext(templatepath)
else:
templateext = None
if (inputext, templateext) in options.outputoptions:
return options.outputoptions[inputext, templateext]
elif (inputext, "*") in options.outputoptions:
outputformat, fileprocessor = options.outputoptions[inputext, "*"]
elif ("*", templateext) in options.outputoptions:
outputformat, fileprocessor = options.outputoptions["*",
templateext]
elif ("*", "*") in options.outputoptions:
outputformat, fileprocessor = options.outputoptions["*", "*"]
elif (inputext, None) in options.outputoptions:
return options.outputoptions[inputext, None]
elif (None, templateext) in options.outputoptions:
return options.outputoptions[None, templateext]
elif ("*", None) in options.outputoptions:
outputformat, fileprocessor = options.outputoptions["*", None]
elif (None, "*") in options.outputoptions:
outputformat, fileprocessor = options.outputoptions[None, "*"]
else:
if self.usetemplates:
if inputext is None:
raise ValueError(
"don't know what to do with input format (no file extension), no template file"
)
elif templateext is None:
raise ValueError(
"don't know what to do with input format %s, no template file"
% (os.extsep + inputext))
else:
raise ValueError(
"don't know what to do with input format %s, template format %s"
% (os.extsep + inputext, os.extsep + templateext))
else:
raise ValueError("don't know what to do with input format %s" %
(os.extsep + inputext))
if outputformat == "*":
if inputext:
outputformat = inputext
elif templateext:
outputformat = templateext
elif ("*", "*") in options.outputoptions:
outputformat = None
else:
if self.usetemplates:
raise ValueError(
"don't know what to do with input format (no file extension), no template file"
)
else:
raise ValueError(
"don't know what to do with input format (no file extension)"
)
return outputformat, fileprocessor
def initprogressbar(self, allfiles, options):
"""Sets up a progress bar appropriate to the options and files."""
if options.progress in ('bar', 'verbose'):
self.progressbar = \
self.progresstypes[options.progress](0, len(allfiles))
# should use .getChild("progress") but that is only in 2.7
logger = logging.getLogger(self.get_prog_name() + ".progress")
logger.setLevel(logging.INFO)
logger.propagate = False
handler = logging.StreamHandler()
handler.setLevel(logging.INFO)
handler.setFormatter(logging.Formatter())
logger.addHandler(handler)
logger.info("processing %d files...", len(allfiles))
else:
self.progressbar = self.progresstypes[options.progress]()
def getfullinputpath(self, options, inputpath):
"""Gets the absolute path to an input file."""
if options.input:
return os.path.join(options.input, inputpath)
else:
return inputpath
def getfulloutputpath(self, options, outputpath):
"""Gets the absolute path to an output file."""
if options.recursiveoutput and options.output:
return os.path.join(options.output, outputpath)
else:
return outputpath
def getfulltemplatepath(self, options, templatepath):
"""Gets the absolute path to a template file."""
if not options.recursivetemplate:
return templatepath
elif (templatepath is not None and self.usetemplates
and options.template):
return os.path.join(options.template, templatepath)
else:
return None
def run(self):
"""Parses the arguments, and runs recursiveprocess with the resulting
options..."""
(options, args) = self.parse_args()
# this is so derived classes can modify the inputformats etc based on
# the options
options.inputformats = self.inputformats
options.outputoptions = self.outputoptions
self.recursiveprocess(options)
def recursiveprocess(self, options):
"""Recurse through directories and process files."""
if self.isrecursive(options.input, 'input') and getattr(
options, "allowrecursiveinput", True):
if not self.isrecursive(options.output, 'output'):
if not options.output:
self.error(
optparse.OptionValueError("No output directory given"))
try:
self.warning(
"Output directory does not exist. Attempting to create"
)
os.mkdir(options.output)
except IOError as e:
self.error(
optparse.OptionValueError(
"Output directory does not exist, attempt to create failed"
))
if isinstance(options.input, list):
inputfiles = self.recurseinputfilelist(options)
else:
inputfiles = self.recurseinputfiles(options)
else:
if options.input:
inputfiles = [os.path.basename(options.input)]
options.input = os.path.dirname(options.input)
else:
inputfiles = [options.input]
options.recursiveoutput = (self.isrecursive(options.output, 'output')
and getattr(options, "allowrecursiveoutput",
True))
options.recursivetemplate = (
self.usetemplates
and self.isrecursive(options.template, 'template')
and getattr(options, "allowrecursivetemplate", True))
self.initprogressbar(inputfiles, options)
for inputpath in inputfiles:
try:
templatepath = self.gettemplatename(options, inputpath)
# If we have a recursive template, but the template doesn't
# have this input file, let's drop it.
if (options.recursivetemplate and templatepath is None
and not self.allowmissingtemplate):
self.warning("No template at %s. Skipping %s." %
(templatepath, inputpath))
continue
outputformat, fileprocessor = self.getoutputoptions(
options, inputpath, templatepath)
fullinputpath = self.getfullinputpath(options, inputpath)
fulltemplatepath = self.getfulltemplatepath(
options, templatepath)
outputpath = self.getoutputname(options, inputpath,
outputformat)
fulloutputpath = self.getfulloutputpath(options, outputpath)
if options.recursiveoutput and outputpath:
self.checkoutputsubdir(options,
os.path.dirname(outputpath))
except Exception:
self.warning("Couldn't handle input file %s" % inputpath,
options, sys.exc_info())
continue
try:
success = self.processfile(fileprocessor, options,
fullinputpath, fulloutputpath,
fulltemplatepath)
except Exception:
self.warning(
"Error processing: input %s, output %s, template %s" %
(fullinputpath, fulloutputpath, fulltemplatepath), options,
sys.exc_info())
success = False
self.reportprogress(inputpath, success)
del self.progressbar
def openinputfile(self, options, fullinputpath):
"""Opens the input file."""
if fullinputpath is None:
return sys.stdin
return open(fullinputpath, 'rb')
def openoutputfile(self, options, fulloutputpath):
"""Opens the output file."""
if fulloutputpath is None:
return sys.stdout
return open(fulloutputpath, 'wb')
def opentempoutputfile(self, options, fulloutputpath):
"""Opens a temporary output file."""
return BytesIO()
def finalizetempoutputfile(self, options, outputfile, fulloutputpath):
"""Write the temp outputfile to its final destination."""
outputfile.reset()
outputstring = outputfile.read()
outputfile = self.openoutputfile(options, fulloutputpath)
outputfile.write(outputstring)
outputfile.close()
def opentemplatefile(self, options, fulltemplatepath):
"""Opens the template file (if required)."""
if fulltemplatepath is not None:
if os.path.isfile(fulltemplatepath):
return open(fulltemplatepath, 'rb')
else:
self.warning("missing template file %s" % fulltemplatepath)
return None
def processfile(self, fileprocessor, options, fullinputpath,
fulloutputpath, fulltemplatepath):
"""Process an individual file."""
inputfile = self.openinputfile(options, fullinputpath)
if (fulloutputpath
and fulloutputpath in (fullinputpath, fulltemplatepath)):
outputfile = self.opentempoutputfile(options, fulloutputpath)
tempoutput = True
else:
outputfile = self.openoutputfile(options, fulloutputpath)
tempoutput = False
templatefile = self.opentemplatefile(options, fulltemplatepath)
passthroughoptions = self.getpassthroughoptions(options)
if fileprocessor(inputfile, outputfile, templatefile,
**passthroughoptions):
if tempoutput:
self.warning("writing to temporary output...")
self.finalizetempoutputfile(options, outputfile,
fulloutputpath)
return True
else:
# remove the file if it is a file (could be stdout etc)
if fulloutputpath and os.path.isfile(fulloutputpath):
outputfile.close()
os.unlink(fulloutputpath)
return False
def reportprogress(self, filename, success):
"""Shows that we are progressing..."""
self.progressbar.amount += 1
self.progressbar.show(filename)
def mkdir(self, parent, subdir):
"""Makes a subdirectory (recursively if neccessary)."""
if not os.path.isdir(parent):
raise ValueError(
"cannot make child directory %r if parent %r does not exist" %
(subdir, parent))
currentpath = parent
subparts = subdir.split(os.sep)
for part in subparts:
currentpath = os.path.join(currentpath, part)
if not os.path.isdir(currentpath):
os.mkdir(currentpath)
def checkoutputsubdir(self, options, subdir):
"""Checks to see if subdir under options.output needs to be created,
creates if neccessary."""
fullpath = os.path.join(options.output, subdir)
if not os.path.isdir(fullpath):
self.mkdir(options.output, subdir)
def isexcluded(self, options, inputpath):
"""Checks if this path has been excluded."""
basename = os.path.basename(inputpath)
for excludename in options.exclude:
if fnmatch.fnmatch(basename, excludename):
return True
return False
def recurseinputfilelist(self, options):
"""Use a list of files, and find a common base directory for them."""
# find a common base directory for the files to do everything
# relative to
commondir = os.path.dirname(os.path.commonprefix(options.input))
inputfiles = []
for inputfile in options.input:
if self.isexcluded(options, inputfile):
continue
if inputfile.startswith(commondir + os.sep):
inputfiles.append(inputfile.replace(commondir + os.sep, "", 1))
else:
inputfiles.append(inputfile.replace(commondir, "", 1))
options.input = commondir
return inputfiles
def recurseinputfiles(self, options):
"""Recurse through directories and return files to be processed."""
dirstack = ['']
join = os.path.join
inputfiles = []
while dirstack:
top = dirstack.pop(-1)
names = os.listdir(join(options.input, top))
dirs = []
for name in names:
inputpath = join(top, name)
if self.isexcluded(options, inputpath):
continue
fullinputpath = self.getfullinputpath(options, inputpath)
# handle directories...
if os.path.isdir(fullinputpath):
dirs.append(inputpath)
elif os.path.isfile(fullinputpath):
if not self.isvalidinputname(options, name):
# only handle names that match recognized input
# file extensions
continue
inputfiles.append(inputpath)
# make sure the directories are processed next time round.
dirs.reverse()
dirstack.extend(dirs)
return inputfiles
def splitext(self, pathname):
"""Splits *pathname* into name and ext, and removes the extsep.
:param pathname: A file path
:type pathname: string
:return: root, ext
:rtype: tuple
"""
root, ext = os.path.splitext(pathname)
ext = ext.replace(os.extsep, "", 1)
return (root, ext)
def splitinputext(self, inputpath):
"""Splits an *inputpath* into name and extension."""
return self.splitext(inputpath)
def splittemplateext(self, templatepath):
"""Splits a *templatepath* into name and extension."""
return self.splitext(templatepath)
def templateexists(self, options, templatepath):
"""Returns whether the given template exists..."""
fulltemplatepath = self.getfulltemplatepath(options, templatepath)
return os.path.isfile(fulltemplatepath)
def gettemplatename(self, options, inputname):
"""Gets an output filename based on the input filename."""
if not self.usetemplates:
return None
if not inputname or not options.recursivetemplate:
return options.template
inputbase, inputext = self.splitinputext(inputname)
if options.template:
for inputext1, templateext1 in options.outputoptions:
if inputext == inputext1:
if templateext1:
templatepath = inputbase + os.extsep + templateext1
if self.templateexists(options, templatepath):
return templatepath
if "*" in options.inputformats:
for inputext1, templateext1 in options.outputoptions:
if (inputext == inputext1) or (inputext1 == "*"):
if templateext1 == "*":
templatepath = inputname
if self.templateexists(options, templatepath):
return templatepath
elif templateext1:
templatepath = inputbase + os.extsep + templateext1
if self.templateexists(options, templatepath):
return templatepath
return None
def getoutputname(self, options, inputname, outputformat):
"""Gets an output filename based on the input filename."""
if not inputname or not options.recursiveoutput:
return options.output
inputbase, inputext = self.splitinputext(inputname)
outputname = inputbase
if outputformat:
outputname += os.extsep + outputformat
return outputname
def isvalidinputname(self, options, inputname):
"""Checks if this is a valid input filename."""
inputbase, inputext = self.splitinputext(inputname)
return ((inputext in options.inputformats)
or ("*" in options.inputformats))