def create_logger(log_level=None,
name=None,
stdout=False,
filename=None,
stderr=False):
"""
Set up a logger to log to a file.
@type filename: string
@param filename: the file to write the logs to. If None, doesn't add a
handler for file output.
@param log_level: one of the level constants in L{logging}; the log level
to give the logger. Defaults to B{INFO}.
@type name: string
@param name: name to give to the logger. By default, uses the filename. If
neither is given, raises an exception.
@type stdout: bool
@param stdout: if True, also adds a handler to output to stdout
@return: the Logger instance created
"""
import sys
from jazzparser.utils.base import check_directory
if name is None:
if filename is None:
raise ValueError, "neither a name nor a filename was given for the "\
"logger"
name = filename
# Create the logger
logger = logging.getLogger(name)
# Set the log level, or leave it as the default
if log_level is None:
logger.setLevel(logging.INFO)
else:
logger.setLevel(log_level)
formatter = logging.Formatter("%(levelname)s %(asctime)s: %(message)s",
"%Y-%m-%d %H:%M:%S")
if filename is not None:
# Make sure the containing directory exists
check_directory(filename)
# Create a file handler
fhandler = logging.FileHandler(filename, 'a')
fhandler.setLevel(logging.DEBUG)
fhandler.setFormatter(formatter)
# Use this handler
logger.addHandler(fhandler)
if stderr:
shandler = logging.StreamHandler()
shandler.setLevel(logging.DEBUG)
shandler.setFormatter(formatter)
logger.addHandler(shandler)
if stdout:
sohandler = logging.StreamHandler(sys.stdout)
sohandler.setLevel(logging.DEBUG)
sohandler.setFormatter(formatter)
logger.addHandler(sohandler)
return logger
def main():
usage = "%prog [options] <midi-input1> [<midi-input2> ...]"
description = "Interactive routine for cutting MIDI files. May take " "multiple MIDI files as input"
parser = OptionParser(usage=usage, description=description)
parser.add_option(
"-o",
"--output",
dest="output_dir",
action="store",
help="directory to send MIDI output to. If not given, they will be sent to a subdirectory 'cut' of that containing the first input",
)
parser.add_option(
"--fragment",
dest="fragment",
action="store",
type="float",
help="length in seconds of fragment to play when asked to play a beginning or ending. Default: 3secs",
default=3.0,
)
parser.add_option(
"--overwrite",
dest="overwrite",
action="store_true",
help="by default, we skip processing any files where there's a file with the same name in the output directory. This forces us to overwrite them",
)
parser.add_option(
"--ignore",
dest="ignore",
action="store",
help="file containing a list of filenames (not paths), one per line: any input files matching these names will be ignored and inputs marked as 'ignore' will be added to the list",
)
parser.add_option(
"--segfile",
dest="segfile",
action="store",
help="output a list of the MIDI files that get written by this script (just the base filename) in the format of segmidi input lists. The list will contain a basic set of default segmentation parameters. Use play_bulk_chunks to validate these. If the file exists, it will be appended",
)
options, arguments = parser.parse_args()
fragment = options.fragment
if len(arguments) == 0:
print >> sys.stderr, "You must specify at least one MIDI file"
sys.exit(1)
# Read in all the MIDI inputs
filenames = arguments
print "Processing %d inputs" % len(filenames)
if options.ignore:
if os.path.exists(options.ignore):
# Existing list
# Open the file to read in the current list and add to it
ignore_file = open(options.ignore, "r+a")
ignore_list = [filename.strip("\n") for filename in ignore_file.readlines()]
print "Loaded ignore list from %s" % options.ignore
else:
# No existing list
# Open the file so we can write new entries
ignore_file = open(options.ignore, "w")
ignore_list = []
print "Created new ignore list in %s" % options.ignore
else:
ignore_file = None
ignore_list = []
if options.segfile:
# Open the file for writing segmidi parameters to
segfile = open(options.segfile, "a")
segcsv = csv.writer(segfile)
else:
segfile = None
try:
# Set up the output directory
if options.output_dir:
output_dir = options.output_dir
else:
# Get the directory of the first input file
output_dir = os.path.join(os.path.dirname(filenames[0]), "cut")
check_directory(output_dir, is_dir=True)
print "Outputing MIDI files to %s" % output_dir
print
for filename in filenames:
basename = os.path.basename(filename)
# Ignore any files in the ignore list
if basename in ignore_list:
print "Skipping input %s, as it's in the ignore list" % basename
continue
out_filename = os.path.join(output_dir, os.path.basename(filename))
# Check whether the output file already exists
if os.path.exists(out_filename):
if options.overwrite:
# Just warn
print "WARNING: writing out this input will overwrite an existing file"
else:
# Don't continue with this input
print "Skipping input %s, since output file already exists" % filename
continue
start = 0
end = None
print "\n####################################"
print "Processing input: %s" % filename
# Read in the midi file
try:
mid = read_midifile(filename)
except Exception, err:
print "Error reading in midi file %s: %s" % (filename, err)
continue
print "Output will be written to: %s" % out_filename
# Start by playing the whole thing
_play(mid, start, end)
try:
while True:
# Print the header information
print "\n>>>>>>>>>>>>>>>>>>>>>>"
if end is None:
end_str = "open"
else:
end_str = "%d ticks" % end
print "Start: %d ticks. End: %s" % (start, end_str)
print ">>>>>>>>>>>>>>>>>>>>>>"
print "Set start time (s); set end time (e)"
print "Play all (p); play beginning ([); play end (], optional length)"
print "Write out and proceed (w); add to ignore list (i); skip to next (n); exit (x)"
# Get a command from the user
try:
command = raw_input(">> ")
except KeyboardInterrupt:
# I quite often send an interrupt by accident, meaning
# to stop the playback, but just after it's stopped
# itself
print "Ignored keyboard interrupt"
continue
command = command.strip()
if command.lower() == "p":
# Play within the selection again
_play(mid, start, end)
elif command.lower() == "n":
break
elif command.lower() == "i":
# Add the filename to the ignore list
if ignore_file:
ignore_file.write("%s\n" % os.path.basename(filename))
else:
print "No ignore file loaded: could not add this file to the list"
break
elif command.lower() == "x":
sys.exit(0)
elif command.lower().startswith("s"):
time = command[1:].strip()
if len(time) == 0:
print "Specify a start tick (T) or time (Ts)"
continue
start = _parse_time(mid, time)
elif command.lower().startswith("e"):
time = command[1:].strip()
if len(time) == 0:
print "Specify an end tick (T) or time (Ts)"
continue
end = _parse_time(mid, time, before=True)
elif command == "[":
# Play the opening few seconds
start_secs = _get_time(mid, start) / 1000.0
frag_end = _time_to_ticks(mid, fragment + start_secs)
_play(mid, start, frag_end)
elif command.startswith("]"):
length = command[1:].strip()
if len(length):
frag_length = float(length)
else:
frag_length = fragment
# Play the last few seconds
end_secs = _get_time(mid, end) / 1000.0
frag_start = _time_to_ticks(mid, max(0.0, end_secs - frag_length), before=True)
_play(mid, frag_start, end)
elif command == "w":
# Write the file out
if start is None:
start = 0
# Trim the midi
trimmed = EventStreamSlice(mid, start, end).to_event_stream(repeat_playing=False)
# Write it out
write_midifile(trimmed, out_filename)
if segfile is not None:
# Also output a row to the segmidi index
SegmentedMidiBulkInput.writeln(segcsv, basename)
print "Output written to %s" % out_filename
# Continue to the next input
break
else:
print "Unknown command: %s" % command
continue
except EOFError:
# User hit ctrl+D: continue to next input
print "Continuing to next input..."
continue
else:
def create_logger(log_level=None, name=None, stdout=False, filename=None, stderr=False):
"""
Set up a logger to log to a file.
@type filename: string
@param filename: the file to write the logs to. If None, doesn't add a
handler for file output.
@param log_level: one of the level constants in L{logging}; the log level
to give the logger. Defaults to B{INFO}.
@type name: string
@param name: name to give to the logger. By default, uses the filename. If
neither is given, raises an exception.
@type stdout: bool
@param stdout: if True, also adds a handler to output to stdout
@return: the Logger instance created
"""
import sys
from jazzparser.utils.base import check_directory
if name is None:
if filename is None:
raise ValueError, "neither a name nor a filename was given for the "\
"logger"
name = filename
# Create the logger
logger = logging.getLogger(name)
# Set the log level, or leave it as the default
if log_level is None:
logger.setLevel(logging.INFO)
else:
logger.setLevel(log_level)
formatter = logging.Formatter("%(levelname)s %(asctime)s: %(message)s",
"%Y-%m-%d %H:%M:%S")
if filename is not None:
# Make sure the containing directory exists
check_directory(filename)
# Create a file handler
fhandler = logging.FileHandler(filename, 'a')
fhandler.setLevel(logging.DEBUG)
fhandler.setFormatter(formatter)
# Use this handler
logger.addHandler(fhandler)
if stderr:
shandler = logging.StreamHandler()
shandler.setLevel(logging.DEBUG)
shandler.setFormatter(formatter)
logger.addHandler(shandler)
if stdout:
sohandler = logging.StreamHandler(sys.stdout)
sohandler.setLevel(logging.DEBUG)
sohandler.setFormatter(formatter)
logger.addHandler(sohandler)
return logger
def main():
usage = "%prog [options] <midi-input1> [<midi-input2> ...]"
description = "Interactive routine for cutting MIDI files. May take "\
"multiple MIDI files as input"
parser = OptionParser(usage=usage, description=description)
parser.add_option(
"-o",
"--output",
dest="output_dir",
action="store",
help=
"directory to send MIDI output to. If not given, they will be sent to a subdirectory 'cut' of that containing the first input"
)
parser.add_option(
"--fragment",
dest="fragment",
action="store",
type="float",
help=
"length in seconds of fragment to play when asked to play a beginning or ending. Default: 3secs",
default=3.0)
parser.add_option(
"--overwrite",
dest="overwrite",
action="store_true",
help=
"by default, we skip processing any files where there's a file with the same name in the output directory. This forces us to overwrite them"
)
parser.add_option(
"--ignore",
dest="ignore",
action="store",
help=
"file containing a list of filenames (not paths), one per line: any input files matching these names will be ignored and inputs marked as 'ignore' will be added to the list"
)
parser.add_option(
"--segfile",
dest="segfile",
action="store",
help=
"output a list of the MIDI files that get written by this script (just the base filename) in the format of segmidi input lists. The list will contain a basic set of default segmentation parameters. Use play_bulk_chunks to validate these. If the file exists, it will be appended"
)
options, arguments = parser.parse_args()
fragment = options.fragment
if len(arguments) == 0:
print >> sys.stderr, "You must specify at least one MIDI file"
sys.exit(1)
# Read in all the MIDI inputs
filenames = arguments
print "Processing %d inputs" % len(filenames)
if options.ignore:
if os.path.exists(options.ignore):
# Existing list
# Open the file to read in the current list and add to it
ignore_file = open(options.ignore, 'r+a')
ignore_list = [
filename.strip("\n") for filename in ignore_file.readlines()
]
print "Loaded ignore list from %s" % options.ignore
else:
# No existing list
# Open the file so we can write new entries
ignore_file = open(options.ignore, 'w')
ignore_list = []
print "Created new ignore list in %s" % options.ignore
else:
ignore_file = None
ignore_list = []
if options.segfile:
# Open the file for writing segmidi parameters to
segfile = open(options.segfile, 'a')
segcsv = csv.writer(segfile)
else:
segfile = None
try:
# Set up the output directory
if options.output_dir:
output_dir = options.output_dir
else:
# Get the directory of the first input file
output_dir = os.path.join(os.path.dirname(filenames[0]), "cut")
check_directory(output_dir, is_dir=True)
print "Outputing MIDI files to %s" % output_dir
print
for filename in filenames:
basename = os.path.basename(filename)
# Ignore any files in the ignore list
if basename in ignore_list:
print "Skipping input %s, as it's in the ignore list" % basename
continue
out_filename = os.path.join(output_dir, os.path.basename(filename))
# Check whether the output file already exists
if os.path.exists(out_filename):
if options.overwrite:
# Just warn
print "WARNING: writing out this input will overwrite an existing file"
else:
# Don't continue with this input
print "Skipping input %s, since output file already exists" % filename
continue
start = 0
end = None
print "\n####################################"
print "Processing input: %s" % filename
# Read in the midi file
try:
mid = read_midifile(filename)
except Exception, err:
print "Error reading in midi file %s: %s" % (filename, err)
continue
print "Output will be written to: %s" % out_filename
# Start by playing the whole thing
_play(mid, start, end)
try:
while True:
# Print the header information
print "\n>>>>>>>>>>>>>>>>>>>>>>"
if end is None:
end_str = "open"
else:
end_str = "%d ticks" % end
print "Start: %d ticks. End: %s" % (start, end_str)
print ">>>>>>>>>>>>>>>>>>>>>>"
print "Set start time (s); set end time (e)"
print "Play all (p); play beginning ([); play end (], optional length)"
print "Write out and proceed (w); add to ignore list (i); skip to next (n); exit (x)"
# Get a command from the user
try:
command = raw_input(">> ")
except KeyboardInterrupt:
# I quite often send an interrupt by accident, meaning
# to stop the playback, but just after it's stopped
# itself
print "Ignored keyboard interrupt"
continue
command = command.strip()
if command.lower() == "p":
# Play within the selection again
_play(mid, start, end)
elif command.lower() == "n":
break
elif command.lower() == "i":
# Add the filename to the ignore list
if ignore_file:
ignore_file.write("%s\n" %
os.path.basename(filename))
else:
print "No ignore file loaded: could not add this file to the list"
break
elif command.lower() == "x":
sys.exit(0)
elif command.lower().startswith("s"):
time = command[1:].strip()
if len(time) == 0:
print "Specify a start tick (T) or time (Ts)"
continue
start = _parse_time(mid, time)
elif command.lower().startswith("e"):
time = command[1:].strip()
if len(time) == 0:
print "Specify an end tick (T) or time (Ts)"
continue
end = _parse_time(mid, time, before=True)
elif command == "[":
# Play the opening few seconds
start_secs = _get_time(mid, start) / 1000.0
frag_end = _time_to_ticks(mid, fragment + start_secs)
_play(mid, start, frag_end)
elif command.startswith("]"):
length = command[1:].strip()
if len(length):
frag_length = float(length)
else:
frag_length = fragment
# Play the last few seconds
end_secs = _get_time(mid, end) / 1000.0
frag_start = _time_to_ticks(
mid, max(0.0, end_secs - frag_length), before=True)
_play(mid, frag_start, end)
elif command == "w":
# Write the file out
if start is None:
start = 0
# Trim the midi
trimmed = EventStreamSlice(
mid, start,
end).to_event_stream(repeat_playing=False)
# Write it out
write_midifile(trimmed, out_filename)
if segfile is not None:
# Also output a row to the segmidi index
SegmentedMidiBulkInput.writeln(segcsv, basename)
print "Output written to %s" % out_filename
# Continue to the next input
break
else:
print "Unknown command: %s" % command
continue
except EOFError:
# User hit ctrl+D: continue to next input
print "Continuing to next input..."
continue
else:
def main():
set_proc_title("jazzparser")
########################################################
usage = "jazzparser [<options>]"
description = "The main parser interface for the Jazz Parser"
## Process the input options
optparser = OptionParser(usage=usage, description=description)
###
# File input options
group = OptionGroup(optparser, "Input", "Input type and location")
optparser.add_option_group(group)
group.add_option("--file", "-f", dest="file", action="store", help="use a file to get parser input from. Use --filetype to specify the type of the file.")
group.add_option("--filetype", "--ft", dest="filetype", action="store", help="select the file type for the input file (--file). Use '--filetype help' for a list of available types. Default: chords", default='chords')
group.add_option("--file-options", "--fopt", dest="file_options", action="store", help="options for the input file (--file). Type '--fopt help', using '--ft <type>' to select file type, for a list of available options.")
group.add_option("--index", "--indices", dest="input_index", action="store", help="select individual inputs to process. Specify as a comma-separated list of indices. All inputs are loaded as usual, but only the ith input is processed, for each i in the list")
group.add_option("--only-load", dest="only_load", action="store_true", help="don't do anything with the inputs, just load and list them. Handy for checking the inputs load and getting their indices")
group.add_option("--partitions", dest="partitions", action="store", type="int", help="divide the input data into this number of partitions and use a different set of models for each. For any parser, tagger and backoff that takes a 'model' argument, the partition number will be appended to the given value")
group.add_option("--seq-parts", "--sequence-partitions", dest="sequence_partitions", action="store", help="use a chord sequence index to partition the inputs. Input type (bulk) must support association of the inputs with chord sequences by id. Sequences in the given sequence index file are partitioned n ways (--partitions) and the inputs are processed according to their associated sequence.")
group.add_option("--continue", "--skip-done", dest="skip_done", action="store_true", help="skip any inputs for which a readable results file already exists. This is useful for continuing a bulk job that was stopped in the middle")
###
group = OptionGroup(optparser, "Parser", "Parser, supertagger and backoff parser")
optparser.add_option_group(group)
group.add_option("-d", "--derivations", dest="derivations", action="store_true", help="keep derivation logs during parse.")
group.add_option("-g", "--grammar", dest="grammar", action="store", help="use the named grammar instead of the default.")
# Parser options
group.add_option("-p", "--parser", dest="parser", action="store", help="use the named parser algorithm instead of the default. Use '-p help' to see the list of available parsers. Default: %s" % settings.DEFAULT_PARSER, default=settings.DEFAULT_PARSER)
group.add_option("--popt", "--parser-options", dest="popts", action="append", help="specify options for the parser. Type '--popt help', using '--parser <name>' to select a parser module, to get a list of options.")
# Tagger options
group.add_option("-t", "--tagger", "--supertagger", dest="supertagger", action="store", help="run the parser using the named supertagger. Use '-t help' to see the list of available taggers. Default: %s" % settings.DEFAULT_SUPERTAGGER, default=settings.DEFAULT_SUPERTAGGER)
group.add_option("--topt", "--tagger-options", dest="topts", action="append", help="specify options for the tagger. Type '--topt help', using '-u <name>' to select a tagger module, to get a list of options.")
# Backoff options
group.add_option("-b", "--backoff", "--noparse", dest="backoff", action="store", help="use the named backoff model as a backoff if the parser produces no results")
group.add_option("--bopt", "--backoff-options", "--backoff-options", "--npo", dest="backoff_opts", action="append", help="specify options for the backoff model. Type '--npo help', using '--backoff <name>' to select a backoff modules, to get a list of options.")
###
# Multiprocessing options
group = OptionGroup(optparser, "Multiprocessing")
optparser.add_option_group(group)
group.add_option("--processes", dest="processes", action="store", type="int", help="number of processes to create to perform parses in parallel. Default: 1, i.e. no process pool. Use -1 to create a process for every input", default=1)
###
# Output options
group = OptionGroup(optparser, "Output")
optparser.add_option_group(group)
group.add_option("--output", dest="output", action="store", help="directory name to output parse results to. A filename specific to the individual input will be appended to this")
group.add_option("--topn", dest="topn", action="store", type="int", help="limit the number of final results to store in the output file to the top n by probability. By default, stores all")
group.add_option("--output-opts", "--oopts", dest="output_opts", action="store", help="options that affect the output formatting. Use '--output-opts help' for a list of options.")
group.add_option("-a", "--atomic-results", dest="atoms_only", action="store_true", help="only include atomic categories in the results.")
group.add_option("-l", "--latex", dest="latex", action="store_true", help="output all results as Latex source. Used to produce a whole Latex document, but doesn't any more")
group.add_option("--all-times", dest="all_times", action="store_true", help="display all timing information on semantics in output.")
group.add_option("-v", "--debug", dest="debug", action="store_true", help="output verbose debugging information.")
group.add_option("--time", dest="time", action="store_true", help="time how long the parse takes and output with the results.")
group.add_option("--no-results", dest="no_results", action="store_true", help="don't print out the parse results at the end. Obviously you'll want to make sure they're going to a file (--output). This is useful for bulk parse jobs, where the results produce a lot of unnecessary output")
group.add_option("--no-progress", dest="no_progress", action="store_true", help="don't output the summary of completed sequences after each one finishes")
###
# Output analysis and harmonical
group = OptionGroup(optparser, "Output processing", "Output analysis and harmonical")
optparser.add_option_group(group)
group.add_option("--harmonical", dest="harmonical", action="store", help="use the harmonical to play the chords justly intoned according to the top result and output to a wave file.")
group.add_option("--enharmonical", dest="enharmonical", action="store", help="use the harmonical to play the chords in equal temperament and output to a wave file.")
group.add_option("--midi", dest="midi", action="store_true", help="generate MIDI files from the harmonical, instead of wave files.")
group.add_option("--tempo", dest="tempo", action="store", type=int, help="tempo to use for the generated music (see --harmonical/--enharmonical). Default: 120", default=120)
group.add_option("--lh-analysis", dest="lh_analysis", action="store_true", help="output the Longuet-Higgins space interpretation of the semantics for each result.")
group.add_option("--lh-coordinates", dest="lh_coord", action="store_true", help="like lh-analysis, but displays the coordinates of the points instead of their names.")
###
# Logging options
group = OptionGroup(optparser, "Logging")
optparser.add_option_group(group)
group.add_option("--long-progress", dest="long_progress", action="store_true", help="print a summary of the chart so far after each chord/word has been processed.")
group.add_option("--progress", "--short-progress", dest="short_progress", action="store_true", help="print a small amount of information out during parsing to indicate progress.")
group.add_option("--logger", dest="logger", action="store", help="directory to put parser logging in. A filename based on an identifier for each individual input will be appended.")
###
# Shell options
group = OptionGroup(optparser, "Shell", "Interactive shell for inspecting results and parser state")
optparser.add_option_group(group)
group.add_option("-i", "--interactive", dest="interactive", action="store_true", help="enter interactive mode after parsing.")
group.add_option("--error", dest="error_shell", action="store_true", help="catch any errors, report them and then enter the interactive shell. This also catches keyboard interrupts, so you can use it to halt parsing and enter the shell.")
# Read in command line options and args
options, clinput = parse_args_with_config(optparser)
########################### Option processing ####################
# Get log level option first, so we can start using the logger
if options.debug:
log_level = logging.DEBUG
else:
log_level = logging.INFO
# Set up a logger
init_logging(log_level)
if options.latex:
settings.OPTIONS.OUTPUT_LATEX = True
if options.logger:
# Directory
parse_logger_dir = options.logger
check_directory(parse_logger_dir)
else:
parse_logger_dir = None
######## Grammar ########
# Check the grammar actually exists
grammar_names = get_grammar_names()
if options.grammar is not None and options.grammar not in grammar_names:
# This is not a valid grammar name
logger.error("The grammar '%s' does not exist. Possible "\
"grammars are: %s." % (options.grammar, ", ".join(grammar_names)))
return 1
grammar = get_grammar(options.grammar)
######## Parser ########
# Load the requested parser
from jazzparser.parsers import PARSERS
if options.parser.lower() == "help":
print "Available parsers are: %s" % ", ".join(PARSERS)
return 0
try:
parser_cls = get_parser(options.parser)
except ParserLoadError:
logger.error("The parser '%s' could not be loaded. Possible "\
"parsers are: %s" % (options.parser, ", ".join(PARSERS)))
return 1
# Get parser options
if options.popts is not None:
poptstr = options.popts
if "help" in [s.strip().lower() for s in poptstr]:
# Output this tagger's option help
from jazzparser.utils.options import options_help_text
print options_help_text(parser_cls.PARSER_OPTIONS, intro="Available options for selected parser")
return 0
poptstr = ":".join(poptstr)
else:
poptstr = ""
popts = ModuleOption.process_option_string(poptstr)
# Check that the options are valid
try:
parser_cls.check_options(popts)
except ModuleOptionError, err:
logger.error("Problem with parser options (--popt): %s" % err)
return 1
# Time the process and output timing info if requested
time_parse = options.time
# Display all time information on semantics in output
settings.OPTIONS.OUTPUT_ALL_TIMES = options.all_times
# Set output options according to the command line spec
grammar.formalism.cl_output_options(options.output_opts)
# Prepare output directory
if options.output is None:
output_dir = None
else:
# Check the output directory exists before starting
output_dir = os.path.abspath(options.output)
check_directory(output_dir, is_dir=True)
if options.partitions and options.partitions > 1:
partitions = options.partitions
else:
partitions = 1
############################ Input processing #####################
stdinput = False
# Try getting a file from the command-line options
input_data = command_line_input(filename=options.file,
filetype=options.filetype,
options=options.file_options)
# Record progress in this for helpful output
global completed_parses
partition_numbers = None