def main(): usage = "%prog <model-name>" description = "Generate chord sequences from a PCFG model" parser = OptionParser(usage=usage, description=description) parser.add_option("-g", "--grammar", dest="grammar", action="store", \ help="use the named grammar instead of the default.") parser.add_option("-d", "--debug", dest="debug", action="store_true", \ help="output debugging information during generation") options, arguments = parse_args_with_config(parser) if options.debug: logger = create_plain_stderr_logger(log_level=logging.DEBUG) else: logger = create_plain_stderr_logger(log_level=logging.WARN) if len(arguments) < 1: print "Specify a model name" sys.exit(1) model_name = arguments[0] grammar = get_grammar(options.grammar) PcfgModel = grammar.formalism.PcfgModel # Load the trained model model = PcfgModel.load_model(model_name) sequence = model.generate(logger=logger) if sequence is None: print "Model did not generate a sequence" else: print sequence
def main(): usage = "%prog <model-name>" description = "Generate chord sequences from a PCFG model" parser = OptionParser(usage=usage, description=description) parser.add_option("-g", "--grammar", dest="grammar", action="store", \ help="use the named grammar instead of the default.") parser.add_option("-d", "--debug", dest="debug", action="store_true", \ help="output debugging information during generation") options, arguments = parse_args_with_config(parser) if options.debug: logger = create_plain_stderr_logger(log_level=logging.DEBUG) else: logger = create_plain_stderr_logger(log_level=logging.WARN) if len(arguments) < 1: print "Specify a model name" sys.exit(1) model_name = arguments[0] grammar = get_grammar(options.grammar) PcfgModel = grammar.formalism.PcfgModel # Load the trained model model = PcfgModel.load_model(model_name) sequence = model.generate(logger=logger) if sequence is None: print "Model did not generate a sequence" else: print sequence
def __init__(self, grammar, tagger, options={}, backoff=None, backoff_options={}, logger=None): """ @param grammar: the L{jazzparser.grammar.Grammar} instance to use for parsing @param tagger: the L{jazzparser.taggers.tagger.Tagger} subclass instance to use to tag the input @param backoff: an optional L{jazzparser.backoff.base.BackoffBuilder} class to use as a fallback if the parser returns no parses. Whether this is used and in what circumstances depends on the type of parser. @param backoff_options: dictionary of options to pass to the backoff model if it gets used. @type logger: C{logging.Logger} @param logger: a logger to which all progress information during parsing will be written. By default, outputs to stderr. """ self.grammar = grammar self.tagger = tagger self.backoff_options = backoff_options if backoff is not None: # Look up the backoff model if one is requested self.backoff = backoff # Pre-check the options dict # This will be done again by the module when instantiated, but # we do it now to verify the options ModuleOption.process_option_dict(backoff_options, backoff.BUILDER_OPTIONS) else: self.backoff = None # Initialize using parser-specific options self.options = type(self).check_options(options) if logger is None: # Output to stderr instead self.logger = create_plain_stderr_logger() else: self.logger = logger self.timed_out = False
def __init__(self, input, options={}, logger=None): # Initialize using tagger-specific options self.options = type(self).check_options(options) # Check what input type we've received and preprocess it datatype, input = detect_input_type(input, allowed=self.INPUT_TYPES) # Store this for the subclass to use as appropriate self.input = input self.original_input = input # Subclasses may redefine self.input to taste # We keep the original wrapped input somewhere where it's sure to remain self.wrapped_input = input # Make sure we have some logger if logger is None: # Output to stderr instead self.logger = create_plain_stderr_logger() else: self.logger = logger
def __init__(self, input, options={}, logger=None): # Initialize using tagger-specific options self.options = type(self).check_options(options) # Check what input type we've received and preprocess it datatype, input = detect_input_type(input, allowed=self.INPUT_TYPES) # Store this for the subclass to use as appropriate self.input = input self.original_input = input # Subclasses may redefine self.input to taste # We keep the original wrapped input somewhere where it's sure to remain self.wrapped_input = input # Make sure we have some logger if logger is None: # Output to stderr instead self.logger = create_plain_stderr_logger() else: self.logger = logger
def __init__(self, grammar, tagger, options={}, backoff=None, backoff_options={}, logger=None): """ @param grammar: the L{jazzparser.grammar.Grammar} instance to use for parsing @param tagger: the L{jazzparser.taggers.tagger.Tagger} subclass instance to use to tag the input @param backoff: an optional L{jazzparser.backoff.base.BackoffBuilder} class to use as a fallback if the parser returns no parses. Whether this is used and in what circumstances depends on the type of parser. @param backoff_options: dictionary of options to pass to the backoff model if it gets used. @type logger: C{logging.Logger} @param logger: a logger to which all progress information during parsing will be written. By default, outputs to stderr. """ self.grammar = grammar self.tagger = tagger self.backoff_options = backoff_options if backoff is not None: # Look up the backoff model if one is requested self.backoff = backoff # Pre-check the options dict # This will be done again by the module when instantiated, but # we do it now to verify the options ModuleOption.process_option_dict(backoff_options, backoff.BUILDER_OPTIONS) else: self.backoff = None # Initialize using parser-specific options self.options = type(self).check_options(options) if logger is None: # Output to stderr instead self.logger = create_plain_stderr_logger() else: self.logger = logger self.timed_out = False
def main(): usage = "%prog [options] <seq-file>" description = "Parses a sequence from a sequence index file using the "\ "annotations stored in the same file." parser = OptionParser(usage=usage, description=description) parser.add_option( "--popt", "--parser-options", dest="popts", action="append", help= "specify options for the parser. Type '--popt help' to get a list of options (we use a DirectedCkyParser)" ) parser.add_option("--derivations", "--deriv", dest="derivations", action="store_true", help="print out derivation traces of all the results") parser.add_option("--index", "-i", dest="index", action="store", type="int", help="parse just the sequence with this index") parser.add_option("--quiet", "-q", dest="quiet", action="store_true", help="show only errors in the output") parser.add_option( "--tonal-space", "--ts", dest="tonal_space", action="store_true", help="show the tonal space path (with -q, shows only paths)") parser.add_option( "--output-set", "-o", dest="output_set", action="store", help="store the analyses to a tonal space analysis set with this name") parser.add_option( "--trace-parse", "-t", dest="trace_parse", action="store_true", help= "output a trace of the shift-reduce parser's operations in producing the full interpretation from the annotations" ) options, arguments = parser.parse_args() if len(arguments) < 1: print "You must specify a sequence file" sys.exit(1) if options.popts is not None: poptstr = options.popts if "help" in [s.strip().lower() for s in poptstr]: # Output this tagger's option help print options_help_text( DirectedCkyParser.PARSER_OPTIONS, intro="Available options for the directed parser") return 0 else: poptstr = "" popts = ModuleOption.process_option_string(poptstr) grammar = get_grammar() if options.quiet: logger = create_plain_stderr_logger(log_level=logging.ERROR) else: logger = create_plain_stderr_logger() if options.trace_parse: parse_logger = logger else: parse_logger = None seq_index = SequenceIndex.from_file(arguments[0]) # Get the chord sequence(s) if options.index is None: seqs = seq_index.sequences else: seqs = [seq_index.sequence_by_index(options.index)] logger.info("%d sequences\n" % len(seqs)) full_analyses = [] stats = { 'full': 0, 'partial': 0, 'fail': 0, } # Try parsing every sequence for seq in seqs: logger.info("====== Sequence %s =======" % seq.string_name) try: results = parse_sequence_with_annotations( seq, grammar, logger=logger, parse_logger=parse_logger) except ParseError, err: logger.error("Error parsing: %s" % err) stats['fail'] += 1 else: # This may have resulted in multiple partial parses logger.info("%d partial parses" % len(results)) if len(results) == 1: stats['full'] += 1 else: stats['partial'] += 1 if options.derivations: # Output the derivation trace for each partial parse for result in results: print print result.derivation_trace if options.tonal_space: # Output the tonal space coordinates path = grammar.formalism.sign_to_coordinates(results[0]) for i, point in enumerate(path): print "%d, %d: %s" % (seq.id, i, point) # Only include a result in the output analyses if it was a full parse if len(results) == 1: full_analyses.append((seq.string_name, results[0].semantics)) else: logger.warn("%s was not included in the output analyses, "\ "since it was not fully parsed" % seq.string_name)
jobs = [] for input in input_getter: if input: # Get an identifier for this input input_identifier = name_getter.next() print "Processing input: %s (%s)" % (input, input_identifier) # Get a filename for a logger for this input if parse_logger_dir: parse_logger = os.path.join(parse_logger_dir, "%s.log" % \ slugify(input_identifier)) print >>sys.stderr, "Logging parser progress to %s" % parse_logger logger = create_logger(filename=parse_logger) else: logger = create_plain_stderr_logger() # Catch any errors and continue to the next input, instead of giving up try: if isinstance(input, str): input = input.rstrip("\n") if len(input) == 0: return input = ChordInput.from_string(input) logger.info("Tagging sequence (%d timesteps)" % len(input)) # Prepare a suitable tagger component tagger = tagger_cls(grammar, input, options=topts.copy(), logger=logger) except KeyboardInterrupt: print "Exiting on keyboard interrupt"
jobs = [] for input in input_getter: if input: # Get an identifier for this input input_identifier = name_getter.next() print "Processing input: %s (%s)" % (input, input_identifier) # Get a filename for a logger for this input if parse_logger_dir: parse_logger = os.path.join(parse_logger_dir, "%s.log" % \ slugify(input_identifier)) print >> sys.stderr, "Logging parser progress to %s" % parse_logger logger = create_logger(filename=parse_logger) else: logger = create_plain_stderr_logger() # Catch any errors and continue to the next input, instead of giving up try: if isinstance(input, str): input = input.rstrip("\n") if len(input) == 0: return input = ChordInput.from_string(input) logger.info("Tagging sequence (%d timesteps)" % len(input)) # Prepare a suitable tagger component tagger = tagger_cls(grammar, input, options=topts.copy(), logger=logger)
def do_parse(grammar, tagger_cls, parser_cls, input, topts, popts, backoff, npopts, options, identifier, multiprocessing=False, logfile=None, partition=None): """ Function called for each input to do tagging and parsing and return the results. It's a separate function so that we can hand it over to worker processes to do multiprocessing. @type logfile: str @param logfile: filename to send logging output to. If None, will log to stderr """ # If the input's a string, preprocess it if isinstance(input, str): input = input.rstrip("\n") if len(input) == 0: return input = ChordInput.from_string(input) print "Processing input: %s (%s)" % (input, identifier) if logfile is None: # Sending logging output to stderr logger = create_plain_stderr_logger() else: logger = create_logger(filename=logfile) print "Logging parser progress to %s" % logfile # Prepare an initial response # We'll fill in some values of this later response = { 'tagger' : None, 'parser' : None, 'input' : input, 'error' : None, 'messages' : [], 'time' : None, 'identifier' : identifier, 'results' : None, 'timed_out' : False, } tagger = None parser = None messages = [] if options.short_progress: # Only output the short form of the progress reports progress = 2 elif options.long_progress: progress = 1 else: progress = 0 # Start a timer now to time the parse timer = ExecutionTimer(clock=True) # Catch any errors and continue to the next input, instead of giving up try: ######### Do that parsing thang logger.info("Tagging sequence (%d timesteps)" % len(input)) # Prepare a suitable tagger component tagger = tagger_cls(grammar, input, options=topts.copy(), logger=logger) if not multiprocessing: response['tagger'] = tagger # Create a parser using this tagger parser = parser_cls(grammar, tagger, options=popts.copy(), backoff=backoff, backoff_options=npopts.copy(), logger=logger) if not multiprocessing: response['parser'] = parser try: # Parse to produce a list of results results = parser.parse(derivations=options.derivations, summaries=progress) except (KeyboardInterrupt, Exception), err: if multiprocessing: # Don't go interactive if we're in a subprocess # Instead, just return with an error response.update({ 'error' : exception_tuple(str_tb=True), }) return response else: # Drop into the shell if type(err) == KeyboardInterrupt: print "Dropping out on keyboard interrupt" print "Entering shell: use 'chart' command to see current state of parse" elif options.error_shell: print >> sys.stderr, "Error parsing %s" % str(input) print >> sys.stderr, "The error was:" traceback.print_exc(file=sys.stderr) # If we keyboard interrupted, always go into the shell, so # the user can see how far we got if options.error_shell or type(err) == KeyboardInterrupt: # Instead of exiting, enter the interactive shell print from jazzparser.shell import interactive_shell env = {} env.update(globals()) env.update(locals()) interactive_shell(parser.chart.parses,options,tagger,parser, grammar.formalism,env,input_data=input) return else: raise except (KeyboardInterrupt, Exception), err: if multiprocessing: response.update({ 'error' : exception_tuple(str_tb=True), }) return response else: if type(err) == KeyboardInterrupt: print "Exiting on keyboard interrupt" sys.exit(1) else: response.update({ 'error' : exception_tuple(str_tb=True), 'messages' : messages, 'time' : timer.get_time(), }) return response