def run(file, start_page=1, end_page=1000000): all_sentences = Sentence.objects.filter( language=Language.get('en')).order_by('id') paginator = Paginator(all_sentences, 100) #pages = ((i,paginator.page(i)) for i in range(start_page,paginator.num_pages)) def do_batch(sentences): preds = [] for sentence in sentences: try: preds.extend(process_sentence(sentence)) # changed to an improbable exception for now except Exception, e: # Add sentence e.sentence = sentence # Extract traceback e_type, e_value, e_tb = sys.exc_info() e.tb = "\n".join( traceback.format_exception(e_type, e_value, e_tb)) # Raise again raise e file.write('\n--- ') yaml.dump_all(preds, file)
def run(file, start_page=1, end_page=1000000): all_sentences = Sentence.objects.filter(language=Language.get('en')).order_by('id') paginator = Paginator(all_sentences,100) #pages = ((i,paginator.page(i)) for i in range(start_page,paginator.num_pages)) def do_batch(sentences): preds = [] for sentence in sentences: try: preds.extend(process_sentence(sentence)) # changed to an improbable exception for now except Exception, e: # Add sentence e.sentence = sentence # Extract traceback e_type, e_value, e_tb = sys.exc_info() e.tb = "\n".join(traceback.format_exception( e_type, e_value, e_tb )) # Raise again raise e file.write('\n--- ') yaml.dump_all(preds, file)
Method: prioritize_sentences All are documented below. Release Notes: Jan 25, 2013 Added more doc. Added method: prioritize_sentences Added methods pretty_print_cmd, pretty_print_cmds Added 'threshold' arguments Made cmd instances that hold an instance of d4d along with the 'actions' and threshold """ import logging #Low level language tools from conceptnet.models import Language en = Language.get('en') en_nl = en.nl import os, sys sys.path.append(os.path.abspath("../")) from d4d import d4d class cmd: """An instance of the 'cmd' class has the properties: name: A string. After its made, the instance can be accessed via cmd.name_of_instance. For example: d4d.c4 is the "default" instance that is automatically created when you load 'cmd'. It uses d4d.c4 (Conceptnet4) for its d4d instance. d4di: The instance of d4d used by this instance, or "any_format_assertions" accepted by d4d. In this case, an instance of d4d is made with the above 'name', and the provided assertions. If you want to use the other arguments to making
Method: prioritize_sentences All are documented below. Release Notes: Jan 25, 2013 Added more doc. Added method: prioritize_sentences Added methods pretty_print_cmd, pretty_print_cmds Added 'threshold' arguments Made cmd instances that hold an instance of d4d along with the 'actions' and threshold """ import logging #Low level language tools from conceptnet.models import Language en = Language.get('en') en_nl = en.nl import os, sys sys.path.append(os.path.abspath("../")) from d4d import d4d class cmd: """An instance of the 'cmd' class has the properties: name: A string. After its made, the instance can be accessed via cmd.name_of_instance. For example: d4d.c4 is the "default" instance that is automatically created when you load 'cmd'. It uses d4d.c4 (Conceptnet4) for its d4d instance. d4di: The instance of d4d used by this instance, or "any_format_assertions" accepted by d4d. In this case, an instance of d4d is made with the above 'name', and the provided assertions. If you want to use the other arguments to making an instance of d4d, create the instance of d4d normally and use it for
from conceptnet.models import Language, Sentence from simplenlp.ja_cabocha.system import * from simplenlp.ja_cabocha.debug import * import operator # Import the data for our test # from test_ja_unit_tests_data import * #################################################################################################### ## Options ######################################################################################### #################################################################################################### assert_on_fail = True use_color = True ## Variables ####################################################################################### ja = Language.get('ja') ja_nl = ja.nl d = JaDebug(use_color) i = "\t" db_parsed = 0 db_successes = 0 db_failures = 0 successes = 0 failures = 0 class JaUnitTest(): def __init__(self, text): self.text = text self.utterance = ja_nl.utterance(text)
from conceptnet.models import Language, Sentence from simplenlp.ja_cabocha.system import * from simplenlp.ja_cabocha.debug import * import operator # Import the data for our test # from test_ja_unit_tests_data import * #################################################################################################### ## Options ######################################################################################### #################################################################################################### assert_on_fail = True use_color = True ## Variables ####################################################################################### ja = Language.get('ja') ja_nl = ja.nl d = JaDebug(use_color) i = "\t" db_parsed = 0 db_successes = 0 db_failures = 0 successes = 0 failures = 0 class JaUnitTest(): def __init__(self, text): self.text = text self.utterance = ja_nl.utterance(text) def do_test(self, name, control):