Python FeatureFactory.readData Examples

Example #1

File: NER.py Project: JeffreyJosanne/NLP

def main(argv):
    if len(argv) < 2:
        print 'USAGE: python NER.py trainFile testFile'
        exit(0)
    
    printOp = ''
    if len(argv) > 2:
        printOp = '-print'

    featureFactory = FeatureFactory()

    # read the train and test data
    trainData = featureFactory.readData(argv[0])
    testData = featureFactory.readData(argv[1])
    
    # add the features
    trainDataWithFeatures = featureFactory.setFeaturesTrain(trainData);
    testDataWithFeatures = featureFactory.setFeaturesTest(testData);

    # write the updated data into JSON files
    featureFactory.writeData(trainDataWithFeatures, 'trainWithFeatures');
    featureFactory.writeData(testDataWithFeatures, 'testWithFeatures');
    
    # run MEMM 
    output = Popen(['java','-cp', 'classes', '-Xmx1G' ,'MEMM'
                    ,'trainWithFeatures.json', 'testWithFeatures.json',
                    printOp], stdout=PIPE).communicate()[0]

    print output

Example #2

File: NER.py Project: varunkapoor/Natural-Language-Processing

def main(argv):
    if len(argv) < 2:
        print 'USAGE: python NER.py trainFile testFile'
        exit(0)

    printOp = ''
    if len(argv) > 2:
        printOp = '-print'

    featureFactory = FeatureFactory()

    # read the train and test data
    trainData = featureFactory.readData(argv[0])
    testData = featureFactory.readData(argv[1])

    # add the features
    trainDataWithFeatures = featureFactory.setFeaturesTrain(trainData)
    testDataWithFeatures = featureFactory.setFeaturesTest(testData)

    # write the updated data into JSON files
    featureFactory.writeData(trainDataWithFeatures, 'trainWithFeatures')
    featureFactory.writeData(testDataWithFeatures, 'testWithFeatures')

    # run MEMM
    output = Popen([
        'java', '-cp', '../java/classes', '-Xmx2G', 'MEMM',
        'trainWithFeatures.json', 'testWithFeatures.json', printOp
    ],
                   stdout=PIPE).communicate()[0]

    # java -cp classes -Xmx1G MEMM trainWithFeatures.json testWithFeatures.json
    # java -cp ../java/classes -Xmx1G MEMM trainWithFeatures.json testWithFeatures.json

    print output

Example #3

File: NER.py Project: M4573R/Natural-Language-Processing-Stanford-Coursera

def main(argv):
    # defaults
    if len(argv) == 0:
        argv.append("../data/train")
        argv.append("../data/dev")
    elif len(argv) < 2:
        print ('USAGE: python NER.py trainFile testFile')
        exit(0)

    # Set this to -print to print
    printOp = ''
    if len(argv) > 2:
        printOp = '-print'

    featureFactory = FeatureFactory()

    # read the train and test data
    trainData = featureFactory.readData(argv[0])
    testData = featureFactory.readData(argv[1])

    # add the features
    trainDataWithFeatures = featureFactory.setFeaturesTrain(trainData);
    testDataWithFeatures = featureFactory.setFeaturesTest(testData);

    # write the updated data into JSON files
    featureFactory.writeData(trainDataWithFeatures, 'trainWithFeatures');
    featureFactory.writeData(testDataWithFeatures, 'testWithFeatures');

    # run MEMM
    output = Popen(['java','-cp', 'classes', '-Xmx1G' ,'MEMM'
                    ,'trainWithFeatures.json', 'testWithFeatures.json',
                    printOp], stdout=PIPE).communicate()[0]

    print (output)

Example #4

def output(partId, ch_aux):
    """Uses the student code to compute the output for test cases."""
    print '== Running your code ...'

    featureFactory = FeatureFactory()

    # read the train and test data
    trainData = featureFactory.readData("../data/train")
    testData = featureFactory.readTestData(ch_aux)

    # add the features
    trainDataWithFeatures = featureFactory.setFeaturesTrain(trainData)
    testDataWithFeatures = featureFactory.setFeaturesTest(testData)

    # write the updated data into JSON files
    featureFactory.writeData(trainDataWithFeatures, "trainWithFeaturesSubmit")
    featureFactory.writeData(testDataWithFeatures, "testWithFeaturesSubmit")

    # run MEMM
    output = Popen([
        'java', '-cp', 'classes', '-Xmx1G', 'MEMM',
        'trainWithFeaturesSubmit.json', 'testWithFeaturesSubmit.json',
        '-submit'
    ],
                   stdout=PIPE).communicate()[0]
    # print output[:100]
    os.remove('trainWithFeaturesSubmit.json')
    os.remove('testWithFeaturesSubmit.json')

    print '== Finished running your code'

    return output

Example #5

File: submit.py Project: CaglarGonul/Natural-Language-Processing

def output(partId, ch_aux):
  """Uses the student code to compute the output for test cases."""
  print '== Running your code ...'

  featureFactory = FeatureFactory()

  # read the train and test data
  trainData = featureFactory.readData("../data/train")
  testData = featureFactory.readTestData(ch_aux)
  
  # add the features
  trainDataWithFeatures = featureFactory.setFeaturesTrain(trainData);
  testDataWithFeatures = featureFactory.setFeaturesTest(testData);

  # write the updated data into JSON files
  featureFactory.writeData(trainDataWithFeatures, "trainWithFeaturesSubmit");
  featureFactory.writeData(testDataWithFeatures, "testWithFeaturesSubmit");

  # run MEMM 
  output = Popen(['java','-cp', 'classes', '-Xmx1G' ,'MEMM'
                  ,'trainWithFeaturesSubmit.json', 'testWithFeaturesSubmit.json',
                  '-submit'], stdout=PIPE).communicate()[0]
  # print output[:100]
  os.remove('trainWithFeaturesSubmit.json')
  os.remove('testWithFeaturesSubmit.json')

  print '== Finished running your code'

  return output

Example #6

File: NER.py Project: mlliu/Dan-Manning-NLP-master

def main():
    
    print 'USAGE: python NER.py trainFile testFile'
    featureFactory = FeatureFactory()

    # read the train and test data
    trainData = featureFactory.readData("../data/train")
    testData = featureFactory.readData("../data/dev")

    # add the features
    trainDataWithFeatures = featureFactory.setFeaturesTrain(trainData);
    testDataWithFeatures = featureFactory.setFeaturesTest(testData);

    # write the updated data into JSON files
    featureFactory.writeData(trainDataWithFeatures, 'trainWithFeatures');
    featureFactory.writeData(testDataWithFeatures, 'testWithFeatures');

    # run MEMM
    output = Popen(['java','-cp', 'classes', '-Xmx1G' ,'MEMM'
                    ,'trainWithFeatures.json', 'testWithFeatures.json', '-print'], 
                    stdout=PIPE).communicate()[0]

    print output

Example #7

def main(argv):
    # defaults
    if len(argv) == 0:
        argv.append("../data/train")
        argv.append("../data/dev")
    elif len(argv) < 2:
        print 'USAGE: python NER.py trainFile testFile'
        exit(0)

    # Set this to -print to print
    printOp = ''
    if len(argv) > 2:
        printOp = '-print'

    featureFactory = FeatureFactory()

    # read the train and test data
    trainData = featureFactory.readData(argv[0])
    testData = featureFactory.readData(argv[1])

    # add the features
    trainDataWithFeatures = featureFactory.setFeaturesTrain(trainData)
    testDataWithFeatures = featureFactory.setFeaturesTest(testData)

    # write the updated data into JSON files
    featureFactory.writeData(trainDataWithFeatures, 'trainWithFeatures')
    featureFactory.writeData(testDataWithFeatures, 'testWithFeatures')

    # run MEMM
    output = Popen([
        'java', '-cp', 'classes', '-Xmx2G', 'MEMM', 'trainWithFeatures.json',
        'testWithFeatures.json', printOp
    ],
                   stdout=PIPE).communicate()[0]

    print output

Example #8

File: submit.py Project: YichenGong/Natural-Language-Processing-Stanford-Coursera

def output(partId, ch_aux):
    """Uses the student code to compute the output for test cases."""
    print "== Running your code ..."

    featureFactory = FeatureFactory()

    # read the train and test data
    trainData = featureFactory.readData("../data/train")
    testData = featureFactory.readTestData(ch_aux)

    # add the features
    trainDataWithFeatures = featureFactory.setFeaturesTrain(trainData)
    testDataWithFeatures = featureFactory.setFeaturesTest(testData)

    # write the updated data into JSON files
    featureFactory.writeData(trainDataWithFeatures, "trainWithFeaturesSubmit")
    featureFactory.writeData(testDataWithFeatures, "testWithFeaturesSubmit")

    # run MEMM
    output = Popen(
        [
            "java",
            "-cp",
            "classes",
            "-Xmx1G",
            "MEMM",
            "trainWithFeaturesSubmit.json",
            "testWithFeaturesSubmit.json",
            "-submit",
        ],
        stdout=PIPE,
    ).communicate()[0]
    # print output[:100]
    os.remove("trainWithFeaturesSubmit.json")
    os.remove("testWithFeaturesSubmit.json")

    print "== Finished running your code"

    return output