def badsgml2text(ldc_name, args):
path = '{0}/bsgml_trees/{1}'.format(args.workspace, ldc_name)
logging.info('Processing %s', path)
n_lines = []
try:
with open(path, 'r') as fi:
with gzip.open('{0}/trees/{1}.gz'.format(args.workspace, ldc_name), 'wb') as fo:
lines = fi.read().split('\n')
doc_re = re.compile('<doc id="(.+)">')
doc_id, doc_lines = None, None
for line in lines:
# try to match <doc ...
m = doc_re.match(line)
if m is not None:
# starts a doc
doc_id = m.group(1)
doc_lines = []
# try to match </doc>
elif line == '</doc>':
# add the doc to an actual SGML file
n_lines.append(len(doc_lines))
writedoctext(fo, doc_lines, id=doc_id)
doc_lines = None
doc_id = None
# if there is an open doc, append lines to it
elif line and doc_lines is not None:
doc_lines.append(line)
#print >> sys.stderr, ptb_str
#print ' '.join(Tree(ptb_str).leaves())
except:
raise Exception(''.join(traceback.format_exception(*sys.exc_info())))
return n_lines
def grids_from_text(ldc_desc, args):
"""Extract grids for documents in a corpus (already parsed)"""
t0 = time()
try:
input_path = '{0}/trees/{1}'.format(args.workspace, ldc_desc['name'])
output_path = '{0}/grids/{1}'.format(args.workspace, ldc_desc['name'])
logging.info('processing: %s', input_path)
if not args.dry_run:
with gzip.open(input_path + '.gz', 'rb') as fi:
with gzip.open(output_path + '.gz', 'wb') as fo:
for lines, attrs in iterdoctext(fi):
logging.debug('document %s', attrs['id'])
cmd_line = args.ExtractGrid
cmd_args = shlex.split(cmd_line)
proc = subprocess.Popen(cmd_args,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE)
(stdoutdata, stderrdata) = proc.communicate(
'{0}\n'.format('\n'.join(lines)))
writedoctext(fo,
stdoutdata.split('\n'),
id=attrs['id'])
logging.info('done: %s', output_path)
except:
raise Exception(''.join(traceback.format_exception(*sys.exc_info())))
#print >> sys.stderr, ptb_str
#print ' '.join(Tree(ptb_str).leaves())
#print
return time() - t0
def grids_from_text(ldc_desc, args):
"""Extract grids for documents in a corpus (already parsed)"""
t0 = time()
try:
input_path = '{0}/trees/{1}'.format(args.workspace, ldc_desc['name'])
output_path = '{0}/grids/{1}'.format(args.workspace, ldc_desc['name'])
logging.info('processing: %s', input_path)
if not args.dry_run:
with gzip.open(input_path + '.gz', 'rb') as fi:
with gzip.open(output_path + '.gz', 'wb') as fo:
for lines, attrs in iterdoctext(fi):
logging.debug('document %s', attrs['id'])
cmd_line = args.ExtractGrid
cmd_args = shlex.split(cmd_line)
proc = subprocess.Popen(cmd_args, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
(stdoutdata, stderrdata) = proc.communicate('{0}\n'.format('\n'.join(lines)))
writedoctext(fo, stdoutdata.split('\n'), id=attrs['id'])
logging.info('done: %s', output_path)
except:
raise Exception(''.join(traceback.format_exception(*sys.exc_info())))
#print >> sys.stderr, ptb_str
#print ' '.join(Tree(ptb_str).leaves())
#print
return time() - t0
def badsgml2text(ldc_name, args):
path = '{0}/bsgml_trees/{1}'.format(args.workspace, ldc_name)
logging.info('Processing %s', path)
n_lines = []
try:
with open(path, 'r') as fi:
with gzip.open('{0}/trees/{1}.gz'.format(args.workspace, ldc_name),
'wb') as fo:
lines = fi.read().split('\n')
doc_re = re.compile('<doc id="(.+)">')
doc_id, doc_lines = None, None
for line in lines:
# try to match <doc ...
m = doc_re.match(line)
if m is not None:
# starts a doc
doc_id = m.group(1)
doc_lines = []
# try to match </doc>
elif line == '</doc>':
# add the doc to an actual SGML file
n_lines.append(len(doc_lines))
writedoctext(fo, doc_lines, id=doc_id)
doc_lines = None
doc_id = None
# if there is an open doc, append lines to it
elif line and doc_lines is not None:
doc_lines.append(line)
#print >> sys.stderr, ptb_str
#print ' '.join(Tree(ptb_str).leaves())
except:
raise Exception(''.join(traceback.format_exception(*sys.exc_info())))
return n_lines
def main(args):
""" Extract documents and shuffle sentences within each document """
try:
fi = open(args.directory, 'r')
with open('{0}.shuffled'.format(args.directory), 'w') as fo:
for lines, attributes in iterdoctext(fi):
random.shuffle(lines)
logging.debug('shuffled: %s', lines)
writedoctext(fo, lines, **attributes)
logging.info('done: %s', args.directory)
except:
raise Exception(''.join(traceback.format_exception(*sys.exc_info())))
def main(args):
""" Extract documents and shuffle sentences within each document """
try:
fi = open(args.directory, 'r')
with open('{0}.shuffled'.format(args.directory), 'w') as fo:
for lines, attributes in iterdoctext(fi):
random.shuffle(lines)
logging.debug('shuffled: %s', lines)
writedoctext(fo, lines, **attributes)
logging.info('done: %s', args.directory)
except:
raise Exception(''.join(traceback.format_exception(*sys.exc_info())))
def main(args):
# reads in documents
for trees, attrs in iterdoctext(args.input):
# generator of d-sequences
sequences = (dseqs(tree,
depth=args.depth,
no_punc=not args.punc,
lexicalised=args.lexicalised,
child_phrase=args.child,
backoff=['*']) for tree in trees)
# writes d-sequences
writedoctext(args.output,
(' '.join(patterns) for patterns in sequences), **attrs)
def main(args):
# reads in documents
for trees, attrs in iterdoctext(args.input):
# generator of d-sequences
sequences = (dseqs(tree,
depth=args.depth,
no_punc=not args.punc,
lexicalised=args.lexicalised,
child_phrase=args.child,
backoff=['*'])
for tree in trees)
# writes d-sequences
writedoctext(args.output,
(' '.join(patterns) for patterns in sequences),
**attrs)
def extract_and_save_txt(sgml_gz, args):
"""Extracts documents from a gzipped sgml file -> file ids"""
try:
ids = []
n = 0
logging.info('Processing %s', sgml_gz)
stem = get_ldc_name(sgml_gz)
with gzip.open(sgml_gz, 'rb') as fi:
with gzip.open('{0}/raw/{1}.gz'.format(args.workspace, stem), 'wb') as fo:
parser = TextFromSGML(fi.read(), text_under='text', root='sgml')
for doc in parser.iterdocs():
if doc['text']:
ids.append(doc['id'])
writedoctext(fo, doc['text'].split('\n'), id=doc['id'])
logging.info('%s contains %d documents', stem, len(ids))
return ids
except:
raise Exception(''.join(traceback.format_exception(*sys.exc_info())))
def main(args):
logging.basicConfig(level=logging.INFO, format='%(levelname)s %(message)s')
# reads docs from input
docs = list(iterdoctext(args.input))
# distributes the jobs
pool = Pool(args.jobs)
logging.info('Distributing %d jobs to %d workers', len(docs), args.jobs)
result = pool.map(partial(wrap_parse, args=args), docs)
# stores the output
times = []
for (content, attrs), (trees, dt) in itertools.izip(docs, result):
writedoctext(args.output, trees, **attrs)
times.append(dt)
# dumps a summary
print >> sys.stderr, tabulate(enumerate(times), headers=['doc', 'time'], tablefmt='pipe')
def main(args):
logging.basicConfig(level=logging.INFO, format='%(levelname)s %(message)s')
# reads docs from input
docs = list(iterdoctext(args.input))
# distributes the jobs
pool = Pool(args.jobs)
logging.info('Distributing %d jobs to %d workers', len(docs), args.jobs)
result = pool.map(partial(wrap_parse, args=args), docs)
# stores the output
times = []
for (content, attrs), (trees, dt) in itertools.izip(docs, result):
writedoctext(args.output, trees, **attrs)
times.append(dt)
# dumps a summary
print >> sys.stderr, tabulate(enumerate(times), headers=['doc', 'time'], tablefmt='pipe')
def extract_and_save_txt(sgml_gz, args):
"""Extracts documents from a gzipped sgml file -> file ids"""
try:
ids = []
n = 0
logging.info('Processing %s', sgml_gz)
stem = get_ldc_name(sgml_gz)
with gzip.open(sgml_gz, 'rb') as fi:
with gzip.open('{0}/raw/{1}.gz'.format(args.workspace, stem),
'wb') as fo:
parser = TextFromSGML(fi.read(),
text_under='text',
root='sgml')
for doc in parser.iterdocs():
if doc['text']:
ids.append(doc['id'])
writedoctext(fo, doc['text'].split('\n'), id=doc['id'])
logging.info('%s contains %d documents', stem, len(ids))
return ids
except:
raise Exception(''.join(traceback.format_exception(*sys.exc_info())))
return todo, done, missing
def wrap_dseqs((i, ipath, opath), depth, **kwargs):
"""
Wrap a call to dseqs. To be used with Pool.map.
"""
try:
logging.info('(%d) %s ', i, ipath)
fi = smart_open(ipath, 'r')
fo = smart_open(opath, 'w')
for trees, attrs in iterdoctext(fi):
sequences = [
' '.join(dseqs(tree, depth=depth, **kwargs)) for tree in trees
]
writedoctext(fo, sequences, **attrs)
except:
raise Exception(''.join(traceback.format_exception(*sys.exc_info())))
def extract_dseqs(corpus, args, namespace, **kwargs):
"""
Extracts dsequences for a certain corpus
"""
logging.info('Extracting d-sequences for: %s', corpus)
input_dir = namespace.trees
output_dir = namespace.dseqs
todo, done, missing = file_check(corpus, input_dir, output_dir)
if not missing:
done = frozenset(os.path.basename(path) for path in glob('{0}/{1}*'.format(output_dir, corpus)))
logging.info('%d files matching %s', len(done), '{0}/{1}*'.format(output_dir, corpus))
missing = todo - done
return todo, done, missing
def wrap_dseqs((i, ipath, opath), depth, **kwargs):
"""
Wrap a call to dseqs. To be used with Pool.map.
"""
try:
logging.info('(%d) %s ', i, ipath)
fi = smart_open(ipath, 'r')
fo = smart_open(opath, 'w')
for trees, attrs in iterdoctext(fi):
sequences = [' '.join(dseqs(tree, depth=depth, **kwargs)) for tree in trees]
writedoctext(fo, sequences, **attrs)
except:
raise Exception(''.join(traceback.format_exception(*sys.exc_info())))
def extract_dseqs(corpus, args, namespace, **kwargs):
"""
Extracts dsequences for a certain corpus
"""
logging.info('Extracting d-sequences for: %s', corpus)
input_dir = namespace.trees
output_dir = namespace.dseqs
todo, done, missing = file_check(corpus, input_dir, output_dir)
if not missing:
logging.info('all d-sequences of depth %d are there, nothing to be done', args.depth)