for xann in xdoc.findall("ANNOTATION"):
annid = xann.get("id")
annset[annid] = xann
anntask = xann.get("task")
if xann.find("EXTENT") is None:
if anntask != "FE":
sys.stderr.write("Warning: extent-free non-full annotation in "+annfile+": "+ET.tostring(xann)+"\n")
continue
continue
xextent = xann.find("EXTENT")
tup = [anntask, docid, xextent.get("start_char") or "None", xextent.get("end_char") or "None", annid or "None", xextent.text or "None"]
if anntask == "NE": # simple ne annotation
tup.append(xann.get("type"))
else: # everything else has category/tag style
try:
tup.append(funornone(xann.find("CATEGORY"), lambda x: x.text))
tup.append(funornone(xann.find("TAG"), lambda x: x.text))
if (anntask == "FE"):
if xann.find("ENTITY") is not None:
eid = xann.find("ENTITY").get("entity_id")
tup.append(eid)
tup.append(annset[eid].find("TAG").text if annset[eid].find("TAG") is not None else "NONE")
elif xann.find("PHRASE") is not None:
tup.append(xann.find("PHRASE").get("phrase_id"))
else:
sys.stderr("Expected ENTITY or PHRASE at "+annfile+"; "+ET.tostring(xann))
continue
elif (anntask == "SSA"):
if xann.find("PREDICATE") is not None:
tup.append(xann.find("PREDICATE").get("predicate_id"))
else:
def main():
import codecs
parser = argparse.ArgumentParser(description="Extract and print laf annotat" \
"ion data from LRLP in a form that is amen" \
"able to insertion into future xml",
formatter_class= \
argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("--rootdir", "-r", default=".", help="root lrlp dir")
# parser.add_argument("--outfile", "-o",
# type=argparse.FileType('w'), default=sys.stdout,
# help="where to write extracted semantic info")
parser.add_argument("--outfile", "-o", help="where to write")
parser.add_argument("--extwtdir", "-et", help="extracted tweet rsd files dir")
try:
args = parser.parse_args()
except IOError as msg:
parser.error(str(msg))
outfile = open(args.outfile, 'w')
twtdir = args.extwtdir
anndir = os.path.join(args.rootdir, 'data', 'annotation')
if not os.path.exists(anndir):
sys.stderr.write("No annotation directory found\n")
sys.exit(0)
# print anndir
for annfile in recursive_file_gen(anndir):
if annfile.endswith("laf.xml") and \
not os.path.basename(annfile).startswith("."):
try:
xobj = ET.parse(annfile)
except:
sys.stderr.write("Problem parsing "+annfile+"\n")
continue
for xdoc in xobj.findall("DOC"):
docid = xdoc.get("id")
if docid.startswith('doc-'): # In NPC annotation, LDC uses "doc-n"
# instead of original docid
docid = os.path.basename(annfile).replace('.laf.xml', '')
if docid.startswith('SN_TWT_'): # No string head for TWT, need rsd file
if not os.path.isfile('%s/%s.rsd.txt' % (twtdir, docid)):
continue
# Store all annotations by id. if they have an extent, spit them out.
# if no extent, check they are entities; nothing else should be
# extent-free PREDICATE, ENTITY, and PHRASE are cross references to ids;
# for ENTITY the core type is copied, for everything else just the cross
# reference
annset = {}
for xann in xdoc.findall("ANNOTATION"):
annid = xann.get("id")
if annid.startswith('doc-'):
annid = re.sub('doc-\d+', docid, annid)
annset[annid] = xann
anntask = xann.get("task")
if xann.find("EXTENT") is None:
if anntask != "FE" and anntask != "SSA":
sys.stderr.write("Warning: extent-free non-full annotation in " \
+annfile+": "+ET.tostring(xann)+"\n")
continue
# map aberrant type
if anntask == "NPchunk":
anntask = "NPC"
xextent = xann.find('EXTENT')
if docid.startswith('SN_TWT_'): # No string head for TWT
strhead = xextent.text
tweet = open('%s/%s.rsd.txt' % (twtdir, docid)).read()
beg = int(xextent.get("start_char"))
end = int(xextent.get("end_char"))
# but don't go negative
if beg < 0 or end > len(tweet):
sys.stderr.write(annfile+" Bad offsets: can't do %d, %d on %s\n" % (beg, end, docid))
continue
strhead = tweet[beg:end+1]
tup = [anntask, docid, xextent.get("start_char") or "None",
xextent.get("end_char") or "None", annid or "None",
strhead or "None"]
else:
tup = [anntask, docid, xextent.get("start_char") or "None",
xextent.get("end_char") or "None", annid or "None",
xextent.text or "None"]
if anntask == "NE": # Simple ne annotation
# old style: in attributes. new style: in tag
if "type" in xann.keys():
tup.append(xann.get("type"))
else:
tup.append(funornone(xann.find("TAG"), lambda x: x.text))
elif (anntask == "NPC" or anntask=="NPchunk"): # NP chunking
# old style: in attributes. new style: in tag
if "type" in xann.keys():
tup.append(xann.get("type"))
else:
tup.append(funornone(xann.find("TAG"), lambda x: x.text))
else: # Everything else has category/tag style
try:
tup.append(funornone(xann.find("CATEGORY"), lambda x: x.text))
tup.append(funornone(xann.find("TAG"), lambda x: x.text))
if (anntask == "FE"):
if xann.find("ENTITY") is not None:
eid = xann.find("ENTITY").get("entity_id")
tup.append(eid)
tup.append(annset[eid].find("TAG").text if \
annset[eid].find("TAG") is not None else "NONE")
elif xann.find("PHRASE") is not None:
tup.append(xann.find("PHRASE").get("phrase_id"))
else:
sys.stderr("Expected ENTITY or PHRASE at "\
+annfile+"; "+ET.tostring(xann))
continue
elif (anntask == "SSA"):
if xann.find("PREDICATE") is not None:
tup.append(xann.find("PREDICATE").get("predicate_id"))
else:
sys.stderr.write(annfile+": Don't know how to process "\
+anntask+"\n")
continue
except:
print(annfile)
print(ET.tostring(xann))
raise
try:
outfile.write("\t".join(map(str,tup))+"\n")
except UnicodeDecodeError:
sys.stderr.write("Warning: Unknown encoding %s:%s-%s\n" % \
(tup[4], tup[2], tup[3]))
def main():
import codecs
parser = argparse.ArgumentParser(description="Extract and print laf annotat" \
"ion data from LRLP in a form that is amen" \
"able to insertion into future xml",
formatter_class= \
argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("--rootdir", "-r", default=".", help="root lrlp dir")
# parser.add_argument("--outfile", "-o",
# type=argparse.FileType('w'), default=sys.stdout,
# help="where to write extracted semantic info")
parser.add_argument("--outfile", "-o", help="where to write")
parser.add_argument("--extwtdir",
"-et",
default=None,
help="extracted tweet rsd files dir")
try:
args = parser.parse_args()
except IOError as msg:
parser.error(str(msg))
outfile = open(args.outfile, 'w')
twtdir = args.extwtdir
anndir = os.path.join(args.rootdir, 'data', 'annotation')
if not os.path.exists(anndir):
sys.stderr.write("No annotation directory found\n")
sys.exit(0)
if twtdir is not None and not os.path.exists(twtdir):
sys.stderr.write("Warning: no {}\n".format(twtdir))
twtdir = None
# print anndir
for annfile in recursive_file_gen(anndir):
if annfile.endswith("laf.xml") and \
not os.path.basename(annfile).startswith("."):
try:
xobj = ET.parse(annfile)
except:
sys.stderr.write("Problem parsing " + annfile + "\n")
continue
for xdoc in xobj.findall("DOC"):
docid = xdoc.get("id")
if docid.startswith(
'doc-'): # In NPC annotation, LDC uses "doc-n"
# instead of original docid
docid = os.path.basename(annfile).replace('.laf.xml', '')
if is_sn(
docid
) and twtdir is not None: # No string head for TWT, need rsd file
if not os.path.isfile('%s/%s.rsd.txt' % (twtdir, docid)):
continue
# Store all annotations by id. if they have an extent, spit them out.
# if no extent, check they are entities; nothing else should be
# extent-free PREDICATE, ENTITY, and PHRASE are cross references to ids;
# for ENTITY the core type is copied, for everything else just the cross
# reference
annset = {}
for xann in xdoc.findall("ANNOTATION"):
annid = xann.get("id")
if annid.startswith('doc-'):
annid = re.sub('doc-\d+', docid, annid)
annset[annid] = xann
anntask = xann.get("task")
if xann.find("EXTENT") is None:
if anntask != "FE" and anntask != "SSA":
sys.stderr.write("Warning: extent-free non-full annotation in " \
+annfile+": "+ET.tostring(xann)+"\n")
continue
# map aberrant type
if anntask == "NPchunk":
anntask = "NPC"
xextent = xann.find('EXTENT')
try:
if is_sn(
docid
) and twtdir is not None: # No string head for TWT
strhead = xextent.text
tweet = open('%s/%s.rsd.txt' %
(twtdir, docid)).read()
beg = int(xextent.get("start_char"))
end = int(xextent.get("end_char"))
# but don't go negative
if beg < 0 or end > len(tweet):
sys.stderr.write(
annfile +
" Bad offsets: can't do %d, %d on %s\n" %
(beg, end, docid))
continue
strhead = tweet[beg:end + 1]
tup = [
anntask, docid,
str(int(xextent.get("start_char"))) or "None",
xextent.get("end_char") or "None", annid
or "None", strhead or "None"
]
else:
tup = [
anntask, docid,
str(int(xextent.get("start_char"))) or "None",
xextent.get("end_char") or "None", annid
or "None", xextent.text or "None"
]
except:
sys.stderr.write(
"Trouble at %s in %s should be investigated\n" %
(docid, annfile))
continue
if anntask == "NE": # Simple ne annotation
# old style: in attributes. new style: in tag
if "type" in xann.keys():
tup.append(xann.get("type"))
else:
tup.append(
funornone(xann.find("TAG"), lambda x: x.text))
elif (anntask == "NPC"
or anntask == "NPchunk"): # NP chunking
# old style: in attributes. new style: in tag
if "type" in xann.keys():
tup.append(xann.get("type"))
else:
tup.append(
funornone(xann.find("TAG"), lambda x: x.text))
else: # Everything else has category/tag style
try:
tup.append(
funornone(xann.find("CATEGORY"),
lambda x: x.text))
tup.append(
funornone(xann.find("TAG"), lambda x: x.text))
if (anntask == "FE"):
if xann.find("ENTITY") is not None:
eid = xann.find("ENTITY").get("entity_id")
tup.append(eid)
tup.append(annset[eid].find("TAG").text if \
annset[eid].find("TAG") is not None else "NONE")
elif xann.find("PHRASE") is not None:
tup.append(
xann.find("PHRASE").get("phrase_id"))
else:
sys.stderr("Expected ENTITY or PHRASE at "\
+annfile+"; "+ET.tostring(xann))
continue
elif (anntask == "SSA"):
if xann.find("PREDICATE") is not None:
tup.append(
xann.find("PREDICATE").get(
"predicate_id"))
else:
sys.stderr.write(annfile+": Don't know how to process "\
+anntask+"\n")
continue
except:
print(annfile)
print(ET.tostring(xann))
raise
try:
outfile.write("\t".join(map(str, tup)) + "\n")
except UnicodeDecodeError:
sys.stderr.write("Warning: Unknown encoding %s:%s-%s\n" % \
(tup[4], tup[2], tup[3]))