def download(self, path):
if self.isJson():
dumpJson(path, self.response.json())
elif self.isJsonp():
data = self.jsonpToJson()
dumpJson(path, data)
else:
print "can't download successfully"
def localGroup(fName):
lines = tools.loadJson(fName)
if lines == -1:
return -1
i = 0
while i < len(lines):
j = i
i = group(lines, i, "PER")
i = group(lines, i, "LOC")
i = group(lines, i, "ORG")
if i == j:
i += 1
tools.dumpJson(lines, fName)
result = []
for line in text.splitlines():
taggedWords = {}
words = line.split(delim)
if len(words) < 2:
for index in header.keys():
taggedWords[header[index]] = ""
else:
for index in header.keys():
taggedWords[header[index]] = words[index]
taggedWords["virtual"] = False
result.append(taggedWords)
return result
if __name__ == "__main__":
if len(sys.argv) < 2:
print "usage : 2json.py fileName"
f = os.path.abspath(sys.argv[1])
exts = f.split(".")
newFile = exts[0] + ".json"
if not os.path.isfile(f):
print "Error " + f + " file does not exists"
sys.exit()
if exts[-1] == "NER":
if exts[-2] == "post":
result = convert(postNERHeader, f, "\t")
else:
sys.exit()
tools.dumpJson(newFile)
def add2Results(partial, store):
for k in partial.keys():
for v in partial[k].keys():
if not store.has_key(k):
store[k] = {}
if not store[k].has_key(v):
store[k][v] = 0
store[k][v] += partial[k][v]
return store
if __name__=="__main__":
amira_dir = settings.amira_dir
results = {}
if len(sys.argv) < 3:
print "usage: getStats.py folder format[json|...]\nFolder should contain files in .t extension"
folder = os.path.abspath(sys.argv[1])
fmt = sys.argv[2]
i = 0
for fName in tools.files(folder, ".*"):
fName = os.path.abspath(fName)
if fmt == 'json':
if i%100 == 0:
print i, " files finished"
partial = parseJson(fName, isNNPVirt, "NER")
i+=1
else:
partial = parsePostNER(fName)
add2Results(partial, results)
statFile = os.path.join(folder, 'stats')
tools.dumpJson(results, statFile)
def tagFile(f, dictionary):
jText = tools.loadJson(f)
if jText == -1:
return -1
jNewText = tag(jText, dictionary)
tools.dumpJson(jNewText, f+".t")
def csv2dict(fName):
result = buildDictionary(csv2dicts(fName), "Arabic")
newFile = fName.split(os.path.extsep)[0]+".json"
tools.dumpJson(result, newFile)
def csv2json(fName):
result = csv2dicts(fName)
newFile = fName.split(os.path.extsep)[0]+".json"
tools.dumpJson(result, newFile)
