def modifyAllTreesFromAllUsers(project):
sql = database.SQL(project)
db, cursor = sql.open()
command = "select sentences.rowid, sentences.nr, sentences.textid, trees.userid, trees.rowid from sentences, texts, trees where sentences.textid=texts.rowid and trees.sentenceid = sentences.rowid;"
cursor.execute(command)
answer = cursor.fetchall()
print len(answer), "trees to update"
lastpourc = -1
for nr, (sid, snr, tid, uid, treeid) in enumerate(answer):
pourc = int(float(nr) / len(answer) * 100)
if pourc != lastpourc:
sys.stdout.write("{pourc}%\r".format(pourc=pourc))
sys.stdout.flush()
dic = sql.gettree(None, None, treeid, indb=db, incursor=cursor)
if dic and dic["tree"]:
sentencetree = dic["tree"]
#print "Resgmentation en cours de ", " ".join([node["t"] for i, node in sentencetree.iteritems()]), "id=", treeid, "by user", uid
#newtree=retokenisation.retokenizeTree(sentencetree)
newtree = retokenisation.segmentationChiffres(sentencetree)
#pprint.pprint(newtree)
sql.enterTree(cursor, newtree, sid, uid, tokensChanged=True)
#print "Tree", treeid," successfully updated."
print "Done."
db.commit()
db.close()
def main():
with open(f'{cwd}/config.json') as f:
config = json.load(f)
url = config['landsnet']['URL']
logging_message = 'power'
with Scraper() as scraper_:
soup = scraper_.scrape(url, description=logging_message)
landsnet = Landsnet()
parsed_data = landsnet.parse(soup=soup)
desired_values = landsnet.extract_desired_values(parsed_data)
timestamp = landsnet.get_timestamp(parsed_data)
final_data = landsnet.merge_data(desired_values, timestamp)
sql = database.SQL()
sql.write(table='landsnet', data=final_data, logging_message='power')
def main():
with open(f'{cwd}/config.json') as f:
config = json.load(f)
stations = config["met"]["stations"]
weather = Forecast()
sql = database.SQL()
logging_message = 'weather'
for station_name, station_id in stations.items():
url = weather.get_url(station_id)
with Scraper() as scraper_:
raw_data = scraper_.scrape(url=url, description=logging_message)
formatted_data = weather.parse(station_name, raw_data)
sql.write(table='weather', data=formatted_data,
logging_message=logging_message)
def updateParseResult(projectname, conlldirpath, filepattern="*.trees.conll14", annotatorName="parser", removeToGetDB="-one-word-per-line.conll14_parse"): sql = database.SQL(projectname) db,cursor=sql.open() print "updateTrees:",glob(os.path.join(conlldirpath, filepattern)) for filename in glob(os.path.join(conlldirpath, filepattern)): print "entering",filename sentences=conll.conllFile2trees(filename) dbtextname = os.path.basename(filename)[:-len(removeToGetDB)] textid = sql.enter(cursor, "texts",["textname"],(dbtextname,)) if not textid: print "couldn't find the database named",textid return enterNewAnnotation(sql, db,cursor, sentences, textid, annotatorName=annotatorName)