def diarize_audio(self, request):
with self.db as db:
#This will lock the DB:
db.check_project(request["projectid"], check_err=False) #DEMIT: check_err?
#Check whether tasks already assigned
if db.project_assigned(request["projectid"]):
raise ConflictError("Tasks have already been assigned")
#Get audiofile path and exists?
row = db.get_project(request["projectid"], ["audiofile"])
if not row["audiofile"]:
raise ConflictError("No audio has been uploaded")
#Set up I/O access and lock the project
inurl = auth.gen_token()
outurl = auth.gen_token()
db.insert_incoming(request["projectid"], url=inurl, servicetype="diarize")
db.insert_outgoing(request["projectid"], url=outurl, audiofile=row["audiofile"])
db.lock_project(request["projectid"], jobid="diarize_audio")
#Make job request:
try:
# TEMPORARILY COMMENTED OUT FOR TESTING WITHOUT SPEECHSERVER:
#if "chunksize" in request:
# chunksize = request["chunksize"]
#else:
# chunksize = self._config["speechservices"]["diarize"]["chunksize"]
#jobreq = {"token" : self._speech.token(), "getaudio": os.path.join(APPSERVER, "projects", outurl),
# "putresult": os.path.join(APPSERVER, "projects", inurl),
# "service" : self._config["speechservices"]["diarize"]["name"],
# "subsystem" : self._config["speechservices"]["diarize"]["subsystem"],
# "chunksize" : chunksize}
if "segmentno" in request:
segmentno = request["segmentno"]
else:
segmentno = self._config["speechservices"]["diarize"]["segmentno"]
jobreq = {"token" : self._speech.token(), "getaudio": os.path.join(APPSERVER, "projects", outurl),
"putresult": os.path.join(APPSERVER, "projects", inurl),
"service" : self._config["speechservices"]["diarize"]["name"],
"subsystem" : self._config["speechservices"]["diarize"]["subsystem"],
"segmentno" : segmentno}
LOG.debug("posting job")
reqstatus = requests.post(os.path.join(SPEECHSERVER, self._config["speechservices"]["diarize"]["url"]), data=json.dumps(jobreq))
reqstatus = reqstatus.json()
#reqstatus = {"jobid": auth.gen_token()} #DEMIT: dummy call for testing!
LOG.debug("Diarize: reqstatus={}".format(reqstatus))
#Check reqstatus from SpeechServ OK?
if not "jobid" in reqstatus:
raise Exception("Diarize request failed, SpeechServ says: {}".format(reqstatus["message"]))
#Update project with jobid from SpeechServ
LOG.info("(username={} projectid={} jobid={}) Diarize request successful".format(username, request["projectid"], reqstatus["jobid"]))
with self.db as db:
db.update_project(request["projectid"], {"jobid": reqstatus["jobid"]})
return "Diarize request successful!"
except:
LOG.debug("(projectid={}) FAIL: Cleaning up DB and unlocking".format(request["projectid"]))
with self.db as db:
db.delete_incoming(request["projectid"])
db.delete_outgoing(request["projectid"])
db.unlock_project(request["projectid"], errstatus="diarize_audio")
raise
def custom_lm(self, request):
""" Add text to build a custom language model
"""
self.authdb.authenticate(request["token"], self._config["role"])
# Bogus projectid
projectid = "clm-{}".format(str(uuid.uuid4()))
# Write text data to temporary file
textfile = os.path.join(self._config["tmpdir"], auth.gen_token())
with open(textfile, 'wb') as f:
f.write(request['file'])
# Add entries in db
inurl = auth.gen_token()
outurl = auth.gen_token()
with self.db as db:
db.insert_incoming(projectid, url=inurl, servicetype="customlm")
db.insert_outgoing(projectid, url=outurl, audiofile=textfile)
# request speech job
try:
jobreq = {
"token": self._speech.token(),
"gettext": os.path.join(APPSERVER, "admin", outurl),
"putresult": os.path.join(APPSERVER, "admin", inurl),
"service": "customlm",
"subsystem": request["subsystem"],
"system_name": request["name"]
}
LOG.debug(
os.path.join(SPEECHSERVER,
self._config["speechservices"]["customlm"]))
reqstatus = requests.post(os.path.join(
SPEECHSERVER, self._config["speechservices"]["customlm"]),
data=json.dumps(jobreq))
reqstatus = reqstatus.json()
LOG.debug("CustomLM: reqstatus={}".format(reqstatus))
#Check reqstatus from SpeechServ OK?
if not "jobid" in reqstatus:
raise Exception(
"CustomLM request failed, SpeechServ says: {}".format(
reqstatus["message"]))
return {"projectid": projectid}
except:
# Cleanup ... things went wrong
with self.db as db:
db.delete_incoming(projectid)
db.delete_outgoing(projectid)
if os.path.exists(textfile):
os.remove(textfile)
raise
def get_token():
data = request.get_json()
username = data.get('username', '*****@*****.**')
password = data.get('password', 'mypwd')
scope = data.get('scope', None)
return gen_token({
'username': username,
'password': password,
'scope': scope
})
def buildmaster(self, request):
"""
Return MS-WORD document from all the files
"""
try:
with self.db as db:
options = db.get_project_text(request["projectid"])
all_text = []
for taskid in sorted(options.keys()):
textfile = options[taskid]
if textfile is None or len(textfile) == 0:
raise NotFoundError("Text file is missing for a task")
if not os.path.exists(textfile):
raise NotFoundError("Cannot find text file")
self._test_read(textfile)
with codecs.open(textfile, "r", "utf-8") as f:
text = f.read()
all_text.append(text)
all_text = u"\n".join(all_text)
LOG.info(all_text)
_html = tempfile.NamedTemporaryFile(delete=False)
with codecs.open(_html.name, "w", "utf-8") as f:
f.write(all_text)
_docx = tempfile.NamedTemporaryFile(delete=False)
_docx.close()
cmd = "pandoc -f html -t docx -o {} {}".format(
_docx.name, _html.name)
ps = subprocess.Popen(cmd.split(),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = ps.communicate()
LOG.info(stdout)
LOG.error(stderr)
outurl = auth.gen_token()
db.insert_outgoing(request["projectid"], outurl, _docx.name,
"-1.0", "-1.0")
os.remove(_html.name)
return {"url": outurl}
except Exception as e:
LOG.error("Get text failed: {}".format(e))
raise
def setUp(self):
self.app = create_app(TestingConfig)
self.client = self.app.test_client
# seed data
self.role = Role('general')
self.role.insert()
self.user = User('Ahmed', 'Hamed', '*****@*****.**',
'ahmedhrayyan', 'secret', self.role.id)
self.user.insert()
self.question = Question(self.user.id, 'Is sal the best QA engine')
self.question.insert()
self.answer = Answer(self.user.id, self.question.id, 'Yes it is')
self.answer.insert()
self.token = gen_token(self.app.config['SECRET_KEY'], self.user)
def register():
data = request.get_json() or {}
required_fields = [
'first_name', 'last_name', 'email', 'username', 'password'
]
# abort if any required field doesnot exist in request body
for field in required_fields:
if field not in data:
abort(400, '%s is required' % field)
first_name = str(data['first_name']).lower().strip()
last_name = str(data['last_name']).lower().strip()
email = str(data['email']).lower().strip()
username = str(data['username']).lower().strip()
password = str(data['password']).lower()
job = str(data['job']).lower().strip() if data.get('job') else None
phone = data.get('phone')
# validating data
if re.match(app.config['EMAIL_PATTERN'], email) is None:
abort(422, 'Email is not valid')
if len(password) < 8:
abort(422, 'Password have to be at least 8 characters in length')
if phone and re.match(app.config['PHONE_PATTERN'], phone) is None:
abort(422, 'Phone is not valid')
if len(username) < 4:
abort(422, 'Username have to be at least 4 characters in length')
default_role = Role.query.filter_by(name="general").one_or_none().id
new_user = User(first_name, last_name, email, username, password,
default_role, job, phone)
try:
new_user.insert()
except IntegrityError:
# Integrity error means a unique value already exist in a different record
if User.query.filter_by(email=email).one_or_none():
msg = 'Email is already in use'
elif User.query.filter_by(username=username).one_or_none():
msg = "Username is already in use"
else:
msg = "Phone is already in use"
abort(422, msg)
return jsonify({
'success': True,
'token': gen_token(SECRET_KEY, new_user),
'data': new_user.format(),
})
def login():
data = request.get_json() or {}
if 'username' not in data or 'password' not in data:
abort(400, 'username and password expected in request body')
username = data['username']
password = data['password']
user = User.query.filter_by(username=username).one_or_none()
if not user or not user.checkpw(str(password)):
abort(422, 'username or password is not correct')
return jsonify({
'success': True,
'token': gen_token(SECRET_KEY, user),
'data': user.format(),
})
return qstr
def valid_pass(p):
return PassValidate().validate(p)
if __name__ == "__main__":
from auth import gen_token
#create_file(1, "yay", "bleh", 1);
"""
f = fetch_file(11)
k = fileid_to_publicid(f["fileid"], f["userid"])
print(f["userid"], f["fileid"])
print(publicid_to_fileid(k))
update_file(11, {"mimeType": "bleh", "custom_attr2": 3})
"""
# qstr = "INSERT INTO authtokens (tokenid,userid,type,permissions,expiration) VALUES("
import datetime
print(sql_update("filedata", ["fileid", "name", "path"], [0, "bleh2", "bleh3"], [sq.int, sq.path, sq.path],
["fileid"], [0], [sq.int]))
print(sql_selectall("users", ["username"], ["joeedh"], [sq.user]))
print(sql_insertinto("authtokens",
["tokenid", "userid", "type", "permissions", "expiration"],
[gen_token("U", 0), 0, 2, 7, datetime.datetime.now()],
[sq.token, sq.int, sq.int, sq.int, sq.datetime]));
def do_GET(self, serv):
elog("fileapi access" + serv.path)
qs = get_qs(serv.path)
if "accessToken" not in qs or ("path" not in qs and "id" not in qs):
serv.send_error(400)
return
tok = qs["accessToken"][0]
userid = do_auth(tok)
if userid == None:
elog("Need user id")
serv.send_error(401)
return
path = qs["path"][0]
if "id" in qs:
fileid = qs["id"][0]
else:
fileid = resolve_path(path)
if fileid == None:
elog("creating new file")
cs = os.path.split(path)
folderid = resolve_path(cs[0])
if folderid == None:
elog("invalid folder " + cs[0])
serv.send_error(401);
return
if len(cs) == 1 or cs[1] == "":
fname = cs[0]
else:
fname = cs[1]
mime = "application/octet-stream"
fileid = create_file(userid, fname, mime, folderid)
meta = fetch_file(fileid);
else:
meta = fetch_file(fileid);
if meta == None:
elog("Invalid file id")
serv.send_error(400)
return
print("\n\nFILE", meta, "\n\n")
if is_folder(meta):
elog("target file is a folder" + meta["name"])
serv.send_error(401)
return
utoken = gen_token("U", userid);
ustatus = UploadStatus()
ustatus.create(utoken, path, userid, fileid, meta["parentid"])
ustatus.commit()
f = open(ustatus.realpath, "w");
f.close();
realpath = ustatus.realpath
cur, con = mysql_connect()
try:
qstr = sql_update("filedata", ["diskpath"], [realpath], [sq.path], ["fileid"], [fileid], [sq.int])
except SQLParamError:
do_param_error("upload start")
serv.send_error(401)
"""
qstr = "UPDATE filedata SET "
qstr += "diskpath=%s"%estr(realpath)
qstr += " WHERE fileid=%d"%fileid
#"""
cur.execute(qstr)
con.commit()
body = json.dumps({"uploadToken" : utoken});
body = bstr(body)
print("\nupload start result:", body, "\n\n\n")
serv.gen_headers("GET", len(body), json_mimetype)
serv.wfile.write(body)
def do_GET(self, serv):
elog("fileapi access" + serv.path)
qs = get_qs(serv.path)
if "accessToken" not in qs or ("path" not in qs and "id" not in qs):
serv.send_error(400)
return
tok = qs["accessToken"][0]
userid = do_auth(tok)
if userid == None:
elog("Need user id")
print("Bad auth")
serv.send_error(401)
return
path = qs["path"][0]
if "id" in qs:
fileid = publicid_to_fileid(qs["id"][0])
else:
fileid = urllib.unquote(path)
meta = File(fileid, userid)
if meta != None:
print("DISKPATH", meta.diskpath)
if meta == None or not os.path.exists(meta.diskpath):
elog("creating new file")
cs = os.path.split(path)
folderid = cs[0]
f = File(folderid, userid)
if not os.path.exists(f.diskpath):
elog("invalid folder " + f.diskpath)
print("invalid folder " + f.diskpath)
serv.send_error(401);
return
if len(cs) == 1 or cs[1] == "":
fname = cs[0]
else:
fname = cs[1]
mime = "application/octet-stream"
#create empty file
f = open(f.diskpath+"/"+fname, "w")
f.close()
meta = File(fileid, userid)
if meta == None:
elog("Invalid file id")
serv.send_error(400)
return
print("\n\nFILE", meta, "\n\n")
if is_folder(meta):
elog("target file is a folder" + meta["name"])
serv.send_error(401)
return
utoken = gen_token("U", userid);
ustatus = UploadStatus()
#ignore fileid/parentid in upload status token
ustatus.create(utoken, path, userid, fileid, -1)
try:
ustatus.commit()
except:
import traceback
elog("USTATUS.COMMIT failed!")
traceback.print_exc()
f = open(ustatus.realpath, "w");
f.close();
realpath = ustatus.realpath
body = json.dumps({"uploadToken" : utoken});
body = bstr(body)
print("\nupload start result:", body, "\n\n\n")
serv.gen_headers("GET", len(body), json_mimetype)
serv.wfile.write(body)
def _speech_job(self, request, project, task):
"""
Submit a speech job
"""
with self.db as db:
db.lock()
#Setup I/O access
inurl = auth.gen_token()
audio_outurl = auth.gen_token()
text_outurl = auth.gen_token()
db.set_jobid(request["projectid"], request["taskid"],
project["year"], "pending")
db.insert_incoming(request["projectid"], request["taskid"], inurl,
request["service"])
db.insert_outgoing(request["projectid"], audio_outurl,
project["audiofile"], task["start"],
task["end"])
db.insert_outgoing(request["projectid"], text_outurl,
task["textfile"], -2.0, -2.0)
#Make job request
#TEMPORARILY COMMENTED OUT FOR TESTING WITHOUT SPEECHSERVER:
#TODO: fix editor reference
jobreq = {
"token": self._speech.token(),
"getaudio": os.path.join(APPSERVER, "editor", audio_outurl),
"gettext": os.path.join(APPSERVER, "editor", text_outurl),
"putresult": os.path.join(APPSERVER, "editor", inurl)
}
jobreq["service"] = request["service"]
jobreq["subsystem"] = request["subsystem"]
LOG.debug(
os.path.join(SPEECHSERVER,
self._config["speechservices"]["API"]["add"]))
LOG.debug("{}".format(jobreq))
reqstatus = requests.post(os.path.join(
SPEECHSERVER, self._config["speechservices"]["API"]["add"]),
data=json.dumps(jobreq))
reqstatus = reqstatus.json()
#reqstatus = {"jobid": auth.gen_token()} #DEMIT: dummy call for testing!
#TODO: handle return status
LOG.debug("{}".format(reqstatus))
#Handle request status
if "jobid" in reqstatus: #no error
with self.db as db:
db.lock()
db.set_jobid(request["projectid"], request["taskid"],
project["year"], reqstatus["jobid"])
LOG.info(
"Speech service request sent for project ID: {}, task ID: {}, job ID: {}"
.format(request["projectid"], request["taskid"],
reqstatus["jobid"]))
return "Request successful!"
#Something went wrong: undo project setup
with self.db as db:
db.lock()
if "message" in reqstatus:
db.set_errstatus(request["projectid"], request["taskid"],
project["year"], reqstatus["message"])
db.delete_incoming_byurl(inurl)
db.delete_outgoing_byurl(audio_outurl)
db.delete_outgoing_byurl(text_outurl)
db.set_jobid(request["projectid"], request["taskid"],
project["year"], None)
LOG.error(
"Speech service request failed for project ID: {}, task ID: {}".
format(request["projectid"], request["taskid"]))
return reqstatus #DEMIT TODO: translate error from speech server!