def assign_tasks(self, request):
"""Assign tasks to editors:
- Create documents associated with speech segments
- Ensure that tasks table is fully completed (i.a. editors assigned)
- Sets permissions appropriately
- Sets project state to `assigned` disallowing revision of task segments
"""
with self.db as db:
#This will lock the DB:
db.check_project(request["projectid"], check_err=False) #DEMIT: check_err?
if db.project_assigned(request["projectid"]):
raise ConflictError("Tasks have already been assigned")
#Fetch tasks and project info
row = db.get_project(request["projectid"], fields=["audiofile"])
tasks = db.get_tasks(request["projectid"])
if not tasks:
raise ConflictError("No tasks found to assign")
#Make sure all required fields are set
undefined = (None, "")
infields = ("taskid", "projectid", "editor", "speaker", "start", "end", "language")
for task in tasks:
if not all(v not in undefined for k, v in task.iteritems() if k in infields):
raise BadRequestError("Not all necessary task fields are defined (use save_project() first)")
#Lock the project
db.lock_project(request["projectid"], jobid="assign_tasks")
try:
#Create files and update fields
textname = "text"
updatefields = ("textfile", "creation", "modified", "commitid")
audiodir = os.path.dirname(row["audiofile"])
textdirs = []
for task in tasks:
textdir = os.path.join(audiodir, str(task["taskid"]).zfill(TASKID_DIR_ZFILL))
os.makedirs(textdir) #should succeed...
textdirs.append(textdir)
repo.init(textdir)
task["textfile"] = os.path.join(textdir, textname)
open(task["textfile"], "wb").close()
task["commitid"], task["creation"] = repo.commit(textdir, textname, "task assigned")
task["modified"] = task["creation"]
#Update fields and unlock project
with self.db as db:
db.update_tasks(request["projectid"], tasks, fields=updatefields)
db.update_project(request["projectid"], data={"collator" : request["collator"], "assigned": "Y"})
db.unlock_project(request["projectid"])
return 'Project tasks assigned!'
except:
LOG.debug("(projectid={}) FAIL: Cleaning up filesystem and unlocking".format(request["projectid"]))
#Cleanup filesystem
for textdir in textdirs:
shutil.rmtree(textdir, ignore_errors=True)
#Unlock the project and set errstatus
with self.db as db:
db.unlock_project(request["projectid"], errstatus="assign_tasks")
raise
def save_text(self, request):
"""
Save the provided text to task
"""
try:
#TODO: check if you can save to this task -- check lock
with self.db as db:
db.check_project_task(request["projectid"],
request["taskid"],
check_err=True)
with self.db as db:
year = db.get_project(request["projectid"],
fields=["year"])["year"]
textfile = db.get_task_field(request["projectid"],
request["taskid"],
year,
fields=["textfile"])["textfile"]
db.set_jobid(request["projectid"], request["taskid"], year,
'save_text')
if textfile is None or len(textfile) == 0:
raise NotFoundError("This task has no text file")
if not os.path.exists(textfile):
raise NotFoundError("Cannot find text file for this task")
self._test_read(textfile)
textdir = os.path.dirname(textfile)
try: # Handle repo errors
#TODO: check errors, not sure how we recover from here
repo.check(textdir)
with codecs.open(textfile, "w", "utf-8") as f:
f.write(request["text"])
commitid, modified = repo.commit(
textdir, os.path.basename(textfile), "Changes saved")
db.save_text(request["projectid"], request["taskid"], year,
commitid, modified)
db.set_jobid(request["projectid"], request["taskid"], year,
None)
except Exception as e:
raise
return "Text Saved!"
except Exception as e: # TODO: exception within exception
LOG.error("Save text failed: {}".format(e))
with self.db as db:
year = db.get_project(request["projectid"],
fields=["year"])["year"]
db.set_errstatus(request["projectid"], request["taskid"], year,
"{}".format(e))
db.set_jobid(request["projectid"], request["taskid"], year,
None)
raise
def save_file(self, event=None):
repo = self.tree.repo
def build_index(n):
files = {}
todo = collections.deque([n])
while todo:
n = todo.popleft()
if n.key:
path = n.get_path()
files[path] = n.key
todo.extend(n.children)
return files
new = build_index(self.tree.root)
if new != self.tree.old_index:
for name, digest in new.items():
if name not in self.tree.old_index:
print('add', digest, name)
repo.add_name(digest, name)
for name, digest in self.tree.old_index.items():
if name not in new:
print('delete', digest, name)
repo.remove_name(digest, name)
repo.commit()
self.tree.old_index = new
def _append_to_textfile(self, projectid, taskid, year, textfile, text):
"""
Append error message textfile
so user knowns what is going on
"""
try: # Handle repo errors
with self.db as db:
out = "<p><font color='red'> {} <font></p>".format(text)
repo.check(os.path.dirname(textfile))
with codecs.open(textfile, "a", "utf-8") as f:
f.write(out)
commitid, modified = repo.commit(os.path.dirname(textfile),
os.path.basename(textfile),
"Changes saved")
db.save_text(projectid, taskid, year, commitid, modified)
db.set_jobid(projectid, taskid, year, None)
db.set_errstatus(projectid, taskid, year, None)
LOG.info(
"Appending error message to textfile, project ID: {}, Task ID: {}"
.format(projectid, taskid))
except Exception as e:
raise
def test():
"""Informal tests...
"""
import os, shutil, tempfile
import repo
### 1
try:
repo.check("/non_existent")
except repo.RepoError as e:
assert str(e) == "Path"
print("TEST_1 SUCCESS:", "Invalid path caught!")
tempd = tempfile.mkdtemp(prefix="repo_")
### 2
try:
repo.check(tempd)
except repo.RepoError as e:
assert str(e) == "Format"
print("TEST_2 SUCCESS:", "Not a repo caught!")
#CREATE REPO
workingdir = os.path.join(tempd, "test")
repo.init(workingdir)
### 3
try:
repo.check(workingdir)
except repo.RepoError as e:
assert str(e) == "Empty"
print("TEST_3 SUCCESS:", "Empty repo caught!")
#CREATE FILE AND COMMIT
fbasename = "a"
filename = os.path.join(workingdir, fbasename)
with open(filename, "w") as wfh:
wfh.write("first edits\n")
cid1 = repo.commit(workingdir, fbasename, "commit 1")
print("\tCommited file to repo, commit id:", cid1)
### 4
try:
repo.check(workingdir, cid1 + "not_correct_id")
except repo.RepoError as e:
assert str(e) == "Inconsistent"
print("TEST_4 SUCCESS:", "Wrong commit ID caught!")
#CREATE PARTIAL COMMIT
repo.check(workingdir, cid1)
with open(filename, "w") as wfh:
wfh.write("unsuccessful edits\n")
r = git.Repo(workingdir)
r.index.add([filename])
### 5
try:
repo.check(workingdir, cid1)
except repo.RepoError as e:
assert str(e) == "Uncommitted"
print("TEST_5 SUCCESS:", "Partial commit process caught!")
#
repo.rollback(workingdir)
repo.check(workingdir, cid1)
with open(filename, "w") as wfh:
wfh.write("successful edits\n")
cid2 = repo.commit(workingdir, fbasename, "commit 2")
print("\tCommited file update to repo, commit id:", cid2)
### 6
repo.check(workingdir, cid2)
cid3 = repo.revert(workingdir, fbasename, cid1)
print("\tReverting file to first commit, new commit id:", cid3)
if repo.check(workingdir,
cid3) and open(filename).read() == "first edits\n":
print("TEST_6 SUCCESSFUL:",
"Successfully reverted file to previous version")
#
shutil.rmtree(tempd)
def _incoming_base(self, data, projectid, taskid, service_name):
"""
Basic incoming data processor
Redirect based on service type
Save result to text file
"""
try:
LOG.debug(
"Speech processing incoming {} (Project ID: {}, Task ID: {})".
format(service_name, projectid, taskid))
with self.db as db:
year = db.get_project(projectid, fields=["year"])["year"]
if not year:
raise ConflictError(
"(projectid={}) Project no longer exists".format(
projectid))
jobid = db.get_task_field(projectid,
taskid,
year,
fields=["jobid"])["jobid"]
if not jobid:
LOG.warning(
"No job expected (Project ID: {}, Task ID: {})".format(
projectid, taskid))
#raise ConflictError("No job expected (Project ID: {}, Task ID: {})".format(projectid, taskid))
textfile = db.get_task_field(projectid,
taskid,
year,
fields=["textfile"])["textfile"]
if textfile is None:
raise NotFoundError("This task has no text file")
self._test_read(textfile)
if "ERROR" in data:
if len(data["ERROR"]) != 0:
#self._append_to_textfile(projectid, taskid, year, textfile, "ERROR: Speech job fail! {}".format(data["ERROR"]))
raise Exception(
"Speech job failed: (Project ID: {}, Task ID: {})".
format(projectid, taskid))
if "CTM" not in data:
#self._append_to_textfile(projectid, taskid, year, textfile, "ERROR: Speech job fail! NO CTM from output!!")
raise Exception(
"Speech service failed, please try manual method: (Project ID: {}, Task ID: {})"
.format(projectid, taskid))
with self.db as db:
repo.check(os.path.dirname(textfile))
with codecs.open(textfile, "w", "utf-8") as f:
f.write(data["CTM"])
commitid, modified = repo.commit(os.path.dirname(textfile),
os.path.basename(textfile),
"Changes saved")
db.save_text(projectid, taskid, year, commitid, modified)
db.set_jobid(projectid, taskid, year, None)
db.set_errstatus(projectid, taskid, year, None)
LOG.info(
"Speech processing result received successfully for project ID: {}, Task ID: {}"
.format(projectid, taskid))
except Exception as e:
LOG.error("Speech processing failure: {}".format(e))
with self.db as db:
if "ERROR" in data:
if len(data["ERROR"]) != 0:
LOG.error("Speech processing failure: {}".format(
data["ERROR"]))
db.set_errstatus(projectid, taskid, year,
data["ERROR"])
elif data["ERROR"] is None:
db.set_errstatus(projectid, taskid, year,
"Requested Speech Service Error!")
else:
db.set_errstatus(projectid, taskid, year,
"{}".format(e))
else:
db.set_errstatus(projectid, taskid, year, "{}".format(e))
db.set_jobid(projectid, taskid, year, None)