def __call__(self, args):
video = session.query(Video).filter(Video.slug == args.slug)
if not video.count():
print "Video {0} does not exist!".format(args.slug)
return
video = video.one()
query = session.query(Path)
query = query.join(Job)
query = query.join(Segment)
query = query.filter(Segment.video == video)
numpaths = query.count()
if numpaths and not args.force:
print ("Video has {0} paths. Use --force to delete."
.format(numpaths))
return
for segment in video.segments:
for job in segment.jobs:
if job.published and not job.completed:
hitid = job.disable()
print "Disabled {0}".format(hitid)
session.delete(video)
session.commit()
print "Deleted video and associated data."
def __call__(self, args):
query = session.query(HIT)
query = query.filter(HIT.useful == True)
if args.hit:
query = query.filter(HIT.hitid == args.id)
else:
worker = session.query(Worker).get(args.id)
if not worker:
print "Worker \"{0}\" not found".format(args.id)
return
if not args.no_block:
worker.block("HIT was invalid.")
print "Blocked worker \"{0}\"".format(args.id)
session.add(worker)
query = query.filter(HIT.workerid == args.id)
for hit in query:
replacement = hit.invalidate()
session.add(hit)
print "Invalidated {0}".format(hit.hitid)
if replacement:
session.add(replacement)
if not args.no_publish:
session.commit()
replacement.publish()
session.add(replacement)
print "Respawned with {0}".format(replacement.hitid)
session.commit()
def __call__(self, args):
videos = session.query(Video)
if args.training:
videos = videos.filter(Video.isfortraining == True)
else:
videos = videos.filter(Video.isfortraining == False)
if args.worker:
videos = videos.join(Segment)
videos = videos.join(Job)
videos = videos.filter(Job.workerid == args.worker)
elif args.published:
videos = videos.join(Segment)
videos = videos.join(Job)
videos = videos.filter(Job.published == True)
elif args.completed:
videos = videos.join(Segment)
videos = videos.join(Job)
videos = videos.filter(Job.completed == True)
if args.count:
print videos.count()
else:
for video in videos.distinct():
print "{0:<25}".format(video.slug),
if args.stats:
print "{0:>3}/{1:<8}".format(video.numcompleted, video.numjobs),
print "${0:<15.2f}".format(video.cost),
print ""
def __call__(self, args):
jobs = session.query(Job)
jobs = jobs.join(Segment).join(Video)
if args.id:
jobs = jobs.filter(Video.slug == args.id)
if args.frame is not None:
jobs = jobs.filter(Segment.start <= args.frame)
jobs = jobs.filter(Segment.stop >= args.frame)
if args.hitid:
jobs = jobs.filter(Job.hitid == args.hitid)
if args.workerid:
jobs = jobs.filter(Job.workerid == args.workerid)
jobs = jobs.filter(CITIS.models.HIT.useful == True)
if jobs.count() > 0:
for job in jobs:
if args.ids:
if job.published:
print job.hitid,
if job.completed:
print job.assignmentid,
print job.workerid,
print ""
else:
print "(not published)"
else:
print job.offlineurl(config.localhost + "ann.html")
else:
print "No jobs matching this criteria."
def markcomplete(hitid, assignmentid, workerid):
"""
Marks a job as complete. Usually this is called right before the
MTurk form is submitted.
"""
hit = session.query(models.HIT).filter(models.HIT.hitid == hitid).one()
hit.markcompleted(workerid, assignmentid)
session.add(hit)
session.commit()
def getuserid(uname, pwd):
user = session.query(Worker)
user = user.filter(Worker.username == uname)
user = user.filter(Worker.password == pwd)
#user = user.filter(Worker.type == 2)
if user.count() == 1:
return user.one().id
else:
return -1
def savedonationstatus(hitid, donation):
"""
Saves the donation statistics
"""
hit = session.query(models.HIT).filter(models.HIT.hitid == hitid).one()
hit.opt2donate = float(donation)
hit.opt2donate = min(max(hit.opt2donate, 0), 1)
session.add(hit)
session.commit()
def savejob(id, tracks):
job = session.query(Job).get(id)
for path in job.paths:
session.delete(path)
session.commit()
for path in readpaths(tracks):
job.paths.append(path)
session.add(job)
session.commit()
def getboxesforjob(id):
job = session.query(Job).get(id)
result = []
for path in job.paths:
attrs = [(x.attributeid, x.frame, x.value) for x in path.attributes]
result.append({
"label": path.labelid,
"boxes": [tuple(x) for x in path.getboxes()],
"attributes": attrs
})
return result
def respawnjob(id):
job = session.query(Job).get(id)
replacement = job.markastraining()
job.worker.verified = True
session.add(job)
session.add(replacement)
session.commit()
replacement.publish()
session.add(replacement)
session.commit()
def readpaths(tracks):
paths = []
logger.debug("Reading {0} total tracks".format(len(tracks)))
for label, track, attributes in tracks:
path = Path()
path.label = session.query(Label).get(label)
logger.debug("Received a {0} track".format(path.label.text))
visible = False
for frame, userbox in track.items():
box = Box(path=path)
box.xtl = max(int(userbox[0]), 0)
box.ytl = max(int(userbox[1]), 0)
box.xbr = max(int(userbox[2]), 0)
box.ybr = max(int(userbox[3]), 0)
box.occluded = int(userbox[4])
box.outside = int(userbox[5])
box.frame = int(frame)
if not box.outside:
visible = True
logger.debug("Received box {0}".format(str(box.getbox())))
if not visible:
logger.warning("Received empty path! Skipping")
continue
for attributeid, timeline in attributes.items():
attribute = session.query(Attribute).get(attributeid)
for frame, value in timeline.items():
aa = AttributeAnnotation()
aa.attribute = attribute
aa.frame = frame
aa.value = value
path.attributes.append(aa)
paths.append(path)
return paths
def saveeventlog(hitid, events):
"""
Records the event log to database.
"""
hit = session.query(models.HIT).filter(models.HIT.hitid == hitid).one()
for timestamp, domain, message in events:
timestamp = datetime.fromtimestamp(int(timestamp) / 1000)
event = EventLog(hit=hit,
domain=domain,
message=message,
timestamp=timestamp)
session.add(event)
session.commit()
def serverstatus(self, session):
available = session.query(HIT).filter(HIT.ready == True).count()
published = session.query(HIT).filter(HIT.published == True).count()
completed = session.query(HIT).filter(HIT.completed == True).count()
compensated = session.query(HIT).filter(
HIT.compensated == True).count()
remaining = published - completed
print "Status:"
print " Available: {0}".format(available)
print " Published: {0}".format(published)
print " Completed: {0}".format(completed)
print " Compensated: {0}".format(compensated)
print " Remaining: {0}".format(remaining)
print ""
if remaining > 0:
print "Server is ONLINE and accepting work!"
else:
if compensated == completed:
print "Server is offline."
else:
print "Server is offline, but some workers are not compensated."
def getjobstats(hitid, workerid):
"""
Returns the worker status as a dictionary for the server.
"""
status = {}
hit = session.query(models.HIT)
hit = hit.filter(models.HIT.hitid == hitid)
hit = hit.one()
status["reward"] = hit.group.cost
status["donationcode"] = hit.group.donation
bonuses = [x.description() for x in hit.group.schedules]
bonuses = [x for x in bonuses if x]
status["bonuses"] = bonuses
worker = session.query(models.Worker)
worker = worker.filter(models.Worker.id == workerid)
try:
worker = worker.one()
except:
status["newuser"] = True
status["numaccepted"] = 0
status["numrejected"] = 0
status["numsubmitted"] = 0
status["verified"] = False
status["blocked"] = False
else:
status["newuser"] = False
status["numaccepted"] = worker.numacceptances
status["numrejected"] = worker.numrejections
status["numsubmitted"] = worker.numsubmitted
status["verified"] = worker.verified
status["blocked"] = worker.blocked
return status
def savejobstats(hitid, timeaccepted, timecompleted, environ):
"""
Saves statistics for a job.
"""
hit = session.query(models.HIT).filter(models.HIT.hitid == hitid).one()
hit.timeaccepted = datetime.fromtimestamp(int(timeaccepted) / 1000)
hit.timecompleted = datetime.fromtimestamp(int(timecompleted) / 1000)
hit.timeonserver = datetime.now()
hit.ipaddress = environ.get("HTTP_X_FORWARDED_FOR", None)
hit.ipaddress = environ.get("REMOTE_ADDR", hit.ipaddress)
session.add(hit)
session.commit()
def __call__(self, args):
session = database.connect()
acceptkeys = []
rejectkeys = []
warnkeys = []
for f in args.accept:
acceptkeys.extend(line.strip() for line in open(f))
for f in args.reject:
rejectkeys.extend(line.strip() for line in open(f))
for f in args.warn:
warnkeys.extend(line.strip() for line in open(f))
try:
query = session.query(HIT)
query = query.filter(HIT.completed == True)
query = query.filter(HIT.compensated == False)
query = query.join(HITGroup)
query = query.filter(HITGroup.offline == False)
if args.limit:
query = query.limit(args.limit)
for hit in query:
if not hit.check():
print "WARNING: {0} failed payment check, ignoring".format(
hit.hitid)
continue
try:
self.process(hit, acceptkeys, rejectkeys, warnkeys,
args.validated, args.default)
if hit.compensated:
if hit.accepted:
print "Accepted HIT {0}".format(hit.hitid)
else:
print "Rejected HIT {0}".format(hit.hitid)
session.add(hit)
except CommunicationError as e:
hit.compensated = True
session.add(hit)
print "Error with HIT {0}: {1}".format(hit.hitid, e)
finally:
session.commit()
session.close()
def getdata(self, args):
response = []
video = session.query(Video).filter(Video.slug == args.slug)
if video.count() == 0:
print "Video {0} does not exist!".format(args.slug)
raise SystemExit()
video = video.one()
if args.merge:
for boxes, paths in merge.merge(video.segments,
threshold = args.merge_threshold):
workers = list(set(x.job.workerid for x in paths))
tracklet = DumpCommand.Tracklet(paths[0].label.text,
paths, boxes, workers)
response.append(tracklet)
else:
for segment in video.segments:
for job in segment.jobs:
if not job.useful:
continue
worker = job.workerid
for path in job.paths:
tracklet = DumpCommand.Tracklet(path.label.text,
[path],
path.getboxes(),
[worker])
response.append(tracklet)
if args.worker:
workers = set(args.worker)
response = [x for x in response if set(x.workers) & workers]
interpolated = []
for track in response:
path = vision.track.interpolation.LinearFill(track.boxes)
tracklet = DumpCommand.Tracklet(track.label, track.paths,
path, track.workers)
interpolated.append(tracklet)
response = interpolated
for tracklet in response:
tracklet.bind()
return video, response
def verify(self, session):
passed = True
print "Testing access to Amazon Mechanical Turk...",
try:
balance = api.server.balance
except Exception as e:
print "ERROR!", e
passed = False
else:
print "OK"
print "Testing access to database server...",
try:
count = session.query(HIT).count()
except Exception as e:
print "ERROR!", e
passed = False
print "OK"
print "Testing access to web server...",
try:
da = urllib2.urlopen("{0}/CITIS/verify.html".format(
config.localhost))
da = da.read().strip()
if da == "1":
print "OK"
else:
print "ERROR!",
print "GOT RESPONSE, BUT INVALID"
print da
passed = False
except Exception as e:
print "ERROR!", e
passed = False
print ""
if passed:
print "All tests passed!"
else:
print "One or more tests FAILED!"
def __call__(self, args):
session = database.connect()
try:
query = session.query(HIT)
query = query.join(HITGroup)
query = query.filter(HITGroup.offline == args.offline)
query = query.filter(HIT.ready == True)
if args.disable:
if args.offline:
print "Cannot disable offline HITs."
return
query = query.filter(HIT.published == True)
query = query.filter(HIT.completed == False)
if args.limit > 0:
query = query.limit(args.limit)
for hit in query:
try:
hitid = hit.disable()
print "Disabled {0}".format(hitid)
except Exception as e:
print "Unable to disable HIT {0}!".format(hit.hitid)
print e
session.add(hit)
else:
query = query.filter(HIT.published == False)
if args.limit > 0:
query = query.limit(args.limit)
for hit in query:
if args.offline:
print hit.offlineurl(config.localhost + "ann.html")
else:
hit.publish()
print "Published {0}".format(hit.hitid)
session.add(hit)
session.commit()
finally:
session.commit()
session.close()
def getjob(id, verified):
job = session.query(Job).get(id)
logger.debug("Found job {0}".format(job.id))
if int(verified) and job.segment.video.trainwith:
# swap segment with the training segment
training = True
segment = job.segment.video.trainwith.segments[0]
logger.debug("Swapping actual segment with training segment")
else:
training = False
segment = job.segment
video = segment.video
labels = dict((l.id, l.text) for l in video.labels)
attributes = {}
for label in video.labels:
attributes[label.id] = dict((a.id, a.text) for a in label.attributes)
logger.debug("Giving user frames {0} to {1} of {2}".format(
video.slug, segment.start, segment.stop))
return {
"start": segment.start,
"stop": segment.stop,
"slug": video.slug,
"width": video.width,
"height": video.height,
"skip": video.skip,
"perobject": video.perobjectbonus,
"completion": video.completionbonus,
"blowradius": video.blowradius,
"jobid": job.id,
"training": int(training),
"labels": labels,
"attributes": attributes
}
#coding=utf-8
from CITIS.database import session
from models import *
jobs = session.query(Job)
jobs = jobs.join(Segment).join(Video)
f = open('public/annotator_generate_list.html', 'w')
f.write('''
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport"content="width = device-width, initial-scale = 1">
<title>标注人员任务</title>
<link rel="stylesheet" href="bootstrap.min.css">
<link rel="stylesheet" type="text/css" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/css/bootstrap.min.css">
<script src="jquery-3.2.0.min.js"></script>
<script type="text/javascript" src="jquery-ui.min.js"></script>
<script type="text/javascript" src="jquery.tmpl.min.js"></script>
<script src="js\bootstrap.min.js"></script>
<style type="text/css">
li{ list-style:none }
li.demo_1{
width:240px;
padding:8px;
margin:0px;
}
li.demo_2{
def __call__(self, args):
hits = session.query(HIT).filter(HIT.donatedamount > 0)
for hit in hits:
print hit.workerid, hit.timeonserver, hit.donatedamount
def __call__(self, args):
if args.load:
for data in pickle.load(open(args.load)):
worker = Worker.lookup(data[0])
worker.numsubmitted = data[1]
worker.numacceptances = data[2]
worker.numrejections = data[3]
worker.blocked = data[4]
worker.donatedamount = data[5]
worker.bonusamount = data[6]
worker.verified = data[7]
print "Loaded {0}".format(worker.id)
session.add(worker)
session.commit()
elif args.dump:
data = []
for worker in session.query(Worker):
data.append((worker.id, worker.numsubmitted,
worker.numacceptances, worker.numrejections,
worker.blocked, worker.donatedamount,
worker.bonusamount, worker.verified))
print "Dumped {0}".format(worker.id)
pickle.dump(data, open(args.dump, "w"))
elif args.block:
worker = Worker.lookup(args.block)
worker.block("Poor quality work.")
session.add(worker)
session.commit()
print "Blocked {0}".format(args.block)
elif args.unblock:
worker = Worker.lookup(args.unblock)
worker.unblock("Continue working.")
session.add(worker)
session.commit()
print "Unblocked {0}".format(args.unblock)
elif args.search:
query = session.query(Worker)
query = query.filter(Worker.id.like(args.search + "%"))
if query.count():
print "Matches:"
for worker in query:
print worker.id
else:
print "No matches."
elif args.summary:
query = session.query(Worker)
query = query.filter(Worker.id == args.summary)
if query.count():
worker = query.one()
print "Submitted: {0}".format(worker.numsubmitted)
print "Accepted: {0}".format(worker.numacceptances)
print "Rejected: {0}".format(worker.numrejections)
print "Bonuses: {0}".format(worker.bonusamount)
print "Donated: {0}".format(worker.donatedamount)
print "Verified: {0}".format(worker.verified)
print "Blocked: {0}".format(worker.blocked)
if args.location:
print "Locations: {0}".format(", ".join(
set(x.country for x in worker.locations)))
else:
print "No matches."
else:
workers = session.query(Worker)
workers = workers.order_by(Worker.numacceptances)
for worker in workers:
extra = ""
if worker.blocked:
extra = "BLOCKED"
if args.location:
locs = set(x.country for x in worker.locations)
if locs:
locs = ", ".join(locs)
extra += " " + locs
extra = extra.strip()
data = (worker.id, worker.numsubmitted, worker.numacceptances,
worker.numrejections, extra)
print "{0:<15} {1:>5} jobs {2:>5} acc {3:>5} rej {4}".format(
*data)
def validatejob(id, tracks):
job = session.query(Job).get(id)
paths = readpaths(tracks)
return job.trainingjob.validator(paths, job.trainingjob.paths)
def __call__(self, args):
try:
os.makedirs(args.directory)
except:
pass
since = None
if args.since:
since = parsedatetime.Calendar().parse(args.since)
since = time.mktime(since[0])
since = datetime.datetime.fromtimestamp(since)
if args.labels:
font = ImageFont.truetype("arial.ttf", 14)
else:
font = None
workers = session.query(CITIS.models.Worker)
for worker in workers:
print "Sampling worker {0}".format(worker.id)
jobs = session.query(Job)
jobs = jobs.filter(Job.worker == worker)
jobs = jobs.join(Segment)
jobs = jobs.join(Video)
jobs = jobs.filter(Video.isfortraining == False)
if since:
jobs = jobs.filter(CITIS.models.HIT.timeonserver >= since)
jobs = jobs.order_by(sqlalchemy.func.rand())
jobs = jobs.limit(args.number)
for job in jobs:
print "Visualizing HIT {0}".format(job.hitid)
paths = [x.getboxes(interpolate = True,
bind = True,
label = True) for x in job.paths]
if args.frames > job.segment.stop - job.segment.start:
frames = range(job.segment.start, job.segment.stop + 1)
else:
frames = random.sample(xrange(job.segment.start,
job.segment.stop + 1),
args.frames)
size = math.sqrt(len(frames))
video = job.segment.video
bannersize = (video.width * int(math.floor(size)),
video.height * int(math.ceil(size)))
image = Image.new(video[0].mode, bannersize)
size = int(math.floor(size))
offset = (0, 0)
horcount = 0
paths = vision.visualize.highlight_paths(video, paths,
font = font)
for frame, framenum in paths:
if framenum in frames:
image.paste(frame, offset)
horcount += 1
if horcount >= size:
offset = (0, offset[1] + video.height)
horcount = 0
else:
offset = (offset[0] + video.width, offset[1])
image.save("{0}/{1}-{2}.jpg".format(args.directory,
worker.id,
job.hitid))
def __call__(self, args, group):
print "Checking integrity..."
# read first frame to get sizes
path = Video.getframepath(0, args.location)
try:
im = Image.open(path)
except IOError:
print "Cannot read {0}".format(path)
return
width, height = im.size
print "Searching for last frame..."
# search for last frame
toplevel = max(int(x)
for x in os.listdir(args.location))
secondlevel = max(int(x)
for x in os.listdir("{0}/{1}".format(args.location, toplevel)))
maxframes = max(int(os.path.splitext(x)[0])
for x in os.listdir("{0}/{1}/{2}"
.format(args.location, toplevel, secondlevel))) + 1
print "Found {0} frames.".format(maxframes)
# can we read the last frame?
path = Video.getframepath(maxframes - 1, args.location)
try:
im = Image.open(path)
except IOError:
print "Cannot read {0}".format(path)
return
# check last frame sizes
if im.size[0] != width and im.size[1] != height:
print "First frame dimensions differs from last frame"
return
if session.query(Video).filter(Video.slug == args.slug).count():
print "Video {0} already exists!".format(args.slug)
return
if args.train_with:
if args.for_training:
print "A training video cannot require training"
return
print "Looking for training video..."
trainer = session.query(Video)
trainer = trainer.filter(Video.slug == args.train_with)
if not trainer.count():
print ("Training video {0} does not exist!"
.format(args.train_with))
return
trainer = trainer.one()
else:
trainer = None
# create video
video = Video(slug = args.slug,
location = os.path.realpath(args.location),
width = width,
height = height,
totalframes = maxframes,
skip = args.skip,
perobjectbonus = args.per_object_bonus,
completionbonus = args.completion_bonus,
trainwith = trainer,
isfortraining = args.for_training,
blowradius = args.blow_radius)
if args.for_training:
video.trainvalidator = qa.tolerable(args.for_training_overlap,
args.for_training_tolerance,
args.for_training_mistakes)
print "Training validator is {0}".format(video.trainvalidator)
session.add(video)
print "Binding labels and attributes..."
# create labels and attributes
labelcache = {}
attributecache = {}
lastlabel = None
for labeltext in args.labels:
if labeltext[0] == "~":
if lastlabel is None:
print "Cannot assign an attribute without a label!"
return
labeltext = labeltext[1:]
attribute = Attribute(text = labeltext)
session.add(attribute)
lastlabel.attributes.append(attribute)
attributecache[labeltext] = attribute
else:
label = Label(text = labeltext)
session.add(label)
video.labels.append(label)
labelcache[labeltext] = label
lastlabel = label
print "Creating symbolic link..."
symlink = "public/frames/{0}".format(video.slug)
try:
os.remove(symlink)
except:
pass
os.symlink(video.location, symlink)
print "Creating segments..."
# create shots and jobs
if args.for_training:
segment = Segment(video = video)
if args.for_training_start:
segment.start = args.for_training_start
if segment.start < 0:
segment.start = 0
else:
segment.start = 0
if args.for_training_stop:
segment.stop = args.for_training_stop
if segment.stop > video.totalframes - 1:
segment.stop = video.totalframes - 1
else:
segment.stop = video.totalframes - 1
job = Job(segment = segment, group = group, ready = False)
session.add(segment)
session.add(job)
elif args.use_frames:
with open(args.use_frames) as useframes:
for line in useframes:
ustart, ustop = line.split()
ustart, ustop = int(ustart), int(ustop)
validlength = float(ustop - ustart)
numsegments = math.ceil(validlength / args.length)
segmentlength = math.ceil(validlength / numsegments)
for start in range(ustart, ustop, int(segmentlength)):
stop = min(start + segmentlength + args.overlap + 1,
ustop)
segment = Segment(start = start,
stop = stop,
video = video)
job = Job(segment = segment, group = group)
session.add(segment)
session.add(job)
else:
startframe = args.start_frame
stopframe = args.stop_frame
if not stopframe:
stopframe = video.totalframes - 1
for start in range(startframe, stopframe, args.length):
stop = min(start + args.length + args.overlap + 1,
stopframe)
segment = Segment(start = start,
stop = stop,
video = video)
job = Job(segment = segment, group = group)
session.add(segment)
session.add(job)
if args.per_object_bonus:
group.schedules.append(
PerObjectBonus(amount = args.per_object_bonus))
if args.completion_bonus:
group.schedules.append(
CompletionBonus(amount = args.completion_bonus))
session.add(group)
if args.for_training and args.for_training_data:
print ("Loading training ground truth annotations from {0}"
.format(args.for_training_data))
with open(args.for_training_data, "r") as file:
pathcache = {}
for line in file:
(id, xtl, ytl, xbr, ybr,
frame, outside, occluded, generated,
label) = line.split(" ")
if int(generated):
continue
if id not in pathcache:
print "Imported new path {0}".format(id)
label = labelcache[label.strip()[1:-1]]
pathcache[id] = Path(job = job, label = label)
box = Box(path = pathcache[id])
box.xtl = int(xtl)
box.ytl = int(ytl)
box.xbr = int(xbr)
box.ybr = int(ybr)
box.frame = int(frame)
box.outside = int(outside)
box.occluded = int(occluded)
pathcache[id].boxes.append(box)
session.commit()
if args.for_training:
if args.for_training and args.for_training_data:
print "Video and ground truth loaded."
else:
print "Video loaded and ready for ground truth:"
print ""
print "\t{0}".format(job.offlineurl(config.localhost))
print ""
print "Visit this URL to provide training with ground truth."
else:
print "Video loaded and ready for publication."
