def __call__(self, args): video = session.query(Video).filter(Video.slug == args.slug) if not video.count(): print "Video {0} does not exist!".format(args.slug) return video = video.one() query = session.query(Path) query = query.join(Job) query = query.join(Segment) query = query.filter(Segment.video == video) numpaths = query.count() if numpaths and not args.force: print ("Video has {0} paths. Use --force to delete." .format(numpaths)) return for segment in video.segments: for job in segment.jobs: if job.published and not job.completed: hitid = job.disable() print "Disabled {0}".format(hitid) session.delete(video) session.commit() print "Deleted video and associated data."
def __call__(self, args): query = session.query(HIT) query = query.filter(HIT.useful == True) if args.hit: query = query.filter(HIT.hitid == args.id) else: worker = session.query(Worker).get(args.id) if not worker: print "Worker \"{0}\" not found".format(args.id) return if not args.no_block: worker.block("HIT was invalid.") print "Blocked worker \"{0}\"".format(args.id) session.add(worker) query = query.filter(HIT.workerid == args.id) for hit in query: replacement = hit.invalidate() session.add(hit) print "Invalidated {0}".format(hit.hitid) if replacement: session.add(replacement) if not args.no_publish: session.commit() replacement.publish() session.add(replacement) print "Respawned with {0}".format(replacement.hitid) session.commit()
def __call__(self, args): videos = session.query(Video) if args.training: videos = videos.filter(Video.isfortraining == True) else: videos = videos.filter(Video.isfortraining == False) if args.worker: videos = videos.join(Segment) videos = videos.join(Job) videos = videos.filter(Job.workerid == args.worker) elif args.published: videos = videos.join(Segment) videos = videos.join(Job) videos = videos.filter(Job.published == True) elif args.completed: videos = videos.join(Segment) videos = videos.join(Job) videos = videos.filter(Job.completed == True) if args.count: print videos.count() else: for video in videos.distinct(): print "{0:<25}".format(video.slug), if args.stats: print "{0:>3}/{1:<8}".format(video.numcompleted, video.numjobs), print "${0:<15.2f}".format(video.cost), print ""
def __call__(self, args): jobs = session.query(Job) jobs = jobs.join(Segment).join(Video) if args.id: jobs = jobs.filter(Video.slug == args.id) if args.frame is not None: jobs = jobs.filter(Segment.start <= args.frame) jobs = jobs.filter(Segment.stop >= args.frame) if args.hitid: jobs = jobs.filter(Job.hitid == args.hitid) if args.workerid: jobs = jobs.filter(Job.workerid == args.workerid) jobs = jobs.filter(CITIS.models.HIT.useful == True) if jobs.count() > 0: for job in jobs: if args.ids: if job.published: print job.hitid, if job.completed: print job.assignmentid, print job.workerid, print "" else: print "(not published)" else: print job.offlineurl(config.localhost + "ann.html") else: print "No jobs matching this criteria."
def markcomplete(hitid, assignmentid, workerid): """ Marks a job as complete. Usually this is called right before the MTurk form is submitted. """ hit = session.query(models.HIT).filter(models.HIT.hitid == hitid).one() hit.markcompleted(workerid, assignmentid) session.add(hit) session.commit()
def getuserid(uname, pwd): user = session.query(Worker) user = user.filter(Worker.username == uname) user = user.filter(Worker.password == pwd) #user = user.filter(Worker.type == 2) if user.count() == 1: return user.one().id else: return -1
def savedonationstatus(hitid, donation): """ Saves the donation statistics """ hit = session.query(models.HIT).filter(models.HIT.hitid == hitid).one() hit.opt2donate = float(donation) hit.opt2donate = min(max(hit.opt2donate, 0), 1) session.add(hit) session.commit()
def savejob(id, tracks): job = session.query(Job).get(id) for path in job.paths: session.delete(path) session.commit() for path in readpaths(tracks): job.paths.append(path) session.add(job) session.commit()
def getboxesforjob(id): job = session.query(Job).get(id) result = [] for path in job.paths: attrs = [(x.attributeid, x.frame, x.value) for x in path.attributes] result.append({ "label": path.labelid, "boxes": [tuple(x) for x in path.getboxes()], "attributes": attrs }) return result
def respawnjob(id): job = session.query(Job).get(id) replacement = job.markastraining() job.worker.verified = True session.add(job) session.add(replacement) session.commit() replacement.publish() session.add(replacement) session.commit()
def readpaths(tracks): paths = [] logger.debug("Reading {0} total tracks".format(len(tracks))) for label, track, attributes in tracks: path = Path() path.label = session.query(Label).get(label) logger.debug("Received a {0} track".format(path.label.text)) visible = False for frame, userbox in track.items(): box = Box(path=path) box.xtl = max(int(userbox[0]), 0) box.ytl = max(int(userbox[1]), 0) box.xbr = max(int(userbox[2]), 0) box.ybr = max(int(userbox[3]), 0) box.occluded = int(userbox[4]) box.outside = int(userbox[5]) box.frame = int(frame) if not box.outside: visible = True logger.debug("Received box {0}".format(str(box.getbox()))) if not visible: logger.warning("Received empty path! Skipping") continue for attributeid, timeline in attributes.items(): attribute = session.query(Attribute).get(attributeid) for frame, value in timeline.items(): aa = AttributeAnnotation() aa.attribute = attribute aa.frame = frame aa.value = value path.attributes.append(aa) paths.append(path) return paths
def saveeventlog(hitid, events): """ Records the event log to database. """ hit = session.query(models.HIT).filter(models.HIT.hitid == hitid).one() for timestamp, domain, message in events: timestamp = datetime.fromtimestamp(int(timestamp) / 1000) event = EventLog(hit=hit, domain=domain, message=message, timestamp=timestamp) session.add(event) session.commit()
def serverstatus(self, session): available = session.query(HIT).filter(HIT.ready == True).count() published = session.query(HIT).filter(HIT.published == True).count() completed = session.query(HIT).filter(HIT.completed == True).count() compensated = session.query(HIT).filter( HIT.compensated == True).count() remaining = published - completed print "Status:" print " Available: {0}".format(available) print " Published: {0}".format(published) print " Completed: {0}".format(completed) print " Compensated: {0}".format(compensated) print " Remaining: {0}".format(remaining) print "" if remaining > 0: print "Server is ONLINE and accepting work!" else: if compensated == completed: print "Server is offline." else: print "Server is offline, but some workers are not compensated."
def getjobstats(hitid, workerid): """ Returns the worker status as a dictionary for the server. """ status = {} hit = session.query(models.HIT) hit = hit.filter(models.HIT.hitid == hitid) hit = hit.one() status["reward"] = hit.group.cost status["donationcode"] = hit.group.donation bonuses = [x.description() for x in hit.group.schedules] bonuses = [x for x in bonuses if x] status["bonuses"] = bonuses worker = session.query(models.Worker) worker = worker.filter(models.Worker.id == workerid) try: worker = worker.one() except: status["newuser"] = True status["numaccepted"] = 0 status["numrejected"] = 0 status["numsubmitted"] = 0 status["verified"] = False status["blocked"] = False else: status["newuser"] = False status["numaccepted"] = worker.numacceptances status["numrejected"] = worker.numrejections status["numsubmitted"] = worker.numsubmitted status["verified"] = worker.verified status["blocked"] = worker.blocked return status
def savejobstats(hitid, timeaccepted, timecompleted, environ): """ Saves statistics for a job. """ hit = session.query(models.HIT).filter(models.HIT.hitid == hitid).one() hit.timeaccepted = datetime.fromtimestamp(int(timeaccepted) / 1000) hit.timecompleted = datetime.fromtimestamp(int(timecompleted) / 1000) hit.timeonserver = datetime.now() hit.ipaddress = environ.get("HTTP_X_FORWARDED_FOR", None) hit.ipaddress = environ.get("REMOTE_ADDR", hit.ipaddress) session.add(hit) session.commit()
def __call__(self, args): session = database.connect() acceptkeys = [] rejectkeys = [] warnkeys = [] for f in args.accept: acceptkeys.extend(line.strip() for line in open(f)) for f in args.reject: rejectkeys.extend(line.strip() for line in open(f)) for f in args.warn: warnkeys.extend(line.strip() for line in open(f)) try: query = session.query(HIT) query = query.filter(HIT.completed == True) query = query.filter(HIT.compensated == False) query = query.join(HITGroup) query = query.filter(HITGroup.offline == False) if args.limit: query = query.limit(args.limit) for hit in query: if not hit.check(): print "WARNING: {0} failed payment check, ignoring".format( hit.hitid) continue try: self.process(hit, acceptkeys, rejectkeys, warnkeys, args.validated, args.default) if hit.compensated: if hit.accepted: print "Accepted HIT {0}".format(hit.hitid) else: print "Rejected HIT {0}".format(hit.hitid) session.add(hit) except CommunicationError as e: hit.compensated = True session.add(hit) print "Error with HIT {0}: {1}".format(hit.hitid, e) finally: session.commit() session.close()
def getdata(self, args): response = [] video = session.query(Video).filter(Video.slug == args.slug) if video.count() == 0: print "Video {0} does not exist!".format(args.slug) raise SystemExit() video = video.one() if args.merge: for boxes, paths in merge.merge(video.segments, threshold = args.merge_threshold): workers = list(set(x.job.workerid for x in paths)) tracklet = DumpCommand.Tracklet(paths[0].label.text, paths, boxes, workers) response.append(tracklet) else: for segment in video.segments: for job in segment.jobs: if not job.useful: continue worker = job.workerid for path in job.paths: tracklet = DumpCommand.Tracklet(path.label.text, [path], path.getboxes(), [worker]) response.append(tracklet) if args.worker: workers = set(args.worker) response = [x for x in response if set(x.workers) & workers] interpolated = [] for track in response: path = vision.track.interpolation.LinearFill(track.boxes) tracklet = DumpCommand.Tracklet(track.label, track.paths, path, track.workers) interpolated.append(tracklet) response = interpolated for tracklet in response: tracklet.bind() return video, response
def verify(self, session): passed = True print "Testing access to Amazon Mechanical Turk...", try: balance = api.server.balance except Exception as e: print "ERROR!", e passed = False else: print "OK" print "Testing access to database server...", try: count = session.query(HIT).count() except Exception as e: print "ERROR!", e passed = False print "OK" print "Testing access to web server...", try: da = urllib2.urlopen("{0}/CITIS/verify.html".format( config.localhost)) da = da.read().strip() if da == "1": print "OK" else: print "ERROR!", print "GOT RESPONSE, BUT INVALID" print da passed = False except Exception as e: print "ERROR!", e passed = False print "" if passed: print "All tests passed!" else: print "One or more tests FAILED!"
def __call__(self, args): session = database.connect() try: query = session.query(HIT) query = query.join(HITGroup) query = query.filter(HITGroup.offline == args.offline) query = query.filter(HIT.ready == True) if args.disable: if args.offline: print "Cannot disable offline HITs." return query = query.filter(HIT.published == True) query = query.filter(HIT.completed == False) if args.limit > 0: query = query.limit(args.limit) for hit in query: try: hitid = hit.disable() print "Disabled {0}".format(hitid) except Exception as e: print "Unable to disable HIT {0}!".format(hit.hitid) print e session.add(hit) else: query = query.filter(HIT.published == False) if args.limit > 0: query = query.limit(args.limit) for hit in query: if args.offline: print hit.offlineurl(config.localhost + "ann.html") else: hit.publish() print "Published {0}".format(hit.hitid) session.add(hit) session.commit() finally: session.commit() session.close()
def getjob(id, verified): job = session.query(Job).get(id) logger.debug("Found job {0}".format(job.id)) if int(verified) and job.segment.video.trainwith: # swap segment with the training segment training = True segment = job.segment.video.trainwith.segments[0] logger.debug("Swapping actual segment with training segment") else: training = False segment = job.segment video = segment.video labels = dict((l.id, l.text) for l in video.labels) attributes = {} for label in video.labels: attributes[label.id] = dict((a.id, a.text) for a in label.attributes) logger.debug("Giving user frames {0} to {1} of {2}".format( video.slug, segment.start, segment.stop)) return { "start": segment.start, "stop": segment.stop, "slug": video.slug, "width": video.width, "height": video.height, "skip": video.skip, "perobject": video.perobjectbonus, "completion": video.completionbonus, "blowradius": video.blowradius, "jobid": job.id, "training": int(training), "labels": labels, "attributes": attributes }
#coding=utf-8 from CITIS.database import session from models import * jobs = session.query(Job) jobs = jobs.join(Segment).join(Video) f = open('public/annotator_generate_list.html', 'w') f.write(''' <!DOCTYPE html> <html> <head> <meta charset="utf-8"> <meta http-equiv="X-UA-Compatible" content="IE=edge"> <meta name="viewport"content="width = device-width, initial-scale = 1"> <title>标注人员任务</title> <link rel="stylesheet" href="bootstrap.min.css"> <link rel="stylesheet" type="text/css" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/css/bootstrap.min.css"> <script src="jquery-3.2.0.min.js"></script> <script type="text/javascript" src="jquery-ui.min.js"></script> <script type="text/javascript" src="jquery.tmpl.min.js"></script> <script src="js\bootstrap.min.js"></script> <style type="text/css"> li{ list-style:none } li.demo_1{ width:240px; padding:8px; margin:0px; } li.demo_2{
def __call__(self, args): hits = session.query(HIT).filter(HIT.donatedamount > 0) for hit in hits: print hit.workerid, hit.timeonserver, hit.donatedamount
def __call__(self, args): if args.load: for data in pickle.load(open(args.load)): worker = Worker.lookup(data[0]) worker.numsubmitted = data[1] worker.numacceptances = data[2] worker.numrejections = data[3] worker.blocked = data[4] worker.donatedamount = data[5] worker.bonusamount = data[6] worker.verified = data[7] print "Loaded {0}".format(worker.id) session.add(worker) session.commit() elif args.dump: data = [] for worker in session.query(Worker): data.append((worker.id, worker.numsubmitted, worker.numacceptances, worker.numrejections, worker.blocked, worker.donatedamount, worker.bonusamount, worker.verified)) print "Dumped {0}".format(worker.id) pickle.dump(data, open(args.dump, "w")) elif args.block: worker = Worker.lookup(args.block) worker.block("Poor quality work.") session.add(worker) session.commit() print "Blocked {0}".format(args.block) elif args.unblock: worker = Worker.lookup(args.unblock) worker.unblock("Continue working.") session.add(worker) session.commit() print "Unblocked {0}".format(args.unblock) elif args.search: query = session.query(Worker) query = query.filter(Worker.id.like(args.search + "%")) if query.count(): print "Matches:" for worker in query: print worker.id else: print "No matches." elif args.summary: query = session.query(Worker) query = query.filter(Worker.id == args.summary) if query.count(): worker = query.one() print "Submitted: {0}".format(worker.numsubmitted) print "Accepted: {0}".format(worker.numacceptances) print "Rejected: {0}".format(worker.numrejections) print "Bonuses: {0}".format(worker.bonusamount) print "Donated: {0}".format(worker.donatedamount) print "Verified: {0}".format(worker.verified) print "Blocked: {0}".format(worker.blocked) if args.location: print "Locations: {0}".format(", ".join( set(x.country for x in worker.locations))) else: print "No matches." else: workers = session.query(Worker) workers = workers.order_by(Worker.numacceptances) for worker in workers: extra = "" if worker.blocked: extra = "BLOCKED" if args.location: locs = set(x.country for x in worker.locations) if locs: locs = ", ".join(locs) extra += " " + locs extra = extra.strip() data = (worker.id, worker.numsubmitted, worker.numacceptances, worker.numrejections, extra) print "{0:<15} {1:>5} jobs {2:>5} acc {3:>5} rej {4}".format( *data)
def validatejob(id, tracks): job = session.query(Job).get(id) paths = readpaths(tracks) return job.trainingjob.validator(paths, job.trainingjob.paths)
def __call__(self, args): try: os.makedirs(args.directory) except: pass since = None if args.since: since = parsedatetime.Calendar().parse(args.since) since = time.mktime(since[0]) since = datetime.datetime.fromtimestamp(since) if args.labels: font = ImageFont.truetype("arial.ttf", 14) else: font = None workers = session.query(CITIS.models.Worker) for worker in workers: print "Sampling worker {0}".format(worker.id) jobs = session.query(Job) jobs = jobs.filter(Job.worker == worker) jobs = jobs.join(Segment) jobs = jobs.join(Video) jobs = jobs.filter(Video.isfortraining == False) if since: jobs = jobs.filter(CITIS.models.HIT.timeonserver >= since) jobs = jobs.order_by(sqlalchemy.func.rand()) jobs = jobs.limit(args.number) for job in jobs: print "Visualizing HIT {0}".format(job.hitid) paths = [x.getboxes(interpolate = True, bind = True, label = True) for x in job.paths] if args.frames > job.segment.stop - job.segment.start: frames = range(job.segment.start, job.segment.stop + 1) else: frames = random.sample(xrange(job.segment.start, job.segment.stop + 1), args.frames) size = math.sqrt(len(frames)) video = job.segment.video bannersize = (video.width * int(math.floor(size)), video.height * int(math.ceil(size))) image = Image.new(video[0].mode, bannersize) size = int(math.floor(size)) offset = (0, 0) horcount = 0 paths = vision.visualize.highlight_paths(video, paths, font = font) for frame, framenum in paths: if framenum in frames: image.paste(frame, offset) horcount += 1 if horcount >= size: offset = (0, offset[1] + video.height) horcount = 0 else: offset = (offset[0] + video.width, offset[1]) image.save("{0}/{1}-{2}.jpg".format(args.directory, worker.id, job.hitid))
def __call__(self, args, group): print "Checking integrity..." # read first frame to get sizes path = Video.getframepath(0, args.location) try: im = Image.open(path) except IOError: print "Cannot read {0}".format(path) return width, height = im.size print "Searching for last frame..." # search for last frame toplevel = max(int(x) for x in os.listdir(args.location)) secondlevel = max(int(x) for x in os.listdir("{0}/{1}".format(args.location, toplevel))) maxframes = max(int(os.path.splitext(x)[0]) for x in os.listdir("{0}/{1}/{2}" .format(args.location, toplevel, secondlevel))) + 1 print "Found {0} frames.".format(maxframes) # can we read the last frame? path = Video.getframepath(maxframes - 1, args.location) try: im = Image.open(path) except IOError: print "Cannot read {0}".format(path) return # check last frame sizes if im.size[0] != width and im.size[1] != height: print "First frame dimensions differs from last frame" return if session.query(Video).filter(Video.slug == args.slug).count(): print "Video {0} already exists!".format(args.slug) return if args.train_with: if args.for_training: print "A training video cannot require training" return print "Looking for training video..." trainer = session.query(Video) trainer = trainer.filter(Video.slug == args.train_with) if not trainer.count(): print ("Training video {0} does not exist!" .format(args.train_with)) return trainer = trainer.one() else: trainer = None # create video video = Video(slug = args.slug, location = os.path.realpath(args.location), width = width, height = height, totalframes = maxframes, skip = args.skip, perobjectbonus = args.per_object_bonus, completionbonus = args.completion_bonus, trainwith = trainer, isfortraining = args.for_training, blowradius = args.blow_radius) if args.for_training: video.trainvalidator = qa.tolerable(args.for_training_overlap, args.for_training_tolerance, args.for_training_mistakes) print "Training validator is {0}".format(video.trainvalidator) session.add(video) print "Binding labels and attributes..." # create labels and attributes labelcache = {} attributecache = {} lastlabel = None for labeltext in args.labels: if labeltext[0] == "~": if lastlabel is None: print "Cannot assign an attribute without a label!" return labeltext = labeltext[1:] attribute = Attribute(text = labeltext) session.add(attribute) lastlabel.attributes.append(attribute) attributecache[labeltext] = attribute else: label = Label(text = labeltext) session.add(label) video.labels.append(label) labelcache[labeltext] = label lastlabel = label print "Creating symbolic link..." symlink = "public/frames/{0}".format(video.slug) try: os.remove(symlink) except: pass os.symlink(video.location, symlink) print "Creating segments..." # create shots and jobs if args.for_training: segment = Segment(video = video) if args.for_training_start: segment.start = args.for_training_start if segment.start < 0: segment.start = 0 else: segment.start = 0 if args.for_training_stop: segment.stop = args.for_training_stop if segment.stop > video.totalframes - 1: segment.stop = video.totalframes - 1 else: segment.stop = video.totalframes - 1 job = Job(segment = segment, group = group, ready = False) session.add(segment) session.add(job) elif args.use_frames: with open(args.use_frames) as useframes: for line in useframes: ustart, ustop = line.split() ustart, ustop = int(ustart), int(ustop) validlength = float(ustop - ustart) numsegments = math.ceil(validlength / args.length) segmentlength = math.ceil(validlength / numsegments) for start in range(ustart, ustop, int(segmentlength)): stop = min(start + segmentlength + args.overlap + 1, ustop) segment = Segment(start = start, stop = stop, video = video) job = Job(segment = segment, group = group) session.add(segment) session.add(job) else: startframe = args.start_frame stopframe = args.stop_frame if not stopframe: stopframe = video.totalframes - 1 for start in range(startframe, stopframe, args.length): stop = min(start + args.length + args.overlap + 1, stopframe) segment = Segment(start = start, stop = stop, video = video) job = Job(segment = segment, group = group) session.add(segment) session.add(job) if args.per_object_bonus: group.schedules.append( PerObjectBonus(amount = args.per_object_bonus)) if args.completion_bonus: group.schedules.append( CompletionBonus(amount = args.completion_bonus)) session.add(group) if args.for_training and args.for_training_data: print ("Loading training ground truth annotations from {0}" .format(args.for_training_data)) with open(args.for_training_data, "r") as file: pathcache = {} for line in file: (id, xtl, ytl, xbr, ybr, frame, outside, occluded, generated, label) = line.split(" ") if int(generated): continue if id not in pathcache: print "Imported new path {0}".format(id) label = labelcache[label.strip()[1:-1]] pathcache[id] = Path(job = job, label = label) box = Box(path = pathcache[id]) box.xtl = int(xtl) box.ytl = int(ytl) box.xbr = int(xbr) box.ybr = int(ybr) box.frame = int(frame) box.outside = int(outside) box.occluded = int(occluded) pathcache[id].boxes.append(box) session.commit() if args.for_training: if args.for_training and args.for_training_data: print "Video and ground truth loaded." else: print "Video loaded and ready for ground truth:" print "" print "\t{0}".format(job.offlineurl(config.localhost)) print "" print "Visit this URL to provide training with ground truth." else: print "Video loaded and ready for publication."