def edit(self): """ Route: POST /api/tag/edit Edit a property of a tag. Require the parameters `property`, `value` and `tagId` to be set. Writes back the edited tag. """ field = self.get_argument('property') value = self.get_argument('value').lower() tagId = self.get_argument('tagId').lower() usage = self.get_argument('usage', default=False) if usage == 'false': usage = False if field == 'relation' or field == 'home': if value == 'false': value = False elif value == 'true': value = True model.getService('tag').set(tagId, field, value) res = model.getService('tag').getById(tagId) res['name'] = res['name'].title() res['value'] = res['value'].title() if usage: self.__count_usage([res]) self.write(json.dumps(res))
def display(self): """ Route: GET /api/home/display Returns the list of videos to be displayed on the home page. The behavior of this function differs depending on what has been configured by the user. By default, it will select n tags among the most used ones. Then from each tag it will select randomly m videos, and returns the selected ones in an array. If some tags have the property 'home' set to True, only these tags will be used. """ nb_vids = int(self.get_argument('nb_vids', default="1")) nb_tags = int(self.get_argument('nb_tags', default="5")) # get tags with the property home set to True tags = model.getService('tag').getHomeTags(returnList=True) if len(tags) == 0: # get all tags tags = model.getService('tag').getAll() selected = tags if len(selected) > 4: # count the usage of the tags self.__count_usage(tags) # select randomly 4 tags among the most used one selected = self.__select_tags(nb_tags, tags) # now for each tag in selected, select randomly 4 videos vids = [] for tag in selected: vids += [ populateMissingData(v) for v in self.__select_vids(nb_vids, tag) ] model.getService('tag').populate(vids) self.write(json.dumps(vids, default=lambda obj: str(obj)))
def get(self): """ Route: POST /api/tag/get Get all avilable tags, or a single tag given by id if the parameter `tagId` is provided. For each tag returned, the number of related videos will be attached **ONLY IF** the parameter 'usage' is set to True. """ tagId = self.get_argument('id', default=None) usage = self.get_argument('usage', default=False) if usage == 'false': usage = False if tagId is None: tagsG = model.getService('tag').getAll(returnList=False, orderBy={ 'name': 1, 'value': 1 }) tags = [] for tag in tagsG: tag['name'] = tag['name'].title() tag['value'] = tag['value'].title() tags.append(tag) if usage: self.__count_usage(tags) self.write(json.dumps(tags)) else: tag = model.getService('tag').getById(tagId) tag['name'] = tag['name'].title() tag['value'] = tag['value'].title() if usage: self.__count_usage([tag]) self.write(json.dumps(tag))
def update(self): """ Route: POST /api/video/update This will update one of the fields of the video object with the given value. This require the parameters `videoId`, `field` and `value` to be defined Note: to send a null/None value, send the string 'null' which will be converted to the None value. """ videoId = self.get_argument('videoId') field = self.get_argument('field') value = self.get_argument('value') if value == 'null': value = None elif value == 'false': value = False elif value == 'true': value = True else: try: value = int(value) except: pass if field == 'thumbnail' and not isinstance(value, int): logging.error( "Video %s's thumbnail has been set to a non-integer value!" % (value)) model.getService('video').set(videoId, field, value) self.write(json.dumps({"success": True}))
def deletePic(self): """ Route: DELETE /api/album/picture Remove a picture from an album. This will also permanentely delete the picture from the data folder. This requires the parameter 'pictureIdx' to be set. This also requires the parameter 'albumId' to be set. """ albumId = self.get_argument('albumId') pictureIdx = int(self.get_argument('pictureIdx')) album = model.getService('album').getById(albumId) album['fullPath'] = '%s%s' % ( Conf['data']['albums']['rootFolder'], album['fullPath']) logging.warning("Deleting picture %s from album %s" % (pictureIdx, album['name'])) model.getService('album').removePicture(albumId, pictureIdx) try: if albumId != 'random' and albumId != 'starred': os.remove(album['fullPath'] + album['picturesDetails'][pictureIdx]['filename']) else: # the pictures array contains the fullpath # (the physical album does not exist) os.remove(album['picturesDetails'][pictureIdx]['filename']) except Exception as e: logging.error("Unable to remove picture %s from album %s." % (album['name'], album['picturesDetails'][pictureIdx])) raise self.write(json.dumps({'success': True}))
def execStart(self, message): """ Start a factorio instance. If a running instance is already existing, attempt to kill it first via ctrl+c to trigger data saving. There might be data loss when starting an instance while another one is running, use carefully. Requires the messsage to hold the field `_id` denoting which instance to start Write back the data for all instances in database """ global instanceProcess if self.instLogtimeout: IOLoop.current().remove_timeout(self.instLogtimeout) if instanceProcess is not None and not \ instanceProcess.killed.value: raise Exception( "An instance is already running (pid: %d, _id=%s)" % ( instanceProcess.subpid.value, instanceProcess._id)) data = getService('instance').getById(message['_id']) instanceProcess = factorio.Instance( data['port'], data['save'], data['_id']) instanceProcess.start() self.instLogtimeout = IOLoop.current().add_timeout( time.time() + 2, self._logInstance) getService('instance').set(message['_id'], 'status', 'running') self.writeMessage({ 'action': 'start', 'instances': getService('instance').getAll() })
def cleanUp(vid): try: with open(vid['path']) as f: pass except IOError: print ("File `%s' doesn't exist. Deleting document." % vid['path']) model.getService('video').deleteById(vid['_id'])
def delete(self, videoId): """ Route: DELETE /api/video/<videoId> Remove any file related to the given video ID. This include the video itself, the snapshots of this video and the db record. """ video = model.getService('video').getById(videoId) video['snapshotsFolder'] = '%s%s' % ( Conf['data']['videos']['rootFolder'], video['snapshotsFolder']) video['path'] = '%s%s' % (Conf['data']['videos']['rootFolder'], video['path']) if video is None: raise HTTPError(404, "Not Found: %s" % videoId) try: shutil.rmtree(video['snapshotsFolder']) except: pass os.remove(video['path']) model.getService('video').deleteById(videoId) self.write(json.dumps({'success': True}))
def _logInstance(self): if instanceProcess is None: return data = instanceProcess.read() if data is not None: logging.info('[Instance] %s' % data) self.writeMessage({ 'action': 'log', 'message': data }) if not instanceProcess.is_alive(): logging.warning( "Cannot find instance log, process isn't alive anymore.") getService('instance').set(instanceProcess._id, 'status', 'stopped') self.writeMessage({ 'action': 'kill', 'instances': [ getService('instance').getById(instanceProcess._id)] }) self.error("Instance seems to have stopped unexpectedly and " "prematurely.") return self.instLogtimeout = IOLoop.current().add_timeout( time.time() + 2, self._logInstance)
def execSave(self, message): """ Save the instance from the given message data. The field `data` should be defined in the message and hold the following fields: * name: name of the instance * port: selected port for this instance * save: selected save for this instance If `_id` field is given as well, the instance will be updated instead. Note that updating a running instance will have no effect until it is restarted. Returned message will hold the fields: * 'action': 'save' * 'instances': [saved data as a list of a single element for consistency with the `load` action.] """ if '_id' in message['data']: _id = message['data']['_id'] getService('instance').update( message['data']['_id'], name=message['data']['name'], save=message['data']['save'], port=message['data']['port']) else: _id = getService('instance').insert( name=message['data']['name'], save=message['data']['save'], port=message['data']['port']) self.writeMessage({ 'action': 'save', 'instances': [getService('instance').getById(_id)] })
def create(self): """ Route: POST /api/tag/create Create a new tag. Requires the parameters `name` and `value` to be set. The parameter `relation` can be set to use this tag to suggest related videos The parameter `autotag` can be set to enable automatic tagging of imported albums and videos """ name = self.get_argument('name').lower() value = self.get_argument('value').lower() relation = self.get_argument('relation', default=False) autotag = self.get_argument('autotag', default='') home = self.get_argument('home', default=False) if relation == 'false': relation = False elif relation == 'true': relation = True if home == 'false': home = False elif home == 'true': home = True _id = model.getService('tag').insert(name, value, _id=None, relation=relation, autotag=autotag, home=home) inserted = model.getService('tag').getById(_id) inserted['name'] = inserted['name'].title() inserted['value'] = inserted['value'].title() self.write(json.dumps(inserted))
def toWatch(self): """ Route: POST /api/video/towatch This will mark the given video as to be watched. Requires the parameter `videoId` to be defined. """ model.getService('video').set(self.get_argument('videoId'), 'toWatch', True)
def migrate(self): albums = model.getService('album').getAll() toRemove = 'Photos\\' for album in albums: logging.info("Migrating: %s" % album['name']) if album['fullPath'].startswith(toRemove): model.getService('album').set( album['_id'], 'fullPath', album['fullPath'][len(toRemove):]) self.write("OK")
def execDelete(self, message): """ Delete the instance from given message id. The message should contain the `_id` of the instance to delete Write back a message with the field 'action' set to 'delete' and the field '_id' set to the deleted instance id. """ getService('instance').deleteById(message['_id']) self.writeMessage({ 'action': 'delete', '_id': message['_id'] })
def display(self): """ Route: GET /api/album/display Return the list of all albums of the database. If the parameter `albumId` is defined, only this album will be returned, alongside with available face detection annotations. """ albumId = self.get_argument('albumId', default=None) if albumId is None: start_t = time.time() albums = model.getService('album').getAll( returnList=True, orderBy={'creation': -1}, projection={ 'album': 1, 'fullPath': 1, 'picturesDetails': 1, 'cover': 1, 'name': 1, 'display': 1, 'picsNumber': 1, 'starredNumber': 1, 'creation': 1, 'lastDisplay': 1, 'lastStarred': 1, 'averageWidth': 1, 'averageHeight': 1, 'tags': 1 }) logging.info("Retrieved %d albums in %s", len(albums), timeFormat(time.time() - start_t)) if not albums: return self.write(json.dumps([])) index, found = next(((i, x) for i, x in enumerate(albums) if x['fullPath'] == 'random'), (-1, False)) if not found: albums.insert(0, model.getService('album').getById('random')) else: albums.pop(index) albums.insert(0, found) index, found = next(((i, x) for i, x in enumerate(albums) if x['fullPath'] == 'starred'), (-1, False)) if not found: albums.insert(0, model.getService('album').getById('starred')) else: albums.pop(index) albums.insert(0, found) albums = [self.__populatePicturesURLs(a) for a in albums] self.write(json.dumps(albums)) else: album = model.getService('album').getById(albumId) if album is None: raise HTTPError(404, 'Not Found') album = model.getService('album').extendAlbumWithFaces(album) album = self.__populatePicturesURLs(album) self.write(json.dumps(album))
def increment(self): """ Route: POST /api/video/increment This will increment one of the incrementable field of the video. WARNING: undefined behaviour if the requested field is not an integer. This requires the parametrs `videoId` and `field` to be defined """ videoId = self.get_argument('videoId') field = self.get_argument('field') model.getService('video').increment(videoId, field) self.write(json.dumps({"success": True}))
def on_analysis_progress(self, videoId, data, skipped=False): if data.get('finished', False) and not skipped: model.getService('video').set(videoId, 'analysis', data['data']) try: self.write_message(json.dumps(data)) except Exception as e: logging.exception(e) # the socket is probably stale, stop receiving update # until a new connection comes in analyzer = memory.getVal(MEMKEY) if analyzer is not None: analyzer.resubscribe(None)
def __count_usage(self, tags): """ Count the number of videos that holds each tags. The result will be added to each tag as a field named 'usage'. """ for tag in tags: _, count = model.getService('video').find( {'tags': [{ '$value': tag['_id'] }]}, returnCount=True) tag['usage'] = count _, count = model.getService('album').find({'tags': [tag['_id']]}, returnCount=True) tag['usage'] += count
def cover(self): """ Route: POST /api/album/cover This will select a new picture to be displayed as album cover This require the `albumId` and the `pictureIdx` to be defined. `pictureIdx` should be the index of the picture to select """ pictureIdx = int(self.get_argument('pictureIdx')) albumId = self.get_argument('albumId') logging.debug("PictureIDx=%d, albumId=%s" % (pictureIdx, albumId)) model.getService('album').selectCover(albumId, pictureIdx) self.write(json.dumps({"success": True}))
def downloadVideo(self, videoId): logging.info("Will download video with id=%s" % videoId) # todo, add a better way to do this, in its dedicated function def asyncDownload(videoPath): videoPath = videoPath.replace('/', '\\') logging.info("Starting download of: %s" % videoPath) videoName = os.path.basename(videoPath) buf_size = 1024 * 1024 self.set_header('Content-Type', self.videoMimeType[videoName.split('.')[-1].lower()]) self.set_header('Content-Disposition', 'attachment; filename=' + videoName) self.set_header('Content-Length', os.path.getsize(videoPath)) self.set_header('Expires', 0) self.set_header('Accept-Ranges', 'bytes') i = 0 total_len = 0 with open(videoPath, 'rb') as f: while True: i += 1 data = f.read(buf_size) if len(data) < buf_size: logging.error("Read %s instead of %s" % (sizeFormat(len(data)), sizeFormat(buf_size))) total_len += len(data) logging.debug("Chunk: %d [%s]" % (i, sizeFormat(total_len))) if not data: data = f.read(buf_size) if not data: break self.write(data) self.flush() logging.debug("Total size should be %s" % (sizeFormat(total_len))) logging.debug("Size is expected to be: %s" % (sizeFormat(os.path.getsize(videoPath)))) self.finish() # get the path for this video. video = model.getService('video').getById(videoId, ['path', 'name']) video['path'] = '%s%s' % ( Conf['data']['videos']['rootFolder'], video['path']) # check that the file exist try: with open(video['path'], 'r') as f: logging.info("The video %s does exist!" % video['path']); except IOError: logging.error("The video: %s cannot be found." % video['path']) raise HTTPError(404, 'Not Found') # increment 'seen' counter model.getService('video').increment(videoId, 'seen') # perform asynchronous download Thread(target=asyncDownload, name="Downloader-%s" % video['name'], args=[video['path']]).start()
def getSingleVid(self): """ Route: GET /api/video/display Will write back to the client a JSON object containing information related to a single video. This require the body parameter `videoId` defined. """ videoId = self.get_argument('videoId') video = model.getService('video').getById(videoId) video = self.__populateMissingData(video) # update 'display' counter model.getService('video').increment(video['_id'], 'display') model.getService('tag').populate([video]) self.write(json.dumps(video))
def downloadAlbum(self, albumId, picNum): """ Writes back to the client the picture number `picNum` of the album given by id. """ picNumber = int(picNum) album = model.getService('album').getById(albumId) album['fullPath'] = '%s%s' % ( Conf['data']['albums']['rootFolder'], album['fullPath']) if picNumber > len(album['picturesDetails']): logging.error("Album has only %d pictures!" % len(album['picturesDetails'])) raise HTTPError(404, "Not Found") if albumId == 'random' or albumId == 'starred': if picNumber > 0: # the first picture is the icon, full exact path does not need to be reconstructed picPath = Conf['data']['albums']['rootFolder'] + album['picturesDetails'][picNumber]['filename'] # path already included else: picPath = album['picturesDetails'][picNumber]['filename'] else: picPath = album['fullPath'] + album['picturesDetails'][picNumber]['filename'] try: with open(picPath, 'rb') as p: buf = p.read() except: logging.error("The picture: %s cannot be found." % picPath) raise HTTPError(404, 'Not Found') self.set_header('Content-Type', self.picMimeType[picPath.split('.')[-1].lower()]) self.set_header('Content-Length', len(buf)) self.write(buf) self.finish()
def start(self, videoId, force=False, **kwargs): """ Action: analysis Parameters: videoId, force Begin the analyze the video by extracting every other frame and submitting them to google cloud vision. The process will be performed on a separate thread. The process is cached and won't be performed more than once, unless `force` is specified and set to `True`. """ video = model.getService('video').getById( videoId, fields=['snapshotsFolder', 'path', 'analysis', 'name', 'duration']) existing_worker = memory.getVal(MEMKEY) if existing_worker is not None and existing_worker.isAlive(): logging.warn("An analyze is still in progress for video: %s" % video['name']) existing_worker.resubscribe(self.callback) return logging.info("Starting analysis of video: %s", video['name']) analyzer = Analyzer(videoId, video['path'], video['snapshotsFolder'], progressCb=self.callback, force=force, annotator=Conf['data']['videos']['annotator'], videoDuration=video['duration']) analyzer.start() memory.setVal(MEMKEY, analyzer)
def openFolder(self): """ Route: GET /api/video/folder Open the containing folder in windows explorer This require the parameter `videoId` to be defined. The function returns nothing. """ vidId = self.get_argument('videoId') video = model.getService('video').getById(vidId, fields=['path']) video['path'] = '%s%s' % (Conf['data']['videos']['rootFolder'], video['path']) # increment 'seen' counter model.getService('video').increment(vidId, 'seen') subprocess.Popen(r'explorer /select,"%s"' % video['path'])
def asyncThumbGen(data): logging.warning("Starting Thumbnail re-generation!") start_t = time.time() return_code = subprocess.call( '{ffmpegPath} -i "{videoPath}" -f image2 -vf fps=fps={frameRate} -s {width}x{height} "{snapFolder}\\thumb%03d.png"' .format(**data), shell=True) logging.warning( "Thumbnails re-generation complete! Done in %.3fs." % (time.time() - start_t)) try: thumbnails = os.listdir(video['snapshotsFolder']) except Exception as e: logging.warning("Couldn't read thumbnails in folder: %s" % (video['snapshotsFolder'])) thumbnails = [] model.getService('video').set(videoId, 'nbSnapshots', len(thumbnails))
def main(): log.init(2, False, filename="fixAlbums.log", colored=False) for album in genAlbums(): if album['_id'] in ['random', 'starred']: model.getService('album')._collection.remove( {'fullPath': album['_id']}) continue data = [] for idx, pic in enumerate(album['pictures']): obj = { 'filename': pic, 'display': 0, 'starred': idx in album['starred'], 'faces': [] } data.append(obj) model.getService('album').set(album['_id'], 'picturesDetails', data)
def tag(self): """ Route: POST /api/album/tag This will add or remove a tag to/from a album. This require the `tagId` and the `albumId` parameters to be defined To remove the tag instead of adding it, provide a `remove` parameter set to True. """ tagId = self.get_argument('tagId') albumId = self.get_argument('albumId') remove = self.get_argument('remove', default=False) if remove: model.getService('album').removeTag(tagId, albumId=albumId) else: model.getService('album').addTag(albumId, tagId) self.write(json.dumps({"success": True}))
def play(self): """ Route: GET /api/video/play Use VLC to play the video This requires the parameter `videoId` to be defined. """ def asyncPlay(videoPath): videoPath = videoPath.replace('/', os.path.sep) videoPath = videoPath.replace('\\', os.path.sep) if (Conf['server']['playVideosVLC']): subprocess.call('%s "%s"' % (Conf['server']['vlcPath'], videoPath)) elif (Conf['server']['playVideosMPC']): subprocess.call('%s "%s"' % (Conf['server']['mpcPath'], videoPath)) else: try: os.startfile(videoPath) except: logging.error("Unable to open file: %s, \ try to enable the `playVideosVLC` option and fill in the VLC binary path to fix this issue." % videoPath) # get the path for this video. videoId = self.get_argument('videoId') video = model.getService('video').getById(videoId, ['path', 'name']) video['path'] = '%s%s' % (Conf['data']['videos']['rootFolder'], video['path']) # check that the file exist try: with open(video['path'], 'r') as f: logging.debug("The video %s does exist!" % video['path']) except IOError: logging.error("The video: %s cannot be found." % video['path']) raise HTTPError(404, 'Not Found') # increment 'seen' counter model.getService('video').increment(videoId, 'seen') # perform asynchronous playing of the video Thread(target=asyncPlay, name="Player-%s" % video['name'], args=[video['path']]).start() self.write(json.dumps({'success': True}))
def main(): log.init(2, False, filename="bulkAnalyze.log", colored=False) for video in genVids(): if ('analysis' in video and video['analysis'] is not None and '__version__' in video['analysis'] and video['analysis']['__version__'] == Analyzer.__version__): logging.info("Skipping video %s - analysis already completed for version %s.", video['path'], Analyzer.__version__) continue start_t = time.time() analyzer = Analyzer( video['_id'], video['path'], video['snapshotsFolder'], async=False, force=False, annotator='dfl-dlib', videoDuration=video['duration'], autoCleanup=True) data = analyzer.run() logging.info("Analysis completed for video %s in %s.", video['path'], timeFormat(time.time() - start_t)) model.getService('video').set(video['_id'], 'analysis', data)
def genVids(): c = 0 videos = model.getService('video').getAll() progress = tqdm(total=len(videos), desc='[Analyzing video: ') for vid in videos: c += 1 progress.set_description('[Analyzing video %s' % os.path.basename(vid['path'])) progress.update() yield vid logging.info("Processed %d videos." % c)
def cleanup(self, videoId, **kwargs): existing_worker = memory.getVal(MEMKEY) video = model.getService('video').getById(videoId, fields=['snapshotsFolder']) if existing_worker is not None: existing_worker.stop() time.sleep(1) Analyzer.cleanup(video['snapshotsFolder'])
def __count_usage(self, tags): """ Count the number of videos that holds each tags. The result will be added to each tag as a field named 'usage'. """ # todo: optimize this function, make only 2 db call for tag in tags: _, count = model.getService('album').find({'tags': [tag['_id']]}, returnCount=True) tag['video_usage'] = count
def genAlbums(): c = 0 albums = model.getService('album').getAll() progress = tqdm(total=len(albums), desc='[Analyzing album: ') for a in albums: c += 1 progress.set_description('[Analyzing album %s' % os.path.basename(a['fullPath'])) progress.update() yield a logging.info("Processed %d albums." % c)
def star(self): """ Route: POST /api/album/star This will save the given picture as starred. This require the `albumId` and the `pictureIdx` to be defined. `pictureIdx` should be the index of the picture to star in the pictures array of the album. If `remove` is prodided and True, the picture will be unstarred instead. """ pictureIdx = int(self.get_argument('pictureIdx')) albumId = self.get_argument('albumId') remove = self.get_argument('remove', default=False) logging.debug("PictureIDx=%d, albumId=%s, remove=%s" % (pictureIdx, albumId, str(remove))) if remove: model.getService('album').removeStar(albumId, pictureIdx) else: model.getService('album').addStar(albumId, pictureIdx) self.write(json.dumps({"success": True}))
def execLoad(self, message): """ Load the data for the given instance and return them as a list of dicts. The list may be of length 1 if the _id of an instance was given, or of any other possible length if '*' is given as an _id in which case all available instances will be returned. This action requires the field `_id` to be set. The message written back will have the following structure: * 'instances': list of instance documents as returned by the database * 'action': 'load' (string litteral) """ _id = message['_id'] if _id == '*': instances = getService('instance').getAll() else: instances = [getService('instance').getById(_id)] # print instances self.writeMessage({ 'instances': instances, 'action': 'load' })
def execKill(self, message): """ Kill a running factorio instance. Requires the messsage to hold the field `_id` denoting which instance to kill Write back the data for all instances in database """ getService('instance').set(message['_id'], 'status', 'stopped') if instanceProcess is None: self.writeMessage({ 'action': 'kill', 'instances': getService('instance').getAll() }) raise Exception("No running instance found") instanceProcess.kill() if self.instLogtimeout: IOLoop.current().remove_timeout(self.instLogtimeout) self.writeMessage({ 'action': 'kill', 'instances': getService('instance').getAll() })