def _handle_internal(self, params): response = {'submissions': [{'id': id, 'status': 'pending'} for id in params.ids]} tracks = lookup_submission_status(self.conn, params.ids) for submission in response['submissions']: id = submission['id'] track_gid = tracks.get(id) if track_gid is not None: submission['status'] = 'imported' submission['result'] = {'id': track_gid} return response
def _handle_internal(self, params): response = { 'submissions': [{ 'id': id, 'status': 'pending' } for id in params.ids] } tracks = lookup_submission_status(self.conn, params.ids) for submission in response['submissions']: id = submission['id'] track_gid = tracks.get(id) if track_gid is not None: submission['status'] = 'imported' submission['result'] = {'id': track_gid} return response
def _handle_internal(self, params): # type: (APIHandlerParams) -> Dict[str, Any] assert isinstance(params, SubmissionStatusHandlerParams) response = { 'submissions': [{ 'id': id, 'status': 'pending' } for id in params.ids] } tracks = lookup_submission_status( self.ctx.db.get_ingest_db(read_only=True), self.ctx.db.get_fingerprint_db(read_only=True), params.ids) for submission in response['submissions']: id = submission['id'] track_gid = tracks.get(id) if track_gid is not None: submission['status'] = 'imported' submission['result'] = {'id': track_gid} return response
def _handle_internal(self, params): # type: (APIHandlerParams) -> Dict[str, Any] assert isinstance(params, SubmitHandlerParams) response = {'submissions': []} # type: Dict[str, Any] ids = set() # type: Set[int] fingerprint_db = self.ctx.db.get_fingerprint_db() ingest_db = self.ctx.db.get_ingest_db() for p in params.submissions: mbids = p['mbids'] or [None] for mbid in mbids: values = { 'mbid': mbid or None, 'puid': p['puid'] or None, 'bitrate': p['bitrate'] or None, 'fingerprint': p['fingerprint'], 'length': p['duration'], 'format': p['format'] or None, 'account_id': params.account_id, 'application_id': params.application_id, 'application_version': params.application_version, } meta_values = dict((n, p[n]) for n in self.meta_fields if p[n]) if any(meta_values.values()): values['meta'] = meta_values if p['foreignid']: values['foreignid'] = p['foreignid'] id = insert_submission(ingest_db, values) ids.add(id) submission = {'id': id, 'status': 'pending'} if p['index']: submission['index'] = p['index'] response['submissions'].append(submission) self.ctx.db.session.flush() self.ctx.db.session.commit() self.ctx.redis.publish('channel.submissions', json.dumps(list(ids))) ingest_db = self.ctx.db.get_ingest_db() fingerprint_db = self.ctx.db.get_fingerprint_db() clients_waiting_key = 'submission.waiting' clients_waiting = self.ctx.redis.incr(clients_waiting_key) - 1 try: max_wait = 10 self.ctx.redis.expire(clients_waiting_key, max_wait) remaining = min(max(0, max_wait - 2**clients_waiting), params.wait) logger.debug('starting to wait at %f %d', remaining, clients_waiting) while remaining > 0 and ids: logger.debug('waiting %f seconds', remaining) time.sleep(0.5) # XXX replace with LISTEN/NOTIFY remaining -= 0.5 tracks = lookup_submission_status(ingest_db, fingerprint_db, ids) if not tracks: continue for submission in response['submissions']: submission_id = submission['id'] assert isinstance(submission_id, int) track_gid = tracks.get(submission_id) if track_gid is not None: submission['status'] = 'imported' submission['result'] = {'id': track_gid} ids.remove(submission_id) finally: self.ctx.redis.decr(clients_waiting_key) return response
def _handle_internal(self, params): response = {'submissions': []} ids = set() with self.conn.begin(): source_id = find_or_insert_source(self.conn, params.application_id, params.account_id, params.application_version) format_ids = {} for p in params.submissions: if p['format']: if p['format'] not in format_ids: format_ids[p['format']] = find_or_insert_format( self.conn, p['format']) p['format_id'] = format_ids[p['format']] for p in params.submissions: mbids = p['mbids'] or [None] for mbid in mbids: values = { 'mbid': mbid or None, 'puid': p['puid'] or None, 'bitrate': p['bitrate'] or None, 'fingerprint': p['fingerprint'], 'length': p['duration'], 'format_id': p.get('format_id'), 'source_id': source_id } meta_values = dict( (n, p[n] or None) for n in self.meta_fields) if any(meta_values.itervalues()): values['meta_id'] = insert_meta(self.conn, meta_values) if p['foreignid']: values['foreignid_id'] = find_or_insert_foreignid( self.conn, p['foreignid']) id = insert_submission(self.conn, values) ids.add(id) submission = {'id': id, 'status': 'pending'} if p['index']: submission['index'] = p['index'] response['submissions'].append(submission) if self.redis is not None: self.redis.publish('channel.submissions', json.dumps(list(ids))) clients_waiting_key = 'submission.waiting' clients_waiting = self.redis.incr(clients_waiting_key) - 1 try: max_wait = 10 self.redis.expire(clients_waiting_key, max_wait) tracks = {} remaining = min(max(0, max_wait - 2**clients_waiting), params.wait) logger.debug('starting to wait at %f %d', remaining, clients_waiting) while remaining > 0 and ids: logger.debug('waiting %f seconds', remaining) time.sleep(0.5) # XXX replace with LISTEN/NOTIFY remaining -= 0.5 tracks = lookup_submission_status(self.conn, ids) if not tracks: continue for submission in response['submissions']: id = submission['id'] track_gid = tracks.get(id) if track_gid is not None: submission['status'] = 'imported' submission['result'] = {'id': track_gid} ids.remove(id) finally: self.redis.decr(clients_waiting_key) return response
def _handle_internal(self, params): response = {'submissions': []} ids = set() with self.conn.begin(): source_id = find_or_insert_source(self.conn, params.application_id, params.account_id, params.application_version) format_ids = {} for p in params.submissions: if p['format']: if p['format'] not in format_ids: format_ids[p['format']] = find_or_insert_format(self.conn, p['format']) p['format_id'] = format_ids[p['format']] for p in params.submissions: mbids = p['mbids'] or [None] for mbid in mbids: values = { 'mbid': mbid or None, 'puid': p['puid'] or None, 'bitrate': p['bitrate'] or None, 'fingerprint': p['fingerprint'], 'length': p['duration'], 'format_id': p.get('format_id'), 'source_id': source_id } meta_values = dict((n, p[n] or None) for n in self.meta_fields) if any(meta_values.itervalues()): values['meta_id'] = insert_meta(self.conn, meta_values) if p['foreignid']: values['foreignid_id'] = find_or_insert_foreignid(self.conn, p['foreignid']) id = insert_submission(self.conn, values) ids.add(id) submission = {'id': id, 'status': 'pending'} if p['index']: submission['index'] = p['index'] response['submissions'].append(submission) if self.redis is not None: self.redis.publish('channel.submissions', json.dumps(list(ids))) clients_waiting_key = 'submission.waiting' clients_waiting = self.redis.incr(clients_waiting_key) - 1 try: max_wait = 10 self.redis.expire(clients_waiting_key, max_wait) tracks = {} remaining = min(max(0, max_wait - 2 ** clients_waiting), params.wait) logger.debug('starting to wait at %f %d', remaining, clients_waiting) while remaining > 0 and ids: logger.debug('waiting %f seconds', remaining) time.sleep(0.5) # XXX replace with LISTEN/NOTIFY remaining -= 0.5 tracks = lookup_submission_status(self.conn, ids) if not tracks: continue for submission in response['submissions']: id = submission['id'] track_gid = tracks.get(id) if track_gid is not None: submission['status'] = 'imported' submission['result'] = {'id': track_gid} ids.remove(id) finally: self.redis.decr(clients_waiting_key) return response