def on_create(self, docs): """Create corresponding item on file upload.""" for doc in docs: if 'media' not in doc or doc['media'] is None: abort(400, description="No media found") file, content_type, metadata = self.get_file_from_document(doc) inserted = [doc['media']] file_type = content_type.split('/')[0] self._set_metadata(doc) try: doc[ITEM_TYPE] = self.type_av.get(file_type) doc[ITEM_STATE] = CONTENT_STATE.PROGRESS rendition_spec = get_renditions_spec(no_custom_crops=True) with timer('archive:renditions'): renditions = generate_renditions(file, doc['media'], inserted, file_type, content_type, rendition_spec, url_for_media) doc['renditions'] = renditions doc['mimetype'] = content_type set_filemeta(doc, metadata) add_activity('upload', 'uploaded media {{ name }}', 'archive', item=doc, name=doc.get('headline', doc.get('mimetype')), renditions=doc.get('renditions')) except Exception as io: logger.exception(io) for file_id in inserted: delete_file_on_error(doc, file_id) abort(500)
def create(self, docs, **kwargs): with multiprocessing.Lock() as lock: with timer("prepopulate"): self._create(docs) if app.config.get("SUPERDESK_TESTING"): for provider in ["paimg", "aapmm"]: if provider not in allowed_search_providers: register_search_provider(provider, provider) return ["OK"]
def create(self, docs, **kwargs): with multiprocessing.Lock() as lock: with timer('prepopulate'): self._create(docs) if app.config.get('SUPERDESK_TESTING'): for provider in ['paimg', 'aapmm']: if provider not in allowed_search_providers: register_search_provider(provider, provider) return ['OK']
def find(self, query, params=None): if params is None: params = {} size = 25 # int(query.get('size', 25)) page = math.ceil((int(query.get('from', 0)) + 1) / size) try: sort = query.get('sort')[0] except (IndexError, AttributeError, TypeError): sort = {'versioncreated': 'desc'} kwargs = { 'pagenumber': page, 'countperpage': size, 'fields': ','.join(self.FIELDS), 'Sort': get_api_sort(sort), 'format': 'json', 'DateFormat': 'u', } query_components = {} try: query_components['Text'] = query['query']['filtered']['query'][ 'query_string']['query'] except (KeyError, TypeError): pass if params: if params.get('mediaTypes'): selected = [k for k, v in params['mediaTypes'].items() if v] if selected: query_components['MediaType'] = '({})'.format( ' OR '.join(selected)) kwargs['query'] = ' '.join([ '{}:{}'.format(key, val) for key, val in query_components.items() if val ]) if params: for param, op in (('from', '>:'), ('to', '<:')): if params.get(param): kwargs['query'] = '{} MediaDate{}{}'.format( kwargs['query'], op, params[param]).strip() with timer('orange'): resp = self._auth_request(SEARCH_API, **kwargs) data = resp.json() with open('/tmp/resp.json', mode='w') as out: out.write(json.dumps(data, indent=2)) items = self._parse_items(data) return OrangelogicListCursor( items, data['APIResponse']['GlobalInfo']['TotalCount'])
def _login(self): with timer('orange.login'): resp = self._request( AUTH_API, method='POST', Login=self.config.get('username'), Password=self.config.get('password'), format='json', ) tokens[self.config['username']] = resp.json()['APIResponse']['Token']
def _login(self): with timer("orange.login"): resp = self._request( AUTH_API, method="POST", Login=self.config.get("username"), Password=self.config.get("password"), format="json", ) tokens[self.config["username"]] = resp.json()["APIResponse"]["Token"]
def on_create(self, docs): """Create corresponding item on file upload.""" for doc in docs: if 'media' not in doc or doc['media'] is None: abort(400, description="No media found") # check content type of video by python-magic content_type = magic.from_buffer(doc['media'].read(1024), mime=True) doc['media'].seek(0) file_type = content_type.split('/')[0] if file_type == 'video' and app.config.get("VIDEO_SERVER_ENABLE"): if not self.videoEditor.check_video_server(): raise SuperdeskApiError( message="Cannot connect to videoserver", status_code=500) # upload media to video server res, renditions, metadata = self.upload_file_to_video_server( doc) # get thumbnails for timeline bar self.videoEditor.get_timeline_thumbnails(doc.get('media'), 40) else: file, content_type, metadata = self.get_file_from_document(doc) inserted = [doc['media']] # if no_custom_crops is set to False the custom crops are generated automatically on media upload # see (SDESK-4742) rendition_spec = get_renditions_spec( no_custom_crops=app.config.get("NO_CUSTOM_CROPS")) with timer('archive:renditions'): renditions = generate_renditions(file, doc['media'], inserted, file_type, content_type, rendition_spec, url_for_media) try: self._set_metadata(doc) doc[ITEM_TYPE] = self.type_av.get(file_type) doc[ITEM_STATE] = CONTENT_STATE.PROGRESS doc['renditions'] = renditions doc['mimetype'] = content_type set_filemeta(doc, metadata) add_activity('upload', 'uploaded media {{ name }}', 'archive', item=doc, name=doc.get('headline', doc.get('mimetype')), renditions=doc.get('renditions')) except Exception as io: logger.exception(io) for file_id in inserted: delete_file_on_error(doc, file_id) if res: self.videoEditor.delete(res.get('_id')) abort(500)
def on_create(self, docs): """Create corresponding item on file upload.""" for doc in docs: if "media" not in doc or doc["media"] is None: abort(400, description="No media found") # check content type of video by python-magic content_type = app.media._get_mimetype(doc["media"]) doc["media"].seek(0) file_type = content_type.split("/")[0] if file_type == "video" and app.config.get("VIDEO_SERVER_ENABLED"): # upload media to video server res, renditions, metadata = self.upload_file_to_video_server( doc) # get thumbnails for timeline bar self.video_editor.create_timeline_thumbnails( doc.get("media"), 60) else: file, content_type, metadata = self.get_file_from_document(doc) inserted = [doc["media"]] # if no_custom_crops is set to False the custom crops are generated automatically on media upload # see (SDESK-4742) rendition_spec = get_renditions_spec( no_custom_crops=app.config.get("NO_CUSTOM_CROPS")) with timer("archive:renditions"): renditions = generate_renditions(file, doc["media"], inserted, file_type, content_type, rendition_spec, url_for_media) try: self._set_metadata(doc) doc[ITEM_TYPE] = self.type_av.get(file_type) doc[ITEM_STATE] = CONTENT_STATE.PROGRESS doc["renditions"] = renditions doc["mimetype"] = content_type set_filemeta(doc, metadata) add_activity( "upload", "uploaded media {{ name }}", "archive", item=doc, name=doc.get("headline", doc.get("mimetype")), renditions=doc.get("renditions"), ) except Exception as io: logger.exception(io) for file_id in inserted: delete_file_on_error(doc, file_id) if res: self.video_editor.delete(res.get("_id")) abort(500)
def _auth_request(self, api, **kwargs): repeats = 2 while repeats > 0: if not self.token: self._login() try: kwargs['token'] = self.token with timer('orange.request'): return self._request(api, **kwargs) except HTTPError as err: logger.error(err) self._login() # auth error repeats -= 1 if repeats == 0: raise
def get_file_from_document(self, doc): file = doc.get("media_fetched") if file: del doc["media_fetched"] else: content = doc["media"] res = process_file_from_stream(content, content_type=content.mimetype) file_name, content_type, metadata = res logger.debug("Going to save media file with %s " % file_name) content.seek(0) with timer("media:put.original"): doc["media"] = app.media.put(content, filename=file_name, content_type=content_type, metadata=metadata) return content, content_type, decode_metadata(metadata) return file, file.content_type, file.metadata
def _do_request(self, doc): desk_id = request.view_args["desk_id"] agg_type = request.view_args["agg_type"] timer_label = f"{agg_type} overview aggregation {desk_id!r}" if agg_type == "users": with timer(timer_label): doc["_items"] = self._users_aggregation(desk_id) return if agg_type == "stages": collection = "archive" desk_field = "task.desk" key = "stage" field = f"task.{key}" elif agg_type == "assignments": collection = "assignments" desk_field = "assigned_to.desk" key = "state" field = f"assigned_to.{key}" else: raise ValueError(f"Invalid overview aggregation type: {agg_type}") agg_query = { "filter": { "bool": { "must_not": [{ "terms": { ITEM_STATE: [ CONTENT_STATE.PUBLISHED, CONTENT_STATE.SPIKED, CONTENT_STATE.PUBLISHED, CONTENT_STATE.KILLED, CONTENT_STATE.CORRECTED, ] } }] } } } filter_bool = agg_query["filter"]["bool"] if desk_id != "all": filter_bool["must"] = [{"term": {desk_field: desk_id}}] agg_query["aggs"] = {"overview": {"terms": {"field": field}}} filters = doc.get("filters") if filters: should = [] filter_bool.setdefault("must", []).append({"bool": { "should": should }}) for f_name, f_data in filters.items(): for text in f_data: should.append({"match": {f_name: text}}) # with filters we need whole documents, we get them with top_hits agg_query["aggs"]["overview"]["aggs"] = { "top_docs": { "top_hits": { "size": 100 } } } with timer(timer_label): response = app.data.elastic.search(agg_query, collection, params={"size": 0}) doc["_items"] = [{ "count": b["doc_count"], key: b["key"] } for b in response.hits["aggregations"]["overview"]["buckets"]] if filters: for idx, bucket in enumerate( response.hits["aggregations"]["overview"]["buckets"]): docs = doc["_items"][idx]["docs"] = [] for hit_doc in bucket["top_docs"]["hits"]["hits"]: docs.append(hit_doc["_source"])
def test_timer(self, logger): with timer('foo'): time.sleep(0.1) logger.info.assert_called_once_with('%s: %.3fms', 'foo', ANY) self.assertAlmostEqual(0.1 * 1000, logger.info.call_args[0][2], 0)
def test_timer(self, logger): with timer("foo"): time.sleep(0.1) logger.info.assert_called_once_with("%s: %.3fms", "foo", ANY) self.assertEqual(100, int(logger.info.call_args[0][2]))