def general_search(): try: args = request.args parameters = {'collection': None, 'size': SEARCH_CHUNK_SIZE, 'from_': 0, 'q': None, 'sort': None, "with_persons": False} parameters.update(PERSONS_SEARCH_DEFAULT_PARAMETERS) got_one_of_required_persons_params = False for param in parameters.keys(): if param in args: if param == "with_persons": parameters[param] = args[param].lower() in ["1", "yes", "true"] else: parameters[param] = args[param] if param in PERSONS_SEARCH_REQUIRES_ONE_OF and parameters[param]: got_one_of_required_persons_params = True if parameters["q"] or (parameters["collection"] == "persons" and got_one_of_required_persons_params): try: rv = es_search(**parameters) except Exception as e: logging.exception(e) return humanify({"error": e.message}, 500) for item in rv['hits']['hits']: enrich_item(item['_source'], collection_name=item['_type']) return humanify(rv) else: return humanify({"error": "You must specify a search query"}, 400) except Exception as e: logging.exception("unexpected error") return humanify(({"error": e.message}, 500))
def get_changes(from_date, to_date): rv = set() # Validate the dates dates = {'start': from_date, 'end': to_date} for date in dates: try: dates[date] = datetime.fromtimestamp(float(dates[date])) except ValueError: abort(400, 'Bad timestamp - {}'.format(dates[date])) log_collection = current_app.data_db['migration_log'] query = {'date': {'$gte': dates['start'], '$lte': dates['end']}} projection = {'item_id': 1, '_id': 0} cursor = log_collection.find(query, projection) if not cursor: return humanify([]) else: for doc in cursor: col, _id = doc['item_id'].split('.') # TODO: remove references to the genTreeIndividuals collection - it is irrelevant and not in use if col == 'genTreeIndividuals': continue else: if filter_doc_id(_id, col): rv.add(doc['item_id']) return humanify(list(rv))
def get_items(slugs): if slugs: items_list = slugs.split(',') elif request.is_json: items_list = request.get_json() # Check if there are items from ugc collection and test their access control ugc_items = [] for item in items_list: if item.startswith('ugc'): ugc_items.append(item) user_oid = current_user.is_authenticated and current_user.id items = fetch_items(items_list) if len(items) == 1 and 'error_code' in items[0]: error = items[0] abort(error['error_code'], error['msg']) else: # Cast items to list if type(items) != list: items = [items] # Check that each of the ugc_items is accessible by the logged in user for ugc_item_id in [i[4:] for i in ugc_items]: for item in items: if item['_id'] == ugc_item_id and item.has_key('owner') and item['owner'] != unicode(user_oid): abort(403, 'You are not authorized to access item ugc.{}'.format(str(item['_id']))) return humanify(items)
def get_geocoded_places(): args = request.args filters = SHOW_FILTER.copy() filters['geometry'] = {'$exists': True} filters['Header.En'] = {'$nin': [None, '']} try: filters['geometry.coordinates.1'] = { '$gte': float(args['sw_lat']), '$lte': float(args['ne_lat']) } filters['geometry.coordinates.0'] = { '$gte': float(args['sw_lng']), '$lte': float(args['ne_lng']) } except KeyError: abort(400, 'Please specify a box using sw_lat, sw_lng, ne_lat, ne_lng') except ValueError: abort( 400, 'Please specify a box using floats in sw_lat, sw_lng, ne_lat, ne_lng' ) points = current_app.data_db['places'].find(filters, { 'Header': True, 'Slug': True, 'geometry': True, 'PlaceTypeDesc': True }) ret = humanify(list(points)) return ret
def linkify(): try: collections = ["places", "personalities", "familyNames"] res = {collection: [] for collection in collections} try: if request.method == "POST": html_lower = request.form["html"].lower() else: html_lower = request.args["html"].lower() except Exception as e: logging.exception(e) raise for i, item in enumerate(get_linkify_items(collections, html_lower)): res[item["collection"]].append(item["item"]) return humanify(res) except Exception as e: return humanify({"error": e.message, "traceback": traceback.format_exc()}, 500)
def ftree_search(): args = request.args keys = args.keys() if len(keys) == 0: em = "At least one search field has to be specified" abort (400, em) if len(keys) == 1 and keys[0]=='sex': em = "Sex only is not enough" abort (400, em) total, items = fsearch(**args) return humanify({"items": items, "total": total})
def get_suggestions(collection,string): ''' This view returns a json with 3 fields: "complete", "starts_with", "phonetic". Each field holds a list of up to 5 strings. ''' rv = {} try: unlistify_item = lambda i: " ".join(i) if isinstance(i, (tuple, list)) else i if collection == "*": rv['starts_with'], rv['phonetic'] = get_completion_all_collections(string) rv['contains'] = {} # make all the words in the suggestion start with a capital letter rv = {k: {kk: [unlistify_item(i).title() for i in vv] for kk, vv in v.items()} for k, v in rv.items()} return humanify(rv) else: rv['starts_with'], rv['phonetic'] = get_completion(collection, string) rv['contains'] = [] # make all the words in the suggestion start with a capital letter rv = {k: [unlistify_item(i).title() for i in v] for k, v in rv.items()} return humanify(rv) except Exception, e: return humanify({"error": "unexpected exception getting completion data: {}".format(e), "traceback": traceback.format_exc()}, 500)
def wizard_search(): ''' We must have either `place` or `name` (or both) of the keywords. If present, the keys must not be empty. ''' args = request.args must_have_keys = ['place', 'name'] keys = args.keys() if not ('place' in keys) and not ('name' in keys): em = "Either 'place' or 'name' key must be present and not empty" abort(400, em) validated_args = {'place': None, 'name': None} for k in must_have_keys: if k in keys: if args[k]: validated_args[k] = args[k] else: abort(400, "{} argument couldn't be empty".format(k)) place = validated_args['place'] name = validated_args['name'] if place == 'havat_taninim' and name == 'tick-tock': return _generate_credits() place_doc = search_by_header(place, 'places', starts_with=False) name_doc = search_by_header(name, 'familyNames', starts_with=False) # fsearch() expects a dictionary of lists and returns Mongo cursor ftree_args = {} if name: ftree_args['last_name'] = [name] if place: ftree_args['birth_place'] = [place] # We turn the cursor to list in order to serialize it ''' TODO: restore family trees tree_found = list(fsearch(max_results=1, **ftree_args)) if not tree_found and name and 'birth_place' in ftree_args: del ftree_args['birth_place'] tree_found = list(fsearch(max_results=1, **ftree_args)) ''' rv = {'place': place_doc, 'name': name_doc} ''' TODO: restore family trees if tree_found: rv['ftree_args'] = ftree_args ''' return humanify(rv)
def get_suggestions(collection, string): ''' This view returns a json with 3 fields: "complete", "starts_with", "phonetic". Each field holds a list of up to 5 strings. ''' rv = {} rv['starts_with'], rv['phonetic'] = get_completion(collection, string) rv['contains'] = [] # make all the words in the suggestion start with a capital letter for k, v in rv.items(): newv = [] for i in v: newv.append(i.title()) rv[k] = newv return humanify(rv)
def general_search(): args = request.args parameters = { 'collection': None, 'size': SEARCH_CHUNK_SIZE, 'from_': 0, 'q': None } for param in parameters.keys(): if param in args: parameters[param] = args[param] if not parameters['q']: abort(400, 'You must specify a search query') else: rv = es_search(**parameters) for item in rv['hits']['hits']: enrich_item(item['_source']) if not rv: abort(500, 'Sorry, the search cluster appears to be down') return humanify(rv)
def fetch_images(image_ids): """Validate the comma separated list of image UUIDs and return a list of links to these images. Will return only 10 first results. """ valid_ids = [] image_urls = [] image_id_list = image_ids.split(',')[:10] for i in image_id_list: if not i: continue try: UUID(i) valid_ids.append(i) except ValueError: current_app.logger.debug('Wrong UUID - {}'.format(i)) continue image_urls = [get_image_url(i) for i in valid_ids] return humanify(image_urls)
def get_story(hash): user = get_user(hash) del user['email'] return humanify (user)
def get_collection(name): items = collect_editors_items(name) return humanify ({'items': items})
def save_user_content(): import magic if not request.files: abort(400, 'No files present!') must_have_key_list = ['title', 'description', 'creator_name'] form = request.form keys = form.keys() # Check that we have a full language specific set of fields must_have_keys = { '_en': {'missing': None, 'error': None}, '_he': {'missing': None, 'error': None} } for lang in must_have_keys: must_have_list = [k+lang for k in must_have_key_list] must_have_set = set(must_have_list) must_have_keys[lang]['missing'] = list(must_have_set.difference(set(keys))) if must_have_keys[lang]['missing']: missing_keys = must_have_keys[lang]['missing'] must_have_keys[lang]['error'] = gen_missing_keys_error(missing_keys) if must_have_keys['_en']['missing'] and must_have_keys['_he']['missing']: em_base = 'You must provide a full list of keys in English or Hebrew. ' em = em_base + must_have_keys['_en']['error'] + ' ' + must_have_keys['_he']['error'] abort(400, em) # Set metadata language(s) to the one(s) without missing fields md_languages = [] for lang in must_have_keys: if not must_have_keys[lang]['missing']: md_languages.append(lang) user_oid = current_user.id file_obj = request.files['file'] filename = secure_filename(file_obj.filename) metadata = dict(form) metadata['user_id'] = str(user_oid) metadata['original_filename'] = filename metadata['Content-Type'] = mimetypes.guess_type(filename)[0] # Pick the first item for all the list fields in the metadata clean_md = {} for key in metadata: if type(metadata[key]) == list: clean_md[key] = metadata[key][0] else: clean_md[key] = metadata[key] # Make sure there are no empty keys for at least one of the md_languages empty_keys = {'_en': [], '_he': []} for lang in md_languages: for key in clean_md: if key.endswith(lang): if not clean_md[key]: empty_keys[lang].append(key) # Check for empty keys of the single language with the full list of fields if len(md_languages) == 1 and empty_keys[md_languages[0]]: abort(400, "'{}' field couldn't be empty".format(empty_keys[md_languages[0]][0])) # Check for existence of empty keys in ALL the languages elif len(md_languages) > 1: if (empty_keys['_en'] and empty_keys['_he']): abort(400, "'{}' field couldn't be empty".format(empty_keys[md_languages[0]][0])) # Create a version of clean_md with the full fields only full_md = {} for key in clean_md: if clean_md[key]: full_md[key] = clean_md[key] # Get the magic file info file_info_str = magic.from_buffer(file_obj.stream.read()) if not _validate_filetype(file_info_str): abort(415, "File type '{}' is not supported".format(file_info_str)) # Rewind the file object file_obj.stream.seek(0) # Convert user specified metadata to BHP6 format bhp6_md = _convert_meta_to_bhp6(clean_md, file_info_str) bhp6_md['owner'] = str(user_oid) # Create a thumbnail and add it to bhp metadata try: binary_thumbnail = binarize_image(file_obj) bhp6_md['thumbnail'] = {} bhp6_md['thumbnail']['data'] = urllib.quote(binary_thumbnail.encode('base64')) except IOError as e: current_app.logger.debug('Thumbnail creation failed for {} with error: {}'.format( file_obj.filename, e.message)) # Add ugc flag to the metadata bhp6_md['ugc'] = True # Insert the metadata to the ugc collection new_ugc = Ugc(bhp6_md) new_ugc.save() file_oid = new_ugc.id bucket = ugc_bucket saved_uri = upload_file(file_obj, bucket, file_oid, full_md, make_public=True) user_email = current_user.email user_name = current_user.name if saved_uri: console_uri = 'https://console.developers.google.com/m/cloudstorage/b/{}/o/{}' http_uri = console_uri.format(bucket, file_oid) mjs = current_user.get_mjs()['mjs'] if mjs == {}: current_app.logger.debug('Creating mjs for user {}'.format(user_email)) # Add main_image_url for images (UnitType 1) if bhp6_md['UnitType'] == 1: ugc_image_uri = 'https://storage.googleapis.com/' + saved_uri.split('gs://')[1] new_ugc['ugc']['main_image_url'] = ugc_image_uri new_ugc.save() # Send an email to editor subject = 'New UGC submission' with open('templates/editors_email_template') as fh: template = jinja2.Template(fh.read()) body = template.render({'uri': http_uri, 'metadata': clean_md, 'user_email': user_email, 'user_name': user_name}) sent = send_gmail(subject, body, editor_address, message_mode='html') if not sent: current_app.logger.error('There was an error sending an email to {}'.format(editor_address)) clean_md['item_page'] = '/item/ugc.{}'.format(str(file_oid)) return humanify({'md': clean_md}) else: abort(500, 'Failed to save {}'.format(filename))