def general_search():
try:
args = request.args
parameters = {'collection': None, 'size': SEARCH_CHUNK_SIZE, 'from_': 0, 'q': None, 'sort': None, "with_persons": False}
parameters.update(PERSONS_SEARCH_DEFAULT_PARAMETERS)
got_one_of_required_persons_params = False
for param in parameters.keys():
if param in args:
if param == "with_persons":
parameters[param] = args[param].lower() in ["1", "yes", "true"]
else:
parameters[param] = args[param]
if param in PERSONS_SEARCH_REQUIRES_ONE_OF and parameters[param]:
got_one_of_required_persons_params = True
if parameters["q"] or (parameters["collection"] == "persons" and got_one_of_required_persons_params):
try:
rv = es_search(**parameters)
except Exception as e:
logging.exception(e)
return humanify({"error": e.message}, 500)
for item in rv['hits']['hits']:
enrich_item(item['_source'], collection_name=item['_type'])
return humanify(rv)
else:
return humanify({"error": "You must specify a search query"}, 400)
except Exception as e:
logging.exception("unexpected error")
return humanify(({"error": e.message}, 500))
def get_changes(from_date, to_date):
rv = set()
# Validate the dates
dates = {'start': from_date, 'end': to_date}
for date in dates:
try:
dates[date] = datetime.fromtimestamp(float(dates[date]))
except ValueError:
abort(400, 'Bad timestamp - {}'.format(dates[date]))
log_collection = current_app.data_db['migration_log']
query = {'date': {'$gte': dates['start'], '$lte': dates['end']}}
projection = {'item_id': 1, '_id': 0}
cursor = log_collection.find(query, projection)
if not cursor:
return humanify([])
else:
for doc in cursor:
col, _id = doc['item_id'].split('.')
# TODO: remove references to the genTreeIndividuals collection - it is irrelevant and not in use
if col == 'genTreeIndividuals':
continue
else:
if filter_doc_id(_id, col):
rv.add(doc['item_id'])
return humanify(list(rv))
def get_items(slugs):
if slugs:
items_list = slugs.split(',')
elif request.is_json:
items_list = request.get_json()
# Check if there are items from ugc collection and test their access control
ugc_items = []
for item in items_list:
if item.startswith('ugc'):
ugc_items.append(item)
user_oid = current_user.is_authenticated and current_user.id
items = fetch_items(items_list)
if len(items) == 1 and 'error_code' in items[0]:
error = items[0]
abort(error['error_code'], error['msg'])
else:
# Cast items to list
if type(items) != list:
items = [items]
# Check that each of the ugc_items is accessible by the logged in user
for ugc_item_id in [i[4:] for i in ugc_items]:
for item in items:
if item['_id'] == ugc_item_id and item.has_key('owner') and item['owner'] != unicode(user_oid):
abort(403, 'You are not authorized to access item ugc.{}'.format(str(item['_id'])))
return humanify(items)
def get_geocoded_places():
args = request.args
filters = SHOW_FILTER.copy()
filters['geometry'] = {'$exists': True}
filters['Header.En'] = {'$nin': [None, '']}
try:
filters['geometry.coordinates.1'] = {
'$gte': float(args['sw_lat']),
'$lte': float(args['ne_lat'])
}
filters['geometry.coordinates.0'] = {
'$gte': float(args['sw_lng']),
'$lte': float(args['ne_lng'])
}
except KeyError:
abort(400, 'Please specify a box using sw_lat, sw_lng, ne_lat, ne_lng')
except ValueError:
abort(
400,
'Please specify a box using floats in sw_lat, sw_lng, ne_lat, ne_lng'
)
points = current_app.data_db['places'].find(filters, {
'Header': True,
'Slug': True,
'geometry': True,
'PlaceTypeDesc': True
})
ret = humanify(list(points))
return ret
def linkify():
try:
collections = ["places", "personalities", "familyNames"]
res = {collection: [] for collection in collections}
try:
if request.method == "POST":
html_lower = request.form["html"].lower()
else:
html_lower = request.args["html"].lower()
except Exception as e:
logging.exception(e)
raise
for i, item in enumerate(get_linkify_items(collections, html_lower)):
res[item["collection"]].append(item["item"])
return humanify(res)
except Exception as e:
return humanify({"error": e.message, "traceback": traceback.format_exc()}, 500)
def ftree_search():
args = request.args
keys = args.keys()
if len(keys) == 0:
em = "At least one search field has to be specified"
abort (400, em)
if len(keys) == 1 and keys[0]=='sex':
em = "Sex only is not enough"
abort (400, em)
total, items = fsearch(**args)
return humanify({"items": items, "total": total})
def get_suggestions(collection,string):
'''
This view returns a json with 3 fields:
"complete", "starts_with", "phonetic".
Each field holds a list of up to 5 strings.
'''
rv = {}
try:
unlistify_item = lambda i: " ".join(i) if isinstance(i, (tuple, list)) else i
if collection == "*":
rv['starts_with'], rv['phonetic'] = get_completion_all_collections(string)
rv['contains'] = {}
# make all the words in the suggestion start with a capital letter
rv = {k: {kk: [unlistify_item(i).title() for i in vv] for kk, vv in v.items()} for k, v in rv.items()}
return humanify(rv)
else:
rv['starts_with'], rv['phonetic'] = get_completion(collection, string)
rv['contains'] = []
# make all the words in the suggestion start with a capital letter
rv = {k: [unlistify_item(i).title() for i in v] for k, v in rv.items()}
return humanify(rv)
except Exception, e:
return humanify({"error": "unexpected exception getting completion data: {}".format(e), "traceback": traceback.format_exc()}, 500)
def wizard_search():
'''
We must have either `place` or `name` (or both) of the keywords.
If present, the keys must not be empty.
'''
args = request.args
must_have_keys = ['place', 'name']
keys = args.keys()
if not ('place' in keys) and not ('name' in keys):
em = "Either 'place' or 'name' key must be present and not empty"
abort(400, em)
validated_args = {'place': None, 'name': None}
for k in must_have_keys:
if k in keys:
if args[k]:
validated_args[k] = args[k]
else:
abort(400, "{} argument couldn't be empty".format(k))
place = validated_args['place']
name = validated_args['name']
if place == 'havat_taninim' and name == 'tick-tock':
return _generate_credits()
place_doc = search_by_header(place, 'places', starts_with=False)
name_doc = search_by_header(name, 'familyNames', starts_with=False)
# fsearch() expects a dictionary of lists and returns Mongo cursor
ftree_args = {}
if name:
ftree_args['last_name'] = [name]
if place:
ftree_args['birth_place'] = [place]
# We turn the cursor to list in order to serialize it
''' TODO: restore family trees
tree_found = list(fsearch(max_results=1, **ftree_args))
if not tree_found and name and 'birth_place' in ftree_args:
del ftree_args['birth_place']
tree_found = list(fsearch(max_results=1, **ftree_args))
'''
rv = {'place': place_doc, 'name': name_doc}
''' TODO: restore family trees
if tree_found:
rv['ftree_args'] = ftree_args
'''
return humanify(rv)
def get_suggestions(collection, string):
'''
This view returns a json with 3 fields:
"complete", "starts_with", "phonetic".
Each field holds a list of up to 5 strings.
'''
rv = {}
rv['starts_with'], rv['phonetic'] = get_completion(collection, string)
rv['contains'] = []
# make all the words in the suggestion start with a capital letter
for k, v in rv.items():
newv = []
for i in v:
newv.append(i.title())
rv[k] = newv
return humanify(rv)
def general_search():
args = request.args
parameters = {
'collection': None,
'size': SEARCH_CHUNK_SIZE,
'from_': 0,
'q': None
}
for param in parameters.keys():
if param in args:
parameters[param] = args[param]
if not parameters['q']:
abort(400, 'You must specify a search query')
else:
rv = es_search(**parameters)
for item in rv['hits']['hits']:
enrich_item(item['_source'])
if not rv:
abort(500, 'Sorry, the search cluster appears to be down')
return humanify(rv)
def fetch_images(image_ids):
"""Validate the comma separated list of image UUIDs and return a list
of links to these images.
Will return only 10 first results.
"""
valid_ids = []
image_urls = []
image_id_list = image_ids.split(',')[:10]
for i in image_id_list:
if not i:
continue
try:
UUID(i)
valid_ids.append(i)
except ValueError:
current_app.logger.debug('Wrong UUID - {}'.format(i))
continue
image_urls = [get_image_url(i) for i in valid_ids]
return humanify(image_urls)
def get_story(hash):
user = get_user(hash)
del user['email']
return humanify (user)
def get_collection(name):
items = collect_editors_items(name)
return humanify ({'items': items})
def save_user_content():
import magic
if not request.files:
abort(400, 'No files present!')
must_have_key_list = ['title',
'description',
'creator_name']
form = request.form
keys = form.keys()
# Check that we have a full language specific set of fields
must_have_keys = {
'_en': {'missing': None, 'error': None},
'_he': {'missing': None, 'error': None}
}
for lang in must_have_keys:
must_have_list = [k+lang for k in must_have_key_list]
must_have_set = set(must_have_list)
must_have_keys[lang]['missing'] = list(must_have_set.difference(set(keys)))
if must_have_keys[lang]['missing']:
missing_keys = must_have_keys[lang]['missing']
must_have_keys[lang]['error'] = gen_missing_keys_error(missing_keys)
if must_have_keys['_en']['missing'] and must_have_keys['_he']['missing']:
em_base = 'You must provide a full list of keys in English or Hebrew. '
em = em_base + must_have_keys['_en']['error'] + ' ' + must_have_keys['_he']['error']
abort(400, em)
# Set metadata language(s) to the one(s) without missing fields
md_languages = []
for lang in must_have_keys:
if not must_have_keys[lang]['missing']:
md_languages.append(lang)
user_oid = current_user.id
file_obj = request.files['file']
filename = secure_filename(file_obj.filename)
metadata = dict(form)
metadata['user_id'] = str(user_oid)
metadata['original_filename'] = filename
metadata['Content-Type'] = mimetypes.guess_type(filename)[0]
# Pick the first item for all the list fields in the metadata
clean_md = {}
for key in metadata:
if type(metadata[key]) == list:
clean_md[key] = metadata[key][0]
else:
clean_md[key] = metadata[key]
# Make sure there are no empty keys for at least one of the md_languages
empty_keys = {'_en': [], '_he': []}
for lang in md_languages:
for key in clean_md:
if key.endswith(lang):
if not clean_md[key]:
empty_keys[lang].append(key)
# Check for empty keys of the single language with the full list of fields
if len(md_languages) == 1 and empty_keys[md_languages[0]]:
abort(400, "'{}' field couldn't be empty".format(empty_keys[md_languages[0]][0]))
# Check for existence of empty keys in ALL the languages
elif len(md_languages) > 1:
if (empty_keys['_en'] and empty_keys['_he']):
abort(400, "'{}' field couldn't be empty".format(empty_keys[md_languages[0]][0]))
# Create a version of clean_md with the full fields only
full_md = {}
for key in clean_md:
if clean_md[key]:
full_md[key] = clean_md[key]
# Get the magic file info
file_info_str = magic.from_buffer(file_obj.stream.read())
if not _validate_filetype(file_info_str):
abort(415, "File type '{}' is not supported".format(file_info_str))
# Rewind the file object
file_obj.stream.seek(0)
# Convert user specified metadata to BHP6 format
bhp6_md = _convert_meta_to_bhp6(clean_md, file_info_str)
bhp6_md['owner'] = str(user_oid)
# Create a thumbnail and add it to bhp metadata
try:
binary_thumbnail = binarize_image(file_obj)
bhp6_md['thumbnail'] = {}
bhp6_md['thumbnail']['data'] = urllib.quote(binary_thumbnail.encode('base64'))
except IOError as e:
current_app.logger.debug('Thumbnail creation failed for {} with error: {}'.format(
file_obj.filename, e.message))
# Add ugc flag to the metadata
bhp6_md['ugc'] = True
# Insert the metadata to the ugc collection
new_ugc = Ugc(bhp6_md)
new_ugc.save()
file_oid = new_ugc.id
bucket = ugc_bucket
saved_uri = upload_file(file_obj, bucket, file_oid, full_md, make_public=True)
user_email = current_user.email
user_name = current_user.name
if saved_uri:
console_uri = 'https://console.developers.google.com/m/cloudstorage/b/{}/o/{}'
http_uri = console_uri.format(bucket, file_oid)
mjs = current_user.get_mjs()['mjs']
if mjs == {}:
current_app.logger.debug('Creating mjs for user {}'.format(user_email))
# Add main_image_url for images (UnitType 1)
if bhp6_md['UnitType'] == 1:
ugc_image_uri = 'https://storage.googleapis.com/' + saved_uri.split('gs://')[1]
new_ugc['ugc']['main_image_url'] = ugc_image_uri
new_ugc.save()
# Send an email to editor
subject = 'New UGC submission'
with open('templates/editors_email_template') as fh:
template = jinja2.Template(fh.read())
body = template.render({'uri': http_uri,
'metadata': clean_md,
'user_email': user_email,
'user_name': user_name})
sent = send_gmail(subject, body, editor_address, message_mode='html')
if not sent:
current_app.logger.error('There was an error sending an email to {}'.format(editor_address))
clean_md['item_page'] = '/item/ugc.{}'.format(str(file_oid))
return humanify({'md': clean_md})
else:
abort(500, 'Failed to save {}'.format(filename))
