def __init__(self, *args, **kwargs):
super(ProfileTests, self).__init__(*args, **kwargs)
USER = '******'
self.USER = USER
USER2 = '2'
self.USER2 = USER2
self.USER_HASH = users.get_userid_hash(self.USER)
self.USER2_HASH = users.get_userid_hash(self.USER2)
# Fake user idtoken verifier
class id_token:
def __init__(self):
pass
def verify_token(self, token, _):
return {'iss': 'accounts.google.com', 'sub': USER}
# Fake user idtoken verifier
class id_token2:
def __init__(self):
pass
def verify_token(self, token, _):
return {'iss': 'accounts.google.com', 'sub': USER2}
self.id_token = id_token()
self.id_token2 = id_token2()
util.id_token = self.id_token
self.HEADERS = {'Content-type': 'application/json', 'X-IDTOKEN': USER}
def user(self, user_id):
client = self._get_datastore_client()
result = flask_users.authn_check(flask.request.headers)
if isinstance(result, flask.Response):
return result
userid_hash = users.get_userid_hash(result)
# User GETting their own record
if self.request.method == 'GET':
if userid_hash == user_id:
return self.get_user(user_id)
else:
return flask.Response('Permission denied', status=403)
# Special case for user PUTting their own initial record
if self.request.method == 'PUT' and \
not users.check_if_user_exists(client, userid_hash) and \
userid_hash == user_id:
return self.put_user(user_id)
# Users can only modify their own records
if userid_hash != user_id:
return flask.Response('Permission denied', status=403)
if self.request.method == 'PUT':
return self.put_user(user_id)
elif self.request.method == 'DELETE':
return self.delete_user(user_id)
elif self.request.method == 'UPDATE':
return self.update_user(user_id)
else:
return flask.Response('Unsupported method', status=405)
def root(self):
client = self._get_datastore_client()
result = flask_users.authn_check(flask.request.headers)
if isinstance(result, flask.Response):
return result
userid_hash = users.get_userid_hash(result)
if not users.check_if_user_exists(client, userid_hash):
return flask.Response('User does not exist', status=404)
result = roles._check_if_user_is_admin(client, userid_hash)
if isinstance(result, flask.Response):
return result
if result is False:
return flask.Response('Permission denied', status=403)
locations = []
query = client.query(kind="User")
cursor = None
while True:
entities_count = 0
entities = query.fetch(start_cursor=cursor, limit=1000)
for entity in entities:
entities_count += 1
if entity.has_key('geocoded_location'):
location = entity['geocoded_location']
locations.append(location)
if entities_count < 1000:
break
cursor = entities.next_page_token
s = flask.jsonify(locations)
return s
def test_profile_disabled(self):
token = get_id_token()
r = gat_requests.Request()
idinfo = util._validate_id_token(token)
userid = users.get_userid(idinfo)
userid_hash = users.get_userid_hash(userid)
self._delete_user_via_datastore_if_exists(userid_hash)
self._get_user_via_api(userid_hash, token)
self._create_user_via_datastore(userid_hash)
self._get_user_via_api(userid_hash, token)
def __init__(self, *args, **kwargs):
super(CountTests, self).__init__(*args, **kwargs)
self.USER = '******'
self.USER2 = '2'
self.USER_HASH = unicode(users.get_userid_hash(self.USER))
self.USER2_HASH = users.get_userid_hash(self.USER2)
# Fake user idtoken verifier
class id_token:
def __init__(self, user):
self.user = user
def verify_token(self, token, _):
return {'iss': 'accounts.google.com', 'sub': self.user}
self.id_token = id_token(self.USER)
self.id_token2 = id_token(self.USER2)
util.id_token = self.id_token
self.HEADERS = {
'Content-type': 'application/json',
'X-IDTOKEN': self.USER
}
def users(self):
client = self._get_datastore_client()
result = flask_users.authn_check(flask.request.headers)
if isinstance(result, flask.Response):
return result
userid_hash = users.get_userid_hash(result)
if not users.check_if_user_exists(client, userid_hash):
return flask.Response('User does not exist', status=404)
result = roles._check_if_user_is_admin(client, userid_hash)
if isinstance(result, flask.Response):
return result
if result is False:
return flask.Response('Permission denied', status=403)
locations = []
query = client.query(kind="User")
cursor = None
while True:
entities_count = 0
entities = query.fetch(start_cursor=cursor, limit=1000)
for entity in entities:
entities_count += 1
if entity.has_key('geocoded_location'):
location = entity['geocoded_location']
locations.append(location)
if entities_count < 1000:
break
cursor = entities.next_page_token
clusters = cluster_points(locations, eps=0.1, min_samples=10, n_jobs=64)
centers, sizes = compute_centers(clusters, locations)
results = { 'points': [] }
if len(centers) != 0:
results['min_size'] = min(sizes.values())
results['max_size'] = max(sizes.values())
p = results['points']
for label in centers:
center = centers[label]
size = sizes[label]
p.append( ((center[0], center[1]), size))
return flask.jsonify(results)
def __init__(self, *args, **kwargs):
super(RolesTest, self).__init__(*args, **kwargs)
self.USER = '******'
self.USER_HASH = users.get_userid_hash(self.USER)
def _upload_post(self):
"""
Request handler for upload POST requests. Writes accepts files in POST
request body and saves them to the local file system as
<self._dir>/<uuid>.<file extension as uploaded>
Creates datastore record for uploaded files and indicates that they
have yet to be uploaded to Cloud Storage.
Returns constants.HTTP_ERROR status if an error occurs with a
short message.
Returns constants.HTTP_OK response on success with no message.
Returns constants.HTTP_OOM status if the server is under too much
load to handle the request.
"""
self.logger.info("Upload POST received")
datastore_client = self.datastore.Client(self.config['PROJECT_ID'])
# Fetch the user's identifier from the request, which
# contains the oauth2 creds.
try:
token = flask.request.headers['X-IDTOKEN']
except Exception as e:
self.logger.error("Missing credential token header")
return flask.Response('Missing credential token header', 405)
try:
upload_session_id = flask.request.headers['X-UPLOADSESSIONID']
except Exception as e:
self.logger.error("Missing session ID")
return flask.Response('Missing session ID', 400)
try:
image_bucket = flask.request.headers['X-IMAGE-BUCKET']
except Exception as e:
self.logger.error("Missing image bucket")
return flask.Response('Missing image bucket', 400)
try:
cc0_agree = flask.request.headers['X-CC0-AGREE']
if cc0_agree != 'true':
raise ValueError('Must accept cc0')
except Exception as e:
self.logger.error("Missing CC0 agreement")
return flask.Response('Missing CC0 agreement', 400)
try:
public_agree = flask.request.headers['X-PUBLIC-AGREE']
if public_agree != 'true':
raise ValueError('Must accept public database')
except Exception as e:
self.logger.error("Missing public dataset agreement")
return flask.Response('Missing public dataset agreement', 400)
# TODO(dek): read and update hashlib object using fix-sized buffers to avoid memory blowup
# Read the content of the upload completely, before returning an error.
file_ = flask.request.files['file']
original_filename = file_.filename
self.logger.info("Reading upload stream")
content = file_.stream.read()
self.logger.info("Read upload stream")
result = flask_users.authn_check(flask.request.headers)
if isinstance(result, flask.Response):
self.logger.error("Failed auth check")
return result
userid_hash = users.get_userid_hash(result)
if not users.check_if_user_exists(datastore_client, userid_hash):
self.logger.error("Failed profile check")
return self.Response('Profile required to upload images.',
status=400)
r = roles.get_user_role(datastore_client, userid_hash)
# Check for a valid bucket.
valid_bucket = False
if image_bucket == 'app':
valid_bucket = True
elif image_bucket == 'megamovie' or image_bucket == 'volunteer_test':
valid_bucket = self._check_role_in_roles(
r, VALID_MEGAMOVIE_UPLOADER_ROLES)
elif image_bucket == 'teramovie':
valid_bucket = self._check_role_in_roles(
r, VALID_TERAMOVIE_UPLOADER_ROLES)
else:
# Not a known bucket.
valid_bucket = False
if not valid_bucket:
self.logger.error("Failed bucket check")
return self.Response(
'Valid role required to upload images to this bucket, or bucket is unknown',
status=400)
content_type = self.request.content_type
name = hashlib.sha256(content).hexdigest()
self.logger.info("Received image with digest: %s" % name)
# Local file system file paths
local_file = self.os.path.join(self._dir, name)
temp_file = local_file + self._file_not_ready_suffix
try:
open(temp_file, "wb").write(content)
except IOError as e:
self.logger.error('Error occured writing to file: {0}'.format(e))
return self.Response('Failed to save file.',
status=constants.HTTP_ERROR)
del content
metadata = _extract_exif_metadata(temp_file)
result = {}
if metadata.has_key('lat'):
result['lat'] = metadata['lat']
if metadata.has_key('lon'):
result['lon'] = metadata['lon']
key = datastore_client.key(self._datastore_kind, name)
entity = datastore_client.get(key)
if entity is not None:
if 'user' in entity:
if entity['user'] == datastore_client.key(
self._user_datastore_kind, userid_hash):
entity['upload_session_id'] = upload_session_id
datastore_client.put(entity)
else:
self.logger.error(
'Duplicate detected but incomplete datastore record')
try:
os.remove(temp_file)
except OSError as e:
self.logger.error('Unable to remove file: {0}'.format(e))
result['warning'] = 'Duplicate file upload.'
return flask.jsonify(**result)
entity = self._create_datastore_entry(
datastore_client,
name,
original_filename,
user=userid_hash,
upload_session_id=upload_session_id,
image_bucket=image_bucket,
cc0_agree=cc0_agree,
public_agree=public_agree)
entity.update(metadata)
if not entity:
self.logger.error('Unable to create datastore entry for %s' % name)
try:
os.remove(temp_file)
except OSError as e:
self.logger.error('Unable to remove file: {0}'.format(e))
return self.Response('Failed to save file.',
status=constants.HTTP_ERROR)
try:
datastore_client.put(entity)
except Exception as e:
self.logger.error('Unable to create datastore entry for %s: %s' %
(name, str(e)))
try:
os.remove(temp_file)
except OSError as e:
self.logger.error('Unable to remove file: {0}'.format(e))
return self.Response('Failed to save file.',
status=constants.HTTP_ERROR)
try:
os.rename(temp_file, local_file)
except Exception as e:
self.logger.error('Error occured rename file: {0}'.format(e))
try:
os.remove(temp_file)
except OSError as e:
self.logger.error('Unable to remove file: {0}'.format(e))
return self.Response('Failed to save file.',
status=constants.HTTP_ERROR)
return flask.jsonify(**result)
def photos(self):
client = self._get_datastore_client()
result = flask_users.authn_check(flask.request.headers)
if isinstance(result, flask.Response):
return result
userid_hash = users.get_userid_hash(result)
if not users.check_if_user_exists(client, userid_hash):
return flask.Response('User does not exist', status=404)
result = roles._check_if_user_is_admin(client, userid_hash)
if isinstance(result, flask.Response):
return result
if result is False:
return flask.Response('Permission denied', status=403)
filters = []
filters.append(('confirmed_by_user', '=', True))
image_buckets = flask.request.args.get('image_buckets', "").split(",")
locations = []
query = client.query(kind="Photo", filters=filters)
cursor = None
while True:
entities_count = 0
entities = query.fetch(start_cursor=cursor, limit=1000)
for entity in entities:
entities_count += 1
if entity.has_key('lat') and entity.has_key('lon'):
# Negate longitude because photo longitude is stored as positive-West, but
# code needs negative-West.
location = entity['lat'], -entity['lon']
if len(image_buckets) == 0:
should_append = True
else:
# Check if this bucket is in the list.
should_append = entity['image_bucket'] in image_buckets
if should_append:
locations.append(location)
if entities_count < 1000:
break
cursor = entities.next_page_token
if len(locations) > 0:
clusters = cluster_points(locations, eps=0.1, min_samples=10, n_jobs=64)
centers, sizes = compute_centers(clusters, locations)
else:
centers = []
sizes = []
results = { 'points': [] }
if len(centers) != 0:
results['min_size'] = min(sizes.values())
results['max_size'] = max(sizes.values())
p = results['points']
for label in centers:
center = centers[label]
size = sizes[label]
p.append( ((center[0], center[1]), size) )
return flask.jsonify(results)
def root(self, cursor=None):
"""Returns list of photos subject to filter criteria."""
client = self._get_datastore_client()
# Auth check: must be logged in, user profile exists, have
# admin or reviewer role
result = flask_users.authn_check(flask.request.headers)
if isinstance(result, flask.Response):
return result
userid_hash = users.get_userid_hash(result)
if not users.check_if_user_exists(client, userid_hash):
return flask.Response('User does not exist', status=404)
result = roles._check_if_user_has_role(
client, userid_hash, set([roles.ADMIN_ROLE, roles.REVIEWER_ROLE]))
if isinstance(result, flask.Response):
return result
if result is False:
return flask.Response('Permission denied', status=403)
# Check if cursor passed in from clinet
cursor = flask.request.args.get('cursor', None)
if cursor is not None:
cursor = str(cursor)
# Filter, and order, items.
order = []
filters = []
num_reviews_max = flask.request.args.get('num_reviews_max', None)
if num_reviews_max is not None:
filters.append(('num_reviews', '<=', int(num_reviews_max)))
order.append('num_reviews')
mask_reviewer = flask.request.args.get('mask_reviewer', False)
image_bucket = flask.request.args.get('image_bucket', None)
if image_bucket is not None:
filters.append(('image_bucket', '=', image_bucket))
order.append('image_bucket')
user_id = flask.request.args.get('user_id', None)
if user_id is not None:
filters.append(('user', '=', client.key("User", user_id)))
order.append('user')
upload_session_id = flask.request.args.get('upload_session_id', None)
if upload_session_id is not None:
filters.append(('upload_session_id', '=', upload_session_id))
order.append('upload_session_id')
if flask.request.args.get('image_datetime_begin'):
image_datetime_begin = datetime.datetime.fromtimestamp(
float(flask.request.args['image_datetime_begin']))
filters.append(('image_datetime', '>', image_datetime_begin))
order.append('image_datetime')
if flask.request.args.get('image_datetime_end'):
image_datetime_end = datetime.datetime.fromtimestamp(
float(flask.request.args['image_datetime_end']))
filters.append(('image_datetime', '<=', image_datetime_end))
if 'image_datetime' not in order:
order.append('image_datetime')
if flask.request.args.get('uploaded_date_begin'):
uploaded_date_begin = datetime.datetime.fromtimestamp(
float(flask.request.args['uploaded_date_begin']))
filters.append(('uploaded_date', '>', uploaded_date_begin))
order.append('uploaded_date')
if flask.request.args.get('uploaded_date_end'):
uploaded_date_end = datetime.datetime.fromtimestamp(
float(flask.request.args['uploaded_date_end']))
filters.append(('uploaded_date', '<=', uploaded_date_end))
if 'uploaded_date' not in order:
order.append('uploaded_date')
# TODO(dek): randomize results(?)
# Fetch start_cursor to limit entites
query = client.query(kind="Photo", filters=filters)
query.order = order
limit = int(flask.request.args.get('limit', 100))
entities = query.fetch(start_cursor=cursor, limit=limit)
page = next(entities.pages)
# Fetch results before getting next page token
e = list(page)
next_cursor = entities.next_page_token
if e is None:
return flask.Response('No entities.', 200)
# Return matching photos by ID
photos = []
client = storage.client.Client(project=config.PROJECT_ID)
bucket = client.bucket(config.GCS_BUCKET)
for entity in e:
blob = storage.Blob(entity.key.name, bucket)
url = blob.public_url
if 'image_type' not in entity:
logging.error("Photo %s does not have an image type." %
entity.key.name)
continue
include_photo = True
location = None
if 'lat' in entity and 'lon' in entity:
location = {"lat": entity['lat'], "lon": entity['lon']}
if entity['image_type'] == 'raw' or entity['image_type'] == 'TIFF':
# If the image isn't a JPG, the browser won't show it.
# Make the URL point at the JPG copy
url = url + '.jpg'
if mask_reviewer and entity.has_key("reviews"):
for review in entity["reviews"]:
# Don't consider photos where we satisfy the
# num_reviews inequality, but one of the reviewers
# is ourself
if review['user_id'] == userid_hash:
include_photo = False
break
if include_photo:
photo = {'name': entity.key.name, 'url': url}
if location is not None:
photo['location'] = location
photos.append(photo)
results = {}
if len(photos):
results['cursor'] = next_cursor
results['photos'] = photos
return flask.jsonify(results)
def confirm(self):
"""For each photo from upload_session_id that matches the files from original_filenames, mark it confirmed_by_user=True"""
client = self._get_datastore_client()
# Auth check: must be logged in, user profile exists, have
# admin or volunteer role
result = flask_users.authn_check(flask.request.headers)
if isinstance(result, flask.Response):
return result
userid_hash = users.get_userid_hash(result)
if not users.check_if_user_exists(client, userid_hash):
return flask.Response('User does not exist', status=404)
if flask.request.headers.has_key(
"X-IDEUM-APP-SECRET"
) and flask.request.headers["X-IDEUM-APP-SECRET"] == IDEUM_APP_SECRET:
logging.info("Request contains Ideum app secret.")
else:
result = roles._check_if_user_has_role(
client, userid_hash,
set([roles.USER_ROLE, roles.ADMIN_ROLE, roles.VOLUNTEER_ROLE]))
if isinstance(result, flask.Response):
return result
if result is False:
return flask.Response('Permission denied', status=403)
json = flask.request.get_json()
if 'upload_session_id' not in json:
return flask.Response('upload_session_id required', status=403)
upload_session_id = json['upload_session_id']
if 'filenames' not in json:
return flask.Response('filenames required', status=403)
filenames = json['filenames']
anonymous_photo = json.get('anonymous_photo', False)
equatorial_mount = json.get('equatorial_mount', False)
filters = [('upload_session_id', '=', upload_session_id)]
query = client.query(kind="Photo", filters=filters)
entities = list(query.fetch())
results = repair_missing_gps.partition_gps(entities)
complete_images, incomplete_images = results
all_entities = complete_images[:]
# If there's one complete image and more than one incomplete images,
# repair the incomplete images using the complete image
logging.info("Received %d complete images" % len(complete_images))
logging.info("Received %d incomplete images" % len(incomplete_images))
if len(incomplete_images) > 0:
logging.info("Repairing incomplete images: %s" %
str(incomplete_images))
if complete_images == []:
complete_image = None
else:
complete_image = pick_image(complete_images)
logging.info("Chose %s as complete image" % complete_image)
for incomplete_image in incomplete_images:
repaired_image = repair_missing_gps.update_incomplete(
complete_image, incomplete_image)
all_entities.append(repaired_image)
logging.info("Repaired: %s" % ([
incomplete_image.key.name
for incomplete_image in incomplete_images
]))
if complete_image is not None:
logging.info("Repair source: %s", complete_image.key.name)
else:
for incomplete_image in incomplete_images:
all_entities.append(incomplete_image)
logging.info("No repairs for: %s" % [
incomplete_image.key.name
for incomplete_image in incomplete_images
])
for entity_chunk in chunks(all_entities, 500):
batch = client.batch()
batch.begin()
for entity in entity_chunk:
if entity.has_key(u'original_filename') and entity[
u'original_filename'] in filenames:
entity['confirmed_by_user'] = True
entity['anonymous_photo'] = anonymous_photo
entity['equatorial_mount'] = equatorial_mount
batch.put(entity)
batch.commit()
return flask.Response('OK', status=200)
def photo(self, photo_id):
"""GET returns photo entity, PATCH modifies photo entity."""
client = self._get_datastore_client()
# Auth check: must be logged in, user profile exists, have
# admin or reviewer role
result = flask_users.authn_check(flask.request.headers)
if isinstance(result, flask.Response):
return result
userid_hash = users.get_userid_hash(result)
if not users.check_if_user_exists(client, userid_hash):
return flask.Response('User does not exist', status=404)
result = roles._check_if_user_has_role(
client, userid_hash, set([roles.ADMIN_ROLE, roles.REVIEWER_ROLE]))
if isinstance(result, flask.Response):
return result
if result is False:
return flask.Response('Permission denied', status=403)
key = client.key("Photo", str(photo_id))
# Dispatch on method
if self.request.method == 'GET':
# if the photo_id exists, return the approved fields
result = {}
entity = client.get(key)
for field in FIELDS:
if field not in entity:
return flask.Response(
'Invalid photo (missing field %s)' % field, 404)
result[field] = entity[field]
result['reviews'] = []
for review in entity['reviews']:
reviewer = review['user_id'].name
vote = review['vote']
result['reviews'].append((reviewer, vote))
result['user'] = entity['user'].name
return flask.jsonify(result)
elif flask.request.method == 'PATCH':
# if the photo_id exists, update it
result = util._validate_json(flask.request)
if result is not True:
return result
json = flask.request.get_json()
json = util._escape_json(json)
if 'vote' not in json:
return flask.Response('Missing vote', 400)
vote = json['vote']
if vote != "up" and vote != "down":
return flask.Response('Invalid vote, use "up" or "down"', 400)
with client.transaction():
# Fetch the existing Photo entity
entity = client.get(key)
review = datastore.Entity(key=client.key('Review'))
review['user_id'] = userid_hash
review['vote'] = vote
# Update the reviews property
if 'reviews' not in entity:
# Case: no existing reviews, create first
entity["reviews"] = [review]
else:
for r in entity["reviews"]:
reviewer_key = r['user_id']
# Case: existing review by user, edit vote
if reviewer_key == userid_hash:
r['vote'] = vote
else:
# Case: existing review by new user, append
entity["reviews"].append(review)
entity["num_reviews"] = len(entity["reviews"])
try:
client.put(entity)
except Exception as e:
logging.error("Datastore update operation failed: %s" %
str(e))
flask.Response('Internal server error', status=500)
return flask.Response('OK', status=200)
def root(self):
client = self._get_datastore_client()
result = flask_users.authn_check(flask.request.headers)
if isinstance(result, flask.Response):
return result
userid_hash = users.get_userid_hash(result)
if not users.check_if_user_exists(client, userid_hash):
return flask.Response('User does not exist', status=404)
result = roles._check_if_user_is_admin(client, userid_hash)
if isinstance(result, flask.Response):
return result
if result is False:
return flask.Response('Permission denied', status=403)
filters = []
image_bucket = flask.request.args.get('image_bucket', None)
if image_bucket is not None:
filters.append(('image_bucket', '=', image_bucket))
filters.append(('confirmed_by_user', '=', True))
total_count = 0
gps_count = 0
type_count = {
'app': {
'total_count': 0,
'gps_count': 0
},
'volunteer_test': {
'total_count': 0,
'gps_count': 0
},
'megamovie': {
'total_count': 0,
'gps_count': 0
},
'teramovie': {
'total_count': 0,
'gps_count': 0
}
}
query = client.query(kind="Photo", filters=filters)
cursor = None
while True:
entities_count = 0
entities = query.fetch(start_cursor=cursor, limit=1000)
for entity in entities:
entities_count += 1
image_bucket = entity['image_bucket']
type_counter = type_count.get(image_bucket, None)
if type_counter is not None:
type_counter['total_count'] += 1
if entity.has_key('lat') and entity.has_key('lon'):
if type_counter is not None:
type_counter['gps_count'] += 1
gps_count += 1
total_count += entities_count
if entities_count < 1000:
break
cursor = entities.next_page_token
photo_stats = {
'total_count': total_count,
'gps_count': gps_count,
'types': type_count
}
s = flask.jsonify(photo_stats)
return s
