Beispiel #1
0
    def root(self):
        client = self._get_datastore_client()
        result = flask_users.authn_check(flask.request.headers)
        if isinstance(result, flask.Response):
            return result
        userid_hash = users.get_userid_hash(result)
        if not users.check_if_user_exists(client, userid_hash):
            return flask.Response('User does not exist', status=404)
        result = roles._check_if_user_is_admin(client, userid_hash)
        if isinstance(result, flask.Response):
            return result
        if result is False:
            return flask.Response('Permission denied', status=403)

        locations = []
        query = client.query(kind="User")
        cursor = None
        while True:
            entities_count = 0
            entities = query.fetch(start_cursor=cursor, limit=1000)
            for entity in entities:
                entities_count += 1
                if entity.has_key('geocoded_location'):
                    location = entity['geocoded_location']
                    locations.append(location)

            if entities_count < 1000:
                break
            cursor = entities.next_page_token

        s = flask.jsonify(locations)
        return s
Beispiel #2
0
    def user(self, user_id):
        client = self._get_datastore_client()
        result = flask_users.authn_check(flask.request.headers)
        if isinstance(result, flask.Response):
            return result
        userid_hash = users.get_userid_hash(result)

        # User GETting their own record
        if self.request.method == 'GET':
            if userid_hash == user_id:
                return self.get_user(user_id)
            else:
                return flask.Response('Permission denied', status=403)

        # Special case for user PUTting their own initial record
        if self.request.method == 'PUT' and \
           not users.check_if_user_exists(client, userid_hash) and \
           userid_hash == user_id:
            return self.put_user(user_id)

        # Users can only modify their own records
        if userid_hash != user_id:
            return flask.Response('Permission denied', status=403)

        if self.request.method == 'PUT':
            return self.put_user(user_id)
        elif self.request.method == 'DELETE':
            return self.delete_user(user_id)
        elif self.request.method == 'UPDATE':
            return self.update_user(user_id)
        else:
            return flask.Response('Unsupported method', status=405)
Beispiel #3
0
    def users(self):
        client = self._get_datastore_client()
        result = flask_users.authn_check(flask.request.headers)
        if isinstance(result, flask.Response):
            return result
        userid_hash = users.get_userid_hash(result)
        if not users.check_if_user_exists(client, userid_hash):
            return flask.Response('User does not exist', status=404)
        result = roles._check_if_user_is_admin(client, userid_hash)
        if isinstance(result, flask.Response):
            return result
        if result is False:
            return flask.Response('Permission denied', status=403)

        locations = []
        query = client.query(kind="User")
        cursor = None
        while True:
            entities_count = 0
            entities = query.fetch(start_cursor=cursor, limit=1000)
            for entity in entities:
                entities_count += 1
                if entity.has_key('geocoded_location'):
                    location = entity['geocoded_location']
                    locations.append(location)

            if entities_count < 1000:
                break
            cursor = entities.next_page_token
        clusters = cluster_points(locations, eps=0.1, min_samples=10, n_jobs=64)
        centers, sizes = compute_centers(clusters, locations)
        results = { 'points': [] }
        if len(centers) != 0:
            results['min_size'] = min(sizes.values())
            results['max_size'] = max(sizes.values())
            p = results['points']
            for label in centers:
                center = centers[label]
                size = sizes[label]
                p.append( ((center[0], center[1]), size))
        return flask.jsonify(results)
    def _upload_post(self):
        """
        Request handler for upload POST requests. Writes accepts files in POST
        request body and saves them to the local file system as
        <self._dir>/<uuid>.<file extension as uploaded>
        Creates datastore record for uploaded files and indicates that they
        have yet to be uploaded to Cloud Storage.
        Returns constants.HTTP_ERROR status if an error occurs with a
            short message.
        Returns constants.HTTP_OK response on success with no message.
        Returns constants.HTTP_OOM status if the server is under too much
            load to handle the request.
        """
        self.logger.info("Upload POST received")
        datastore_client = self.datastore.Client(self.config['PROJECT_ID'])

        # Fetch the user's identifier from the request, which
        # contains the oauth2 creds.
        try:
            token = flask.request.headers['X-IDTOKEN']
        except Exception as e:
            self.logger.error("Missing credential token header")
            return flask.Response('Missing credential token header', 405)
        try:
            upload_session_id = flask.request.headers['X-UPLOADSESSIONID']
        except Exception as e:
            self.logger.error("Missing session ID")
            return flask.Response('Missing session ID', 400)
        try:
            image_bucket = flask.request.headers['X-IMAGE-BUCKET']
        except Exception as e:
            self.logger.error("Missing image bucket")
            return flask.Response('Missing image bucket', 400)

        try:
            cc0_agree = flask.request.headers['X-CC0-AGREE']
            if cc0_agree != 'true':
                raise ValueError('Must accept cc0')
        except Exception as e:
            self.logger.error("Missing CC0 agreement")
            return flask.Response('Missing CC0 agreement', 400)

        try:
            public_agree = flask.request.headers['X-PUBLIC-AGREE']
            if public_agree != 'true':
                raise ValueError('Must accept public database')
        except Exception as e:
            self.logger.error("Missing public dataset agreement")
            return flask.Response('Missing public dataset agreement', 400)

        # TODO(dek): read and update hashlib object using fix-sized buffers to avoid memory blowup
        # Read the content of the upload completely, before returning an error.
        file_ = flask.request.files['file']
        original_filename = file_.filename
        self.logger.info("Reading upload stream")
        content = file_.stream.read()
        self.logger.info("Read upload stream")

        result = flask_users.authn_check(flask.request.headers)
        if isinstance(result, flask.Response):
            self.logger.error("Failed auth check")
            return result
        userid_hash = users.get_userid_hash(result)
        if not users.check_if_user_exists(datastore_client, userid_hash):
            self.logger.error("Failed profile check")
            return self.Response('Profile required to upload images.',
                                 status=400)
        r = roles.get_user_role(datastore_client, userid_hash)

        # Check for a valid bucket.
        valid_bucket = False
        if image_bucket == 'app':
            valid_bucket = True
        elif image_bucket == 'megamovie' or image_bucket == 'volunteer_test':
            valid_bucket = self._check_role_in_roles(
                r, VALID_MEGAMOVIE_UPLOADER_ROLES)
        elif image_bucket == 'teramovie':
            valid_bucket = self._check_role_in_roles(
                r, VALID_TERAMOVIE_UPLOADER_ROLES)
        else:
            # Not a known bucket.
            valid_bucket = False

        if not valid_bucket:
            self.logger.error("Failed bucket check")
            return self.Response(
                'Valid role required to upload images to this bucket, or bucket is unknown',
                status=400)

        content_type = self.request.content_type

        name = hashlib.sha256(content).hexdigest()
        self.logger.info("Received image with digest: %s" % name)
        # Local file system file paths
        local_file = self.os.path.join(self._dir, name)
        temp_file = local_file + self._file_not_ready_suffix
        try:
            open(temp_file, "wb").write(content)
        except IOError as e:
            self.logger.error('Error occured writing to file: {0}'.format(e))
            return self.Response('Failed to save file.',
                                 status=constants.HTTP_ERROR)
        del content

        metadata = _extract_exif_metadata(temp_file)
        result = {}
        if metadata.has_key('lat'):
            result['lat'] = metadata['lat']
        if metadata.has_key('lon'):
            result['lon'] = metadata['lon']

        key = datastore_client.key(self._datastore_kind, name)
        entity = datastore_client.get(key)
        if entity is not None:
            if 'user' in entity:
                if entity['user'] == datastore_client.key(
                        self._user_datastore_kind, userid_hash):
                    entity['upload_session_id'] = upload_session_id
                    datastore_client.put(entity)
            else:
                self.logger.error(
                    'Duplicate detected but incomplete datastore record')
            try:
                os.remove(temp_file)
            except OSError as e:
                self.logger.error('Unable to remove file: {0}'.format(e))
            result['warning'] = 'Duplicate file upload.'
            return flask.jsonify(**result)
        entity = self._create_datastore_entry(
            datastore_client,
            name,
            original_filename,
            user=userid_hash,
            upload_session_id=upload_session_id,
            image_bucket=image_bucket,
            cc0_agree=cc0_agree,
            public_agree=public_agree)
        entity.update(metadata)
        if not entity:
            self.logger.error('Unable to create datastore entry for %s' % name)
            try:
                os.remove(temp_file)
            except OSError as e:
                self.logger.error('Unable to remove file: {0}'.format(e))
            return self.Response('Failed to save file.',
                                 status=constants.HTTP_ERROR)
        try:
            datastore_client.put(entity)
        except Exception as e:
            self.logger.error('Unable to create datastore entry for %s: %s' %
                              (name, str(e)))
            try:
                os.remove(temp_file)
            except OSError as e:
                self.logger.error('Unable to remove file: {0}'.format(e))
            return self.Response('Failed to save file.',
                                 status=constants.HTTP_ERROR)

        try:
            os.rename(temp_file, local_file)
        except Exception as e:
            self.logger.error('Error occured rename file: {0}'.format(e))
            try:
                os.remove(temp_file)
            except OSError as e:
                self.logger.error('Unable to remove file: {0}'.format(e))
            return self.Response('Failed to save file.',
                                 status=constants.HTTP_ERROR)

        return flask.jsonify(**result)
Beispiel #5
0
    def photos(self):
        client = self._get_datastore_client()
        result = flask_users.authn_check(flask.request.headers)
        if isinstance(result, flask.Response):
            return result
        userid_hash = users.get_userid_hash(result)
        if not users.check_if_user_exists(client, userid_hash):
            return flask.Response('User does not exist', status=404)
        result = roles._check_if_user_is_admin(client, userid_hash)
        if isinstance(result, flask.Response):
            return result
        if result is False:
            return flask.Response('Permission denied', status=403)

        filters = []
        filters.append(('confirmed_by_user', '=', True))

        image_buckets = flask.request.args.get('image_buckets', "").split(",")

        locations = []
        query = client.query(kind="Photo", filters=filters)
        cursor = None
        while True:
            entities_count = 0
            entities = query.fetch(start_cursor=cursor, limit=1000)
            for entity in entities:
                entities_count += 1
                if entity.has_key('lat') and entity.has_key('lon'):
                    # Negate longitude because photo longitude is stored as positive-West, but
                    # code needs negative-West.
                    location = entity['lat'], -entity['lon']
                    if len(image_buckets) == 0:
                        should_append = True
                    else:
                        # Check if this bucket is in the list.
                        should_append = entity['image_bucket'] in image_buckets

                    if should_append:
                        locations.append(location)

            if entities_count < 1000:
                break
            cursor = entities.next_page_token

        if len(locations) > 0:
            clusters = cluster_points(locations, eps=0.1, min_samples=10, n_jobs=64)
            centers, sizes = compute_centers(clusters, locations)
        else:
            centers = []
            sizes = []

        results = { 'points': [] }
        if len(centers) != 0:
            results['min_size'] = min(sizes.values())
            results['max_size'] = max(sizes.values())
            p = results['points']
            for label in centers:
                center = centers[label]
                size = sizes[label]
                p.append( ((center[0], center[1]), size) )
        return flask.jsonify(results)
Beispiel #6
0
    def root(self, cursor=None):
        """Returns list of photos subject to filter criteria."""
        client = self._get_datastore_client()

        # Auth check: must be logged in, user profile exists, have
        # admin or reviewer role
        result = flask_users.authn_check(flask.request.headers)
        if isinstance(result, flask.Response):
            return result
        userid_hash = users.get_userid_hash(result)
        if not users.check_if_user_exists(client, userid_hash):
            return flask.Response('User does not exist', status=404)
        result = roles._check_if_user_has_role(
            client, userid_hash, set([roles.ADMIN_ROLE, roles.REVIEWER_ROLE]))
        if isinstance(result, flask.Response):
            return result
        if result is False:
            return flask.Response('Permission denied', status=403)

        # Check if cursor passed in from clinet
        cursor = flask.request.args.get('cursor', None)
        if cursor is not None:
            cursor = str(cursor)

        # Filter, and order, items.
        order = []
        filters = []
        num_reviews_max = flask.request.args.get('num_reviews_max', None)
        if num_reviews_max is not None:
            filters.append(('num_reviews', '<=', int(num_reviews_max)))
            order.append('num_reviews')
        mask_reviewer = flask.request.args.get('mask_reviewer', False)
        image_bucket = flask.request.args.get('image_bucket', None)
        if image_bucket is not None:
            filters.append(('image_bucket', '=', image_bucket))
            order.append('image_bucket')
        user_id = flask.request.args.get('user_id', None)
        if user_id is not None:
            filters.append(('user', '=', client.key("User", user_id)))
            order.append('user')
        upload_session_id = flask.request.args.get('upload_session_id', None)
        if upload_session_id is not None:
            filters.append(('upload_session_id', '=', upload_session_id))
            order.append('upload_session_id')

        if flask.request.args.get('image_datetime_begin'):
            image_datetime_begin = datetime.datetime.fromtimestamp(
                float(flask.request.args['image_datetime_begin']))
            filters.append(('image_datetime', '>', image_datetime_begin))
            order.append('image_datetime')
        if flask.request.args.get('image_datetime_end'):
            image_datetime_end = datetime.datetime.fromtimestamp(
                float(flask.request.args['image_datetime_end']))
            filters.append(('image_datetime', '<=', image_datetime_end))
            if 'image_datetime' not in order:
                order.append('image_datetime')
        if flask.request.args.get('uploaded_date_begin'):
            uploaded_date_begin = datetime.datetime.fromtimestamp(
                float(flask.request.args['uploaded_date_begin']))
            filters.append(('uploaded_date', '>', uploaded_date_begin))
            order.append('uploaded_date')
        if flask.request.args.get('uploaded_date_end'):
            uploaded_date_end = datetime.datetime.fromtimestamp(
                float(flask.request.args['uploaded_date_end']))
            filters.append(('uploaded_date', '<=', uploaded_date_end))
            if 'uploaded_date' not in order:
                order.append('uploaded_date')

        # TODO(dek): randomize results(?)

        # Fetch start_cursor to limit entites
        query = client.query(kind="Photo", filters=filters)
        query.order = order
        limit = int(flask.request.args.get('limit', 100))
        entities = query.fetch(start_cursor=cursor, limit=limit)
        page = next(entities.pages)
        # Fetch results before getting next page token
        e = list(page)
        next_cursor = entities.next_page_token

        if e is None:
            return flask.Response('No entities.', 200)
        # Return matching photos by ID
        photos = []
        client = storage.client.Client(project=config.PROJECT_ID)
        bucket = client.bucket(config.GCS_BUCKET)
        for entity in e:
            blob = storage.Blob(entity.key.name, bucket)
            url = blob.public_url
            if 'image_type' not in entity:
                logging.error("Photo %s does not have an image type." %
                              entity.key.name)
                continue
            include_photo = True
            location = None
            if 'lat' in entity and 'lon' in entity:
                location = {"lat": entity['lat'], "lon": entity['lon']}

            if entity['image_type'] == 'raw' or entity['image_type'] == 'TIFF':
                # If the image isn't a JPG, the browser won't show it.
                # Make the URL point at the JPG copy
                url = url + '.jpg'
            if mask_reviewer and entity.has_key("reviews"):
                for review in entity["reviews"]:
                    # Don't consider photos where we satisfy the
                    # num_reviews inequality, but one of the reviewers
                    # is ourself
                    if review['user_id'] == userid_hash:
                        include_photo = False
                        break

            if include_photo:
                photo = {'name': entity.key.name, 'url': url}
                if location is not None:
                    photo['location'] = location
                photos.append(photo)
        results = {}
        if len(photos):
            results['cursor'] = next_cursor
        results['photos'] = photos
        return flask.jsonify(results)
Beispiel #7
0
    def confirm(self):
        """For each photo from upload_session_id that matches the files from original_filenames, mark it confirmed_by_user=True"""
        client = self._get_datastore_client()
        # Auth check: must be logged in, user profile exists, have
        # admin or volunteer role
        result = flask_users.authn_check(flask.request.headers)
        if isinstance(result, flask.Response):
            return result
        userid_hash = users.get_userid_hash(result)
        if not users.check_if_user_exists(client, userid_hash):
            return flask.Response('User does not exist', status=404)
        if flask.request.headers.has_key(
                "X-IDEUM-APP-SECRET"
        ) and flask.request.headers["X-IDEUM-APP-SECRET"] == IDEUM_APP_SECRET:
            logging.info("Request contains Ideum app secret.")
        else:
            result = roles._check_if_user_has_role(
                client, userid_hash,
                set([roles.USER_ROLE, roles.ADMIN_ROLE, roles.VOLUNTEER_ROLE]))
            if isinstance(result, flask.Response):
                return result
            if result is False:
                return flask.Response('Permission denied', status=403)

        json = flask.request.get_json()
        if 'upload_session_id' not in json:
            return flask.Response('upload_session_id required', status=403)
        upload_session_id = json['upload_session_id']

        if 'filenames' not in json:
            return flask.Response('filenames required', status=403)
        filenames = json['filenames']
        anonymous_photo = json.get('anonymous_photo', False)
        equatorial_mount = json.get('equatorial_mount', False)

        filters = [('upload_session_id', '=', upload_session_id)]
        query = client.query(kind="Photo", filters=filters)
        entities = list(query.fetch())
        results = repair_missing_gps.partition_gps(entities)
        complete_images, incomplete_images = results
        all_entities = complete_images[:]
        # If there's one complete image and more than one incomplete images,
        # repair the incomplete images using the complete image
        logging.info("Received %d complete images" % len(complete_images))
        logging.info("Received %d incomplete images" % len(incomplete_images))
        if len(incomplete_images) > 0:
            logging.info("Repairing incomplete images: %s" %
                         str(incomplete_images))
            if complete_images == []:
                complete_image = None
            else:
                complete_image = pick_image(complete_images)
            logging.info("Chose %s as complete image" % complete_image)
            for incomplete_image in incomplete_images:
                repaired_image = repair_missing_gps.update_incomplete(
                    complete_image, incomplete_image)
                all_entities.append(repaired_image)
            logging.info("Repaired: %s" % ([
                incomplete_image.key.name
                for incomplete_image in incomplete_images
            ]))
            if complete_image is not None:
                logging.info("Repair source: %s", complete_image.key.name)
        else:
            for incomplete_image in incomplete_images:
                all_entities.append(incomplete_image)
            logging.info("No repairs for: %s" % [
                incomplete_image.key.name
                for incomplete_image in incomplete_images
            ])

        for entity_chunk in chunks(all_entities, 500):
            batch = client.batch()
            batch.begin()
            for entity in entity_chunk:
                if entity.has_key(u'original_filename') and entity[
                        u'original_filename'] in filenames:
                    entity['confirmed_by_user'] = True
                    entity['anonymous_photo'] = anonymous_photo
                    entity['equatorial_mount'] = equatorial_mount
                    batch.put(entity)

            batch.commit()
        return flask.Response('OK', status=200)
Beispiel #8
0
    def photo(self, photo_id):
        """GET returns photo entity, PATCH modifies photo entity."""
        client = self._get_datastore_client()

        # Auth check: must be logged in, user profile exists, have
        # admin or reviewer role
        result = flask_users.authn_check(flask.request.headers)
        if isinstance(result, flask.Response):
            return result
        userid_hash = users.get_userid_hash(result)
        if not users.check_if_user_exists(client, userid_hash):
            return flask.Response('User does not exist', status=404)
        result = roles._check_if_user_has_role(
            client, userid_hash, set([roles.ADMIN_ROLE, roles.REVIEWER_ROLE]))
        if isinstance(result, flask.Response):
            return result
        if result is False:
            return flask.Response('Permission denied', status=403)

        key = client.key("Photo", str(photo_id))

        # Dispatch on method
        if self.request.method == 'GET':
            # if the photo_id exists, return the approved fields
            result = {}
            entity = client.get(key)
            for field in FIELDS:
                if field not in entity:
                    return flask.Response(
                        'Invalid photo (missing field %s)' % field, 404)
                result[field] = entity[field]
            result['reviews'] = []
            for review in entity['reviews']:
                reviewer = review['user_id'].name
                vote = review['vote']
                result['reviews'].append((reviewer, vote))
            result['user'] = entity['user'].name
            return flask.jsonify(result)
        elif flask.request.method == 'PATCH':
            # if the photo_id exists, update it
            result = util._validate_json(flask.request)
            if result is not True:
                return result
            json = flask.request.get_json()
            json = util._escape_json(json)
            if 'vote' not in json:
                return flask.Response('Missing vote', 400)
            vote = json['vote']
            if vote != "up" and vote != "down":
                return flask.Response('Invalid vote, use "up" or "down"', 400)

            with client.transaction():
                # Fetch the existing Photo entity
                entity = client.get(key)
                review = datastore.Entity(key=client.key('Review'))
                review['user_id'] = userid_hash
                review['vote'] = vote
                # Update the reviews property
                if 'reviews' not in entity:
                    # Case: no existing reviews, create first
                    entity["reviews"] = [review]
                else:
                    for r in entity["reviews"]:
                        reviewer_key = r['user_id']
                        # Case: existing review by user, edit vote
                        if reviewer_key == userid_hash:
                            r['vote'] = vote
                        else:
                            # Case: existing review by new user, append
                            entity["reviews"].append(review)
                entity["num_reviews"] = len(entity["reviews"])
                try:
                    client.put(entity)
                except Exception as e:
                    logging.error("Datastore update operation failed: %s" %
                                  str(e))
                    flask.Response('Internal server error', status=500)
            return flask.Response('OK', status=200)
Beispiel #9
0
    def root(self):
        client = self._get_datastore_client()
        result = flask_users.authn_check(flask.request.headers)
        if isinstance(result, flask.Response):
            return result
        userid_hash = users.get_userid_hash(result)
        if not users.check_if_user_exists(client, userid_hash):
            return flask.Response('User does not exist', status=404)
        result = roles._check_if_user_is_admin(client, userid_hash)
        if isinstance(result, flask.Response):
            return result
        if result is False:
            return flask.Response('Permission denied', status=403)

        filters = []
        image_bucket = flask.request.args.get('image_bucket', None)
        if image_bucket is not None:
            filters.append(('image_bucket', '=', image_bucket))

        filters.append(('confirmed_by_user', '=', True))

        total_count = 0
        gps_count = 0
        type_count = {
            'app': {
                'total_count': 0,
                'gps_count': 0
            },
            'volunteer_test': {
                'total_count': 0,
                'gps_count': 0
            },
            'megamovie': {
                'total_count': 0,
                'gps_count': 0
            },
            'teramovie': {
                'total_count': 0,
                'gps_count': 0
            }
        }
        query = client.query(kind="Photo", filters=filters)
        cursor = None
        while True:
            entities_count = 0
            entities = query.fetch(start_cursor=cursor, limit=1000)
            for entity in entities:
                entities_count += 1
                image_bucket = entity['image_bucket']
                type_counter = type_count.get(image_bucket, None)
                if type_counter is not None:
                    type_counter['total_count'] += 1

                if entity.has_key('lat') and entity.has_key('lon'):
                    if type_counter is not None:
                        type_counter['gps_count'] += 1
                    gps_count += 1

            total_count += entities_count
            if entities_count < 1000:
                break
            cursor = entities.next_page_token

        photo_stats = {
            'total_count': total_count,
            'gps_count': gps_count,
            'types': type_count
        }
        s = flask.jsonify(photo_stats)
        return s