Beispiel #1
0
    async def delete(self, uid, variation_name):
        """Remove the variation from the asset"""

        asset = self.get_asset(uid,
                               projection={
                                   'expires': True,
                                   'name': True,
                                   'secure': True,
                                   'uid': True,
                                   f'variations.{variation_name}': {
                                       '$sub': Variation
                                   }
                               })

        variation = self.get_variation(asset, variation_name)

        # Remove file for the variation
        backend = self.get_backend(asset.secure)

        await backend.async_delete(variation.get_store_key(
            asset, variation_name),
                                   loop=asyncio.get_event_loop())

        # Remove the variation from the asset
        Asset.get_collection().update(
            (Q._id == asset._id).to_dict(),
            {'$unset': {
                f'variations.{variation_name}': ''
            }})

        # Update the asset stats
        Stats.inc(self.account, today_tz(tz=self.config['TIMEZONE']), {
            'variations': -1,
            'length': -variation.meta['length']
        })

        self.set_status(204)
        self.finish()
Beispiel #2
0
def get_highlights(state):

    # Get the stats for the account
    api_call_keys = Stats.get_key_range(today_tz(), 'days', 7, 'api_calls')

    state.has_stats = False
    stats = Stats.one(Q.scope == state.account,
                      projection={
                          'values.all.assets': True,
                          'values.all.length': True,
                          'values.all.variations': True,
                          **{f'values.{k}': True
                             for k in api_call_keys}
                      })

    if stats:
        state.has_stats = True
        state.highlights = {
            'api_calls': stats.sum_stats(api_call_keys),
            'assets': stats.get_stat('all.assets'),
            'length': stats.get_stat('all.length'),
            'variations': stats.get_stat('all.variations')
        }
Beispiel #3
0
def get_activity(state):

    # Determine the period to view
    unit = 'days'
    length = 90
    state.period = '90d'

    if request.args.get('period') == '12m':
        unit = 'months'
        length = 12
        state.period = '12m'

    elif request.args.get('period') == '10y':
        unit = 'years'
        length = 10
        state.period = '10y'

    # Build the keys required for the period
    today = today_tz()
    api_call_keys = Stats.get_key_range(today, unit, length, 'api_calls')
    asset_keys = Stats.get_key_range(today, unit, length, 'assets')
    length_keys = Stats.get_key_range(today, unit, length, 'length')
    variations_keys = Stats.get_key_range(today, unit, length, 'variations')

    # Get the stats for the account
    state.has_stats = False
    stats = Stats.one(Q.scope == 'all',
                      projection={
                          **{f'values.{k}': True
                             for k in api_call_keys},
                          **{f'values.{k}': True
                             for k in asset_keys},
                          **{f'values.{k}': True
                             for k in length_keys},
                          **{f'values.{k}': True
                             for k in variations_keys}
                      })

    if stats:
        state.has_stats = True

        # Build the period totals
        state.totals = {
            'api_calls': stats.sum_stats(api_call_keys),
            'assets': stats.sum_stats(asset_keys),
            'length': stats.sum_stats(length_keys),
            'variations': stats.sum_stats(variations_keys)
        }

        # Build the chart data
        color = {
            'backgroundColor': '#67B3DA64',
            'borderColor': '#67B3DA',
            'pointBackgroundColor': '#67B3DA64',
            'pointBorderColor': '#67B3DA'
        }

        state.data_series = {
            'api_calls': {
                'datasets': [{
                    'data': stats.get_series(api_call_keys),
                    **color
                }],
                'labels': stats.get_series_labels(api_call_keys)
            },
            'assets': {
                'datasets': [{
                    'data': stats.get_series(asset_keys),
                    **color
                }],
                'labels': stats.get_series_labels(asset_keys)
            },
            'length': {
                'datasets': [{
                    'data': [
                        round(v / 1000000, 2)
                        for v in stats.get_series(length_keys)
                    ],
                    **color
                }],
                'labels':
                stats.get_series_labels(length_keys)
            },
            'variations': {
                'datasets': [{
                    'data': stats.get_series(variations_keys),
                    **color
                }],
                'labels': stats.get_series_labels(variations_keys)
            }
        }
Beispiel #4
0
    def _store_variation(
        self,
        config,
        asset,
        variation_name,
        versioned,
        ext,
        meta,
        file
    ):
        """
        Store a new variation of the asset. This method both stores the
        variation (and removes any existing variation with the same name) as
        well as updating the asset's `variations` field with details of the
        new variation.
        """

        # Get the account and backend associated with the asset
        account = Account.by_id(
            asset.account,
            projection={
                'public_backend_settings': True,
                'secure_backend_settings': True
            }
        )

        if asset.secure:
            backend = account.secure_backend
        else:
            backend = account.public_backend

        assert backend, 'No backend configured for the asset'

        # Ensure the asset's variation value is a dictionary (in case it's
        # never been set before).
        if not asset.variations:
            asset.variations = {}

        # Create the new variation
        old_variation = asset.variations.get(variation_name)

        new_variation = Variation(
            content_type=mimetypes.guess_type(f'f.{ext}')[0] if ext else '',
            ext=ext,
            meta=meta,
            version=(
                Variation.next_version(
                    old_variation.version if old_variation else None
                )
                if versioned else None
            )
        )

        # Store the new variation
        new_store_key = new_variation.get_store_key(asset, variation_name)
        file.seek(0)
        backend.store(file, new_store_key)

        # Add the new variation's details to the asset's `variations` field
        asset.variations[variation_name] = new_variation
        asset.modified = datetime.utcnow()

        # Apply the updated to the database
        Asset.get_collection().update(
            (Q._id == asset._id).to_dict(),
            {
                '$set': {
                    f'variations.{variation_name}': \
                            new_variation.to_json_type(),
                    'modified': asset.modified
                }
            }
        )

        # Remove the existing variation
        if old_variation:
            old_store_key = old_variation.get_store_key(asset, variation_name)
            if new_store_key != old_store_key:
                backend.delete(old_store_key)

        # Update the asset stats
        new_length = new_variation.meta['length']
        old_length = 0
        if old_variation:
            old_length = old_variation.meta['length']

        Stats.inc(
            account,
            today_tz(tz=config['TIMEZONE']),
            {
                'variations': 0 if old_variation else 1,
                'length': new_length - old_length
            }
        )
Beispiel #5
0
    async def put(self):
        """Store the uploaded file as an asset"""

        # Make sure a file was received
        files = self.request.files.get('file')
        if not files:
            raise APIError(
                'invalid_request',
                arg_errors={'file': ['No file received.']}
            )
        file = files[0]

        # Validate the arguments
        form = PutForm(to_multi_dict(self.request.body_arguments))
        if not form.validate():
            raise APIError(
                'invalid_request',
                arg_errors=form.errors
            )

        if self.config['ANTI_VIRUS_ENABLED']:

            # Check the file for viruses
            av_client = clamd.ClamdUnixSocket(
                self.config['ANTI_VIRUS_CLAMD_PATH']
            )
            av_scan_result = av_client.instream(io.BytesIO(file.body))

            if av_scan_result['stream'][0] == 'FOUND':
                raise APIError(
                    'invalid_request',
                    arg_errors={
                        'file': ['File appears to be a virus.']
                    }
                )

        form_data = form.data

        # Create a name for the asset
        fname, fext = os.path.splitext(file.filename)
        name = slugify(
            form_data['name'] or fname,
            regex_pattern=ALLOWED_SLUGIFY_CHARACTERS,
            max_length=200
        )

        # Determine the files extension
        ext = fext[1:] if fext else imghdr.what(file.filename, file.body)

        # Determine the asset type/content type for the image
        content_type = mimetypes.guess_type(f'f.{ext}')[0] \
                or 'application/octet-stream'
        asset_type = self.config['CONTENT_TYPE_TO_TYPES'].get(
            content_type,
            'file'
        )

        # Build the meta data for the asset
        meta = {
            'filename': file.filename,
            'length': len(file.body)
        }

        if asset_type == 'audio':
            try:
                au_file = io.BytesIO(file.body)
                au_file.name = file.filename
                au = mutagen.File(au_file)

            except:
                raise APIError(
                    'invalid_request',
                    arg_errors={
                        'file': ['Unable to open the file as an audio file.']
                    }
                )

            if au is not None:
                meta['audio'] = {
                    'channels': getattr(au.info, 'channels', -1),
                    'length': getattr(au.info, 'length', -1),
                    'mode': {
                        0: 'stereo',
                        1: 'joint_stereo',
                        2: 'dual_channel',
                        3: 'mono'
                    }.get(getattr(au.info, 'mode', ''), ''),
                    'sample_rate': getattr(au.info, 'sample_rate', -1)
                }

        if asset_type == 'image':

            im = None

            try:
                im = Image.open(io.BytesIO(file.body))
                meta['image'] = {
                    'mode': im.mode,
                    'size': im.size
                }

            except:
                raise APIError(
                    'invalid_request',
                    arg_errors={
                        'file': ['Unable to open the file as an image.']
                    }
                )

            finally:
                if im:
                    im.close()

        # Create the asset
        asset = Asset(
            uid=Asset.generate_uid(),
            account=self.account,
            secure=form_data['secure'],
            name=name,
            ext=ext,
            type=asset_type,
            content_type=content_type,
            expires=(time.time() + form_data['expire'])
                    if form_data['expire'] else None,
            meta=meta
        )

        # Store the file
        backend = self.get_backend(asset.secure)

        await backend.async_store(
            io.BytesIO(file.body),
            asset.store_key,
            loop=asyncio.get_event_loop()
        )

        # Save the asset
        asset.insert()

        # Update the asset stats
        Stats.inc(
            self.account,
            today_tz(tz=self.config['TIMEZONE']),
            {
                'assets': 1,
                'length': asset.meta['length']
            }
        )

        self.write(asset.to_json_type())
Beispiel #6
0
    async def prepare(self):

        from blueprints.accounts.models import Account, Stats

        # (Re)Set the account attribute
        self._account = None

        # (Re)Set the write log
        self._write_log = ''

        # (Re)Set the call timer
        self._call_timer = time.time()

        # Check a valid API key has been provided
        api_key = self.request.headers.get('X-H51-APIKey')

        if not api_key:
            raise APIError('unauthorized', 'No authorization key provided.')

        # Find the calling account
        account = Account.one(
            Q.api_key == api_key,
            projection={
                'api_allowed_ip_addresses': True,
                'api_rate_limit_per_second': True
            }
        )

        # Store a reference to the account document against the request handler
        self._account = account

        if not account:
            raise APIError('unauthorized', 'API key not recognized.')

        if account.api_allowed_ip_addresses:

            # Check the caller's IP address is allowed
            ip_address = self.request.headers.get('X-Real-Ip', '')
            if ip_address not in account.api_allowed_ip_addresses:
                raise APIError(
                    'forbidden',
                    (
                        f'The IP address {ip_address} is not allowed to call '
                        'the API for this account.'
                    )
                )

        # Record this request
        rate_key = account.get_rate_key()
        ttl = await self.redis.pttl(rate_key)

        if ttl > 0:
            await self.redis.incr(rate_key)

        else:
            multi = self.redis.multi_exec()
            multi.incr(rate_key)
            multi.expire(rate_key, 1)
            await multi.execute()

        # Apply rate limit
        request_count = int((await self.redis.get(rate_key)) or 0)
        rate_limit = account.api_rate_limit_per_second \
                or self.config['API_RATE_LIMIT_PER_SECOND']

        if request_count > rate_limit:
            raise APIError('request_limit_exceeded')

        # Set the remaining requests allowed this second in the response
        # headers.
        rate_key_reset = max(0, ttl / 1000.0)
        rate_key_reset += time.time()

        self.set_header('X-H51-RateLimit-Limit', str(rate_limit))
        self.set_header(
            'X-H51-RateLimit-Remaining',
            str(rate_limit - request_count)
        )
        self.set_header('X-H51-RateLimit-Reset', str(rate_key_reset))

        # Update the API call stats
        Stats.inc(
            account,
            today_tz(tz=self.config['TIMEZONE']),
            {'api_calls': 1}
        )
Beispiel #7
0
def purge():
    """Purge assets that have expired"""

    # Get expired assets
    now = time.time()
    max_delete_period = 48 * 60 * 60

    assets = Asset.many(And(
        Q.expires <= now,
        Q.expires > now - max_delete_period,
    ),
                        projection={
                            'expires': True,
                            'ext': True,
                            'meta.length': True,
                            'name': True,
                            'secure': True,
                            'uid': True,
                            'variations': {
                                '$sub.': Variation
                            },
                            'account': {
                                '$ref': Account,
                                'public_backend_settings': True,
                                'secure_backend_settings': True
                            }
                        })

    # Delete the assets
    for asset in assets:

        variation_count = 0
        length = asset.meta['length']

        # Remove all stored files for the asset
        if asset.secure:
            backend = asset.account.secure_backend
        else:
            backend = asset.account.public_backend

        if backend:

            if asset.variations:

                # Count the variations for the asset
                variation_count = len(asset.variations)

                for variation_name, variation in asset.variations.items():

                    # Tally up the assets total footprint
                    length += variation.meta['length']

                    # Remove variation files
                    backend.delete(
                        variation.get_store_key(asset, variation_name))

            # Remove the asset file
            backend.delete(asset.store_key)

        # Delete the asset from the database
        asset.delete()

        # Update the asset stats
        Stats.inc(asset.account, today_tz(), {
            'assets': -1,
            'variations': -variation_count,
            'length': -length
        })