def get_url(self, fieldname, transcoded=False, expiry_seconds=60): """ Get URL of fieldname from S3 :param fieldname: Name of the field on the adapted object :type fieldname: str :param transcoded: Whether or not to attempt to get a transcoded version of a video :return: URL on S3 of given field's content :rtype: str """ storage = self._getStorage() if fieldname not in storage['cloud_available']: logger.warn('%s on %s not available in the cloud', fieldname, self.context.absolute_url()) return None fieldinfo = self.field_info(fieldname) if not fieldinfo: logger.error('Field info for %s missing from context %s', fieldname, self.context.absolute_url()) return bucket_name = 'netsight-cloudstorage-%s' % get_value_from_registry( 'bucket_name' ) s3 = self._get_s3_connection() response_headers = {} key = None if transcoded: transcoded_bucket_name = '%s-transcoded' % bucket_name bucket = s3.lookup(transcoded_bucket_name) if bucket is not None: # Check if it is available in the transcoded bucket key = bucket.get_key( '%s-%s' % (fieldname, self.context.UID()) ) else: logger.warn('Transcode bucket does not exist %s', transcoded_bucket_name) if key is None: bucket = s3.lookup(bucket_name) if bucket is not None: key = bucket.get_key( '%s-%s' % (fieldname, self.context.UID()) ) if key is None: logger.error('File not found: %s on %s', fieldname, self.context.absolute_url()) return else: logger.error('Bucket %s does not exist', bucket_name) return if not transcoded: response_headers = { 'response-content-disposition': 'attachment; filename="%s"' % fieldinfo['filename'], 'response-content-type': fieldinfo['mimetype'] } return key.generate_url(expiry_seconds, response_headers=response_headers)
def _get_bucket(self, postfix=None): """Look up the a bucket set in the registry""" s3 = self._get_s3_connection() bucket_name = 'netsight-cloudstorage-{reg_value}'.format( reg_value=get_value_from_registry('bucket_name') ) if postfix: bucket_name += postfix bucket = s3.lookup(bucket_name) if bucket is None: logger.warn('Bucket does not exist %s', bucket_name) return bucket
def _get_s3_connection(self): """Set up an S3 connection using the registry settings""" aws_key = get_value_from_registry('aws_access_key') aws_secret_key = get_value_from_registry('aws_secret_access_key') return S3Connection(aws_key, aws_secret_key)
def enqueue(self, enforce_file_size=True): """ Dispatch any relevant file fields off to Celery :param enforce_file_size: Allow manually uploading files smaller than the configured minimum :type enforce_file_size: bool """ logger.info('Enqueue called for %s' % self.context.absolute_url()) in_progress = self._getStorage()['in_progress'] cloud_available = self._getStorage()['cloud_available'] for field in self._getFields(): if not field['size'] > 0: # Ignore empty fields continue # Remove existing cloud info, assuming file data has changed if field['name'] in cloud_available: del cloud_available[field['name']] min_size = get_value_from_registry('min_file_size') if field['size'] < min_size * 1024 * 1024 and enforce_file_size: logger.info('Field %s on %s is too small (< %sMB)', field['name'], self.context.absolute_url(), min_size) continue # unique token for this job security_token = uuid4().hex in_progress[field['name']] = security_token # make sure storage token is stored before # job goes on queue transaction.commit() path = '/'.join(self.context.getPhysicalPath()) plone_url = get_value_from_config('plone_url') root_url = '%s/%s' % (plone_url, path) logger.info('Queuing field %s on %s to be uploaded', field['name'], self.context.absolute_url()) source_url = '%s/@@cloudstorage-retrieve' % root_url callback_url = '%s/@@cloudstorage-callback' % root_url errorback_url = '%s/@@cloudstorage-error' % root_url bucket_name = 'netsight-cloudstorage-%s' % get_value_from_registry( 'bucket_name' ) aws_key = get_value_from_registry('aws_access_key') aws_secret_key = get_value_from_registry('aws_secret_access_key') pipeline_name = get_value_from_registry('pipeline_name') upload_task = upload_to_s3.s( bucket_name=bucket_name, source_url=source_url, callback_url=callback_url, errorback_url=errorback_url, field=field, security_token=security_token, aws_key=aws_key, aws_secret_key=aws_secret_key, pipeline_name=pipeline_name, ) logger.info('File mimetype: %s', field['mimetype']) transcoding_enabled = get_value_from_registry( 'transcoding_enabled' ) if transcoding_enabled and field['mimetype'].startswith('video'): links = group(upload_callback.s(), transcode_video.s(), transcode_callback.s()) else: links = group(upload_callback.s()) upload_task.link(links) upload_task.apply_async()