def download(self, id, filename=None): ''' Provide a download by either redirecting the user to the url stored or downloading the uploaded file from S3. ''' if not self.use_filename or filename is None: filename = os.path.basename(self.url) filename = munge.munge_filename(filename) key_path = self.get_path(id, filename) key = filename if key is None: log.warning("Key '%s' not found in bucket '%s'", key_path, self.bucket_name) try: url = self.get_signed_url_to_key(key_path) h.redirect_to(url) except ClientError as ex: if ex.response['Error']['Code'] in ['NoSuchKey', '404']: # attempt fallback default_resource_upload = DefaultResourceUpload(self.resource) return default_resource_upload.download(id, self.filename) else: # Controller will raise 404 for us raise OSError(errno.ENOENT)
def _handle_files(self, action, data_dict, context, files): if action not in ['resource_create', 'resource_update']: raise CKANAPIError("LocalCKAN.call_action only supports file uploads for resources.") new_data_dict = dict(data_dict) if action == 'resource_create': if 'url' not in new_data_dict or new_data_dict['url']: new_data_dict['url'] = '/tmp-file' # url needs to be set, otherwise there is a ValidationError resource = self._get_action(action)(dict(context), new_data_dict) else: resource = new_data_dict from ckan.lib.uploader import ResourceUpload resource_upload = ResourceUpload({'id': resource['id']}) # get first upload, ignore key source_file = files.values()[0] if not resource_upload.storage_path: raise CKANAPIError("No storage configured, unable to upload files") directory = resource_upload.get_directory(resource['id']) filepath = resource_upload.get_path(resource['id']) try: os.makedirs(directory) except OSError, e: ## errno 17 is file already exists if e.errno != 17: raise
def filesystem_resource_download(self, id, resource_id, filename=None): """ A fallback controller action to download resources from the filesystem. A copy of the action from `ckan.controllers.package:PackageController.resource_download`. Provide a direct download by either redirecting the user to the url stored or downloading an uploaded file directly. """ context = { 'model': model, 'session': model.Session, 'user': c.user or c.author, 'auth_user_obj': c.userobj } try: rsc = get_action('resource_show')(context, {'id': resource_id}) get_action('package_show')(context, {'id': id}) except NotFound: abort(404, _('Resource not found')) except NotAuthorized: abort(401, _('Unauthorised to read resource %s') % resource_id) if rsc.get('url_type') == 'upload': upload = DefaultResourceUpload(rsc) try: return upload.download(rsc['id'], filename) except OSError: # includes FileNotFoundError abort(404, _('Resource data not found')) elif 'url' not in rsc: abort(404, _('No download is available')) redirect(rsc['url'])
def get_local_upload_path(resource_id): u''' Returns the local path to an uploaded file give an id Note: it does not check if the resource or file actually exists ''' upload = ResourceUpload({u'url': u'foo'}) return upload.get_path(resource_id)
def filesystem_resource_download(package_type, id, resource_id, filename=None): """ A fallback view action to download resources from the filesystem. A copy of the action from `ckan.views.resource:download`. Provide a direct download by either redirecting the user to the url stored or downloading an uploaded file directly. """ context = { u'model': model, u'session': model.Session, u'user': g.user, u'auth_user_obj': g.userobj } preview = request.args.get(u'preview', False) try: rsc = get_action(u'resource_show')(context, {u'id': resource_id}) get_action(u'package_show')(context, {u'id': id}) except NotFound: return abort(404, _(u'Resource not found')) except NotAuthorized: return abort(401, _('Unauthorised to read resource %s') % resource_id) mimetype, enc = mimetypes.guess_type(rsc.get('url', '')) if rsc.get(u'url_type') == u'upload': if hasattr(DefaultResourceUpload, 'download'): upload = DefaultResourceUpload(rsc) try: return upload.download(rsc['id'], filename) except OSError: # includes FileNotFoundError return abort(404, _('Resource data not found')) path = get_storage_path() storage_path = os.path.join(path, 'resources') directory = os.path.join(storage_path, resource_id[0:3], resource_id[3:6]) filepath = os.path.join(directory, resource_id[6:]) if preview: return flask.send_file(filepath, mimetype=mimetype) else: return flask.send_file(filepath) elif u'url' not in rsc: return abort(404, _(u'No download is available')) return redirect(rsc[u'url'])
def metadata(self, id, filename=None): if filename is None: filename = os.path.basename(self.url) filename = munge.munge_filename(filename) key_path = self.get_path(id, filename) key = filename if key is None: log.warning("Key '%s' not found in bucket '%s'", key_path, self.bucket_name) try: # Small workaround to manage downloading of large files # We are using redirect to minio's resource public URL client = self.get_s3_client() metadata = client.head_object(Bucket=self.bucket_name, Key=key_path) metadata['content_type'] = metadata['ContentType'] # Drop non public metadata metadata.pop('ServerSideEncryption', None) metadata.pop('SSECustomerAlgorithm', None) metadata.pop('SSECustomerKeyMD5', None) metadata.pop('SSEKMSKeyId', None) metadata.pop('StorageClass', None) metadata.pop('RequestCharged', None) metadata.pop('ReplicationStatus', None) metadata.pop('ObjectLockLegalHoldStatus', None) metadata['size'] = metadata['ContentLength'] metadata['hash'] = metadata['ETag'] return self.as_clean_dict(metadata) except ClientError as ex: if ex.response['Error']['Code'] in ['NoSuchKey', '404']: if config.get( 'ckanext.s3filestore.filesystem_download_fallback', False): log.info('Attempting filesystem fallback for resource %s', id) default_resource_upload = DefaultResourceUpload( self.resource) return default_resource_upload.metadata(id) #Uploader interface does not know about s3 errors raise OSError(errno.ENOENT)
def migrate(self): ''' Migrate filestore over in our very HDXish way :) ''' results = Session.execute( "select id, revision_id, url from resource " "where resource_type = 'file.upload' " "and (url_type <> 'upload' or url_type is null)" "and url like '%storage%'") for id, revision_id, url in results: # Give it a second, would you? time.sleep(0.7) url_parts = urlparse(url) url_parts = url_parts.path.split("/") filename = url_parts[len(url_parts) - 1] response = requests.get(url, stream=True) if response.status_code != 200: print "failed to fetch %s (code %s)" % (url, response.status_code) continue resource_upload = ResourceUpload({'id': id}) assert resource_upload.storage_path, "no storage configured aborting" directory = resource_upload.get_directory(id) filepath = resource_upload.get_path(id) try: os.makedirs(directory) except OSError, e: ## errno 17 is file already exists if e.errno != 17: raise with open(filepath, 'wb+') as out: for chunk in response.iter_content(1024): if chunk: out.write(chunk) Session.execute("update resource set url_type = 'upload', " "url = '%s' where id = '%s'" % (filename, id)) Session.execute( "update resource_revision set url_type = 'upload', " "url = '%s' where id = '%s' and " "revision_id = '%s'" % (filename, id, revision_id)) Session.commit() print "Saved url %s" % url
def test_resource_without_upload(self, mock_uploads_enabled, mock_open, send_file): # this test data is based on real observation using a browser res = { u'clear_upload': u'true', u'format': u'PNG', u'url': u'https://example.com/data.csv', u'description': u'', u'upload': FileStorage(filename=u''), u'package_id': u'dataset1', u'id': u'8a3a874e-5ee1-4e43-bdaf-e2569cf72344', u'name': u'data.csv' } res_upload = ResourceUpload(res) eq_(res_upload.filename, None)
def test_resource_without_upload(self, ckan_config, monkeypatch, tmpdir): monkeypatch.setitem(ckan_config, u'ckan.storage_path', str(tmpdir)) # this test data is based on real observation using a browser res = { u'clear_upload': u'true', u'format': u'PNG', u'url': u'https://example.com/data.csv', u'description': u'', u'upload': FileStorage(filename=u''), u'package_id': u'dataset1', u'id': u'8a3a874e-5ee1-4e43-bdaf-e2569cf72344', u'name': u'data.csv' } res_upload = ResourceUpload(res) assert res_upload.filename is None
def get_resource_uploader(self, data_dict): return ResourceUpload(data_dict)