def resource_download_corrected_data(self, id, resource_id): """ Provides a direct download by either redirecting the user to the url stored or downloading an uploaded file directly. """ context = {'model': model, 'session': model.Session, 'user': c.user or c.author, 'auth_user_obj': c.userobj} result = p.toolkit.get_action('get_file_path')(context, {'resourceId': resource_id, 'suffix': '_CorrectedData'}) filePath = result.get('path', None) if os.path.isfile(filePath): #upload = uploader.ResourceUpload(rsc) #filepath = upload.get_path(rsc['id']) fileapp = paste.fileapp.FileApp(filePath) try: status, headers, app_iter = request.call_application(fileapp) except OSError: abort(404, _('Resource data not found')) response.headers.update(dict(headers)) #content_type, content_enc = mimetypes.guess_type(rsc.get('url','')) #if content_type: #It's CSV, because this method can be only access when a usgin model doesn't validate a CSV File response.headers['Content-Type'] = 'text/csv' response.headers['Content-Disposition'] = 'attachment; filename="%s"' % 'correctedData.csv' response.status = status return app_iter abort(404, _('No download is available'))
def resource_download(self, id, resource_id, filename=None): """ Provides a direct download by either redirecting the user to the url stored or downloading an uploaded file directly. """ context = {'model': model, 'session': model.Session, 'user': c.user or c.author, 'auth_user_obj': c.userobj} try: rsc = get_action('resource_show')(context, {'id': resource_id}) pkg = get_action('package_show')(context, {'id': id}) except NotFound: abort(404, _('Resource not found')) except NotAuthorized: abort(401, _('Unauthorized to read resource %s') % id) if rsc.get('url_type') == 'upload': upload = uploader.ResourceUpload(rsc) filepath = upload.get_path(rsc['id']) #### s3archive new code access_key = config.get('ckanext.s3archive.access_key') secret_key = config.get('ckanext.s3archive.secret_key') bucket_name = config.get('ckanext.s3archive.bucket') if not os.path.exists(filepath): content_type, content_enc = mimetypes.guess_type(rsc.get('url','')) key_name = filepath[len(filepath)-39:] conn = s3connection.S3Connection(access_key, secret_key) bucket = conn.get_bucket(bucket_name) key = None for key in bucket.list(prefix=key_name.lstrip('/')): pass if not key: abort(404, _('Resource data not found')) headers = {} if content_type: headers['response-content-type'] = content_type url = key.generate_url(300, method='GET', response_headers=headers) redirect(url) #### code finish fileapp = paste.fileapp.FileApp(filepath) try: status, headers, app_iter = request.call_application(fileapp) except OSError: abort(404, _('Resource data not found')) response.headers.update(dict(headers)) content_type, content_enc = mimetypes.guess_type(rsc.get('url','')) response.headers['Content-Type'] = content_type response.status = status return app_iter elif not 'url' in rsc: abort(404, _('No download is available')) redirect(rsc['url'])
def resource_download(self, id, resource_id): """Handles resource downloads for CKAN going through S3""" context = { 'model': model, 'session': model.Session, 'user': toolkit.c.user or toolkit.c.author, 'auth_user_obj': toolkit.c.userobj } try: rsc = toolkit.get_action('resource_show')(context, {'id': resource_id}) except toolkit.ObjectNotFound: toolkit.abort(404, _('Resource not found')) except toolkit.NotAuthorized: toolkit.abort(401, _('Unauthorized to read resource %s') % resource_id) # Check where the resource is located # If rsc.get('url_type') == 'upload' then the resource is in CKAN # file system if rsc.get('url_type') == 'upload': upload = uploader.ResourceUpload(rsc) filepath = upload.get_path(rsc['id']) fileapp = paste.fileapp.FileApp(filepath) try: status, headers, app_iter = request.call_application(fileapp) except OSError: abort(404, _('Resource data not found')) response.headers.update(dict(headers)) content_type, _ = mimetypes.guess_type(rsc.get('url', '')) if content_type: response.headers['Content-Type'] = content_type response.status = status return app_iter # If resource is not in CKAN file system, it should have a URL # directly to the resource elif not 'url' in rsc: abort(404, _('No download is available')) # Track download try: toolkit.get_action('track_resource_download')(context, rsc) except Exception as exception: # Log the error logger = logging.getLogger(__name__) logger.error("Error tracking resource download - %s" % exception) # Redirect the request to the URL for the resource zip pkg = toolkit.get_action('package_show')(context, {'id': id}) redirect(self.s3_url_prefix + pkg['name'] + '/' + 'resources' + '/' + slugify(rsc.get('name'), to_lower=True) + '.zip')
def _prepare_cached_response(self, full_file_path): try: fileapp = paste.fileapp.FileApp(full_file_path) status, headers, app_iter = request.call_application(fileapp) response.headers.update(dict(headers)) content_type, content_enc = mimetypes.guess_type(full_file_path) if content_type: response.headers['Content-Type'] = content_type response.status = status return app_iter except OSError: abort(404, _('Resource data not found'))
def resource_download(self, id, resource_id, filename=None): # custom_base.g_analitics() """ Provides a direct download by either redirecting the user to the url stored or downloading an uploaded file directly. """ context = {'model': model, 'session': model.Session, 'user': c.user or c.author, 'auth_user_obj': c.userobj} try: rsc = get_action('resource_show')(context, {'id': resource_id}) # removes the host to make it relative if config.get('ckan.upload_file_url'): url = rsc['url'] if config.get('ckan.upload_file_url') in url: url = url.split(config.get('ckan.upload_file_url')) rsc['url'] = url[1] # c.resource['url'] = rsc['url'] except NotFound: abort(404, _('Resource not found')) except NotAuthorized: abort(401, _('Unauthorized to read resource %s') % id) # analytics section pack_dict = get_action('package_show')(context, {'id': id}) analytics_helpers.update_analytics_code_by_organization(pack_dict['organization']['id']) analytics_helpers.send_analytic_event_server_side( u'{}~{}'.format(pack_dict.get('organization').get('title'), u'Resource_Download'), pack_dict.get('title'), rsc.get('name')) if rsc.get('url_type') == 'upload': upload = uploader.ResourceUpload(rsc) filepath = upload.get_path(rsc['id']) fileapp = paste.fileapp.FileApp(filepath) try: status, headers, app_iter = request.call_application(fileapp) except OSError: abort(404, _('Resource data not found')) response.headers.update(dict(headers)) content_type, content_enc = mimetypes.guess_type( rsc.get('url', '')) if content_type: response.headers['Content-Type'] = content_type response.status = status return app_iter elif not 'url' in rsc: abort(404, _('No download is available')) h.redirect_to(rsc['url'])
def global_file_download(self, filename): upload = GlobalUpload({ 'filename': filename, 'upload': None }) filepath = upload.get_path() fapp = fileapp.FileApp(filepath) try: status, headers, app_iter = request.call_application(fapp) response.headers.update(dict(headers)) content_type, content_enc = mimetypes.guess_type(filename) if content_type: response.headers['Content-Type'] = content_type response.status = status return app_iter except OSError: base.abort(404, _('Resource data not found'))
def resource_download(self, id, resource_id, filename=None): custom_base.g_analitics() """ Provides a direct download by either redirecting the user to the url stored or downloading an uploaded file directly. """ context = {'model': model, 'session': model.Session, 'user': c.user or c.author, 'auth_user_obj': c.userobj} try: rsc = get_action('resource_show')(context, {'id': resource_id}) get_action('package_show')(context, {'id': id}) # removes the host to make it relative if config.get('ckan.upload_file_url'): url = rsc['url'] if config.get('ckan.upload_file_url') in url: url = url.split(config.get('ckan.upload_file_url')) rsc['url'] = url[1] #c.resource['url'] = rsc['url'] except NotFound: abort(404, _('Resource not found')) except NotAuthorized: abort(401, _('Unauthorized to read resource %s') % id) if rsc.get('url_type') == 'upload': upload = uploader.ResourceUpload(rsc) filepath = upload.get_path(rsc['id']) fileapp = paste.fileapp.FileApp(filepath) try: status, headers, app_iter = request.call_application(fileapp) except OSError: abort(404, _('Resource data not found')) response.headers.update(dict(headers)) content_type, content_enc = mimetypes.guess_type( rsc.get('url', '')) if content_type: response.headers['Content-Type'] = content_type response.status = status return app_iter elif not 'url' in rsc: abort(404, _('No download is available')) h.redirect_to(rsc['url'])
def get_thumbnail(self, resolution, image): import ckan.lib.uploader as uploader import paste.fileapp import mimetypes filepath = uploader.get_storage_path() + '/thumbnails/' + resolution + '/' + image log.warn(filepath) fileapp = paste.fileapp.FileApp(filepath) try: status, headers, app_iter = request.call_application(fileapp) except OSError: base.abort(404, _('Resource data not found')) response.headers.update(dict(headers)) content_type, content_enc = mimetypes.guess_type(image) if content_type: response.headers['Content-Type'] = content_type response.status = status return app_iter
def resource_download(self, id, resource_id, filename=None): """ Provides a direct download by either redirecting the user to the url stored or downloading an uploaded file directly. """ context = {'model': model, 'session': model.Session, 'user': c.user or c.author, 'auth_user_obj': c.userobj} if request.method == 'POST': data = data or clean_dict(dict_fns.unflatten(tuplize_dict(parse_params( request.POST)))) # check if resources in package are publicly accessible pkg_dict = get_action('package_show')(context, {'id': id}) res_access = True; if 'res_access' in pkg_dict: res_access = pkg_dict['res_access'] try: rsc = get_action('resource_show')(context, {'id': resource_id, 'res_access': res_access}) get_action('package_show')(context, {'id': id}) except NotFound: abort(404, _('Resource not found')) except NotAuthorized: abort(401, _('Unauthorized to read resource %s') % id) if rsc.get('url_type') == 'upload': upload = uploader.ResourceUpload(rsc) filepath = upload.get_path(rsc['id']) fileapp = paste.fileapp.FileApp(filepath) try: status, headers, app_iter = request.call_application(fileapp) except OSError: abort(404, _('Resource data not found')) response.headers.update(dict(headers)) content_type, content_enc = mimetypes.guess_type( rsc.get('url', '')) if content_type: response.headers['Content-Type'] = content_type response.status = status return app_iter elif not 'url' in rsc: abort(404, _('No download is available')) redirect(rsc['url'])
def filesystem_resource_download(self, id, resource_id, filename=None): """ A fallback controller action to download resources from the filesystem. A copy of the action from `ckan.controllers.package:PackageController.resource_download`. Provide a direct download by either redirecting the user to the url stored or downloading an uploaded file directly. """ context = { 'model': model, 'session': model.Session, 'user': c.user or c.author, 'auth_user_obj': c.userobj } try: rsc = get_action('resource_show')(context, {'id': resource_id}) get_action('package_show')(context, {'id': id}) except NotFound: abort(404, _('Resource not found')) except NotAuthorized: abort(401, _('Unauthorized to read resource %s') % id) if rsc.get('url_type') == 'upload': upload = uploader.ResourceUpload(rsc) filepath = upload.get_path(rsc['id']) fileapp = paste.fileapp.FileApp(filepath) try: status, headers, app_iter = request.call_application(fileapp) except OSError: abort(404, _('Resource data not found')) response.headers.update(dict(headers)) content_type, content_enc = mimetypes.guess_type(rsc.get( 'url', '')) if content_type: response.headers['Content-Type'] = content_type response.status = status return app_iter elif 'url' not in rsc: abort(404, _('No download is available')) redirect(str(rsc['url']))
def filesystem_resource_download(self, id, resource_id, filename=None): """ A fallback controller action to download resources from the filesystem. A copy of the action from `ckan.controllers.package:PackageController.resource_download`. Provide a direct download by either redirecting the user to the url stored or downloading an uploaded file directly. """ context = {'model': model, 'session': model.Session, 'user': c.user or c.author, 'auth_user_obj': c.userobj} try: rsc = get_action('resource_show')(context, {'id': resource_id}) get_action('package_show')(context, {'id': id}) except NotFound: abort(404, _('Resource not found')) except NotAuthorized: abort(401, _('Unauthorized to read resource %s') % id) if rsc.get('url_type') == 'upload': upload = uploader.ResourceUpload(rsc) filepath = upload.get_path(rsc['id']) fileapp = paste.fileapp.FileApp(filepath) try: status, headers, app_iter = request.call_application(fileapp) except OSError: abort(404, _('Resource data not found')) response.headers.update(dict(headers)) content_type, content_enc = mimetypes.guess_type(rsc.get('url', '')) if content_type: response.headers['Content-Type'] = content_type response.status = status return app_iter elif 'url' not in rsc: abort(404, _('No download is available')) redirect(str(rsc['url']))
def resource_download(self, id, resource_id, filename=None): ''' Provide a download by either redirecting the user to the url stored or downloading the uploaded file from S3. ''' context = { 'model': model, 'session': model.Session, 'user': c.user or c.author, 'auth_user_obj': c.userobj } try: rsc = get_action('resource_show')(context, {'id': resource_id}) get_action('package_show')(context, {'id': id}) except NotFound: abort(404, _('Resource not found')) except NotAuthorized: abort(401, _('Unauthorized to read resource %s') % id) if rsc.get('url_type') == 'upload': upload = uploader.get_resource_uploader(rsc) bucket_name = config.get('ckanext.s3filestore.aws_bucket_name') region = config.get('ckanext.s3filestore.region_name') bucket = upload.get_s3_bucket(bucket_name) if filename is None: filename = os.path.basename(rsc['url']) key_path = upload.get_path(rsc['id'], filename) key = filename if key is None: log.warn('Key \'{0}\' not found in bucket \'{1}\''.format( key_path, bucket_name)) try: obj = bucket.Object(key_path) contents = str(obj.get()['Body'].read()) except ClientError as ex: if ex.response['Error']['Code'] == 'NoSuchKey': # attempt fallback if config.get( 'ckanext.s3filestore.filesystem_download_fallback', False): log.info( 'Attempting filesystem fallback for resource {0}'. format(resource_id)) url = toolkit.url_for( controller= 'ckanext.s3filestore.controller:S3Controller', action='filesystem_resource_download', id=id, resource_id=resource_id, filename=filename) redirect(url) abort(404, _('Resource data not found')) else: raise ex dataapp = paste.fileapp.DataApp(contents) try: status, headers, app_iter = request.call_application(dataapp) except OSError: abort(404, _('Resource data not found')) response.headers.update(dict(headers)) response.status = status content_type, x = mimetypes.guess_type(rsc.get('url', '')) if content_type: response.headers['Content-Type'] = content_type return app_iter elif 'url' not in rsc: abort(404, _('No download is available')) redirect(str(rsc['url']))
def index(self, id): print 'index' context = { 'model': ckan.model, 'session': ckan.model.Session, 'user': pylons.c.user or pylons.c.author } try: plugins.toolkit.c.pkg_dict = plugins.toolkit.get_action( 'package_show')(context, { 'id': id }) plugins.toolkit.c.pkg = context['package'] plugins.toolkit.c.resources_json = h.json.dumps( plugins.toolkit.c.pkg_dict.get('resources', [])) except plugins.toolkit.ObjectNotFound: plugins.toolkit.abort(404, plugins.toolkit._('Dataset not found')) except plugins.toolkit.NotAuthorized: plugins.toolkit.abort( 401, plugins.toolkit._('Unauthorized to read package %s') % id) vars = { 'errors': {}, 'data': { 'title': '', #plugins.toolkit._('Clone of {dataset}').format(dataset=plugins.toolkit.c.pkg_dict['title'])', 'name': '' } } if plugins.toolkit.request.method == 'POST': post_data = plugins.toolkit.request.POST if post_data['action-type'] == 'clone': context = { 'model': ckan.model, 'session': ckan.model.Session, 'user': pylons.c.user or pylons.c.author } try: plugins.toolkit.check_access('package_create', context) plugins.toolkit.check_access('package_update', context, {'id': id}) del context['package'] except plugins.toolkit.NotAuthorized: plugins.toolkit.abort( 401, plugins.toolkit._( 'Unauthorized to clone this package')) #get current package... pkg_dict = plugins.toolkit.get_action('package_show')(None, { 'id': id }) #update necessary fields title = ckan.plugins.toolkit.request.params.getone('title') name = ckan.plugins.toolkit.request.params.getone('name') dt = datetime.datetime.now() pkg_dict['title'] = title pkg_dict['name'] = name pkg_dict['metadata_created'] = dt pkg_dict['metadata_modified'] = dt del pkg_dict['id'] del pkg_dict['revision_id'] del pkg_dict['revision_timestamp'] resources = pkg_dict['resources'] for resource in resources: if resource['url_type'] == 'upload': #copy file upload = uploader.ResourceUpload(resource) filepath = upload.get_path(resource['id']) cfs = FieldStorage() cfs.file = open(filepath) cfs.filename = resource['url'].split('/')[-1] resource['upload'] = cfs resource['created'] = dt del resource['id'] del resource['revision_id'] del resource['revision_timestamp'] del pkg_dict['resources'] #create a new one based on existing one... try: #for some reason, the pkg_dict given to 'package_create' still has the old id pkg_dict_new = plugins.toolkit.get_action( 'package_create')(context, pkg_dict) for resource in resources: resource['package_id'] = pkg_dict_new['id'] plugins.toolkit.get_action('resource_create')(context, resource) #if package already has a review date set, return it... if pkg_dict.get('next_review_date'): package_review = get_package_review( ckan.model.Session, pkg_dict_new['id']) if package_review: package_review.next_review_date = pkg_dict.get( 'next_review_date') update_package_review(context['session'], package_review) else: add_package_review( context['session'], pkg_dict_new['id'], pkg_dict.get('next_review_date')) except plugins.toolkit.ValidationError as ve: plugins.toolkit.c.pkg_dict = plugins.toolkit.get_action( 'package_show')(context, { 'id': id }) plugins.toolkit.c.pkg = context['package'] plugins.toolkit.c.resources_json = h.json.dumps( plugins.toolkit.c.pkg_dict.get('resources', [])) errorsOther = dict(ve.error_dict) if 'name' in errorsOther: del errorsOther['name'] vars = { 'errors': ve.error_dict, 'errorsOther': errorsOther, 'data': { 'title': title, 'name': name } } return plugins.toolkit.render("dsaction-index.html", extra_vars=vars) ckan.plugins.toolkit.redirect_to(controller="package", action="edit", id=pkg_dict_new['id']) else: get_data = plugins.toolkit.request.GET if 'action-type' in get_data and get_data[ 'action-type'] == 'export': print 'export' #task 1: work out if the dataset has items in filestore #get package pid = convert_to_id(id, context) query = ckan.model.Session.query( ckan.model.Package).filter(ckan.model.Package.id == pid) file_zip_path = exportPackages(query) #serve zip file fileapp = paste.fileapp.FileApp(file_zip_path) fileapp.content_disposition(filename='%s.zip' % id) status, headers, app_iter = request.call_application(fileapp) response.headers.update(dict(headers)) content_type = 'application/zip' response.headers['Content-Type'] = content_type response.status = status #remove tmp zip file - not sure if this will cause issues deleting the file before it has been fully served? os.remove(file_zip_path) return app_iter return plugins.toolkit.render("dsaction-index.html", extra_vars=vars)
def resource_download(self, environ, id, resource_id, filename=None): context = { 'model': model, 'session': model.Session, 'user': c.user, 'auth_user_obj': c.userobj } try: rsc = t.get_action('resource_show')(context, {'id': resource_id}) except (logic.NotFound, logic.NotAuthorized): base.abort(404, _('Resource not found')) headers = { 'X-Forwarded-For': environ.get('REMOTE_ADDR'), 'User-Agent': environ.get('HTTP_USER_AGENT'), 'Accept-Language': environ.get('HTTP_ACCEPT_LANGUAGE', ''), 'Accept-Encoding': environ.get('HTTP_ACCEPT_ENCODING', '') } if rsc.get('token_required') == 'Yes': authentication = environ.get('HTTP_AUTHORIZATION', '') url_redirect = "%s/tokens?resource_id=%s&package_id=%s" % ( config.get('ckan.site_url'), resource_id, rsc['package_id']) if authentication == '': return redirect(url_redirect.encode('utf-8')) dbd = parse_db_config('ckan.drupal.url') drupal_conn_string = "host='%s' dbname='%s' port='%s' user='******' password='******'" % ( dbd['db_host'], dbd['db_name'], dbd['db_port'], dbd['db_user'], dbd['db_pass']) drupal_conn = psycopg2.connect(drupal_conn_string) drupal_cursor = drupal_conn.cursor( cursor_factory=psycopg2.extras.DictCursor) if not rsc.get('token_type'): drupal_cursor.execute( """select id_usuario from opendata_tokens where tkn_usuario=%s""", (authentication, )) else: drupal_cursor.execute( """SELECT t.*, pu.*, p.*, u.name, u.mail, u.uid FROM opendata_tokens t LEFT JOIN opendata_tokens_provider_user pu ON pu.id_usuario=t.id_usuario LEFT JOIN opendata_tokens_provider p ON (pu.provider = p.id OR p.id='bsm') LEFT JOIN users u ON t.id_usuario = u.uid WHERE t.tkn_usuario = %s AND (p.id IS NULL OR p.id = %s)""", (authentication, rsc.get('token_type'))) if drupal_cursor.rowcount < 1: return redirect(url_redirect.encode('utf-8')) elif rsc.get('token_type'): record = drupal_cursor.fetchone() api = None if rsc.get('token_type') == 'bsm': api = bsm.BsmApi(rsc, app_token=record['app_token'], consumer_key=record['consumer_key'], consumer_secret=record['consumer_secret'], user_token=record['token'], user_id=record['uid'], user_key=record['key'], user_secret=record['secret'], username=record['name'], email=record['mail']) pprint.pprint(record['app_token']) api_content, status, headers = api.execute() # Save download to tracking_raw CustomTrackingController.update(environ['REQUEST_URI'], 'resource', environ) if rsc.get('url_type') == 'upload': # Internal redirect upload = uploader.get_resource_uploader(rsc) filepath = upload.get_path(rsc['id']) fileapp = paste.fileapp.FileApp(filepath) try: status, headers, app_iter = request.call_application(fileapp) except OSError: base.abort(404, _('Resource data not found')) response.headers.update(dict(headers)) content_type, content_enc = m.guess_type(rsc.get('url', '')) if content_type and content_type == 'application/xml': response.headers['Content-Type'] = 'application/octet-stream' elif content_type: response.headers['Content-Type'] = content_type response.status = status return app_iter h.redirect_to(rsc['url'].encode('utf-8')) elif api_content: response.headers['Content-Type'] = headers['content-type'] response.status = status return api_content elif 'url' not in rsc: base.abort(404, _('No download is available')) else: # External redirect return redirect(rsc['url'].encode('utf-8'))
def _prepare_packager_parameters(self, email, params): """Prepare the parameters for the ckanpackager service for the current request @param resource_id: The resource id @param params: A dictionary of parameters @return: A tuple defining an URL and a dictionary of parameters """ print(type(params)) print(params) packager_url = config['url'] request_params = { 'secret': config['secret'], 'email': email, # default to csv format, this can be overridden in the params 'format': u'csv', } context = { 'model': model, 'session': model.Session, 'user': t.c.user, 'auth_user_obj': t.c.userobj } # TODO check this makes sense... request_params['download_payload'] = params['download_payload'] download_payload = json.loads(params['download_payload']) resource_file_paths = [] for resource_id in download_payload['download_list']: if resource_id: print '\n\n', resource_id try: rsc = get_action('resource_show')(context, { 'id': resource_id }) # get_action('package_show')(context, {'id': id}) if rsc.get('url_type') == 'upload': upload = uploader.get_resource_uploader(rsc) filepath = upload.get_path(rsc['id']) print filepath resource_file_paths.append({ 'name': rsc['url'].split('/')[-1], 'file_path': filepath, 'size': rsc['size'] }) fileapp = paste.fileapp.FileApp(filepath) try: status, headers, app_iter = request.call_application( fileapp) except OSError: pass # abort(404, _('Resource data not found')) # response.headers.update(dict(headers)) # content_type, content_enc = mimetypes.guess_type( # rsc.get('url', '')) # if content_type: # response.headers['Content-Type'] = content_type # response.status = status # return app_iter elif 'url' not in rsc: pass # abort(404, _('No download is available')) except (NotFound, NotAuthorized): pass request_params['file_paths'] = json.dumps( {'paths': resource_file_paths}) packager_url += '/package_url' return packager_url, request_params