zfile.extractall(path=upload_dir)
def dir_filter(s):
if os.path.isdir(os.path.join(upload_dir, s)):
return False
return True
resource_list = filter(dir_filter, zfile.namelist())
status, err_msg = self._validate_uploadfile(datafilepath, upload_dir,
resource_list)
if status == "INVALID":
import_helper.delete_files(
file_path=upload_dir,
ignore_files=[datafilename, resourcesfilename])
self._create_bulk_upload_record(c.user or c.author, datafilename,
resourcesfilename, upload_dir, status,
err_msg)
url = h.url_for(
controller=
'ckanext.ngds.ngdsui.controllers.contribute:ContributeController',
action='bulk_upload')
redirect(url)
def _validate_uploadfile(self, data_file, resource_path, resource_list):
"""
This function is responsible for validating a bulk upload.
self.importpackagedata(bulk_upload_record.id,
file_path=data_file_path,
resource_dir=bulk_upload_record.path,
ckanclient=self.ckanclient)
bulk_upload_record.status = "COMPLETED"
except Exception, e:
log.debug(e)
log.debug(
"Exception while processing bulk upload for the file : %s",
bulk_upload_record.data_file)
bulk_upload_record.status = "FAILURE"
bulk_upload_record.comments = e.message
finally:
import_helper.delete_files(file_path=bulk_upload_record.path,
ignore_files=[
bulk_upload_record.data_file,
bulk_upload_record.resources
])
bulk_upload_record.last_updated = None
bulk_upload_record.save()
def importpackagedata(self,
bulk_upload_id,
file_path=None,
resource_dir=None,
ckanclient=None):
"""
This is an entry point for bulk upload of one record(bulk uploaded template). Loads the package details as
dict from xls file and loads them into CKAN. Each successfully loaded pacakges are referenced against bulk upload
record for tracking.
"""
for bulk_upload_record in query.all():
log.debug("Processing the file: %s", bulk_upload_record.data_file)
try:
data_file_path = os.path.join(bulk_upload_record.path,bulk_upload_record.data_file)
self.importpackagedata(bulk_upload_record.id,file_path=data_file_path,resource_dir=bulk_upload_record.path,ckanclient=self.ckanclient)
bulk_upload_record.status = "COMPLETED"
except Exception, e:
log.debug(e)
log.debug("Exception while processing bulk upload for the file : %s" , bulk_upload_record.data_file )
bulk_upload_record.status = "FAILURE"
bulk_upload_record.comments = e.message
finally:
import_helper.delete_files(file_path=bulk_upload_record.path,ignore_files=[bulk_upload_record.data_file,bulk_upload_record.resources])
bulk_upload_record.last_updated = None
bulk_upload_record.save()
def importpackagedata(self,bulk_upload_id,file_path=None,resource_dir=None,ckanclient=None):
"""
This is an entry point for bulk upload of one record(bulk uploaded template). Loads the package details as
dict from xls file and loads them into CKAN. Each successfully loaded pacakges are referenced against bulk upload
record for tracking.
"""
from ckan.lib.navl.dictization_functions import unflatten
from ckan.logic import (tuplize_dict,clean_dict )
from ckanext.ngds.importer.loader import ResourceLoader
zfile = zipfile.ZipFile(resfilepath)
zfile.extractall(path=upload_dir)
def dir_filter(s):
if os.path.isdir(os.path.join(upload_dir, s)):
return False
return True
resource_list = filter(dir_filter, zfile.namelist())
status, err_msg = self._validate_uploadfile(datafilepath, upload_dir, resource_list)
if status == "INVALID":
import_helper.delete_files(file_path=upload_dir, ignore_files=[datafilename, resourcesfilename])
self._create_bulk_upload_record(c.user or c.author, datafilename, resourcesfilename, upload_dir, status,
err_msg)
url = h.url_for(controller='ckanext.ngds.ngdsui.controllers.contribute:ContributeController',
action='bulk_upload')
redirect(url)
def _validate_uploadfile(self, data_file, resource_path, resource_list):
"""
This function is responsible for validating a bulk upload.
"""
err_msg = ""
try:
validator = ngdsvalidator.NGDSValidator(filepath=data_file, resource_path=resource_path,
