zfile.extractall(path=upload_dir) def dir_filter(s): if os.path.isdir(os.path.join(upload_dir, s)): return False return True resource_list = filter(dir_filter, zfile.namelist()) status, err_msg = self._validate_uploadfile(datafilepath, upload_dir, resource_list) if status == "INVALID": import_helper.delete_files( file_path=upload_dir, ignore_files=[datafilename, resourcesfilename]) self._create_bulk_upload_record(c.user or c.author, datafilename, resourcesfilename, upload_dir, status, err_msg) url = h.url_for( controller= 'ckanext.ngds.ngdsui.controllers.contribute:ContributeController', action='bulk_upload') redirect(url) def _validate_uploadfile(self, data_file, resource_path, resource_list): """ This function is responsible for validating a bulk upload.
self.importpackagedata(bulk_upload_record.id, file_path=data_file_path, resource_dir=bulk_upload_record.path, ckanclient=self.ckanclient) bulk_upload_record.status = "COMPLETED" except Exception, e: log.debug(e) log.debug( "Exception while processing bulk upload for the file : %s", bulk_upload_record.data_file) bulk_upload_record.status = "FAILURE" bulk_upload_record.comments = e.message finally: import_helper.delete_files(file_path=bulk_upload_record.path, ignore_files=[ bulk_upload_record.data_file, bulk_upload_record.resources ]) bulk_upload_record.last_updated = None bulk_upload_record.save() def importpackagedata(self, bulk_upload_id, file_path=None, resource_dir=None, ckanclient=None): """ This is an entry point for bulk upload of one record(bulk uploaded template). Loads the package details as dict from xls file and loads them into CKAN. Each successfully loaded pacakges are referenced against bulk upload record for tracking. """
for bulk_upload_record in query.all(): log.debug("Processing the file: %s", bulk_upload_record.data_file) try: data_file_path = os.path.join(bulk_upload_record.path,bulk_upload_record.data_file) self.importpackagedata(bulk_upload_record.id,file_path=data_file_path,resource_dir=bulk_upload_record.path,ckanclient=self.ckanclient) bulk_upload_record.status = "COMPLETED" except Exception, e: log.debug(e) log.debug("Exception while processing bulk upload for the file : %s" , bulk_upload_record.data_file ) bulk_upload_record.status = "FAILURE" bulk_upload_record.comments = e.message finally: import_helper.delete_files(file_path=bulk_upload_record.path,ignore_files=[bulk_upload_record.data_file,bulk_upload_record.resources]) bulk_upload_record.last_updated = None bulk_upload_record.save() def importpackagedata(self,bulk_upload_id,file_path=None,resource_dir=None,ckanclient=None): """ This is an entry point for bulk upload of one record(bulk uploaded template). Loads the package details as dict from xls file and loads them into CKAN. Each successfully loaded pacakges are referenced against bulk upload record for tracking. """ from ckan.lib.navl.dictization_functions import unflatten from ckan.logic import (tuplize_dict,clean_dict ) from ckanext.ngds.importer.loader import ResourceLoader
zfile = zipfile.ZipFile(resfilepath) zfile.extractall(path=upload_dir) def dir_filter(s): if os.path.isdir(os.path.join(upload_dir, s)): return False return True resource_list = filter(dir_filter, zfile.namelist()) status, err_msg = self._validate_uploadfile(datafilepath, upload_dir, resource_list) if status == "INVALID": import_helper.delete_files(file_path=upload_dir, ignore_files=[datafilename, resourcesfilename]) self._create_bulk_upload_record(c.user or c.author, datafilename, resourcesfilename, upload_dir, status, err_msg) url = h.url_for(controller='ckanext.ngds.ngdsui.controllers.contribute:ContributeController', action='bulk_upload') redirect(url) def _validate_uploadfile(self, data_file, resource_path, resource_list): """ This function is responsible for validating a bulk upload. """ err_msg = "" try: validator = ngdsvalidator.NGDSValidator(filepath=data_file, resource_path=resource_path,