def setup_class(cls):
super(TestxloaderDataIntoDatastore, cls).setup_class()
cls.host = 'www.ckan.org'
cls.api_key = 'my-fake-key'
cls.resource_id = 'foo-bar-42'
factories.Resource(id=cls.resource_id)
jobs_db.init(config, echo=False)
# drop test table
engine, conn = cls.get_datastore_engine_and_connection()
conn.execute('DROP TABLE IF EXISTS "{}"'.format(cls.resource_id))
def setup_class(self):
self.host = "www.ckan.org"
self.api_key = "my-fake-key"
self.resource_id = "foo-bar-42"
factories.Resource(id=self.resource_id)
jobs_db.init(config, echo=False)
# drop test table
engine, conn = self.get_datastore_engine_and_connection()
conn.execute('DROP TABLE IF EXISTS "{}"'.format(self.resource_id))
yield
if "_datastore" in dir(self):
connection = self._datastore[1]
connection.close()
def xspatialloader_data_into_datastore_(input, job_dict):
'''This function:
* downloads the resource (metadata) from CKAN
* downloads the data
* calls the loader to load the data into DataStore
* calls back to CKAN with the new status
(datapusher called this function 'push_to_datastore')
'''
job_id = get_current_job().id
db.init(config)
# Store details of the job in the db
try:
db.add_pending_job(job_id, **input)
except sa.exc.IntegrityError:
raise JobError('job_id {} already exists'.format(job_id))
# Set-up logging to the db
handler = StoringHandler(job_id, input)
level = logging.DEBUG
handler.setLevel(level)
logger = logging.getLogger(job_id)
handler.setFormatter(logging.Formatter('%(message)s'))
logger.addHandler(handler)
# also show logs on stderr
logger.addHandler(logging.StreamHandler())
logger.setLevel(logging.DEBUG)
validate_input(input)
data = input['metadata']
resource_id = data['resource_id']
try:
resource, dataset = get_resource_and_dataset(resource_id)
except JobError, e:
# try again in 5 seconds just in case CKAN is slow at adding resource
time.sleep(5)
resource, dataset = get_resource_and_dataset(resource_id)