def test_delete_datastore_table(self):
with patch('requests.post') as mock_delete_datastore_table:
# Validate Success
mocked_res = {
'success': True,
'message': 'Table deleted successfully.'
}
# Set the status code as 200
mock_delete_datastore_table.return_value.status_code = 200
mock_delete_datastore_table.return_value.json.return_value = \
mocked_res
self.assertEqual(
delete_datastore_table(RESOURCE_ID, CKAN_API_KEY, CKAN_URL),
mocked_res)
# Validate Failure
mocked_res_error = {
"success": False,
"message": "Failed to Delete Datastore Table.",
}
mocked_res = {"error": "Failed to Delete Datastore Table."}
# Set the status code as 400
mock_delete_datastore_table.return_value.status_code = 400
mock_delete_datastore_table.return_value.json.return_value = \
mocked_res
self.assertEqual(
delete_datastore_table(RESOURCE_ID, CKAN_API_KEY, CKAN_URL),
str(mocked_res_error))
def task_create_datastore_table(**context):
ti = context["ti"]
logging.info("Invoking Create Datastore")
resource_id = context["params"].get("resource", {}).get("ckan_resource_id")
ckan_api_key = context["params"].get("ckan_config", {}).get("api_key")
ckan_site_url = context["params"].get("ckan_config", {}).get("site_url")
xcom_result = ti.xcom_pull(task_ids="fetch_resource_data")
schema = xcom_result["resource"].get("schema", {}).get("fields", [])
logging.info("Invoking Delete Datastore")
delete_datastore_table(resource_id, ckan_api_key, ckan_site_url)
create_datastore_table(resource_id, schema, ckan_api_key, ckan_site_url)
def full_load(resource_id, schema_fields, csv_input, json_output, **kwargs):
logging.info('Deleting Datastore if exists')
delete_datastore_table(resource_id, Variable.get('CKAN_SYSADMIN_API_KEY'),
Variable.get('CKAN_SITE_URL'))
logging.info('Invoking Create Datastore')
data_resource_fields = ast.literal_eval(schema_fields)
create_datastore_table(resource_id, data_resource_fields,
Variable.get('CKAN_SYSADMIN_API_KEY'),
Variable.get('CKAN_SITE_URL'))
logging.info('Converting resources to json')
convert(csv_input, json_output)
logging.info('Loading CSV via API')
try:
with open(json_output) as f:
records = json.load(f)
return load_resource_via_api(resource_id, records,
Variable.get('CKAN_SYSADMIN_API_KEY'),
Variable.get('CKAN_SITE_URL'))
except Exception as e:
return {"success": False, "errors": [e]}
def task_delete_datastore_table(resource_id, **kwargs):
logging.info('Invoking Delete Datastore')
return delete_datastore_table(resource_id, Variable.get('CKAN_SYSADMIN_API_KEY'), Variable.get('CKAN_SITE_URL'))
def task_delete_datastore_table(**context):
logging.info('Invoking Delete Datastore')
resource_id = context['params'].get('resource', {}).get('ckan_resource_id')
ckan_api_key = context['params'].get('ckan_config', {}).get('api_key')
ckan_site_url = context['params'].get('ckan_config', {}).get('site_url')
return delete_datastore_table(resource_id, ckan_api_key, ckan_site_url)