def get_package_list_for_all(self):
api_list = self.get_ckan_api_list()
for api in api_list:
ckan = CkanClient(base_location=api,
api_key=config.ckan_api_key)
for package in ckan.package_list():
break
def _process_upload(context, data):
"""
When provided with a filename this function will process each row
within the file and then return a tuple. The tuple will contain
- a list of error messages (if any)
- a list of dicts where each dict contains ...
{
'package': 'a_package_id',
'action': 'Added' or 'Updated'
}
"""
log = inventory_upload.get_logger()
errors = []
results = []
filename = data['file']
publisher_name = data['publisher']
import urlparse
client = CkanClient(base_location=urlparse.urljoin(context['site_url'],
'api'),
api_key=context['apikey'])
tableset = None
try:
_, ext = os.path.splitext(filename)
tableset = messytables.any_tableset(open(filename, 'r'),
extension=ext[1:])
except Exception, e:
if str(e) == "Unrecognized MIME type: text/plain":
tableset = messytables.any_tableset(f, mimetype="text/csv")
else:
errors.append("Unable to load file: {0}".format(e))
def oct10(self):
client = CkanClient(base_location=self.options.api_url,
api_key=self.options.api_key)
change_licenses = ChangeLicenses(client,
dry_run=self.options.dry_run,
force=self.options.force)
change_licenses.change_oct_2010(self.options.license_id)
def _clean_test_database(self, package_name, id):
base_location = self._get_ckan_base_api_url()
api_key = self._get_user_api_key()
testclient = CkanClient(base_location, api_key)
# package_name ='spatialize_test_resource_3'
testclient.package_entity_delete(package_name)
# also remove table from database using id
data_dict = {}
data_dict['connection_url'] = pylons.config.get(
'ckan.datastore.write_url',
'postgresql://*****:*****@localhost/test_datastore')
engine = db._get_engine(None, data_dict)
connection = engine.connect()
resources_sql = 'DROP TABLE IF EXISTS "' + id + '";'
# resources_sql = 'DROP TABLE "b11351a2-5bbc-4f8f-8078-86a4eef1c7b0";'
try:
print '>>>>>>>>>>>>> Executing command: ', resources_sql
trans = connection.begin()
results = connection.execute(resources_sql)
trans.commit()
except Exception, e:
print "exception", e
assert False
def all(self):
client = CkanClient(base_location=self.options.api_url,
api_key=self.options.api_key,
http_user=self.options.username,
http_pass=self.options.password)
change_licenses = ChangeLicenses(client,
dry_run=self.options.dry_run,
force=self.options.force)
change_licenses.change_all_packages(self.options.license_id)
def command(self):
super(OnsAnalysisCommand, self).command()
# now do command
client = CkanClient(base_location=self.options.api_url,
api_key=self.options.api_key,
http_user=self.options.username,
http_pass=self.options.password)
change_licenses = OnsAnalysis(client)
change_licenses.run()
def init_ckanclient(self):
"""Init the CKAN client from options."""
if not self.options.ckan_api_location:
print "Warning: CKAN API location not provided."
if not self.options.ckan_api_key:
print "Warning: CKAN API key not provided."
self.ckanclient = CkanClient(
base_location=self.options.ckan_api_location,
api_key=self.options.ckan_api_key,
)
def setup_class(self):
self.pid = self._start_ckan_server()
self.test_base_location = 'http://127.0.0.1:5000/api'
self._wait_for_url(url=self.test_base_location)
self._recreate_ckan_server_testdata(config_path)
# this is api key created for tester user by create-test-data in ckan
test_api_key = 'tester'
test_api_key2 = 'tester2'
self.c = CkanClient(
base_location=self.test_base_location,
api_key=test_api_key,
is_verbose=True,
)
self.c2 = CkanClient(
base_location=self.test_base_location,
api_key=test_api_key2,
is_verbose=True,
)
def setup_class(self):
if hasattr(super(TestLoaderBase, self), 'setup_class'):
super(TestLoaderBase, self).setup_class()
CreateTestData.create_arbitrary([], extra_user_names=[USER])
user = model.User.by_name(USER)
assert user
if WSGI_CLIENT:
self.testclient = WsgiCkanClient(self.app, api_key=user.apikey)
else:
self.sub_proc = self._start_ckan_server('test.ini')
self.testclient = CkanClient(
base_location='http://localhost:5000/api', api_key=user.apikey)
self._wait_for_url(url='http://localhost:5000/api')
def command(self):
super(TransferUrlCommand, self).command()
if self.options.license_id is None:
self.parser.error("Please specify a license ID")
if len(self.args) != 1:
self.parser.error("Command is required")
client = CkanClient(base_location=self.options.api_url,
api_key=self.options.api_key,
http_user=self.options.username,
http_pass=self.options.password)
transfer_url = TransferUrl(client,
dry_run=self.options.dry_run,
force=self.options.force)
transfer_url.transfer_url()
def _setup_test_database(self, package_name):
print ">>>>>>>>>>>>>>>>>> creating package: ", package_name
try:
base_location = self._get_ckan_base_api_url()
api_key = self._get_user_api_key()
testclient = CkanClient(base_location, api_key)
file_url, status = testclient.upload_file(
"./testData/small_with_lat_long.csv")
print "created file_url:", file_url
except Exception, e:
print "exception", e
assert False
def _setup_test_database(self, package_name):
print ">>>>>>>>>>>>>>>>>> creating package: ", package_name
base_location = self._get_ckan_base_api_url()
api_key = self._get_user_api_key()
testclient = CkanClient(base_location, api_key)
print "base.. ", testclient.base_location
file_url, status = testclient.upload_file(
"./testData/small_with_lat_long.csv")
print "created file_url:", file_url
print "status: ", status
assert True
package_dict = {
u'name':
package_name,
u'title':
u'Serialize test 1',
u'notes':
u'dummy notes',
'owner_org':
'public',
u'private':
u'False',
u'state':
u'active',
'resources': [{
'description': u'Resource Document Description',
'format': u'csv',
'url': file_url,
'name': u'Resource somewhere'
}]
}
#print "package_dict: at test: ", package_dict
try:
ret_pack = testclient.package_register_post(package_dict)
resources = ret_pack['resources']
self.database_id = resources[0]['id']
print ">>>>>>>>>>>>>>>>>>>>>>>> database_id:", self.database_id
except Exception, e:
print "Exception: ", e
assert False
return ""
def traverse(pkg_func, query='*:*'):
client = CkanClient(base_location=HOST, api_key=API_KEY)
for page in count(1):
results_page = client.package_search(query,
search_options={
'offset': page * PAGE_SIZE,
'limit': PAGE_SIZE
})
#pprint(results_page)
if not len(results_page.get('results', [])):
break
for pkg_name in results_page.get('results', []):
print "Traversing", pkg_name
pkg = client.package_entity_get(pkg_name)
ret = pkg_func(client, pkg)
if ret is not None:
client.package_entity_put(ret, package_name=pkg_name)
def command(self):
super(ApiCommand, self).command()
if not self.options.api_key:
self.parser.error('Please specify an API Key')
if not self.options.api_url:
self.parser.error('Please specify an API URL')
if self.options.api_url:
if not (self.options.api_url.startswith('http://') or \
self.options.api_url.startswith('https://')):
self.parser.error('--host must start with "http://"')
if not '/api' in self.options.api_url:
self.parser.error('--host must have "/api" towards the end')
user_agent = self.user_agent if hasattr(self, 'user_agent') else 'ckanext-importlib/ApiCommand'
self.client = CkanClient(base_location=self.options.api_url,
api_key=self.options.api_key,
http_user=self.options.username,
http_pass=self.options.password,
is_verbose=True,
user_agent=user_agent)
def _clean_all_tables_and_packages_in_database(self):
base_location = self._get_ckan_base_api_url()
api_key = self._get_user_api_key()
testclient = CkanClient(base_location, api_key)
# TODO: clean all packages
# also remove table from database using id
data_dict = {}
data_dict['connection_url'] = pylons.config.get(
'ckan.datastore.write_url',
'postgresql://*****:*****@localhost/test_datastore')
engine = db._get_engine(None, data_dict)
connection = engine.connect()
resources_sql = "SELECT * FROM pg_tables;"
# resources_sql = 'DROP TABLE "b11351a2-5bbc-4f8f-8078-86a4eef1c7b0";'
try:
print '>>>>>>>>>>>>> Executing command: ', resources_sql
trans = connection.begin()
results_cursor = connection.execute(resources_sql)
trans.commit()
allTables = results_cursor.fetchall()
filteredTables = []
for table in allTables:
tableName = table[1]
if not "pg_" in tableName and not "sql_" in tableName and not tableName == "geometry_columns" and not tableName == "spatial_ref_sys":
filteredTables.append(tableName)
trans = connection.begin()
for name in filteredTables:
print "dropping table: ", name
resource_sql = 'DROP TABLE IF EXISTS "' + name + '";'
results = connection.execute(resource_sql)
trans.commit()
except Exception, e:
print "exception", e
assert False
def __init__(self, api_base):
self.ckan = CkanClient(base_location=api_base)
def ckan_client():
ckan_api = config_get('ckan-api.url')
return CkanClient(base_location='http://data.gov.uk/api')
def command(ckan_api_url):
from ckanext.dgu.ons.importer import OnsImporter
# sources pasted here from http://www.statistics.gov.uk/hub/statistics-producers/index.html
sources = '''
Agri-Food and Biosciences Institute
Agriculture and Rural Development (Northern Ireland)
Business, Innovation and Skills
Cabinet Office
Communities and Local Government
Culture, Media and Sport
Defence
Education
Education (Northern Ireland)
Employment and Learning (Northern Ireland)
Energy and Climate Change
Enterprise, Trade and Investment (Northern Ireland)
Environment (Northern Ireland)
Environment, Food and Rural Affairs
Food Standards Agency
Forestry Commission
Health
Health and Safety Executive
Health Protection Agency
Health, Social Service and Public Safety (Northern Ireland)
HM Revenue and Customs
HM Treasury
Home Office
ISD Scotland (part of NHS National Services Scotland)
International Development
Justice
Justice (Northern Ireland)
Marine Management Organisation
National Records of Scotland
National Treatment Agency
Northern Ireland Statistics and Research Agency
Office for National Statistics
Office for Rail Regulation
Office for Standards in Education, Children\'s Services and Skills
Office of Qualifications and Examinations Regulation
Office of the First and Deputy First Minister
Passenger Focus
Police Service of Northern Ireland (PSNI)
Public Health England
Regional Development (Northern Ireland)
Scottish Government
Social Development (Northern Ireland)
Transport
Welsh Government
Work and Pensions
Cancer Registry (Northern Ireland)
Civil Aviation Authority
Child Maintenance and Enforcement Commission
Health and Social Care Information Centre
Higher Education Statistics Agency
Independent Police Complaints Commission
NHS England
Scottish Consortium for Learning Disability
Student Loans Company
Eurostat
'''
# These are extra sources seen in the past ONS data, picked up from
# the ons_merge_duplicates tool:
sources += '''
Cancer Registry Northern Ireland
Welsh Assembly Government
'''
pasted_lines_to_ignore = (
'Government Statistical Departments',
'Other statistics producers',
'International statistics organisations',
)
ckanclient = CkanClient(base_location=ckan_api_url)
num_errors = 0
sources = sources.split('\n')
for source in sources:
if not source.strip() or source in pasted_lines_to_ignore:
continue
publisher = OnsImporter._source_to_publisher_(source.strip(),
ckanclient)
if not publisher:
log.error('Publisher not found: %s', source)
num_errors += 1
log.info('Completed with %i errors from %i sources', num_errors,
len(sources))
