def user_confirm(msg):
if check_ckan_version(min_version='2.9'):
import click
return click.confirm(msg)
else:
from ckan.lib.cli import query_yes_no
return query_yes_no(msg) == 'yes'
def run_validation(self):
if self.options.resource_id:
for resource_id in self.options.resource_id:
resource = get_action('resource_show')({}, {'id': resource_id})
self._run_validation_on_resource(resource['id'],
resource['package_id'])
else:
query = _search_datasets()
if query['count'] == 0:
error('No suitable datasets, exiting...')
elif not self.options.assume_yes:
msg = ('\nYou are about to start validation for {0} datasets' +
'.\n Do you want to continue?')
confirm = query_yes_no(msg.format(query['count']))
if confirm == 'no':
error('Command aborted by user')
result = get_action('resource_validation_run_batch')(
{
'ignore_auth': True
}, {
'dataset_ids': self.options.dataset_id,
'query': self.options.search_params
})
print(result['output'])
def spc_del_datasets_from_list(self):
"""
Purge all datasets from provided list of IDs
"""
if self.args and len(self.args) == 2:
file = self.args[1]
site_user = logic.get_action(u'get_site_user')(
{u'ignore_auth': True}, {})
else:
error('Please, provide only a file path to CSV file with package IDs.')
# read all package ids from file
with open(file) as file:
file = csv.reader(file, quoting=csv.QUOTE_NONE)
next(file) # skip headers
pkg_ids = {pkg[0].split(';')[0] for pkg in file}
print('Are you sure you want to purge {} packages?'.format(len(pkg_ids)))
print('This action is irreversible!')
confirm = query_yes_no('Do you want to proceed?:', default='no')
if confirm == 'no':
error('Command aborted by user')
return
for idx, pkg_id in enumerate(pkg_ids, start=1):
print('Purging packages in progress: {}/{}'.format(idx, len(pkg_ids)))
try:
context = {u'user': site_user[u'name']}
tk.get_action('dataset_purge')(context, {'id': pkg_id})
print('Package {} was purged'.format(pkg_id))
except logic.NotFound:
print('Dataset with id {} wasn\'t found. Skipping...'.format(pkg_id))
print('DONE')
def _confirm_or_abort(self):
question = (
"Data in any datastore resource that isn't in their source files "
"(e.g. data added using the datastore API) will be permanently "
"lost. Are you sure you want to proceed?")
answer = cli.query_yes_no(question, default=None)
if not answer == 'yes':
print "Aborting..."
sys.exit(0)
def _run_on_datasets(self):
log = logging.getLogger(__name__)
page = 1
count_resources = 0
while True:
query = self._search_datasets(page)
if page == 1 and query['count'] == 0:
error('No suitable datasets, exiting...')
elif page == 1 and not self.options.assume_yes:
msg = ('\nYou are about to start validation for {0} datasets' +
'.\n Do you want to continue?')
confirm = query_yes_no(msg.format(query['count']))
if confirm == 'no':
error('Command aborted by user')
if query['results']:
for dataset in query['results']:
if not dataset.get('resources'):
continue
for resource in dataset['resources']:
if (not resource.get(u'format', u'').lower()
in settings.SUPPORTED_FORMATS):
continue
try:
self._run_validation_on_resource(
resource['id'], dataset['name'])
count_resources += 1
except ValidationError as e:
log.warning(
u'Could not run validation for resource {} ' +
u'from dataset {}: {}'.format(
resource['id'], dataset['name'], str(e)))
if len(query['results']) < self._page_size:
break
page += 1
else:
break
log.info('Done. {} resources sent to the validation queue'.format(
count_resources))
def _confirm_or_abort(self):
question = (
"Data in any datastore resource that isn't in their source files "
"(e.g. data added using the datastore API) will be permanently "
"lost. Are you sure you want to proceed?"
)
answer = cli.query_yes_no(question, default=None)
if not answer == 'yes':
print "Aborting..."
sys.exit(0)
def spc_find_detached_datasets(self):
"""
You must provide the creator_user_id
Use a comma as a separator to provide multiple ids
"""
if self.args and len(self.args) == 2:
creator_user_ids = self.args[1].split(',')
else:
error('Please, provide only the creator_user_id(s)')
# check if the file is already exist
# ask for rewrite confirmation
if os.path.isfile('detached_datasets.csv'):
print('File detached_datasets.csv is already exist.')
confirm = query_yes_no('Do you want to rewrite it?:', default='no')
if confirm == 'no':
error('Command aborted by user')
return
obj = harvest_model.HarvestObject
pkg = model.Package
# getting all packages that have no related harvest_object
subquery = model.Session.query(obj.package_id).distinct().subquery()
detached_pkgs = model.Session.query(pkg) \
.outerjoin(subquery, pkg.id == subquery.c.package_id) \
.filter(pkg.creator_user_id.in_(creator_user_ids),
subquery.c.package_id.is_(None)) \
.filter(pkg.type != 'harvest')
pkg_list = detached_pkgs.all()
if not pkg_list:
print('There is no detached datasets at all')
return
print('{} detached datasets found'.format(len(pkg_list)))
print('Creating report .csv file')
# generating report with detached datasets
try:
self._generate_detached_report(pkg_list)
except csv.Error as e:
error('Failed. An error occured during writing report: {}'.format(e))
print('DONE')
def _submit_all(self):
question = (
"Data in any datastore resource that isn't in their source files "
"(e.g. data added using the datastore API) will be permanently "
"lost. Are you sure you want to proceed?")
answer = cli.query_yes_no(question, default=None)
if answer == 'yes':
resources_ids = datastore_db.get_all_resources_ids_in_datastore()
print 'Submitting %d datastore resources' % len(resources_ids)
datapusher_submit = p.toolkit.get_action('datapusher_submit')
for resource_id in resources_ids:
print('Submitting %s...' % resource_id),
data_dict = {
'resource_id': resource_id,
'ignore_hash': True,
}
if datapusher_submit(None, data_dict):
print 'OK'
else:
print 'Fail'
def _submit_all(self):
question = (
"Data in any datastore resource that isn't in their source files "
"(e.g. data added using the datastore API) will be permanently "
"lost. Are you sure you want to proceed?"
)
answer = cli.query_yes_no(question, default=None)
if answer == 'yes':
resources_ids = datastore_db.get_all_resources_ids_in_datastore()
print 'Submitting %d datastore resources' % len(resources_ids)
datapusher_submit = p.toolkit.get_action('datapusher_submit')
for resource_id in resources_ids:
print ('Submitting %s...' % resource_id),
data_dict = {
'resource_id': resource_id,
'ignore_hash': True,
}
if datapusher_submit(None, data_dict):
print 'OK'
else:
print 'Fail'
def command(self):
output_file = self.args[0]
msg = "Are you sure you want to reset ALL of the user passwords?"
confirm = query_yes_no(msg, default='no')
if confirm == 'no':
sys.stderr.write("Aborting...\n")
sys.exit(1)
self._load_config()
import ckan.model as model
import ckan.lib.search as search
with open(output_file, 'wb') as f:
writer = csv.writer(f)
writer.writerow(['id', 'email-address', 'reset-key'])
for user in model.User.all():
create_reset_key(user)
user.save()
writer.writerow([user.id, user.email, user.reset_key])