def purge_task_data(context, data_dict):
'''
Purge data from the task_status and kombu_message tables
(used by CKAN tasks and Celery).
To just clear the Celery data (and not the task_status table),
see the 'celery clean' command in CKAN core.
:returns: number of task_status and Celery (kombu_message) rows deleted.
:rtype: dictionary
'''
logic.check_access('purge_task_data', context, data_dict)
model = context['model']
engine = model.meta.engine
purge_task_status = 'DELETE FROM task_status;'
purge_celery_data = 'DELETE FROM kombu_message;'
try:
task_status_rows_deleted = engine.execute(purge_task_status).rowcount
celery_rows_deleted = engine.execute(purge_celery_data).rowcount
except Exception, e:
raise logic.ActionError('Error executing sql: %s' % e)
def purge_revision_history(context, data_dict):
'''
Purge a given publisher's unused revision history.
:param group: the name or id of the publisher
:type group: string
:returns: number of resources and revisions purged.
:rtype: dictionary
'''
logic.check_access('purge_revision_history', context, data_dict)
model = context['model']
engine = model.meta.engine
group_id = logic.get_or_bust(data_dict, 'group')
group = model.Group.get(group_id)
if not group:
raise logic.NotFound('Publisher {0} not found'.format(group_id))
RESOURCE_IDS_SQL = '''
SELECT resource.id FROM resource
JOIN resource_group ON resource.resource_group_id = resource_group.id
JOIN member ON member.table_id = resource_group.package_id
JOIN "group" ON "group".id = member.group_id
WHERE "group".name = %s
AND "group".type = 'organization'
AND member.table_name = 'package'
AND resource.state = 'deleted'
'''
DELETE_REVISIONS_SQL = '''
DELETE FROM resource_revision
WHERE id IN ({sql})
'''.format(sql=RESOURCE_IDS_SQL)
# Not necessary to use a sub-select, but it allows re-use of sql statement
# and this isn't performance critical code.
DELETE_RESOURCES_SQL = '''
DELETE FROM resource WHERE id IN ({sql})
'''.format(sql=RESOURCE_IDS_SQL)
try:
number_revisions_deleted = engine.execute(
DELETE_REVISIONS_SQL,
group.name
).rowcount
number_resources_deleted = engine.execute(
DELETE_RESOURCES_SQL,
group.name
).rowcount
except Exception, e:
raise logic.ActionError('Error executing sql: %s' % e)
def purge_publisher_datasets(context, data_dict):
"""
Purge all deleted datasets belonging to a given publisher.
:returns: Info about the command's result
:rtype: dictionary
"""
def do_purge(dataset_ids):
model.repo.new_revision()
# Every single purge is a rather heavy operation
for ds_id in dataset_ids:
dataset = model.Package.get(ds_id)
dataset.purge()
model.repo.commit_and_remove()
logic.check_access('purge_publisher_datasets', context, data_dict)
model = context['model']
engine = model.meta.engine
publisher_name = logic.get_or_bust(data_dict, 'name')
group = model.Group.get(publisher_name)
if not group:
error_text = "Publisher '{0}' not found; please specify an existing publisher".format(publisher_name)
raise logic.NotFound(error_text)
deleted_datasets = '''
SELECT DISTINCT package.id
FROM package
INNER JOIN member ON (member.table_name='package' AND member.table_id=package.id)
INNER JOIN "group" ON ("group".id=member.group_id)
WHERE "group".name='{publisher_name}' AND package.state='deleted';
'''.format(publisher_name=publisher_name)
try:
db_rows = engine.execute(deleted_datasets)
datasets_to_delete = [elent[0] for elent in db_rows] # list comprehension
#datasets_to_delete = map(lambda elent : elent[0], db_rows) # lambda
num_datasets_to_delete = len(datasets_to_delete)
except Exception, e:
raise logic.ActionError('Error executing sql: %s' % e)
def purge_package_extra_revision(context, data_dict):
'''
Purge old data from the package_extra_revision table.
:returns: number of revisions purged.
:rtype: dictionary
'''
logic.check_access('purge_package_extra_revision', context, data_dict)
model = context['model']
engine = model.meta.engine
delete_old_extra_revisions = '''
DELETE FROM package_extra_revision WHERE current=false;
'''
try:
revision_rows_deleted = engine.execute(
delete_old_extra_revisions).rowcount
except Exception, e:
raise logic.ActionError('Error executing sql: %s' % e)
def skos_hierarchy_update(context, data_dict):
'''
Truncate organization hierarchy table and fill it again.
'''
model = context['model']
#logic.check_access('member_create', context, data_dict)
q_group = ckan_model.Session.query(ckan_model.Group)
try:
q = ckan_model.Session.query(ckan_model.Member)\
.filter(ckan_model.Member.table_name == 'group')\
.filter(ckan_model.Member.state == 'active')
for member in q.all():
member_dict = {
'id': member.group_id,
'object': member.table_id,
'object_type': 'group',
'capacity': 'public',
}
ckan.logic.get_action('member_delete')(context, member_dict)
#engine.execute(delete_org_parent)
#row_counts_int = engine.execute(row_count).cursor.rownumber
if len(q.all()) > 0:
raise logic.ActionError('Not all elements has been deleted')
#all_groups = engine.execute(groups_rows).fetchall()
all_groups = q_group.all()
full_list = data_dict.get('results', {}).get('bindings', [])
for current_elem in full_list:
cb_value = current_elem.get('cb_name', {}).get('value', None)
p_value = current_elem.get('p_name', {}).get('value', None)
p_group = None
cb_group = None
if cb_value is not None:
q_cb = ckan_model.Session.query(ckan_model.Group)\
.filter(ckan_model.Group.name == cb_value.lower())
cb_group_fetch = q_cb.all()
if len(cb_group_fetch)>0:
cb_group = cb_group_fetch[0]
if p_value is not None:
q_p = ckan_model.Session.query(ckan_model.Group)\
.filter(ckan_model.Group.name == p_value.lower())
p_group_fetch = q_p.all()
if len(p_group_fetch)>0:
p_group = p_group_fetch[0]
is_addable = True
if cb_group not in all_groups:
log.error('The group %s does not exists in the list of the groups' % cb_value)
is_addable = False
if p_group is None:
log.error('The group %s has no child' % p_value)
is_addable = False
elif p_group not in all_groups:
log.error('The group %s does not exists in the list of the groups' % p_value)
is_addable = False
if is_addable:
#key = hashlib.md5(cb_value).hexdigest()
#key = "'" + key + "'"
#INSERT_ORG_TABLE = '''
#INSERT INTO org_parent VALUES ({key}, {org_name}, {org_parent_name})
#'''.format(key=key, org_name=cb_value,
# org_parent_name=p_value)
if cb_group is not None:
cb_val = cb_group.id
else:
cb_val = None
if p_group is not None:
p_val = p_group.id
else:
p_val = None
member_dict = {
'object': cb_val,
'id': p_val,
'object_type': 'group',
'capacity': 'public',
}
logic.get_action('member_create')(context, member_dict)
except Exception, e:
log.error('Error during process skos_hierarchy_update: %s' % e)