def portal_update_worker(self, source):
"""
a process that accepts package ids on stdin which are passed to
the package_show API on the remote CKAN instance and compared
to the local version of the same package. The local package is
then created, updated, deleted or left unchanged. This process
outputs that action as a string 'created', 'updated', 'deleted'
or 'unchanged'
"""
registry = RemoteCKAN(source)
portal = LocalCKAN()
now = datetime.now()
for package_id in iter(sys.stdin.readline, ''):
try:
data = registry.action.package_show(id=package_id.strip())
source_pkg = data['result']
except NotAuthorized:
source_pkg = None
_trim_package(source_pkg)
if source_pkg:
# treat unpublished packages same as deleted packages
if not source_pkg['portal_release_date'] or isodate(
source_pkg['portal_release_date'], None) > now:
source_pkg = None
try:
# don't pass user in context so deleted packages
# raise NotAuthorized
target_pkg = portal.call_action('package_show',
{'id':package_id.strip()}, {})
except (NotFound, NotAuthorized):
target_pkg = None
_trim_package(target_pkg)
if target_pkg is None and source_pkg is None:
result = 'unchanged'
elif target_pkg is None:
# CREATE
portal.action.package_create(**source_pkg)
result = 'created'
elif source_pkg is None:
# DELETE
portal.action.package_delete(id=package_id.strip())
result = 'deleted'
elif source_pkg == target_pkg:
result = 'unchanged'
else:
# UPDATE
portal.action.package_update(**source_pkg)
result = 'updated'
sys.stdout.write(result + '\n')
try:
sys.stdout.flush()
except IOError:
break
def publish(self):
lc = LocalCKAN(username=c.user)
publish_date = date_str_to_datetime(request.str_POST['publish_date']
).strftime("%Y-%m-%d %H:%M:%S")
# get a list of package id's from the for POST data
for key, package_id in request.str_POST.iteritems():
if key == 'publish':
old = lc.action.package_show(id=package_id)
lc.call_action('package_update', dict(old,
portal_release_date=publish_date))
#return us to the publishing interface
url = h.url_for(controller='ckanext.canada.controller:PublishController',
action='search')
redirect(url)
def test_invalid_url(self):
lc = LocalCKAN()
try:
dataset = factories.Dataset()
lc.call_action(
'resource_create',
{
'package_id': dataset['name'],
'name': 'Test-File',
'url': 'https://example.com]'
}
)
except ValidationError as e:
eq_(
e.error_dict['url'],
[u'Please provide a valid URL']
)
else:
raise AssertionError('ValidationError not raised')
def copy_datasets(self, remote, package_ids=None):
"""
a process that accepts packages on stdin which are compared
to the local version of the same package. The local package is
then created, updated, deleted or left unchanged. This process
outputs that action as a string 'created', 'updated', 'deleted'
or 'unchanged'
"""
portal = LocalCKAN()
now = datetime.now()
packages = iter(sys.stdin.readline, '')
for package in packages:
source_pkg = json.loads(package)
package_id = source_pkg['id']
reason = None
target_deleted = False
if source_pkg and source_pkg['state'] == 'deleted':
source_pkg = None
if source_pkg and source_pkg['type'] not in DATASET_TYPES:
# non-default dataset types ignored
source_pkg = None
_trim_package(source_pkg)
action = None
if source_pkg and not self.options.mirror:
if source_pkg.get('ready_to_publish') == 'false':
source_pkg = None
reason = 'marked not ready to publish'
elif not source_pkg.get('portal_release_date'):
source_pkg = None
reason = 'release date not set'
elif isodate(source_pkg['portal_release_date'], None) > now:
source_pkg = None
reason = 'release date in future'
else:
# portal packages published public
source_pkg['private'] = False
if action != 'skip':
try:
target_pkg = portal.call_action('package_show', {
'id': package_id
})
except (NotFound, NotAuthorized):
target_pkg = None
except (CKANAPIError, urllib2.URLError), e:
sys.stdout.write(
json.dumps([
package_id,
'target error',
unicode(e.args)
]) + '\n'
)
raise
if target_pkg and target_pkg['state'] == 'deleted':
target_pkg = None
target_deleted = True
_trim_package(target_pkg)
if action == 'skip':
pass
elif target_pkg is None and source_pkg is None:
action = 'unchanged'
reason = reason or 'deleted on registry'
elif target_deleted:
action = 'updated'
reason = 'undeleting on target'
portal.action.package_update(**source_pkg)
elif target_pkg is None:
action = 'created'
portal.action.package_create(**source_pkg)
elif source_pkg is None:
action = 'deleted'
portal.action.package_delete(id=package_id)
elif source_pkg == target_pkg:
action = 'unchanged'
reason = 'no difference found'
else:
action = 'updated'
portal.action.package_update(**source_pkg)
sys.stdout.write(json.dumps([package_id, action, reason]) + '\n')
sys.stdout.flush()
def copy_datasets(self, remote, package_ids=None):
"""
a process that accepts packages on stdin which are compared
to the local version of the same package. The local package is
then created, updated, deleted or left unchanged. This process
outputs that action as a string 'created', 'updated', 'deleted'
or 'unchanged'
"""
portal = LocalCKAN()
now = datetime.now()
packages = iter(sys.stdin.readline, '')
for package in packages:
source_pkg = json.loads(package)
package_id = source_pkg['id']
reason = None
target_deleted = False
if source_pkg and source_pkg['state'] == 'deleted':
source_pkg = None
if source_pkg and source_pkg['type'] not in DATASET_TYPES:
# non-default dataset types ignored
source_pkg = None
_trim_package(source_pkg)
action = None
if source_pkg and not self.options.mirror:
if source_pkg.get('ready_to_publish') == 'false':
source_pkg = None
reason = 'marked not ready to publish'
elif not source_pkg.get('portal_release_date'):
source_pkg = None
reason = 'release date not set'
elif isodate(source_pkg['portal_release_date'], None) > now:
source_pkg = None
reason = 'release date in future'
else:
# portal packages published public
source_pkg['private'] = False
if action != 'skip':
try:
target_pkg = portal.call_action('package_show', {
'id': package_id
})
except (NotFound, NotAuthorized):
target_pkg = None
except (CKANAPIError, urllib2.URLError), e:
sys.stdout.write(
json.dumps([
package_id,
'target error',
unicode(e.args)
]) + '\n'
)
raise
if target_pkg and target_pkg['state'] == 'deleted':
target_pkg = None
target_deleted = True
_trim_package(target_pkg)
if action == 'skip':
pass
elif target_pkg is None and source_pkg is None:
action = 'unchanged'
reason = reason or 'deleted on registry'
elif target_deleted:
action = 'updated'
reason = 'undeleting on target'
portal.action.package_update(**source_pkg)
elif target_pkg is None:
action = 'created'
portal.action.package_create(**source_pkg)
elif source_pkg is None:
action = 'deleted'
portal.action.package_delete(id=package_id)
elif source_pkg == target_pkg:
action = 'unchanged'
reason = 'no difference found'
else:
action = 'updated'
portal.action.package_update(**source_pkg)
sys.stdout.write(json.dumps([package_id, action, reason]) + '\n')
sys.stdout.flush()
