def test_14_resource_no_id(self):
context = {"model": model,
"session": model.Session}
model.repo.new_revision()
model.Session.commit()
new_resource = {
'alt_url': u'empty resource group id',
'description': u'Full text. Needs escaping: " Umlaut: \xfc',
'size': u'123',
'format': u'plain text',
'hash': u'abc123',
'position': 0,
'state': u'active',
'url': u'test'
}
model.repo.new_revision()
resource_dict_save(new_resource, context)
model.Session.commit()
model.Session.remove()
res = model.Session.query(model.Resource).filter_by(url=u'test').one()
res_dictized = self.remove_changable_columns(resource_dictize(res, context))
assert res_dictized == new_resource, res_dictized
def test_14_resource_no_id(self):
context = {"model": model, "session": model.Session}
model.repo.new_revision()
model.Session.commit()
new_resource = {
'alt_url': u'empty resource group id',
'description': u'Full text. Needs escaping: " Umlaut: \xfc',
'size': u'123',
'format': u'plain text',
'hash': u'abc123',
'position': 0,
'state': u'active',
'url': u'test'
}
model.repo.new_revision()
resource_dict_save(new_resource, context)
model.Session.commit()
model.Session.remove()
res = model.Session.query(model.Resource).filter_by(url=u'test').one()
res_dictized = self.remove_changable_columns(
resource_dictize(res, context))
assert res_dictized == new_resource, res_dictized
def resource_update(context, data_dict):
model = context["model"]
session = context["session"]
user = context["user"]
id = data_dict["id"]
schema = context.get("schema") or default_update_resource_schema()
model.Session.remove()
resource = model.Resource.get(id)
context["resource"] = resource
if not resource:
logging.error("Could not find resource " + id)
raise NotFound(_("Resource was not found."))
check_access("resource_update", context, data_dict)
data, errors = validate(data_dict, schema, context)
if errors:
model.Session.rollback()
raise ValidationError(errors, resource_error_summary(errors))
rev = model.repo.new_revision()
rev.author = user
if "message" in context:
rev.message = context["message"]
else:
rev.message = _(u"REST API: Update object %s") % data.get("name", "")
resource = resource_dict_save(data, context)
if not context.get("defer_commit"):
model.repo.commit()
return resource_dictize(resource, context)
def resource_update(context, data_dict):
model = context['model']
session = context['session']
user = context['user']
id = data_dict["id"]
schema = context.get('schema') or default_update_resource_schema()
model.Session.remove()
resource = model.Resource.get(id)
context["resource"] = resource
if not resource:
raise NotFound(_('Resource was not found.'))
check_access('resource_update', context, data_dict)
data, errors = validate(data_dict, schema, context)
if errors:
model.Session.rollback()
raise ValidationError(errors, resource_error_summary(errors))
rev = model.repo.new_revision()
rev.author = user
if 'message' in context:
rev.message = context['message']
else:
rev.message = _(u'REST API: Update object %s') % data.get("name")
resource = resource_dict_save(data, context)
if not context.get('defer_commit'):
model.repo.commit()
return resource_dictize(resource, context)
def resource_update(context, data_dict):
model = context['model']
user = context['user']
id = data_dict["id"]
schema = context.get('schema') or ckan.logic.schema.default_update_resource_schema()
model.Session.remove()
resource = model.Resource.get(id)
context["resource"] = resource
if not resource:
logging.error('Could not find resource ' + id)
raise NotFound(_('Resource was not found.'))
check_access('resource_update', context, data_dict)
data, errors = validate(data_dict, schema, context)
if errors:
model.Session.rollback()
raise ValidationError(errors, error_summary(errors))
rev = model.repo.new_revision()
rev.author = user
if 'message' in context:
rev.message = context['message']
else:
rev.message = _(u'REST API: Update object %s') % data.get("name", "")
resource = model_save.resource_dict_save(data, context)
if not context.get('defer_commit'):
model.repo.commit()
return model_dictize.resource_dictize(resource, context)
def test_14_resource_no_id(self):
context = {"model": model, "session": model.Session}
model.Session.commit()
new_resource = {
"mimetype": None,
u"alt_url": u"empty resource group id",
"hash": u"abc123",
"description": u'Full text. Needs escaping: " Umlaut: \xfc',
"format": u"plain text",
"url": u"http://test_new",
"cache_url": None,
"cache_last_updated": None,
"state": u"active",
"mimetype_inner": None,
"url_type": None,
"last_modified": None,
"position": 0,
"size": None,
"size_extra": u"123",
"resource_type": None,
"name": None,
"package_id": "", # Just so we can save
}
resource_dict_save(new_resource, context)
model.Session.commit()
model.Session.remove()
# Remove the package id
del new_resource["package_id"]
res = (
model.Session.query(model.Resource)
.filter_by(url=u"http://test_new")
.one()
)
res_dictized = self.remove_changable_columns(
resource_dictize(res, context), True
)
assert res_dictized == new_resource, res_dictized
def test_14_resource_no_id(self):
context = {"model": model, "session": model.Session}
model.repo.new_revision()
model.Session.commit()
new_resource = {
'mimetype': None,
u'alt_url': u'empty resource group id',
'hash': u'abc123',
'description': u'Full text. Needs escaping: " Umlaut: \xfc',
'format': u'plain text',
'url': u'http://test_new',
'cache_url': None,
'webstore_url': None,
'cache_last_updated': None,
'state': u'active',
'mimetype_inner': None,
'webstore_last_updated': None,
'url_type': None,
'last_modified': None,
'position': 0,
'size': None,
'size_extra': u'123',
'resource_type': None,
'name': None,
'package_id': '' # Just so we can save
}
model.repo.new_revision()
resource_dict_save(new_resource, context)
model.Session.commit()
model.Session.remove()
# Remove the package id
del new_resource['package_id']
res = model.Session.query(
model.Resource).filter_by(url=u'http://test_new').one()
res_dictized = self.remove_changable_columns(
resource_dictize(res, context), True)
assert res_dictized == new_resource, res_dictized
def test_14_resource_no_id(self):
context = {"model": model,
"session": model.Session}
model.repo.new_revision()
model.Session.commit()
new_resource = {
'mimetype': None,
u'alt_url': u'empty resource group id',
'hash': u'abc123',
'description': u'Full text. Needs escaping: " Umlaut: \xfc',
'format': u'plain text',
'url': u'http://test_new',
'cache_url': None,
'webstore_url': None,
'cache_last_updated': None,
'state': u'active',
'mimetype_inner': None,
'webstore_last_updated': None,
'url_type': None,
'last_modified': None,
'position': 0,
'size': None,
'size_extra': u'123',
'resource_type': None,
'name': None,
'package_id':'' # Just so we can save
}
model.repo.new_revision()
resource_dict_save(new_resource, context)
model.Session.commit()
model.Session.remove()
# Remove the package id
del new_resource['package_id']
res = model.Session.query(model.Resource).filter_by(url=u'http://test_new').one()
res_dictized = self.remove_changable_columns(resource_dictize(res, context), True)
assert res_dictized == new_resource, res_dictized
def test_14_resource_no_id(self):
context = {"model": model, "session": model.Session}
model.repo.new_revision()
model.Session.commit()
new_resource = {
"mimetype": None,
u"alt_url": u"empty resource group id",
"hash": u"abc123",
"description": u'Full text. Needs escaping: " Umlaut: \xfc',
"format": u"plain text",
"url": u"http://test_new",
"cache_url": None,
"cache_last_updated": None,
"state": u"active",
"mimetype_inner": None,
"url_type": None,
"last_modified": None,
"position": 0,
"size": None,
"size_extra": u"123",
"resource_type": None,
"name": None,
"package_id": "", # Just so we can save
}
model.repo.new_revision()
resource_dict_save(new_resource, context)
model.Session.commit()
model.Session.remove()
# Remove the package id
del new_resource["package_id"]
res = model.Session.query(model.Resource).filter_by(url=u"http://test_new").one()
res_dictized = self.remove_changable_columns(resource_dictize(res, context), True)
assert res_dictized == new_resource, res_dictized
def test_package_dictize_resource_upload_with_url_and_striped(self):
dataset = factories.Dataset()
resource = factories.Resource(package=dataset['id'],
name='test_pkg_dictize',
url_type='upload',
url='http://some_filename.csv')
context = {'model': model, 'session': model.Session}
result = model_save.resource_dict_save(resource, context)
expected_dict = {u'url': u'some_filename.csv', u'url_type': u'upload'}
assert expected_dict['url'] == result.url
def resource_update(context, data_dict):
'''Update a resource.
To update a resource you must be authorized to update the dataset that the
resource belongs to.
For further parameters see ``resource_create()``.
:param id: the id of the resource to update
:type id: string
:returns: the updated resource
:rtype: string
'''
model = context['model']
user = context['user']
id = _get_or_bust(data_dict, "id")
resource = model.Resource.get(id)
context["resource"] = resource
if not resource:
logging.error('Could not find resource ' + id)
raise NotFound(_('Resource was not found.'))
_check_access('resource_update', context, data_dict)
if 'schema' in context:
schema = context['schema']
else:
package_plugin = lib_plugins.lookup_package_plugin(
resource.resource_group.package.type)
schema = package_plugin.update_package_schema()['resources']
data, errors = _validate(data_dict, schema, context)
if errors:
model.Session.rollback()
raise ValidationError(errors)
rev = model.repo.new_revision()
rev.author = user
if 'message' in context:
rev.message = context['message']
else:
rev.message = _(u'REST API: Update object %s') % data.get("name", "")
resource = model_save.resource_dict_save(data, context)
if not context.get('defer_commit'):
model.repo.commit()
return model_dictize.resource_dictize(resource, context)
def resource_update(context, data_dict):
'''Update a resource.
To update a resource you must be authorized to update the dataset that the
resource belongs to.
For further parameters see ``resource_create()``.
:param id: the id of the resource to update
:type id: string
:returns: the updated resource
:rtype: string
'''
model = context['model']
user = context['user']
id = _get_or_bust(data_dict, "id")
resource = model.Resource.get(id)
context["resource"] = resource
if not resource:
logging.error('Could not find resource ' + id)
raise NotFound(_('Resource was not found.'))
_check_access('resource_update', context, data_dict)
if 'schema' in context:
schema = context['schema']
else:
package_plugin = lib_plugins.lookup_package_plugin(
resource.resource_group.package.type)
schema = package_plugin.update_package_schema()['resources']
data, errors = _validate(data_dict, schema, context)
if errors:
model.Session.rollback()
raise ValidationError(errors)
rev = model.repo.new_revision()
rev.author = user
if 'message' in context:
rev.message = context['message']
else:
rev.message = _(u'REST API: Update object %s') % data.get("name", "")
resource = model_save.resource_dict_save(data, context)
if not context.get('defer_commit'):
model.repo.commit()
return model_dictize.resource_dictize(resource, context)
def test_package_dictize_resource_upload_with_url_and_striped(self):
dataset = factories.Dataset()
resource = factories.Resource(
package=dataset["id"],
name="test_pkg_dictize",
url_type="upload",
url="http://some_filename.csv",
)
context = {"model": model, "session": model.Session}
result = model_save.resource_dict_save(resource, context)
expected_dict = {u"url": u"some_filename.csv", u"url_type": u"upload"}
assert expected_dict["url"] == result.url
def test_package_dictize_resource_upload_with_url_and_striped(self):
dataset = factories.Dataset()
resource = factories.Resource(package=dataset['id'],
name='test_pkg_dictize',
url_type='upload',
url='http://some_filename.csv')
context = {'model': model, 'session': model.Session}
result = model_save.resource_dict_save(resource, context)
expected_dict = {
u'url': u'some_filename.csv',
u'url_type': u'upload'
}
assert expected_dict['url'] == result.url
def resource_update(context, data_dict):
"""Update a resource.
To update a resource you must be authorized to update the dataset that the
resource belongs to.
For further parameters see ``resource_create()``.
:param id: the id of the resource to update
:type id: string
:returns: the updated resource
:rtype: string
"""
model = context["model"]
user = context["user"]
id = _get_or_bust(data_dict, "id")
schema = context.get("schema") or ckan.logic.schema.default_update_resource_schema()
model.Session.remove()
resource = model.Resource.get(id)
context["resource"] = resource
if not resource:
logging.error("Could not find resource " + id)
raise NotFound(_("Resource was not found."))
_check_access("resource_update", context, data_dict)
data, errors = _validate(data_dict, schema, context)
if errors:
model.Session.rollback()
raise ValidationError(errors)
rev = model.repo.new_revision()
rev.author = user
if "message" in context:
rev.message = context["message"]
else:
rev.message = _(u"REST API: Update object %s") % data.get("name", "")
resource = model_save.resource_dict_save(data, context)
if not context.get("defer_commit"):
model.repo.commit()
return model_dictize.resource_dictize(resource, context)
def resource_update(context, data_dict):
model = context["model"]
session = context["session"]
user = context["user"]
id = data_dict["id"]
schema = context.get("schema") or default_update_resource_schema()
model.Session.remove()
resource = model.Resource.get(id)
context["resource"] = resource
if not resource:
raise NotFound(_("Resource was not found."))
# check authentication against the resource package
# TODO: can check_access be used against a resource?
query = (
session.query(model.Package)
.join(model.ResourceGroup)
.join(model.Resource)
.filter(model.ResourceGroup.id == resource.resource_group_id)
)
pkg = query.first()
if not pkg:
raise NotFound(_("No package found for this resource, cannot check auth."))
check_access("package_update", context, package_dictize(pkg, context))
data, errors = validate(data_dict, schema, context)
if errors:
model.Session.rollback()
raise ValidationError(errors, resource_error_summary(errors))
rev = model.repo.new_revision()
rev.author = user
if "message" in context:
rev.message = context["message"]
else:
rev.message = _(u"REST API: Update object %s") % data.get("name")
resource = resource_dict_save(data, context)
model.repo.commit()
return resource_dictize(resource, context)
## update package positions
try:
pkg_dict = _get_action('package_update')(context, pkg_dict)
except ValidationError, e:
errors = e.error_dict['resources'][-1]
raise ValidationError(errors)
## end update package positions
rev = model.repo.new_revision()
rev.author = user
if 'message' in context:
rev.message = context['message']
else:
rev.message = _(u'REST API: Update object %s') % data.get("name", "")
resource = model_save.resource_dict_save(data, context)
if not context.get('defer_commit'):
model.repo.commit()
return model_dictize.resource_dictize(resource, context)
def package_update(context, data_dict):
'''Update a dataset (package).
You must be authorized to edit the dataset and the groups that it belongs
to.
Plugins may change the parameters of this function depending on the value
of the dataset's ``type`` attribute, see the ``IDatasetForm`` plugin
interface.
def after_create(self, context, data):
log.info(
'After Create GOOGLE DRIVE PLUGIN: Context: {}, Data: {}'.format(
context, data))
""" SAMPLE LOGs
Context = {
'allow_state_change': True,
'allow_partial_update': True,
'auth_user_obj': "USER OBJECT" ,
'__auth_user_obj_checked': True,
'session': " OBJECT sqlalchemy.orm.scoping.scoped_session",
'user': '******',
'__auth_audit': [],
'model': 'MODULE ckan.model from /srv/app/src/ckan/ckan/model/__init__.pyc',
'save': True,
'message': ''
}
Data = {
'maintainer': '',
'name': 'polonia55644',
'author': '',
'author_email': '',
'title': 'Dataset title',
'notes': '',
'owner_org': '60xxxxxxxxxxxxxxxxxxxx',
'private': True,
'maintainer_email': '',
'url': 'https://docs.google.com/spreadsheets/d/1dUgDpVYDywwdqtWw0A8qqaCstgtb5GygsaTDRMIgUAo/edit?usp=drive_web&ouid=100530708543483213226',
'state': 'draft',
'version': '',
'creator_user_id': '4eexxxxxxxxxxxxxxxxxxxxx',
'id': '4f420a31-xxxxxxxxxxxxxxxx',
'license_id': 'cc-by',
'type': 'dataset',
'tag_string': '',
'tags': [],
'extras': []
}
"""
url = data.get('url', None)
log.info('GOOGLE DRIVE PLUGIN url: {}'.format(url))
# detect google drive URLs
# https://docs.google.com/spreadsheets/d/1dUgDpVYDywwdqtWw0A8qqaCstgtb5GygsaTDRMIgUAo/edit#gid=1559509753
if url is None or not url.startswith(
'https://docs.google.com/spreadsheets'):
return
# I supouse never exists resources in creation time
resources = data.get('resources', None)
log.info('GOOGLE DRIVE PLUGIN Resources: {}'.format(resources))
if resources is not None and len(resources) > 0:
return
#TODO transform URL to get data as CSV
#TODO Download CSV
# save the resources (it works)
#TODO lear URL an other params
res_dict = {
# id: 0,
'url': 'http://no-se.com',
'format': 'CSV',
'description':
'Auto generated CSV from Google Drive Sheet Download',
'hash': '',
'package_id': data.get('id')
}
log.info('Save CSV data GOOGLE DRIVE PLUGIN as Resources: {}'.format(
res_dict))
resource = resource_dict_save(res_dict=res_dict, context=context)
