def _handle_files(self, action, data_dict, context, files):
if action not in ['resource_create', 'resource_update']:
raise CKANAPIError("LocalCKAN.call_action only supports file uploads for resources.")
new_data_dict = dict(data_dict)
if action == 'resource_create':
if 'url' not in new_data_dict or new_data_dict['url']:
new_data_dict['url'] = '/tmp-file' # url needs to be set, otherwise there is a ValidationError
resource = self._get_action(action)(dict(context), new_data_dict)
else:
resource = new_data_dict
from ckan.lib.uploader import ResourceUpload
resource_upload = ResourceUpload({'id': resource['id']})
# get first upload, ignore key
source_file = files.values()[0]
if not resource_upload.storage_path:
raise CKANAPIError("No storage configured, unable to upload files")
directory = resource_upload.get_directory(resource['id'])
filepath = resource_upload.get_path(resource['id'])
try:
os.makedirs(directory)
except OSError, e:
## errno 17 is file already exists
if e.errno != 17:
raise
def get_local_upload_path(resource_id):
u'''
Returns the local path to an uploaded file give an id
Note: it does not check if the resource or file actually exists
'''
upload = ResourceUpload({u'url': u'foo'})
return upload.get_path(resource_id)
def migrate(self):
'''
Migrate filestore over in our very HDXish way :)
'''
results = Session.execute(
"select id, revision_id, url from resource "
"where resource_type = 'file.upload' "
"and (url_type <> 'upload' or url_type is null)"
"and url like '%storage%'")
for id, revision_id, url in results:
# Give it a second, would you?
time.sleep(0.7)
url_parts = urlparse(url)
url_parts = url_parts.path.split("/")
filename = url_parts[len(url_parts) - 1]
response = requests.get(url, stream=True)
if response.status_code != 200:
print "failed to fetch %s (code %s)" % (url,
response.status_code)
continue
resource_upload = ResourceUpload({'id': id})
assert resource_upload.storage_path, "no storage configured aborting"
directory = resource_upload.get_directory(id)
filepath = resource_upload.get_path(id)
try:
os.makedirs(directory)
except OSError, e:
## errno 17 is file already exists
if e.errno != 17:
raise
with open(filepath, 'wb+') as out:
for chunk in response.iter_content(1024):
if chunk:
out.write(chunk)
Session.execute("update resource set url_type = 'upload', "
"url = '%s' where id = '%s'" % (filename, id))
Session.execute(
"update resource_revision set url_type = 'upload', "
"url = '%s' where id = '%s' and "
"revision_id = '%s'" % (filename, id, revision_id))
Session.commit()
print "Saved url %s" % url