def get_local_upload_path(resource_id):
u'''
Returns the local path to an uploaded file give an id
Note: it does not check if the resource or file actually exists
'''
upload = ResourceUpload({u'url': u'foo'})
return upload.get_path(resource_id)
def test_resource_without_upload(self, mock_uploads_enabled, mock_open,
send_file):
# this test data is based on real observation using a browser
res = {
u'clear_upload': u'true',
u'format': u'PNG',
u'url': u'https://example.com/data.csv',
u'description': u'',
u'upload': FileStorage(filename=u''),
u'package_id': u'dataset1',
u'id': u'8a3a874e-5ee1-4e43-bdaf-e2569cf72344',
u'name': u'data.csv'
}
res_upload = ResourceUpload(res)
eq_(res_upload.filename, None)
def test_resource_without_upload(self, ckan_config, monkeypatch, tmpdir):
monkeypatch.setitem(ckan_config, u'ckan.storage_path', str(tmpdir))
# this test data is based on real observation using a browser
res = {
u'clear_upload': u'true',
u'format': u'PNG',
u'url': u'https://example.com/data.csv',
u'description': u'',
u'upload': FileStorage(filename=u''),
u'package_id': u'dataset1',
u'id': u'8a3a874e-5ee1-4e43-bdaf-e2569cf72344',
u'name': u'data.csv'
}
res_upload = ResourceUpload(res)
assert res_upload.filename is None
def migrate(self):
'''
Migrate filestore over in our very HDXish way :)
'''
results = Session.execute(
"select id, revision_id, url from resource "
"where resource_type = 'file.upload' "
"and (url_type <> 'upload' or url_type is null)"
"and url like '%storage%'")
for id, revision_id, url in results:
# Give it a second, would you?
time.sleep(0.7)
url_parts = urlparse(url)
url_parts = url_parts.path.split("/")
filename = url_parts[len(url_parts) - 1]
response = requests.get(url, stream=True)
if response.status_code != 200:
print "failed to fetch %s (code %s)" % (url,
response.status_code)
continue
resource_upload = ResourceUpload({'id': id})
assert resource_upload.storage_path, "no storage configured aborting"
directory = resource_upload.get_directory(id)
filepath = resource_upload.get_path(id)
try:
os.makedirs(directory)
except OSError, e:
## errno 17 is file already exists
if e.errno != 17:
raise
with open(filepath, 'wb+') as out:
for chunk in response.iter_content(1024):
if chunk:
out.write(chunk)
Session.execute("update resource set url_type = 'upload', "
"url = '%s' where id = '%s'" % (filename, id))
Session.execute(
"update resource_revision set url_type = 'upload', "
"url = '%s' where id = '%s' and "
"revision_id = '%s'" % (filename, id, revision_id))
Session.commit()
print "Saved url %s" % url
def get_resource_uploader(self, data_dict):
return ResourceUpload(data_dict)