def __init__(self, size, filename):
self.file = StringIO("x" * size)
self.file.filename = filename
self.file.headers = []
self.file.name = "file1"
self.file.file = self.file
FileUpload.__init__(self, self.file)
def _prepare_file(self, filename='test.bin', type_='text/plain'):
fp = TemporaryFile('w+b')
fp.write('\x00' + 'x' * (1 << 19))
fp.seek(0)
env = {'REQUEST_METHOD': 'POST'}
headers = {'Content-Type': type_,
'content-length': 1 << 19,
'content-disposition': 'attachment; filename=%s' % filename}
fs = FieldStorage(fp=fp, environ=env, headers=headers)
fu = FileUpload(fs)
fu.read = fp.read
fu.seek = fp.seek
fu.tell = fp.tell
return fu
def _prepare_file(self, filename='test.bin', type_='text/plain'):
fp = TemporaryFile('w+b')
fp.write('\x00' + 'x' * (1 << 19))
fp.seek(0)
env = {'REQUEST_METHOD': 'POST'}
headers = {'Content-Type': type_,
'content-length': 1 << 19,
'content-disposition': 'attachment; filename=%s' % filename}
fs = FieldStorage(fp=fp, environ=env, headers=headers)
fu = FileUpload(fs)
fu.read = fp.read
fu.seek = fp.seek
fu.tell = fp.tell
return fu
def test_suite():
tarball = FileUpload(FileHolder(os.path.join(datadir, 'project-macosx-10.3-fat.tar.gz')))
egg = FileUpload(FileHolder(os.path.join(datadir, 'project.egg')))
globs = globals()
globs['tarball'] = tarball
globs['egg'] = egg
return Suite(os.path.basename('pypi.txt'),
os.path.basename('permissions.txt'),
optionflags=OPTIONFLAGS,
package='Products.PloneSoftwareCenter.tests',
test_class=PSCFunctionalTestCase,
setUp=setUp, globs=globs)
def HTTPUpload(fp, content_type=None, filename=None):
""" Create a FileUpload instance from a file handle (and content_type) """
if isinstance(fp, FileUpload):
if content_type:
fp.headers['content-type'] = content_type
else:
environ = {'REQUEST_METHOD': 'POST'}
if content_type:
environ['CONTENT_TYPE'] = content_type
elif hasattr(fp, 'headers') and fp.headers.has_key('content-type'):
environ['CONTENT_TYPE'] = fp.headers['content-type']
fp = FileUpload(FieldStorage(fp, environ=environ))
if filename:
fp.filename = filename
return fp
def HTTPUpload(fp, content_type=None, filename=None):
""" Create a FileUpload instance from a file handle (and content_type) """
if isinstance(fp, FileUpload):
if content_type:
fp.headers["content-type"] = content_type
else:
environ = {"REQUEST_METHOD": "POST"}
if content_type:
environ["CONTENT_TYPE"] = content_type
elif hasattr(fp, "headers") and fp.headers.has_key("content-type"):
environ["CONTENT_TYPE"] = fp.headers["content-type"]
fp = FileUpload(FieldStorage(fp, environ=environ))
if filename:
fp.filename = filename
return fp
def HTTPUpload(fp, content_type=None, filename=None):
""" Create a FileUpload instance from a file handle (and content_type) """
if isinstance(fp, FileUpload):
if content_type:
fp.headers['content-type'] = content_type
else:
environ = {'REQUEST_METHOD': 'POST'}
if content_type:
environ['CONTENT_TYPE'] = content_type
elif hasattr(fp, 'headers') and fp.headers.has_key('content-type'):
environ['CONTENT_TYPE'] = fp.headers['content-type']
fp = FileUpload(FieldStorage(fp, environ=environ))
if filename:
fp.filename = filename
return fp
def test_first_step_validate_csv_encoding_nok(self):
"""Ensure that we can decode csv file"""
request = self.layer["request"]
csv = StringIO()
lines = [
[u"猫", u"èè", u"ùù"],
["", "key1", "Key 1"],
[u"猫", u"ààà", u"ééé"],
]
for line in lines:
csv.write(";".join(line).encode("utf-16") + "\n")
csv.seek(0)
source = FileUpload(
type(
"obj",
(object, ),
{
"file": csv,
"filename": "foo.csv",
"headers": "text/csv"
},
)())
request.form = {
"form.buttons.continue": u"continuer",
"form.widgets.separator": [u";"],
"form.widgets.separator-empty-marker": u"1",
"form.widgets.source": source,
"form.widgets.has_header": u"True",
}
form = importform.ImportFormFirstStep(self.container, request)
form.update()
data, errors = form.extractData()
self.assertEqual(1, len(errors))
self.assertEqual("File encoding is not utf8", errors[0].error.message)
def test_first_step_validate_line_columns_nok(self):
"""Ensure that every lines have the same number of columns"""
request = self.layer["request"]
csv = StringIO()
lines = [
["", "key1", "Key 1"],
["key1", "key1.1"],
["key1.1", "key1.1.1", "Key 1.1.1", "foo"],
]
for line in lines:
csv.write(";".join(line) + "\n")
csv.seek(0)
source = FileUpload(
type(
"obj",
(object, ),
{
"file": csv,
"filename": "foo.csv",
"headers": "text/csv"
},
)())
request.form = {
"form.buttons.continue": u"continuer",
"form.widgets.separator": [u";"],
"form.widgets.separator-empty-marker": u"1",
"form.widgets.source": source,
"form.widgets.has_header": u"False",
}
form = importform.ImportFormFirstStep(self.container, request)
form.update()
data, errors = form.extractData()
self.assertEqual(1, len(errors))
self.assertTrue("Lines 2, 3" in translate(errors[0].error.message))
def test_tidy_validator_with_upload_wrong_encoding(self):
doc = self._ATCT
field = doc.getField('text')
request = self.app.REQUEST
setattr(request, 'text_text_format', 'text/html')
input_file_name = 'tidy1-in.html'
in_file = open(input_file_path(input_file_name))
env = {'REQUEST_METHOD': 'PUT'}
headers = {
'content-type': 'text/html',
'content-length': len(in_file.read()),
'content-disposition':
'attachment; filename=%s' % input_file_name
}
in_file.seek(0)
fs = FieldStorage(fp=in_file, environ=env, headers=headers)
f = FileUpload(fs)
tcv = TidyHtmlWithCleanupValidator('tidy_validator_with_cleanup')
result = tcv.__call__(f, field=field, REQUEST=request)
self.assertEquals(result, 1)
expected_file = open(input_file_path('tidy1-out.html'))
expected = expected_file.read()
expected_file.close()
self.assertEquals(request['text_tidier_data'], expected)
def makeFileUpload(file, content_type=None, filename=None):
headers = {}
if type(file) == type(''):
file = open(file, 'rb')
if content_type:
headers['content-type'] = content_type
fs = DummyFieldStorage(file, filename, headers)
return FileUpload(fs)
def test_upload_zip(self):
# upload a zip in bccvl bagit format
view = self.getview()
from ZPublisher.HTTPRequest import FileUpload
from cgi import FieldStorage
from StringIO import StringIO
data = resource_string(__name__, 'spc_obl_merc.zip')
env = {'REQUEST_METHOD': 'PUT'}
headers = {'content-type': 'text/csv',
'content-length': str(len(data)),
'content-disposition': 'attachment; filename=spc_obl_merc.zip'}
fileupload = FileUpload(FieldStorage(fp=StringIO(data),
environ=env, headers=headers))
view.request.form.update({
'climatefuture.buttons.save': u'Save',
'climatefuture.widgets.description': u'some test.tif file',
'climatefuture.widgets.file': fileupload,
'climatefuture.widgets.title': u'test smulti layer title',
'climatefuture.widgets.legalcheckbox': [u'selected'],
'climatefuture.widgets.legalcheckbox-empty-marker': u'1',
'climatefuture.widgets.rightsstatement': u'test rights',
'climatefuture.widgets.rightsstatement.mimeType': u'text/html',
'climatefuture.widgets.emsc': u'SRESB2',
'climatefuture.widgets.gcm': u'cccma-cgcm31',
'climatefuture.widgets.resolution': u'Resolution5m',
'climatefuture.widgets.temporal': u'2015',
})
_ = view()
self.assertEqual(self.portal.REQUEST.response.status, 302)
self.assertEqual(self.portal.REQUEST.response.getHeader('Location'),
'http://nohost/plone/datasets')
ds = self.portal.datasets['spc_obl_merc.zip']
self.assertEqual(ds.rightsstatement.raw, u'test rights')
self.assertEqual(ds.file.data, data)
from org.bccvl.site.interfaces import IBCCVLMetadata
md = IBCCVLMetadata(ds)
self.assertEqual(md['genre'], 'DataGenreFC')
self.assertEqual(md['resolution'], u'Resolution5m')
self.assertEqual(md['temporal'], u'2015')
self.assertEqual(md['emsc'], u'SRESB2')
self.assertEqual(md['gcm'], u'cccma-cgcm31')
layermd = md['layers']['spc_obl_merc/data/spc_obl_merc_1.tif']
self.assertEqual(layermd['filename'], 'spc_obl_merc/data/spc_obl_merc_1.tif')
self.assertEqual(layermd['min'], 19.0)
self.assertEqual(layermd['max'], 128.0)
self.assertEqual(layermd['datatype'], 'continuous')
self.assertEqual(layermd['height'], 200)
self.assertEqual(layermd['width'], 200)
self.assertEqual(layermd['srs'], None)
layermd = md['layers']['spc_obl_merc/data/spc_obl_merc_2.tif']
self.assertEqual(layermd['filename'], 'spc_obl_merc/data/spc_obl_merc_2.tif')
self.assertEqual(layermd['min'], 19.0)
self.assertEqual(layermd['max'], 128.0)
self.assertEqual(layermd['datatype'], 'continuous')
self.assertEqual(layermd['height'], 200)
self.assertEqual(layermd['width'], 200)
self.assertEqual(layermd['srs'], None)
def loadfile(self, rel_filename, ctype='image/png'):
""" load a file
"""
home = package_home(product_globals)
filename = os.path.sep.join([home, rel_filename])
data = open(filename, 'r').read()
fp = StringIO(data)
fp.seek(0)
env = {'REQUEST_METHOD': 'PUT'}
headers = {'content-type': ctype,
'content-length': len(data),
'content-disposition': 'attachment; filename=test.txt'}
fs = FieldStorage(fp=fp, environ=env, headers=headers)
ufile = FileUpload(fs)
ufile.name = None
return ufile
def loadfile(self, rel_filename, ctype='image/png'):
""" load a file
"""
home = package_home(product_globals)
filename = os.path.sep.join([home, rel_filename])
data = open(filename, 'r').read()
fp = StringIO(data)
fp.seek(0)
env = {'REQUEST_METHOD': 'PUT'}
headers = {'content-type': ctype,
'content-length': len(data),
'content-disposition': 'attachment; filename=test.txt'}
fs = FieldStorage(fp=fp, environ=env, headers=headers)
ufile = FileUpload(fs)
ufile.name = None
return ufile
def make_file_upload(data, content_type=None, filename=None):
"""Create a FileUpload object.
"""
headers = {}
if isinstance(data, str):
data = open(data, 'rb')
if content_type:
headers['content-type'] = content_type
fs = DummyFieldStorage(data, filename, headers)
return FileUpload(fs)
def _prepareFormTarball(self):
"""we could use our @@export-easyform view,
but we want a more atomic test, so we make
a tarfile for our test. the approach to making
a tarfile is a bit strange, but does the job
"""
in_fname = "test_form_1_easyform.tar.gz"
test_dir = os.path.dirname(__file__)
def _add_form_structure_to_archive(archive):
form_relative_path = os.path.join(
"profiles",
"testing",
"structure",
"Members",
"test_user_1_",
"test_form_1_easyform",
)
abs_path = os.path.join(test_dir, form_relative_path)
# add structure folder
os.chdir(os.path.join(test_dir, "profiles", "testing"))
archive.add("structure", recursive=False)
for f in os.listdir(abs_path):
os.chdir(abs_path) # add form data w/o full directory tree
archive.add(f, arcname=os.path.join("structure", f))
# Capture the current working directory for later when we need to
# clean up the environment.
working_directory = os.path.abspath(os.curdir)
# make me a tarfile in the current dir
os.chdir(test_dir)
archive = TarFile.open(name=in_fname, mode="w:gz")
_add_form_structure_to_archive(archive)
archive.close()
# Change back to the working directory in case something tries to
# write files (e.g. collective.xmltestreport).
os.chdir(working_directory)
# get it and upload
in_file = open(os.path.join(test_dir, in_fname))
env = {"REQUEST_METHOD": "PUT"}
headers = {
"content-type": "text/html",
"content-length": len(in_file.read()),
"content-disposition":
"attachment; filename={0}".format(in_file.name),
} # noqa
in_file.seek(0)
fs = FieldStorage(fp=in_file, environ=env, headers=headers)
return FileUpload(fs)
def makeFileUpload(self,
filename='test.txt',
value='test text',
diskname=''):
if diskname:
filename = diskname
value = open(os.path.join(test_folder,
diskname)).read().replace('\r',
'').strip()
from ZPublisher.HTTPRequest import FileUpload
return FileUpload(DummyFieldStorage(filename, value))
def prepare_request(self):
fieldstorage = cgi.FieldStorage()
fieldstorage.file = resource_stream("opengever.bumblebee.tests.assets",
"vertragsentwurf.pdf")
fieldstorage.filename = 'test.pdf'
file = FileUpload(fieldstorage)
self.request.set('status', "success")
self.request.set('pdf', file)
self.request.set('token', get_demand_callback_access_token())
self.request.set('opaque_id', self.save_token)
self.request.method = "POST"
def _createBinaryFile():
from cgi import FieldStorage
from ZPublisher.HTTPRequest import FileUpload
from tempfile import TemporaryFile
fp = TemporaryFile('w+b')
fp.write('\x00' + 'x' * (1 << 19))
fp.seek(0)
env = {'REQUEST_METHOD':'PUT'}
headers = {'content-type':'application/msword',
'content-length': 1 << 19,
'content-disposition':'attachment; filename=test.doc'}
fs = FieldStorage(fp=fp, environ=env, headers=headers)
return FileUpload(fs)
def createFileUpload(data, filename):
fp = TemporaryFile('w+b')
fp.write(data)
fp.seek(0)
env = {'REQUEST_METHOD': 'PUT'}
headers = {'content-type':'text/plain',
'content-length': len(data),
'content-disposition':'attachment; filename=%s' % \
filename}
fs = FieldStorage(fp=fp, environ=env, headers=headers)
result = FileUpload(fs)
fp.close()
return result
def setUp(self):
self.portal = self.layer['portal']
file_ = open("%s/testfile_mimetypes.txt" %
os.path.split(__file__)[0], 'r')
self.file_data = file_.read()
fieldstorage = cgi.FieldStorage()
fieldstorage.file = file_
fieldstorage.filename = 'testfile_mimetypes.txt'
self.fileupload = FileUpload(fieldstorage)
mtr = getToolByName(self.portal, 'mimetypes_registry')
mtr.manage_addMimeType(
'Only globs mimetype', ['application/x-only-glob'], [],
'application.png', globs=['*.onlyglob'])
mtr.manage_addMimeType(
'Only extension mimetype', ['application/x-only-ext'],
['onlyext'], 'application.png')
def test_file_upload_binary(self):
from cgi import FieldStorage
from ZPublisher.HTTPRequest import FileUpload
from tempfile import TemporaryFile
fp = TemporaryFile('w+b')
fp.write('\x00' + 'x' * (1 << 19))
fp.seek(0)
env = {'REQUEST_METHOD': 'PUT'}
headers = {'content-type': 'text/plain',
'content-length': 1 << 19,
'content-disposition': 'attachment; filename=test.bin'}
fs = FieldStorage(fp=fp, environ=env, headers=headers)
f = FileUpload(fs)
v, m, f = self.field._process_input(f, instance=self.instance)
self.assertTrue(isinstance(v, self.factory), (type(v), self.factory))
self.assertEqual(m, 'application/octet-stream')
self.assertEqual(f, 'test.bin')
def loadxmlfile(rel_filename):
ctype = 'text/xml'
home = package_home(globals())
filename = os.path.sep.join([home, rel_filename])
data = open(filename, 'r').read()
fp = StringIO(data)
fp.seek(0)
header_filename = rel_filename.split('/')[-1]
env = {'REQUEST_METHOD': 'PUT'}
headers = {
'content-type': ctype,
'content-length': len(data),
'content-disposition': 'attachment; filename=%s' % (header_filename, )
}
fs = FieldStorage(fp=fp, environ=env, headers=headers)
return FileUpload(fs)
def test_upload_file(self):
fp = TemporaryFile('w+b')
fp.write('foobar')
fp.seek(0)
filename = 'test.xml'
env = {'REQUEST_METHOD': 'PUT'}
headers = {'content-type': 'text/plain',
'content-length': str(len('foobar')),
'content-disposition': 'attachment; filename=%s' % filename}
files_ = FileUpload(FieldStorage(fp=fp, environ=env, headers=headers))
fileids = self.context.name + 'fileids'
request = TestRequest(environ={'HTTP_ACCEPT_LANGUAGE': 'en', 'REQUEST_METHOD': 'POST', 'dummy': files_, fileids: 'fXXXXX'})
setattr(request, "dummy", "stuff")
setattr(request, "URL1", "http://127.0.0.1")
upload = Upload(self.context, request)
setattr(request, "REQUEST_METHOD", "POST")
json_value = upload()
self.assertTrue('"size": 6' in json_value)
self.assertTrue('"name": "test.xml"' in json_value)
request = TestRequest(environ={'HTTP_ACCEPT_LANGUAGE': 'en', 'REQUEST_METHOD': 'POST', 'dummy': None})
def test_05_EditSelectionWithFileUpload(self, quiet=quiet, run=run_all_test):
"""Listbox edits selection with request parameters. Special care must be
taken for FileUpload objects that cannot be pickled, thus cannot be stored
in the ZODB.
"""
portal = self.getPortal()
portal.ListBoxZuite_reset()
listbox = portal.FooModule_viewFooList.listbox
# XXX isn't Selection automatically created ?
name = listbox.get_value('selection_name')
portal.portal_selections.setSelectionFor(name, Selection(name))
request = get_request()
request['here'] = portal.foo_module
request.form['my_file_upload'] = FileUpload(DummyFieldStorage())
listbox.get_value('default', render_format='list', REQUEST=request)
try:
self.commit()
except TypeError, e:
self.fail('Unable to commit transaction: %s' % e)
def test_upload_occurrence(self):
view = self.getview()
from ZPublisher.HTTPRequest import FileUpload
from cgi import FieldStorage
from StringIO import StringIO
data = "species,lon,lat\nSpecies,1,2\nSpecies,2,3\n"
env = {'REQUEST_METHOD': 'PUT'}
headers = {'content-type': 'text/csv',
'content-length': str(len(data)),
'content-disposition': 'attachment; filename=test.csv'}
fileupload = FileUpload(FieldStorage(fp=StringIO(data),
environ=env, headers=headers))
view.request.form.update({
'speciesoccurrence.buttons.save': u'Save',
'speciesoccurrence.widgets.description': u'some test.csv file',
'speciesoccurrence.widgets.file': fileupload,
'speciesoccurrence.widgets.title': u'test species title',
'speciesoccurrence.widgets.legalcheckbox': [u'selected'],
'speciesoccurrence.widgets.legalcheckbox-empty-marker': u'1',
'speciesoccurrence.widgets.rightsstatement': u'test rights',
'speciesoccurrence.widgets.rightsstatement.mimeType': u'text/html',
'speciesoccurrence.widgets.scientificName': u'test species',
'speciesoccurrence.widgets.taxonID': u'test taxonid',
'speciesoccurrence.widgets.vernacularName': u'test it'
})
_ = view()
self.assertEqual(self.portal.REQUEST.response.status, 302)
self.assertEqual(self.portal.REQUEST.response.getHeader('Location'),
'http://nohost/plone/datasets')
ds = self.portal.datasets['test.csv']
self.assertEqual(ds.rightsstatement.raw, u'test rights')
self.assertEqual(ds.file.data, 'species,lon,lat\nSpecies,1,2\nSpecies,2,3\n')
from org.bccvl.site.interfaces import IBCCVLMetadata
md = IBCCVLMetadata(ds)
self.assertEqual(md['genre'], 'DataGenreSpeciesOccurrence')
self.assertEqual(md['species']['taxonID'], u'test taxonid')
self.assertEqual(md['species']['scientificName'], u'test species')
self.assertEqual(md['species']['vernacularName'], u'test it')
self.assertEqual(md['rows'], 2)
def test_formlib_form_import(self):
"""Interacting with our formlib form we should be able
to successfully upload an exported PFG form and have it
correctly configured.
"""
self._makeForm()
fields = self.ff1.fields_model
actions = self.ff1.actions_model
form_folder_export = getMultiAdapter((self.ff1, self.app.REQUEST),
name='export-easyform')
in_file = StringIO(form_folder_export())
env = {'REQUEST_METHOD': 'PUT'}
headers = {
'content-type': 'text/html',
'content-length': len(in_file.read()),
'content-disposition': 'attachment; filename=ff1.tar.gz'
}
in_file.seek(0)
fs = FieldStorage(fp=in_file, environ=env, headers=headers)
# setup a reasonable request
request = self.app.REQUEST
request.form = {
'form.widgets.upload': FileUpload(fs),
'form.buttons.import': 'import'
}
request.RESPONSE = request.response
# get the form object
import_form = getMultiAdapter((self.ff1, request),
name='import-easyform')
# call update (aka submit) on the form, see TestRequest above
import_form.update()
self.assertEqual(self.ff1.fields_model, fields)
self.assertEqual(self.ff1.actions_model, actions)
def loadfile(self, rel_filename, ctype='text/xml', zope=False):
""" load a file
"""
home = package_home(product_globals)
filename = os.path.sep.join([home, rel_filename])
data = open(filename, 'r').read()
fp = StringIO(data)
fp.seek(0)
if not zope:
return fp
header_filename = rel_filename.split('/')[-1]
env = {'REQUEST_METHOD': 'PUT'}
headers = {
'content-type': ctype,
'content-length': len(data),
'content-disposition': 'attachment; filename=%s' % header_filename
}
fs = FieldStorage(fp=fp, environ=env, headers=headers)
return FileUpload(fs)
def test_first_step_validate_columns_number_correct(self):
"""Ensure that csv file contains at least 2 columns"""
request = self.layer["request"]
source = FileUpload(
type(
"obj",
(object, ),
{
"file": self._csv,
"filename": "foo.csv",
"headers": "text/csv"
},
)())
request.form = {
"form.buttons.continue": u"Continuer",
"form.widgets.separator": [u";"],
"form.widgets.separator-empty-marker": u"1",
"form.widgets.source": source,
"form.widgets.has_header": u"False",
}
form = importform.ImportFormFirstStep(self.container, request)
form.update()
data, errors = form.extractData()
self.assertEqual(0, len(errors))
def test_upload_multi_csv_mac(self, mock_move=None):
testcsv = resource_string(__name__,
'mock_data/multi_occurrence_mac.csv')
view = self.getview()
from ZPublisher.HTTPRequest import FileUpload
from cgi import FieldStorage
from StringIO import StringIO
env = {'REQUEST_METHOD': 'PUT'}
headers = {
'content-type': 'text/csv',
'content-length': str(testcsv),
'content-disposition': 'attachment; filename=test.csv'
}
fileupload = FileUpload(
FieldStorage(fp=StringIO(testcsv), environ=env, headers=headers))
view.request.form.update({
'multispeciesoccurrence.buttons.save':
u'Save',
'multispeciesoccurrence.widgets.file':
fileupload,
'multispeciesoccurrence.widgets.title':
u'test species title',
'multispeciesoccurrence.widgets.description':
u'some test.csv file',
'multispeciesoccurrence.widgets.legalcheckbox': [u'selected'],
'multispeciesoccurrence.widgets.legalcheckbox-empty-marker':
u'1',
'multispeciesoccurrence.widgets.rights':
u'test rights',
})
_ = view()
self.assertEqual(self.portal.REQUEST.response.status, 302)
self.assertEqual(
self.portal.REQUEST.response.getHeader('Location'),
'http://{0}:{1}/plone/datasets'.format(self.layer.get('host'),
self.layer.get('port')))
ds = self.portal.datasets.species.user['test.csv']
self.assertEqual(ds.rights, u'test rights')
self.assertEqual(ds.file.data, testcsv)
md = IBCCVLMetadata(ds)
self.assertEqual(md['genre'], 'DataGenreSpeciesCollection')
jt = IJobTracker(ds)
self.assertEqual(jt.state, 'PENDING')
# patch movelib.move
def move_occurrence_data(*args, **kw):
# try to move test.csv from dataset
src, dst = (urlsplit(x['url']) for x in args)
if src.scheme == 'http' and dst.scheme == 'file':
# copy test.csv to dst
dest_file = os.path.join(dst.path, 'somerandomfilename')
shutil.copyfileobj(
resource_stream(__name__,
'mock_data/multi_occurrence_mac.csv'),
open(dest_file, 'w'))
elif src.scheme == 'file' and dst.scheme == 'scp':
# copy result back
shutil.copyfile(src.path, dst.path)
else:
raise Exception('Data move failed')
mock_move.side_effect = move_occurrence_data
# triger background process
transaction.commit()
# 6 move should have happened
self.assertEqual(mock_move.call_count, 5)
self.assertEqual(
mock_move.call_args_list[0][0][0]['url'],
'http://{0}:{1}/plone/datasets/species/user/test.csv/@@download/file/test.csv'
.format(self.layer.get('host'), self.layer.get('port')))
# TODO: should test other call orguments as well
# job state should be complete
self.assertEqual(jt.state, 'COMPLETED')
# metadata should be up to date
self.assertEqual(md['rows'], 999)
# check other datasets:
for name, rows in (('abbreviata.csv', 65), ('acinacea.csv', 596),
('acanthoclada.csv', 322), ('acanthaster.csv', 16)):
self.assertIn(name, self.portal.datasets.species.user)
tds = self.portal.datasets.species.user[name]
tmd = IBCCVLMetadata(tds)
self.assertEqual(tmd['rows'], rows)
self.assertEqual(len(ds.parts), 4)
self.assertEqual(len(ds.parts), len(set(ds.parts)))
def test_upload_zip(self, mock_move):
# upload a zip in bccvl bagit format
view = self.getview()
from ZPublisher.HTTPRequest import FileUpload
from cgi import FieldStorage
from StringIO import StringIO
data = resource_string(__name__, 'mock_data/spc_obl_merc.zip')
env = {'REQUEST_METHOD': 'PUT'}
headers = {
'content-type': 'text/csv',
'content-length': str(len(data)),
'content-disposition': 'attachment; filename=spc_obl_merc.zip'
}
fileupload = FileUpload(
FieldStorage(fp=StringIO(data), environ=env, headers=headers))
view.request.form.update({
'climatefuture.buttons.save':
u'Save',
'climatefuture.widgets.description':
u'some test.tif file',
'climatefuture.widgets.domain':
u'Terrestrialdatasets',
'climatefuture.widgets.file':
fileupload,
'climatefuture.widgets.title':
u'test smulti layer title',
'climatefuture.widgets.legalcheckbox': [u'selected'],
'climatefuture.widgets.legalcheckbox-empty-marker':
u'1',
'climatefuture.widgets.rights':
u'test rights',
'climatefuture.widgets.emsc':
u'SRESB2',
'climatefuture.widgets.gcm':
u'cccma-cgcm31',
'climatefuture.widgets.resolution':
u'Resolution5m',
})
_ = view()
self.assertEqual(self.portal.REQUEST.response.status, 302)
self.assertEqual(
self.portal.REQUEST.response.getHeader('Location'),
'http://{0}:{1}/plone/datasets'.format(self.layer.get('host'),
self.layer.get('port')))
ds = self.portal.datasets.climate.user['spc_obl_merc.zip']
self.assertEqual(ds.rights, u'test rights')
self.assertEqual(ds.file.data, data)
md = IBCCVLMetadata(ds)
self.assertEqual(md['genre'], 'DataGenreFC')
self.assertEqual(md['resolution'], u'Resolution5m')
self.assertEqual(md['emsc'], u'SRESB2')
self.assertEqual(md['gcm'], u'cccma-cgcm31')
jt = IJobTracker(ds)
self.assertEqual(jt.state, 'PENDING')
# patch movelib.move
def move_occurrence_data(*args, **kw):
# try to move test.csv from dataset
src, dst = (urlsplit(x['url']) for x in args)
# copy test.csv to dst
shutil.copyfileobj(
resource_stream(__name__, 'mock_data/spc_obl_merc.zip'),
open(dst.path, 'w'))
mock_move.side_effect = move_occurrence_data
# triger background process
transaction.commit()
# one move should have happened
self.assertEqual(
mock_move.call_args[0][0]['url'],
'http://{0}:{1}/plone/datasets/climate/user/spc_obl_merc.zip/@@download/file/spc_obl_merc.zip'
.format(self.layer.get('host'), self.layer.get('port')))
# job state should be complete
self.assertEqual(jt.state, 'COMPLETED')
layermd = md['layers']['spc_obl_merc/data/spc_obl_merc_1.tif']
self.assertEqual(layermd['filename'],
'spc_obl_merc/data/spc_obl_merc_1.tif')
self.assertEqual(layermd['min'], 19.0)
self.assertEqual(layermd['max'], 128.0)
self.assertEqual(layermd['datatype'], 'continuous')
self.assertEqual(layermd['height'], 200)
self.assertEqual(layermd['width'], 200)
self.assertEqual(layermd['srs'], None)
layermd = md['layers']['spc_obl_merc/data/spc_obl_merc_2.tif']
self.assertEqual(layermd['filename'],
'spc_obl_merc/data/spc_obl_merc_2.tif')
self.assertEqual(layermd['min'], 19.0)
self.assertEqual(layermd['max'], 128.0)
self.assertEqual(layermd['datatype'], 'continuous')
self.assertEqual(layermd['height'], 200)
self.assertEqual(layermd['width'], 200)
self.assertEqual(layermd['srs'], None)
def test_upload_occurrence(self, mock_move):
testcsv = resource_string(__name__, 'mock_data/ala_occurrence.csv')
view = self.getview()
from ZPublisher.HTTPRequest import FileUpload
from cgi import FieldStorage
from StringIO import StringIO
env = {'REQUEST_METHOD': 'PUT'}
headers = {
'content-type': 'text/csv',
'content-length': str(len(testcsv)),
'content-disposition': 'attachment; filename=test.csv'
}
fileupload = FileUpload(
FieldStorage(fp=StringIO(testcsv), environ=env, headers=headers))
view.request.form.update({
'speciesoccurrence.buttons.save':
u'Save',
'speciesoccurrence.widgets.file':
fileupload,
'speciesoccurrence.widgets.title':
u'test species title',
'speciesoccurrence.widgets.description':
u'some test.csv file',
'speciesoccurrence.widgets.legalcheckbox': [u'selected'],
'speciesoccurrence.widgets.legalcheckbox-empty-marker':
u'1',
'speciesoccurrence.widgets.rights':
u'test rights',
'speciesoccurrence.widgets.scientificName':
u'test species',
'speciesoccurrence.widgets.taxonID':
u'test taxonid',
'speciesoccurrence.widgets.vernacularName':
u'test it'
})
_ = view()
self.assertEqual(self.portal.REQUEST.response.status, 302)
self.assertEqual(
self.portal.REQUEST.response.getHeader('Location'),
'http://{0}:{1}/plone/datasets'.format(self.layer.get('host'),
self.layer.get('port')))
# dataset should exist now
ds = self.portal.datasets.species.user['test.csv']
self.assertEqual(ds.rights, u'test rights')
self.assertEqual(ds.file.data, testcsv)
md = IBCCVLMetadata(ds)
self.assertEqual(md['genre'], 'DataGenreSpeciesOccurrence')
self.assertEqual(md['species']['taxonID'], u'test taxonid')
self.assertEqual(md['species']['scientificName'], u'test species')
self.assertEqual(md['species']['vernacularName'], u'test it')
jt = IJobTracker(ds)
self.assertEqual(jt.state, 'PENDING')
# patch movelib.move
def move_occurrence_data(*args, **kw):
# try to move test.csv from dataset
src, dst = (urlsplit(x['url']) for x in args)
# copy test.csv to dst
shutil.copyfileobj(
resource_stream(__name__, 'mock_data/ala_occurrence.csv'),
open(dst.path, 'w'))
mock_move.side_effect = move_occurrence_data
# triger background process
transaction.commit()
# one move should have happened
self.assertEqual(
mock_move.call_args[0][0]['url'],
'http://{0}:{1}/plone/datasets/species/user/test.csv/@@download/file/test.csv'
.format(self.layer.get('host'), self.layer.get('port')))
# job state should be complete
self.assertEqual(jt.state, 'COMPLETED')
# metadata should be up to date
self.assertEqual(md['rows'], 29)
def create_test_file_field(self, data, filename):
field_storage = ZopeFieldStorage()
field_storage.file = BytesIO(data)
field_storage.filename = filename
file_field = FileUpload(field_storage)
return file_field
def _create_test_file_field(self):
field_storage = ZopeFieldStorage()
field_storage.file = BytesIO('test')
field_storage.filename = 'test.pdf'
file_field = FileUpload(field_storage)
return file_field