def __init__(self, size, filename):
     self.file = StringIO("x" * size)
     self.file.filename = filename
     self.file.headers = []
     self.file.name = "file1"
     self.file.file = self.file
     FileUpload.__init__(self, self.file)
 def _prepare_file(self, filename='test.bin', type_='text/plain'):
     fp = TemporaryFile('w+b')
     fp.write('\x00' + 'x' * (1 << 19))
     fp.seek(0)
     env = {'REQUEST_METHOD': 'POST'}
     headers = {'Content-Type': type_,
                'content-length': 1 << 19,
                'content-disposition': 'attachment; filename=%s' % filename}
     fs = FieldStorage(fp=fp, environ=env, headers=headers)
     fu = FileUpload(fs)
     fu.read = fp.read
     fu.seek = fp.seek
     fu.tell = fp.tell
     return fu
Exemple #3
0
 def _prepare_file(self, filename='test.bin', type_='text/plain'):
     fp = TemporaryFile('w+b')
     fp.write('\x00' + 'x' * (1 << 19))
     fp.seek(0)
     env = {'REQUEST_METHOD': 'POST'}
     headers = {'Content-Type': type_,
                'content-length': 1 << 19,
                'content-disposition': 'attachment; filename=%s' % filename}
     fs = FieldStorage(fp=fp, environ=env, headers=headers)
     fu = FileUpload(fs)
     fu.read = fp.read
     fu.seek = fp.seek
     fu.tell = fp.tell
     return fu
Exemple #4
0
def test_suite():
    tarball = FileUpload(FileHolder(os.path.join(datadir, 'project-macosx-10.3-fat.tar.gz')))
    egg = FileUpload(FileHolder(os.path.join(datadir, 'project.egg')))
 
    globs = globals()
    globs['tarball'] = tarball
    globs['egg'] = egg

    return Suite(os.path.basename('pypi.txt'),
                 os.path.basename('permissions.txt'),
                 optionflags=OPTIONFLAGS,
                 package='Products.PloneSoftwareCenter.tests',
                 test_class=PSCFunctionalTestCase,
                 setUp=setUp, globs=globs)
Exemple #5
0
def HTTPUpload(fp, content_type=None, filename=None):
    """ Create a FileUpload instance from a file handle (and content_type) """
    if isinstance(fp, FileUpload):
        if content_type:
            fp.headers['content-type'] = content_type
    else:
        environ = {'REQUEST_METHOD': 'POST'}
        if content_type:
            environ['CONTENT_TYPE'] = content_type
        elif hasattr(fp, 'headers') and fp.headers.has_key('content-type'):
            environ['CONTENT_TYPE'] = fp.headers['content-type']
        fp = FileUpload(FieldStorage(fp, environ=environ))
    if filename:
        fp.filename = filename
    return fp
Exemple #6
0
def HTTPUpload(fp, content_type=None, filename=None):
    """ Create a FileUpload instance from a file handle (and content_type) """
    if isinstance(fp, FileUpload):
        if content_type:
            fp.headers["content-type"] = content_type
    else:
        environ = {"REQUEST_METHOD": "POST"}
        if content_type:
            environ["CONTENT_TYPE"] = content_type
        elif hasattr(fp, "headers") and fp.headers.has_key("content-type"):
            environ["CONTENT_TYPE"] = fp.headers["content-type"]
        fp = FileUpload(FieldStorage(fp, environ=environ))
    if filename:
        fp.filename = filename
    return fp
Exemple #7
0
def HTTPUpload(fp, content_type=None, filename=None):
    """ Create a FileUpload instance from a file handle (and content_type) """
    if isinstance(fp, FileUpload):
        if content_type:
            fp.headers['content-type'] = content_type
    else:
        environ = {'REQUEST_METHOD': 'POST'}
        if content_type:
            environ['CONTENT_TYPE'] = content_type
        elif hasattr(fp, 'headers') and fp.headers.has_key('content-type'):
            environ['CONTENT_TYPE'] = fp.headers['content-type']
        fp = FileUpload(FieldStorage(fp, environ=environ))
    if filename:
        fp.filename = filename
    return fp
Exemple #8
0
 def test_first_step_validate_csv_encoding_nok(self):
     """Ensure that we can decode csv file"""
     request = self.layer["request"]
     csv = StringIO()
     lines = [
         [u"猫", u"èè", u"ùù"],
         ["", "key1", "Key 1"],
         [u"猫", u"ààà", u"ééé"],
     ]
     for line in lines:
         csv.write(";".join(line).encode("utf-16") + "\n")
     csv.seek(0)
     source = FileUpload(
         type(
             "obj",
             (object, ),
             {
                 "file": csv,
                 "filename": "foo.csv",
                 "headers": "text/csv"
             },
         )())
     request.form = {
         "form.buttons.continue": u"continuer",
         "form.widgets.separator": [u";"],
         "form.widgets.separator-empty-marker": u"1",
         "form.widgets.source": source,
         "form.widgets.has_header": u"True",
     }
     form = importform.ImportFormFirstStep(self.container, request)
     form.update()
     data, errors = form.extractData()
     self.assertEqual(1, len(errors))
     self.assertEqual("File encoding is not utf8", errors[0].error.message)
Exemple #9
0
 def test_first_step_validate_line_columns_nok(self):
     """Ensure that every lines have the same number of columns"""
     request = self.layer["request"]
     csv = StringIO()
     lines = [
         ["", "key1", "Key 1"],
         ["key1", "key1.1"],
         ["key1.1", "key1.1.1", "Key 1.1.1", "foo"],
     ]
     for line in lines:
         csv.write(";".join(line) + "\n")
     csv.seek(0)
     source = FileUpload(
         type(
             "obj",
             (object, ),
             {
                 "file": csv,
                 "filename": "foo.csv",
                 "headers": "text/csv"
             },
         )())
     request.form = {
         "form.buttons.continue": u"continuer",
         "form.widgets.separator": [u";"],
         "form.widgets.separator-empty-marker": u"1",
         "form.widgets.source": source,
         "form.widgets.has_header": u"False",
     }
     form = importform.ImportFormFirstStep(self.container, request)
     form.update()
     data, errors = form.extractData()
     self.assertEqual(1, len(errors))
     self.assertTrue("Lines 2, 3" in translate(errors[0].error.message))
Exemple #10
0
        def test_tidy_validator_with_upload_wrong_encoding(self):
            doc = self._ATCT

            field = doc.getField('text')
            request = self.app.REQUEST
            setattr(request, 'text_text_format', 'text/html')
            input_file_name = 'tidy1-in.html'
            in_file = open(input_file_path(input_file_name))
            env = {'REQUEST_METHOD': 'PUT'}
            headers = {
                'content-type': 'text/html',
                'content-length': len(in_file.read()),
                'content-disposition':
                'attachment; filename=%s' % input_file_name
            }
            in_file.seek(0)
            fs = FieldStorage(fp=in_file, environ=env, headers=headers)
            f = FileUpload(fs)

            tcv = TidyHtmlWithCleanupValidator('tidy_validator_with_cleanup')
            result = tcv.__call__(f, field=field, REQUEST=request)

            self.assertEquals(result, 1)

            expected_file = open(input_file_path('tidy1-out.html'))
            expected = expected_file.read()
            expected_file.close()
            self.assertEquals(request['text_tidier_data'], expected)
Exemple #11
0
def makeFileUpload(file, content_type=None, filename=None):
    headers = {}
    if type(file) == type(''):
        file = open(file, 'rb')
    if content_type:
        headers['content-type'] = content_type
    fs = DummyFieldStorage(file, filename, headers)
    return FileUpload(fs)
    def test_upload_zip(self):
        # upload a zip in bccvl bagit format
        view = self.getview()
        from ZPublisher.HTTPRequest import FileUpload
        from cgi import FieldStorage
        from StringIO import StringIO
        data = resource_string(__name__, 'spc_obl_merc.zip')
        env = {'REQUEST_METHOD': 'PUT'}
        headers = {'content-type': 'text/csv',
                   'content-length': str(len(data)),
                   'content-disposition': 'attachment; filename=spc_obl_merc.zip'}
        fileupload = FileUpload(FieldStorage(fp=StringIO(data),
                                             environ=env, headers=headers))

        view.request.form.update({
            'climatefuture.buttons.save': u'Save',
            'climatefuture.widgets.description': u'some test.tif file',
            'climatefuture.widgets.file': fileupload,
            'climatefuture.widgets.title': u'test smulti layer title',
            'climatefuture.widgets.legalcheckbox': [u'selected'],
            'climatefuture.widgets.legalcheckbox-empty-marker': u'1',
            'climatefuture.widgets.rightsstatement': u'test rights',
            'climatefuture.widgets.rightsstatement.mimeType': u'text/html',
            'climatefuture.widgets.emsc': u'SRESB2',
            'climatefuture.widgets.gcm': u'cccma-cgcm31',
            'climatefuture.widgets.resolution': u'Resolution5m',
            'climatefuture.widgets.temporal': u'2015',
        })
        _ = view()
        self.assertEqual(self.portal.REQUEST.response.status, 302)
        self.assertEqual(self.portal.REQUEST.response.getHeader('Location'),
                         'http://nohost/plone/datasets')
        ds = self.portal.datasets['spc_obl_merc.zip']
        self.assertEqual(ds.rightsstatement.raw, u'test rights')
        self.assertEqual(ds.file.data, data)
        from org.bccvl.site.interfaces import IBCCVLMetadata
        md = IBCCVLMetadata(ds)
        self.assertEqual(md['genre'], 'DataGenreFC')
        self.assertEqual(md['resolution'], u'Resolution5m')
        self.assertEqual(md['temporal'], u'2015')
        self.assertEqual(md['emsc'], u'SRESB2')
        self.assertEqual(md['gcm'], u'cccma-cgcm31')
        layermd = md['layers']['spc_obl_merc/data/spc_obl_merc_1.tif']
        self.assertEqual(layermd['filename'], 'spc_obl_merc/data/spc_obl_merc_1.tif')
        self.assertEqual(layermd['min'], 19.0)
        self.assertEqual(layermd['max'], 128.0)
        self.assertEqual(layermd['datatype'], 'continuous')
        self.assertEqual(layermd['height'], 200)
        self.assertEqual(layermd['width'], 200)
        self.assertEqual(layermd['srs'], None)
        layermd = md['layers']['spc_obl_merc/data/spc_obl_merc_2.tif']
        self.assertEqual(layermd['filename'], 'spc_obl_merc/data/spc_obl_merc_2.tif')
        self.assertEqual(layermd['min'], 19.0)
        self.assertEqual(layermd['max'], 128.0)
        self.assertEqual(layermd['datatype'], 'continuous')
        self.assertEqual(layermd['height'], 200)
        self.assertEqual(layermd['width'], 200)
        self.assertEqual(layermd['srs'], None)
Exemple #13
0
    def loadfile(self, rel_filename, ctype='image/png'):
        """ load a file
        """
        home = package_home(product_globals)
        filename = os.path.sep.join([home, rel_filename])
        data = open(filename, 'r').read()

        fp = StringIO(data)
        fp.seek(0)

        env = {'REQUEST_METHOD': 'PUT'}
        headers = {'content-type': ctype,
                   'content-length': len(data),
                   'content-disposition': 'attachment; filename=test.txt'}

        fs = FieldStorage(fp=fp, environ=env, headers=headers)
        ufile = FileUpload(fs)
        ufile.name = None
        return ufile
Exemple #14
0
    def loadfile(self, rel_filename, ctype='image/png'):
        """ load a file
        """
        home = package_home(product_globals)
        filename = os.path.sep.join([home, rel_filename])
        data = open(filename, 'r').read()

        fp = StringIO(data)
        fp.seek(0)

        env = {'REQUEST_METHOD': 'PUT'}
        headers = {'content-type': ctype,
                   'content-length': len(data),
                   'content-disposition': 'attachment; filename=test.txt'}

        fs = FieldStorage(fp=fp, environ=env, headers=headers)
        ufile = FileUpload(fs)
        ufile.name = None
        return ufile
Exemple #15
0
def make_file_upload(data, content_type=None, filename=None):
    """Create a FileUpload object.
    """
    headers = {}
    if isinstance(data, str):
        data = open(data, 'rb')
    if content_type:
        headers['content-type'] = content_type
    fs = DummyFieldStorage(data, filename, headers)
    return FileUpload(fs)
Exemple #16
0
    def _prepareFormTarball(self):
        """we could use our @@export-easyform view,
           but we want a more atomic test, so we make
           a tarfile for our test.  the approach to making
           a tarfile is a bit strange, but does the job
        """
        in_fname = "test_form_1_easyform.tar.gz"
        test_dir = os.path.dirname(__file__)

        def _add_form_structure_to_archive(archive):
            form_relative_path = os.path.join(
                "profiles",
                "testing",
                "structure",
                "Members",
                "test_user_1_",
                "test_form_1_easyform",
            )
            abs_path = os.path.join(test_dir, form_relative_path)

            # add structure folder
            os.chdir(os.path.join(test_dir, "profiles", "testing"))
            archive.add("structure", recursive=False)

            for f in os.listdir(abs_path):
                os.chdir(abs_path)  # add form data w/o full directory tree
                archive.add(f, arcname=os.path.join("structure", f))

        # Capture the current working directory for later when we need to
        # clean up the environment.
        working_directory = os.path.abspath(os.curdir)

        # make me a tarfile in the current dir
        os.chdir(test_dir)
        archive = TarFile.open(name=in_fname, mode="w:gz")
        _add_form_structure_to_archive(archive)
        archive.close()

        # Change back to the working directory in case something tries to
        # write files (e.g. collective.xmltestreport).
        os.chdir(working_directory)

        # get it and upload
        in_file = open(os.path.join(test_dir, in_fname))
        env = {"REQUEST_METHOD": "PUT"}
        headers = {
            "content-type": "text/html",
            "content-length": len(in_file.read()),
            "content-disposition":
            "attachment; filename={0}".format(in_file.name),
        }  # noqa
        in_file.seek(0)
        fs = FieldStorage(fp=in_file, environ=env, headers=headers)
        return FileUpload(fs)
Exemple #17
0
 def makeFileUpload(self,
                    filename='test.txt',
                    value='test text',
                    diskname=''):
     if diskname:
         filename = diskname
         value = open(os.path.join(test_folder,
                                   diskname)).read().replace('\r',
                                                             '').strip()
     from ZPublisher.HTTPRequest import FileUpload
     return FileUpload(DummyFieldStorage(filename, value))
 def prepare_request(self):
     fieldstorage = cgi.FieldStorage()
     fieldstorage.file = resource_stream("opengever.bumblebee.tests.assets",
                                         "vertragsentwurf.pdf")
     fieldstorage.filename = 'test.pdf'
     file = FileUpload(fieldstorage)
     self.request.set('status', "success")
     self.request.set('pdf', file)
     self.request.set('token', get_demand_callback_access_token())
     self.request.set('opaque_id', self.save_token)
     self.request.method = "POST"
Exemple #19
0
 def _createBinaryFile():
     from cgi import FieldStorage
     from ZPublisher.HTTPRequest import FileUpload
     from tempfile import TemporaryFile
     fp = TemporaryFile('w+b')
     fp.write('\x00' + 'x' * (1 << 19))
     fp.seek(0)
     env = {'REQUEST_METHOD':'PUT'}
     headers = {'content-type':'application/msword',
                'content-length': 1 << 19,
                'content-disposition':'attachment; filename=test.doc'}
     fs = FieldStorage(fp=fp, environ=env, headers=headers)
     return FileUpload(fs)
Exemple #20
0
def createFileUpload(data, filename):
    fp = TemporaryFile('w+b')
    fp.write(data)
    fp.seek(0)
    env = {'REQUEST_METHOD': 'PUT'}
    headers = {'content-type':'text/plain',
               'content-length': len(data),
               'content-disposition':'attachment; filename=%s' % \
                 filename}
    fs = FieldStorage(fp=fp, environ=env, headers=headers)
    result = FileUpload(fs)
    fp.close()

    return result
Exemple #21
0
 def setUp(self):
     self.portal = self.layer['portal']
     file_ = open("%s/testfile_mimetypes.txt" %
                  os.path.split(__file__)[0], 'r')
     self.file_data = file_.read()
     fieldstorage = cgi.FieldStorage()
     fieldstorage.file = file_
     fieldstorage.filename = 'testfile_mimetypes.txt'
     self.fileupload = FileUpload(fieldstorage)
     mtr = getToolByName(self.portal, 'mimetypes_registry')
     mtr.manage_addMimeType(
         'Only globs mimetype', ['application/x-only-glob'], [],
         'application.png', globs=['*.onlyglob'])
     mtr.manage_addMimeType(
         'Only extension mimetype', ['application/x-only-ext'],
         ['onlyext'], 'application.png')
Exemple #22
0
 def test_file_upload_binary(self):
     from cgi import FieldStorage
     from ZPublisher.HTTPRequest import FileUpload
     from tempfile import TemporaryFile
     fp = TemporaryFile('w+b')
     fp.write('\x00' + 'x' * (1 << 19))
     fp.seek(0)
     env = {'REQUEST_METHOD': 'PUT'}
     headers = {'content-type': 'text/plain',
                'content-length': 1 << 19,
                'content-disposition': 'attachment; filename=test.bin'}
     fs = FieldStorage(fp=fp, environ=env, headers=headers)
     f = FileUpload(fs)
     v, m, f = self.field._process_input(f, instance=self.instance)
     self.assertTrue(isinstance(v, self.factory), (type(v), self.factory))
     self.assertEqual(m, 'application/octet-stream')
     self.assertEqual(f, 'test.bin')
Exemple #23
0
def loadxmlfile(rel_filename):
    ctype = 'text/xml'
    home = package_home(globals())
    filename = os.path.sep.join([home, rel_filename])
    data = open(filename, 'r').read()
    fp = StringIO(data)
    fp.seek(0)

    header_filename = rel_filename.split('/')[-1]
    env = {'REQUEST_METHOD': 'PUT'}
    headers = {
        'content-type': ctype,
        'content-length': len(data),
        'content-disposition': 'attachment; filename=%s' % (header_filename, )
    }

    fs = FieldStorage(fp=fp, environ=env, headers=headers)
    return FileUpload(fs)
Exemple #24
0
 def test_upload_file(self):
     fp = TemporaryFile('w+b')
     fp.write('foobar')
     fp.seek(0)
     filename = 'test.xml'
     env = {'REQUEST_METHOD': 'PUT'}
     headers = {'content-type': 'text/plain',
                'content-length': str(len('foobar')),
                'content-disposition': 'attachment; filename=%s' % filename}
     files_ = FileUpload(FieldStorage(fp=fp, environ=env, headers=headers))
     fileids = self.context.name + 'fileids'
     request = TestRequest(environ={'HTTP_ACCEPT_LANGUAGE': 'en', 'REQUEST_METHOD': 'POST', 'dummy': files_, fileids: 'fXXXXX'})
     setattr(request, "dummy", "stuff")
     setattr(request, "URL1", "http://127.0.0.1")
     upload = Upload(self.context, request)
     setattr(request, "REQUEST_METHOD", "POST")
     json_value = upload()
     self.assertTrue('"size": 6' in json_value)
     self.assertTrue('"name": "test.xml"' in json_value)
     request = TestRequest(environ={'HTTP_ACCEPT_LANGUAGE': 'en', 'REQUEST_METHOD': 'POST', 'dummy': None})
  def test_05_EditSelectionWithFileUpload(self, quiet=quiet, run=run_all_test):
    """Listbox edits selection with request parameters. Special care must be
    taken for FileUpload objects that cannot be pickled, thus cannot be stored
    in the ZODB.
    """
    portal = self.getPortal()
    portal.ListBoxZuite_reset()
    listbox = portal.FooModule_viewFooList.listbox
    # XXX isn't Selection automatically created ?
    name = listbox.get_value('selection_name')
    portal.portal_selections.setSelectionFor(name, Selection(name))

    request = get_request()
    request['here'] = portal.foo_module
    request.form['my_file_upload'] = FileUpload(DummyFieldStorage())
    listbox.get_value('default', render_format='list', REQUEST=request)
    try:
      self.commit()
    except TypeError, e:
      self.fail('Unable to commit transaction: %s' % e)
    def test_upload_occurrence(self):
        view = self.getview()
        from ZPublisher.HTTPRequest import FileUpload
        from cgi import FieldStorage
        from StringIO import StringIO
        data = "species,lon,lat\nSpecies,1,2\nSpecies,2,3\n"
        env = {'REQUEST_METHOD': 'PUT'}
        headers = {'content-type': 'text/csv',
                   'content-length': str(len(data)),
                   'content-disposition': 'attachment; filename=test.csv'}
        fileupload = FileUpload(FieldStorage(fp=StringIO(data),
                                             environ=env, headers=headers))

        view.request.form.update({
            'speciesoccurrence.buttons.save': u'Save',
            'speciesoccurrence.widgets.description': u'some test.csv file',
            'speciesoccurrence.widgets.file': fileupload,
            'speciesoccurrence.widgets.title': u'test species title',
            'speciesoccurrence.widgets.legalcheckbox': [u'selected'],
            'speciesoccurrence.widgets.legalcheckbox-empty-marker': u'1',
            'speciesoccurrence.widgets.rightsstatement': u'test rights',
            'speciesoccurrence.widgets.rightsstatement.mimeType': u'text/html',
            'speciesoccurrence.widgets.scientificName': u'test species',
            'speciesoccurrence.widgets.taxonID': u'test taxonid',
            'speciesoccurrence.widgets.vernacularName': u'test it'
        })
        _ = view()
        self.assertEqual(self.portal.REQUEST.response.status, 302)
        self.assertEqual(self.portal.REQUEST.response.getHeader('Location'),
                         'http://nohost/plone/datasets')
        ds = self.portal.datasets['test.csv']
        self.assertEqual(ds.rightsstatement.raw, u'test rights')
        self.assertEqual(ds.file.data, 'species,lon,lat\nSpecies,1,2\nSpecies,2,3\n')
        from org.bccvl.site.interfaces import IBCCVLMetadata
        md = IBCCVLMetadata(ds)
        self.assertEqual(md['genre'], 'DataGenreSpeciesOccurrence')
        self.assertEqual(md['species']['taxonID'], u'test taxonid')
        self.assertEqual(md['species']['scientificName'], u'test species')
        self.assertEqual(md['species']['vernacularName'], u'test it')
        self.assertEqual(md['rows'], 2)
    def test_formlib_form_import(self):
        """Interacting with our formlib form we should be able
           to successfully upload an exported PFG form and have it
           correctly configured.
        """
        self._makeForm()

        fields = self.ff1.fields_model
        actions = self.ff1.actions_model

        form_folder_export = getMultiAdapter((self.ff1, self.app.REQUEST),
                                             name='export-easyform')
        in_file = StringIO(form_folder_export())
        env = {'REQUEST_METHOD': 'PUT'}
        headers = {
            'content-type': 'text/html',
            'content-length': len(in_file.read()),
            'content-disposition': 'attachment; filename=ff1.tar.gz'
        }
        in_file.seek(0)
        fs = FieldStorage(fp=in_file, environ=env, headers=headers)

        # setup a reasonable request
        request = self.app.REQUEST
        request.form = {
            'form.widgets.upload': FileUpload(fs),
            'form.buttons.import': 'import'
        }
        request.RESPONSE = request.response

        # get the form object
        import_form = getMultiAdapter((self.ff1, request),
                                      name='import-easyform')

        # call update (aka submit) on the form, see TestRequest above
        import_form.update()
        self.assertEqual(self.ff1.fields_model, fields)
        self.assertEqual(self.ff1.actions_model, actions)
Exemple #28
0
    def loadfile(self, rel_filename, ctype='text/xml', zope=False):
        """ load a file
        """
        home = package_home(product_globals)
        filename = os.path.sep.join([home, rel_filename])
        data = open(filename, 'r').read()

        fp = StringIO(data)
        fp.seek(0)

        if not zope:
            return fp

        header_filename = rel_filename.split('/')[-1]
        env = {'REQUEST_METHOD': 'PUT'}
        headers = {
            'content-type': ctype,
            'content-length': len(data),
            'content-disposition': 'attachment; filename=%s' % header_filename
        }

        fs = FieldStorage(fp=fp, environ=env, headers=headers)
        return FileUpload(fs)
Exemple #29
0
 def test_first_step_validate_columns_number_correct(self):
     """Ensure that csv file contains at least 2 columns"""
     request = self.layer["request"]
     source = FileUpload(
         type(
             "obj",
             (object, ),
             {
                 "file": self._csv,
                 "filename": "foo.csv",
                 "headers": "text/csv"
             },
         )())
     request.form = {
         "form.buttons.continue": u"Continuer",
         "form.widgets.separator": [u";"],
         "form.widgets.separator-empty-marker": u"1",
         "form.widgets.source": source,
         "form.widgets.has_header": u"False",
     }
     form = importform.ImportFormFirstStep(self.container, request)
     form.update()
     data, errors = form.extractData()
     self.assertEqual(0, len(errors))
Exemple #30
0
    def test_upload_multi_csv_mac(self, mock_move=None):
        testcsv = resource_string(__name__,
                                  'mock_data/multi_occurrence_mac.csv')
        view = self.getview()
        from ZPublisher.HTTPRequest import FileUpload
        from cgi import FieldStorage
        from StringIO import StringIO
        env = {'REQUEST_METHOD': 'PUT'}
        headers = {
            'content-type': 'text/csv',
            'content-length': str(testcsv),
            'content-disposition': 'attachment; filename=test.csv'
        }
        fileupload = FileUpload(
            FieldStorage(fp=StringIO(testcsv), environ=env, headers=headers))
        view.request.form.update({
            'multispeciesoccurrence.buttons.save':
            u'Save',
            'multispeciesoccurrence.widgets.file':
            fileupload,
            'multispeciesoccurrence.widgets.title':
            u'test species title',
            'multispeciesoccurrence.widgets.description':
            u'some test.csv file',
            'multispeciesoccurrence.widgets.legalcheckbox': [u'selected'],
            'multispeciesoccurrence.widgets.legalcheckbox-empty-marker':
            u'1',
            'multispeciesoccurrence.widgets.rights':
            u'test rights',
        })
        _ = view()
        self.assertEqual(self.portal.REQUEST.response.status, 302)
        self.assertEqual(
            self.portal.REQUEST.response.getHeader('Location'),
            'http://{0}:{1}/plone/datasets'.format(self.layer.get('host'),
                                                   self.layer.get('port')))
        ds = self.portal.datasets.species.user['test.csv']
        self.assertEqual(ds.rights, u'test rights')
        self.assertEqual(ds.file.data, testcsv)
        md = IBCCVLMetadata(ds)
        self.assertEqual(md['genre'], 'DataGenreSpeciesCollection')

        jt = IJobTracker(ds)
        self.assertEqual(jt.state, 'PENDING')

        # patch movelib.move
        def move_occurrence_data(*args, **kw):
            # try to move test.csv from dataset
            src, dst = (urlsplit(x['url']) for x in args)
            if src.scheme == 'http' and dst.scheme == 'file':
                # copy test.csv to dst
                dest_file = os.path.join(dst.path, 'somerandomfilename')
                shutil.copyfileobj(
                    resource_stream(__name__,
                                    'mock_data/multi_occurrence_mac.csv'),
                    open(dest_file, 'w'))
            elif src.scheme == 'file' and dst.scheme == 'scp':
                # copy result back
                shutil.copyfile(src.path, dst.path)
            else:
                raise Exception('Data move failed')

        mock_move.side_effect = move_occurrence_data
        # triger background process
        transaction.commit()
        # 6 move should have happened
        self.assertEqual(mock_move.call_count, 5)
        self.assertEqual(
            mock_move.call_args_list[0][0][0]['url'],
            'http://{0}:{1}/plone/datasets/species/user/test.csv/@@download/file/test.csv'
            .format(self.layer.get('host'), self.layer.get('port')))
        # TODO: should test other call orguments as well
        # job state should be complete
        self.assertEqual(jt.state, 'COMPLETED')
        # metadata should be up to date
        self.assertEqual(md['rows'], 999)
        # check other datasets:
        for name, rows in (('abbreviata.csv', 65), ('acinacea.csv', 596),
                           ('acanthoclada.csv', 322), ('acanthaster.csv', 16)):
            self.assertIn(name, self.portal.datasets.species.user)
            tds = self.portal.datasets.species.user[name]
            tmd = IBCCVLMetadata(tds)
            self.assertEqual(tmd['rows'], rows)
        self.assertEqual(len(ds.parts), 4)
        self.assertEqual(len(ds.parts), len(set(ds.parts)))
Exemple #31
0
    def test_upload_zip(self, mock_move):
        # upload a zip in bccvl bagit format
        view = self.getview()
        from ZPublisher.HTTPRequest import FileUpload
        from cgi import FieldStorage
        from StringIO import StringIO
        data = resource_string(__name__, 'mock_data/spc_obl_merc.zip')
        env = {'REQUEST_METHOD': 'PUT'}
        headers = {
            'content-type': 'text/csv',
            'content-length': str(len(data)),
            'content-disposition': 'attachment; filename=spc_obl_merc.zip'
        }
        fileupload = FileUpload(
            FieldStorage(fp=StringIO(data), environ=env, headers=headers))

        view.request.form.update({
            'climatefuture.buttons.save':
            u'Save',
            'climatefuture.widgets.description':
            u'some test.tif file',
            'climatefuture.widgets.domain':
            u'Terrestrialdatasets',
            'climatefuture.widgets.file':
            fileupload,
            'climatefuture.widgets.title':
            u'test smulti layer title',
            'climatefuture.widgets.legalcheckbox': [u'selected'],
            'climatefuture.widgets.legalcheckbox-empty-marker':
            u'1',
            'climatefuture.widgets.rights':
            u'test rights',
            'climatefuture.widgets.emsc':
            u'SRESB2',
            'climatefuture.widgets.gcm':
            u'cccma-cgcm31',
            'climatefuture.widgets.resolution':
            u'Resolution5m',
        })
        _ = view()
        self.assertEqual(self.portal.REQUEST.response.status, 302)
        self.assertEqual(
            self.portal.REQUEST.response.getHeader('Location'),
            'http://{0}:{1}/plone/datasets'.format(self.layer.get('host'),
                                                   self.layer.get('port')))
        ds = self.portal.datasets.climate.user['spc_obl_merc.zip']
        self.assertEqual(ds.rights, u'test rights')
        self.assertEqual(ds.file.data, data)
        md = IBCCVLMetadata(ds)
        self.assertEqual(md['genre'], 'DataGenreFC')
        self.assertEqual(md['resolution'], u'Resolution5m')
        self.assertEqual(md['emsc'], u'SRESB2')
        self.assertEqual(md['gcm'], u'cccma-cgcm31')

        jt = IJobTracker(ds)
        self.assertEqual(jt.state, 'PENDING')

        # patch movelib.move
        def move_occurrence_data(*args, **kw):
            # try to move test.csv from dataset
            src, dst = (urlsplit(x['url']) for x in args)
            # copy test.csv to dst
            shutil.copyfileobj(
                resource_stream(__name__, 'mock_data/spc_obl_merc.zip'),
                open(dst.path, 'w'))

        mock_move.side_effect = move_occurrence_data

        # triger background process
        transaction.commit()
        # one move should have happened
        self.assertEqual(
            mock_move.call_args[0][0]['url'],
            'http://{0}:{1}/plone/datasets/climate/user/spc_obl_merc.zip/@@download/file/spc_obl_merc.zip'
            .format(self.layer.get('host'), self.layer.get('port')))
        # job state should be complete
        self.assertEqual(jt.state, 'COMPLETED')

        layermd = md['layers']['spc_obl_merc/data/spc_obl_merc_1.tif']
        self.assertEqual(layermd['filename'],
                         'spc_obl_merc/data/spc_obl_merc_1.tif')
        self.assertEqual(layermd['min'], 19.0)
        self.assertEqual(layermd['max'], 128.0)
        self.assertEqual(layermd['datatype'], 'continuous')
        self.assertEqual(layermd['height'], 200)
        self.assertEqual(layermd['width'], 200)
        self.assertEqual(layermd['srs'], None)
        layermd = md['layers']['spc_obl_merc/data/spc_obl_merc_2.tif']
        self.assertEqual(layermd['filename'],
                         'spc_obl_merc/data/spc_obl_merc_2.tif')
        self.assertEqual(layermd['min'], 19.0)
        self.assertEqual(layermd['max'], 128.0)
        self.assertEqual(layermd['datatype'], 'continuous')
        self.assertEqual(layermd['height'], 200)
        self.assertEqual(layermd['width'], 200)
        self.assertEqual(layermd['srs'], None)
Exemple #32
0
    def test_upload_occurrence(self, mock_move):
        testcsv = resource_string(__name__, 'mock_data/ala_occurrence.csv')
        view = self.getview()
        from ZPublisher.HTTPRequest import FileUpload
        from cgi import FieldStorage
        from StringIO import StringIO
        env = {'REQUEST_METHOD': 'PUT'}
        headers = {
            'content-type': 'text/csv',
            'content-length': str(len(testcsv)),
            'content-disposition': 'attachment; filename=test.csv'
        }
        fileupload = FileUpload(
            FieldStorage(fp=StringIO(testcsv), environ=env, headers=headers))

        view.request.form.update({
            'speciesoccurrence.buttons.save':
            u'Save',
            'speciesoccurrence.widgets.file':
            fileupload,
            'speciesoccurrence.widgets.title':
            u'test species title',
            'speciesoccurrence.widgets.description':
            u'some test.csv file',
            'speciesoccurrence.widgets.legalcheckbox': [u'selected'],
            'speciesoccurrence.widgets.legalcheckbox-empty-marker':
            u'1',
            'speciesoccurrence.widgets.rights':
            u'test rights',
            'speciesoccurrence.widgets.scientificName':
            u'test species',
            'speciesoccurrence.widgets.taxonID':
            u'test taxonid',
            'speciesoccurrence.widgets.vernacularName':
            u'test it'
        })
        _ = view()
        self.assertEqual(self.portal.REQUEST.response.status, 302)
        self.assertEqual(
            self.portal.REQUEST.response.getHeader('Location'),
            'http://{0}:{1}/plone/datasets'.format(self.layer.get('host'),
                                                   self.layer.get('port')))
        # dataset should exist now
        ds = self.portal.datasets.species.user['test.csv']
        self.assertEqual(ds.rights, u'test rights')
        self.assertEqual(ds.file.data, testcsv)
        md = IBCCVLMetadata(ds)
        self.assertEqual(md['genre'], 'DataGenreSpeciesOccurrence')
        self.assertEqual(md['species']['taxonID'], u'test taxonid')
        self.assertEqual(md['species']['scientificName'], u'test species')
        self.assertEqual(md['species']['vernacularName'], u'test it')

        jt = IJobTracker(ds)
        self.assertEqual(jt.state, 'PENDING')

        # patch movelib.move
        def move_occurrence_data(*args, **kw):
            # try to move test.csv from dataset
            src, dst = (urlsplit(x['url']) for x in args)
            # copy test.csv to dst
            shutil.copyfileobj(
                resource_stream(__name__, 'mock_data/ala_occurrence.csv'),
                open(dst.path, 'w'))

        mock_move.side_effect = move_occurrence_data

        # triger background process
        transaction.commit()
        # one move should have happened
        self.assertEqual(
            mock_move.call_args[0][0]['url'],
            'http://{0}:{1}/plone/datasets/species/user/test.csv/@@download/file/test.csv'
            .format(self.layer.get('host'), self.layer.get('port')))
        # job state should be complete
        self.assertEqual(jt.state, 'COMPLETED')
        # metadata should be up to date
        self.assertEqual(md['rows'], 29)
 def create_test_file_field(self, data, filename):
     field_storage = ZopeFieldStorage()
     field_storage.file = BytesIO(data)
     field_storage.filename = filename
     file_field = FileUpload(field_storage)
     return file_field
Exemple #34
0
 def _create_test_file_field(self):
     field_storage = ZopeFieldStorage()
     field_storage.file = BytesIO('test')
     field_storage.filename = 'test.pdf'
     file_field = FileUpload(field_storage)
     return file_field