Exemple #1
0
def migrate_response_attachments_to_blobstorage(context):
    logger.info('Migrating response attachments to blob storage.')
    catalog = getToolByName(context, 'portal_catalog')
    already_migrated = 0
    migrated = 0
    for brain in catalog.unrestrictedSearchResults(portal_type='PoiIssue'):
        path = brain.getPath()
        try:
            issue = brain.getObject()
        except (AttributeError, ValueError, TypeError):
            logger.warn('Error getting object from catalog for path %s', path)
            continue
        folder = IResponseContainer(issue)
        for id, response in enumerate(folder):
            if response is None:
                # Has been removed.
                continue
            attachment = response.attachment
            if attachment is None:
                continue
            if isinstance(attachment, NamedBlobFile):
                # Already migrated
                logger.debug('Response %d already migrated, at %s.', id, path)
                already_migrated += 1
                continue
            content_type = getattr(attachment, 'content_type', '')
            filename = getattr(attachment, 'filename', '')
            if not filename and hasattr(attachment, 'getId'):
                filename = attachment.getId()
            data = attachment.data
            # Data can be 'nested' in OFS.Image.Pdata.
            if base_hasattr(data, 'data'):
                data = data.data
            filename = safe_unicode(filename)
            try:
                blob = NamedBlobFile(data,
                                     contentType=content_type,
                                     filename=filename)
            except ConstraintNotSatisfied:
                # Found in live data: a filename that includes a newline...
                logger.info('Trying to normalize filename %s', filename)
                filename = normalize_filename(filename, context.REQUEST)
                logger.info('Normalize to %s', filename)
                blob = NamedBlobFile(data,
                                     contentType=content_type,
                                     filename=filename)
            response.attachment = blob
            logger.debug('Response %d migrated, at %s.', id, path)
            migrated += 1

    logger.info(
        'Migrated %d response attachments to blobs. '
        '%d already migrated.', migrated, already_migrated)
Exemple #2
0
 def test_second_step_import_encoding_form(self):
     """Test importing csv data with special chars in header and content"""
     form = importform.ImportFormSecondStep(self.container,
                                            self.layer["request"])
     annotations = IAnnotations(self.container)
     annotation = annotations[importform.ANNOTATION_KEY] = PersistentDict()
     annotation["has_header"] = True
     annotation["separator"] = u";"
     csv = StringIO()
     lines = [
         [u"猫".encode("utf8"), u"èè".encode("utf8"), u"ùù".encode("utf8")],
         ["", u"kéy1".encode("utf8"), u"Kèy 1".encode("utf8")],
         [
             u"kéy1".encode("utf8"), u"kéy1.1".encode("utf8"),
             u"猫".encode("utf8")
         ],
     ]
     for line in lines:
         csv.write(";".join(line) + "\n")
     csv.seek(0)
     annotation["source"] = NamedBlobFile(
         data=csv.read(),
         contentType=u"text/csv",
         filename=u"test.csv",
     )
     form.update()
     exception = None
     try:
         render = form.render()
     except UnicodeDecodeError as e:
         exception = e
     self.assertIsNone(exception)
     self.assertTrue(u"Column {0}".format(u"猫") in render)
Exemple #3
0
 def createFile(self, path, title=None):
     obj = self._createObject('File', path, title=title)
     named_file = NamedBlobFile(pdf_data,
                                filename=u'test.pdf',
                                contentType='application/pdf')
     obj.file = named_file
     obj.reindexObject()
Exemple #4
0
 def test_second_step_import_single_column(self):
     """Test importing csv data"""
     form = importform.ImportFormSecondStep(self.container,
                                            self.layer["request"])
     annotations = IAnnotations(self.container)
     annotation = annotations[importform.ANNOTATION_KEY] = PersistentDict()
     annotation["has_header"] = False
     annotation["separator"] = u";"
     csv = StringIO()
     lines = [
         ["", "key1", "Key 1"],
         ["", "key2", "Key 2"],
     ]
     for line in lines:
         csv.write(";".join(line) + "\n")
     csv.seek(0)
     annotation["source"] = NamedBlobFile(
         data=csv.read(),
         contentType=u"text/csv",
         filename=u"test.csv",
     )
     data = {
         "column_0": None,
         "column_1": "identifier",
         "column_2": None,
         "decimal_import": False,
         "allow_empty": False,
     }
     form._import(data)
     self.assertEqual(2, len(self.container))
     self.assertEqual(["key1", "key2"],
                      sorted(
                          [e.identifier for e in self.container.values()]))
     self.assertEqual(["key1", "key2"],
                      sorted([e.title for e in self.container.values()]))
Exemple #5
0
 def fetch_data(self, field_obj, field_value, _obj, row_buffer):
     field_type = field_obj[1].__class__.__name__
     field_title = field_obj[1].title
     field_name = field_obj[0]
     if field_type in ('NamedBlobImage', 'NamedBlobFile'):
         request = urllib2.Request(url=field_value)
         try:
             response = urllib2.urlopen(request)
             filename = os.path.basename(response.url)
             if field_type == 'NamedBlobFile':
                 blob_file = NamedBlobFile(data=response.read(),
                                           filename=safe_unicode(filename))
             elif field_type == 'NamedBlobImage':
                 blob_file = NamedBlobImage(data=response.read(),
                                            filename=safe_unicode(filename))
             setattr(_obj, field_name, blob_file)
             return True
         except Exception:
             field_title = translate(_(field_title), context=self.request)
             error_doc = translate(
                 _("""The URL you have provided could not be reached."""),
                 context=self.request)
             msg = u'{0}: {1}; '.format(error_doc, field_title)
             row_buffer[2] += msg
             return False
Exemple #6
0
 def test_second_step_basic_delimiter(self):
     """Test edge case related to csv delimiter"""
     form = importform.ImportFormSecondStep(self.container,
                                            self.layer["request"])
     annotations = IAnnotations(self.container)
     annotation = annotations[importform.ANNOTATION_KEY] = PersistentDict()
     annotation["has_header"] = False
     annotation["separator"] = u","
     csv = StringIO()
     lines = [
         ["", "key1", "Key 1"],
         ["key1", "key1.1", '"Key 1,1"'],
         ["key1.1", "key1.1.1", '"Key 1.1.1"'],
     ]
     for line in lines:
         csv.write(",".join(line) + "\n")
     csv.seek(0)
     annotation["source"] = NamedBlobFile(
         data=csv.read(),
         contentType=u"text/csv",
         filename=u"test.csv",
     )
     exception = None
     try:
         form.update()
     except Exception as e:
         exception = e
     self.assertIsNone(exception)
Exemple #7
0
 def test_second_step_optional_columns_data_ok(self):
     """Test validation of optional columns data"""
     request = self.layer["request"]
     request.form = {
         "form.buttons.import": u"Importer",
         "form.widgets.column_0": u"parent_identifier",
         "form.widgets.column_1": u"identifier",
         "form.widgets.column_2": u"title",
         "form.widgets.column_3": u"informations",
         "form.widgets.decimal_import": u"False",
         "form.widgets.allow_empty": u"False",
     }
     annotations = IAnnotations(self.container)
     annotation = annotations[importform.ANNOTATION_KEY] = PersistentDict()
     annotation["has_header"] = False
     annotation["separator"] = u";"
     csv = StringIO()
     lines = [
         ["", "key1", "Key 1", "infos"],
         ["key1", "key1.1", "Key 1.1", ""],
         ["key1.1", "key1.1.1", "Key 1.1.1", ""],
     ]
     for line in lines:
         csv.write(";".join(line) + "\n")
     csv.seek(0)
     annotation["source"] = NamedBlobFile(
         data=csv.read(),
         contentType=u"text/csv",
         filename=u"test.csv",
     )
     form = importform.ImportFormSecondStep(self.container, request)
     form.updateFieldsFromSchemata()
     form.updateWidgets()
     data, errors = form.extractData()
     self.assertEqual(0, len(errors))
Exemple #8
0
 def test_second_step_basic_encoding(self):
     """Ensure that form can be displayed even with special characters"""
     form = importform.ImportFormSecondStep(self.container,
                                            self.layer["request"])
     annotations = IAnnotations(self.container)
     annotation = annotations[importform.ANNOTATION_KEY] = PersistentDict()
     annotation["has_header"] = False
     annotation["separator"] = u";"
     csv = StringIO()
     lines = [
         ["null", "key1", "key1.1", "Key 1.1", "informations"],
         [
             "null",
             "",
             u"key1 éà$€".encode("utf8"),
             u"Key 1 éà$€".encode("utf8"),
             u"informations éà$€".encode("utf8"),
         ],
     ]
     for line in lines:
         csv.write(";".join(line) + "\n")
     csv.seek(0)
     annotation["source"] = NamedBlobFile(
         data=csv.read(),
         contentType=u"text/csv",
         filename=u"test.csv",
     )
     exception = None
     try:
         form.update()
     except UnicodeDecodeError as e:
         exception = e
     self.assertIsNone(exception)
Exemple #9
0
    def _add_dashboard_pod_template_export_users_groups(self):
        """Add the export users and groups DashboardPODTemplate in the contacts directory."""
        logger.info("Adding 'Export users and groups' to 'contacts' directory...")
        pod_template_id = 'export-users-groups'
        contacts = self.portal.contacts
        if pod_template_id in contacts.objectIds():
            self._already_migrated()
            return

        profile_path = self.ps._getImportContext(self.profile_name)._profile_path
        odt_path = profile_path + '/../examples_fr/templates/users-groups-export.ods'
        odt_file = open(odt_path, 'rb')
        odt_binary = odt_file.read()
        odt_file.close()
        data = {'title': 'Export utilisateurs et groupes',
                'pod_formats': ['ods', 'xls'],
                'dashboard_collections': contacts.get('orgs-searches').all_orgs.UID(),
                'odt_file': NamedBlobFile(
                    data=odt_binary,
                    contentType='application/vnd.oasis.opendocument.text',
                    filename=u'users-groups-export.ods'),
                'use_objects': False,
                }
        pod_template = api.content.create(
            id=pod_template_id,
            type='DashboardPODTemplate',
            container=contacts,
            **data)
        pod_template.reindexObject()
        logger.info('Done.')
Exemple #10
0
 def test_second_step_required_columns_nok(self):
     """Test validation of required columns"""
     request = self.layer["request"]
     request.form = {
         "form.buttons.import": u"Importer",
         "form.widgets.column_0": u"--NOVALUE--",
         "form.widgets.column_1": u"--NOVALUE--",
         "form.widgets.column_2": u"--NOVALUE--",
         "form.widgets.decimal_import": u"False",
         "form.widgets.allow_empty": u"False",
     }
     annotations = IAnnotations(self.container)
     annotation = annotations[importform.ANNOTATION_KEY] = PersistentDict()
     annotation["has_header"] = False
     annotation["separator"] = u";"
     annotation["source"] = NamedBlobFile(
         data=self._csv.read(),
         contentType=u"text/csv",
         filename=u"test.csv",
     )
     form = importform.ImportFormSecondStep(self.container, request)
     form.updateFieldsFromSchemata()
     form.updateWidgets()
     data, errors = form.extractData()
     self.assertEqual(1, len(errors))
     self.assertEqual(
         "The following required columns are missing: identifier",
         translate(errors[0].error.message),
     )
Exemple #11
0
def _getBlob(_type='audio', _format='mp3'):
    if _type == 'audio':
        filename = u'test.%s' % _format
    else:
        filename = u'test.%s' % _format

    newpath = mktemp()
    origpath = os.path.join(test_file_dir, filename)
    copyfile(origpath, newpath)
    fi = open(newpath, 'rb')
    blob = NamedBlobFile(fi, filename=filename)
    fi.close()
    return blob
Exemple #12
0
 def setUp(self):
     self.portal = self.layer['portal']
     self.request = self.layer['request']
     setRoles(self.portal, TEST_USER_ID, TEST_USER_ROLES + ['Manager'])
     self.portal.invokeFactory(
         type_name=TEST_CONTENT_TYPE_ID,
         id='obj1',
         title=u'Object 1 Title',
         description=u'Description of obect number 1',
         text=u'Object 1 some footext.',
         file=NamedBlobFile(filename=u'object_1_file.txt',
                            data='Object 1 Data'),
     )
     self.obj1 = self.portal['obj1']
 def __exit__(self, exc_type, exc_value, traceback):
     # push the result back into the site pod templates
     for filename in self.changed_files:
         with open(filename, "rb") as replaced_file:
             podtemplate = self.templates_by_filename[filename]["obj"]
             result = NamedBlobFile(
                 data=replaced_file.read(),
                 contentType=mimetypes.guess_type(filename)[0],
                 filename=podtemplate.odt_file.filename,
             )
             podtemplate.odt_file = result
     # clean tmp file
     for filename in self.templates_by_filename.keys():
         if os.path.isfile(filename):
             os.remove(filename)
Exemple #14
0
    def test_query_filter_by_friendly_type(self):
        img_path = resource_filename('ploneintranet',
                                     'userprofile/tests/test_avatar.jpg')
        with open(img_path, 'rb') as fp:
            img_data = fp.read()
        self.image1 = self._create_content(
            type='Image',
            container=self.layer['portal'],
            title=u'A Test image',
            description=u'Info about this image',
            file=NamedBlobFile(
                data=img_data,
                contentType='image/jpeg',
                filename=fp.name.decode('utf-8'),
            ))
        transaction.commit()

        util = self._make_utility()
        response = util.query(u'Test',
                              filters={'friendly_type_name': ['Image']})
        self.assertEqual(response.total_results, 1)
        result = next(iter(response))
        self.assertEqual(result.title, self.image1.title)

        response = util.query(u'Test',
                              filters={'friendly_type_name': [
                                  'Page',
                              ]})
        self.assertEqual(response.total_results, 3)
        self.assertEqual(
            set([x.friendly_type_name for x in response]),
            {'Page'},
        )

        response = util.query(
            u'Test', filters={'friendly_type_name': [
                'Image',
                'Page',
            ]})
        self.assertEqual(response.total_results, 4)
        self.assertEqual(
            set([x.friendly_type_name for x in response]),
            {
                'Page',
                'Image',
            },
        )
 def add_failing_template(self):
     current_path = os.path.dirname(__file__)
     failing_template_data = open(
         os.path.join(current_path, "failing_template.odt"), "r").read()
     self.failing_template = api.content.create(
         type="ConfigurablePODTemplate",
         id="failing_template",
         title=_(u"Failing template"),
         odt_file=NamedBlobFile(
             data=failing_template_data,
             contentType="application/vnd.oasis.opendocument.text",
             filename=u"modele_general.odt",
         ),
         pod_formats=["odt"],
         container=self.portal.podtemplates,
         exclude_from_nav=True,
     )
     self.failing_template.reindexObject()
 def setUp(self):
     super(TestFriendlyName, self).setUp()
     self.portal = api.portal.get()
     self.catalog = api.portal.get_tool(name='portal_catalog')
     self.doc1 = self._create_content(
         type='Document',
         title='Test Doc',
         container=self.portal,
     )
     self.file1 = self._create_content(type='File',
                                       title='Test File',
                                       container=self.portal,
                                       file=NamedBlobFile(
                                           data='blah blah',
                                           filename=u'test-file.pdf',
                                           contentType='application/pdf',
                                       ))
     transaction.commit()
Exemple #17
0
def pop_temporary_attachment(workspace, file_field, token):
    """
    Replace a temporary attachment on the workspace with
    the uploaded data
    """
    temp_attachments = IAttachmentStorage(workspace)
    temp_id = getUtility(IURLNormalizer).normalize(u'{0}-{1}'.format(
        token, safe_unicode(file_field.filename)))
    if temp_id in temp_attachments.keys():
        temp_att = aq_base(temp_attachments.get(temp_id))
        temp_att.id = file_field.filename
        temp_att.file = NamedBlobFile(
            data=file_field.read(),
            filename=file_field.filename.decode('utf-8'),
        )
        temp_attachments.remove(temp_id)
        return temp_att
    return None
Exemple #18
0
    def test_second_step_import_encoding(self):
        """Test importing csv data with special chars in header and content"""
        form = importform.ImportFormSecondStep(self.container,
                                               self.layer["request"])
        annotations = IAnnotations(self.container)
        annotation = annotations[importform.ANNOTATION_KEY] = PersistentDict()
        annotation["has_header"] = True
        annotation["separator"] = u";"
        csv = StringIO()
        lines = [
            [u"猫".encode("utf8"), u"èè".encode("utf8"), u"ùù".encode("utf8")],
            ["", u"kéy1".encode("utf8"), u"Kèy 1".encode("utf8")],
            [
                u"kéy1".encode("utf8"), u"kéy1.1".encode("utf8"),
                u"猫".encode("utf8")
            ],
        ]
        for line in lines:
            csv.write(";".join(line) + "\n")
        csv.seek(0)
        annotation["source"] = NamedBlobFile(
            data=csv.read(),
            contentType=u"text/csv",
            filename=u"test.csv",
        )
        data = {
            "column_0": "parent_identifier",
            "column_1": "identifier",
            "column_2": "title",
            "decimal_import": False,
            "allow_empty": False,
        }
        form._import(data)
        self.assertEqual(1, len(self.container))
        self.assertEqual([u"kéy1"],
                         [e.identifier for e in self.container.values()])

        key1 = self.container.get_by("identifier", u"kéy1")
        self.assertEqual(1, len(key1))
        self.assertEqual([u"kéy1.1"], [e.identifier for e in key1.values()])

        key1_1 = key1.get_by("identifier", u"kéy1.1")
        self.assertEqual(u"猫", key1_1.title)
Exemple #19
0
    def _delete_attachments(self, attachments):
        if not attachments:
            return

        attachment_names = [
            attachment.get('filename', '[no filename]').decode('utf-8')
            for attachment in attachments
        ]
        positions = [attachment['position'] for attachment in attachments]

        # Flag the `message` attribute as having changed
        desc = Attributes(IAttachmentsDeletedEvent, "message")
        notify(AttachmentsDeleted(self, attachment_names, desc))

        # set the new message file
        msg = remove_attachments(self.msg, positions)
        self.message = NamedBlobFile(data=msg.as_string(),
                                     contentType=self.message.contentType,
                                     filename=self.message.filename)
Exemple #20
0
    def test_second_step_import_basic(self):
        """Test importing csv data"""
        form = importform.ImportFormSecondStep(self.container,
                                               self.layer["request"])
        annotations = IAnnotations(self.container)
        annotation = annotations[importform.ANNOTATION_KEY] = PersistentDict()
        annotation["has_header"] = False
        annotation["separator"] = u";"
        annotation["source"] = NamedBlobFile(
            data=self._csv.read(),
            contentType=u"text/csv",
            filename=u"test.csv",
        )
        data = {
            "column_0": "parent_identifier",
            "column_1": "identifier",
            "column_2": "title",
            "decimal_import": False,
            "allow_empty": False,
        }
        form._import(data)
        self.assertEqual(2, len(self.container))
        self.assertEqual(["key1", "key2"],
                         sorted(
                             [e.identifier for e in self.container.values()]))

        key1 = self.container.get_by("identifier", "key1")
        self.assertEqual(3, len(key1))
        self.assertEqual(
            ["key1.1", "key1.2", "key1.3"],
            sorted([e.identifier for e in key1.values()]),
        )

        key1_1 = key1.get_by("identifier", "key1.1")
        self.assertEqual(1, len(key1_1))
        self.assertEqual(["key1.1.1"],
                         sorted([e.identifier for e in key1_1.values()]))

        key2 = self.container.get_by("identifier", "key2")
        self.assertEqual(1, len(key2))
        self.assertEqual(["key2.1"],
                         sorted([e.identifier for e in key2.values()]))
Exemple #21
0
def create_attachment(filename, data):
    """ Set up a contextless dexterity file to hold the attachment data """
    if not isinstance(filename, unicode):
        filename = filename.decode('utf-8')
    namedfile = NamedBlobFile(
        data=data,
        filename=filename,
    )
    if namedfile.contentType.startswith('image'):
        fti = queryUtility(IDexterityFTI, name='Image')
        thefile = createObject(fti.factory, id=filename, image=namedfile)
    else:
        fti = queryUtility(IDexterityFTI, name='File')
        thefile = createObject(fti.factory, id=filename, file=namedfile)

    # assign a uuid, needed for c.documentviewer
    generator = queryUtility(IUUIDGenerator)
    setattr(thefile, ATTRIBUTE_NAME, generator())

    return thefile
Exemple #22
0
 def test_file_content_matches(self):
     path = resource_filename('ploneintranet.search.tests',
                              'fixtures/lorum-ipsum.pdf')
     with open(path, 'rb') as fp:
         data = fp.read()
     self._create_content(type='File',
                          container=self.layer['portal'],
                          title=u'Test File 1',
                          description=(u'This is a test file. '),
                          safe_id=False,
                          file=NamedBlobFile(
                              data=data,
                              contentType='application/pdf',
                              filename=fp.name.decode('utf-8')))
     transaction.commit()
     util = self._make_utility()
     query = util.query
     response = query(u'Maecenas urna elit')
     results = list(response)
     self.assertEqual(len(results), 1)
     self.assertEqual(results[0].title, u'Test File 1')
Exemple #23
0
 def test_second_step_columns_data_format_nok(self):
     """Test validation of columns data format"""
     request = self.layer["request"]
     request.form = {
         "form.buttons.import": u"Importer",
         "form.widgets.column_0": u"identifier",
         "form.widgets.column_1": u"title",
         "form.widgets.decimal_import": u"selected",
         "form.widgets.allow_empty": u"False",
     }
     annotations = IAnnotations(self.container)
     annotation = annotations[importform.ANNOTATION_KEY] = PersistentDict()
     annotation["has_header"] = False
     annotation["separator"] = u";"
     csv = StringIO()
     lines = [
         ["-1", "key1"],
         [".10", "key2"],
         ["-1.1", "key3"],
         ["-1 11", "Key4"],
     ]
     for line in lines:
         csv.write(";".join(line) + "\n")
     csv.seek(0)
     annotation["source"] = NamedBlobFile(
         data=csv.read(),
         contentType=u"text/csv",
         filename=u"test.csv",
     )
     form = importform.ImportFormSecondStep(self.container, request)
     form.updateFieldsFromSchemata()
     form.updateWidgets()
     data, errors = form.extractData()
     self.assertEqual(1, len(errors))
     self.assertEqual(
         "Bad format values: Line 4, col 1: '-1 11'",
         translate(errors[0].error.message),
     )
def consumeMessage(message, event):
    """Consume message"""

    uuid = message.header_frame.correlation_id
    minutes = uuidToObject(uuid)

    site = getUtility(ISiteRoot)
    mtool = getToolByName(site, "portal_membership")
    creator = mtool.getMemberById(minutes.creators[0])
    date = minutes.modified()

    kwargs = {
        "date": "%s.%s.%s" % (date.day(), date.month(), date.year()),
        "title": minutes.title,
        "author": creator.getProperty("fullname"),
        "email": creator.getProperty("email"),
        "body": minutes.body.output,
        "site": site.Title()
    }

    # Read receipt template from disk
    path = os.path.join(os.path.dirname(__file__), "sfs2487.pt")
    template = PageTemplateFile(path)

    # Render template into HTML
    html = template(**kwargs)

    # Create a PDF from HTML
    pdf = StringIO()
    pisa.CreatePDF(html, pdf)
    pdf.seek(0)

    # Save the PDF
    blob = NamedBlobFile(pdf.read(), filename=u"minutes.pdf")
    bound = IMinutes["deliverable"].bind(minutes)
    bound.set(minutes, blob)

    message.ack()
Exemple #25
0
    def test_first_step_set_data(self):
        form = importform.ImportFormFirstStep(self.container,
                                              self.layer["request"])
        data = {
            "source":
            NamedBlobFile(
                data=self._csv.read(),
                contentType=u"text/csv",
                filename=u"test.csv",
            ),
            "separator":
            u";",
            "has_header":
            False,
        }
        form._set_data(data)
        annotations = IAnnotations(self.container)
        self.assertTrue(importform.ANNOTATION_KEY in annotations)

        annotation = annotations[importform.ANNOTATION_KEY]
        self.assertEqual(data["separator"], annotation["separator"])
        self.assertEqual(data["source"], annotation["source"])
        self.assertEqual(data["has_header"], annotation["has_header"])
Exemple #26
0
 def test_second_step_import_basic(self):
     """Test importing csv data"""
     form = importform.ImportFormSecondStep(self.folders,
                                            self.layer["request"])
     form.data = []
     annotations = IAnnotations(self.folders)
     annotation = annotations[importform.ANNOTATION_KEY] = PersistentDict()
     annotation["separator"] = u";"
     annotation["has_header"] = False
     annotation["source"] = NamedBlobFile(
         data=self._csv.read(),
         contentType=u"text/csv",
         filename=u"test.csv",
     )
     data = {
         "treating_groups": None,
         "column_1": "classification_categories",
         "column_3": "parent_identifier",
         "column_4": "internal_reference_no",
         "column_5": "title",
     }
     self.assertEqual(0, len(self.folders))
     form.update()
     form._import(data)
Exemple #27
0
 def test_second_step_required_columns_data_ok(self):
     """Test validation of required columns data"""
     request = self.layer["request"]
     request.form = {
         "form.buttons.import": u"Importer",
         "form.widgets.column_0": u"parent_identifier",
         "form.widgets.column_1": u"identifier",
         "form.widgets.column_2": u"title",
         "form.widgets.decimal_import": u"False",
     }
     annotations = IAnnotations(self.container)
     annotation = annotations[importform.ANNOTATION_KEY] = PersistentDict()
     annotation["has_header"] = False
     annotation["separator"] = u";"
     annotation["source"] = NamedBlobFile(
         data=self._csv.read(),
         contentType=u"text/csv",
         filename=u"test.csv",
     )
     form = importform.ImportFormSecondStep(self.container, request)
     form.updateFieldsFromSchemata()
     form.updateWidgets()
     data, errors = form.extractData()
     self.assertEqual(0, len(errors))
Exemple #28
0
def _convertFormat(context):
    # reset these...
    context.video_file_ogv = None
    context.video_file_webm = None

    video = context.video_file
    context.video_converted = True
    try:
        opened = openBlob(video._blob)
        bfilepath = opened.name
        opened.close()
    except IOError:
        logger.warn('error opening blob file')
        return

    tmpdir = mkdtemp()
    tmpfilepath = os.path.join(tmpdir, video.filename)
    copyfile(bfilepath, tmpfilepath)

    try:
        metadata = avprobe.info(tmpfilepath)
    except:
        logger.warn('not a valid video format')
        return
    context.metadata = metadata

    conversion_types = {
        'mp4': 'video_file'
    }

    portal = getToolByName(context, 'portal_url').getPortalObject()
    settings = GlobalSettings(portal)
    for type_ in settings.additional_video_formats:
        format = getFormat(type_)
        if format:
            conversion_types[format.extension] = 'video_file_%s' % (
                format.extension
            )

    # sometimes force full video conversion
    force = settings.force

    for video_type, fieldname in conversion_types.items():
        if video_type == video.contentType.split('/')[-1] and not force:
            setattr(context, fieldname, video)
        else:
            output_filepath = os.path.join(tmpdir, 'output.' + video_type)
            try:
                avconv.convert(tmpfilepath, output_filepath, video_type, context)
            except:
                logger.warn('error converting to %s' % video_type)
                continue
            if os.path.exists(output_filepath):
                fi = open(output_filepath)
                namedblob = NamedBlobFile(
                    fi, filename=switchFileExt(video.filename,  video_type))
                setattr(context, fieldname, namedblob)
                fi.close()

    # try and grab one from video
    output_filepath = os.path.join(tmpdir, u'screengrab.png')
    try:
        avconv.grab_frame(tmpfilepath, output_filepath)
        if os.path.exists(output_filepath):
            with open(output_filepath, 'rb') as fi:
                data = fi.read()
            context.image = NamedBlobImage(data, filename=u'screengrab.png')
            fi.close()
    except:
        logger.warn('error getting thumbnail from video')
    rmtree(tmpdir)
Exemple #29
0
def _convertFormat(context):
    # reset these...
    context.video_file_webm = None

    video = context.video_file
    context.video_converted = True
    try:
        opened = openBlob(video._blob)
        bfilepath = opened.name
        opened.close()
    except IOError:
        logger.warn('error opening blob file')
        return

    tmpdir = mkdtemp()
    tmpfilepath = os.path.join(tmpdir, video.filename)
    copyfile(bfilepath, tmpfilepath)

    try:
        metadata = avprobe.info(tmpfilepath)
    except:
        logger.warn('NOT a valid video format')
        return
    context.metadata = metadata
    logger.info('Valid video format')

    try:
        duration = _get_duration(tmpfilepath)
    except:
        logger.warn('cannot recover duration from file')
        return
    context.duration = duration

    conversion_types = {}

    portal = getToolByName(context, 'portal_url').getPortalObject()
    settings = GlobalSettings(portal)
    for type_ in settings.additional_video_formats:
        format = getFormat(type_)
        if format:
            conversion_types[format.type_] = '%s' % (format.quality)

    for video_type, quality in conversion_types.items():
        vt = video_type.split('_')[0]
        if video_type == video.contentType.split('/')[-1]:
            setattr(context, vt, video)
        else:
            output_filepath = os.path.join(tmpdir,
                                           'output_' + video_type + '.' + vt)
            try:
                avconv.convert(tmpfilepath, output_filepath, vt, quality,
                               context)
            except:
                logger.warn('error converting to %s' % vt)
                continue
            if os.path.exists(output_filepath):
                fi = open(output_filepath)
                namedblob = NamedBlobFile(fi,
                                          filename=switchFileExt(
                                              video.filename, vt))
                setattr(context, video_type, namedblob)
                fi.close()
            import transaction
            transaction.commit()

    # try and grab one from video
    output_filepath = os.path.join(tmpdir, u'screengrab.png')
    try:
        avconv.grab_frame(tmpfilepath, output_filepath)
        if os.path.exists(output_filepath):
            with open(output_filepath, 'rb') as fi:
                data = fi.read()
            context.image = NamedBlobImage(data, filename=u'screengrab.png')
            fi.close()
    except:
        logger.warn('error getting thumbnail from video')
    logger.warn('CONVERSIONS FINISHED')
    rmtree(tmpdir)
Exemple #30
0
    def test_second_step_import_decimal_basic(self):
        """Test importing csv data with decimal codes"""
        form = importform.ImportFormSecondStep(self.container,
                                               self.layer["request"])
        annotations = IAnnotations(self.container)
        annotation = annotations[importform.ANNOTATION_KEY] = PersistentDict()
        annotation["has_header"] = False
        annotation["separator"] = u";"
        csv = StringIO()
        lines = [
            ["100", "Key 1"],
            ["100.1", "Key 1.1"],
            ["100.2", "Key 1.2"],
            ["200", "Key 2"],
            ["200.1", "Key 2.1"],
            ["200.10", "Key 2.10"],
        ]
        for line in lines:
            csv.write(";".join(line) + "\n")
        csv.seek(0)
        annotation["source"] = NamedBlobFile(
            data=csv.read(),
            contentType=u"text/csv",
            filename=u"test.csv",
        )
        data = {
            "column_0": "identifier",
            "column_1": "title",
            "decimal_import": True,
            "allow_empty": False,
        }
        form._import(data)
        self.assertEqual(2, len(self.container))
        self.assertEqual(["1", "2"],
                         sorted(
                             [e.identifier for e in self.container.values()]))

        code_1 = self.container.get_by("identifier", "1")
        self.assertEqual("1", code_1.title)
        self.assertEqual(1, len(code_1))
        self.assertEqual(["10"], [e.identifier for e in code_1.values()])

        code_10 = code_1.get_by("identifier", "10")
        self.assertEqual("10", code_10.title)
        self.assertEqual(1, len(code_10))
        self.assertEqual(["100"], [e.identifier for e in code_10.values()])

        code_100 = code_10.get_by("identifier", "100")
        self.assertEqual("Key 1", code_100.title)
        self.assertEqual(2, len(code_100))
        self.assertEqual(
            ["100.1", "100.2"],
            sorted([e.identifier for e in code_100.values()]),
        )
        self.assertEqual(
            ["Key 1.1", "Key 1.2"],
            sorted([e.title for e in code_100.values()]),
        )

        code_2 = self.container.get_by("identifier", "2")
        self.assertEqual("2", code_2.title)
        self.assertEqual(1, len(code_2))
        self.assertEqual(["20"], [e.identifier for e in code_2.values()])

        code_20 = code_2.get_by("identifier", "20")
        self.assertEqual("20", code_20.title)
        self.assertEqual(1, len(code_20))
        self.assertEqual(["200"], [e.identifier for e in code_20.values()])

        code_200 = code_20.get_by("identifier", "200")
        self.assertEqual("Key 2", code_200.title)
        self.assertEqual(1, len(code_200))
        self.assertEqual(["200.1"], [e.identifier for e in code_200.values()])

        code_2001 = code_200.get_by("identifier", "200.1")
        self.assertEqual("Key 2.1", code_2001.title)
        self.assertEqual(1, len(code_2001))
        self.assertEqual(["200.10"],
                         [e.identifier for e in code_2001.values()])
        self.assertEqual(["Key 2.10"], [e.title for e in code_2001.values()])