Example #1
0
def importer_process_worker(iid):
    """Do the real import work

    :param iid: import id we need to pull and work on

    """
    logger = importer_process_worker.get_logger()

    trans = transaction.begin()
    initialize_sql(ini)
    import_job = ImportQueueMgr.get(iid)
    logger.info("IMPORT: RUNNING for {username}".format(**dict(import_job)))
    try:
        # process the file using the import script
        import_file = open(import_job.file_path)
        importer = Importer(
            import_file,
            import_job.username)
        importer.process()
        import_job.mark_done()
        logger.info(
            "IMPORT: COMPLETE for {username}".format(**dict(import_job)))
    except Exception, exc:
        # we need to log this and probably send an error email to the
        # admin
        logger = importer_process_worker.get_logger()
        logger.error(str(exc))
        import_job.mark_error()
Example #2
0
    def test_import_process(self):
        """Verify importer inserts the correct records"""
        good_file = self._get_del_file()
        imp = Importer(good_file, username="******")
        imp.process()

        # now let's do some db sanity checks
        _delicious_data_test()
Example #3
0
    def test_import_process(self):
        """Verify importer inserts the correct records"""
        good_file = self._get_del_file()
        imp = Importer(good_file, username="******")
        imp.process()

        # now let's do some db sanity checks
        _delicious_data_test()
Example #4
0
    def test_import_process(self):
        """Verify importer inserts the correct google bookmarks"""
        good_file = self._get_google_file()
        imp = Importer(good_file, username="******")
        imp.process()

        # now let's do some db sanity checks
        _google_data_test()
Example #5
0
    def test_import_process(self):
        """Verify importer inserts the correct google bookmarks"""
        good_file = self._get_google_file()
        imp = Importer(good_file)
        imp.process()

        # now let's do some db sanity checks
        _google_data_test()
Example #6
0
    def test_bookmarklet_file(self):
        """Verify we can import a file with a bookmarklet in it."""
        loc = os.path.dirname(__file__)
        bmarklet_file = os.path.join(loc, 'bookmarklet_error.htm')
        fh = open(bmarklet_file)

        imp = Importer(fh, username=u"admin")
        imp.process()

        res = Bmark.query.all()
        self.assertEqual(len(res), 3)
Example #7
0
    def test_bookmarklet_file(self):
        """Verify we can import a file with a bookmarklet in it."""
        loc = os.path.dirname(__file__)
        bmarklet_file = os.path.join(loc, 'bookmarklet_error.htm')
        fh = open(bmarklet_file)

        imp = Importer(fh, username=u"admin")
        imp.process()

        res = Bmark.query.all()
        self.assertEqual(len(res), 3)
Example #8
0
File: utils.py Project: akn/Bookie
def import_bmarks(request):
    """Allow users to upload a delicious bookmark export"""
    data = {}
    post = request.POST
    LOG.error(request.registry.settings.get('api_key', ''))
    LOG.error(post.get('api_key'))
    if post:
        # we have some posted values
        with Authorize(request.registry.settings.get('api_key', ''),
                       post.get('api_key', None)):

            # if auth fails, it'll raise an HTTPForbidden exception
            files = post.get('import_file', None)

            if files is not None:
                # upload is there for use
                # process the file using the import script
                importer = Importer(files.file)

                # we want to store fulltext info so send that along to the
                # import processor
                conn_str = request.registry.settings.get('sqlalchemy.url',
                                                         False)
                searcher = get_fulltext_handler(conn_str)
                importer.process(fulltext=searcher)

                # @todo get a count of the imported bookmarks and setup a flash
                # message. Forward to / and display the import message

                # request.session.flash("Error something")
                return HTTPFound(location=request.route_url('home'))
            else:
                msg = request.session.pop_flash()

                if msg:
                    data['error'] = msg
                else:
                    data['error'] = None

            return data
    else:
        # just display the form
        return {}
Example #9
0
def import_bmarks(request):
    """Allow users to upload a delicious bookmark export"""
    data = {}
    post = request.POST
    LOG.error(request.registry.settings.get('api_key', ''))
    LOG.error(post.get('api_key'))
    if post:
        # we have some posted values
        with Authorize(request.registry.settings.get('api_key', ''),
                       post.get('api_key', None)):

            # if auth fails, it'll raise an HTTPForbidden exception
            files = post.get('import_file', None)

            if files is not None:
                # upload is there for use
                # process the file using the import script
                importer = Importer(files.file)

                # we want to store fulltext info so send that along to the
                # import processor
                conn_str = request.registry.settings.get(
                    'sqlalchemy.url', False)
                searcher = get_fulltext_handler(conn_str)
                importer.process(fulltext=searcher)

                # @todo get a count of the imported bookmarks and setup a flash
                # message. Forward to / and display the import message

                # request.session.flash("Error something")
                return HTTPFound(location=request.route_url('home'))
            else:
                msg = request.session.pop_flash()

                if msg:
                    data['error'] = msg
                else:
                    data['error'] = None

            return data
    else:
        # just display the form
        return {}
Example #10
0
    def test_dupe_imports(self):
        """If we import twice, we shouldn't end up with duplicate bmarks"""
        good_file = self._get_del_file()
        imp = Importer(good_file, username=u"admin")
        imp.process()

        good_file = self._get_del_file()
        imp = Importer(good_file, username=u"admin")
        imp.process()

        # Now let's do some db sanity checks.
        self._delicious_xml_data_test()
Example #11
0
    def test_factory_gives_google(self):
        """"Verify that the base importer will give GBookmarkImporter"""
        loc = os.path.dirname(__file__)
        google_file = os.path.join(loc, 'googlebookmarks.html')

        with open(google_file) as google_io:
            imp = Importer(google_io, username=u"admin")

            self.assertTrue(isinstance(imp, GBookmarkImporter),
                            "Instance should be a GBookmarkImporter instance")
Example #12
0
    def test_factory_gives_delicious(self):
        """"Verify that the base importer will give DelImporter"""
        loc = os.path.dirname(__file__)
        del_file = os.path.join(loc, 'delicious.html')

        with open(del_file) as del_io:
            imp = Importer(del_io, username=u"admin")

            self.assertTrue(isinstance(imp, DelImporter),
                            "Instance should be a delimporter instance")
Example #13
0
def import_bmarks(request):
    """Allow users to upload a delicious bookmark export"""
    rdict = request.matchdict
    username = rdict.get("username")

    # if auth fails, it'll raise an HTTPForbidden exception
    with ReqAuthorize(request):
        data = {}
        post = request.POST
        if post:
            # we have some posted values
            files = post.get("import_file", None)

            if files is not None:
                # upload is there for use
                # process the file using the import script
                importer = Importer(files.file, username=username)
                importer.process()

                # @todo get a count of the imported bookmarks and setup a flash
                # message. Forward to / and display the import message

                # request.session.flash("Error something")
                return HTTPFound(location=request.route_url("user_home", username=username))
            else:
                msg = request.session.pop_flash()

                if msg:
                    data["error"] = msg
                else:
                    data["error"] = None

            return data
        else:
            # just display the form
            return {}
Example #14
0
    def test_dupe_imports(self):
        """If we import twice, we shouldn't end up with duplicate bmarks"""
        good_file = self._get_del_file()
        imp = Importer(good_file, username=u"admin")
        imp.process()

        good_file = self._get_del_file()
        imp = Importer(good_file, username=u"admin")
        imp.process()

        # Now let's do some db sanity checks.
        self._delicious_xml_data_test()
Example #15
0
def importer_process_worker(import_id):
    """Do the real import work

    :param import_id: import id we need to pull and work on

    """
    trans = transaction.begin()
    import_job = ImportQueueMgr.get(import_id)
    logger.info("IMPORT: RUNNING for {username}".format(**dict(import_job)))

    try:
        # process the file using the import script
        import_file = open(import_job.file_path)
        importer = Importer(
            import_file,
            import_job.username)
        importer.process()

        # Processing kills off our transaction so we need to start a new one
        # to update that our import is complete.
        trans = transaction.begin()
        import_job = ImportQueueMgr.get(import_id)
        import_job.mark_done()
        user = UserMgr.get(username=import_job.username)
        from bookie.lib.message import UserImportSuccessMessage
        msg = UserImportSuccessMessage(
            user.email,
            'Bookie: Your requested import has completed.',
            INI)
        msg.send({
            'username': import_job.username,
        })

        logger.info(
            "IMPORT: COMPLETE for {username}".format(**dict(import_job)))
        trans.commit()

    except Exception, exc:
        # We need to log this and probably send an error email to the
        # admin
        from bookie.lib.message import ImportFailureMessage
        from bookie.lib.message import UserImportFailureMessage

        trans = transaction.begin()
        import_job = ImportQueueMgr.get(import_id)
        user = UserMgr.get(username=import_job.username)

        msg = ImportFailureMessage(
            INI.get('email.from'),
            'Import failure!',
            INI)
        msg.send({
            'username': import_job.username,
            'file_path': import_job.file_path,
            'exc': str(exc)
        })

        # Also send an email to the user that their import failed.
        msg = UserImportFailureMessage(
            user.email,
            'Bookie: We are sorry, your import failed.',
            INI)
        msg.send({
            'username': import_job.username,
            'exc': str(exc)
        })

        logger.error(exc)
        logger.error(str(exc))
        import_job.mark_error()
        logger.info(
            "IMPORT: ERROR for {username}".format(**dict(import_job)))
        logger.info(exc)
        trans.commit()
Example #16
0
 def test_doesnt_implement_can_handle(self):
     """Verify we get the exception expected when running can_handle"""
     Importer.can_handle("")
Example #17
0
 def test_doesnt_implement_process(self):
     """Verify we get the exception expected when running process"""
     some_io = StringIO.StringIO()
     imp = Importer(some_io)
     imp.process()
Example #18
0
 def test_doesnt_implement_can_handle(self):
     """Verify we get the exception expected when running can_handle"""
     Importer.can_handle("")
Example #19
0
 def test_doesnt_implement_process(self):
     """Verify we get the exception expected when running process"""
     some_io = StringIO.StringIO()
     imp = Importer(some_io)
     imp.process()
Example #20
0
 def test_doesnt_implement_process(self):
     """Verify we get the exception expected when running process"""
     some_io = StringIO.StringIO()
     imp = Importer(some_io)
     self.assertRaises(NotImplementedError, imp.process)