Exemplo n.º 1
0
def importer_process_worker(iid):
    """Do the real import work

    :param iid: import id we need to pull and work on

    """
    logger = importer_process_worker.get_logger()

    trans = transaction.begin()
    initialize_sql(ini)
    import_job = ImportQueueMgr.get(iid)
    logger.info("IMPORT: RUNNING for {username}".format(**dict(import_job)))
    try:
        # process the file using the import script
        import_file = open(import_job.file_path)
        importer = Importer(
            import_file,
            import_job.username)
        importer.process()
        import_job.mark_done()
        logger.info(
            "IMPORT: COMPLETE for {username}".format(**dict(import_job)))
    except Exception, exc:
        # we need to log this and probably send an error email to the
        # admin
        logger = importer_process_worker.get_logger()
        logger.error(str(exc))
        import_job.mark_error()
Exemplo n.º 2
0
def importer_process_worker(iid):
    """Do the real import work

    :param iid: import id we need to pull and work on

    """
    logger = importer_process_worker.get_logger()

    trans = transaction.begin()
    initialize_sql(ini)
    import_job = ImportQueueMgr.get(iid)
    logger.info("IMPORT: RUNNING for {username}".format(**dict(import_job)))
    try:
        # process the file using the import script
        import_file = open(import_job.file_path)
        importer = Importer(import_file, import_job.username)
        importer.process()
        import_job.mark_done()
        logger.info(
            "IMPORT: COMPLETE for {username}".format(**dict(import_job)))
    except Exception, exc:
        # we need to log this and probably send an error email to the
        # admin
        logger = importer_process_worker.get_logger()
        logger.error(str(exc))
        import_job.mark_error()
Exemplo n.º 3
0
    def import_bmarks(self):
        """Allow users to upload a bookmark export file for processing"""
        username = self.matchdict.get('username')

        # if auth fails, it'll raise an HTTPForbidden exception
        with ReqAuthorize(self.request):
            data = {}
            post = self.POST

            # We can't let them submit multiple times, check if this user has
            # an import in process.
            if ImportQueueMgr.get(username=username, status=NEW):
                # They have an import, get the information about it and shoot
                # to the template.
                return {
                    'existing': True,
                    'import_stats': ImportQueueMgr.get_details(
                        username=username)
                }

            if post:
                # we have some posted values
                files = post.get('import_file', None)

                if files is not None:
                    storage_dir_tpl = self.settings.get('import_files',
                                                        '/tmp/bookie')
                    storage_dir = storage_dir_tpl.format(
                        here=self.settings.get('app_root'))

                    out_fname = store_import_file(storage_dir, username, files)

                    # Mark the system that there's a pending import that needs
                    # to be completed
                    q = ImportQueue(username, unicode(out_fname))
                    DBSession.add(q)
                    DBSession.flush()
                    # Schedule a task to start this import job.
                    tasks.importer_process.delay(q.id)

                    return HTTPFound(
                        location=self.request.route_url('user_import',
                                                        username=username))
                else:
                    msg = self.request.session.pop_flash()
                    if msg:
                        data['error'] = msg
                    else:
                        data['error'] = None

                return data
            else:
                # we need to see if they've got
                # just display the form
                return {
                    'existing': False
                }
Exemplo n.º 4
0
def import_reset(request):
    """Reset an import to try again"""
    rdict = request.matchdict
    import_id = rdict.get('id', None)

    if not id:
        request.response.status_int = 400
        ret = {'error': "Bad request, missing parameters"}
        return ret

    imp = ImportQueueMgr.get(int(import_id))
    imp.status = 0
    tasks.importer_process.delay(imp.id)

    ret = {'import': dict(imp)}
    return ret
Exemplo n.º 5
0
def import_reset(request):
    """Reset an import to try again"""
    rdict = request.matchdict
    import_id = rdict.get('id', None)

    if not id:
        request.response.status_int = 400
        ret = {'error': "Bad request, missing parameters"}
        return _api_response(request, ret)

    imp = ImportQueueMgr.get(int(import_id))
    imp.status = 0
    tasks.importer_process.delay(imp.id)

    ret = {
        'import': dict(imp)
    }
    return _api_response(request, ret)
Exemplo n.º 6
0
def importer_process(import_id):
    """Start the process of running the import.

    We load it, mark it as running, and begin begin a task to process.

    :param import_id: import id we need to pull and work on

    """
    trans = transaction.begin()
    imp = ImportQueueMgr.get(import_id)
    import_id = imp.id

    # Log that we've scheduled it
    logger.info("IMPORT: SCHEDULED for {0}.".format(imp.username))
    # We need to mark that it's running to prevent it getting picked up
    # again.
    imp.mark_running()
    trans.commit()
    importer_process_worker.delay(import_id)
Exemplo n.º 7
0
def importer_process_worker(import_id):
    """Do the real import work

    :param import_id: import id we need to pull and work on

    """
    trans = transaction.begin()
    import_job = ImportQueueMgr.get(import_id)
    logger.info("IMPORT: RUNNING for {username}".format(**dict(import_job)))

    try:
        # process the file using the import script
        import_file = open(import_job.file_path)
        importer = Importer(
            import_file,
            import_job.username)
        importer.process()

        # Processing kills off our transaction so we need to start a new one
        # to update that our import is complete.
        trans = transaction.begin()
        import_job = ImportQueueMgr.get(import_id)
        import_job.mark_done()
        user = UserMgr.get(username=import_job.username)
        from bookie.lib.message import UserImportSuccessMessage
        msg = UserImportSuccessMessage(
            user.email,
            'Bookie: Your requested import has completed.',
            INI)
        msg.send({
            'username': import_job.username,
        })

        logger.info(
            "IMPORT: COMPLETE for {username}".format(**dict(import_job)))
        trans.commit()

    except Exception, exc:
        # We need to log this and probably send an error email to the
        # admin
        from bookie.lib.message import ImportFailureMessage
        from bookie.lib.message import UserImportFailureMessage

        trans = transaction.begin()
        import_job = ImportQueueMgr.get(import_id)
        user = UserMgr.get(username=import_job.username)

        msg = ImportFailureMessage(
            INI.get('email.from'),
            'Import failure!',
            INI)
        msg.send({
            'username': import_job.username,
            'file_path': import_job.file_path,
            'exc': str(exc)
        })

        # Also send an email to the user that their import failed.
        msg = UserImportFailureMessage(
            user.email,
            'Bookie: We are sorry, your import failed.',
            INI)
        msg.send({
            'username': import_job.username,
            'exc': str(exc)
        })

        logger.error(exc)
        logger.error(str(exc))
        import_job.mark_error()
        logger.info(
            "IMPORT: ERROR for {username}".format(**dict(import_job)))
        logger.info(exc)
        trans.commit()
Exemplo n.º 8
0
def import_bmarks(request):
    """Allow users to upload a delicious bookmark export"""
    rdict = request.matchdict
    username = rdict.get('username')

    # if auth fails, it'll raise an HTTPForbidden exception
    with ReqAuthorize(request):
        data = {}
        post = request.POST

        # we can't let them submit multiple times, check if this user has an
        # import in process
        if ImportQueueMgr.get(username=username, status=NEW):
            # they have an import, get the information about it and shoot to
            # the template
            return {
                'existing': True,
                'import_stats': ImportQueueMgr.get_details(username=username)
            }

        if post:
            # we have some posted values
            files = post.get('import_file', None)

            if files is not None:
                # save the file off to the temp storage
                out_dir = "{storage_dir}/{randdir}".format(
                    storage_dir=request.registry.settings.get(
                        'import_files',
                        '/tmp/bookie').format(
                            here=request.registry.settings.get('app_root')),
                    randdir=random.choice(string.letters),
                )

                # make sure the directory exists
                # we create it with parents as well just in case
                if not os.path.isdir(out_dir):
                    os.makedirs(out_dir)

                out_fname = "{0}/{1}.{2}".format(
                    out_dir, username, files.filename)
                out = open(out_fname, 'w')
                out.write(files.file.read())
                out.close()

                # mark the system that there's a pending import that needs to
                # be completed
                q = ImportQueue(username, out_fname)
                DBSession.add(q)
                DBSession.flush()
                # Schedule a task to start this import job.
                tasks.importer_process.delay(q.id)

                return HTTPFound(location=request.route_url('user_import',
                                                            username=username))
            else:
                msg = request.session.pop_flash()
                if msg:
                    data['error'] = msg
                else:
                    data['error'] = None

            return data
        else:
            # we need to see if they've got
            # just display the form
            return {
                'existing': False
            }