Ejemplo n.º 1
0
Archivo: base.py Proyecto: 4iji/crawley
    def syncdb(self, syncb_command):
        """
            Checks for storages configuration in the settings.py file
        """

        self.connector = None
        syncb_command.sessions = []

        documents_sessions = { 'JSON_DOCUMENT' : json_session,
                               'XML_DOCUMENT' : xml_session,
                               'CSV_DOCUMENT' : csv_session,
                               'MONGO_DB_HOST' : mongo_session,
                               'COUCH_DB_HOST' : couch_session,
                             }

        for storage_name, session in documents_sessions.iteritems():

            if has_valid_attr(syncb_command.settings, storage_name):

                session.set_up(syncb_command.settings, storage_name)
                syncb_command.sessions.append(session)

        if has_valid_attr(syncb_command.settings, "DATABASE_ENGINE"):

            import_user_module("models", exit=False)
            syncb_command.sessions.append(database_session)
            self.connector = connectors[syncb_command.settings.DATABASE_ENGINE](syncb_command.settings)
Ejemplo n.º 2
0
    def _install_opener(self):

        if has_valid_attr(self.settings,'PROXY_HOST') and has_valid_attr(self.settings,'PROXY_PORT'):

            proxy_info = {        #proxy information
                'user' : getattr(self.settings, 'PROXY_USER', ''),
                'pass' : getattr(self.settings, 'PROXY_PASS', ''),
                'host' : getattr(self.settings, 'PROXY_HOST', ''), #localhost
                'port' : getattr(self.settings, 'PROXY_PORT', 80)
            }

            # build a new opener that uses a proxy requiring authorization
            proxy = urllib2.ProxyHandler({"http" :"http://%(user)s:%(pass)s@%(host)s:%(port)d" % proxy_info})
            self.opener = urllib2.build_opener(proxy, self.cookie_handler)

        else:
            self.opener = urllib2.build_opener(self.cookie_handler)
Ejemplo n.º 3
0
    def _check_project_type(self):
        """
            Check for the project's type
        """

        if has_valid_attr(self.settings, "PROJECT_TYPE"):
            project_type = self.settings.PROJECT_TYPE
        else:
            meta_data = import_user_module("__init__")
            project_type = meta_data.project_type

        self.project_type = project_types[project_type]()
Ejemplo n.º 4
0
    def _install_opener(self):

        if has_valid_attr(self.settings, 'PROXY_HOST') and has_valid_attr(
                self.settings, 'PROXY_PORT'):

            proxy_info = {  #proxy information
                'user': getattr(self.settings, 'PROXY_USER', ''),
                'pass': getattr(self.settings, 'PROXY_PASS', ''),
                'host': getattr(self.settings, 'PROXY_HOST', ''),  #localhost
                'port': getattr(self.settings, 'PROXY_PORT', 80)
            }

            # build a new opener that uses a proxy requiring authorization
            proxy = urllib2.ProxyHandler({
                "http":
                "http://%(user)s:%(pass)s@%(host)s:%(port)d" % proxy_info
            })
            self.opener = urllib2.build_opener(proxy, self.cookie_handler)

        else:
            self.opener = urllib2.build_opener(self.cookie_handler)
Ejemplo n.º 5
0
    def syncdb(self, syncb_command):
        """
            Checks for storages configuration in the settings.py file
        """

        self.connector = None
        syncb_command.sessions = []

        documents_sessions = { 'JSON_DOCUMENT' : json_session,
                               'XML_DOCUMENT' : xml_session,
                               'CSV_DOCUMENT' : csv_session, }

        for document_name, session in documents_sessions.iteritems():

            if has_valid_attr(syncb_command.settings, document_name):

                session.file_name = getattr(syncb_command.settings, document_name)
                syncb_command.sessions.append(session)

        if has_valid_attr(syncb_command.settings, "DATABASE_ENGINE"):

            syncb_command.sessions.append(database_session)
            self.connector = connectors[syncb_command.settings.DATABASE_ENGINE](syncb_command.settings)