Пример #1
0
def main(project_name, template, **kwargs):
    cur_dir = os.getcwd()
    project_dir = os.path.join(cur_dir, project_name)

    if template is None:
        grab_root = os.path.dirname(os.path.dirname(
            os.path.realpath(__file__)))
        template_path = os.path.join(grab_root, 'util/default_project')
    else:
        template_path = template

    if os.path.exists(project_dir):
        raise GrabError('Directory %s already exists' % project_dir)
    else:
        logger.debug('Copying %s to %s' % (template_path, project_dir))
        shutil.copytree(template_path, project_dir)

        project_name_camelcase = underscore_to_camelcase(project_name)
        context = {
            'PROJECT_NAME': project_name,
            'PROJECT_NAME_CAMELCASE': project_name_camelcase,
        }
        for base, dir_names, file_names in os.walk(project_dir):
            for file_name in file_names:
                if file_name.endswith(('._py', '.py')):
                    file_path = os.path.join(base, file_name)
                    changed, content = process_macros(
                        open(file_path).read(), context)
                    if changed:
                        with open(normalize_extension(file_path), 'w') as out:
                            out.write(content)
                        os.unlink(file_path)
Пример #2
0
    def request(self):
        try:
            if self.config['method'] == 'post':
                post_params = json.dumps(self.config['post'])
                script = """
                    var form = document.createElement("form");
                    form.setAttribute("method", "POST");
                    form.setAttribute("action", "%s");

                    var params = %s;

                    for(var key in params) {
                        if(params.hasOwnProperty(key)) {
                            var hiddenField = document.createElement("input");
                            hiddenField.setAttribute("type", "hidden");
                            hiddenField.setAttribute("name", key);
                            hiddenField.setAttribute("value", params[key]);

                            form.appendChild(hiddenField);
                         }
                    }

                    document.body.appendChild(form);
                    form.submit();
                """ % (self.config['url'], post_params)

                self.browser.execute_script(script)
            else:
                self.browser.get(self.config['url'])
                time.sleep(self.config['wait'])
        except Exception as ex:
            logging.error('', exc_info=ex)
            raise GrabError(999, 'Error =8-[ ]')
Пример #3
0
    def set_source(self, source_type='file', proxy_type='http', **kwargs):
        """
        Configure proxy list source and load proxies from that source.
        """

        if source_type in SOURCE_TYPE_ALIAS:
            source_cls = SOURCE_TYPE_ALIAS[source_type]
            self.source = source_cls(proxy_type=proxy_type, **kwargs)
            self.reload(force=True)
        else:
            raise GrabError('Unknown source type: %s' % source_type)
Пример #4
0
def parse_proxyline(line):
    """
    Extract proxy details from the text line.
    """

    match = RE_SIMPLE_PROXY.search(line)
    if match:
        host, port = match.groups()
        return host, port, None, None
    else:
        match = RE_AUTH_PROXY.search(line)
        if match:
            host, port, user, pwd = match.groups()
            return host, port, user, pwd
    raise GrabError('Invalid proxy line: %s' % line)
Пример #5
0
def parse_proxy_data(data, data_format='text', proxy_type='http'):
    """
    Yield `Proxy` objects found in the given `data`.
    """
    if data_format == 'text':
        for line in data.splitlines():
            if not PY3K and isinstance(line, unicode):
                line = line.encode('utf-8')
            line = line.strip().replace(' ', '')
            if line and not line.startswith('#'):
                try:
                    host, port, user, pwd = parse_proxy_line(line)
                except GrabError as ex:
                    logger.error('Invalid proxy line: %s' % line)
                else:
                    yield Proxy(host, port, user, pwd, proxy_type)
    else:
        raise GrabError('Unknown proxy data format: %s' % data_format)
Пример #6
0
def parse_proxy_line(line):
    """
    Extract proxy details from the text line.

    The text line could be in one of the following formats:
    * host:port
    * host:port:username:password
    """

    match = RE_SIMPLE_PROXY.search(line)
    if match:
        host, port = match.groups()
        return host, port, None, None
    else:
        match = RE_AUTH_PROXY.search(line)
        if match:
            host, port, user, pwd = match.groups()
            return host, port, user, pwd
    raise GrabError('Invalid proxy line: %s' % line)
Пример #7
0
def main(project_name, template, **kwargs):  # pylint: disable=unused-argument
    cur_dir = os.getcwd()
    project_dir = os.path.join(cur_dir, project_name)

    if template is None:
        grab_root = os.path.dirname(os.path.dirname(
            os.path.realpath(__file__)))
        template_path = os.path.join(grab_root, 'util/default_project')
    else:
        template_path = template

    if os.path.exists(project_dir):
        raise GrabError('Directory %s already exists' % project_dir)
    else:
        logger.debug('Copying %s to %s', template_path, project_dir)
        shutil.copytree(template_path, project_dir)

        project_name_camelcase = underscore_to_camelcase(project_name)
        context = {
            'PROJECT_NAME': project_name,
            'PROJECT_NAME_CAMELCASE': project_name_camelcase,
        }
        for base, _, file_names in os.walk(project_dir):
            for file_name in file_names:
                if file_name.endswith('.py'):
                    file_path = os.path.join(base, file_name)
                    content = process_content(open(file_path).read(), context)
                    new_file_path = process_file_path(file_path, context)
                    with open(new_file_path, 'w') as out:
                        out.write(content)

                    if file_path != new_file_path:
                        os.unlink(file_path)
                        print('%s: OK' % new_file_path)
                    else:
                        print('%s: OK' % file_path)