def main(project_name, template, **kwargs):
cur_dir = os.getcwd()
project_dir = os.path.join(cur_dir, project_name)
if template is None:
grab_root = os.path.dirname(os.path.dirname(
os.path.realpath(__file__)))
template_path = os.path.join(grab_root, 'util/default_project')
else:
template_path = template
if os.path.exists(project_dir):
raise GrabError('Directory %s already exists' % project_dir)
else:
logger.debug('Copying %s to %s' % (template_path, project_dir))
shutil.copytree(template_path, project_dir)
project_name_camelcase = underscore_to_camelcase(project_name)
context = {
'PROJECT_NAME': project_name,
'PROJECT_NAME_CAMELCASE': project_name_camelcase,
}
for base, dir_names, file_names in os.walk(project_dir):
for file_name in file_names:
if file_name.endswith(('._py', '.py')):
file_path = os.path.join(base, file_name)
changed, content = process_macros(
open(file_path).read(), context)
if changed:
with open(normalize_extension(file_path), 'w') as out:
out.write(content)
os.unlink(file_path)
def request(self):
try:
if self.config['method'] == 'post':
post_params = json.dumps(self.config['post'])
script = """
var form = document.createElement("form");
form.setAttribute("method", "POST");
form.setAttribute("action", "%s");
var params = %s;
for(var key in params) {
if(params.hasOwnProperty(key)) {
var hiddenField = document.createElement("input");
hiddenField.setAttribute("type", "hidden");
hiddenField.setAttribute("name", key);
hiddenField.setAttribute("value", params[key]);
form.appendChild(hiddenField);
}
}
document.body.appendChild(form);
form.submit();
""" % (self.config['url'], post_params)
self.browser.execute_script(script)
else:
self.browser.get(self.config['url'])
time.sleep(self.config['wait'])
except Exception as ex:
logging.error('', exc_info=ex)
raise GrabError(999, 'Error =8-[ ]')
def set_source(self, source_type='file', proxy_type='http', **kwargs):
"""
Configure proxy list source and load proxies from that source.
"""
if source_type in SOURCE_TYPE_ALIAS:
source_cls = SOURCE_TYPE_ALIAS[source_type]
self.source = source_cls(proxy_type=proxy_type, **kwargs)
self.reload(force=True)
else:
raise GrabError('Unknown source type: %s' % source_type)
def parse_proxyline(line):
"""
Extract proxy details from the text line.
"""
match = RE_SIMPLE_PROXY.search(line)
if match:
host, port = match.groups()
return host, port, None, None
else:
match = RE_AUTH_PROXY.search(line)
if match:
host, port, user, pwd = match.groups()
return host, port, user, pwd
raise GrabError('Invalid proxy line: %s' % line)
def parse_proxy_data(data, data_format='text', proxy_type='http'):
"""
Yield `Proxy` objects found in the given `data`.
"""
if data_format == 'text':
for line in data.splitlines():
if not PY3K and isinstance(line, unicode):
line = line.encode('utf-8')
line = line.strip().replace(' ', '')
if line and not line.startswith('#'):
try:
host, port, user, pwd = parse_proxy_line(line)
except GrabError as ex:
logger.error('Invalid proxy line: %s' % line)
else:
yield Proxy(host, port, user, pwd, proxy_type)
else:
raise GrabError('Unknown proxy data format: %s' % data_format)
def parse_proxy_line(line):
"""
Extract proxy details from the text line.
The text line could be in one of the following formats:
* host:port
* host:port:username:password
"""
match = RE_SIMPLE_PROXY.search(line)
if match:
host, port = match.groups()
return host, port, None, None
else:
match = RE_AUTH_PROXY.search(line)
if match:
host, port, user, pwd = match.groups()
return host, port, user, pwd
raise GrabError('Invalid proxy line: %s' % line)
def main(project_name, template, **kwargs): # pylint: disable=unused-argument
cur_dir = os.getcwd()
project_dir = os.path.join(cur_dir, project_name)
if template is None:
grab_root = os.path.dirname(os.path.dirname(
os.path.realpath(__file__)))
template_path = os.path.join(grab_root, 'util/default_project')
else:
template_path = template
if os.path.exists(project_dir):
raise GrabError('Directory %s already exists' % project_dir)
else:
logger.debug('Copying %s to %s', template_path, project_dir)
shutil.copytree(template_path, project_dir)
project_name_camelcase = underscore_to_camelcase(project_name)
context = {
'PROJECT_NAME': project_name,
'PROJECT_NAME_CAMELCASE': project_name_camelcase,
}
for base, _, file_names in os.walk(project_dir):
for file_name in file_names:
if file_name.endswith('.py'):
file_path = os.path.join(base, file_name)
content = process_content(open(file_path).read(), context)
new_file_path = process_file_path(file_path, context)
with open(new_file_path, 'w') as out:
out.write(content)
if file_path != new_file_path:
os.unlink(file_path)
print('%s: OK' % new_file_path)
else:
print('%s: OK' % file_path)