def login(username, password):
"""
输入自己的账号密码，模拟登录知乎
"""
# 检测到11位数字则是手机登录
if re.match(r'\d{11}$', account):
url = 'http://www.zhihu.com/login/phone_num'
data = {'_xsrf': get_xsrf(),
'password': password,
'remember_me': 'true',
'phone_num': username
}
else:
url = 'https://www.zhihu.com/login/email'
data = {'_xsrf': get_xsrf(),
'password': password,
'remember_me': 'true',
'email': username
}
# 若不用验证码，直接登录
post_data = urlparse.urlencode(data).encode('utf-8')
r = opener.open(url, post_data)
result = r.read().decode('utf-8')
# 打印返回的响应，r = 1代表响应失败，msg里是失败的原因
# 要用验证码，post后登录
if (json.loads(result))["r"] == 1:
data['captcha'] = get_captcha()
post_data = urlparse.urlencode(data).encode('utf-8')
r = opener.open(url, post_data)
result = r.read().decode('utf-8')
print((json.loads(result))['msg'])
# 保存cookie到本地
cookie.save(ignore_discard=True, ignore_expires=True)
def render(self, context):
try:
start = self.start.resolve(context)
current_url = self.current_url.resolve(context)
page_arg_names = self.page_arg_names.resolve(context)
except Exception:
raise template.TemplateSyntaxError(
"Invalid variables passed to start_url tag")
names = page_arg_names.split(",")
params = {names[0]: start}
url_parts = list(url_parse.urlparse(current_url))
query = dict(parse_qsl(url_parts[4]))
for name in names[1:]:
if name not in query:
query.update({name: 1})
query.update(params)
try:
url_parts[4] = urllib.urlencode(query)
except AttributeError:
url_parts[4] = url_parse.urlencode(query)
return url_parse.urlunparse(url_parts)
def to_url(self):
"""Serialize as a URL for a GET request."""
base_url = urlparse.urlparse(self.url)
try:
query = base_url.query
except AttributeError:
# must be python <2.5
query = base_url[4]
query = parse_qs(query)
for k, v in self.items():
query.setdefault(k, []).append(v)
try:
scheme = base_url.scheme
netloc = base_url.netloc
path = base_url.path
params = base_url.params
fragment = base_url.fragment
except AttributeError:
# must be python <2.5
scheme = base_url[0]
netloc = base_url[1]
path = base_url[2]
params = base_url[3]
fragment = base_url[5]
try:
url = (scheme, netloc, path, params,
urllib.urlencode(query, True), fragment)
return urllib.urlunparse(url)
except AttributeError:
url = (scheme, netloc, path, params,
urlparse.urlencode(query, True), fragment)
return urlparse.urlunparse(url)
def __str__(self):
data = {'oauth_consumer_key': self.key,
'oauth_consumer_secret': self.secret}
try:
return urllib.urlencode(data)
except AttributeError:
return urlparse.urlencode(data)
def post(url, data=None, json=None, headers=None, ctx=None):
"""
.. function:: post(url,data=None,json=None,headers=None,ctx=None)
Implements the POST method of the HTTP protocol. A tcp connection is made to the host:port given in the url using the default net driver.
If *headers* is given as a dictionary, each pair (key, value) is appropriately sent as a HTTP request header. Mandatory headers are transparently handled: "Host:" is always derived by parsing *url*;
other headers are set to defaults if not given: for example "Connection: close" is sent if no value for "Connection" is specified in *headers*. To request a permanent connection,
*headers* must contain the pair {"Connection":"Keep-Alive"}.
If *data* is provided (always as dictionary), each pair (key, value) will be form-encoded and send in the body of the request with {"content-type":"application/x-www-form-urlencoded"} appended in the headers.
If *json* is provided (always as dictionary), json data will send in the body of the request with {"content-type":"application/json"} appended in the headers.
.. note:: if both (*data* and *json*) dict are provided, json data are ignored and post request is performed with urlencoded data.
*post* returns a :class:`Response` instance.
Exceptions can be raised: :exc:`HTTPConnectionError` when the HTTP server can't be contacted; :exc:`IOError` when the source of error lies at the socket level (i.e. closed sockets, invalid sockets, etc..)
"""
pdata = None
if data is not None:
pdata = [urlparse.urlencode(data), "application/x-www-form-urlencoded"]
elif json is not None:
pdata = [json_encoder.dumps(json), "application/json"]
return _verb(url, pdata, None, headers, None, "POST", ctx)
def _request(self, **kwargs):
# Check for missing variables
resource = self.interface
def validate_kwarg(key, target):
# Always allow list
if isinstance(key, list):
return all([validate_kwarg(x, target[0]) for x in key])
return isinstance(key, target)
for k in resource.get('required', []):
if k not in [x.split(':')[0] for x in kwargs.keys()]:
raise ValueError('Missing required argument: %s' % k)
if isinstance(kwargs.get(k), list) and not isinstance(resource['required'][k], list):
raise ValueError('Wrong argument type: %s is not a list' % k)
elif not validate_kwarg(kwargs.get(k), resource['required'][k]):
if isinstance(resource['required'][k], list):
raise ValueError('Wrong arguemnt type: %s is not a list of %ss' % (k, resource['required'][k][0]))
raise ValueError('Wrong arguemnt type: %s is not a %s' % (k, resource['required'][k]))
conduit = self.api.conduit
if conduit:
# Already authenticated, add session key to json data
kwargs['__conduit__'] = conduit
elif self.method == 'conduit' and self.endpoint == 'connect':
# Not authenticated, requesting new session key
token = str(int(time.time()))
kwargs['authToken'] = token
kwargs['authSignature'] = self.api.generate_hash(token)
else:
# Authorization is required, silently auth the user
self.api.connect()
kwargs['__conduit__'] = self.api.conduit
url = urlparse.urlparse(self.api.host)
if url.scheme == 'https':
conn = httplib.HTTPSConnection(url.netloc, timeout=self.api.timeout)
else:
conn = httplib.HTTPConnection(url.netloc, timeout=self.api.timeout)
path = url.path + '%s.%s' % (self.method, self.endpoint)
headers = {
'User-Agent': 'python-phabricator/%s' % str(self.api.clientVersion),
'Content-Type': 'application/x-www-form-urlencoded'
}
body = urlparse.urlencode({
"params": json.dumps(kwargs),
"output": self.api.response_format
})
# TODO: Use HTTP "method" from interfaces.json
conn.request('POST', path, body, headers)
response = conn.getresponse()
data = self._parse_response(response.read())
return Result(data['result'])
def _post_ST2(server, payload):
data = urlencode(payload)
try:
content = urlopen(server, data).read()
except IOError:
return None
else:
return content
def get_authorization_url(self, nexturl='/'):
params = {
'client_id': self.client_id,
'response_type': 'code',
'redirect_uri': self.callback_url + '&next_url=' + nexturl
}
url = self.AUTH_URL + "?" + urlparse.urlencode(params)
return url
def authorize(self, url, state):
base_url = "https://www.wunderlist.com/oauth/authorize?"
params = {
'client_id': self.client_ID,
'redirect_uri': url,
'state': state
}
return base_url + urlparse.urlencode(params)
def get(url, params):
full_url = url
if params != None and len(params) > 0:
data = urlparse.urlencode(params)
full_url = full_url + "?" + data
f = urllib2.urlopen(full_url)
content = f.read()
return content
def get_authorization_url(self, nexturl='/'):
params = {
'client_id': self.client_id,
'response_type': 'code',
'redirect_uri': self.callback_url,
'scope': 'openid email',
}
# url = self.AUTH_URL + "?" + urllib.parse.urlencode(params, quote_via=urllib.parse.quote)
url = self.AUTH_URL + "?" + urlparse.urlencode(params)
return url
def replace_query_param(url, key, val):
"""
Given a URL and a key/val pair, set or replace an item in the query
parameters of the URL, and return the new URL.
"""
(scheme, netloc, path, query, fragment) = urlparse.urlsplit(url)
query_dict = urlparse.parse_qs(query)
query_dict[key] = [val]
query = urlparse.urlencode(sorted(list(query_dict.items())), doseq=True)
return urlparse.urlunsplit((scheme, netloc, path, query, fragment))
def get(url, params):
full_url = url
if params != None and len(params) > 0:
data = urlparse.urlencode(params)
full_url = full_url + "?" + data
with urllib2.urlopen(full_url) as f:
content = f.read()
return content
def to_postdata(self):
"""Serialize as post data for a POST request."""
d = {}
for k, v in self.items():
d[k.encode('utf-8')] = to_utf8_optional_iterator(v)
# tell urlencode to deal with sequence values and map them correctly
# to resulting querystring. for example self["k"] = ["v1", "v2"] will
# result in 'k=v1&k=v2' and not k=%5B%27v1%27%2C+%27v2%27%5D
try:
return urllib.urlencode(d, True).replace('+', '%20')
except AttributeError:
return urlparse.urlencode(d, True).replace('+', '%20')
def get_pdata(data, json):
pdata = None
if data is not None:
if type(data) == PDICT:
pdata = [
urlparse.urlencode(data), "application/x-www-form-urlencoded"
]
else:
pdata = [data, ""]
elif json is not None:
pdata = [json_encoder.dumps(json), "application/json"]
return pdata
def SynchronizedRequest(connection_policy,
method,
base_url,
path,
request_data,
query_params,
headers):
"""Performs one synchronized http request according to the parameters.
:Parameters:
- `connection_policy`: documents.ConnectionPolicy
- `method`: str
- `base_url`: str
- `path`: str
- `request_data`: str, unicode, file-like stream object, dict, list
or None
- `query_params`: dict
- `headers`: dict
:Returns:
tuple of (result, headers), where both result and headers
are dicts.
"""
request_body = None
if request_data:
request_body = _RequestBodyFromData(request_data)
if not request_body:
raise errors.UnexpectedDataType(
'parameter data must be a JSON object, string or' +
' readable stream.')
request_options = {}
parse_result = urlparse.urlparse(base_url)
request_options['host'] = parse_result.hostname
request_options['port'] = parse_result.port
request_options['path'] = path
request_options['method'] = method
if query_params:
request_options['path'] += '?' + urlparse.urlencode(query_params)
request_options['headers'] = headers
if (request_body and
(type(request_body) in [str, bytes] or
(unicode is not None and type(request_body) is unicode))):
request_options['headers'][http_constants.HttpHeaders.ContentLength] = (
len(request_body))
elif request_body is None:
request_options['headers'][http_constants.HttpHeaders.ContentLength] = 0
return _InternalRequest(connection_policy, request_options, request_body)
def post(url, params):
if params != None and len(params) > 0:
data = urlparse.urlencode(params)
else:
data = ""
data = data.encode('utf-8')
request = urllib2.Request(url)
request.add_header("Content-Type", "application/x-www-form-urlencoded;charset=utf-8")
with urllib2.urlopen(request, data) as f:
content = f.read()
return content
def post(url, params):
if params != None and len(params) > 0:
data = urlparse.urlencode(params)
else:
data = ""
data = data.encode('utf-8')
request = urllib2.Request(url)
request.add_header("Content-Type", "application/x-www-form-urlencoded;charset=utf-8")
with urllib2.urlopen(request, data) as f:
content = f.read()
return content
def get_normalized_parameters(self):
"""Return a string that contains the parameters that must be signed."""
items = []
for key, value in self.items():
if key == 'oauth_signature':
continue
# 1.0a/9.1.1 states that kvp must be sorted by key, then by value,
# so we unpack sequence values into multiple items for sorting.
try:
if isinstance(value, basestring):
items.append((to_utf8_if_string(key), to_utf8(value)))
else:
try:
value = list(value)
except TypeError as e:
assert 'is not iterable' in str(e)
items.append((to_utf8_if_string(key), to_utf8_if_string(value)))
else:
items.extend((to_utf8_if_string(key), to_utf8_if_string(item)) for item in value)
except NameError:
if isinstance(value, str):
items.append((to_utf8_if_string(key), to_utf8(value)))
else:
try:
value = list(value)
except TypeError as e:
assert 'is not iterable' in str(e)
items.append((to_utf8_if_string(key), to_utf8_if_string(value)))
else:
items.extend((to_utf8_if_string(key), to_utf8_if_string(item)) for item in value)
# Include any query string parameters from the provided URL
query = urlparse.urlparse(self.url)[4]
url_items = self._split_url_string(query).items()
url_items = [(to_utf8(k), to_utf8(v)) for k, v in url_items if k != 'oauth_signature' ]
items.extend(url_items)
items.sort()
try:
encoded_str = urllib.urlencode(items)
except AttributeError:
encoded_str = urlparse.urlencode(items)
# Encode signature parameters per Oauth Core 1.0 protocol
# spec draft 7, section 3.6
# (http://tools.ietf.org/html/draft-hammer-oauth-07#section-3.6)
# Spaces must be encoded with "%20" instead of "+"
return encoded_str.replace('+', '%20').replace('%7E', '~')
def wrapper(self, *args, **kwargs):
if not self.get_current_user():
url = self.get_login_url()
if "?" not in url:
if urlparse.urlsplit(url).scheme:
# if login url is absolute, make next absolute too
next_url = self.request.full_url()
else:
next_url = self.request.uri
url += "?" + urlparse.urlencode(dict(next=next_url))
if settings.REDIRECT_LOGIN:
self.redirect(url)
return
else:
raise HTTPError(403)
return method(self, *args, **kwargs)
def generate(uri, key, ttl, policy_name=None):
"""
.. function:: generate(uri, key, ttl, policy_name=None)
Generate a SAS for target :samp:`uri` signed with :samp:`key` valid till :samp:`ttl` (passed as epoch) and with optional :samp:`policy_name`.
"""
sign_key = urlparse.quote_plus(uri) + '\n' + str(int(ttl))
hh = hmac.HMAC(base64.standard_b64decode(key),
sha2.SHA2(hashtype=sha2.SHA256))
hh.update(sign_key)
signature = base64.standard_b64encode(hh.digest())
rawtoken = {'sr': uri, 'sig': signature, 'se': str(int(ttl))}
if policy_name is not None:
rawtoken['skn'] = policy_name
return 'SharedAccessSignature ' + urlparse.urlencode(rawtoken)
def to_string(self):
"""Returns this token as a plain string, suitable for storage.
The resulting string includes the token's secret, so you should never
send or store this string where a third party can read it.
"""
data = {
'oauth_token': self.key,
'oauth_token_secret': self.secret,
}
if self.callback_confirmed is not None:
data['oauth_callback_confirmed'] = self.callback_confirmed
try:
return urllib.urlencode(data)
except AttributeError:
return urlparse.urlencode(data)
def testProtourlencodeUnrecognizedField(self):
"""Test that unrecognized fields are saved and can be accessed."""
class MyMessage(messages.Message):
number = messages.IntegerField()
decoded = to_url.decode_message(MyMessage,
self.unexpected_tag_message)
self.assertEqual(1, len(decoded.all_unrecognized_fields()))
self.assertEqual('unexpected', decoded.all_unrecognized_fields()[0])
# Unknown values set to a list of however many values had that name.
self.assertEqual((['whatever'], messages.Variant.STRING),
decoded.get_unrecognized_field_info('unexpected'))
repeated_unknown = urlencode([('repeated', 400),
('repeated', 'test'),
('repeated', '123.456')])
decoded2 = to_url.decode_message(MyMessage, repeated_unknown)
self.assertEqual((['400', 'test', '123.456'], messages.Variant.STRING),
decoded2.get_unrecognized_field_info('repeated'))
def request(self,
resource,
method="get",
args=None,
body=None,
parts=None,
headers={}):
params = None
path = resource
headers['User-Agent'] = 'Basic Agent'
BOUNDARY = r'00hoYUXOnLD5RQ8SKGYVgLLt64jejnMwtO7q8XE1'
CRLF = r'\r\n'
if parts:
# Attempt to find the Mimetype
headers[
'Content-Type'] = 'multipart/form-data; boundary=' + BOUNDARY
encode_string = StringIO()
for part in parts:
if part.type != 'file':
continue
encode_string.write(r'--' + BOUNDARY)
encode_string.write(CRLF)
body = None
if part.type == 'file':
encode_string.write(
r'Content-Disposition: form-data; name="%s"; filename="%s"'
% (part.key, os.path.basename(part.text)))
encode_string.write(CRLF)
content_type = self.get_content_type(part.text)
encode_string.write(r'Content-Type: %s' % content_type)
encode_string.write(CRLF)
encode_string.write(r'Content-Transfer-Encoding: base64')
encode_string.write(CRLF)
encode_string.write(CRLF)
with open(part.text, "rb") as file:
body = base64.b64encode(file.read()).decode()
#body=str(file.read())
else:
encode_string.write(
r'Content-Disposition: form-data; name="%s"' %
(part.key))
encode_string.write(CRLF)
encode_string.write(r'Content-Type: %s' % part.contentType)
encode_string.write(CRLF)
encode_string.write(CRLF)
body = part.text
encode_string.write(body)
encode_string.write(CRLF)
encode_string.write(r'--' + BOUNDARY + r'--' + CRLF)
body = encode_string.getvalue()
headers['Content-Length'] = str(len(body))
elif body:
if not headers.get('Content-Type', None):
headers['Content-Type'] = 'text/xml'
headers['Content-Length'] = str(len(body))
else:
if not headers.get('Content-Type', None):
headers['Content-Type'] = 'text/xml'
domain = self.base_url.split("//")[1].split("/")[0]
if args:
if utils.getPythonVersion() == '2':
path += r"?" + urllib.urlencode(args)
else:
path += r"?" + urlparse.urlencode(args)
request_path = []
if self.path != "/":
if self.path.endswith('/'):
request_path.append(self.path[:-1])
else:
request_path.append(self.path)
if path.startswith('/'):
request_path.append(path[1:])
else:
request_path.append(path)
# if self.host=='211.136.86.203':# kuan chang com IP
# url = r"%s://%s%s" % (self.scheme, self.host, ''.join(request_path))
# else:# normal flow
url = r"%s://%s%s" % (self.scheme, self.host, ''.join(request_path))
url = url.rstrip('/')
logger.info('request: %s' % url)
resp, content = self.h.request(url,
method.upper(),
body=body,
headers=headers)
encoding = 'UTF-8'
if 'content-type' in resp and 'charset' in resp['content-type']:
items = resp['content-type'].split('=')
encoding = items[1]
if 'content-type' in resp and 'application' in resp[
'content-type'] and 'json' not in resp['content-type']:
encoding = None
if 'content-type' in resp and 'image' in resp['content-type']:
encoding = None
if encoding is not None:
return {r'headers': resp, r'body': content.decode(encoding)}
else:
# not return binary data
return {r'headers': resp, r'body': ''}
def _verb(url,
data=None,
params=None,
headers=None,
connection=None,
verb=None,
ctx=None,
stream_callback=None,
stream_chunk=512):
urlp = urlparse.parse(url)
netl = urlparse.parse_netloc(urlp[1])
host = netl[2]
# print(verb,urlp,netl)
if connection:
sock = connection
else:
sock = _connect(host, netl[3], urlp[0], ctx)
# print("CREATED SOCKET",sock.channel)
#Generate and send Request Line
msg = bytearray(BUFFER_LEN)
p = 0
endline = "\r\n"
p = add_to_buffer(msg, verb, p)
p = add_to_buffer(msg, " ", p)
if not urlp[2]:
p = add_to_buffer(msg, "/", p)
else:
p = add_to_buffer(msg, urlp[2], p)
if (verb in zverbs) and (urlp[-2] or params):
p = add_to_buffer(msg, "?", p)
if urlp[-2]:
p = add_to_buffer(msg, urlp[-2], p)
if params:
p = add_to_buffer(msg, "&", p)
if params:
p = add_to_buffer(msg, urlparse.urlencode(params), p)
if msg[p - 1] != __ORD(" "):
p = add_to_buffer(msg, " ", p)
p = add_to_buffer(msg, "HTTP/1.1", p)
p = add_to_buffer(msg, endline, p)
__elements_set(msg, p) #clamp buffer
#send cmd
try:
# print(">>",msg)
sock.sendall(msg)
except Exception as e:
sock.close()
raise HTTPConnectionError
__elements_set(msg, BUFFER_LEN)
p = 0
#Generated and send Request headers
p = add_to_buffer(msg, "Host: ", p)
p = add_to_buffer(msg, host, p)
if netl[3]:
p = add_to_buffer(msg, ":", p)
p = add_to_buffer(msg, netl[3], p)
p = add_to_buffer(msg, endline, p)
__elements_set(msg, p)
# print(">>",msg)
sock.sendall(msg)
__elements_set(msg, BUFFER_LEN)
p = 0
rh = {}
if headers:
for k in headers:
rh[k.lower()] = headers[k]
if "connection" not in rh:
rh["connection"] = "close"
if data is not None:
rh["content-length"] = str(len(data[0])) #data[0] is actual data
rh["content-type"] = data[1] #data[1] is data type header
for k, v in rh.items():
p = add_to_buffer(msg, k, p)
p = add_to_buffer(msg, ": ", p)
p = add_to_buffer(msg, v, p)
p = add_to_buffer(msg, endline, p)
__elements_set(msg, p)
# print(">>",msg)
sock.sendall(msg)
__elements_set(msg, BUFFER_LEN)
p = 0
# Generate and send Body
p = add_to_buffer(msg, endline, p)
if data is not None:
p = add_to_buffer(msg, data[0], p)
__elements_set(msg, p)
# print(">>",msg)
sock.sendall(msg)
__elements_set(msg, BUFFER_LEN)
p = 0
#Parse Response
rr = Response()
ssock = streams.SocketStream(sock)
buffer = msg
msg = _readline(ssock, buffer, 0, BUFFER_LEN)
# print("<<",msg)
if msg.startswith("HTTP/1.1"):
rr.status = int(msg[9:12])
__elements_set(msg, BUFFER_LEN)
msg = _readline(ssock, buffer, 0, BUFFER_LEN)
# print("<<",msg)
#print(msg)
#print(">[",msg,"]",msg=="\n",msg==endline)
while not (msg == endline or msg == "\n"):
idx_cl = msg.find(__ORD(":"))
if idx_cl < 0:
sock.close()
raise HTTPResponseError
rr.headers[str(msg[0:idx_cl].lower())] = str(msg[idx_cl +
1:-2].strip(endline))
__elements_set(msg, BUFFER_LEN)
msg = _readline(ssock, buffer, 0, BUFFER_LEN)
# print("<<",msg)
# print(msg)
#print(">[",msg,"]",msg=="\n",msg==endline)
#print(rr.headers)
rr.connection = sock
if "content-length" in rr.headers:
bodysize = int(rr.headers["content-length"])
#print("bodysize",bodysize)
contentsize = bodysize
reset_content = False
if stream_callback is not None:
reset_content = True
contentsize = stream_chunk
rr.content = bytearray(contentsize)
tmp = 0
rdr = 0
while bodysize > 0:
if reset_content:
ofs = 0
else:
ofs = tmp
rdr = sock.recv_into(rr.content,
min(bodysize, contentsize),
ofs=ofs)
if rdr and stream_callback:
__elements_set(rr.content, rdr)
stream_callback(rr.content)
#print(rdr,rr.content[tmp:tmp+rdr])
tmp += rdr
bodysize -= rdr
if not rdr:
break
#print("CLOSED SOCKET A",sock.channel,rdr,tmp,bodysize)
sock.close()
rr.connection = None
elif "transfer-encoding" in rr.headers:
while True:
__elements_set(msg, BUFFER_LEN)
msg = _readline(ssock, buffer, 0, BUFFER_LEN)
chsize = int(msg, 16)
#print("chsize",chsize)
if chsize:
msg = sock.recv(chsize)
#print(msg)
if msg:
rr.content.extend(msg)
else:
break
__elements_set(buffer, BUFFER_LEN)
msg = _readline(ssock, buffer, 0, BUFFER_LEN)
else:
__elements_set(buffer, BUFFER_LEN)
msg = _readline(ssock, buffer, 0, BUFFER_LEN) #terminator
break
#print("CLOSED SOCKET B",sock.channel)
sock.close()
else:
while True:
tmp = sock.recv(32)
if tmp:
rr.content.extend(tmp)
#print(tmp)
else:
break
#print("CLOSED SOCKET C",sock.channel)
sock.close()
rr.connection = None
return rr
class ProtourlencodeConformanceTest(test_util.TestCase,
test_util.PytractsConformanceTestBase):
PROTOLIB = to_url
encoded_partial = urlencode(sorted([('double_value', 1.23),
('int64_value', -100000000000),
('int32_value', 1020),
('string_value', u'a string'),
('enum_value', 'VAL2'),
], key=lambda kv: kv[0]))
encoded_full = urlencode(sorted([('double_value', 1.23),
('float_value', -2.5),
('int64_value', -100000000000),
('uint64_value', 102020202020),
('int32_value', 1020),
('bool_value', 'true'),
('string_value', u'a string\u044f'.encode('utf-8')),
('bytes_value', 'YSBieXRlc//+'),
('enum_value', 'VAL2'),
], key=lambda kv: kv[0]))
encoded_repeated = urlencode(sorted([('double_value', 1.23),
('double_value', 2.3),
('float_value', -2.5),
('float_value', 0.5),
('int64_value', -100000000000),
('int64_value', 20),
('uint64_value', 102020202020),
('uint64_value', 10),
('int32_value', 1020),
('int32_value', 718),
('bool_value', 'true'),
('bool_value', 'false'),
('string_value', u'a string\u044f'.encode('utf-8')),
('string_value', u'another string'.encode('utf-8')),
('bytes_value', 'YSBieXRlc//+'),
('bytes_value', 'YW5vdGhlciBieXRlcw=='),
('enum_value', 'VAL2'),
('enum_value', 'VAL1'),
], key=lambda kv: kv[0]))
encoded_nested = urlencode(sorted([('nested.a_value', 'a string')], key=lambda kv: kv[0]))
encoded_repeated_nested = urlencode(sorted([('repeated_nested-0.a_value', 'a string'),
('repeated_nested-1.a_value', 'another string'),
], key=lambda kv: kv[0]))
unexpected_tag_message = 'unexpected=whatever'
encoded_default_assigned = urlencode(sorted([('a_value', 'a default'),], key=lambda kv: kv[0]))
encoded_nested_empty = urlencode(sorted([('nested', '')], key=lambda kv: kv[0]))
encoded_repeated_nested_empty = urlencode(sorted([('repeated_nested-0', ''),
('repeated_nested-1', '')], key=lambda kv: kv[0]))
encoded_extend_message = urlencode(sorted([('int64_value-0', 400),
('int64_value-1', 50),
('int64_value-2', 6000)], key=lambda kv: kv[0]))
encoded_string_types = urlencode(
sorted([('string_value', 'Latin')], key=lambda kv: kv[0]))
encoded_invalid_enum = urlencode([('enum_value', 'undefined')])
def testParameterPrefix(self):
"""Test using the 'prefix' parameter to encode_message."""
class MyMessage(messages.Message):
number = messages.IntegerField()
names = messages.StringField(repeated=True)
message = MyMessage()
message.number = 10
message.names = [u'Fred', u'Lisa']
encoded_message = to_url.encode_message(message, prefix='prefix-')
self.assertEqual({'prefix-number': ['10'],
'prefix-names': ['Fred', 'Lisa']},
parse_qs(encoded_message))
self.assertEqual(message, to_url.decode_message(MyMessage,
encoded_message,
prefix='prefix-'))
def testProtourlencodeUnrecognizedField(self):
"""Test that unrecognized fields are saved and can be accessed."""
class MyMessage(messages.Message):
number = messages.IntegerField()
decoded = to_url.decode_message(MyMessage,
self.unexpected_tag_message)
self.assertEqual(1, len(decoded.all_unrecognized_fields()))
self.assertEqual('unexpected', decoded.all_unrecognized_fields()[0])
# Unknown values set to a list of however many values had that name.
self.assertEqual((['whatever'], messages.Variant.STRING),
decoded.get_unrecognized_field_info('unexpected'))
repeated_unknown = urlencode([('repeated', 400),
('repeated', 'test'),
('repeated', '123.456')])
decoded2 = to_url.decode_message(MyMessage, repeated_unknown)
self.assertEqual((['400', 'test', '123.456'], messages.Variant.STRING),
decoded2.get_unrecognized_field_info('repeated'))
def testDecodeUUID(self):
from uuid import UUID
class Foo(messages.Message):
bar = messages.UUIDField()
m = to_url.decode_message(Foo, 'bar=06335e84-2872-4914-8c5d-3ed07d2a2f16')
self.assertEqual(UUID("06335e84-2872-4914-8c5d-3ed07d2a2f16"), m.bar)
def testDecodeUUIDInvalid(self):
from uuid import UUID
class Foo(messages.Message):
bar = messages.UUIDField()
with self.assertRaises(messages.DecodeError):
to_url.decode_message(Foo, 'bar=bad')
def testDecodeDateTimeIso8601(self):
class MyMessage(messages.Message):
a_datetime = messages.DateTimeISO8601Field()
m = to_url.decode_message(MyMessage, 'a_datetime=2012-09-30T15:31:50.262000')
self.assertEqual(datetime(2012, 9, 30, 15, 31, 50, 262000), m.a_datetime)
m = to_url.decode_message(MyMessage, 'a_datetime=2012-09-30T15%3A31%3A50.262000')
self.assertEqual(datetime(2012, 9, 30, 15, 31, 50, 262000), m.a_datetime)
def testDecodeInvalidDateTimeIso8601(self):
class MyMessage(messages.Message):
a_datetime = messages.DateTimeISO8601Field()
self.assertRaises(messages.DecodeError, to_url.decode_message,
MyMessage, 'a_datetime=invalid')
def testDecodeDateTimeMsInteger(self):
class MyMessage(messages.Message):
a_datetime = messages.DateTimeMsIntegerField()
m = to_url.decode_message(MyMessage, 'a_datetime=1349019110262')
self.assertEqual(datetime(2012, 9, 30, 15, 31, 50, 262000), m.a_datetime)
m = to_url.decode_message(MyMessage, 'a_datetime=1349019110262')
self.assertEqual(datetime(2012, 9, 30, 15, 31, 50, 262000), m.a_datetime)
def testDecodeInvalidDateTimeMsInteger(self):
class MyMessage(messages.Message):
a_datetime = messages.DateTimeISO8601Field()
self.assertRaises(messages.DecodeError, to_url.decode_message,
MyMessage, 'a_datetime=invalid')
def test_decode_message_from_url(self):
class AnimalSounds(messages.Message):
cow = messages.StringField()
dog = messages.StringField()
foo = AnimalSounds(cow='moo', dog='woof')
self.assertEqual(foo, to_url.decode_message_from_url(AnimalSounds, "http://example.com?cow=moo&dog=woof"))
def test_decode_message_from_url_repeated(self):
class Animals(messages.Message):
animals = messages.StringField(repeated=True)
number = messages.IntegerField()
tmp = Animals(animals=['dog', 'cat'], number=2)
self.assertEqual(tmp, to_url.decode_message_from_url(Animals, "http://example.com?animals=dog&animals=cat&number=2"))
def test_decode_message_from_url_repeated_alias(self):
class Animals(messages.Message):
animals = messages.StringField(name='a', repeated=True)
number = messages.IntegerField()
tmp = Animals(animals=['dog', 'cat'], number=2)
self.assertEqual(tmp, to_url.decode_message_from_url(Animals, "http://example.com?a=dog&a=cat&number=2"))
def test_decode_message_from_url_repeated_alias_dashes(self):
class Animals(messages.Message):
animals = messages.StringField(name='a-m', repeated=True)
number = messages.IntegerField()
tmp = Animals(animals=['dog', 'cat'], number=2)
self.assertEqual(tmp, to_url.decode_message_from_url(Animals, "http://example.com?a-m=dog&a-m=cat&number=2"))
def test_encode_message_from_url_repeated_alias(self):
class Animals(messages.Message):
animals = messages.StringField(name='a', repeated=True)
number = messages.IntegerField()
tmp = Animals(animals=['dog', 'cat'], number=2)
self.assertEqual("a=dog&a=cat&number=2", to_url.encode_message(tmp))
def test_decode_message_from_url_repeated_not_filled_out(self):
class Animals(messages.Message):
animals = messages.StringField(repeated=True)
number = messages.IntegerField()
result = to_url.decode_message_from_url(Animals, "http://example.com?number=2")
self.assertFalse(Animals.animals.is_set(result))
self.assertEqual([], result.animals)
def test_decode_message_from_url_repeated_message(self):
class Animal(messages.Message):
name = messages.StringField()
size = messages.IntegerField()
class Animals(messages.Message):
animals = messages.MessageField(Animal, repeated=True)
number = messages.IntegerField()
dog = Animal(name='dog', size=12)
cat = Animal(name='cat', size=10)
tmp = Animals(animals=[dog, cat], number=2)
self.assertEqual(tmp, to_url.decode_message_from_url(Animals, "http://example.com?animals-0.name=dog&animals-0.size=12&animals-1.name=cat&animals-1.size=10&number=2"))
def test_encode_message_repeated_message_field(self):
class Animal(messages.Message):
name = messages.StringField()
size = messages.IntegerField()
class Animals(messages.Message):
animals = messages.MessageField(Animal, repeated=True)
number = messages.IntegerField()
dog = Animal(name='dog', size=12)
cat = Animal(name='cÄt', size=10)
tmp = Animals(animals=[dog, cat], number=2)
encoded_message = to_url.encode_message(tmp)
self.assertEqual({'number': ['2'],
'animals-0.name': ['dog'],
'animals-0.size': ['12'],
'animals-1.name': ['cÄt'],
'animals-1.size': ['10']},
parse_qs(encoded_message))
self.assertEqual(tmp, to_url.decode_message(Animals, encoded_message))
def _on_request(self,
_request,
session,
method,
url,
data=None,
headers=None,
**kw):
# Create request contract based on incoming params
req = Request(method)
req.headers = headers or {}
req.body = data
# Expose extra variadic arguments
req.extra = kw
# Compose URL
if not kw.get('params'):
req.url = str(url)
else:
req.url = str(url) + '?' + urlencode(
[(x, y) for x, y in kw['params'].items()])
# Match the request against the registered mocks in pook
mock = self.engine.match(req)
# If cannot match any mock, run real HTTP request if networking
# or silent model are enabled, otherwise this statement won't
# be reached (an exception will be raised before).
if not mock:
return _request(session,
method,
url,
data=data,
headers=headers,
**kw)
# Simulate network delay
if mock._delay:
yield from asyncio.sleep(mock._delay / 1000) # noqa
# Shortcut to mock response
res = mock._response
# Aggregate headers as list of tuples for interface compatibility
headers = []
for key in res._headers:
headers.append((key, res._headers[key]))
# Create mock equivalent HTTP response
_res = HTTPResponse(req.method, self._url(urlunparse(req.url)))
# response status
_res.version = (1, 1)
_res.status = res._status
_res.reason = http_reasons.get(res._status)
_res._should_close = False
# Add response headers
_res._raw_headers = tuple(headers)
_res._headers = multidict.CIMultiDictProxy(
multidict.CIMultiDict(headers))
if res._body:
_res.content = SimpleContent(
res._body.encode('utf-8', errors='replace'), )
else:
# Define `_content` attribute with an empty string to
# force do not read from stream (which won't exists)
_res.content = EmptyStreamReader()
# Return response based on mock definition
return _res
def acs(r):
try:
import urlparse as _urlparse
from urllib import unquote
except:
import urllib.parse as _urlparse
from urllib.parse import unquote
saml_client = _get_saml_client(get_current_domain(r))
resp = r.POST.get('SAMLResponse', None)
next_url = r.session.get('login_next_url', _default_next_url())
# Use RelayState if available, else fall back to next_url.
next_url = r.POST.get('RelayState', next_url)
if not resp:
return HttpResponseRedirect(
get_reverse([denied, 'denied', 'django_saml2_auth:denied']))
authn_response = saml_client.parse_authn_request_response(
resp, entity.BINDING_HTTP_POST)
if authn_response is None:
return HttpResponseRedirect(
get_reverse([denied, 'denied', 'django_saml2_auth:denied']))
print('return addy')
print(authn_response.return_addrs)
user_identity = authn_response.get_identity()
print('authn_response')
print(authn_response)
print('user_identity')
print(user_identity)
if user_identity is None:
return HttpResponseRedirect(
get_reverse([denied, 'denied', 'django_saml2_auth:denied']))
user_email = user_identity[settings.SAML2_AUTH.get(
'ATTRIBUTES_MAP', {}).get('email', 'Email')][0]
user_name = user_identity[settings.SAML2_AUTH.get(
'ATTRIBUTES_MAP', {}).get('username', 'UserName')][0]
user_first_name = user_identity[settings.SAML2_AUTH.get(
'ATTRIBUTES_MAP', {}).get('first_name', 'FirstName')][0]
user_last_name = user_identity[settings.SAML2_AUTH.get(
'ATTRIBUTES_MAP', {}).get('last_name', 'LastName')][0]
target_user = None
is_new_user = False
try:
target_user = User.objects.get(username=user_name)
if settings.SAML2_AUTH.get('TRIGGER', {}).get('BEFORE_LOGIN', None):
import_string(
settings.SAML2_AUTH['TRIGGER']['BEFORE_LOGIN'])(user_identity)
except User.DoesNotExist:
new_user_should_be_created = settings.SAML2_AUTH.get(
'CREATE_USER', True)
if new_user_should_be_created:
target_user = _create_new_user(user_name, user_email,
user_first_name, user_last_name)
if settings.SAML2_AUTH.get('TRIGGER', {}).get('CREATE_USER', None):
import_string(settings.SAML2_AUTH['TRIGGER']['CREATE_USER'])(
user_identity)
is_new_user = True
else:
return HttpResponseRedirect(
get_reverse([denied, 'denied', 'django_saml2_auth:denied']))
r.session.flush()
if target_user.is_active:
target_user.backend = 'django.contrib.auth.backends.ModelBackend'
login(r, target_user)
else:
return HttpResponseRedirect(
get_reverse([denied, 'denied', 'django_saml2_auth:denied']))
if settings.SAML2_AUTH.get('USE_JWT') is True:
# We use JWT auth send token to frontend
jwt_token = jwt_encode(target_user)
params = {"uid": target_user.id, "token": jwt_token}
frontend_url = settings.SAML2_AUTH.get('FRONTEND_URL', next_url)
if next_url and next_url != _default_next_url():
frontend_url = next_url
# Reconstruct URL with added parameters.
url_parts = list(
_urlparse.urlparse(frontend_url, allow_fragments=False))
query = dict(_urlparse.parse_qsl(url_parts[4]))
query.update(params)
url_parts[4] = _urlparse.urlencode(query)
return HttpResponseRedirect(_urlparse.urlunparse(url_parts))
if is_new_user:
try:
return render(r, 'django_saml2_auth/welcome.html',
{'user': r.user})
except TemplateDoesNotExist:
return HttpResponseRedirect(next_url)
else:
return HttpResponseRedirect(next_url)
def encode_message(message, prefix=''):
"""Encode Message instance to url-encoded string.
Args:
message: Message instance to encode in to url-encoded string.
prefix: Prefix to append to field names of contained values.
Returns:
String encoding of Message in URL encoded format.
Raises:
messages.ValidationError if message is not initialized.
"""
message.check_initialized()
parameters = []
def build_message(parent, prefix):
"""Recursively build parameter list for URL response.
Args:
parent: Message to build parameters for.
prefix: Prefix to append to field names of contained values.
Returns:
True if some value of parent was added to the parameters list,
else False, meaning the object contained no values.
"""
has_any_values = False
for field in sorted(parent.all_fields(), key=lambda f: f.name):
if not parent.has_value_assigned(field.name):
continue
values = parent.get_assigned_value(field.name)
# Found a value. Ultimate return value should be True.
has_any_values = True
# Normalize all values in to a list.
if not field.repeated:
values = [values]
for index, item in enumerate(values):
# Create a name with an index if it is a repeated field.
if field.repeated and isinstance(field, messages.MessageField):
field_name = '%s%s-%s' % (prefix, field.name, index)
else:
field_name = prefix + field.name
if isinstance(field, messages.DateTimeISO8601Field):
# DateTimeField stores its data as a ISO8601 compliant string.
parameters.append((field_name, item.isoformat()))
elif isinstance(field, messages.DateTimeMsIntegerField):
parameters.append((field_name, util.datetime_to_ms(item)))
elif isinstance(field, messages.MessageField):
# Message fields must be recursed in to in order to construct
# their component parameter values.
if not build_message(item, field_name + '.'):
# The nested message is empty. Append an empty value to
# represent it.
parameters.append((field_name, ''))
elif isinstance(field, messages.BooleanField):
parameters.append((field_name, item and 'true' or 'false'))
elif isinstance(field, messages.BytesField):
parameters.append(
(field_name, base64.b64encode(item).decode('utf-8')))
else:
#if isinstance(item, unicode):
# item = item.encode('utf-8')
parameters.append((field_name, str(item)))
return has_any_values
build_message(message, prefix)
# Also add any unrecognized values from the decoded string.
for key in message.all_unrecognized_fields():
values, _ = message.get_unrecognized_field_info(key)
if not isinstance(values, (list, tuple)):
values = (values, )
for value in values:
parameters.append((key, value))
return urlencode(parameters)
import urllib
import urlparse
import re
import urlparse
##x = urllib.urlopen('http://www.google.com')
##print x.read()
url = 'http://www.pythonprgramming.net'
values = {'s':'basic',
'submit':'search'}
data = urlparse.urlencode(values)
data = data.encode('utf-8')
req = urllib.request.Request(url, data)
resp = urllib.request.urlopen(req)
respData = resp.read()
print (respData)
##paragraphs = re.findall(r'<p>(.*?)</p>', str(respData))
for eachP in paragraphs:
print(eachP)
def build_query(self, params):
params = urlparse.urlencode(params)
return params
def _request(self, **kwargs):
# Check for missing variables
resource = self.interface
def validate_kwarg(key, target):
# Always allow list
if isinstance(key, list):
return all([validate_kwarg(x, target[0]) for x in key])
return isinstance(key, target)
for k in resource.get('required', []):
if k not in [x.split(':')[0] for x in kwargs.keys()]:
raise ValueError('Missing required argument: %s' % k)
if isinstance(kwargs.get(k), list) and not isinstance(
resource['required'][k], list):
raise ValueError('Wrong argument type: %s is not a list' % k)
elif not validate_kwarg(kwargs.get(k), resource['required'][k]):
if isinstance(resource['required'][k], list):
raise ValueError(
'Wrong arguemnt type: %s is not a list of %ss' %
(k, resource['required'][k][0]))
raise ValueError('Wrong arguemnt type: %s is not a %s' %
(k, resource['required'][k]))
conduit = self.api.conduit
if conduit:
# Already authenticated, add session key to json data
kwargs['__conduit__'] = conduit
elif self.method == 'conduit' and self.endpoint == 'connect':
# Not authenticated, requesting new session key
token = str(int(time.time()))
kwargs['authToken'] = token
kwargs['authSignature'] = self.api.generate_hash(token)
else:
# Authorization is required, silently auth the user
self.api.connect()
kwargs['__conduit__'] = self.api.conduit
url = urlparse.urlparse(self.api.host)
if url.scheme == 'https':
conn = httplib.HTTPSConnection(url.netloc,
timeout=self.api.timeout)
else:
conn = httplib.HTTPConnection(url.netloc, timeout=self.api.timeout)
path = url.path + '%s.%s' % (self.method, self.endpoint)
headers = {
'User-Agent':
'python-phabricator/%s' % str(self.api.clientVersion),
'Content-Type': 'application/x-www-form-urlencoded'
}
body = urlparse.urlencode({
"params": json.dumps(kwargs),
"output": self.api.response_format
})
# TODO: Use HTTP "method" from interfaces.json
conn.request('POST', path, body, headers)
response = conn.getresponse()
data = self._parse_response(response.read())
return Result(data['result'])
def request(self, resource, method = "get", args = None, body = None, parts=None, headers={}):
params = None
path = resource
headers['User-Agent'] = 'Basic Agent'
BOUNDARY = r'00hoYUXOnLD5RQ8SKGYVgLLt64jejnMwtO7q8XE1'
CRLF = r'\r\n'
if parts:
# Attempt to find the Mimetype
headers['Content-Type']='multipart/form-data; boundary='+BOUNDARY
encode_string = StringIO()
for part in parts:
if part.type != 'file':
continue
encode_string.write(r'--' + BOUNDARY)
encode_string.write(CRLF)
body = None
if part.type == 'file':
encode_string.write(r'Content-Disposition: form-data; name="%s"; filename="%s"' % (part.key, os.path.basename(part.text)))
encode_string.write(CRLF)
content_type = self.get_content_type(part.text)
encode_string.write(r'Content-Type: %s' % content_type)
encode_string.write(CRLF)
encode_string.write(r'Content-Transfer-Encoding: base64')
encode_string.write(CRLF)
encode_string.write(CRLF)
with open(part.text, "rb") as file:
body = base64.b64encode(file.read()).decode()
#body=str(file.read())
else:
encode_string.write(r'Content-Disposition: form-data; name="%s"' % (part.key))
encode_string.write(CRLF)
encode_string.write(r'Content-Type: %s' % part.contentType)
encode_string.write(CRLF)
encode_string.write(CRLF)
body = part.text
encode_string.write(body)
encode_string.write(CRLF)
encode_string.write(r'--' + BOUNDARY + r'--' + CRLF)
body = encode_string.getvalue()
headers['Content-Length'] = str(len(body))
elif body:
if not headers.get('Content-Type', None):
headers['Content-Type']='text/xml'
headers['Content-Length'] = str(len(body))
else:
if not headers.get('Content-Type', None):
headers['Content-Type']='text/xml'
domain=self.base_url.split("//")[1].split("/")[0]
if args:
if utils.getPythonVersion() == '2':
path += r"?" + urllib.urlencode(args)
else:
path += r"?" + urlparse.urlencode(args)
request_path = []
if self.path != "/":
if self.path.endswith('/'):
request_path.append(self.path[:-1])
else:
request_path.append(self.path)
if path.startswith('/'):
request_path.append(path[1:])
else:
request_path.append(path)
# if self.host=='211.136.86.203':# kuan chang com IP
# url = r"%s://%s%s" % (self.scheme, self.host, ''.join(request_path))
# else:# normal flow
url = r"%s://%s%s" % (self.scheme, self.host, ''.join(request_path))
url = url.rstrip('/')
logger.info('request: %s' % url)
resp, content = self.h.request(url, method.upper(), body=body, headers=headers )
encoding = 'UTF-8'
if 'content-type' in resp and 'charset' in resp['content-type']:
items = resp['content-type'].split('=')
encoding = items[1]
if 'content-type' in resp and 'application' in resp['content-type'] and 'json' not in resp['content-type']:
encoding = None
if 'content-type' in resp and 'image' in resp['content-type']:
encoding = None
if encoding is not None:
return {r'headers':resp, r'body':content.decode(encoding)}
else:
# not return binary data
return {r'headers':resp, r'body':''}
def __init__(
self,
url,
cache_dir,
pkg_dir,
proxy_addr="",
proxy_user="",
proxy_pass="",
gpg_verify=True,
channel_label=None,
):
self.url = url
parts = url.rsplit('/dists/', 1)
self.base_url = [parts[0]]
parsed_url = urlparse.urlparse(url)
query = urlparse.parse_qsl(parsed_url.query)
new_query = []
suite = None
component = None
arch = None
for qi in query:
if qi[0] == "uyuni_suite":
suite = qi[1]
elif qi[0] == "uyuni_component":
component = qi[1]
elif qi[0] == "uyuni_arch":
arch = qi[1]
else:
new_query.append(qi)
if suite:
parsed_url = parsed_url._replace(
query=urlparse.urlencode(new_query))
base_url = urlparse.urlunparse(parsed_url)
path_list = parsed_url.path.split("/")
log2(0, 0, "Base URL: {}".format(base_url))
log2(0, 0, "Suite: {}".format(suite))
log2(0, 0, "Component: {}".format(component))
if "/" not in suite:
path_list.append("dists")
path_list.extend(suite.split("/"))
if component:
path_list.extend(component.split("/"))
if "/" not in suite:
if arch is None:
rhnSQL.initDB()
h = rhnSQL.prepare("""
SELECT ca.label AS arch_label
FROM rhnChannel AS c
LEFT JOIN rhnChannelArch AS ca
ON c.channel_arch_id = ca.id
WHERE c.label = :channel_label
""")
h.execute(channel_label=channel_label)
row = h.fetchone_dict()
if row and "arch_label" in row:
aspl = row["arch_label"].split("-")
if len(aspl) == 3 and aspl[0] == "channel" and aspl[
2] == "deb":
arch_trans = {
"ia32": "i386",
"arm": "armhf",
}
if aspl[1] in arch_trans:
arch = arch_trans[aspl[1]]
else:
arch = aspl[1]
if arch:
log2(0, 0, "Channel architecture: {}".format(arch))
path_list.append("binary-{}".format(arch))
while "" in path_list:
path_list.remove("")
parsed_url = parsed_url._replace(path="/".join(path_list))
self.url = url = urlparse.urlunparse(parsed_url)
self.base_url = [base_url]
# Make sure baseurl ends with / and urljoin will work correctly
if self.base_url[0][-1] != '/':
self.base_url[0] += '/'
self.urls = self.base_url
self.sslclientcert = self.sslclientkey = self.sslcacert = None
self.proxy = proxy_addr
self.proxy_username = proxy_user
self.proxy_password = proxy_pass
self.gpg_verify = gpg_verify
self.basecachedir = cache_dir
if not os.path.isdir(self.basecachedir):
fileutils.makedirs(self.basecachedir, user='******', group='www')
self.includepkgs = []
self.exclude = []
self.pkgdir = pkg_dir
self.http_headers = {}
