def normalise_url(url):
"""
Take a URL and turn it into a form which is suitable for normalised comparison with other normalised
URLs.
The function does the following:
* strips leading/trailing whitespace
* validates the URL is realistic
* strips the scheme (so, removes http, https, ftp, ftps, etc)
:param url:
:return:
"""
if url is None:
return url
schemes = ["http", "https", "ftp", "ftps"]
url = url.strip()
if url.startswith("//"):
url = "http:" + url
if "://" not in url:
url = "http://" + url
u = urlparse.urlparse(url)
if u.netloc is None or u.netloc == "":
raise ValueError(
"Could not extract a normalised URL from '{x}'".format(x=url))
if u.scheme not in schemes:
raise ValueError(
"URL must be at http(s) or ftp(s), found '{x}'".format(x=u.netloc))
n = urlparse.ParseResult(None, u.netloc, u.path, u.params, u.query,
u.fragment)
return urlparse.urlunparse(n)
def __sanitize_url(server_url):
"""
Parses a URL and makes sure it has a scheme and no extra / and path.
..note:: Calling this method only once might yield incorrect result. Always call
the sanitize_url function instead.
:param str server_url: URL to clean up.
:returns: The cleaned up URL.
"""
# The given url https://192.168.1.250:30/path?a=b is parsed such that
# scheme => https
# netloc => 192.168.1.250:30
# path = /path
# query = a=b
# As such, when sanitizing a url, we want to keep only the scheme and
# network location
# Then break up the url into chunks
tokens_parsed = urlparse.urlparse(server_url)
# Then extract the good parts from the url
clean_url_tokens = urlparse.ParseResult(
# We want https when there is no specified scheme.
scheme=tokens_parsed.scheme or "https",
# If only a host has been provided, path will be set.
# If a scheme was set, then use the netloc
netloc=tokens_parsed.netloc or tokens_parsed.path,
path="",
params="",
query="",
fragment="")
return urlparse.urlunparse(clean_url_tokens)
def can_fetch(self, url):
# construct URL of robots.txt
parts = urlparse.urlparse(url)
parts2 = urlparse.ParseResult(parts.scheme, parts.netloc,
'/robots.txt', None, None, None)
robotsurl = parts2.geturl()
print url, robotsurl
# create new parser, if there is none in cache
if robotsurl not in self.cache:
rp = robotparser.RobotFileParser()
rp.set_url(robotsurl)
self.cache[robotsurl] = rp
# fetch and parse robots.txt if existing one is too old
if time.time() - self.cache[robotsurl].mtime() > 60 * 60 * 24:
try:
status, header, content = self.http_get(robotsurl)
except:
return False
if status >= 400:
return True
rp.parse(content)
rp.modified()
# lookup current given url with current parser
return self.cache[robotsurl].can_fetch(SPIDERAGENT, url)
def _build_network_relative_path(url):
p = urlparse.urlparse(url)
return urlparse.urlunparse(
urlparse.ParseResult(p.scheme, p.netloc, os.path.dirname(p.path), '',
'', ''))
def geturl(self):
return urlparse.ParseResult(*self.gettuple()).geturl()
def get_url(self):
query = werkzeug.url_encode(self._query)
return urlparse.ParseResult(self.scheme, self.netloc, self.path,
self.params, query,
self.fragment).geturl()
def pdns_reverse_request_handler(search_value,
result_format,
**dynamic_fields):
"""method to allow for custom pdns reverse requests handler
for DNSDB as a pdns source
Note: the method name must be "pdns_reverse_request_handler"
"""
scheme = "https"
netloc = "api.dnsdb.info"
path = "lookup/rdata".split('/')
query = ["limit=%d" % ((int(dynamic_fields['limit'])))]
results = {'success': False}
if not config.myConfig['apikey']:
results['error'] = 'No DNSDB key.'
return results
try:
value = _verify_type(search_value, dynamic_fields['type'])
except Exception as e:
results['error'] = 'Unable to verify input'
return results
path.extend([dynamic_fields['type'], urllib.quote(value)])
# If 'any' is in rrtypes and anything else too, just default to 'any'
if 'any' in dynamic_fields['rrtypes']:
if 'any-dnssec' in dynamic_fields['rrtypes']:
dynamic_fields['rrtypes'] = ['any', 'any-dnssec']
else:
dynamic_fields['rrtypes'] = ['any']
results['data'] = {}
for rrtype in dynamic_fields['rrtypes']:
local_path = list(path) + [rrtype]
local_path = "/".join(local_path)
local_url = urlparse.ParseResult(scheme,
netloc,
local_path,
"",
"&".join(query),
"")
url = urlparse.urlunparse(local_url)
try:
headers = {'Accept': 'application/json',
'X-API-Key': config.myConfig['apikey']}
r = requests.get(url,
proxies=settings.PROXIES,
headers=headers,
verify=config.myConfig["ssl_verify"])
except Exception as e:
results['error'] = str(e)
return results
if r.status_code not in [200, 404]:
return check_return_code(r)
if r.status_code == 200:
# Each line of the response is an individual JSON blob.
for line in r.text.split('\n'):
# Skip empty lines.
if not line:
continue
try:
tmp = json.loads(line)
except Exception as e:
results['error'] = \
"%s: %s" % (str(e), cgi.escape(line, quote=True))
return results
# Convert epoch timestamps to human readable.
for key in ['time_first', 'time_last']:
if key in tmp:
tmp[key] = time.strftime("%Y-%m-%d %H:%M:%S",
time.gmtime(tmp[key]))
rrtype = tmp['rrtype']
# Strip the MX weight
if rrtype == 'MX':
tmp['rdata'] = [tmp['rdata'].split()[1]]
else:
tmp['rdata'] = [tmp['rdata']]
if result_format in ['none', 'list']:
if tmp['rrname'][-1] == ".":
tmp['rrname'] = tmp['rrname'][:-1]
for i in range(len(tmp['rdata'])):
if tmp['rdata'][i][-1] == ".":
tmp['rdata'][i] = tmp['rdata'][i][:-1]
try:
results['data'][rrtype].append(tmp)
except KeyError:
results['data'][rrtype] = [tmp]
results['success'] = True
rate = {'limit': r.headers['X-RateLimit-Limit'],
'remaining': r.headers['X-RateLimit-Remaining'],
'reset': r.headers['X-RateLimit-Reset']}
if rate['reset'] != 'n/a':
rate['reset'] = datetime.datetime.utcfromtimestamp(
float(rate['reset'])).strftime('%Y-%m-%d %H:%M:%S GMT')
results['rate'] = rate
if result_format != 'none':
results = _format_results(results, result_format, dynamic_fields)
return results
urls = defaultdict(list)
for s in (a, b):
filename = os.path.join(args.prefix, domain, s)
html = read_file(filename)
url = original_url(html)
url = "http://" + url
# print url
parsed_url = urlparse.urlparse(url)
stripped_path = stripper.strip(parsed_url.path).replace("//", "/")
stripped_query = stripper.strip(parsed_url.query).replace(
"//", "/")
stripped_url = urlparse.ParseResult(parsed_url.scheme,
parsed_url.netloc,
stripped_path,
parsed_url.params,
stripped_query,
parsed_url.fragment).geturl()
urls[stripped_url].append(url)
if stripped_url != url:
was_stripped += 1
if len(urls) == 1:
correct += 1
print len(urls), was_stripped, correct, urls.items()
print "correct: ", correct
def delayed_shipping(request, template):
apitoolsusername = '******' + request.user.username
appname_id = MainSDKApp.objects.using('apiserver_db').all()
if request.method == 'GET':
form = DelayedShippingForm()
return render(
request, template, {
'form': form,
'username': apitoolsusername,
'appname_id': appname_id,
'guestlist': settings.GUEST
})
elif request.method == 'POST':
if apitoolsusername in settings.GUEST:
form = DelayedShippingForm(request.POST)
return render(
request, template, {
'form': form,
'username': apitoolsusername,
'appname_id': appname_id,
'guestlist': settings.GUEST
})
form = DelayedShippingForm(request.POST)
if form.is_valid():
cleaned_data = form.cleaned_data
if cleaned_data['captcha'].lower() != request.session['captcha']:
form.add_error('captcha', '验证码错误')
else:
appid = cleaned_data['appid']
orderid = cleaned_data['orderid']
price = cleaned_data['price'] * 100
productid = cleaned_data['productid']
try:
app = MainSDKApp.objects.using('apiserver_db').get(
appid=appid)
except MainSDKApp.DoesNotExist:
form.add_error('appid', 'appid在SDK服务器中不存在')
else:
try:
U8Uorder.objects.using('u8server_db').get(
orderId=orderid)
except Exception:
form.add_error('orderid', 'U8服务器上不存在此订单')
return render(
request, template, {
'form': form,
'username': apitoolsusername,
'appname_id': appname_id,
'guestlist': settings.GUEST
})
pay_callback_url = app.pay_callback_url
parsed_u8_callback_url = urlparse.urlparse(
pay_callback_url)
user = MainSDKUser.objects.using('apiserver_db').get(
username='******')
request_args = [('AppOrderID', orderid), ('Price', price),
('Uid', user.id),
('ChannelOrderID', uuid.uuid4().get_hex()),
('ProductID', productid)]
request_query_str = '&'.join(
['='.join(map(str, item)) for item in request_args])
new_u8_parsed_callback_url = urlparse.ParseResult(
scheme=parsed_u8_callback_url.scheme,
netloc=parsed_u8_callback_url.netloc,
path=parsed_u8_callback_url.path,
params=parsed_u8_callback_url.params,
query=request_query_str,
fragment=parsed_u8_callback_url.fragment)
new_u8_callback_url = urlparse.urlunparse(
new_u8_parsed_callback_url)
callback_sign = get_signature(
app.appsecret.encode('utf-8'), new_u8_callback_url)
request_args.append(('sign', callback_sign))
request_args_map = dict(request_args)
request_obj = urllib2.Request(pay_callback_url)
request_obj.add_data(urllib.urlencode(request_args_map))
try:
response = urllib2.urlopen(request_obj,
timeout=6).read()
messages.info(request, 'U8 said: %s' % response)
except Exception as e:
messages.error(request, 'Exception: %s' % str(e))
else:
if json.loads(response)['status'] == 'success':
infostr = '订单发货成功'
try:
ApiToolsShippingHistory2.objects.using(
'default').create(
username=apitoolsusername[1:],
time=time.strftime(
'%Y-%m-%d %H:%M:%S',
time.localtime(time.time() +
28800)),
orderid=orderid,
price=price,
appid=appid)
except Exception:
infostr = infostr + ',写入缓存失败'
messages.success(request, infostr)
else:
messages.error(request, '修改U8服务器失败')
return render(
request, template, {
'form': form,
'username': apitoolsusername,
'appname_id': appname_id,
'guestlist': settings.GUEST
})
def get(self, sid):
memcache_client = memcache.Client()
results = memcache_client.get(sid)
if results:
try:
results = json.loads(zlib.decompress(results))
except Exception:
results = None
if not results:
try:
access_token = crypto.decrypt(sid,
self.app.config["SECRET_KEY"])
except (ValueError, TypeError):
self.response.status_int = 403
return
logging.debug("access_token: %s", access_token)
content = urlfetch.fetch(
"https://api.weibo.com/2/statuses/home_timeline.json?" +
urllib.urlencode({
"count": 100,
"base_app": 0,
"feature": 0,
"trim_user": 0,
"isGetLongText": "1",
}),
headers={
"Authorization": "OAuth2 " + access_token
}).content
body = json.loads(content)
if "error" in body:
logging.error("error: %s", content)
self.response.status_int = 500
self.response.write(body["error"])
return
results = body["statuses"]
# 读取超过140字的全文
long_text_ids = []
long_text_map = {}
for status in results:
if status.get("isLongText", True):
long_text_ids.append(status["idstr"])
status = status.get("retweeted_status")
if status is not None:
if status.get("isLongText", True):
long_text_ids.append(status["idstr"])
logging.debug("long_text_ids before cache size=%d",
len(long_text_ids))
if len(long_text_ids):
cached_result = memcache_client.get_multi(
long_text_ids, "long#")
long_text_map.update(cached_result)
long_text_ids = filter(lambda x: x not in long_text_map,
long_text_ids)
logging.debug("long_text_ids after cache size=%d",
len(long_text_ids))
long_text_ids = sorted(list(set(long_text_ids)))
if len(long_text_ids) > 0:
rpcs = []
max_size = 50
for chunk in (
long_text_ids[x:x + max_size]
for x in xrange(0, len(long_text_ids), max_size)):
ids = ",".join(chunk)
rpc = urlfetch.create_rpc()
urlfetch.make_fetch_call(
rpc,
"https://api.weibo.com/2/statuses/show_batch.json?" +
urllib.urlencode({
"ids": ids,
"isGetLongText": "1",
}),
headers={"Authorization": "OAuth2 " + access_token})
rpc.ids = ids
rpcs.append(rpc)
for rpc in rpcs:
try:
body = json.loads(rpc.get_result().content)
except Exception as e:
body = {"error": str(e)}
if "error" in body:
logging.warn("show_batch %s error: %s", rpc.ids,
str(body["error"]))
elif "statuses" in body:
for status in body["statuses"]:
if "longText" in status:
long_text_map[status["idstr"]] = status[
"longText"]["longTextContent"]
logging.debug("long_text_map size=%d", len(long_text_map))
if len(long_text_map) > 0:
memcache_client.set_multi(long_text_map,
time=86400,
key_prefix="long#")
def expand_long_text(status):
if status.get("isLongText", True):
text = long_text_map.get(status["idstr"])
if text is not None:
logging.debug("replace long text for %s",
status["idstr"])
status["text"] = text
status["isLongText"] = False
for status in results:
expand_long_text(status)
status = status.get("retweeted_status")
if status is not None:
expand_long_text(status)
# 将t.cn短链接展开
tcn_id2url = defaultdict(set)
all_tcn_urls = set()
def extract_tcn_urls(status):
tcn_urls = tcn_regex.findall(status["text"])
idstr = status["idstr"]
tcn_id2url[idstr].update(tcn_urls)
all_tcn_urls.update(tcn_urls)
for status in results:
extract_tcn_urls(status)
status = status.get("retweeted_status")
if status is not None:
extract_tcn_urls(status)
logging.debug("all_tcn_urls before cache size=%d",
len(all_tcn_urls))
all_tcn_urls = list(all_tcn_urls)
tcn_short2long = {}
cached_result = memcache_client.get_multi(all_tcn_urls, "tcn#")
tcn_short2long.update(cached_result)
all_tcn_urls = filter(lambda x: x not in tcn_short2long,
all_tcn_urls)
logging.debug("all_tcn_urls after cache size=%d",
len(all_tcn_urls))
rpcs = []
max_size = 20
for chunk in (all_tcn_urls[x:x + max_size]
for x in xrange(0, len(all_tcn_urls), max_size)):
rpc = urlfetch.create_rpc()
rpc.chunk = chunk
urlfetch.make_fetch_call(
rpc,
"https://api.weibo.com/2/short_url/expand.json?" +
urllib.urlencode([("url_short", x) for x in chunk]),
headers={"Authorization": "OAuth2 " + access_token})
rpcs.append(rpc)
for rpc in rpcs:
try:
result = json.loads(rpc.get_result().content)
except Exception as e:
result = {"error": str(e)}
if "error" in result:
logging.warn("expand %s error: %s", str(rpc.chunk),
result["error"])
elif "urls" in result:
for u in result["urls"]:
url_long = u["url_long"]
if u["result"] and url_long != "":
o = urlparse.urlparse(url_long,
scheme="http",
allow_fragments=True)
qsl = filter(lambda x: x[0] not in utm_queries,
urlparse.parse_qsl(o.query, True))
o = urlparse.ParseResult(
o.scheme, o.netloc, o.path, o.params,
urllib.urlencode(encode_obj(qsl)), o.fragment)
url_long = urlparse.urlunparse(o)
if url_long:
tcn_short2long[u["url_short"]] = url_long
logging.debug("tcn_short2long size=%d", len(tcn_short2long))
if len(tcn_short2long) > 0:
memcache_client.set_multi(tcn_short2long,
time=86400,
key_prefix="tcn#")
def expand_url(status):
idstr = status["idstr"]
tcn_urls = tcn_id2url[idstr]
if len(tcn_urls) > 0:
text = status["text"]
for short_url in tcn_urls:
long_url = tcn_short2long.get(short_url)
if long_url is not None:
text = text.replace(short_url, long_url)
status["text"] = text
for status in results:
expand_url(status)
status = status.get("retweeted_status")
if status is not None:
expand_url(status)
# 将结果缓存
memcache_client.set(sid,
zlib.compress(json.dumps(results), 9),
time=120)
else:
logging.debug("sid %s from cache", sid)
self.response.headers[
"Content-Type"] = "application/rss+xml; charset=utf-8"
self.render_response("rss.xml",
results=results,
abs_uri_for=functools.partial(self.uri_for,
_full=True,
_scheme="https"))
stripped_path = language_stripper.strip_path(parsed_uri.path)
stripped_path = re.sub(r'//+', '/', stripped_path)
stripped_path = re.sub(r'__+', '_', stripped_path)
stripped_path = re.sub(r'--+', '-', stripped_path)
stripped_query = language_stripper.strip_query(parsed_uri.query)
stripped_netloc = parsed_uri.netloc
if '@' in stripped_netloc:
stripped_netloc = stripped_netloc.split('@')[1]
if ':' in stripped_netloc:
stripped_netloc = stripped_netloc.split(':')[0]
if not stripped_netloc:
continue
stripped_uri = urlparse.ParseResult(scheme="http",
netloc=stripped_netloc,
path=stripped_path,
params='',
query=stripped_query,
fragment='').geturl()
# remove new trailing /
if stripped_uri and stripped_uri[-1] == '/' \
and parsed_uri.path and parsed_uri.path[-1] != '/':
stripped_uri = stripped_uri[:-1]
if stripped_uri in candidates:
print_match(stripped_uri, uri, args.crawl, candidates)
continue
def add_wfs_layer(endpoint,
version,
typename,
metadata_string=None,
title=None,
bbox=None,
user=None,
password=None):
endpoint_parsed = urlparse.urlparse(endpoint)
q_dict = {
'version': version,
'typename': typename,
'outputFormat': 'shape-zip',
'request': 'GetFeature',
'service': 'WFS',
'srsName': 'EPSG:4326',
}
if bbox:
q_dict['bbox'] = ','.join(bbox)
parsed_url = urlparse.ParseResult(scheme=endpoint_parsed.scheme,
netloc=endpoint_parsed.netloc,
path=endpoint_parsed.path,
params=None,
query=urllib.urlencode(q_dict),
fragment=None)
tmpfile = download_file(parsed_url.geturl(), user=user, password=password)
# args = [
# 'ogr2ogr',
# '-nlt POLYGON',
# '-skipfailures',
# '%s.shp' % tmpfile,
# tmpfile,
# 'OGRGeoJSON'
# ]
#
# retval = subprocess.call(args)
# # get metadata file
# if metadata_string:
# if not isinstance(metadata_string, unicode):
# metadata_string = unicode(metadata_string, 'utf-8')
# metadata_file = '%s.xml' % tmpfile
# with io.open(metadata_file, mode='w', encoding='utf-8') as f:
# f.write(metadata_string)
#
# saved_layer = None
# if retval == 0:
# saved_layer = file_upload(
# '%s.shp' % tmpfile,
# overwrite=True)
# saved_layer.set_default_permissions()
# saved_layer.title = title or typename
# saved_layer.save()
#
# # cleanup
# dir_name = os.path.dirname(tmpfile)
# for root, dirs, files in os.walk(dir_name):
# for f in files:
# if tmpfile in f:
# try:
# os.remove(os.path.join(root, f))
# except:
# pass
dir_name = os.path.dirname(tmpfile)
saved_layer = None
metadata_file = None
with ZipFile(tmpfile) as zf:
zf.extractall(path=dir_name)
for name in zf.namelist():
basename, ext = os.path.splitext(name)
if '.shp' in ext:
# get metadata file
if metadata_string:
if not isinstance(metadata_string, unicode):
metadata_string = unicode(metadata_string, 'utf-8')
metadata_file = '%s.xml' % basename
metadata_file = os.path.join(dir_name, metadata_file)
metadata_string = cleanup_metadata(metadata_string)
with io.open(metadata_file, mode='w',
encoding='utf-8') as f:
f.write(metadata_string)
# process shapefile layer
saved_layer = file_upload(os.path.join(dir_name, name),
overwrite=True)
saved_layer.set_default_permissions()
saved_layer.title = title or typename
saved_layer.save()
break
# cleanup
for name in zf.namelist():
filepath = os.path.join(dir_name, name)
try:
os.remove(filepath)
except BaseException:
pass
if metadata_file:
try:
os.remove(metadata_file)
except BaseException:
pass
# cleanup
try:
os.remove(tmpfile)
except BaseException:
pass
return saved_layer
list_only = True
elif a[0] == '-i':
interactive = True
elif a[0] == '-h':
usage()
sys.exit(0)
try:
in_app = urlparse.urlparse(leftover[0])
except:
error("%s is not a valid Acre app" % leftover[0])
try:
out_app = urlparse.urlparse(leftover[1])
except IndexError, e:
out_app = urlparse.ParseResult('','',in_app.path.split('/')[-1],'','','')
if in_app.scheme != "" and in_app.scheme != "freebase":
error("%s is not a valid acre URL" % leftover[0])
if out_app.scheme != "" and out_app.scheme != "freebase":
error("%s is not a valid acre URL" % leftover[1])
if in_app.scheme == "" and in_app.scheme == out_app.scheme:
error("You must specify at least one URL")
if in_app.netloc:
username = in_app.username or username
password = in_app.password or password
msb = MetawebServicesBundle(in_app.netloc, None, username, password)
if version:
def unauth(self):
if not self.is_auth():
return self
return URL.objectify(urlparse.ParseResult(
self.scheme, '%s:%s' % (self.hostname, self.port or {'https': 443, 'http': 80}[self.scheme]),
self.path.replace('//', '/'), self.params, self.query, self.fragment))
def _parse_url(self, rawUrl):
"""
Get direct image URL for web pages.
"""
url = urlparse.urlparse(rawUrl)
# check whether the url is accessible
req = urllib2.Request(rawUrl,
headers={'User-Agent': credentials.USER_AGENT})
fd = urllib2.urlopen(req, context=self.context)
p = url.path.lower()
if p.endswith(".jpg") or p.endswith(".jpeg") or p.endswith(
".png") or p.endswith(".gif"):
return rawUrl
# get direct url from imgur (skip sets and albums)
if "imgur.com" in url.netloc and "a/" not in url.path and (
"," not in url.path) and ("gifv" not in url.path):
newloc = "i." + url.netloc
newpath = url.path
if newpath.endswith("/new"):
newpath.replace("/new", "")
newpath += ".jpg"
newpath = newpath.replace("gallery/", "")
newUrl = urlparse.ParseResult(url.scheme, newloc, newpath,
url.params, url.query, url.fragment)
return newUrl.geturl()
# get direct url from flickr
if "flickr.com" in url.netloc:
path = filter(lambda x: x != '', url.path.split('/'))
if path[0] == 'photos' and len(path) >= 3:
newpath = '/photos/%s/%s/sizes/l' % (path[1], path[2])
newUrl = urlparse.ParseResult(url.scheme, url.netloc, newpath,
url.params, url.query,
url.fragment)
try:
file = urllib2.urlopen(newUrl.geturl(),
context=self.context)
tree = etree.HTML(file.read())
staticUrl = tree.xpath(
'//div[@id="allsizes-photo"]/img/@src')
if len(staticUrl):
return staticUrl[0]
except:
pass
if "apod.nasa.gov" in url.netloc:
try:
file = urllib2.urlopen(url.geturl(), context=self.context)
tree = etree.HTML(file.read())
directUrl = tree.xpath('//img/@src')
if len(directUrl):
return "http://apod.nasa.gov/apod/" + directUrl[0]
except:
pass
if "wikipedia.org" in url.netloc and "File:" in url.path:
try:
file = urllib2.urlopen(url.geturl(), context=self.context)
tree = etree.HTML(file.read())
directUrl = tree.xpath('//div[@class="fullMedia"]/a/@href')[0]
if len(directUrl):
return "http:" + directUrl
except:
pass
return None
try:
developer_key = developer_key_file.read().strip()
except IOError, io_error:
print 'Error loading developer key from file %s' % (
FLAGS.developer_key_file)
print 'Error details: %s' % str(io_error)
sys.exit()
finally:
developer_key_file.close()
s = urlparse.urlparse(uri)
query = 'key=' + developer_key
if s.query:
query = s.query + '&key=' + developer_key
d = urlparse.ParseResult(s.scheme,
s.netloc,
s.path,
s.params,
query,
s.fragment)
uri = urlparse.urlunparse(d)
if FLAGS.dump_request:
print '--request-start--'
print '%s %s' % (method, uri)
if headers:
for (h, v) in headers.iteritems():
print '%s: %s' % (h, v)
print ''
if body:
print json.dumps(json.loads(body),
sort_keys=True,
indent=2)
print '--request-end--'
def upload(request):
"""
view that handles file upload via Ajax
"""
# check upload permission
error = ''
new_file_name = ''
allowed_upload_file_types = ALLOWED_UPLOAD_FILE_TYPE
max_upload_file_size = MAX_UPLOAD_FILE_SIZE
try:
base_file_name = str(time.time()).replace(
'.', str(random.randint(0, 100000)))
file_storage, new_file_name = store_uploaded_file(
request,
'file-upload',
allowed_upload_file_types,
base_file_name,
max_file_size=max_upload_file_size)
logger.info(
"docreaderxblock FileStoreAPI.upload try uploaded {}: success".
format(str(new_file_name)))
except Exception as e:
logger.warning(
"docreaderxblock FileStoreAPI.upload try uploaded {}: fail -- {}"
.format(str(new_file_name), e))
file = request.POST["file-upload"].file
request.FILES = {}
request.FILES["file-upload"] = file
try:
file_storage, new_file_name = store_uploaded_file(
request,
'file-upload',
allowed_upload_file_types,
base_file_name,
max_file_size=max_upload_file_size)
logger.info(
"docreaderxblock FileStoreAPI.upload try uploaded {}: success"
.format(str(new_file_name)))
except Exception as e:
error = str(type(e)) + " - " + str(e)
logger.error(
"docreaderxblock FileStoreAPI.upload except uploaded {}: fail"
.format(str(new_file_name), e))
raise Exception
if error == '':
result = 'SUCCESS'
file_url = file_storage.url(new_file_name)
parsed_url = urlparse.urlparse(file_url)
file_url = urlparse.urlunparse(
urlparse.ParseResult(parsed_url.scheme, parsed_url.netloc,
parsed_url.path, '', '', ''))
else:
result = ''
file_url = ''
return HttpResponse(json.dumps({
'result': "SUCCESS",
'msg': result,
'error': error,
'file_url': file_url,
}),
content_type="application/json")
def parseUri(packageUri, path=None, packageloc=None):
""" static method to parse the uri
@throws - AgentException
"""
if (path == None):
from agent.lib.packagemgr import PackageMgr
path = PackageMgr.packagePath()
uri = urlparse.urlparse(packageUri)
if (uri.scheme != 'http'):
raise AgentException(
Errors.PACKAGE_SCHEME_ERROR,
'uri (%s) scheme(%s) not supported' % (packageUri, uri.scheme))
if (uri.path == ''):
raise AgentException(Errors.PACKAGE_PATH_ERROR,
'uri (%s) path is empty' % (packageUri))
# now parse the path. get the name and then verify that it matches the convention
if packageloc is not None:
# check if torrent download
if uri.path.endswith('.torrent'):
if not packageloc.endswith('.torrent'):
packageloc = packageloc + '.torrent'
packName = packageloc
else:
packName = uri.path.rpartition('/')[2]
match = PackageUtil.nameRe.match(packName)
if (match == None or match.group(0) != packName):
raise AgentException(
Errors.PACKAGE_PATH_ERROR,
'cannot find package name in path %s' % (uri.path))
# ok now we can fill out the dictionary
uriDict = {}
uriDict['uri'] = packageUri
uriDict['scheme'] = uri.scheme
uriDict['uripath'] = uri.path
uriDict['package'] = match.group(1)
uriDict['packageNameVersion'] = match.group(2)
uriDict['inProgressPackage'] = PackageUtil.inProgressPath(
uriDict['package'])
uriDict['packageName'] = match.group(3)
uriDict['propName'] = uriDict['package'] + '.prop'
uriDict['packageVersion'] = match.group(4)
uriDict['packageVersionMajor'] = match.group(5)
uriDict['packageVersionMinor'] = match.group(6)
uriDict['packageVersionBuild'] = match.group(7)
uriDict['packagePlat'] = match.group(8)
# path specific attributes
uriDict['packagePath'] = os.path.join(path, uriDict['package'])
uriDict['inProgressPackagePath'] = os.path.join(
path, uriDict['inProgressPackage'])
uriDict['propPath'] = os.path.join(path, uriDict['propName'])
# check if bittorrent transfer and define the necessary keys
if match.group(9) is not None and match.group(9) == '.torrent':
uriDict['scheme'] = 'bittorrent'
uriDict['torrentName'] = uriDict['package'] + '.torrent'
uriDict['torrentPath'] = os.path.join(path, uriDict['torrentName'])
# calculate prop url
# append path with .prop - mons: leaving the below code in place in case we support other protocols in future
uriScheme = uri.scheme
if (uriScheme !=
"http"): #only use http to download .prop and .torrent files
uriScheme = "http"
if uri.path.endswith('.torrent'):
uripath = uri.path.strip('.torrent')
else:
uripath = uri.path
propParseResult = urlparse.ParseResult(uriScheme, uri.netloc,
uripath + '.prop', uri.params,
uri.query, uri.fragment)
propUri = urlparse.urlunparse(propParseResult)
uriDict['propUri'] = propUri
# calculate the torrent url
# append path with .torrent
torrentParseResult = urlparse.ParseResult(uriScheme, uri.netloc,
uripath + '.torrent',
uri.params, uri.query,
uri.fragment)
torrentUri = urlparse.urlunparse(torrentParseResult)
uriDict['torrentUri'] = torrentUri
return uriDict
def googlepay_info2u8(self, context, meta, appid, packagename, productid,
token, userid, app_order_id, amount, game_callback_url,
good_name, passthrough, platform):
pay_channel = '96'
context['pay_channel'] = pay_channel # google支付，代码96
event_name = settings.API_IMPORTANT_EVENTS.ORDER_CREATED
try:
user = User.objects.get(id=userid)
app = App.objects.get(appid=appid)
except Exception:
context['reason'] = 'user or app error'
_track(event_name, context, meta)
#创建本地订单
order = UserGameOrder.create_order(user=user,
app=app,
game_order_id=app_order_id,
amount=amount,
real_amount=amount,
callback_url=game_callback_url,
good_name=good_name,
passthrough=passthrough,
platform=platform,
pay_channel=pay_channel)
# 本地系统为本订单生成trade id
trade_id = uuid.uuid4().get_hex()
order.trade_id = trade_id
order.order_status = "I" # 发起支付接口，状态设置为I：待支付
order.save()
event_name = settings.API_IMPORTANT_EVENTS.GOOGLE_PAY_VERIFY_INFO
try:
servertoken = googlepay_token_refresh(appid)
except Exception:
event_name = settings.API_IMPORTANT_EVENTS.GOOGLE_PAY_REFRESH_TOKEN
context['reason'] = 'google refresh token error'
_track(event_name, context, meta)
try:
purchaseState = googlepay_product_status(packagename, productid, token,
servertoken)
except Exception:
event_name = settings.API_IMPORTANT_EVENTS.GOOGLE_PAY_VERIFY_ERROR
context['reason'] = 'google server connect error'
_track(event_name, context, meta)
else:
if str(purchaseState) == '0':
event_name = settings.API_IMPORTANT_EVENTS.REUQEST_U8
# 只有在订单状态是购买成功的情况下，通知U8服务器
request_args = get_callback_arg_tuples(order,
others=[('ProductID', '')])
request_query_str = '&'.join(
['='.join(item) for item in request_args])
pay_callback_url = order.app.pay_callback_url
parsed_u8_callback_url = urlparse.urlparse(pay_callback_url)
new_u8_parsed_callback_url = urlparse.ParseResult(
scheme=parsed_u8_callback_url.scheme,
netloc=parsed_u8_callback_url.netloc,
path=parsed_u8_callback_url.path,
params=parsed_u8_callback_url.params,
query=request_query_str,
fragment=parsed_u8_callback_url.fragment)
new_u8_callback_url = urlparse.urlunparse(
new_u8_parsed_callback_url)
callback_sign = get_signature(order.app.appsecret.encode('utf-8'),
new_u8_callback_url)
request_args.append(('sign', callback_sign))
request_args_map = dict(request_args)
request_obj = urllib2.Request(pay_callback_url)
request_obj.add_data(urllib.urlencode(request_args_map))
try:
response = urllib2.urlopen(request_obj, timeout=6).read()
except Exception:
context['reason'] = 'u8 connect error'
_track(event_name, context, meta)
else:
#若U8返回成功，则修改本地订单状态为S
if json.loads(response)['status'] == 'success':
order.order_status = "S"
order.save()
event_name = settings.API_IMPORTANT_EVENTS.PAY_SUCCESS
_track(event_name, context, meta)
else:
context['reason'] = 'u8 response is failed'
_track(event_name, context, meta)
elif str(purchaseState) == '1':
#订单状态是已取消
context['reason'] = 'order state is cancelled'
_track(event_name, context, meta)
else:
#理论上不会出现这种情况
context['reason'] = 'order state error'
_track(event_name, context, meta)
def pdns_request_handler(domain, result_format, **dynamic_data):
scheme = "https"
netloc = "api.dnsdb.info"
path = "lookup/rrset/name".split('/')
query = ["limit=%d" % (int(dynamic_data['limit']))]
results = {'success': False}
if not config.myConfig['apikey']:
results['error'] = 'No DNSDB key.'
return results
# If 'any' is in rrtypes and anything else too, just default to 'any'
if 'any' in dynamic_data['rrtypes']:
if 'any-dnssec' in dynamic_data['rrtypes']:
dynamic_data['rrtypes'] = ['any', 'any-dnssec']
else:
dynamic_data['rrtypes'] = ['any']
results['data'] = {}
wildcard = "*."
if dynamic_data['absolute']:
wildcard = ""
owner_name = wildcard + urllib.quote(domain)
path.append(owner_name)
for rrtype in dynamic_data['rrtypes']:
local_path = list(path) + [rrtype]
local_path = "/".join(local_path)
local_url = urlparse.ParseResult(scheme,
netloc,
local_path,
"",
"&".join(query),
"")
url = urlparse.urlunparse(local_url)
try:
headers = {'Accept': 'application/json',
'X-API-Key': config.myConfig['apikey']}
r = requests.get(url,
proxies=settings.PROXIES,
headers=headers,
verify=config.myConfig["ssl_verify"])
except Exception as e:
results['error'] = str(e)
return results
if r.status_code not in [200, 404]:
return check_return_code(r)
if r.status_code == 200:
# Each line of the response is an individual JSON blob.
for line in r.text.split('\n'):
# Skip empty lines.
if not line:
continue
try:
tmp = json.loads(line)
except Exception as e:
results['error'] = \
"%s: %s" % (str(e), cgi.escape(line, quote=True))
return results
# Convert epoch timestamps to human readable.
for key in ['time_first', 'time_last']:
if key in tmp:
tmp[key] = time.strftime("%Y-%m-%d %H:%M:%S",
time.gmtime(tmp[key]))
rrtype = tmp['rrtype']
# Strip the MX weight.
if rrtype == 'MX':
tmp['rdata'] = [rd.split()[1] for rd in tmp['rdata']]
if result_format in ['none', 'list']:
if tmp['rrname'][-1] == ".":
tmp['rrname'] = tmp['rrname'][:-1]
for (idx, rdat) in enumerate(list(tmp['rdata'])):
if rdat and rdat[-1] == ".":
tmp['rdata'][idx] = rdat[:-1]
try:
results['data'][rrtype].append(tmp)
except KeyError:
results['data'][rrtype] = [tmp]
results['success'] = True
rate = {'limit': r.headers['X-RateLimit-Limit'],
'remaining': r.headers['X-RateLimit-Remaining'],
'reset': r.headers['X-RateLimit-Reset']}
if rate['reset'] != 'n/a':
rate['reset'] = datetime.datetime.utcfromtimestamp(
float(rate['reset'])).strftime('%Y-%m-%d %H:%M:%S GMT')
results['rate'] = rate
if result_format != 'none':
results = _format_results(results, result_format, dynamic_data)
return results
except exceptions.PermissionDenied, err:
error = unicode(err)
except Exception, err:
print err
logging.critical(unicode(err))
error = _(
'Error uploading file. Please contact the site administrator. Thank you.'
)
if error == '':
result = _('Good')
file_url = file_storage.url(new_file_name)
parsed_url = urlparse.urlparse(file_url)
file_url = urlparse.urlunparse(
urlparse.ParseResult(parsed_url.scheme, parsed_url.netloc,
parsed_url.path, '', '', ''))
else:
result = ''
file_url = ''
return JsonResponse(
{'result': {
'msg': result,
'error': error,
'file_url': file_url,
}})
@require_GET
@login_required
def users(request, course_id):
def strip_path(input_url):
"""srips path, params and hash fragments of the url"""
purl = urlparse.urlparse(input_url)
return urlparse.urlunparse(
urlparse.ParseResult(purl.scheme, purl.netloc, '', '', '', ''))
def parse(self):
url_dict = {}
url_parse = urlparse.urlparse(self.url)
base_url = urlparse.ParseResult(url_parse.scheme, url_parse.netloc,
'/', None, None, None).geturl()
base_url_domain = urlutils.getdomain(self.url)
LOG.info(
_("Url: %(url)s,Get BaseUrl: %(baseurl)s and base_domain:%(basedomain)s"
), {
'url': self.url,
'baseurl': base_url,
'basedomain': base_url_domain
})
soup = BeautifulSoup(self.content)
for a_tag in soup.findAll('a'):
if not a_tag.has_key('href'):
continue
if a_tag['href'].lower().find('javascript') != -1:
continue
if CONF.filter_no_follow and a_tag.has_key('nofollow'):
continue
if CONF.filter_onclick and a_tag.has_key('onclick'):
continue
new_url = a_tag['href']
if base_url and not new_url.startswith("http"):
if new_url.startswith('/'):
new_url = new_url[1:]
new_url = base_url + new_url
ret, reason = self.__filter.filter(new_url)
if ret:
LOG.info(_("Filter Url: %(url)s,Reason: %(reason)s"), {
'url': new_url,
'reason': reason
})
continue
if CONF.extract_indomain_link:
domain = urlutils.getdomain(new_url)
if not domain.lower() == base_url_domain.lower():
LOG.info(_("Filter Url: %(url)s,Reason: NotInDomain"),
{'url': new_url})
continue
if new_url in url_dict.keys():
if not a_tag.string in url_dict[new_url]:
url_dict[new_url].append(a_tag.string)
LOG.debug(
_("Add outlink Text Url: %(url)s,value: %(value)s"), {
'url': new_url,
'value': url_dict[new_url]
})
else:
l = list()
l.append(a_tag.string)
url_dict[new_url] = l
LOG.debug(_("Extract Outlink: url: %(url)s,text: %(text)s "), {
'url': new_url,
'text': a_tag.string
})
for key, value in url_dict.iteritems():
ol = OutLink(url=key, text='$@$'.join(value))
self.outlinks.append(ol)
def iap_verify_receipt(self, verify_from, appid, amount, app_order_id,
good_name, pay_channel, userid, **kwargs):
'''使用rawdata作为凭证请求苹果的服务器，获取支付回执信息'''
args_string = '\t'.join([
verify_from, appid, amount, app_order_id, good_name, pay_channel,
userid
]) # for logging
raw_data = kwargs.get('raw_data')
raw_data_digest = hashlib.sha1(
raw_data).hexdigest() # 用于判断是否重复使用了一个支付凭证，如果重复直接拒绝
iap_receipt_history, created = IAPReceiptHistory2.objects.get_or_create(
iap_digest=raw_data_digest)
# 本处理逻辑中涉及到的重要事件
IAP_VERIFY_ERROR = settings.API_IMPORTANT_EVENTS.IAP_VERIFY_ERROR
IAP_VERIFY_INFO = settings.API_IMPORTANT_EVENTS.IAP_VERIFY_INFO
REUQEST_U8 = settings.API_IMPORTANT_EVENTS.REUQEST_U8
raw_args = {
'appid': appid,
'amount': amount,
'app_order_id': app_order_id,
'good_name': good_name,
'pay_channel': pay_channel,
'userid': userid,
}
context = {}
context['args_map'] = raw_args
context['pay_channel'] = pay_channel
if (not created) and (iap_receipt_history.state == 1):
context['reason'] = 'duplicated receipt'
_track(IAP_VERIFY_ERROR, context)
return
if verify_from != '1':
context['reason'] = 'verify_from should be 1'
_track(IAP_VERIFY_ERROR, context)
return
try:
IS_SANDBOX = True
response = None
try:
with itunesiap.env.review:
response = itunesiap.verify(raw_data)
except itunesiap.exceptions.InvalidReceipt as e:
context['reason'] = 'exception when request iap endpoint %s' % str(
e)
_track(IAP_VERIFY_ERROR, context)
if not response:
context['reason'] = 'response from iap is none'
_track(IAP_VERIFY_ERROR, context)
return
if response.status == 0:
IS_SANDBOX = response['environment'].lower() == 'sandbox'
NEED_NOTIFY = True
try:
app = App.objects.get(appid=appid)
except App.DoesNotExist:
context['reason'] = 'app not exists %s' % appid
_track(IAP_VERIFY_ERROR, context)
return
###### 校验包名
try:
response.receipt.in_app.sort(
key=lambda x: int(getattr(x, 'original_purchase_date_ms')))
last_in_app = response.receipt.last_in_app
ios_receipt_logger.info('{}\t{}\t{}\t{}'.format(
raw_data_digest, raw_data, args_string,
response)) # 记录支付凭证摘要何其原始值的对应关系到日志
# 通过original_transaction_id防止重复发货
try:
_original_transaction_id = getattr(
last_in_app, 'original_transaction_id')
except:
_original_transaction_id = None
original_transaction_ids = IAPReceiptHistory2.objects.filter(
original_transaction_id=_original_transaction_id)
if original_transaction_ids.exists():
context['reason'] = 'duplicated original_transaction_id'
context[
'original_transaction_id'] = _original_transaction_id
_track(IAP_VERIFY_ERROR, context)
return
# 通过original_transaction_id防止重复发货 END
except IndexError:
context['reason'] = 'no last_in_app found'
_track(IAP_VERIFY_ERROR, context)
return
try:
bundle_id = response.receipt['bundle_id']
except:
bundle_id = None
if not bundle_id:
try:
bundle_id = last_in_app.bid
except AttributeError as _:
bundle_id = None
if not bundle_id:
context['reason'] = 'bundle_id is empty'
_track(IAP_VERIFY_ERROR, context)
return
else:
package_names = app.package_names
try:
package_names_info = json.loads(package_names)
except Exception as e:
context['reason'] = 'bundle_id not configured properly'
_track(IAP_VERIFY_ERROR, context)
return
if bundle_id not in package_names_info: # 首先，确保包名是合法的
context['reason'] = 'invalid bundle_id'
_track(IAP_VERIFY_ERROR, context)
return
else:
# 包名（bundle_id）确定是合法的，就要再进一步从product_id（bundleid-currency-goodid-realmoney）中获取到，此支付订单的真实的订单价格
SEP = '_'
product_id = last_in_app.product_id
try:
_bundleid, _currency, _goodid, realmoney = product_id.split(
SEP)
except:
context[
'reason'] = 'cannot extract the 4 parts from product_id %s' % product_id
_track(IAP_VERIFY_ERROR, context)
return
if bundle_id != _bundleid:
context[
'info'] = 'bundle_id from iap mismatch bundle_id extracted from product_id %s %s' % (
bundle_id, _bundleid)
_track(IAP_VERIFY_INFO, context)
try:
realmoney = float(realmoney) # 单位为元
real_amount = int(realmoney * 100) # 元转换为分
except Exception as _:
context[
'reason'] = 'error when reading realmony from product_id'
_track(IAP_VERIFY_ERROR, context)
return
###### 校验包名 END
###### 校验包的审核状态，如果过审，则禁用沙箱支付
package_online = package_names_info[bundle_id]['production'] == '1'
if not IS_SANDBOX:
order_status = 'S'
else:
if not package_online:
order_status = 'SS'
else:
NEED_NOTIFY = False # 如果已经上线，禁用沙箱支付
order_status = 'E' # 同时将订单状态设置为异常
###### 校验包的审核状态，如果过审，则禁用沙箱支付 END
try:
user = User.objects.get(id=userid)
except User.DoesNotExist:
context['reason'] = 'user not exists'
_track(IAP_VERIFY_ERROR, context)
return
if pay_channel != '99': # 苹果iTunes支付
context['reason'] = 'invalid pay_channel'
_track(IAP_VERIFY_ERROR, context)
return
orders = UserGameOrder.objects.filter(game_order_id=app_order_id)
if not orders.exists(): # 如果订单不存在，创建订单
platform = 2 # 手游
passthrough = kwargs.get('passthrough', '')
game_callback_url = kwargs.get('game_callback_url', '')
# 创建本地订单
order = UserGameOrder.create_order(
user=user,
real_amount=real_amount,
currency=_currency,
app=app,
game_order_id=app_order_id,
amount=amount,
callback_url=game_callback_url,
good_name=good_name,
passthrough=passthrough,
platform=platform,
pay_channel=pay_channel)
# 本地系统为本订单生成trade id
order.trade_id = uuid.uuid4().get_hex()
else:
order = orders[0]
order.order_status = order_status # 根据上下文信息修改订单状态
try: # 保存订单
order.save()
# 防止应用内购买重复刷单状态更新
def _getattr(name):
try:
return getattr(last_in_app, name)
except AttributeError:
return None
attr_names = [
'quantity', 'product_id', 'transaction_id',
'purchase_date_ms', 'original_transaction_id',
'original_purchase_date_ms'
]
attr_values = map(_getattr, attr_names)
for item in zip(attr_names, attr_values):
setattr(iap_receipt_history, item[0], item[1])
iap_receipt_history.bundle_id = bundle_id
iap_receipt_history.trade_id = order.trade_id
iap_receipt_history.is_sandbox = IS_SANDBOX
iap_receipt_history.state = 0 # 当前处于未验证状态
iap_receipt_history.save()
# 防止应用内购买重复刷单状态更新 END
except Exception as save_exc:
context['info'] = 'failed to create local order'
_track(IAP_VERIFY_INFO, context)
# 虽然本地订单保存不成功，但是还是要通知U8服务器，故此处不返回
if not NEED_NOTIFY:
context[
'reason'] = 'sandbox receipt trying to buy in production envronment'
_track(IAP_VERIFY_ERROR, context)
return
request_args = get_callback_arg_tuples(order,
others=[
('ProductID',
product_id)
]) # 获取回调参数，用于请求U8服务器
request_query_str = '&'.join(
['='.join(item) for item in request_args])
pay_callback_url = app.pay_callback_url
context['pay_callback_url'] = pay_callback_url # 日志记录
parsed_u8_callback_url = urlparse.urlparse(pay_callback_url)
new_u8_parsed_callback_url = urlparse.ParseResult(
scheme=parsed_u8_callback_url.scheme,
netloc=parsed_u8_callback_url.netloc,
path=parsed_u8_callback_url.path,
params=parsed_u8_callback_url.params,
query=request_query_str,
fragment=parsed_u8_callback_url.fragment)
new_u8_callback_url = urlparse.urlunparse(
new_u8_parsed_callback_url)
callback_sign = get_signature(app.appsecret.encode('utf-8'),
new_u8_callback_url)
request_args.append(('Sign', callback_sign))
args_map = dict(request_args)
request_obj = urllib2.Request(pay_callback_url) # 创建请求对象
request_obj.add_data(urllib.urlencode(args_map)) # 添加请求参数
response = urllib2.urlopen(
request_obj, timeout=settings.PAY_CALLBACK_TIMEOUT).read()
response_map = json.loads(response)
context['response_map'] = response_map # 日志记录
if response_map['status'] == 'success':
_track(REUQEST_U8, context)
iap_receipt_history.state = 1 # 标记为验证成功
iap_receipt_history.save()
set_user_ispay_cache(app.appid, user.id, real_amount) # 设置付费标记
event_name = settings.API_IMPORTANT_EVENTS.PAY_SUCCESS
_track(event_name, context)
else:
iap_receipt_history.state = 2 # 标记为验证失败
iap_receipt_history.save()
raise U8ResponseException(
response_map['description']) # 向外层传播本异常，以引入重试机制
else:
context['reason'] = 'iap endpoint return none zero code'
_track(IAP_VERIFY_ERROR, context)
except Exception as ee:
# 有异常的情况下，重试机制介入
raise self.retry(
exc=ee,
max_retries=settings.CELERY_TASK_RETRY_POLICY_MAX_RETRIES,
countdown=settings.CELERY_TASK_RETRY_POLICY[self.request.retries])
def mod_repo(repo, **kwargs):
'''
Modify one or more values for a repo. If the repo does not exist, it will
be created, so long as the following values are specified:
repo or alias
alias by which the zypper refers to the repo
url or mirrorlist
the URL for zypper to reference
enabled
enable or disable (True or False) repository,
but do not remove if disabled.
refresh
enable or disable (True or False) auto-refresh of the repository.
cache
Enable or disable (True or False) RPM files caching.
gpgcheck
Enable or disable (True or False) GOG check for this repository.
Key/Value pairs may also be removed from a repo's configuration by setting
a key to a blank value. Bear in mind that a name cannot be deleted, and a
url can only be deleted if a mirrorlist is specified (or vice versa).
CLI Examples:
.. code-block:: bash
salt '*' pkg.mod_repo alias alias=new_alias
salt '*' pkg.mod_repo alias url= mirrorlist=http://host.com/
'''
repos_cfg = _get_configured_repos()
added = False
# An attempt to add new one?
if repo not in repos_cfg.sections():
url = kwargs.get("url", kwargs.get("mirrorlist"))
if not url:
raise CommandExecutionError(
'Repository "{0}" not found and no URL passed to create one.'.format(repo))
if not urlparse.urlparse(url).scheme:
raise CommandExecutionError(
'Repository "{0}" not found and passed URL looks wrong.'.format(repo))
# Is there already such repo under different alias?
for alias in repos_cfg.sections():
repo_meta = _get_repo_info(alias, repos_cfg=repos_cfg)
# Complete user URL, in case it is not
new_url = urlparse.urlparse(url)
if not new_url.path:
new_url = urlparse.ParseResult(scheme=new_url.scheme, # pylint: disable=E1123
netloc=new_url.netloc,
path='/',
params=new_url.params,
query=new_url.query,
fragment=new_url.fragment)
base_url = urlparse.urlparse(repo_meta["baseurl"])
if new_url == base_url:
raise CommandExecutionError(
'Repository "{0}" already exists as "{1}".'.format(repo, alias))
# Add new repo
doc = None
try:
# Try to parse the output and find the error,
# but this not always working (depends on Zypper version)
doc = dom.parseString(__salt__['cmd.run'](("zypper -x ar {0} '{1}'".format(url, repo)),
output_loglevel='trace'))
except Exception:
# No XML out available, but it is still unknown the state of the result.
pass
if doc:
msg_nodes = doc.getElementsByTagName("message")
if msg_nodes:
msg_node = msg_nodes[0]
if msg_node.getAttribute("type") == "error":
raise CommandExecutionError(msg_node.childNodes[0].nodeValue)
# Verify the repository has been added
repos_cfg = _get_configured_repos()
if repo not in repos_cfg.sections():
raise CommandExecutionError(
'Failed add new repository "{0}" for unknown reason. Please look into Zypper logs.'.format(repo))
added = True
# Modify added or existing repo according to the options
cmd_opt = []
if "enabled" in kwargs:
cmd_opt.append(kwargs["enabled"] and "--enable" or "--disable")
if "refresh" in kwargs:
cmd_opt.append(kwargs["refresh"] and "--refresh" or "--no-refresh")
if "cache" in kwargs:
cmd_opt.append(kwargs["cache"] and "--keep-packages" or "--no-keep-packages")
if "gpgcheck" in kwargs:
cmd_opt.append(kwargs["gpgcheck"] and "--gpgcheck" or "--no-gpgcheck")
if cmd_opt:
__salt__['cmd.run'](("zypper -x mr {0} '{1}'".format(' '.join(cmd_opt), repo)),
output_loglevel='trace')
# If repo nor added neither modified, error should be thrown
if not added and not cmd_opt:
raise CommandExecutionError('Modification of the repository "{0}" was not specified.'.format(repo))
return {}
def mk_url(self, **params):
query = dict(urlparse.parse_qsl(self.url_params.query))
query.update(params)
prlist = list(self.url_params)
prlist[4] = urllib.urlencode(query)
return urlparse.ParseResult(*prlist).geturl()
def CreateFirstSearchPageUrl(self):
query_string = {'q': self.keyword, 'count': '50'}
url = urlparse.ParseResult('https', self.hostname, '/search', '',
urllib.urlencode(query_string), '')
return url.geturl()
