def upload_attachment_url(self, url, urlname, model, id, callback):
Model = request.session.model('ir.attachment')
out = """<script language="javascript" type="text/javascript">
var win = window.top.window;
win.jQuery(win).trigger(%s, %s);
</script>"""
try:
#url = "http://"+url;
my_url = urlparse(url)
if not my_url.scheme:
my_url = urlparse('%s%s' % ('http://', url))
attachment_id = Model.create({
'name': urlname,
'type': 'url',
'url': my_url.geturl(),
'res_id': int(id),
'res_model': model,
}, request.context)
args = {
'name': urlname,
'url': my_url.geturl(),
'id': attachment_id,
'type': 'url',
}
except Exception:
args = {'error': "Something horrible happened"}
_logger.exception("Fail to upload attachment %s" % urlname)
return out % (simplejson.dumps(callback), simplejson.dumps(args))
def auth(email, password, client_id, scope):
def split_key_value(kv_pair):
kv = kv_pair.split("=")
return kv[0], kv[1]
# Authorization form
def auth_user(email, password, client_id, scope, opener):
response = opener.open(
"http://oauth.vk.com/oauth/authorize?" + \
"redirect_uri=http://oauth.vk.com/blank.html&response_type=token&" + \
"client_id=%s&scope=%s&display=wap" % (client_id, ",".join(scope))
)
doc = response.read()
parser = FormParser()
parser.feed(doc)
parser.close()
if not parser.form_parsed or parser.url is None or "pass" not in parser.params or \
"email" not in parser.params:
raise RuntimeError("Something wrong")
parser.params["email"] = email
parser.params["pass"] = password
if parser.method == "POST":
response = opener.open(parser.url, urllib.urlencode(parser.params))
else:
raise NotImplementedError("Method '%s'" % parser.method)
return response.read(), response.geturl()
# Permission request form
def give_access(doc, opener):
parser = FormParser()
parser.feed(doc)
parser.close()
if not parser.form_parsed or parser.url is None:
raise RuntimeError("Something wrong")
if parser.method == "POST":
response = opener.open(parser.url, urllib.urlencode(parser.params))
else:
raise NotImplementedError("Method '%s'" % parser.method)
return response.geturl()
if not isinstance(scope, list):
scope = [scope]
opener = build_opener(
HTTPCookieProcessor(cookielib.CookieJar()),
HTTPRedirectHandler())
doc, url = auth_user(email, password, client_id, scope, opener)
if urlparse(url).path != "/blank.html":
# Need to give access to requested scope
url = give_access(doc, opener)
if urlparse(url).path != "/blank.html":
raise RuntimeError("Expected success here")
answer = dict(split_key_value(kv_pair) for kv_pair in urlparse(url).fragment.split("&"))
if "access_token" not in answer or "user_id" not in answer:
raise RuntimeError("Missing some values in answer")
return answer["access_token"], answer["user_id"]
def _determineProtocol(self, url):
"""
Determine the protocol to be used e.g ftp,cifs,rsync,...
"""
elements = urllib.urlparse(url)
j.logger.log('Determined protocol: %s' % str(elements.scheme))
return elements.scheme
def make_request(self,
url,
method='GET',
query=None,
data=None,
headers=None):
url = urlparse.urljoin(self.config.url, url)
pieces = urlparse.urlsplit(url)
local_url = pieces.path
if pieces.query or query:
local_url += '?' + pieces.query
if query:
if not local_url.endswith('?'):
local_url += '&'
local_url += urllib.urlencode(query)
headers = self.make_headers(headers)
body = None
if data is not None:
body = StringIO(urllib.urlparse(data))
headers['Content-Type'] = 'application/x-www-form-urlencoded'
key = (url, local_url, method, tuple(headers.items()))
if key in self._request_cache:
return self._request_cache[key]
try:
con = self.make_connection(pieces.scheme, pieces.netloc)
con.request(method, local_url, body, headers)
rv = Response(con)
except IOError:
rv = Response(None)
self._request_cache[key] = rv
return rv
def migrate_url(apps, schema_editor):
Domain = apps.get_model("projects", "Domain")
Domain.objects.filter(count=0).delete()
for domain in Domain.objects.all():
if domain.project.superprojects.count() or domain.project.main_language_project:
print("{project} is a subproject or translation. Deleting domain.".format(
project=domain.project.slug))
domain.delete()
continue
parsed = urlparse(domain.url)
if parsed.scheme or parsed.netloc:
domain_string = parsed.netloc
else:
domain_string = parsed.path
try:
domain.domain = domain_string
domain.save()
print(u"Added {domain} from {url}".format(url=domain.url, domain=domain_string))
except Exception as e:
print(e)
print(u"Failed {domain} from {url}".format(url=domain.url, domain=domain_string))
dms = Domain.objects.filter(domain=domain_string).order_by('-count')
if dms.count() > 1:
for dm in list(dms)[1:]:
dm.delete()
def ask_resolution(name,results):
ans=None
while ans is None:
print "0: None"
for i in range(0,len(results)):
print "%d: %s (%s)" % ((i+1), results[i]['label'], results[i]['uri'])
line=sys.stdin.readline()
line.strip()
if len(line)==0:
return None
try:
n=int(line)
if n==0:
return None
n -= 1
if n < len(results):
return results[n]['uri']
except ValueError:
print >>sys.stderr, "Not a number"
try:
url=urlparse(line)
if line.startswith('http://dbpedia.org/'):
return line
print >>sys.stderr, "Resource should start with http://dbpedia.org: %s" % line
except:
print >>sys.stderr, "Not an url: %s" % line
def migrate_url(apps, schema_editor):
Domain = apps.get_model("projects", "Domain")
Domain.objects.filter(count=0).delete()
for domain in Domain.objects.all():
if domain.project.superprojects.count(
) or domain.project.main_language_project:
print("{project} is a subproject or translation. Deleting domain.".
format(project=domain.project.slug))
domain.delete()
continue
parsed = urlparse(domain.url)
if parsed.scheme or parsed.netloc:
domain_string = parsed.netloc
else:
domain_string = parsed.path
try:
domain.domain = domain_string
domain.save()
print(u"Added {domain} from {url}".format(url=domain.url,
domain=domain_string))
except Exception as e:
print(e)
print(u"Failed {domain} from {url}".format(url=domain.url,
domain=domain_string))
dms = Domain.objects.filter(domain=domain_string).order_by('-count')
if dms.count() > 1:
for dm in list(dms)[1:]:
dm.delete()
def __init__(self, address, username=None, password=None, debug=False):
"""
Constructs a new EventHubClient with the given address URL.
:param address: The full URI string of the Event Hub. This can optionally
include URL-encoded access name and key.
:type address: str
:param username: The name of the shared access policy. This must be supplied
if not encoded into the address.
:type username: str
:param password: The shared access key. This must be supplied if not encoded
into the address.
:type password: str
:param debug: Whether to output network trace logs to the logger. Default
is `False`.
:type debug: bool
"""
self.container_id = "eventhub.pysdk-" + str(uuid.uuid4())[:8]
self.address = urlparse(address)
url_username = unquote_plus(
self.address.username) if self.address.username else None
username = username or url_username
url_password = unquote_plus(
self.address.password) if self.address.password else None
password = password or url_password
if not username or not password:
raise ValueError("Missing username and/or password.")
auth_uri = "sb://{}{}".format(self.address.hostname, self.address.path)
self.auth = self._create_auth(auth_uri, username, password)
self.connection = None
self.debug = debug
self.clients = []
self.stopped = False
log.info("{}: Created the Event Hub client".format(self.container_id))
def get_from_iex_v1(
url,
verbose=False):
"""get_from_iex_v1
Get data from the IEX Trading API (v1)
https//api.iextrading.com/1.0/
:param url: IEX V1 Resource URL
:param verbose: optional - bool turn on logging
"""
url = (
f'{iex_consts.IEX_URL_BASE_V1}{url}')
resp = requests.get(
urlparse(url).geturl(),
proxies=iex_consts.IEX_PROXIES)
if resp.status_code == 200:
res_data = resp.json()
if verbose:
proxy_str = ''
if iex_consts.IEX_PROXIES:
proxy_str = (
f'proxies={iex_consts.IEX_PROXIES} ')
log.info(
f'IEXAPI_V1 - url={url} '
f'{proxy_str}'
f'status_code={resp.status_code} '
f'data={ae_consts.ppj(res_data)}')
return res_data
raise Exception(
f'Failed to get data from IEX V1 API with '
f'function=get_from_iex_v1 '
f'url={url} which sent '
f'response {resp.status_code} - {resp.text}')
def _make_request(method,
bucket,
key,
body=None,
authenticated=False,
response_headers=None,
request_headers=None,
expires_in=100000,
path_style=True,
timeout=None):
"""
issue a request for a specified method, on a specified <bucket,key>,
with a specified (optional) body (encrypted per the connection), and
return the response (status, reason).
If key is None, then this will be treated as a bucket-level request.
If the request or response headers are None, then default values will be
provided by later methods.
"""
if not path_style:
conn = bucket.connection
request_headers['Host'] = conn.calling_format.build_host(
conn.server_name(), bucket.name)
if authenticated:
urlobj = None
if key is not None:
urlobj = key
elif bucket is not None:
urlobj = bucket
else:
raise RuntimeError('Unable to find bucket name')
url = urlobj.generate_url(expires_in,
method=method,
response_headers=response_headers,
headers=request_headers)
o = urlparse(url)
path = o.path + '?' + o.query
else:
bucketobj = None
if key is not None:
path = '/{obj}'.format(obj=key.name)
bucketobj = key.bucket
elif bucket is not None:
path = '/'
bucketobj = bucket
else:
raise RuntimeError('Unable to find bucket name')
if path_style:
path = '/{bucket}'.format(bucket=bucketobj.name) + path
return _make_raw_request(host=s3.main.host,
port=s3.main.port,
method=method,
path=path,
body=body,
request_headers=request_headers,
secure=s3.main.is_secure,
timeout=timeout)
def _getSourceHandler(self, sourcepath, is_dir=False,recursive=True, tempdir=j.dirs.tmpDir, Atype='copy'):
"""
Handle all protocol related stuff
Returns a dict with the src and dst
"""
src_proto = self._determineProtocol(sourcepath)
if(src_proto == "cifs" or src_proto == "smb"):
src_elements = self._parseCifsURL(sourcepath)
else:
src_elements = urllib.urlparse(sourcepath)
j.logger.log('PARSING SRC RETURNED %s' %str(src_elements))
# Determine the object we need to call
j.logger.log("_getSourceHandler: source protocol [%s]" % (src_proto))
# for the source
if(src_proto == "cifs" or src_proto == "smb"):
src_fs = CifsFS('src',server=src_elements.hostname,share=src_elements.path,username=src_elements.username,password=src_elements.password,is_dir=is_dir,recursive=recursive,tempdir=tempdir, Atype=Atype)
elif (src_proto == "ftp"):
src_fs = FtpFS('src',server=src_elements.hostname,path=src_elements.path,username=src_elements.username,password=src_elements.password,is_dir=is_dir,recursive=recursive,tempdir=tempdir, Atype=Atype)
elif (src_proto == 'file'):
src_fs = FileFS('src',path=src_elements.path,is_dir=is_dir,recursive=recursive,tempdir=tempdir, Atype=Atype)
elif (src_proto == 'http'):
src_fs = HttpFS('src', server=src_elements.netloc,path=src_elements.path,tempdir=tempdir, Atype=Atype)
elif (src_proto == 'sftp'):
src_fs = SshFS('src', server=src_elements.hostname,directory=src_elements.path,username=src_elements.username,password=src_elements.password,is_dir=is_dir,recursive=recursive,tempdir=tempdir, Atype=Atype)
else:
q.eventhandler.raiseError('Unsupported protocol [%s] for the sourcepath [%s]'%(src_proto, sourcepath))
return src_fs
def __init__(self, ftpURL, mode=''):
parse = urlparse(ftpURL)
if parse[0] != 'ftp':
raise IOError("error reading %s: malformed ftp URL" % ftpURL)
# ftp URL; syntax: ftp://[username:password@]hostname/filename
self.mode = mode
authIndex = parse[1].find('@')
if authIndex == -1:
auth = None
ftphost = parse[1]
else:
auth = parse[1][:authIndex]
ftphost = parse[1][authIndex + 1:]
self.ftp = ftplib.FTP(ftphost)
if auth is None:
self.ftp.login()
else:
# the URL has username/password
pwdIndex = auth.find(':')
if pwdIndex == -1:
raise IOError("error reading %s: malformed ftp URL" % ftpURL)
user = auth[:pwdIndex]
password = auth[pwdIndex + 1:]
self.ftp.login(user, password)
self.path = parse[2][1:]
self.filename = os.path.basename(self.path)
self.dirname = os.path.dirname(self.path)
self.isConnectionOpen = 1
self.currentLine = 0
def make_request(self, url, method='GET', query=None, data=None,
headers=None):
url = urlparse.urljoin(self.config.url, url)
pieces = urlparse.urlsplit(url)
local_url = pieces.path
if pieces.query or query:
local_url += '?' + pieces.query
if query:
if not local_url.endswith('?'):
local_url += '&'
local_url += urllib.urlencode(query)
headers = self.make_headers(headers)
body = None
if data is not None:
body = StringIO(urllib.urlparse(data))
headers['Content-Type'] = 'application/x-www-form-urlencoded'
key = (url, local_url, method, tuple(headers.items()))
if key in self._request_cache:
return self._request_cache[key]
try:
con = self.make_connection(pieces.scheme, pieces.netloc)
con.request(method, local_url, body, headers)
rv = Response(con)
except IOError:
rv = Response(None)
self._request_cache[key] = rv
return rv
def solve_cf_challenge(self, resp, headers, **kwargs):
headers = headers.copy()
url = resp.url
parsed = urlparse(url)
domain = parsed.netloc
page = resp.content
kwargs.pop("params", None) # Don't pass on params
try:
# Extract the arithmetic operation
challenge = re.search(r'name="jschl_vc" value="(\w+)"', page).group(1)
builder = re.search(
r"setTimeout\(function\(\){\s+(var t,r,a,f.+?\r?\n[\s\S]+?a\.value =.+?)\r?\n", page
).group(1)
builder = re.sub(r"a\.value =(.+?) \+ .+?;", r"\1", builder)
builder = re.sub(r"\s{3,}[a-z](?: = |\.).+", "", builder)
except AttributeError:
# Something is wrong with the page. This may indicate Cloudflare has changed their
# anti-bot technique. If you see this and are running the latest version,
# please open a GitHub issue so I can update the code accordingly.
raise IOError(
"Unable to parse Cloudflare anti-bots page. Try upgrading cfscrape, or "
"submit a bug report if you are running the latest version."
)
# Lock must be added explicitly, because PyV8 bypasses the GIL
ctxt = execjs.get("PhantomJS")
# Safely evaluate the Javascript expression
answer = str(int(ctxt.eval(builder)) + len(domain))
params = {"jschl_vc": challenge, "jschl_answer": answer}
submit_url = "%s://%s/cdn-cgi/l/chk_jschl" % (parsed.scheme, domain)
headers["Referer"] = url
return requests.get(submit_url, params=params, headers=headers, **kwargs)
def _getDestinationHandler(self,destinationpath,is_dir=False,recursive=True, tempdir=j.dirs.tmpDir, Atype='copy'):
"""
Handle all protocol related stuff
Returns a dict with the src and dst
"""
dst_proto = self._determineProtocol(destinationpath)
if(dst_proto == "cifs" or dst_proto == "smb"):
dst_elements = self._parseCifsURL(destinationpath)
else:
dst_elements = urllib.urlparse(destinationpath)
j.logger.log('PARSING DEST RETURNED %s' %str(dst_elements))
# Determine the object we need to call
j.logger.log("_getDestinationHandler: destination protocol [%s]" % (dst_proto))
if(dst_proto == "cifs" or dst_proto == "smb"):
dst_fs = CifsFS('dst',server=dst_elements.hostname,share=dst_elements.path,username=dst_elements.username,password=dst_elements.password,is_dir=is_dir,recursive=recursive,tempdir=tempdir, Atype=Atype)
elif (dst_proto == 'ftp'):
dst_fs = FtpFS('dst',server=dst_elements.hostname,path=dst_elements.path,username=dst_elements.username,password=dst_elements.password,is_dir=is_dir,recursive=recursive,tempdir=tempdir, Atype=Atype)
elif (dst_proto == 'file'):
dst_fs = FileFS('dst',path=dst_elements.path,is_dir=is_dir,recursive=recursive,tempdir=tempdir, Atype=Atype)
elif (dst_proto == 'http'):
raise RuntimeError('http as a destination is not supported')
elif (dst_proto == 'sftp'):
dst_fs = SshFS('dst', server=dst_elements.hostname,directory=dst_elements.path,username=dst_elements.username,password=dst_elements.password,is_dir=is_dir,recursive=recursive,tempdir=tempdir, Atype=Atype)
else:
q.eventhandler.raiseError('Unsupported protocol [%s] for the destinationpath [%s]'%(dst_proto, destinationpath))
return dst_fs
def get_video_url(hrefs, session):
"""
这个函数的作用是根据获取到的a链接得到视频真正的下载地址
:param hrefs:
:param session:
:return:
"""
download_video_dict = {}
for href in hrefs:
series_page = session.get(href)
series_page_soup = BeautifulSoup(series_page.text)
video_src = base_url + \
series_page_soup.find('a', {'title': 'Download Video'})['href']
video_get_location = session.get(video_src, allow_redirects=False)
video_location = 'https:' + video_get_location.headers['location']
get_video = requests.get(
video_location, allow_redirects=False, proxies=proxies)
video_download_location = get_video.headers['location']
# 获取文件的文件名
pased_location = urlparse(video_download_location)
print(video_download_location)
with open('url.txt', 'a') as o:
o.write(video_download_location + '\n')
pased_location_query = parse_qs(pased_location.query)
download_video_dict[
pased_location_query['filename'][0]] = video_download_location
def _process_redirect_uri(self, redirect):
redirect_uri = redirect.address.decode('utf-8')
auth_uri, _, _ = redirect_uri.partition("/ConsumerGroups")
self.address = urlparse(auth_uri)
self.auth_uri = "sb://{}{}".format(self.address.hostname, self.address.path)
self.eh_name = self.address.path.lstrip('/')
self.mgmt_target = redirect_uri
def listDir(self, path):
"""
List content of specified path
"""
is_dir = True
recursive = False
j.logger.log("listDir: supplied path is [%s]" % path )
proto = self._determineProtocol(path)
if(proto == "cifs" or proto == "smb"):
path_elements = self._parseCifsURL(path)
j.logger.log('CIFS LISTDIR path_elements: %s' %str(path_elements))
else:
path_elements = urllib.urlparse(path)
# Determine the object we need to call
j.logger.log("listDir: protocol [%s]" % proto )
# for the source
if(proto == "cifs" or proto == "smb"):
fs = CifsFS('src',server=path_elements.hostname,share=path_elements.path,username=path_elements.username,password=path_elements.password,is_dir=is_dir,recursive=recursive)
elif (proto == "ftp"):
fs = FtpFS('src',server=path_elements.hostname,path=path_elements.path,username=path_elements.username,password=path_elements.password,is_dir=is_dir,recursive=recursive)
elif (proto == 'file'):
fs = FileFS('src',path=path_elements.path,is_dir=is_dir,recursive=recursive)
elif (proto == 'sftp'):
fs = SshFS('src', server=path_elements.hostname,directory=path_elements.path,username=path_elements.username,password=path_elements.password,is_dir=is_dir,recursive=recursive)
else:
q.eventhandler.raiseError('Unsupported protocol [%s] for the path [%s]'%(proto, path))
dir_list = fs.list()
fs.cleanup()
return dir_list
def __init__(self, ftpURL, mode=''):
parse = urlparse(ftpURL)
if parse[0] != 'ftp':
raise IOError("error reading %s: malformed ftp URL" % ftpURL)
# ftp URL; syntax: ftp://[username:password@]hostname/filename
self.mode = mode
authIndex = parse[1].find('@')
if authIndex == -1:
auth = None
ftphost = parse[1]
else:
auth = parse[1][:authIndex]
ftphost = parse[1][authIndex+1:]
self.ftp = ftplib.FTP(ftphost)
if auth == None:
self.ftp.login()
else:
# the URL has username/password
pwdIndex = auth.find(':')
if pwdIndex == -1:
raise IOError("error reading %s: malformed ftp URL" % ftpURL)
user = auth[:pwdIndex]
password = auth[pwdIndex+1:]
self.ftp.login(user, password)
self.path = parse[2][1:]
self.filename = os.path.basename(self.path)
self.dirname = os.path.dirname(self.path)
self.isConnectionOpen = 1
self.currentLine = 0
def _determineProtocol(self, url):
"""
Determine the protocol to be used e.g ftp,cifs,rsync,...
"""
elements = urllib.urlparse(url)
j.logger.log('Determined protocol: %s' %str(elements.scheme))
return elements.scheme
def insert_read_only_node(c, p, name):
if name == "":
name = g.app.gui.runOpenFileDialog(
c,
title="Open",
filetypes=[("All files", "*")],
)
c.setHeadString(p, "@read-only %s" % name)
c.redraw()
parse = urlparse(name)
try:
if parse[0] == 'ftp':
f = FTPurl(name) # FTP URL
elif parse[0] == 'http':
f = urlopen(name) # HTTP URL
else:
f = open(name, "r") # local file
g.es("..." + name)
new = f.read()
f.close()
except IOError: # as msg:
# g.es("error reading %s: %s" % (name, msg))
# g.es("...not found: " + name)
c.setBodyString(p, "") # Clear the body text.
return True # Mark the node as changed.
else:
ext = os.path.splitext(parse[2])[1]
if ext.lower() in ['.htm', '.html']:
#@+<< convert HTML to text >>
#@+node:edream.110203113231.895: *3* << convert HTML to text >>
fh = StringIO()
fmt = AbstractFormatter(DumbWriter(fh))
# the parser stores parsed data into fh (file-like handle)
parser = HTMLParser(fmt)
# send the HTML text to the parser
parser.feed(new)
parser.close()
# now replace the old string with the parsed text
new = fh.getvalue()
fh.close()
# finally, get the list of hyperlinks and append to the end of the text
hyperlinks = parser.anchorlist
numlinks = len(hyperlinks)
if numlinks > 0:
hyperlist = ['\n\n--Hyperlink list follows--']
for i in range(numlinks):
hyperlist.append("\n[%d]: %s" %
(i + 1, hyperlinks[i])) # 3/26/03: was i.
new = new + ''.join(hyperlist)
#@-<< convert HTML to text >>
previous = p.b
c.setBodyString(p, new)
changed = (g.toUnicode(new) != g.toUnicode(previous))
if changed and previous != "":
g.es("changed: %s" % name) # A real change.
return changed
def _build_uri(address, entity):
parsed = urlparse(address)
if parsed.path:
return address
if not entity:
raise ValueError("No EventHub specified")
address += "/" + str(entity)
return address
def _checklogin(request, *args, **kwargs):
if request.user.is_active and request.user.is_staff:
return view_func(request, *args, **kwargs)
# If the login url is the same scheme and net location then just
# use the path as the "next" url.
path = request.build_absolute_uri()
login_scheme, login_netloc = urllib.urlparse(login_url)[:2]
current_scheme, current_netloc = urllib.urlparse(path)[:2]
if ((not login_scheme or login_scheme == current_scheme) and
(not login_netloc or login_netloc == current_netloc)):
path = request.get_full_path()
messages.warning(request, _("You must log in to access this page"))
from django.contrib.auth.views import redirect_to_login
return redirect_to_login(path, login_url, REDIRECT_FIELD_NAME)
def _getAuthHeaders(self, method, url, params=None, headers=None):
"""
api request http headers
"""
headers = headers or {}
params = params or {}
if self._isCloudUser == False:
return headers
urlResult = urlparse(url)
for kv in urlResult.query.strip().split('&'):
if kv:
k, v = kv.split('=')
params[k] = v
# UTC timestamp
timestamp = datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')
headers['Host'] = urlResult.hostname
headers['x-bce-date'] = timestamp
version, expire = '1', '1800'
# 1 Generate SigningKey
val = "bce-auth-v%s/%s/%s/%s" % (version, self._apiKey, timestamp,
expire)
signingKey = hmac.new(self._secretKey.encode('utf-8'),
val.encode('utf-8'), hashlib.sha256).hexdigest()
# 2 Generate CanonicalRequest
# 2.1 Genrate CanonicalURI
canonicalUri = quote(urlResult.path)
# 2.2 Generate CanonicalURI: not used here
# 2.3 Generate CanonicalHeaders: only include host here
canonicalHeaders = []
for header, val in headers.items():
canonicalHeaders.append(
'%s:%s' %
(quote(header.strip(), '').lower(), quote(val.strip(), '')))
canonicalHeaders = '\n'.join(sorted(canonicalHeaders))
# 2.4 Generate CanonicalRequest
canonicalRequest = '%s\n%s\n%s\n%s' % (
method.upper(), canonicalUri, '&'.join(
sorted(urlencode(params).split('&'))), canonicalHeaders)
# 3 Generate Final Signature
signature = hmac.new(signingKey.encode('utf-8'),
canonicalRequest.encode('utf-8'),
hashlib.sha256).hexdigest()
headers['authorization'] = 'bce-auth-v%s/%s/%s/%s/%s/%s' % (
version, self._apiKey, timestamp, expire, ';'.join(
headers.keys()).lower(), signature)
return headers
def action_add_gdrive(self, docs):
"""Adds the Google Drive Document with an ir.attachment record."""
context = self.env.context
if not context.get('active_model'):
return
for doc in docs:
url = urlparse(doc['url'])
if not url.scheme:
url = urlparse('%s%s' % ('http://', url))
for active_id in context.get('active_ids', []):
self.env['ir.attachment'].create({
'name': doc['name'],
'type': 'url',
'url': url.geturl(),
'res_id': active_id,
'res_model': context['active_model'],
})
return {'type': 'ir.actions.act_close_wizard_and_reload_view'}
def is_torrent(self, url):
parse = urlparse(url)
if parse.scheme == "magnet":
return True
if len(parse.path) > 8 and parse.path[-8:] == ".torrent":
return True
if parse.netloc == "cuelgame.net":
return True
return False
def insert_read_only_node (c,p,name):
if name=="":
name = g.app.gui.runOpenFileDialog(c,
title="Open",
filetypes=[("All files", "*")],
)
c.setHeadString(p,"@read-only %s" % name)
c.redraw()
parse = urlparse(name)
try:
if parse[0] == 'ftp':
f = FTPurl(name) # FTP URL
elif parse[0] == 'http':
f = urlopen(name) # HTTP URL
else:
f = open(name,"r") # local file
g.es("..." + name)
new = f.read()
f.close()
except IOError: # as msg:
# g.es("error reading %s: %s" % (name, msg))
# g.es("...not found: " + name)
c.setBodyString(p,"") # Clear the body text.
return True # Mark the node as changed.
else:
ext = os.path.splitext(parse[2])[1]
if ext.lower() in ['.htm', '.html']:
#@+<< convert HTML to text >>
#@+node:edream.110203113231.895: *3* << convert HTML to text >>
fh = StringIO()
fmt = AbstractFormatter(DumbWriter(fh))
# the parser stores parsed data into fh (file-like handle)
parser = HTMLParser(fmt)
# send the HTML text to the parser
parser.feed(new)
parser.close()
# now replace the old string with the parsed text
new = fh.getvalue()
fh.close()
# finally, get the list of hyperlinks and append to the end of the text
hyperlinks = parser.anchorlist
numlinks = len(hyperlinks)
if numlinks > 0:
hyperlist = ['\n\n--Hyperlink list follows--']
for i in range(numlinks):
hyperlist.append("\n[%d]: %s" % (i+1,hyperlinks[i])) # 3/26/03: was i.
new = new + ''.join(hyperlist)
#@-<< convert HTML to text >>
previous = p.b
c.setBodyString(p,new)
changed = (g.toUnicode(new) != g.toUnicode(previous))
if changed and previous != "":
g.es("changed: %s" % name) # A real change.
return changed
def listDir(self, path):
"""
List content of specified path
"""
is_dir = True
recursive = False
j.logger.log("listDir: supplied path is [%s]" % path)
proto = self._determineProtocol(path)
if (proto == "cifs" or proto == "smb"):
path_elements = self._parseCifsURL(path)
j.logger.log('CIFS LISTDIR path_elements: %s' % str(path_elements))
else:
path_elements = urllib.urlparse(path)
# Determine the object we need to call
j.logger.log("listDir: protocol [%s]" % proto)
# for the source
if (proto == "cifs" or proto == "smb"):
fs = CifsFS('src',
server=path_elements.hostname,
share=path_elements.path,
username=path_elements.username,
password=path_elements.password,
is_dir=is_dir,
recursive=recursive)
elif (proto == "ftp"):
fs = FtpFS('src',
server=path_elements.hostname,
path=path_elements.path,
username=path_elements.username,
password=path_elements.password,
is_dir=is_dir,
recursive=recursive)
elif (proto == 'file'):
fs = FileFS('src',
path=path_elements.path,
is_dir=is_dir,
recursive=recursive)
elif (proto == 'sftp'):
fs = SshFS('src',
server=path_elements.hostname,
directory=path_elements.path,
username=path_elements.username,
password=path_elements.password,
is_dir=is_dir,
recursive=recursive)
else:
q.eventhandler.raiseError(
'Unsupported protocol [%s] for the path [%s]' % (proto, path))
dir_list = fs.list()
fs.cleanup()
return dir_list
def uamqp_send_simple():
msg_content = b"Hello world"
parsed_uri = urlparse(uri)
plain_auth = authentication.SASLPlain(parsed_uri.hostname, key_name,
access_key)
uamqp.send_message(uri, msg_content, auth=plain_auth)
print("Message sent!")
def register_node(self, address):
"""
Add a new node to the list of nodes
:param address: <str> Address of node. Eg. 'http://192.168.0.5:5000'
:return: None
"""
parsed_url = urlparse(address)
self.nodes.add(parsed_url.netloc)
def urlformat(url):
'''
策略是构建一个三元组
第一项为url的netloc
第二项为path中每项的拆分长度
第三项为query的每个参数名称(参数按照字母顺序排序，避免由于顺序不同而导致的重复问题)
'''
if urllib.urlparse(url)[2] == '':
url = url + '/'
url_structure = urllib.urlparse(url)
netloc = url_structure[1]
path = url_structure[2]
query = url_structure[4]
temp = (netloc, tuple([len(i) for i in path.split('/')]),
tuple(sorted([i.split('=')[0] for i in query.split('&')])))
# print temp
return temp
def get_host(url, host):
try:
#TODO adapt to urllib py3
# https://stackoverflow.com/questions/12772190/urllib-module-object-is-not-callable
u = urlparse(url)
if host == '127.0.0.1' or host == 'localhost':
nu = u.scheme + '://' + host + ':' + str(u.port) + u.path
return nu
except:
return url
def modify_header(self, *args):
'''Generate the content for the :mailheader:`List-ID` header.
:returns: The ID of the group concatenated with the ``netloc`` of the site
URL.
:rtype: unicode'''
canonicalHost = urlparse(self.groupInfo.siteInfo.url).netloc
retval = '{groupInfo.name} <{groupInfo.id}.{canonicalHost}>'.format(
groupInfo=self.groupInfo, canonicalHost=canonicalHost)
return retval
def __init__(self,
address,
username=None,
password=None,
debug=False,
http_proxy=None,
auth_timeout=60):
"""
Constructs a new EventHubClient with the given address URL.
:param address: The full URI string of the Event Hub. This can optionally
include URL-encoded access name and key.
:type address: str
:param username: The name of the shared access policy. This must be supplied
if not encoded into the address.
:type username: str
:param password: The shared access key. This must be supplied if not encoded
into the address.
:type password: str
:param debug: Whether to output network trace logs to the logger. Default
is `False`.
:type debug: bool
:param http_proxy: HTTP proxy settings. This must be a dictionary with the following
keys: 'proxy_hostname' (str value) and 'proxy_port' (int value).
Additionally the following keys may also be present: 'username', 'password'.
:type http_proxy: dict[str, Any]
:param auth_timeout: The time in seconds to wait for a token to be authorized by the service.
The default value is 60 seconds. If set to 0, no timeout will be enforced from the client.
:type auth_timeout: int
"""
self.container_id = "eventhub.pysdk-" + str(uuid.uuid4())[:8]
self.address = urlparse(address)
self.eh_name = self.address.path.lstrip('/')
self.http_proxy = http_proxy
self.mgmt_target = "amqps://{}/{}".format(self.address.hostname,
self.eh_name)
url_username = unquote_plus(
self.address.username) if self.address.username else None
username = username or url_username
url_password = unquote_plus(
self.address.password) if self.address.password else None
password = password or url_password
if not username or not password:
raise ValueError("Missing username and/or password.")
self.auth_uri = "sb://{}{}".format(self.address.hostname,
self.address.path)
self._auth_config = {'username': username, 'password': password}
self.get_auth = functools.partial(self._create_auth)
self.debug = debug
self.auth_timeout = auth_timeout
self.clients = []
self.stopped = False
log.info("%r: Created the Event Hub client", self.container_id)
def action_add_url(self):
"""Adds the URL with the given name as an ir.attachment record."""
if not self._context.get('active_model'):
return
attachment_obj = self.env['ir.attachment']
for form in self:
url = urlparse(form.url)
if not url.scheme:
url = urlparse('%s%s' % ('http://', form.url))
for active_id in self._context.get('active_ids', []):
attachment = {
'name': form.name,
'type': 'url',
'url': url.geturl(),
'res_id': active_id,
'res_model': self._context.get('active_model')
}
attachment_obj.create(attachment)
return False
def format_url(template, params={}, **kwparams):
args = {**get_formatdata(params), **kwparams}
parts = list(urllib.urlparse(template))
for i in range(0, len(parts)):
if i == 4:
parts[i] = QuerystringFormatter().format(parts[i], **args)
else:
parts[i] = format(parts[i], **args)
return urllib.urlunparse(parts)
def _make_url(row, band):
parse = urlparse(
'http://storage.googleapis.com/gcp-public-data-landsat/LC08/01/037/029/'
'LC08_L1TP_037029_20130101_20190131_01_T1/LC08_L1TP_037029_20130101_20190131_01_T1_B9.TIF'
)
base = row.BASE_URL.replace('gs://', '')
path = '{}/{}_{}'.format(base, row.PRODUCT_ID, band)
url = urlunparse([parse.scheme, parse.netloc, path, '', '', ''])
return url
def action_add_gdrive(self, name, url):
"""Adds the Google Drive Document with an ir.attachment record."""
context = self.env.context
if context is None:
context = {}
if not context.get('active_model'):
return
attachment_obj = self.pool['ir.attachment']
url = urlparse(url)
if not url.scheme:
url = urlparse('%s%s' % ('http://', url))
for active_id in context.get('active_ids', []):
attachment = {
'name': name,
'type': 'url',
'url': url.geturl(),
'res_id': active_id,
'res_model': context['active_model'],
}
attachment_obj.create(self.env.cr, context['uid'], attachment, context=context)
return {'type': 'ir.actions.act_close_wizard_and_reload_view'}
def action_add_gdrive(self, name, url):
"""Adds the Google Drive Document with an ir.attachment record."""
context = self.env.context
if context is None:
context = {}
if not context.get("active_model"):
return
attachment_obj = self.pool["ir.attachment"]
url = urlparse(url)
if not url.scheme:
url = urlparse("%s%s" % ("http://", url))
for active_id in context.get("active_ids", []):
attachment = {
"name": name,
"type": "url",
"url": url.geturl(),
"res_id": active_id,
"res_model": context["active_model"],
}
attachment_obj.create(self.env.cr, context["uid"], attachment, context=context)
return {"type": "ir.actions.act_close_wizard_and_reload_view"}
def authenticate(self, host):
output = None
try:
start = self.startup(host)
if not start.url: return None
query = parse.parse_qs(parse.urlparse(start.url).query)
if 'client_id' in query.keys(): cid = query['client_id']
else: return None
uri = '/dialog/authorize'
payload = {'redirect_uri':'https://cms.lingotek.com/tms-ui/html/portal/sso_redirect.html','response_type':'token','client_id':cid}
# r = requests.get(host + uri, headers={'Host': 'cmssso.lingotek.com', 'Referer': 'https://cmssso.lingotek.com/login', 'Cache-Control':'max-age=0', 'Upgrade-Insecure-Requests':'1', 'Cookie':'__ctmid=58220c510010e8c8dc704410; _gat=1; _ga=GA1.2.831256021.1467748163; connect.sid=s%3AxU6QRRV9jDVSX3SeYAOElBOI1Y5HdMRK.yU%2FTgKno2PqlKGljl50dJ8HarhRUT71zT0rF6aniDvw'}, data=payload)
# r = requests.get(host + uri, headers={'Cookie':'connect.sid=s%3Aq4dTUpbJVb8uIgbM7s2T0txtHR6qpkhE.5dFEBdjsPtlcDGgG9MO9yNQMhyrkMpJVjhLH84J2mKI'}, params=payload)
r = requests.get(host + uri, headers={'Cookie': self.cookie}, params=payload)
log_api('GET', uri, r)
# r = requests.get(host + uri, headers=self.headers, params=payload)
fragment = parse.parse_qs(parse.urlparse(r.url).fragment)
if 'access_token' in fragment.keys() and len(fragment['access_token']) > 0: return fragment['access_token'][0]
else: return None
except Exception as e:
print("authenticate", e)
self.handleError()
return None
def scrape_videos(self):
"""Scrape all videos a user is tagged in, and all videos created by the user."""
videos_dir = self._create_dir("facebook", "videos")
my_videos = self.api_request("/me/videos/uploaded")["data"]
video_tags = self.api_request("/me/videos")["data"]
for video in my_videos + video_tags:
name = self._clean(video["name"])
fn, ext = os.path.splitext(urlparse(video["source"]).path)
vurl = video["source"]
filename = os.path.join(videos_dir, "%s%s" % (name, ext))
self.pool.apply_async(save_file, [vurl, filename])
def add_url_params(url, params):
url = unquote(url)
parsed_url = urlparse(url)
get_args = parsed_url.query
parsed_get_args = dict(parse_qsl(get_args))
parsed_get_args.update(params)
encoded_get_args = urlencode(parsed_get_args, doseq=True)
new_url = ParseResult(parsed_url.scheme, parsed_url.netloc,
parsed_url.path, parsed_url.params, encoded_get_args,
parsed_url.fragment).geturl()
return new_url
def _parseCifsURL(self,url):
"""
If URL starts with cifs:// we need to parse it ourselves since urllib does not support this
"""
# Warning: Dirty HACK since urllib does not support cifs/smb but the URI is the same as ftp or http
durl = url
durl = durl.replace("cifs://","ftp://")
durl = durl.replace("smb://","ftp://")
elements = urllib.urlparse(durl)
ret_elements = {}
j.logger.log('_parseCifsURL returned %s' %elements.hostname)
return elements
def _parseCifsURL(self, url):
"""
If URL starts with cifs:// we need to parse it ourselves since urllib does not support this
"""
# Warning: Dirty HACK since urllib does not support cifs/smb but the URI is the same as ftp or http
durl = url
durl = durl.replace("cifs://", "ftp://")
durl = durl.replace("smb://", "ftp://")
elements = urllib.urlparse(durl)
ret_elements = {}
j.logger.log('_parseCifsURL returned %s' % elements.hostname)
return elements
def action_add_url(self, cr, uid, ids, context=None):
"""Adds the URL with the given name as an ir.attachment record."""
if context is None:
context = {}
if not context.get('active_model'):
return
attachment_obj = self.pool['ir.attachment']
for form in self.browse(cr, uid, ids, context=context):
url = urlparse(form.url)
if not url.scheme:
url = urlparse('%s%s' % ('http://', form.url))
for active_id in context.get('active_ids', []):
attachment = {
'name': form.name,
'type': 'url',
'url': url.geturl(),
'user_id': uid,
'res_id': active_id,
'res_model': context['active_model'],
}
attachment_obj.create(cr, uid, attachment, context=context)
return {'type': 'ir.actions.act_close_wizard_and_reload_view'}
def action_add_gdrive(self, docs):
"""Adds the Google Drive Document with an ir.attachment record."""
context = self.env.context
if not context.get('active_model'):
return
for doc in docs:
url = urlparse(doc['url'])
if not url.scheme:
url = urlparse('%s%s' % ('http://', url))
for active_id in context.get('active_ids', []):
self.env['ir.attachment'].create({
'name':
doc['name'],
'type':
'url',
'url':
url.geturl(),
'res_id':
active_id,
'res_model':
context['active_model'],
})
return {'type': 'ir.actions.act_close_wizard_and_reload_view'}
def load_state_dict_from_url(url, model_dir=None, map_location=None, progress=True):
r"""Loads the Torch serialized object at the given URL.
If the object is already present in `model_dir`, it's deserialized and
returned. The filename part of the URL should follow the naming convention
``filename-<sha256>.ext`` where ``<sha256>`` is the first eight or more
digits of the SHA256 hash of the contents of the file. The hash is used to
ensure unique names and to verify the contents of the file.
The default value of `model_dir` is ``$TORCH_HOME/checkpoints`` where
environment variable ``$TORCH_HOME`` defaults to ``$XDG_CACHE_HOME/torch``.
``$XDG_CACHE_HOME`` follows the X Design Group specification of the Linux
filesytem layout, with a default value ``~/.cache`` if not set.
Args:
url (string): URL of the object to download
model_dir (string, optional): directory in which to save the object
map_location (optional): a function or a dict specifying how to remap storage locations (see torch.load)
progress (bool, optional): whether or not to display a progress bar to stderr
Example:
>>> state_dict = torch.hub.load_state_dict_from_url('https://s3.amazonaws.com/pytorch/models/resnet18-5c106cde.pth')
"""
# Issue warning to move data if old env is set
if os.getenv('TORCH_MODEL_ZOO'):
warnings.warn('TORCH_MODEL_ZOO is deprecated, please use env TORCH_HOME instead')
if model_dir is None:
torch_home = _get_torch_home()
model_dir = os.path.join(torch_home, 'checkpoints')
try:
os.makedirs(model_dir)
except OSError as e:
if e.errno == errno.EEXIST:
# Directory already exists, ignore.
pass
else:
# Unexpected OSError, re-raise.
raise
parts = urlparse(url)
filename = os.path.basename(parts.path)
cached_file = os.path.join(model_dir, filename)
if not os.path.exists(cached_file):
sys.stderr.write('Downloading: "{}" to {}\n'.format(url, cached_file))
hash_prefix = HASH_REGEX.search(filename).group(1)
_download_url_to_file(url, cached_file, hash_prefix, progress=progress)
return torch.load(cached_file, map_location=map_location)
def get_success_url(self):
url = None
if self.request.POST.get('next'):
url = self.request.POST.get('next')
elif 'HTTP_REFERER' in self.request.META:
url = self.request.META['HTTP_REFERER']
if url:
# We only allow internal URLs so we see if the url resolves
try:
resolve(urlparse(url).path)
except Http404:
url = None
if url is None:
url = reverse('basket:summary')
return url
def scrape_videos(self):
"""Scrape all tagged videos and uploaded videos"""
videos_dir = self._create_dir("facebook", "videos")
videos = self.api_request("/me/videos/uploaded")
tags = self.api_request("/me/videos")
if not videos or not tags:
print "Error: Could not scrape your movies"
return
for video in videos["data"] + tags["data"]:
name = self._clean(video["name"])
fn, ext = os.path.splitext(urlparse(video["source"]).path)
vurl = video["source"]
filename = os.path.join(videos_dir, "%s%s" % (name, ext))
self.pool.apply_async(save_file, [vurl, filename])
def targeting(request):
if not (request.user and request.user.has_perm("ads.change_targetingoverride")):
return HttpResponseForbidden(
'{"detail": "You do not have permission to perform this action."}'
)
url = request.GET.get("url")
if url is None:
return HttpResponseNotFound()
if url.startswith("http://") or url.startswith("https://"):
parsed = urlparse(url)
url = parsed.path
try:
view, args, kwargs = resolve(url)
except Resolver404:
return HttpResponseNotFound()
req = RequestFactory().get(url)
kwargs['request'] = req
response = view(*args, **kwargs)
if response.status_code != 200:
return HttpResponseNotFound()
targeting = response.context_data.get('targeting', {})
if request.method == "POST":
override_data = json.loads(request.body)
for key, value in targeting.items():
if value == override_data.get(key):
del override_data[key]
override, created = TargetingOverride.objects.get_or_create(url=url)
override.targeting = override_data
override.save()
targeting.update(override_data)
else:
try:
override = TargetingOverride.objects.get(url=url)
targeting.update(override.targeting)
except TargetingOverride.DoesNotExist:
pass
return HttpResponse(json.dumps(targeting), content_type="application/json")
def loadPage(self, url, uri=None, method="GET", params=""):
if not url:
logging.error("Request URL undefined")
return
if not uri:
urlData = urlparse(url)
url = urlData.netloc
uri = urlData.path + '?' + urlData.query
# prepare params, append to uri
if params :
params = urlencode(params)
if method == "GET":
uri += ('?' if uri.find('?') == -1 else '&') + params
params = ""
# insert local cookies in request
headers = {
"Cookie": '; '.join( [ key+'='+self.cookies[key] for key in self.cookies.keys() ] )
}
if method == "POST":
headers["Content-type"] = "application/x-www-form-urlencoded"
#logging.debug("Request URL: %s:/%s > %s # %s", url, uri, unquote(params), headers["Cookie"])
conn = httplib.HTTPSConnection(url)
conn.request(method, uri, params, headers)
response = conn.getresponse()
data = response.read()
conn.close()
# logging.debug("Response : %s > %s", response.status, response.getheaders())
result = Struct(status=response.status, location=response.getheader('location', None), data=data)
# update local cookies
sk = Cookie.SimpleCookie(response.getheader("Set-Cookie", ""))
for key in sk:
self.cookies[key] = sk[key].value
return result
def note(request, pk):
#this isn't right
note = get_object_or_404(Note, id=pk)
#all_notes = Note.objects.all()
notesegment = get_object_or_404(NoteSegment, id=pk)
all_segments = list(note.notesegment_set.all())
#all_segments = NoteSegment.objects.all()
pageURL = request.get_full_path()
urlData = urlparse(pageURL)
theQuery = urlData.query.strip('=')
context = {
'tempvar': theQuery,
'note': note,
'notesegment': notesegment,
'all_segments': all_segments,
}
return render(request, "capitolHoundApp/note.html", context)
def importVolume(self, sourcepath, destinationpath,format='vdi',tempdir=j.dirs.tmpDir):
"""
Import volume from specified source
@param sourcepath: location to import the volume from e.g. ftp://login:[email protected]/myroot/mymachine1/test.vdi, if .vdi.tgz at end then compression will happen automatically
@type sourcepath: string
@param destinationpath: name of the device to import to e.g. E: F on windows, or /dev/sda5 on linux
@type destinationpath: string
@param tempdir: (optional) directory whereto will be exported; default is the default temp-directory as determined by underlying system
@type tempdir: string
"""
prefix = 'file://'
j.logger.log("CloudSystemFS: importVolume source [%s] to path [%s]" % (sourcepath, destinationpath))
if sourcepath.endswith('.tgz'):
compressImage = True
else:
compressImage = False
protocol = self._determineProtocol(sourcepath)
if protocol == "file":
elements = urllib.urlparse(sourcepath)
j.logger.log("Source is a local file:// not running copyFile... for %s" % elements.path)
tmp_inputFileName = elements.path
elif protocol == "smb" or protocol == "cifs":
src_elements = self._parseCifsURL(sourcepath)
src_fs = CifsFS('src',server=src_elements.hostname,share=src_elements.path,username=src_elements.username,password=src_elements.password,is_dir=False,recursive=False,tempdir=tempdir)
tmp_inputFileName = src_fs.download()
j.logger.log("Source is a CIFS/SMB share, not running copyFile,using %s" %tmp_inputFileName)
else:
tmp_inputFileName = j.system.fs.getTempFileName(tempdir)
self.copyFile(sourcepath, ''.join([prefix,tmp_inputFileName]),tempdir=tempdir)
j.cmdtools.disktools.qemu_img.convert(tmp_inputFileName, format, destinationpath, 'raw', compressTargetImage=compressImage)
if not protocol == "file" and not protocol == "smb" and not protocol == "cifs":
j.system.fs.remove(tmp_inputFileName)
elif protocol == "smb" or protocol == "cifs":
src_fs.cleanup()
def do_GET(self):
global ACCESS_TOKEN
self.send_response(200)
self.send_header("Content-type", "text/html")
self.end_headers()
params = parse_qs(urlparse(self.path).query)
ACCESS_TOKEN = params.get('access_token', [None])[0]
if ACCESS_TOKEN:
data = {'scope': AUTH_SCOPE,
'access_token': ACCESS_TOKEN}
expiration = params.get('expires_in', [None])[0]
if expiration:
if expiration == '0':
# this is what's returned when offline_access is requested
data['expires_at'] = 'never'
else:
data['expires_at'] = int(time.time()+int(expiration))
open(ACCESS_TOKEN_FILE,'w').write(json.dumps(data))
self.wfile.write(b(AUTH_SUCCESS_HTML))
else:
self.wfile.write(b('<html><head>'
'<script>location = "?"+location.hash.slice(1);</script>'
'</head></html>'))
def add_headers(self, container):
'''Add the stadard headers to the message
:param container: The email message container to add the headers to.
:type container: email.mime.multipart.MIMEMultipart
:returns: The container (message).
The following headers are added
* :mailheader:`Subject`: The subject set during initialisation
* :mailheader:`From`: the group email address
* :mailheader:`To`: the group email address
* :mailheader:`Sender`: the support email address
* :mailheader:`Precedence`: ``bulk``
* :mailheader:`Organization`: the site name
* :mailheader:`User-Agent`: Identifies GroupServer and this module
* :mailheader:`List-Post`: the group email address
* :mailheader:`List-Unsubscribe`: The unsubscribe ``mailto`` for the group
* :mailheader:`List-Archive`: the address for the group page
* :mailheader:`List-Help`: the address for the help page
* :mailheader:`List-Owner`: the support email address
* :mailheader:`List-ID`: The unique identifier for the group
'''
# Required headers
container['Subject'] = self.h(self.subject)
container['From'] = self.fromAddress
container['To'] = self.toAddress
# Sender?
s = formataddr((self.h('{0} Support'.format(self.siteInfo.name)),
self.siteInfo.get_support_email()))
container['Sender'] = s
# Nice-to-have headers
container['Precedence'] = b'Bulk'
container['Organization'] = self.h(self.siteInfo.name)
# User-Agent is not actually a real header, but Mozilla and MS use
# it.
brag = 'GroupServer <http://groupserver.org/> '\
'(gs.groups.messages.digest.base)'
container['User-Agent'] = self.h(brag)
# RFC2369 headers: <http://tools.ietf.org/html/rfc2369>
try:
p = '<mailto:{0}>'.format(self.rawFromAddress)
container['List-Post'] = self.h(p)
u = '<mailto:{0}?subject=Unsubscribe>'.format(
self.rawFromAddress)
container['List-Unsubscribe'] = self.h(u)
a = '<{0}>'.format(self.groupInfo.url)
container['List-Archive'] = self.h(a)
helpS = '<{0}/help> (Help)'.format(self.siteInfo.url)
container['List-Help'] = self.h(helpS)
s = '<mailto:{0}>'.format(self.siteInfo.get_support_email())
container['List-Owner'] = self.h(s)
# List-ID <http://tools.ietf.org/html/rfc2919>
canonicalHost = urlparse(self.siteInfo.url).netloc
gid = '{groupInfo.name} '\
'<{groupInfo.id}.{canonicalHost}>'.format(
groupInfo=self.groupInfo, canonicalHost=canonicalHost)
container['List-ID'] = gid
except UnicodeDecodeError:
# FIXME: Sometimes data is just too messed up.
# http://farmdev.com/talks/unicode/
pass
return container
def create_from_referrer(url):
obj = Campaign(Campaign.TYPE_REFERRAL)
parse_rslt = urlparse(url)
obj.source = parse_rslt.netloc
obj.content = parse_rslt.path
return obj
#encoding:UTF-8
import urllib2, urllib, HTMLParser, re
from urllib import urlencode
from urllib import urlparse
url = "http://www.baidu.com"
data = urllib2.urlopen(url).read()
print("This should be the baidu's website's code")
a=urllib2.urlopen(url)
type(a)
#
data={} #
data['word']='Jecvay Notes'
url="http://www.baidu.com/s?"
data=urllib.urlencode(data)
url_values=urllib.urlparse(data)
url_full=url+url_values
data=urllib2.urlopen(url_full).read()
data=data.decode('UTF-8')
printf(data)
def proxy_request(path, proxy_host, proxy_port, proxy_prefix, proxy_auth):
request_headers = {}
for h in ["Cookie", "Referer", "X-Csrf-Token"]:
if h in request.headers:
request_headers[h] = request.headers[h]
proxy_path = path
if request.query_string:
proxy_path = "%s?%s" % (path, request.query_string)
if proxy_prefix:
proxy_path = "/%s/%s" % (proxy_prefix.strip('/'), proxy_path)
else:
proxy_path = "/%s" + path
logger.info("Forward request to : '%s@%s:%s%s" %
(proxy_auth,proxy_host, proxy_port, proxy_path))
request_headers["Authorization"] = 'Basic %s' % base64.b64encode(proxy_auth)
request_headers["Content-Type"] = 'application/json'
if request.method == "POST" or request.method == "PUT":
form_data = json.dumps(request.json)
request_headers["Content-Length"] = len(form_data)
else:
form_data = None
conn = httplib.HTTPConnection(proxy_host, proxy_port)
conn.request(request.method,
proxy_path,
body=form_data,
headers=request_headers)
resp = conn.getresponse()
# Clean up response headers for forwarding
d = {}
response_headers = Headers()
for key, value in resp.getheaders():
logger.debug(" | %s: %s" % (key, value))
d[key.lower()] = value
if key in ["content-length", "connection", "content-type", "transfer-encoding","connection"]:
continue
if key == "set-cookie":
cookies = value.split(",")
[response_headers.add(key, c) for c in cookies]
else:
response_headers.add(key, value)
# If this is a redirect, munge the Location URL
if "location" in response_headers:
redirect = response_headers["location"]
parsed = urlparse(request.url)
redirect_parsed = urlparse(redirect)
redirect_host = redirect_parsed.netloc
if not redirect_host:
redirect_host = "%s:%d" % (proxy_host, proxy_port)
redirect_path = redirect_parsed.path
if redirect_parsed.query:
redirect_path += "?" + redirect_parsed.query
munged_path = url_for("proxy_old_api",
path=redirect_path[1:])
url = "%s://%s%s" % (parsed.scheme, parsed.netloc, munged_path)
response_headers["location"] = url
contents = resp.read()
# Restructing Contents.
if d["content-type"].find("application/json") >= 0:
kwargs = {'ensure_ascii': False,
'cls': JSONEncoder,
'indent': 4,
'separators': (',', ': ')}
contents = json.dumps(json.loads(contents), **kwargs)
root_prefix = "/%s/%s" % (proxy_prefix.strip('/'), path)
contents = contents.replace(root_prefix, "")
flask_response = Response(response=contents,
status=resp.status,
headers=response_headers,
content_type=resp.getheader('content-type'))
return flask_response
