def test_sni_urls(self):
"""
Test SNI urls
:return:
"""
print ""
# Just checking all providers - we should make this error on non-existent urls.
for provider in [
provider
for provider in providers.makeProviderList()
if provider.name not in self.self_signed_cert_providers
]:
print "Checking %s" % provider.name
try:
requests.head(provider.url, verify=certifi.old_where(), timeout=10)
except requests.exceptions.Timeout:
pass
except requests.exceptions.SSLError as error:
if u"SSL3_GET_SERVER_CERTIFICATE" not in ex(error):
print "SSLError on %s: %s" % (provider.name, ex(error.message))
raise
else:
print "Cannot verify certificate for %s" % provider.name
except Exception: # pylint: disable=broad-except
pass
def get_html(url):
"""
Given a URL, will return the HTML using urllib3.
:param url: The url to extract the HTML from
:return: If extracted successfully, the HTML is returned. If there is a failure, a message with HTTP status. If an exception is thrown, -1 is returned witha description of the error
"""
try:
# urllib3.disable_warnings()
# Try with new where function, but sometimes it failes
# so then try old where function
# Read more: https://github.com/certifi/python-certifi#usage
try:
http = urllib3.PoolManager(cert_reqs='CERT_REQUIRED',
ca_certs=certifi.where())
except:
http = urllib3.PoolManager(cert_reqs='CERT_REQUIRED',
ca_certs=certifi.old_where())
r = http.request('GET', url, timeout=5.0)
if str(r.status).startswith("2"):
html = r.data.decode("utf-8")
return html
else:
return "Failed to get html, status: " + str(r.status)
except Exception as e:
sys.stdout.write(str(e))
return "-1: " + str(e)
def authenticate_withings(username, password):
global pem
if args.warning:
try:
requests.packages.urllib3.disable_warnings()
except Exception:
pass
if args.insecure:
pem = False
else:
try:
import certifi
pem = certifi.old_where()
except Exception:
pem = True
auth_data = "email=" + str(username) + "&is_admin=&password="******"[-] Authenticating at account.withings.com"
s.request("HEAD", URL_USAGE, timeout=3, headers=HEADER, allow_redirects=True, verify=pem)
response = restpost(URL_AUTH, auth_data)
if 'session_key' in s.cookies.get_dict():
jar = s.cookies.get_dict()
else:
sys.exit("[-] Session key negotiation failed, check username and/or password" + "\n")
accountid = re.sub("[^0-9]", "", str(re.search('(?<=accountId)(.*)', response.content)))
payload = "accountid=" + str(accountid) + "&action=getbyaccountid&appliver=c7726fda&appname=my2&apppfm=web&enrich=t&sessionid=" + \
jar['session_key'] + "&type=-1"
response = restpost(URL_ASSO, payload)
deviceid = response['body']['associations'][0]['deviceid']
sessionkey = jar['session_key']
return deviceid, sessionkey
def authenticate_withings(username, password):
"""Authenticate based on username and md5 hashed password."""
global pem
if args.warning:
try:
requests.packages.urllib3.disable_warnings()
except Exception:
pass
if args.insecure:
pem = False
else:
try:
import certifi
pem = certifi.old_where()
except Exception:
pem = True
requests.head(URL_USAGE,
timeout=3,
headers=HEADER,
allow_redirects=True,
verify=pem)
payload = {
'email': username,
'hash': hashlib.md5(password.encode('utf-8')).hexdigest(),
'duration': '900'
}
print("[-] Authenticating at scalews.withings.net")
response = requests.post(URL_AUTH, data=payload)
iddata = response.json()
sessionkey = iddata['body']['sessionid']
response = requests.get(URL_ASSO + sessionkey)
iddata = response.json()
deviceid = iddata['body']['associations'][0]['deviceid']
return deviceid, sessionkey
def _get_file_meta(self, key):
self.logger.debug(
"Getting metainformation for a file at key {}".format(key))
try:
if self.auth and self.auth.get_token():
# pyrebase download does not work with files that require
# authentication...
# Need to rewrite
# storageobj.download(local_file_path, self.auth.get_token())
headers = {
"Authorization": "Firebase " + self.auth.get_token()
}
else:
headers = {}
escaped_key = key.replace('/', '%2f')
url = "{}/o/{}".format(self.app.storage().storage_bucket,
escaped_key)
response = self.app.requests.get(url,
headers=headers,
verify=certifi.old_where())
if response.status_code != 200:
self.logger.debug("Response error with code {}".format(
response.status_code))
return (None, None)
return (json.loads(response.content.decode()), url)
except Exception as err:
self.logger.warn(("Getting metainfo of file {} " +
"raised an exception: {}").format(key, err))
self.logger.exception(err)
return (None, None)
def __init__(self, client_id, client_secret):
self.AccessUrl = "https://oxford-speech.cloudapp.net/token/issueToken"
self._clientId = client_id
# provided by MS
self._clientSecret = client_secret
# provided by MS
self.request_data = 'grant_type=client_credentials&client_id=' + self._clientId+'&client_secret='
self.request_data += self._clientSecret + '&scope=https://speech.platform.bing.com'
# opt must be a string object not a unicode object
data_l = len(self.request_data)
http_header = [
"Content-Type:application/x-www-form-urlencoded"
]
storage = StringIO()
# the way to get response and print it
c = pycurl.Curl()
c.setopt(pycurl.CAINFO, certifi.old_where())
c.setopt(pycurl.URL, self.AccessUrl)
c.setopt(c.HTTPHEADER, http_header)
c.setopt(c.POST, 1)
c.setopt(c.POSTFIELDSIZE, data_l)
c.setopt(c.CONNECTTIMEOUT, 30)
c.setopt(c.TIMEOUT, 30)
c.setopt(c.POSTFIELDS, self.request_data)
# --------------------------------------------
c.setopt(c.WRITEFUNCTION, storage.write)
# --------------------------------------------
c.perform()
c.close()
body = storage.getvalue()
storage.close()
self._token = byteify(json.loads(body.decode()))
def __init__(self, proxies=None, **kwargs):
"""Create base Medusa session instance."""
# Set default ssl verify
self.verify = certifi.old_where() if all(
[app.SSL_VERIFY, kwargs.pop('verify', False)]) else False
# Add response hooks
self.my_hooks = kwargs.pop('hooks', [])
# Pop the cache_control config
cache_control = kwargs.pop('cache_control', None)
# Initialize request.session after we've done the pop's.
super(MedusaSession, self).__init__(**kwargs)
# Add cache control of provided as a dict. Needs to be attached after super init.
if cache_control:
factory.add_cache_control(self, cache_control)
# add proxies
self.proxies = proxies or factory.add_proxies()
# Configure global session hooks
self.hooks['response'].append(hooks.log_url)
# Extend the hooks with kwargs provided session hooks
self.hooks['response'].extend(self.my_hooks)
# Set default headers.
self.headers.update(self.default_headers)
def __init__(self):
self._auth = None
self._sj_client = HTTPClient.from_url(
"https://{0}{1}".format(SJ_DOMAIN, SJ_URL),
concurrency=20,
network_timeout=15,
ssl_options={
"ca_certs": certifi.old_where(),
})
self._pl_client = HTTPClient.from_url(
"https://{0}{1}".format(SJ_DOMAIN, SJ_URL),
concurrency=1,
network_timeout=120,
ssl_options={
"ca_certs": certifi.old_where(),
})
def _delete_file(self, key):
self.logger.debug("Deleting file at key {}".format(key))
try:
if self.auth:
headers = {
"Authorization": "Firebase " + self.auth.get_token()
}
else:
headers = {}
escaped_key = key.replace('/', '%2f')
url = "{}/o/{}?alt=media".format(self.app.storage().storage_bucket,
escaped_key)
response = self.app.requests.delete(url,
headers=headers,
verify=certifi.old_where())
if response.status_code != 204:
raise ValueError("Response error with code {}, text {}".format(
response.status_code, response.text))
self.logger.debug("Done")
except Exception as err:
self.logger.warning(("Deleting file {} from storage " +
"raised an exception: {}").format(key, err))
def call_server():
tornado.httpclient.AsyncHTTPClient().configure(
None, defaults=dict(ca_certs=certifi.old_where()))
http_client = tornado.httpclient.AsyncHTTPClient()
url = 'https://testapi.taxrating.net/Services/V02/SureTax.asmx/PostRequest'
body = """request=<?xml version="1.0" encoding="UTF-8"?><Request xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"><ClientNumber>000000911</ClientNumber> <BusinessUnit></BusinessUnit> <ValidationKey>f35fcde8-d112-4192-b4ba-86169bfe887a</ValidationKey> <DataYear>2019</DataYear> <DataMonth>07</DataMonth> <TotalRevenue>25.25</TotalRevenue> <ReturnFileCode>0</ReturnFileCode> <ClientTracking>PORTAL001</ClientTracking> <IndustryExemption></IndustryExemption> <ResponseGroup>03</ResponseGroup> <ResponseType>D2</ResponseType> <ItemList> <Item> <LineNumber>USCAZ1000009</LineNumber> <InvoiceNumber></InvoiceNumber> <CustomerNumber>000000911</CustomerNumber> <OrigNumber></OrigNumber> <TermNumber></TermNumber> <BillToNumber></BillToNumber> <Zipcode>19107</Zipcode> <Plus4>0000</Plus4> <P2PZipcode></P2PZipcode> <P2PPlus4></P2PPlus4> <TransDate>07/28/2019</TransDate> <Revenue>25.25</Revenue> <Units>1</Units> <UnitType>00</UnitType> <Seconds>1</Seconds> <TaxIncludedCode>1</TaxIncludedCode> <TaxSitusRule>04</TaxSitusRule> <TransTypeCode>500002</TransTypeCode> <SalesTypeCode>R</SalesTypeCode> <RegulatoryCode>05</RegulatoryCode> <TaxExemptionCodeList> <string>00</string></TaxExemptionCodeList></Item></ItemList></Request>"""
response = yield http_client.fetch(request=url, method='POST', body=body)
print response.body # python2+使用
def _get_ssl_cert(verify):
"""
Configure the ssl verification.
We need to overwrite this in the request method. As it's not available in the session init.
:param verify: SSL verification on or off.
"""
return certifi.old_where() if all([app.SSL_VERIFY, verify]) else False
def get_auth_url(self, level):
logger.info('using %s', keyring.get_keyring().__module__)
client_id = key_store.get('picasa', 'client_id')
self.session = OAuth2Session(client_id,
scope=self.scope[level],
redirect_uri='urn:ietf:wg:oauth:2.0:oob')
self.session.verify = certifi.old_where()
return self.session.authorization_url(
'https://accounts.google.com/o/oauth2/v2/auth')[0]
def client(self, *args, **kwargs):
# We have certifi installed, which causes botocore to use the newer
# CA Bundle which does not have the 1024 bit roots that Amazon requires
# so instead we'll override this so that it uses the old bundle when
# talking to Amazon.
if kwargs.get("verify") is None:
kwargs["verify"] = certifi.old_where()
return super().client(*args, **kwargs)
def domain_verify_options(url):
for domain in WHITELIST_SHA1_DOMAINS:
if url.startswith(domain):
return certifi.old_where()
for domain in WHITELIST_INSECURE_DOMAINS:
if url.startswith(domain):
logging.warn("SKIPPING HTTPS VERIFICATION.")
return False
return True
def client(self, *args, **kwargs):
# We have certifi installed, which causes botocore to use the newer
# CA Bundle which does not have the 1024 bit roots that Amazon requires
# so instead we'll override this so that it uses the old bundle when
# talking to Amazon.
if kwargs.get("verify") is None:
kwargs["verify"] = certifi.old_where()
return super().client(*args, **kwargs)
def domain_verify_options(url):
for domain in WHITELIST_SHA1_DOMAINS:
if url.startswith(domain):
return certifi.old_where()
for domain in WHITELIST_INSECURE_DOMAINS:
if url.startswith(domain):
logging.warn("SKIPPING HTTPS VERIFICATION.")
return False
return True
def _Download(url, to):
print 'Downloading %s to %s...' % (url, to)
# Using certifi.old_where() because old versions of OpenSSL sometimes fails
# to validate certificate chains that use the strong roots [certifi.where()].
http = urllib3.PoolManager(cert_reqs='CERT_REQUIRED',
ca_certs=certifi.old_where())
response = http.request('GET', url, preload_content=False)
with open(to, 'w') as to_file:
for chunk in response.stream(1024):
to_file.write(chunk)
response.release_conn()
def lipper_job():
certifi.old_where()
d = datetime.date.today()
os.chdir(Path)
if not os.path.isdir(str(d.year) +"_"+ str(d.month)+"_"+str(d.day)):
os.mkdir(str(d.year) +"_"+ str(d.month)+"_"+str(d.day))
newdir = Path + '\\'+str(d.year) +'_'+ str(d.month)+'_'+str(d.day)
os.chdir(newdir)
ftp=ftplib.FTP()
ftp.connect('lipperftp.thomsonreuters.com',21)
ftp.login('hedgeworld_st','LipperHedge')
ftp.retrlines('LIST')
filenames = ftp.nlst()
for filename in filenames:
host_file = os.path.join(newdir, filename)
try:
with open(host_file, 'wb') as local_file:
ftp.retrbinary('RETR %s' % filename, local_file.write) #Download in filename in binary transfer mode to local file
except ftplib.error_perm:
pass
def sign_in_with_custom_token(self, token):
request_ref = ("https://www.googleapis.com/identitytoolkit" +
"/v3/relyingparty/verifyCustomToken?key={0}").format(
self.api_key)
headers = {"content-type": "application/json; charset=UTF-8"}
data = json.dumps({"returnSecureToken": True, "token": token})
request_object = requests.post(request_ref,
headers=headers,
data=data,
verify=certifi.old_where())
raise_detailed_error(request_object)
return request_object.json()
def ready(self):
super().ready()
# Configure Elasticsearch connections for connection pooling.
connections.configure(
default={
'hosts': settings.ES_HOST,
'verify_certs': True,
# We need to use old_where as long as our openssl version is below 1.0.2
# and ES found uses a CA which is cross signed by a 1024-bit root.
'ca_certs': certifi.old_where(),
'timeout': 60.0,
},
)
def verify_password_reset_code(self, reset_code, new_password):
request_ref = "https://www.googleapis.com/identitytoolkit" + \
"/v3/relyingparty/resetPassword?key={0}" \
.format(self.api_key)
headers = {"content-type": "application/json; charset=UTF-8"}
data = json.dumps({"oobCode": reset_code, "newPassword": new_password})
request_object = requests.post(request_ref,
headers=headers,
data=data,
verify=certifi.old_where())
raise_detailed_error(request_object)
return request_object.json()
def ready(self):
super(DocsConfig, self).ready()
# Configure Elasticsearch connections for connection pooling.
connections.configure(
default={
'hosts': settings.ES_HOST,
'verify_certs': True,
# We need to use old_where as long as our openssl version is below 1.0.2
# and ES found uses a CA which is cross signed by a 1024-bit root.
'ca_certs': certifi.old_where(),
'timeout': 60.0,
},
)
def get_account_info(self, id_token):
request_ref = ("https://www.googleapis.com/identitytoolkit/" +
"v3/relyingparty/getAccountInfo?key={0}") \
.format(self.api_key)
headers = {"content-type": "application/json; charset=UTF-8"}
data = json.dumps({"idToken": id_token})
request_object = requests.post(request_ref,
headers=headers,
data=data,
verify=certifi.old_where())
raise_detailed_error(request_object)
return request_object.json()
def send_email_verification(self, id_token):
request_ref = "https://www.googleapis.com/identitytoolkit/" + \
"v3/relyingparty/getOobConfirmationCode?key={0}" \
.format(self.api_key)
headers = {"content-type": "application/json; charset=UTF-8"}
data = json.dumps({"requestType": "VERIFY_EMAIL", "idToken": id_token})
request_object = requests.post(request_ref,
headers=headers,
data=data,
verify=certifi.old_where())
raise_detailed_error(request_object)
return request_object.json()
def send_password_reset_email(self, email):
request_ref = "https://www.googleapis.com/identitytoolkit/" + \
"v3/relyingparty/getOobConfirmationCode?key={0}" \
.format(self.api_key)
headers = {"content-type": "application/json; charset=UTF-8"}
data = json.dumps({"requestType": "PASSWORD_RESET", "email": email})
request_object = requests.post(request_ref,
headers=headers,
data=data,
verify=certifi.old_where())
raise_detailed_error(request_object)
return request_object.json()
def _certifi_where_for_ssl_version():
"""Gets the right location for certifi certifications for the current SSL
version.
Older versions of SSL don't support the stronger set of root certificates.
"""
if not ssl:
return
if ssl.OPENSSL_VERSION_INFO < (1, 0, 2):
warnings.warn("You are using an outdated version of OpenSSL that " "can't use stronger root certificates.")
return certifi.old_where()
return certifi.where()
def _certifi_where_for_ssl_version():
"""Gets the right location for certifi certifications for the current SSL
version.
Older versions of SSL don't support the stronger set of root certificates.
"""
if not ssl:
return
if ssl.OPENSSL_VERSION_INFO < (1, 0, 2):
warnings.warn('You are using an outdated version of OpenSSL that '
'can\'t use stronger root certificates.')
return certifi.old_where()
return certifi.where()
def create_user_with_email_and_password(self, email, password):
request_ref = "https://www.googleapis.com/identitytoolkit/" + \
"v3/relyingparty/signupNewUser?key={0}" \
.format(self.api_key)
headers = {"content-type": "application/json; charset=UTF-8"}
data = json.dumps(
{"email": email, "password": password, "returnSecureToken": True})
request_object = requests.post(
request_ref,
headers=headers,
data=data,
verify=certifi.old_where())
raise_detailed_error(request_object)
return request_object.json()
def permitted(self, level):
refresh_token = keyring.get_password('photini', 'picasa')
if not refresh_token:
self.session = None
self.token = None
return False
if not self.token:
# create expired token
self.token = {
'access_token': 'xxx',
'refresh_token': refresh_token,
'expires_in': -30,
}
self.session = None
if not self.session:
# create new session
client_id = key_store.get('picasa', 'client_id')
client_secret = key_store.get('picasa', 'client_secret')
auto_refresh_kwargs = {
'client_id': client_id,
'client_secret': client_secret,
}
if self.auto_refresh:
self.session = OAuth2Session(
client_id,
token=self.token,
token_updater=self._save_token,
auto_refresh_kwargs=auto_refresh_kwargs,
auto_refresh_url=self.token_url,
)
else:
self.session = OAuth2Session(client_id, token=self.token)
self.session.verify = certifi.old_where()
# refresh manually to get a valid token now
self.token = self.session.refresh_token(self.token_url,
**auto_refresh_kwargs)
self.session.headers.update({'GData-Version': '2'})
# verify the token
resp = self._check_response(
self.session.get(
'https://www.googleapis.com/oauth2/v3/tokeninfo',
params={'access_token':
self.token['access_token']})).json()
if resp['scope'] != self.scope[level] or resp['aud'] != client_id:
return False
return True
def authenticate_withings(username, password):
global pem
if args.warning:
try:
requests.packages.urllib3.disable_warnings()
except Exception:
pass
if args.insecure:
pem = False
else:
try:
import certifi
pem = certifi.old_where()
except Exception:
pem = True
login = s.get(URL_AUTH)
login_html = lxml.html.fromstring(login.text)
hidden_inputs = login_html.xpath(r'//form//input[@type="hidden"]')
auth_data = {x.attrib["name"]: x.attrib["value"] for x in hidden_inputs}
auth_data['email'] = username
auth_data['password'] = password
print "[-] Authenticating at account.withings.com"
s.request("HEAD",
URL_USAGE,
timeout=3,
headers=HEADER,
allow_redirects=True,
verify=pem)
response = restpost(URL_AUTH, auth_data)
if 'session_key' in s.cookies.get_dict():
jar = s.cookies.get_dict()
else:
sys.exit(
"[-] Session key negotiation failed, check username and/or password"
+ "\n")
accountid = re.sub("[^0-9]", "",
str(re.search('(?<=accountId)(.*)', response.content)))
payload = "accountid=" + str(accountid) + "&action=getbyaccountid&appliver=c7726fda&appname=my2&apppfm=web&enrich=t&sessionid=" + \
jar['session_key'] + "&type=-1"
response = restpost(URL_ASSO, payload)
deviceid = response['body']['associations'][0]['deviceid']
sessionkey = jar['session_key']
return deviceid, sessionkey
def refresh(self, refresh_token):
request_ref = "https://securetoken.googleapis.com/v1/token?key={0}" \
.format(self.api_key)
headers = {"content-type": "application/json; charset=UTF-8"}
data = json.dumps({"grantType": "refresh_token",
"refreshToken": refresh_token})
request_object = requests.post(
request_ref,
headers=headers,
data=data,
verify=certifi.old_where())
raise_detailed_error(request_object)
request_object_json = request_object.json()
# handle weirdly formatted response
user = {
"userId": request_object_json["user_id"],
"idToken": request_object_json["id_token"],
"refreshToken": request_object_json["refresh_token"]
}
return user
def _connectivity_test():
"""
Generate tests
:param self:
:return: test to run
"""
if not _provider.url:
print '%s has no url set, skipping' % _provider.name
return
try:
requests.head(_provider.url, verify=certifi.old_where(), timeout=10)
except requests.exceptions.SSLError as error:
if 'certificate verify failed' in str(error):
print 'Cannot verify certificate for %s' % _provider.name
else:
print 'SSLError on %s: %s' % (_provider.name, ex(error.message))
raise
except requests.exceptions.Timeout:
print 'Provider timed out'
def _connectivity_test():
"""
Generate tests
:param self:
:return: test to run
"""
if not _provider.url:
print '{0!s} has no url set, skipping'.format(_provider.name)
return
try:
requests.head(_provider.url, verify=certifi.old_where(), timeout=10)
except requests.exceptions.SSLError as error:
if 'certificate verify failed' in str(error):
print 'Cannot verify certificate for {0!s}'.format(_provider.name)
else:
print 'SSLError on {0!s}: {1!s}'.format(_provider.name, ex(error.message))
raise
except requests.exceptions.Timeout:
print 'Provider timed out'
def _connectivity_test(self): # pylint: disable=unused-argument
"""
Generate tests
:param self:
:return: test to run
"""
if not _provider.url:
print '%s has no url set, skipping' % _provider.name
return
try:
requests.head(_provider.url, verify=certifi.old_where(), timeout=10)
except requests.exceptions.SSLError as error:
if 'certificate verify failed' in str(error):
print 'Cannot verify certificate for %s' % _provider.name
else:
print 'SSLError on %s: %s' % (_provider.name, ex(error.message))
raise
except requests.exceptions.Timeout:
print 'Provider timed out'
def _download_file(self, key, local_file_path):
self.logger.debug("Downloading file at key {} to local path {}..."
.format(key, local_file_path))
try:
storageobj = self.app.storage().child(key)
if self.auth:
# pyrebase download does not work with files that require
# authentication...
# Need to rewrite
# storageobj.download(local_file_path, self.auth.get_token())
headers = {"Authorization": "Firebase " +
self.auth.get_token()}
escaped_key = key.replace('/', '%2f')
url = "{}/o/{}?alt=media".format(
self.app.storage().storage_bucket,
escaped_key)
response = self.app.requests.get(
url,
stream=True,
headers=headers,
verify=certifi.old_where())
if response.status_code == 200:
with open(local_file_path, 'wb') as f:
for chunk in response:
f.write(chunk)
else:
raise ValueError("Response error with code {}"
.format(response.status_code))
else:
storageobj.download(local_file_path)
self.logger.debug("Done")
except Exception as err:
self.logger.warn(
("Downloading file {} to local path {} from storage " +
"raised an exception: {}") .format(
key,
local_file_path,
err))
def downImg(imgURL, dirPath, name):
with open(dirPath + '/URLlist.txt', 'a') as f:
f.write(imgURL + '\n')
try:
ir = requests.get(imgURL, verify=certifi.old_where(), timeout=15)
# if str(ir.status_code)[0] == '4':
# print("\n%s:%s" % (str(ir.status_code), imgURL))
# return False
except Exception as e:
print("\nERROR:%s" % imgURL)
print(e)
return False
imgtype = imgURL.split('.')[-1]
if imgtype.find('/'):
imgtype = 'jpg'
with open((dirPath + '/%d.' + imgtype) % name, 'wb') as f:
f.write(ir.content)
return True
def getUrlFromPage(keyword, page_number):
params = {
'tn': 'resultjson_com',
'ipn': 'rj',
'ct': 201326592,
'is': '',
'fp': 'result',
'queryWord': keyword,
'cl': 2,
'lm': -1,
'ie': 'utf-8',
'oe': 'utf-8',
'adpicid': '',
'st': -1,
'z': '',
'ic': 0,
'word': keyword,
's': '',
'se': '',
'tab': '',
'width': '',
'height': '',
'face': 0,
'istype': 2,
'qc': '',
'nc': 1,
'fr': '',
'pn': page_number * 30,
'rn': 30,
'gsm': '1e',
'1488942260214': ''
}
url = 'https://image.baidu.com/search/acjson'
urlcontent = requests.get(
url, params=params, verify=certifi.old_where()).content.decode('utf-8')
urls = [uncompile(x) for x in re_url.findall(urlcontent)]
return urls
def test_weak_cacerts_installed(self):
yield self.get_configured_transport()
self.assertEqual(os.environ["SSL_CERT_FILE"], certifi.old_where())
def setup_cacerts(self):
# TODO: This installs an older CA certificate chain that allows
# some weak CA certificates. We should switch to .where() when
# Vumi Go's certificate doesn't rely on older intermediate
# certificates.
os.environ["SSL_CERT_FILE"] = certifi.old_where()
# License for the specific language governing permissions and limitations
# under the License.
# pylint: disable=no-member
# known issue regarding ssl module and pylint.
import ssl
import certifi
import requests
from requests.adapters import HTTPAdapter
from requests.packages.urllib3.poolmanager import PoolManager
# https://github.com/certifi/python-certifi#1024-bit-root-certificates
if ssl.OPENSSL_VERSION_INFO and ssl.OPENSSL_VERSION_INFO < (1, 0, 2, 0, 0):
_TRUSTED_CERT_FILE = certifi.old_where()
else:
_TRUSTED_CERT_FILE = certifi.where()
class _SSLAdapter(HTTPAdapter):
def init_poolmanager(self, connections, maxsize, block=False):
self.poolmanager = PoolManager(num_pools=connections,
maxsize=maxsize,
block=block,
cert_reqs=ssl.CERT_REQUIRED,
ca_certs=_TRUSTED_CERT_FILE,
ssl_version=ssl.PROTOCOL_TLSv1)
def pinned_session(pool_maxsize=8):
def recommend(request):
"""inputs : {'email' : EMAIL ,
'password' : PASS,
'lat' : CURRENT_LATITUDE,
'lng' : CURRENT_LONGITUDE,
**kwargs(required: see notes)}
output : {SUCCESS or FAILURE, 'results':[{'cafe_distance': KM, 'cafe_lat': LAT, 'cafe_lng':LNG,
'cafe_name':NAME', 'noise_level': dB, 'pic_available': BOOL,
'pic_url': (see notes)}]}
NOTES: The keyword arguments for the inputs are mandatory, I purposefully disallowed them to fail
silently. If no values are provided there are default values in the API. For any key if the default
value is okay, please pass "" as the value for that key. The keys are:
'diameter' : (int or float) the diamater around the user, default is 4 kilometers,
'granularity' : (int) number of grid nodes in each direction, default 4,
'image_max_width: (int) this is required by the google api, default is 400,
'noise_mu' : (int or float) the mean value of the sound intesity, default 70 dB,
'noise_sigma' : (int or float) the STD value of the sound intensity, default 30 dB,
'how_many_silent':(int) for each node, how many silent cafes should googel return? ,default 5. Also
please be mindful of this number as each added number is another api call on my account.
'radius_of_search':(int or float) how far should google search from a node to find cafes? default 500,
Please note if this radius is bigger than the node distances then extraneous calls will be made
In outputs, if no venues are found, a cute cactus image will be returned, along with the message.
if the venue has no pictures, a url to the cocoon logo will be returned.(could have made this
a kwarg too lol)"""
response_message = {0:'Success',1:'request method must be POST',2:'password cannot be empty',
3:'invalid email address',4:'always send a "name" key, login ? name="" : name=name',
5:'Seems that someone else with the same email is in the dataset, plumSemPy has been notified',
6:'success', 7:'Invalid email or password', 8:'No venues found :('}
response_status = 0
data = get_data_from_request(request,response_message)
email = data['email']
password = data['password']
lat = float(data['lat'])
lng = float(data['lng'])
kwargs = data['kwargs']
diameter = try_or_default(kwargs['diameter'],4)
image_max_width = try_or_default(kwargs['image_max_width'],400)
granularity = try_or_default(kwargs['granularity'],4)
noise_mu = try_or_default(kwargs['mu'],70)
noise_sigma = try_or_default(kwargs['sigma'],30)
how_many_silent = try_or_default(kwargs['how_many_silent'],1)
radius_of_search = try_or_default(kwargs['radius_of_search'],500)
#Verify user
user = Users.objects.filter(email=email)
if user.exists()==1:
#if verified:
if user[0].password == password:
#Make noise distribution grid around them, query the n most silent ones
#defining the bounding box
start_lat = lat - (1/110.574) * (diameter/2)
start_lng = lng - (1/(111.320*np.cos(start_lat))) * (diameter/2)
end_lat = lat + (1/110.574) * (diameter/2)
end_lng = lng + (1/(111.320*np.cos(start_lat))) * (diameter/2)
#Making a mesh grid
lat_grid, lng_grid = np.meshgrid(np.linspace(start_lat,end_lat,granularity),
np.linspace(start_lng, end_lng, granularity))
#Distributing noise levels
noise_levels = np.random.randn(granularity**2)*noise_sigma+noise_mu
#Making the noise data
data = np.array([(lat,lng, noise) for lat,lng,noise in zip(lat_grid.ravel(),
lng_grid.ravel(),
noise_levels)])
top_n_silent_index = data[:,2].argsort()[:how_many_silent]
top_n_silent = data[top_n_silent_index]
#call google to return cafes in 500 meters around that silent place
key = os.environ['GOOGLE_API_KEY']
results=[]
for i,point in enumerate(top_n_silent):
to_go_lat = point[0]
to_go_lng = point[1]
url = r'https://maps.googleapis.com/maps/api/place/nearbysearch/json?location={0},{1}&radius={2}&type=cafe&key={3}'.format(to_go_lat,
to_go_lng,
radius_of_search,
key)
r = requests.post(url,verify=certifi.old_where())
try:
cafe_lat = r.json()['results'][0]['geometry']['location']['lat']
cafe_lng = r.json()['results'][0]['geometry']['location']['lng']
cafe_name = r.json()['results'][0]['name']
cafe_distance = distance(lng, lat, cafe_lng, cafe_lat)
except IndexError:
if i==how_many_silent-1:
response_status +=1
return HttpResponse(simplejson.dumps({'Response Status':response_status,
'Response Description':response_message[8],
'results':['http://s2.favim.com/orig/37/aww-cactus-cute-friendship-green-Favim.com-304829.jpg']},
separators=(',', ':'), sort_keys=True) )
else:
continue
try:
cafe_photo_reference = r.json()['results'][0]['photos'][0]['photo_reference']
pic_url = r'https://maps.googleapis.com/maps/api/place/photo?maxwidth={0}&photoreference={1}&key={2}'.format(image_max_width,
cafe_photo_reference,
key)
pic_available = 1
except KeyError:
pic_url = r'http://s32.postimg.org/up9zw3udx/circle_logo.png'
pic_available = 0
#get the name of the place, lat,long, and the image url and send back
results.append({'cafe_lat':cafe_lat, 'cafe_lng':cafe_lng,'cafe_name':cafe_name,'pic_url':pic_url,
'noise_level':point[2], 'pic_available':pic_available,
'cafe_distance':cafe_distance})
response_status +=1
return HttpResponse(simplejson.dumps({'Response Status':response_status,
'Response Description':response_message[0],
'results':results},
separators=(',', ':'), sort_keys=True) )
#if not verified : invalid user/pass
else:
#Invalid password
return HttpResponse(simplejson.dumps({'Response Status':response_status,
'Response Description':response_message[7]},
separators=(',', ':'), sort_keys=True) )
else:
#invalid user name
return HttpResponse(simplejson.dumps({'Response Status':response_status,
'Response Description':response_message[7]},
separators=(',', ':'), sort_keys=True) )
import yaml
# TODO[mike]: This should be removed once we've updated python to 2.7.9
# This tells urllib3 to use pyopenssl, which has the latest tls protocols and is
# more secure than the default python ssl module in python 2.7.4
import requests
import urllib3.contrib.pyopenssl
urllib3.contrib.pyopenssl.inject_into_urllib3()
urllib3.disable_warnings()
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
# TODO[mike]: This shold be removed once we've updated our base OS. openssl 1.0.1 doesn't support cross-signed certs
# https://github.com/certifi/python-certifi/issues/26#issuecomment-138322515
import certifi
os.environ["REQUESTS_CA_BUNDLE"] = certifi.old_where()
__all__ = ['config']
if 'NYLAS_ENV' in os.environ:
assert os.environ['NYLAS_ENV'] in ('dev', 'test', 'staging', 'prod'), \
"NYLAS_ENV must be either 'dev', 'test', staging, or 'prod'"
env = os.environ['NYLAS_ENV']
else:
env = 'prod'
def is_live_env():
return env == 'prod' or env == 'staging'
def _requests_setup(self, requests, **adapter_kws):
session, requests_version = None, requests.__version__
log.debug('Using "requests" module version: %r', requests_version)
try: requests_version = tuple(it.imap(int, requests_version.split('.')))
except: requests_version = 999, 0, 0 # most likely some future version
if requests_version < (0, 14, 0):
raise RuntimeError( (
'Version of the "requests" python module (used by python-onedrive)'
' is incompatible - need at least 0.14.0, but detected {}.'
' Please update it (or file an issue if it worked before).' )\
.format(requests.__version__) )
if requests_version >= (1, 0, 0):
session = requests.Session()
session.mount('https://', requests.adapters.HTTPAdapter(**adapter_kws))
else:
log.warn( 'Not using request_adapter_settings, as these should not be'
' supported by detected requests module version: %s', requests_version )
if hasattr(sys, '_MEIPASS'):
# Fix cacert.pem path for running from PyInstaller bundle
cacert_pem = requests.certs.where()
if not exists(cacert_pem):
from pkg_resources import resource_filename
cacert_pem = resource_filename('requests', 'cacert.pem')
if not exists(cacert_pem):
cacert_pem = join(sys._MEIPASS, 'requests', 'cacert.pem')
if not exists(cacert_pem):
cacert_pem = join(sys._MEIPASS, 'cacert.pem')
if not exists(cacert_pem):
raise OneDriveInteractionError(
'Failed to find requests cacert.pem bundle when running under PyInstaller.' )
self._requests_base_keywords = (self._requests_base_keywords or dict()).copy()
self._requests_base_keywords.setdefault('verify', cacert_pem)
log.debug( 'Adjusted "requests" default ca-bundle'
' path (to run under PyInstaller) to: %s', cacert_pem )
if requests_version >= (2, 4, 0):
# Workaround for https://github.com/certifi/python-certifi/issues/26
import ssl
if ssl.OPENSSL_VERSION_INFO < (1, 0, 2):
try: import certifi
except ImportError: pass
else:
certifi_issue_url = 'https://github.com/certifi/python-certifi/issues/26'
if hasattr(certifi, 'old_where'): cacert_pem = certifi.old_where()
else:
cacert_pem = join(dirname(requests.certs.__file__), 'cacert.pem')
if not exists(cacert_pem):
cacert_pem = None
log.warn( 'Failed to find requests'
' certificate bundle for woraround to %s', certifi_issue_url )
if cacert_pem:
self._requests_base_keywords = (self._requests_base_keywords or dict()).copy()
self._requests_base_keywords.setdefault('verify', cacert_pem)
log.debug( 'Adjusted "requests" default ca-bundle path, to work around %s '
' [OpenSSL version %s, requests %s (>2.4.0) and certifi available at %r], to: %s',
certifi_issue_url, ssl.OPENSSL_VERSION_INFO,
requests_version, certifi.__file__, cacert_pem )
self._requests_setup_done = True
return session
def test_boto3_session_client_defaults_old_certifi(self):
session = aws._Boto3Session()
client = session.client(service_name="s3")
assert client._endpoint.verify == certifi.old_where()
the environnement variable SEARX_DEBUG is 0 or false
(whatever the value in settings.yml)
or general.debug=False in settings.yml
'''
searx_debug_env = environ.get('SEARX_DEBUG', '').lower()
if searx_debug_env == 'true' or searx_debug_env == '1':
searx_debug = True
elif searx_debug_env == 'false' or searx_debug_env == '0':
searx_debug = False
else:
searx_debug = settings.get('general', {}).get('debug')
if searx_debug:
logging.basicConfig(level=logging.DEBUG)
else:
logging.basicConfig(level=logging.WARNING)
logger = logging.getLogger('searx')
logger.debug('read configuration from %s', settings_path)
# Workaround for openssl versions <1.0.2
# https://github.com/certifi/python-certifi/issues/26
if OPENSSL_VERSION_INFO[0:3] < (1, 0, 2):
if hasattr(certifi, 'old_where'):
environ['REQUESTS_CA_BUNDLE'] = certifi.old_where()
logger.warning('You are using an old openssl version({0}), please upgrade above 1.0.2!'.format(OPENSSL_VERSION))
logger.info('Initialisation done')
if 'SEARX_SECRET' in environ:
settings['server']['secret_key'] = environ['SEARX_SECRET']
