def filter(self):
url = "https://stream.twitter.com/1.1/statuses/filter.json"
query = {}
querystring =""
if self.track:
query["track"] = self.track
querystring += "track="+quote(self.track)
if self.locations:
query["locations"] = self.locations
if querystring:
querystring += "&"
querystring += "locations="+quote(self.locations)
self.start_time = int(time.time())
running = True
while running:
try:
auth_header = self.generate_authorization_header("POST",url,query)
logging.getLogger("twitstream").debug("calling: https://stream.twitter.com/1.1/statuses/filter.json?"+querystring)
conn = HTTPSConnection("stream.twitter.com")
conn.request("POST","/1.1/statuses/filter.json?"+querystring,"",{'User-agent':'Mozilla/5.0','Authorization':auth_header})
running = self.stream(conn)
except Exception as ex:
logging.getLogger("twitstream").error(str(ex))
def send(self, message):
"""
Sends a specified message with id "message" or as object.
"""
# Only send message if message is instance of PushoverMessage
if type(message) is PushoverMessage:
# get dict contaning "message" and any other key-values declared
# by the "set" method.
data = message.get()
data['token'] = self.token
# define which user to send notification to if any
if self.user is not None:
data['user'] = self.user_token
# define which device to send notification to if any
if self.user_device is not None:
data['device'] = self.user_device
# initialise HTTPS connection
conn = HTTPSConnection(self.PUSHOVER_SERVER)
# send HTTPS request
conn.request(
"POST",
self.PUSHOVER_ENDPOINT,
urlencode(data),
self.PUSHOVER_CONTENT_TYPE
)
# get response form server
response = conn.getresponse().read().decode('utf-8')
data = json.loads(response)
# check response for error
if data['status'] != 1:
raise PushoverError(response)
else:
return True
else:
raise PushoverError("Wrong type passed to Pushover.send()!")
def __init__(self, scheme='http', host=None, port=80, timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
''' initial '''
self.__via_proxy = False
# TODO config file support with exceptions when proxy is needed and when is not
# Is it really needed? We can use PROXY_IGNORE_HOSTS
if scheme not in _ALLOWED_SCHEMES:
raise UnknownConnectionSchemeException(scheme)
if host not in PROXY_IGNORE_HOSTS:
if PROXIES.get(scheme):
parsed_proxy = parse_url(PROXIES[scheme])
host = parsed_proxy['host']
port = parsed_proxy['port']
self.__via_proxy = True
if not host:
raise UndefinedHostException(host)
if scheme == 'http':
self.__conn = HTTPConnection(host, port, timeout=timeout)
elif scheme == 'https':
self.__conn = HTTPSConnection(host, port, timeout=timeout)
else:
raise UnknownConnectionSchemeException(scheme)
def _url_retrieve(self, url, outfd, reporthook, binary):
#Like the one in urllib. Unlike urllib.retrieve url_retrieve
#can be interrupted. KeyboardInterrupt exception is raised when
#interrupted.
count = 0
blockSize = 1024 * 8
parsed_url = urlparse(url)
host = parsed_url.netloc
try:
connection = HTTPSConnection(host, timeout=15)
headers = { "Accept-Encoding": "identity", "Host": host, "User-Agent": "Python/3" }
connection.request("GET", parsed_url.path, headers=headers)
urlobj = connection.getresponse()
assert urlobj.getcode() == 200
totalSize = int(urlobj.info()['content-length'])
while not self._is_aborted():
data = urlobj.read(blockSize)
count += 1
if not data:
break
if not binary:
data = data.decode("utf-8")
outfd.write(data)
ui_thread_do(reporthook, count, blockSize, totalSize)
except Exception as e:
raise e
def push(event, msg):
global config
data = {'apikey': config["API_KEY"],
'application': __application__,
'event': event,
'description': msg,
'priority': 0}
h = Https(config["API_DOMAIN"])
h.request("POST",
"/publicapi/add",
headers={'Content-type': "application/x-www-form-urlencoded",
'User-Agent': __application__ + '/' + str(__version__)
},
body=urlencode(data))
response = h.getresponse()
request_status = response.status
if request_status == 200:
return True
else:
return False
def query(self, endpoint, arguments=None):
if not endpoint.startswith('/'):
endpoint = '/' + endpoint
if not endpoint.endswith(ApiClient.ENDPOINT_EXT):
endpoint += ApiClient.ENDPOINT_EXT
if arguments is None:
arguments = {}
query_string = '?'.join([endpoint, urlencode(arguments)])
result = self.cache.get(query_string)
if result is not None:
return objectify.fromstring(result)
connection = HTTPSConnection(ApiClient.API_HOST)
connection.request('GET', query_string)
response = connection.getresponse()
if response.status != 200:
raise ApiError('Error response: ' + str(response.status))
result = response.read()
result_object = objectify.fromstring(result)
cached_until = datetime.strptime(result_object.cachedUntil.text,
ApiClient.TIMESTAMP_FORMAT)
self.cache.set(query_string, result, cached_until)
return result_object
def submit_xml_request(xml_file):
'''
Submit the xml request to MAST.
Parameters:
xml_file : string
The xml request string.
Returns:
response : httplib object
The xml request submission result.
'''
home = os.environ.get("HOME")
user = os.environ.get("USER")
signer = SignStsciRequest()
request_xml_str = signer.signRequest("{0}/.ssh/privkey.pem".format(home),
xml_file)
params = urllib.urlencode({
"dadshost": "dmsops1.stsci.edu",
"dadsport": 4703,
"mission": "HST",
"request": request_xml_str})
headers = {"Accept": "text/html",
"User-Agent": "{0}PythonScript".format(user)}
req = HTTPSConnection("archive.stsci.edu")
req.request("POST", "/cgi-bin/dads.cgi", params, headers)
response = req.getresponse().read()
return response
def _send(self, message):
"""
Sends the specified PushoverMessage object via the Pushover API.
"""
kwargs = message.get()
kwargs['token'] = self.token
assert 'message' in kwargs
assert self.token is not None
if not 'user' in kwargs:
if self.set_user is not None:
kwargs['user'] = self.user_token
if self.user_device is not None:
kwargs['device'] = self.user_device
else:
kwargs['user'] = os.environ['PUSHOVER_USER']
data = urlencode(kwargs)
conn = HTTPSConnection(Pushover.PUSHOVER_SERVER)
conn.request("POST", Pushover.PUSHOVER_ENDPOINT, data,
Pushover.PUSHOVER_CONTENT_TYPE)
output = conn.getresponse().read().decode('utf-8')
data = json.loads(output)
if data['status'] != 1:
raise PushoverError(output)
else:
return True
def send(self, type, text):
message_time = time.time()
message_timestamp = time.ctime(message_time)
if check_time_restriction(self.starttime, self.endtime):
self.msg_to_send = text[:10000].encode('utf8') + " Message Sent at: " + message_timestamp
self.event = NMA_EVENT.encode('utf8')
self.content_type = NMA_CONTENT_TYPE
notify_data = {
'application': self.app_name,
'description': self.msg_to_send,
'event': self.event,
'priority': self.priority,
'content-type': self.content_type,
'apikey': self.api_key
}
headers = { 'User-Agent': NMA_USER_AGENT }
headers['Content-type'] = NMA_HEADER_CONTENT_TYPE
http_handler = HTTPSConnection(NMA_URL)
http_handler.request(NMA_METHOD, NMA_PATH, urlencode(notify_data), headers)
http_response = http_handler.getresponse()
try:
res = self._parse_response(http_response.read())
except Exception as e:
res = {
'type': 'NMA Notify Error',
'code': 800,
'message': str(e)
}
current_app.logger.info('Event NotifyMyAndroid Notification Failed: {0}'.format(str(e)))
raise Exception('NotifyMyAndroid Failed: {0}' . format(str(e)))
def callapi(self, method, path, args):
# Set headers.
headers = {
'User-Agent': USER_AGENT
}
# Set content type if we want to post the data.
if method == "POST":
headers['Content-type'] = "application/x-www-form-urlencoded"
# Create the HTTPSConnection and do the api call.
http_handler = HTTPSConnection(API_SERVER)
http_handler.request(method, path, urlencode(args), headers)
# Get the response.
resp = http_handler.getresponse()
try:
# Return the API-Response.
return self._parse_reponse_json(resp.read())
except Exception as e:
# Return response could not be parsed.
return {
'type': "RapidPush parse error",
'code': 600,
'message': str(e)
}
def connect(self):
"""
Override the connect() function to intercept calls to certain
host/ports.
If no app at host/port has been registered for interception then
a normal HTTPSConnection is made.
"""
if debuglevel:
sys.stderr.write('connect: %s, %s\n' % (self.host, self.port,))
try:
(app, script_name) = self.get_app(self.host, self.port)
if app:
if debuglevel:
sys.stderr.write('INTERCEPTING call to %s:%s\n' %
(self.host, self.port,))
self.sock = wsgi_fake_socket(app, self.host, self.port,
script_name, self.environ)
else:
HTTPSConnection.connect(self)
except Exception:
if debuglevel: # intercept & print out tracebacks
traceback.print_exc()
raise
def do_request(self, method, url, params = {}):
final_url = url
if method == 'GET' and params:
final_url += '?' + urlencode(params)
request_body = None
elif (method == 'POST' or method == 'PUT') and params:
request_body = urlencode(params)
else:
request_body = None
print("Starting {method} {final_url}".format(method = method, final_url = final_url))
if self.use_ssl:
con = HTTPSConnection(self.base_url)
else:
con = HTTPConnection(self.base_url)
headers = {}
headers["Content-Type"] = "application/x-www-form-urlencoded"
headers["ISV_API_KEY"] = self.key
headers["ISV_API_SECRET"] = self.secret
headers["LCNS_DISABLE_SIGN"] = "true"
con.request(method, final_url, request_body, headers)
res = con.getresponse()
body = res.read()
status = res.status
if self.debug:
debug_message = ""
debug_message += "################################\n"
debug_message += "REQUEST: {method} {uri}\n".format(method = method.upper(), uri = self.request_url(final_url) )
if params and ( method == 'POST' or method == 'GET' ):
debug_message += "PARAMETERS: {params}\n".format(params = params)
debug_message += "RESPONSE CODE: {status}\n".format(status = status)
debug_message += "BODY:\n{body}\n".format(body = body)
debug_message += "================================\n"
print(debug_message)
return {'body': body, 'status': status}
def call(self, method='', params=None):
""" Call T411 API """
if params is not None:
params = urlencode(params)
headers = HTTP_HEADERS
if method != 'auth':
headers['Authorization'] = self._token
conn = HTTPSConnection(API_URL)
conn.request('POST', '/%s' % method, body=params, headers=headers)
r = conn.getresponse()
if r.status == HTTP_OK:
#import pdb; pdb.set_trace()
rt = r.read()
try:
pass
rt = rt.decode('utf-8')
conn.close()
except:
conn.close()
return rt
else:
conn.close()
return False
def send(self, type, text, raw):
message_time = time.time()
message_timestamp = time.ctime(message_time)
if check_time_restriction(self.starttime, self.endtime):
if self.suppress_timestamp == False:
self.msg_to_send = text[:10000].encode('utf8') + " Message Sent at: " + message_timestamp
else:
self.msg_to_send = text[:10000].encode('utf8')
notify_data = {
'apikey': self.api_key,
'application': self.app_name,
'event': self.event,
'description': self.msg_to_send,
'priority': self.priority
}
if sys.version_info >= (2,7,9):
http_handler = HTTPSConnection(PROWL_URL, context=ssl._create_unverified_context())
else:
http_handler = HTTPSConnection(PROWL_URL)
http_handler.request(PROWL_METHOD, PROWL_PATH, headers=self.headers,body=urlencode(notify_data))
http_response = http_handler.getresponse()
if http_response.status == 200:
return True
else:
current_app.logger.info('Event Prowl Notification Failed: {0}'. format(http_response.reason))
raise Exception('Prowl Notification Failed: {0}' . format(http_response.reason))
def __init__(self, host, port, key_file, cert_file, ca_file, timeout=None):
HTTPSConnection.__init__(self, host, key_file=key_file, cert_file=cert_file)
self.key_file = key_file
self.cert_file = cert_file
self.ca_file = ca_file
self.timeout = timeout
self.port = port
def getRemoteDBModifiedTS():
"""
Performs a HEAD request to get the Last-Modified date-time
of a database dump file and parses it into a UNIX timestamp.
"""
debug_msg = "Unable to get timestamp of remote database dump - {0}"
logging.info("Getting timestamp of database dump at '%s'", HOST + PATH)
try:
# Removing the scheme from the URL
conn = HTTPSConnection(HOST[8:], timeout=TIMEOUT)
conn.request("HEAD", PATH)
except gaierror as e:
logging.debug(debug_msg.format("Cannot connect to '%s', error: %s"), HOST + PATH, e)
exit(1)
rsp = conn.getresponse()
if rsp.status != 200:
logging.debug(debug_msg.format("Server responded with: %d %s"), rsp.status, rsp.reason)
exit(1)
last_modified = rsp.getheader("last-modified", None)
if last_modified is None:
logging.debug(debug_msg.format("Response doesnt include Last-Modified Header"))
exit(1)
last_m_dt = datetime.strptime(last_modified.split(", ")[1], "%d %b %Y %H:%M:%S %Z")
return timegm(last_m_dt.timetuple())
def httpsGet(self, url):
Log.d("GET " + url)
url = self.completeUrl(url)
headers = self.createHeaders()
conn = HTTPSConnection(self.server)
conn.request("GET", url, None, headers)
self.lastResponse = self.parseResponse(conn.getresponse())
return self.lastResponse
def __init__(self, host, port=None, key_file=None, cert_file=None,
strict=None,
timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
connect_timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
HTTPSConnection.__init__(self, host, port=port, key_file=key_file,
cert_file=cert_file, strict=strict, timeout=timeout)
self.connect_timeout = connect_timeout
def normalise_package(name):
"""
"""
http = HTTPSConnection('pypi.python.org')
http.request('HEAD', '/pypi/%s/' % name)
r = http.getresponse()
if r.status not in (200, 301):
raise ValueError(r.reason)
return r.getheader('location', name).split('/')[-1]
def retrieve_apikey(self, providerkey=None, token=None):
"""
Get an API key from a registration token retrieved in retrieve/token.
The user must have approved your request first, or you will get an
error response.
The parameters are :
- providerkey (required) : your provider API key.
- token (required): the token returned from retrieve_token.
This returns a dictionary such as:
{'apikey': u'16b776682332cf11102b67d6db215821f2c233a3',
'code': u'200',
'remaining': u'999',
'resetdate': u'1299535575'}
"""
h = Https(API_DOMAIN)
data = {'apikey': self.apikey}
if providerkey is not None:
data['providerkey'] = providerkey
else:
raise Exception("Provider Key is required for retrieving API key")
if token is not None:
data['token'] = token
else:
raise Exception("Token is required for retrieving API key.\
Call retrieve_token to request it.")
h.request("GET",
"/publicapi/retrieve/apikey?" + urlencode(data),
headers=self.headers)
request = h.getresponse()
request_status = request.status
if request_status == 200:
dom = minidom.parseString(request.read())
code = dom.getElementsByTagName('prowl')[0].\
getElementsByTagName('success')[0].\
getAttribute('code')
remaining = dom.getElementsByTagName('prowl')[0].\
getElementsByTagName('success')[0].\
getAttribute('remaining')
resetdate = dom.getElementsByTagName('prowl')[0].\
getElementsByTagName('success')[0].\
getAttribute('resetdate')
users_api_key = dom.getElementsByTagName('prowl')[0].\
getElementsByTagName('retrieve')[0].\
getAttribute('apikey')
return dict(apikey=users_api_key, code=code, remaining=remaining,
resetdate=resetdate)
else:
self._relay_error(request_status)
def __init__(self, access_token=None, timeout=None):
self.access_token = access_token
self.timeout = timeout
try:
self.conn = HTTPSConnection('graph.facebook.com', timeout=self.timeout)
except TypeError:
# Timeout support for Python <2.6
if self.timeout:
socket.setdefaulttimeout(self.timeout)
self.conn = HTTPSConnection('graph.facebook.com')
def lookForAddress(loc):
print("Google GEOCODE Query")
conn = HTTPSConnection("maps.googleapis.com")
parametres = "latlng="+loc.latlng()+"&sensor=false&key="+commons.googleAPIKey
conn.request("GET", "/maps/api/geocode/xml?"+parametres)
response = conn.getresponse()
data = response.read()
dom = parseString(data)
for d in dom.getElementsByTagNameNS("*", 'formatted_address'):
loc.desc = d.firstChild.data
return
def httpclient(host, port, path, method='GET', contentType=None, content=None, user=None, password=None, ssl=False):
if ssl:
conn = HTTPSConnection(host, port, context=_create_unverified_context())
else:
conn = HTTPConnection(host, port)
headers = {}
if content and contentType:
headers['Content-type']=contentType
if user and password:
headers['Authorization']='Basic '+b64encode((user+':'+password).encode()).decode()
conn.request(method, path, (content if content else None), headers)
return conn.getresponse().read().decode()
def get_xml_from_directory_api(url):
assert url.startswith('https://')
xml_path = url.replace('https://', '')
xml_path = xml_path.replace(DIRECTORY_WEB_SERVICE, '')
c = HTTPSConnection(DIRECTORY_WEB_SERVICE)
b = bytes(DIRECTORY_USERNAME + ':' + DIRECTORY_PASSWORD, 'utf-8')
userAndPass = base64.b64encode(b).decode("ascii")
headers = { 'Authorization' : 'Basic %s' % userAndPass }
c.request('GET', xml_path, headers=headers)
result = c.getresponse()
return result.read()
def request_token(self):
connection = HTTPSConnection(self._host)
request = oauth.OAuthRequest.from_consumer_and_token(self._consumer,
http_url='https://' + self._host + '/oauth/request_token')
request.sign_request(self._signature_method, self._consumer, None)
connection.request(request.http_method, request.to_url())
resp = connection.getresponse()
if resp.status == 200:
self._oauth_token = oauth.OAuthToken.from_string(resp.read().decode())
return {'key': self._oauth_token.key, 'secret': self._oauth_token.secret}
else:
raise InsTwitterResponseError(resp.code, resp.read().decode())
def post(self, host, url, data, headers=None):
tmp_data = json.dumps(data)
tmp_data = tmp_data.encode(encoding='utf-8')
# req = httpreq.Request(url=url,
# data=tmp_data, method='POST')
# if headers is not None:
# req.headers = headers
con = HTTPSConnection(host)
con.request("POST", url, body=tmp_data, headers=headers)
r = con.getresponse()
# res = r.read()
return r
def post(self, application=None, event=None,
description=None, priority=0, providerkey=None,
url=None):
"""
Post a notification..
You must provide either event or description or both.
The parameters are :
- application ; The name of your application or the application
generating the event.
- providerkey (optional) : your provider API key.
Only necessary if you have been whitelisted.
- priority (optional) : default value of 0 if not provided.
An integer value ranging [-2, 2] representing:
-2. Very Low
-1. Moderate
0. Normal
1. High
2. Emergency (note : emergency priority messages may bypass
quiet hours according to the user's settings)
- event : the name of the event or subject of the notification.
- description : a description of the event, generally terse.
- url (optional) : The URL which should be attached to the
notification.
"""
# Create the http object
h = Https(API_DOMAIN)
# Perform the request and get the response headers and content
data = {'apikey': self.apikey,
'application': application,
'event': event,
'description': description,
'priority': priority}
if providerkey is not None:
data['providerkey'] = providerkey
if url is not None:
data['url'] = url[0:512] # API limits to 512 characters
h.request("POST",
"/publicapi/add",
headers=self.headers,
body=urlencode(data))
response = h.getresponse()
request_status = response.status
if request_status == 200:
return True
else:
self._relay_error(request_status, response.reason)
def apiRequest(self, request):
base_url = "api.github.com"
c = HTTPSConnection(base_url)
userAndPass = base64.b64encode(("%s:%s" % (self.u, self.p)).encode("ascii")).decode("ascii")
headers = {"Authorization": "Basic %s" % userAndPass}
c.request("GET", request, headers=headers)
res = c.getresponse()
return json.loads(res.read().decode("ascii"))
def sample(self):
url = "https://stream.twitter.com/1.1/statuses/sample.json"
query = {}
self.start_time = int(time.time())
while True:
try:
auth_header = self.generate_authorization_header("GET",url,query)
conn = HTTPSConnection("stream.twitter.com")
logging.getLogger("twitstream").debug("calling: https://stream.twitter.com/1.1/statuses/sample.json")
conn.request("GET","/1.1/statuses/sample.json",None,{'User-agent':'Mozilla/5.0','Authorization':auth_header})
self.stream(conn)
except Exception as ex:
logging.getLogger("twitstream").error(str(ex))
def login(request):
try:
conn = HTTPSConnection('ip.oauth')
conn.request('POST', '/oauth')
r1 = conn.getresponse()
if r1.status == 200:
access_token = r1.read()
else:
raise myExceptions.OauthError('password error')
return HttpResponse(status=200)
except myExceptions.OauthError as e:
return HttpResponse(json.dumps({'message': e.value}), status=403)
def requests_test(self):
# requests without timeout
requests.delete('http://localhost')
requests.get('http://localhost')
requests.head('http://localhost')
requests.options('http://localhost')
requests.patch('http://localhost')
requests.post('http://localhost')
requests.put('http://localhost')
requests.request('call', 'http://localhost')
delete_r('http://localhost')
get_r('http://localhost')
head_r('http://localhost')
options_r('http://localhost')
patch_r('http://localhost')
post_r('http://localhost')
put_r('http://localhost')
request_r('call', 'http://localhost')
delete('http://localhost')
get('http://localhost')
head('http://localhost')
options('http://localhost')
patch('http://localhost')
post('http://localhost')
put('http://localhost')
request('call', 'http://localhost')
# requests valid cases
requests.delete('http://localhost', timeout=10)
requests.get('http://localhost', timeout=10)
requests.head('http://localhost', timeout=10)
requests.options('http://localhost', timeout=10)
requests.patch('http://localhost', timeout=10)
requests.post('http://localhost', timeout=10)
requests.put('http://localhost', timeout=10)
requests.request('call', 'http://localhost', timeout=10)
delete_r('http://localhost', timeout=10)
get_r('http://localhost', timeout=10)
head_r('http://localhost', timeout=10)
options_r('http://localhost', timeout=10)
patch_r('http://localhost', timeout=10)
post_r('http://localhost', timeout=10)
put_r('http://localhost', timeout=10)
request_r('call', 'http://localhost', timeout=10)
delete('http://localhost', timeout=10)
get('http://localhost', timeout=10)
head('http://localhost', timeout=10)
options('http://localhost', timeout=10)
patch('http://localhost', timeout=10)
post('http://localhost', timeout=10)
put('http://localhost', timeout=10)
request('call', 'http://localhost', timeout=10)
# urllib without timeout
urllib.request.urlopen('http://localhost')
urlopen('http://localhost')
urlopen_r('http://localhost')
# urllib valid cases
urllib.request.urlopen('http://localhost', timeout=10)
urlopen('http://localhost', timeout=10)
urlopen_r('http://localhost', timeout=10)
# suds without timeout
suds.client.Client('http://localhost')
Client('http://localhost')
Client_r('http://localhost')
# suds valid cases
suds.client.Client('http://localhost', timeout=10)
Client('http://localhost', timeout=10)
Client_r('http://localhost', timeout=10)
# http.client without timeout
http.client.HTTPConnection('http://localhost')
http.client.HTTPSConnection('http://localhost')
HTTPConnection('http://localhost')
HTTPSConnection('http://localhost')
HTTPConnection_r('http://localhost')
HTTPSConnection_r('http://localhost')
# http.client valid cases
http.client.HTTPConnection('http://localhost', timeout=10)
http.client.HTTPSConnection('http://localhost', timeout=10)
HTTPConnection('http://localhost', timeout=10)
HTTPSConnection('http://localhost', timeout=10)
HTTPConnection_r('http://localhost', timeout=10)
HTTPSConnection_r('http://localhost', timeout=10)
# smtplib without timeout
smtplib.SMTP('http://localhost')
smtplib_r.SMTP('http://localhost')
SMTP('http://localhost')
SMTP_r('http://localhost')
# smtplib valid cases
smtplib.SMTP('http://localhost', timeout=10)
smtplib_r.SMTP('http://localhost', timeout=10)
SMTP('http://localhost', timeout=10)
SMTP_r('http://localhost', timeout=10)
# Serial without timeout
serial.Serial('/dev/ttyS1')
serial_r.Serial('/dev/ttyS1')
Serial('/dev/ttyS1')
Serial_r('/dev/ttyS1')
# serial valid cases
serial.Serial('/dev/ttyS1', timeout=10)
serial_r.Serial('/dev/ttyS1', timeout=10)
Serial('/dev/ttyS1', timeout=10)
Serial_r('/dev/ttyS1', timeout=10)
# odoo.addons.iap without timeout
odoo.addons.iap.models.iap.jsonrpc('http://localhost')
jsonrpc('http://localhost')
iap.jsonrpc('http://localhost')
# odoo.addons.iap valid cases
odoo.addons.iap.models.iap.jsonrpc('http://localhost', timeout=10)
jsonrpc('http://localhost', timeout=10)
iap.jsonrpc('http://localhost', timeout=10)
def getConnection(self, host, timeout=None):
return HTTPSConnection(host, key_file=cert, cert_file=cert)
def get_episode_urls(cartmn: HTTPSConnection, season_url: str):
cartmn.request("GET", season_url)
parser = LinkExtractor()
resp = cartmn.getresponse()
parser.feed(str(resp.read()))
return (href for href in parser.hrefs if href.startswith("/episodes"))
def _connect(self):
"""Establish HTTPS Connection"""
self.connection = HTTPSConnection(*self.sig[1:], timeout=30,
context=self.SSLCONTEXT)
def find_speeches_by_year(host, this_url, print_test=False):
'''
Takes the host and a url for a given year
and returns infromation about the speeches and links to the web site
containing the text of the speeches. This function is used to create
the list of all web sites that contain the individual speeches that
need to be scraped.
INPUTS:
host the host (for the Federal Reserve 'www.federalreserve.gov)
this_url the path to the speeches for a given year
print_test an optional field that will print out summary statistics
OUTPUT:
date_lst list of speech dates
speaker_lst list of speaker names
title_lst list containing titles of speeches
link_lst list of htm links to the actual speeches
NOTES:
1. There are video links on some of the urls that we need to removed.
These videos are represented by the 'watchLive' class.
'''
conn = HTTPSConnection(host = host)
conn.request(method='GET', url = this_url)
resp = conn.getresponse()
body = resp.read()
# check that we received the correct response code
if resp.status != 200:
print('Error from Web Site! Response code: ', resp.status)
else:
soup=BeautifulSoup(body, 'html.parser')
event_list = soup.find('div', class_='row eventlist')
# creating the list of dates, titles, speakers and html articles from web page
date_lst =[]
title_lst = []
speaker_lst = []
link_lst = []
for row in event_list.find_all('div', class_='row'):
tmp_date= [x.text for x in row.find_all('time')]
date_lst.append(tmp_date)
tmp_speaker = [x.text for x in row.find_all('p', class_='news__speaker')]
speaker_lst.append(tmp_speaker)
tmp_title = [x.text for x in row.find_all('em')]
title_lst.append(tmp_title)
# some of the links include video with the transcript. We are deleteing these here
for link in event_list.find_all('a', href=True, class_ = lambda x: x != 'watchLive'):
link_lst.append(link['href'])
if print_test:
print('length of dates: ', len(date_lst))
print('length of speakers: ', len(speaker_lst))
print('length of titles: ', len(title_lst))
print('length of href: ', len(link_lst))
return date_lst, speaker_lst, title_lst, link_lst
def _conectar_servico(self, servico, envio, resposta, ambiente=None):
print(' CT-e versao...', self.versao)
print(' Conectando ao webservice.........')
if ambiente is None:
ambiente = self.ambiente
if self.versao == u'3.00':
self._soap_envio = SOAPEnvio_300()
self._soap_envio.webservice = webservices.METODO_WS[servico][
'webservice']
self._soap_envio.metodo = webservices.METODO_WS[servico]['metodo']
self._soap_envio.cUF = UF_CODIGO[self.estado]
self._soap_envio.envio = envio
self._soap_retorno = SOAPRetorno_300()
self._soap_retorno.webservice = webservices.METODO_WS[servico][
'webservice']
self._soap_retorno.metodo = webservices.METODO_WS[servico][
'metodo']
self._soap_retorno.resposta = resposta
self._servidor = webservices.ESTADO_WS[
self.estado][ambiente][u'servidor']
self._url = webservices.ESTADO_WS[self.estado][ambiente][servico]
#try:
self.certificado.prepara_certificado_arquivo_pfx()
#
# Salva o certificado e a chave privada para uso na conexão HTTPS
# Salvamos como um arquivo de nome aleatório para evitar o conflito
# de uso de vários certificados e chaves diferentes na mesma máquina
# ao mesmo tempo
#
self.caminho_temporario = self.caminho_temporario or u'/tmp/'
nome_arq_chave = self.caminho_temporario + uuid4().hex
arq_tmp = open(nome_arq_chave, 'w')
arq_tmp.write(self.certificado.chave)
arq_tmp.close()
nome_arq_certificado = self.caminho_temporario + uuid4().hex
arq_tmp = open(nome_arq_certificado, 'w')
arq_tmp.write(self.certificado.certificado)
arq_tmp.close()
print(" servidor: ", self._servidor)
con = HTTPSConnection(self._servidor,
key_file=nome_arq_chave,
cert_file=nome_arq_certificado)
#con = ConexaoHTTPS(self._servidor, key_file=nome_arq_chave, cert_file=nome_arq_certificado)
con.request(u'POST', u'/' + self._url,
self._soap_envio.xml.encode(u'utf-8'),
self._soap_envio.header)
resp = con.getresponse()
#
# Apagamos os arquivos do certificado e o da chave privada, para evitar
# um potencial risco de segurança; muito embora o uso da chave privada
# para assinatura exija o uso da senha, pode haver serviços que exijam
# apenas o uso do certificado para validar a identidade, independente
# da existência de assinatura digital
#
os.remove(nome_arq_chave)
os.remove(nome_arq_certificado)
# Dados da resposta salvos para possível debug
self._soap_retorno.resposta.version = resp.version
self._soap_retorno.resposta.status = resp.status
#self._soap_retorno.resposta.reason = unicode(resp.reason.decode('utf-8'))
self._soap_retorno.resposta.reason = unicode(resp.reason)
self._soap_retorno.resposta.msg = resp.msg
self._soap_retorno.resposta.original = unicode(
resp.read().decode('utf-8'))
# Tudo certo!
#print ('servidor:', self._servidor)
#print('url: ', self._url)
#print ('STATUS__-', self._soap_retorno.resposta.original)
if self._soap_retorno.resposta.status == 200:
self._soap_retorno.xml = self._soap_retorno.resposta.original
con.close()
def __init__(self, host, port, ca_certs):
HTTPSConnection.__init__(self, host, port)
self.ca_certs = ca_certs
def __init__(self, *a, **k):
k['context'] = ssl._create_unverified_context()
HTTPSConnection.__init__(self, *a, **k)
def _new_conn(self):
# Performs the NTLM handshake that secures the connection. The socket
# must be kept open while requests are performed.
self.num_connections += 1
log.debug('Starting NTLM HTTPS connection no. %d: https://%s%s',
self.num_connections, self.host, self.authurl)
headers = {}
headers['Connection'] = 'Keep-Alive'
req_header = 'Authorization'
resp_header = 'www-authenticate'
conn = HTTPSConnection(host=self.host, port=self.port)
# Send negotiation message
headers[req_header] = (
'NTLM %s' % ntlm.create_NTLM_NEGOTIATE_MESSAGE(self.rawuser))
log.debug('Request headers: %s', headers)
conn.request('GET', self.authurl, None, headers)
res = conn.getresponse()
reshdr = dict(res.getheaders())
log.debug('Response status: %s %s', res.status, res.reason)
log.debug('Response headers: %s', reshdr)
log.debug('Response data: %s [...]', res.read(100))
# Remove the reference to the socket, so that it can not be closed by
# the response object (we want to keep the socket open)
res.fp = None
# Server should respond with a challenge message
auth_header_values = reshdr[resp_header].split(', ')
auth_header_value = None
for s in auth_header_values:
if s[:5] == 'NTLM ':
auth_header_value = s[5:]
if auth_header_value is None:
raise Exception('Unexpected %s response header: %s' %
(resp_header, reshdr[resp_header]))
# Send authentication message
ServerChallenge, NegotiateFlags = \
ntlm.parse_NTLM_CHALLENGE_MESSAGE(auth_header_value)
auth_msg = ntlm.create_NTLM_AUTHENTICATE_MESSAGE(
ServerChallenge, self.user, self.domain, self.pw, NegotiateFlags)
headers[req_header] = 'NTLM %s' % auth_msg
log.debug('Request headers: %s', headers)
conn.request('GET', self.authurl, None, headers)
res = conn.getresponse()
log.debug('Response status: %s %s', res.status, res.reason)
log.debug('Response headers: %s', dict(res.getheaders()))
log.debug('Response data: %s [...]', res.read()[:100])
if res.status != 200:
if res.status == 401:
raise Exception('Server rejected request: wrong '
'username or password')
raise Exception('Wrong server response: %s %s' %
(res.status, res.reason))
res.fp = None
log.debug('Connection established')
return conn
def request(self, method, url, body=None, headers={}):
self.request_length = 0
HTTPSConnection.request(self, method, url, body, headers)
def send(self, astr):
HTTPSConnection.send(self, astr)
self.request_length += len(astr)
def __init__(self, host, port=None, strict=None):
strict = strict
HTTPSConnection.__init__(self, host, port, strict)
self.request_length = 0
def __init__(self, host, port=None):
HTTPSConnection.__init__(self, host, port)
self.request_length = 0
def connect():
return HTTPSConnection("discordapp.com", 443)
import ssl
import os
from base64 import b64encode
from http.client import HTTPSConnection
username = os.environ["HAW_USER"]
password = os.environ["HAW_PASSWORD"]
userpass_string = username + ":" + password
userpass_string = userpass_string.encode()
# Crawl the Container Harbor to get the current containers.
# Only works, when there is just one set of PNS containers.
c = HTTPSConnection("pnskss.informatik.haw-hamburg.de",
context=ssl._create_unverified_context())
userAndPass = b64encode(userpass_string).decode("ascii")
headers = {'Authorization': 'Basic %s' % userAndPass,
'Accept': 'application/json'}
with open('servers') as file:
content = file.readlines()
content = [x.strip() for x in content]
for server in content:
request_url = '/containers/{}/execution'.format(server)
print(request_url)
r = c.request('POST', request_url, headers=headers)
res = c.getresponse()
print(res.read())
c.close()
def endheaders(self, *args, **kwargs):
if self._proxy_auth is None:
self._proxy_auth = self._encode_auth()
HTTPSConnection.endheaders(self, *args, **kwargs)
def request(self, method, url, body=None, headers={}):
self.request_length = 0
if self.has_proxy:
self.set_tunnel(self.request_host, 443)
headers.setdefault("Host", self.request_host)
HTTPSConnection.request(self, method, url, body, headers)
class urlopen(object):
"""HTTP(S) urlopen that handles compressed connections
Errors that respond with bencoded data can be read
Raises IOError on other errors
"""
conntypes = {'http': HTTPConnection, 'https': HTTPSConnection}
def __init__(self, url):
self.tries = 0
self.error_return = None
self.connection = None
self.url = None
self._open(url.strip())
def _setconn(self, url):
scheme, host, path, params, query, _ = urllib.parse.urlparse(url)
if scheme not in self.conntypes:
raise IOError(('url error', 'unknown url type', scheme, url))
if self.connection is not None and not (
isinstance(self.connection, self.conntypes[scheme])
and host == self.connection.host):
try:
self.connection.close()
except socket.error:
pass
self.connection = None
if self.connection is None:
if scheme == 'http':
self.connection = HTTPConnection(host, timeout=30)
else:
self.connection = HTTPSConnection(host,
timeout=30,
context=SSLCONTEXT)
# a[:len(b)] == (a if b else '')
self.url = path + ';'[:len(params)] + params + '?'[:len(query)] + query
def _open(self, url):
try:
self._setconn(url)
except HTTPException as e:
raise IOError(('http error', str(e)))
for _ in range(MAX_REDIRECTS):
try:
self.connection.request('GET', self.url, None, {
'User-Agent': VERSION,
'Accept-Encoding': 'gzip'
})
self.response = self.connection.getresponse()
if self.response.status == 200: # Success
return
if self.response.status in (301, 302): # Redirect
self._setconn(self.response.getheader('Location'))
continue
except HTTPException as e:
raise IOError(('http error', str(e)))
# Handle bencoded errors
try:
data = self._read()
d = bdecode(data)
if 'failure reason' in d:
self.error_return = data
return
except (IOError, ValueError):
pass
# General HTTP error
raise IOError(
('http error', self.response.status, self.response.reason))
else:
raise IOError(('http error', 500,
"Internal Server Error: Redirect Recursion"))
def read(self):
"""Read response"""
if self.error_return:
return self.error_return
return self._read()
def _read(self):
data = self.response.read()
if self.response.getheader('Content-Encoding', '').find('gzip') >= 0:
try:
data = gzip.GzipFile(fileobj=io.BytesIO(data)).read()
except IOError:
raise IOError(('http error', 'got corrupt response'))
return data
def close(self):
"""Close connection
Always succeeds"""
if self.connection is not None:
try:
self.connection.close()
except socket.error:
pass
def __enter__(self):
return self
def __exit__(self, _type, _value, _traceback):
self.close()
class Client(object):
"""
Provides methods to access the taskforce http service. These are basically
for convenience in clients, and are particularly useful when using
Unix domain sockets (thanks to Erik van Zijst for the nice approach --
https://pypi.python.org/pypi/uhttplib).
Parameters:
address - The address to listen on, defaults to "httpd.def_address".
This may be specified as "[host][:port]" for TCP, or
as "path" to select a Udom service (path must contain
at least one "/" character).
use_ssl - If None (default) the connection will not use SSL.
If False, SSL will be used but the certificate will
not be verified.
If True, SSL will be used and the server must have a
valid certificate (assumes python >= 2.7.9)
timeout - The timeout in seconds (float) for query I/O.
log - A 'logging' object to log errors and activity.
"""
def __init__(self, address=None, use_ssl=None, timeout=5, log=None):
if log:
self.log = log
else:
self.log = logging.getLogger(__name__)
self.log.addHandler(logging.NullHandler())
if address:
self.address = address
else:
self.address = httpd.def_address
if self.address.find('/') >= 0:
if use_ssl is None:
self.http = udomHTTPConnection(self.address, timeout)
else:
ssl_params = self._build_params(use_ssl, timeout)
self.http = udomHTTPSConnection(self.address, **ssl_params)
else:
port = None
m = re.match(r'^(.*):(.*)$', self.address)
if m:
self.log.debug("Matched host '%s', port '%s'", m.group(1),
m.group(2))
host = m.group(1)
try:
port = int(m.group(2))
except:
raise HttpError(
code=500,
content_type='text/plain',
content="TCP listen port must be an integer")
else:
host = self.address
self.log.debug("No match, proceding with host '%s'", host)
if use_ssl is None:
if not port:
port = httpd.def_port
self.log.debug("Connecting to host '%s', port '%s'", host,
port)
self.http = HTTPConnection(host, port, timeout=timeout)
else:
if not port:
port = httpd.def_sslport
ssl_params = self._build_params(use_ssl, timeout)
self.http = HTTPSConnection(host, port, **ssl_params)
self.log.debug("Connecting via ssl to host '%s', port '%s'",
host, port)
self.address = "%s:%d" % (host, port)
self.http.connect()
self.sock = self.http.sock
self.lastpath = None
self.log.info("HTTP connected via %s", self.http.sock)
if use_ssl and hasattr(self.http.sock, 'cipher'): # pragma: no cover
self.log.debug("Cipher: %s", self.http.sock.cipher())
def _build_params(self, use_ssl, timeout):
ssl_params = {'timeout': timeout}
try:
ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
except AttributeError: # pragma: no cover
self.log.info("No ssl.SSLContext(), assuming older python")
return ssl_params
if use_ssl is False:
ctx.verify_mode = ssl.CERT_NONE
else: # pragma: no cover
ctx.verify_mode = ssl.CERT_REQUIRED
if 'OP_NO_SSLv2' in ssl.__dict__:
ctx.options |= ssl.OP_NO_SSLv2
else: # pragma: no cover
self.log.info(
"Implementation does not offer ssl.OP_NO_SSLv2 which may allow less secure connections"
)
if 'OP_NO_SSLv3' in ssl.__dict__:
ctx.options |= ssl.OP_NO_SSLv3
else: # pragma: no cover
self.log.info(
"Implementation does not offer ssl.OP_NO_SSLv3 which may allow less secure connections"
)
ssl_params['context'] = ctx
return ssl_params
def get(self, path, query=None):
"""
Issue a GET request. If specfied, "query" should be a dict of name/value
pairs. Names should be normal identifiers which start with an alpha
followed by alnums or underscores. The values are url-encoded and
become the "query" part of the request header (ie, the part after
the '?' in the URI).
The result is the tuple:
(code, content, content_type)
If the request is unsuccessful returning code 400 or higher, the
http.HttpError exception is raised.
"""
self.lastpath = path
if query is not None:
self.lastpath += '?' + urlencode(query)
self.http.request('GET', self.lastpath)
resp = self.http.getresponse()
ctype = resp.getheader('Content-Type')
data = resp.read().decode('utf-8')
self.log.debug("Request '%s' status %d, %s length %d", self.lastpath,
resp.status, ctype, len(data))
if resp.status < 400:
return (resp.status, data, ctype)
else:
raise HttpError(code=resp.status, content_type=ctype, content=data)
def getmap(self, path, query=None):
"""
Performs a GET request where the response content type is required to be
"application/json" and the content is a JSON-encoded data structure.
The decoded structure is returned.
"""
code, data, ctype = self.get(path, query)
if ctype != 'application/json':
self.log.error("Expecting JSON from GET of '%s', got '%s'",
self.lastpath, ctype)
raise HttpError(code=400,
content_type='text/plain',
content='Remote returned invalid content type: ' +
ctype)
try:
result = json.loads(data)
except Exception as e: # pragma: no cover
self.log.error("Could not load JSON content from GET %r -- %s",
self.lastpath, e)
raise HttpError(code=400,
content_type='text/plain',
content='Could not load JSON content')
return result
def post(self, path, valuemap=None, query=None):
"""
Performs a POST request. "valuemap" is a dict sent as "application/x-www-form-urlencoded".
"query" is as for get(). Return is same as get().
"""
self.lastpath = path
if query is not None:
self.lastpath += '?' + urlencode(query)
if valuemap:
self.http.request(
'POST', self.lastpath, urlencode(valuemap),
{"Content-type": "application/x-www-form-urlencoded"})
else:
self.http.request('POST', self.lastpath, '')
resp = self.http.getresponse()
ctype = resp.getheader('Content-Type')
data = resp.read().decode('utf-8')
self.log.debug("Request '%s' status %d, %s length %d", self.lastpath,
resp.status, ctype, len(data))
if resp.status < 400:
return (resp.status, data, ctype)
else:
raise HttpError(code=resp.status, content_type=ctype, content=data)
def postmap(self, path, valuemap=None, query=None):
"""
Performs a POST request as per post() but the response content type
is required to be "application/json" and is processed as with getmap().
"""
code, data, ctype = self.post(path, valuemap, query)
if ctype != 'application/json':
self.log.error("Expecting JSON from POST of '%s', got '%s'",
self.lastpath, ctype)
raise HttpError(code=400,
content_type='text/plain',
content='Remote returned invalid content type: ' +
ctype)
try:
result = json.loads(data)
except Exception as e: # pragma: no cover
self.log.error("Could not load JSON content from POST %r -- %s",
self.lastpath, e)
raise HttpError(code=400,
content_type='text/plain',
content='Could not load JSON content')
return result
def request(self, method, url, *args):
"""
Pass-thru method to make this class behave a little like HTTPConnection
"""
return self.http.request(method, url, *args)
def getresponse(self):
"""
Pass-thru method to make this class behave a little like HTTPConnection
"""
resp = self.http.getresponse()
self.log.info("resp is %s", str(resp))
if resp.status < 400:
return resp
else:
errtext = resp.read()
content_type = resp.getheader('Content-Type', 'text/plain')
raise HttpError(code=resp.status,
content_type=content_type,
content=errtext)
class Client:
""" Responsable for requests to Blitz RESTful API """
def __init__(self, user, api_key, host=None, port=None, connect=True):
self.username = user
self.api_key = api_key
self.host = 'www.blitz.io' if host is None else host
self.port = 443 if port is None else port
self.private_key = None
if connect:
self.connect()
def connect(self):
""" Connects the client. """
self.connection = HTTPSConnection(self.host, self.port)
def get_headers(self):
""" Returns the headers need for a auccessful request to blitz.io. """
private = self.private_key
headers = {
"Content-type": "application/json",
'X-API-User': self.username,
'X-API-Key': self.api_key if private is None else private,
'X-API-Client' : 'python'
}
return headers
def set_private_key(self, key):
""" Sets the user private key to be used in the request header. """
self.private_key = key
def execute(self, post_data):
""" Sends a queue request to blitz.io RESTful API. """
path = "/api/1/curl/execute"
data = json.dumps(post_data)
self.connection.request("POST", path, data, self.get_headers())
response = self.connection.getresponse()
response_string = response.read().decode('UTF-8')
return json.loads(response_string)
def login(self):
""" Login to blitz.io RESTful API. """
path = "/login/api"
self.connection.request("GET", path, None, self.get_headers())
response = self.connection.getresponse()
response_string = response.read().decode('UTF-8')
return json.loads(response_string)
def job_status(self, job_id):
""" Sends a job status request to blitz.io RESTful API. """
path = "/api/1/jobs/{}/status".format(job_id)
self.connection.request("GET", path, None, self.get_headers())
response = self.connection.getresponse()
response_string = response.read().decode('UTF-8')
return json.loads(response_string)
def abort_job(self, job_id):
""" Send a abort request to blitz.io RESTful API. """
path = "/api/1/jobs/{}/abort".format(job_id)
self.connection.request("PUT", path, '', self.get_headers())
response = self.connection.getresponse()
response_string = response.read().decode('UTF-8')
return json.loads(response_string)
def close(self):
""" Closes the connection. """
self.connection.close()
def parse(self, post_data):
""" Sends a parse request to blitz.io RESTful API. """
path = "/api/1/parse"
data = json.dumps(post_data)
self.connection.request("POST", path, data, self.get_headers())
response = self.connection.getresponse()
response_string = response.read().decode('UTF-8')
return json.loads(response_string)
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import json
try:
from http.client import HTTPSConnection
except ImportError: # python2
from httplib import HTTPSConnection
import argparse
deezer_api = "/2.0/"
connection = HTTPSConnection("api.deezer.com")
def get_data(url):
connection.request("GET", url)
response = connection.getresponse()
return json.loads(response.read().decode("utf-8"))
def get_user_info(userid):
url = deezer_api + "user/%d" % userid
return get_data(url)
def get_user_playlists(userid):
url = deezer_api + "user/%d/playlists" % userid
return get_data(url)
def get_playlist(playlistid):
def connect(self):
""" Connects the client. """
self.connection = HTTPSConnection(self.host, self.port)
def __init__(self, host, port=None, timeout=20, **kwargs):
HTTPSConnection.__init__(self, str(host), port, **kwargs)
self.timeout = timeout
def get_access_token(self) -> str:
conn = HTTPSConnection(self.auth_url)
conn.request("POST", "/api/token", self.auth_params, self.auth_headers)
response = conn.getresponse()
return json.loads(response.read().decode("utf-8"))["access_token"]
import ssl
from time import sleep
from html.parser import HTMLParser
from http.client import HTTPSConnection
HOST = "www.southparkstudios.com"
conn = HTTPSConnection(HOST, context=ssl._create_unverified_context())
class LinkExtractor(HTMLParser):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.__hrefs = set()
def handle_starttag(self, tag, attrs):
if tag == "a":
self.__hrefs.add(dict(attrs)["href"])
@property
def hrefs(self):
return self.__hrefs
def get_season_urls(cartmn: HTTPSConnection):
cartmn.request("GET", "/seasons/south-park")
parser = LinkExtractor()
resp = cartmn.getresponse()
parser.feed(str(resp.read()))
return (href for href in parser.hrefs
if href.startswith("/seasons/south-park/"))
def checkEWS(host, port, mode, domain, user, data):
ews_url = "/EWS/Exchange.asmx"
if port == 443:
try:
uv_context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
session = HTTPSConnection(host, port, context=uv_context)
except AttributeError:
session = HTTPSConnection(host, port)
else:
session = HTTPConnection(host, port)
# Use impacket for NTLM
ntlm_nego = ntlm.getNTLMSSPType1(host, domain)
#Negotiate auth
negotiate = base64.b64encode(ntlm_nego.getData())
# Headers
headers = {
"Authorization":
'NTLM %s' % negotiate.decode('utf-8'),
"Content-type":
"text/xml; charset=utf-8",
"Accept":
"text/xml",
"User-Agent":
"Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.129 Safari/537.36"
}
session.request("POST", ews_url, POST_BODY, headers)
res = session.getresponse()
res.read()
if res.status != 401:
print(
'Status code returned: %d. Authentication does not seem required for URL'
% (res.status))
return False
try:
if 'NTLM' not in res.getheader('WWW-Authenticate'):
print('NTLM Auth not offered by URL, offered protocols: %s' %
(res.getheader('WWW-Authenticate')))
return False
except (KeyError, TypeError):
print('No authentication requested by the server for url %s' %
(ews_url))
return False
print('[*] Got 401, performing NTLM authentication')
# Get negotiate data
try:
ntlm_challenge_b64 = re.search(
'NTLM ([a-zA-Z0-9+/]+={0,2})',
res.getheader('WWW-Authenticate')).group(1)
ntlm_challenge = base64.b64decode(ntlm_challenge_b64)
except (IndexError, KeyError, AttributeError):
print('No NTLM challenge returned from server')
return False
if mode == 'plaintext':
password1 = data
nt_hash = ''
elif mode == 'ntlmhash':
password1 = ''
nt_hash = binascii.unhexlify(data)
else:
print('[!]Wrong parameter')
return False
lm_hash = ''
ntlm_auth, _ = ntlm.getNTLMSSPType3(ntlm_nego, ntlm_challenge, user,
password1, domain, lm_hash, nt_hash)
auth = base64.b64encode(ntlm_auth.getData())
headers = {
"Authorization":
'NTLM %s' % auth.decode('utf-8'),
"Content-type":
"text/xml; charset=utf-8",
"Accept":
"text/xml",
"User-Agent":
"Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.129 Safari/537.36"
}
session.request("POST", ews_url, POST_BODY, headers)
res = session.getresponse()
body = res.read()
if res.status == 401:
print('[!] Server returned HTTP status 401 - authentication failed')
return False
else:
print('[+] Valid:%s %s' % (user, data))
#print(body)
return True
def __init__(self, host, ca_file=None):
HTTPSConnection.__init__(self, host)
self.__ca_file = ca_file
from urllib.parse import quote
from xml.etree.ElementTree import fromstring
from pymongo.mongo_client import MongoClient
from datetime import datetime
# https://developers.naver.com/apps/#/myapps/lA5FivJ5MenI3OvwU03a/overview
# 네이버 개발자센터
# https://developers.naver.com/docs/search/news/
# 검색-뉴스
# lA5FivJ5MenI3OvwU03a
# Tra6RYPObW
# https://openapi.naver.com/v1/search/news.xml
huc = HTTPSConnection("openapi.naver.com")
# 인터넷 주소에 한글 - x
# ㅋ -> %2D
# urllib.parse
# URL인코딩
w = quote("코로나")
# 헤더
h = {
"X-Naver-Client-Id": "lA5FivJ5MenI3OvwU03a",
"X-Naver-Client-Secret": "Tra6RYPObW"
}
huc.request("GET", "/v1/search/news.xml?query=" + w, headers=h)
resBody = huc.getresponse().read()
class HTTPGetter:
def __init__(self, baseUrl, maxPending=10, auth=""):
self.baseUrl = baseUrl
self.parsedBaseUrl = urlparse(baseUrl)
self.maxPending = maxPending
self.requests = []
self.pendingRequests = []
if self.parsedBaseUrl.scheme == "http":
self.httpConnection = HTTPConnection(self.parsedBaseUrl.netloc)
elif self.parsedBaseUrl.scheme == "https":
context = ssl.create_default_context()
self.httpConnection = HTTPSConnection(self.parsedBaseUrl.netloc,
context=context)
else:
raise UnsupportedURLScheme(self.parsedBaseUrl.scheme)
self.httpRequestHeaders = headers = {
'Host': self.parsedBaseUrl.netloc,
'Content-Length': 0,
'Connection': 'Keep-Alive',
'User-Agent': 'FlightGear terrasync.py'
}
if (auth and not auth.isspace()):
self.httpRequestHeaders['Authorization'] = 'Basic %s' % b64encode(
auth.encode("utf-8")).decode("ascii")
def assemblePath(self, httpGetCallback):
"""Return the path-on-server for the file to download.
Example: '/scenery/Airports/N/E/4/.dirindex'
"""
assert not self.parsedBaseUrl.path.endswith('/'), \
repr(self.parsedBaseUrl)
return self.parsedBaseUrl.path + str(httpGetCallback.src)
def assembleUrl(self, httpGetCallback):
"""Return the URL of the file to download."""
baseUrl = self.parsedBaseUrl.geturl()
assert not baseUrl.endswith('/'), repr(baseUrl)
return urljoin(baseUrl + '/', httpGetCallback.src.asRelative())
def doGet(self, httpGetCallback):
time.sleep(1.25) # throttle the rate
pathOnServer = self.assemblePath(httpGetCallback)
self.httpConnection.request("GET", pathOnServer, None,
self.httpRequestHeaders)
httpResponse = self.httpConnection.getresponse()
# 'httpResponse' is an http.client.HTTPResponse instance
return httpGetCallback.callback(self.assembleUrl(httpGetCallback),
httpResponse)
def get(self, httpGetCallback):
nbRetries = nbRetriesLeft = 5
while True:
try:
return self.doGet(httpGetCallback)
except HTTPException as exc:
if nbRetriesLeft == 0:
raise NetworkError(
"after {nbRetries} retries for URL {url}: {errMsg}".
format(nbRetries=nbRetries,
url=self.assembleUrl(httpGetCallback),
errMsg=exc)) from exc
# Try to reconnect
self.httpConnection.close()
time.sleep(1)
self.httpConnection.connect()
nbRetriesLeft -= 1
def request(self, method, url, body=None, headers=None):
self.request_length = 0
HTTPSConnection.request(self, method, url, body,
{} if headers is None else headers)
