def main(reactor):
print('List of tuples')
resp = yield treq.get('http://httpbin.org/get',
params=[('foo', 'bar'), ('baz', 'bax')])
content = yield resp.text()
print(content)
print('Single value dictionary')
resp = yield treq.get('http://httpbin.org/get',
params={'foo': 'bar', 'baz': 'bax'})
content = yield resp.text()
print(content)
print('Multi value dictionary')
resp = yield treq.get('http://httpbin.org/get',
params={'foo': ['bar', 'baz', 'bax']})
content = yield resp.text()
print(content)
print('Mixed value dictionary')
resp = yield treq.get('http://httpbin.org/get',
params={'foo': ['bar', 'baz'], 'bax': 'quux'})
content = yield resp.text()
print(content)
print('Preserved query parameters')
resp = yield treq.get('http://httpbin.org/get?foo=bar',
params={'baz': 'bax'})
content = yield resp.text()
print(content)
def test_body_schema(self):
class Api(object):
app = Klein()
@app.route('/')
@json_body
@validate(body_schema({'properties': {'foo': {'type': 'string'}}}))
def route(self, req, body):
pass
srv = yield ToyServer.from_test(self, Api().app)
resp = yield treq.get(
srv.url,
persistent=False,
data=json.dumps({'foo': 23}))
self.assertEqual(json.loads((yield resp.content())), {
'errors': [{
'type': 'invalid_body',
'message': "23 is not of type 'string'"
}]
})
resp = yield treq.get(
srv.url,
persistent=False,
data=json.dumps({'foo': 'bar'}))
self.assertEqual(resp.code, http.OK)
def test_has_header(self):
class Api(object):
app = Klein()
@app.route('/')
@validate(has_header('X-Foo'))
def route(self, req):
pass
srv = yield ToyServer.from_test(self, Api().app)
resp = yield treq.get(srv.url, persistent=False)
self.assertEqual(json.loads((yield resp.content())), {
'errors': [{
'type': 'header_missing',
'message': "Header 'X-Foo' is missing"
}]
})
resp = yield treq.get(
srv.url,
headers={'X-Foo': ['bar']},
persistent=False)
self.assertEqual(resp.code, http.OK)
def test_custom_agent(self):
"""
A custom Agent is used if specified.
"""
custom_agent = mock.Mock()
treq.get('https://www.example.org/', agent=custom_agent)
self.HTTPClient.assert_called_once_with(custom_agent)
def main(reactor):
tor = yield txtorcon.connect(
reactor,
UNIXClientEndpoint(reactor, "/var/run/tor/control")
)
print("Connected to Tor version {}".format(tor.version))
url = 'https://www.torproject.org:443'
print("Downloading {}".format(url))
resp = yield treq.get(url, agent=tor.web_agent())
print(" {} bytes".format(resp.length))
data = yield resp.text()
print("Got {} bytes:\n{}\n[...]{}".format(
len(data),
data[:120],
data[-120:],
))
print("Creating a circuit")
state = yield tor.create_state()
circ = yield state.build_circuit()
yield circ.when_built()
print(" path: {}".format(" -> ".join([r.ip for r in circ.path])))
print("Downloading meejah's public key via above circuit...")
config = yield tor.get_config()
resp = yield treq.get(
'https://meejah.ca/meejah.asc',
agent=circ.web_agent(reactor, config.socks_endpoint(reactor)),
)
data = yield resp.text()
print(data)
def test_body_schema(self):
class Api(object):
app = Klein()
@app.route('/')
@json_body
@validate(body_schema({'properties': {'foo': {'type': 'string'}}}))
def route(self, req, body):
pass
srv = yield ToyServer.from_test(self, Api().app)
resp = yield treq.get(
srv.url,
persistent=False,
data=json.dumps({'foo': 23}))
content = yield resp.json()
self.assertEqual(content['result'], {
'errors': [{
'type': 'invalid_body',
'message': "23 is not of type 'string'",
'schema_path': ['properties', 'foo', 'type'],
}]
})
self.assertEqual(content['status'], 400)
self.assertEqual(content['code'], 'Bad Request')
self.assertEqual(content['description'], 'api usage error')
resp = yield treq.get(
srv.url,
persistent=False,
data=json.dumps({'foo': 'bar'}))
self.assertEqual(resp.code, http.OK)
def process_item(self, item, spider):
try:
if type(item) is UrlItem:
# Call Pagespeed API
url = item.get('url', '')
params={'url' : url, 'key' : self.api_key }
#logging.debug('PageSpeedPipeline:' + self.PAGESPEED_URL + str(params))
treq.get(self.PAGESPEED_URL, params=params).addCallback(self.api_done)
#r = requests.get(self.PAGESPEED_URL, params=params)
'''
response = yield treq.get(self.PAGESPEED_URL, params=params)
code = yield treq.code(response)
if (code is 200):
result = yield treq.text(response)
with self.connection.cursor() as cursor:
# Store result in DB
cursor.execute("""INSERT INTO pagespeed (url, result) VALUES(%s, %s)""", ( url, result ) )
self.connection.commit()
'''
except:
logging.exception("PageSpeedPipeline Error %s", item.get('url', ''))
return item
def start_streaming(host, user):
print "Initial Sync for testuser_%d" % (user,)
r = yield treq.get(
"%s/_matrix/client/api/v1/events" % (host,),
params={
"access_token": "token_%d" % (user,),
"timeout": "0",
},
)
initialSync = yield r.json()
assert r.code == 200, "InitialSync %d. %r" % (r.code, initialSync,)
from_token = initialSync["end"]
yield sleep(5 * random.random()) # This is to even requests out.
print "Starting event stream for testuser_%d" % (user,)
while True:
try:
start = get_time()
r, i = yield defer.DeferredList(
[
treq.get(
"%s/_matrix/client/api/v1/events" % (host,),
params={
"access_token": "token_%d" % (user,),
"from": from_token,
"timeout": "30000",
},
),
sleep(45),
],
fireOnOneCallback=True,
)
end = get_time()
if i == 1:
# We timedout.
print "Cancelled after %ds, retrying" % ((end-start)/1000,)
continue
stream = yield r.json()
assert r.code == 200, "Event stream %d. %r" % (r.code, stream,)
if end - start > 35000:
print "Took more than %ds to get response" % ((end-start)/1000,)
from_token = stream["end"]
except Exception as e:
print "Failed to do event stream for %d: %s" % (user, e)
yield sleep(5)
def test_cached_pool(self):
pool = self.HTTPConnectionPool.return_value
treq.get('http://test.com')
self.HTTPConnectionPool.return_value = mock.Mock()
treq.get('http://test.com')
self.Agent.assert_called_with(mock.ANY, pool=pool)
def test_api_dump_invalid_querystring(self):
yield ShortenerTables(self.account, self.conn).create_tables()
url = "http://en.wikipedia.org/wiki/Cthulhu"
yield self.service.shorten_url(url, "test-user")
yield treq.get(self.make_url("/qr0"), allow_redirects=False, pool=self.pool)
resp = yield treq.get(self.make_url("/api/handler/dump"), allow_redirects=False, pool=self.pool)
self.assertEqual(resp.code, 400)
def lookupUri(self, message):
if not message.target in self.config:
return
for uri in re.findall("https?://[^\s]+", message.message):
treq.get(uri, unbuffered = True).addCallbacks(
callback = self._gotResponse,
errback = self._errorResult,
callbackArgs = (message, uri),
errbackArgs = (message, uri)
)
def check(self):
# Hax
try:
irc = self.master.modules["irc"]
words = yield self.config.get("words", [])
response = yield treq.get("http://a.4cdn.org/a/threads.json")
threads = yield treq.json_content(response)
nthreads, check = {}, []
for o in threads:
for t in o["threads"]:
nthreads[t["no"]] = t["last_modified"]
if t["no"] not in self.threads:
self.threads[t["no"]] = 0
if t["last_modified"] > self.threads[t["no"]]:
check.append(t["no"])
if not self.initial:
threads = []
for t in check:
response = yield treq.get("http://a.4cdn.org/a/res/{:d}.json".format(t))
if response.code == 200:
data = yield treq.json_content(response)
found, posts = set(), []
for p in data["posts"]:
if p["time"] > self.threads[t] and "com" in p:
f = set(filter(lambda x: re.search(r"\b" + x.lower() + r"\b", p["com"].lower()), words))
if f:
found.update(f)
posts.append((p["no"], p["com"]))
if found and posts:
threads.append((t, found, posts))
if len(threads) < 5:
for t, found, posts in threads:
p, also = posts[0], (u"(also {} in same thread)".format(", ".join([str(x[0]) for x in posts[1:]])) if posts[1:] else u"")
url = "https://archive.foolz.us/a/thread/{:d}/#{:d}".format(t, p[0])
#response = yield treq.post("https://www.googleapis.com/urlshortener/v1/url?key="+API_KEY, json.dumps({"longUrl": url}), headers={'Content-Type': ['application/json']})
#data = yield treq.json_content(response)
#url = data["id"]
excerpt = p[1].replace("<br>", "\n")
excerpt = BeautifulSoup(excerpt).get_text()
excerpt = re.sub("\s+", " ", excerpt)
excerpt = excerpt if len(excerpt) <= 100 else excerpt[:97]+"..."
irc.msg(u"#commie-subs", u"\u00039>{} mentioned on /a/ at {} [{}] {}".format(u", ".join(found), url, excerpt, also))
else:
t, found = [str(x[0]) for x in threads], reduce(lambda a,b: a|b, [x[1] for x in threads])
irc.msg(u"#commie-subs", u"\u00039>{} flooded /a/ in threads {}".format(u", ".join(found), u", ".join(t)))
self.threads = nthreads
self.initial = False
except:
self.err(u"4chan module broke horribly :(")
def lookupCharacters(member, characters):
response = yield treq.get("http://www.bungie.net/Platform/User/GetBungieAccount/" + member["membershipId"].encode("UTF-8") + "/254/")
data = yield treq.json_content(response)
for account in data["Response"]["destinyAccounts"]:
if account["userInfo"]["membershipType"] == 1:
platform = "xbox"
elif account["userInfo"]["membershipType"] == 2:
platform = "playstation"
else:
continue
for character in account["characters"]:
character_data = {
"bungieId": member["membershipId"],
"accountId": account["userInfo"]["membershipId"],
"characterId": character["characterId"],
"name": account["userInfo"]["displayName"],
"race": character["race"]["raceName"],
"gender": character["gender"]["genderName"],
"class": character["characterClass"]["className"],
"level": character["level"],
"levelString": "{:,d}".format(character["level"]),
"icon": "http://bungie.net" + character["emblemPath"],
"background": "http://bungie.net" + character["backgroundPath"],
# Default values for extra data (in case it fails)
"light": 0,
"lightString": "{:,d}".format(0),
"grimoire": 0,
"grimoireString": "{:,d}".format(0),
"minutesPlayed": 0,
"minutesPlayedString": "{:,d}".format(0),
"lastSeen": "",
"lastSeenString": ""
}
character_data["style"] = 'background: url("' + character_data["background"] + '")'
try:
response = yield treq.get("http://www.bungie.net/Platform/Destiny/{!s}/Account/{!s}/Character/{!s}/".format(account["userInfo"]["membershipType"], account["userInfo"]["membershipId"], character["characterId"]))
extra_data = yield treq.json_content(response)
extra_data = extra_data["Response"]["data"]["characterBase"]
character_data["light"] = extra_data["stats"]["STAT_LIGHT"]["value"]
character_data["lightString"] = "{:,d}".format(character_data["light"])
character_data["grimoire"] = extra_data["grimoireScore"]
character_data["grimoireString"] = "{:,d}".format(character_data["grimoire"])
character_data["minutesPlayed"] = int(extra_data["minutesPlayedTotal"])
character_data["minutesPlayedString"] = "{:,d}".format(character_data["minutesPlayed"])
character_data["lastSeen"] = extra_data["dateLastPlayed"]
character_data["lastSeenString"] = datetime.datetime.strptime(extra_data["dateLastPlayed"], "%Y-%m-%dT%H:%M:%SZ").strftime("%B %d, %I:%M%p")
except:
pass
characters[platform].append(character_data)
def lookup_timezone(loc, api_key=''):
"""
Determine the timezone of a lat/long pair.
@param loc: lat/long coordinates of a location
@type loc: dict
@type api_key: str
@rtype: defer.Deferred yielding a second offset representing the timezone
"""
try:
res = yield treq.get(("https://maps.googleapis.com/maps/api/timezone/json"),
params={'location': str(loc['lat']) + ',' + str(loc['lng']),
'timestamp': str(now_in_utc_secs()), 'sensor': 'false', 'key': api_key})
if res and res.code == 200:
data = yield treq.json_content(res)
if data['status'] == 'OK':
# API returned timezone info. What we care about: rawOffset
defer.returnValue(int(data['rawOffset'])+int(data['dstOffset']))
else:
log.warn("Bad status response from Google Geocode API: %s" % (data['status']))
elif res is not None:
log.warn("Bad HTTP status from Google Timezone API: %s" % (res.code))
except TypeError:
log.warn("Bad lat/long parameter passed to lookup_timezone: %s" % (loc), exc_info=True)
def lookup_geocode(location, api_key=''):
"""
Determine the lat/long coordinates for a location name.
:param location: location name
:type location: str
:type api_key: str
:return: a dict with keys 'lat', 'lng', 'loc' that contain
the lat/long coordinates and placename for the location.
:rtype: defer.Deferred
"""
try:
res = yield treq.get("http://maps.googleapis.com/maps/api/geocode/json",
params={'address': location, 'sensor': 'false', 'key': api_key})
if res and res.code == 200:
data = yield treq.json_content(res)
if data['status'] == 'OK':
# API returned at least one geocode
ret = data['results'][0]['geometry']['location']
ret['loc'] = data['results'][0]['formatted_address']
defer.returnValue(ret)
else:
log.warn("Bad status response from Google Geocode API: %s" % (data['status']))
elif res is not None:
log.warn("Bad HTTP status from Google Geocode API: %s" % (res.code))
except TypeError:
log.warn("Bad location passed to lookup_geocode", exc_info=True)
def _startRequest(self, url):
self._buffer = ''
log.info('start request to %r', url)
d = treq.get(url)
d.addCallback(treq.collect, self._dataReceived)
# not sure how to stop this
return d
def do_request(self):
self.log('requesting backend')
d = treq.get(self.create_url())
d.addCallback(self.on_request_success)
d.addErrback(self.on_request_error)
return d
def _send_request(self):
if not self.queue:
if self.request_loop.running:
self.request_loop.stop()
return
now = time.time() - 1 # 1 second buffer
if (self.rate_remaining < 1+1 and self.rate_reset > now or
DiscordRestApiLoop.global_wait > now):
self.log.warn("Rate limited: {}".format(self.channel_id))
return
payload = self.queue.pop()
method = payload['method']
url = payload['url']
content = payload['content']
# url = '{}/channels/{}/messages'.format(HOST, self.channel_id)
content = json.dumps({"content": content})
self.log.debug('at _send_request: {} url {}'.format(self.channel_id,
url))
if method == 'post':
d = treq.post(url, content, headers=HEADERS)
elif method == 'patch':
d = treq.patch(url, content, headers=HEADERS)
elif method == 'delete':
d = treq.delete(url, headers=HEADERS)
elif method == 'get':
d = treq.get(url, headers=HEADERS)
d.addCallback(self.update_rate_limits)
if not self.queue:
self.request_loop.stop()
def geocode(self, address):
"""
This method makes a call to Google's geocoding API. You shouldn't
call this more than 5 times per second
"""
# The url for this API
#endpoint = 'https://maps.googleapis.com/maps/api/geocode/json'
endpoint = 'http://web:9312/maps/api/geocode/json'
# Do the call
parms = [('address', address), ('sensor', 'false')]
response = yield treq.get(endpoint, params=parms)
# Decode the response as json
content = yield response.json()
# If the status isn't ok, return it as a string
if content['status'] != 'OK':
raise Exception('Unexpected status="%s" for address="%s"' %
(content['status'], address))
# Extract the address and geo-point and set item's fields
geo = content['results'][0]["geometry"]["location"]
# Return the final value
defer.returnValue({"lat": geo["lat"], "lon": geo["lng"]})
def forceBuild(branch, reason, tests=None, reactor=None):
"""
Force a build of a given branch.
@return: URL where build results can be found.
"""
if not branch.startswith('/branches/'):
branch = '/branches/' + branch
args = [
('username', 'twisted'),
('passwd', 'matrix'),
('forcescheduler', 'force-supported'),
('revision', ''),
('submit', 'Force Build'),
('branch', branch),
('reason', reason)]
if tests is not None:
args += [('test-case-name', tests)]
url = "http://buildbot.twistedmatrix.com/builders/_all/forceall"
url = url + '?' + '&'.join([k + '=' + urllib.quote(v) for (k, v) in args])
headers = {'user-agent': [USER_AGENT]}
# We don't actually care about the result and buildbot returns a
# relative redirect here. Until recently (#5434) twisted didn't
# handle them, so avoid following the redirect to support released
# versions of twisted.
d = treq.get(url, headers, allow_redirects=False, reactor=reactor)
d.addCallback(lambda _: getURLForBranch(branch))
return d
def sms(self, number, message):
message_encoded = message.encode('utf-8')
params = {'from': '12342492074',
'to': number,
'type': 'unicode',
'text': message_encoded
}
params.update(self.params)
d = treq.get("https://rest.nexmo.com/sms/json", params=params)
def handle_response(response):
def parse_content(content):
result = json.loads(content)
log.msg("Nexmo returned: %s" % content)
errors = []
for message in result['messages']:
if message['status'] != "0":
errors.append((message['status'], message['error-text']))
if len(errors):
raise NexmoException(errors)
response.content().addCallback(parse_content)
d.addCallback(handle_response)
return d
def lookupMembers(id):
page = 1
hasMore = True if id else False
members = []
characters = {
"playstation": [],
"xbox": []
}
deferreds = []
while hasMore:
# No idea what is different between V1, V2, and V3...
response = yield treq.get("http://www.bungie.net/Platform/Group/{!s}/MembersV3/".format(id), params={
"itemsPerPage": 50,
"currentPage": page
})
data = yield treq.json_content(response)
page += 1
hasMore = data["Response"]["hasMore"]
members.extend(data["Response"]["results"])
# Load character data in parallel
for member in members:
deferreds.append(lookupCharacters(member, characters))
yield DeferredList(deferreds, fireOnOneErrback=True, consumeErrors=True)
for platform_characters in characters.values():
platform_characters.sort(key=lambda c: (c["level"], c["light"]), reverse=True)
returnValue(characters)
def test_validate_fail(self):
class Api(object):
app = Klein()
@app.route('/')
@validate(
lambda _: errs1,
lambda _: None,
lambda _: errs2)
def route(self, req):
pass
errs1 = [{
'type': '1',
'message': 'a'
}]
errs2 = [{
'type': 'b',
'message': 'B'
}]
srv = yield ToyServer.from_test(self, Api().app)
resp = yield treq.get(srv.url, persistent=False)
self.assertEqual(resp.code, http.BAD_REQUEST)
self.assertEqual(json.loads((yield resp.content())), {
'errors': errs1 + errs2
})
def check_http_server(host, port):
"""
Check if an HTTP server is running.
Attempts a request to an HTTP server and indicate the success
or failure of the request.
:param bytes host: Host to connect to.
:param int port: Port to connect to.
:return Deferred: Fires with True if the request received a response,
False if the request failed.
"""
req = get(
"http://{host}:{port}".format(host=host, port=port),
timeout=SOCKET_TIMEOUT_FOR_POLLING,
persistent=False,
)
def failed(failure):
return False
def succeeded(result):
return True
req.addCallbacks(succeeded, failed)
return req
def check_status(request):
if not request.args.get("cname"):
pass
# abort(400)
# elif not re.match(request.args.get("cname")):
else:
get_url = "http://india.alibaba.com/supplier_list.htm?bizType=&SearchText="
search_query = request.args.get("cname",[0])[0].strip().replace('.','').replace(',','').upper()
if re.search(comp_name_regex,search_query):
search_query = search_query.replace(" ","%20")
# print search_query
try:
response = yield treq.get(get_url+search_query)
# print response1.status_code
except Exception,e:
print e
log.msg(e,logLevel=logging.DEBUG)
# response.close()
abort(404)
content = yield response.content()
soup = BeautifulSoup(content,"html.parser")
# print soup.find('title').get_text()
name_comps = soup.find_all("a",{"class":"dot-company"})
name_url_list = []
for names in name_comps:
name_url_dict = {}
name_url_dict['Name'] = names.get("title")
name_url_dict['href'] = names.get("href")
name_url_list.append(name_url_dict)
# response.close()
returnValue(json.dumps(name_url_list))
def _monitor(self):
all_deferreds = []
for name, url in self._to_monitor.iteritems():
d = treq.get(url, timeout=5, persistent=False)
d.addBoth(lambda resp, name: (name, resp), name)
all_deferreds.append(d)
all_resp = yield defer.DeferredList(all_deferreds)
status = {}
for (success, (name, resp)) in all_resp:
if not success:
print "deferred error"
elif isinstance(resp, Failure):
print "got failure: %r" % resp
elif resp.code == 200:
json_resp = yield resp.json()
status[name] = (
len(json_resp.get('running', [])),
len(json_resp.get('finished', [])),
len(json_resp.get('pending', [])),
)
to_print = []
for name in sorted(status.keys()):
to_print.append("%-20s running: %d, finished: %d, pending: %d" %
((name,) + status[name]))
print "\033c" + "\n".join(to_print)
def test_validate_fail(self):
class Api(object):
app = Klein()
@app.route('/')
@validate(
lambda _: errs1,
lambda _: None,
lambda _: errs2)
def route(self, req):
pass
errs1 = [{
'type': '1',
'message': 'a'
}]
errs2 = [{
'type': 'b',
'message': 'B'
}]
srv = yield ToyServer.from_test(self, Api().app)
resp = yield treq.get(srv.url, persistent=False)
self.assertEqual(resp.code, http.BAD_REQUEST)
content = yield resp.json()
self.assertEqual(content['result'], {
'errors': sorted(errs1 + errs2)
})
self.assertEqual(content['status'], 400)
self.assertEqual(content['code'], 'Bad Request')
self.assertEqual(content['description'], 'api usage error')
def joke(bot):
"""Chuck Norris jokes from http://icndb.com"""
url = "http://api.icndb.com/jokes/random/1"
@defer.inlineCallbacks
def _tell_joke(response, channel, name=None):
data = yield response.json()
cnjoke = data['value'][0]['joke']
cnjoke = cnjoke.replace(""", "\"")
if name:
cnjoke = cnjoke.replace("Chuck Norris", name)
bot.msg(channel, str(cnjoke), length=510)
while True:
args, sender, senderhost, channel = yield
get(url).addCallback(_tell_joke, channel, " ".join(args))
def main(reactor, *args):
d = treq.get(
'http://httpbin.org/basic-auth/treq/treq',
auth=('treq', 'treq')
)
d.addCallback(print_response)
return d
def get_details(request):
if not request.args.get('url',[0]):
# abort(400)
pass
else:
add_url = "/trustpass_profile.html?certification_type=intl_av"
try:
url = request.args.get('url',[0])[0].strip()
print url
response = yield treq.get(url.split(".html")[0] + add_url)
except Exception ,e:
print e
log.msg(e,logLevel=logging.DEBUG)
# response.close()
# abort(404)
pass
# print response2.status_code
try:
content = yield response.content()
soup = BeautifulSoup(content,"html.parser")
details = soup.find_all("table",{"class":"table"})
basic_details = details[0].find_all("tr")
personl_details = details[1].find_all("tr")
outdict = {}
for idx,lines in enumerate(basic_details):
outdict[clean_text(lines,'th')] = clean_text(lines,'td')
# print clean_text(lines,'td')
for lines in personl_details:
outdict[clean_text(lines,'th')] = clean_text(lines,'td')
returnValue(json.dumps(outdict))
except KeyError:
# abort(400)
pass
def scanip(self, entry):
"""
Scan IP against GreyNoise API
"""
def message(query):
if query["noise"]:
log.msg(
eventid="cowrie.greynoise.result",
format=f"GreyNoise: {query['ip']} has been observed scanning the Internet. GreyNoise "
f"classification is {query['classification']} and the believed owner is {query['name']}",
)
if query["riot"]:
log.msg(
eventid="cowrie.greynoise.result",
format=f"GreyNoise: {query['ip']} belongs to a benign service or provider. "
f"The owner is {query['name']}.",
)
gn_url = f"{GNAPI_URL}{entry['src_ip']}".encode("utf8")
headers = {"User-Agent": [COWRIE_USER_AGENT], "key": self.apiKey}
try:
response = yield treq.get(url=gn_url, headers=headers, timeout=10)
except (
defer.CancelledError,
error.ConnectingCancelledError,
error.DNSLookupError,
):
log.msg("GreyNoise requests timeout")
return
if response.code == 404:
rsp = yield response.json()
log.err(f"GreyNoise: {rsp['ip']} - {rsp['message']}")
return
if response.code != 200:
rsp = yield response.text()
log.err(f"GreyNoise: got error {rsp}")
return
j = yield response.json()
if self.debug:
log.msg("GreyNoise: debug: " + repr(j))
if j["message"] == "Success":
message(j)
else:
log.msg("GreyNoise: no results for for IP {}".format(entry["src_ip"]))
def main(reactor, *args):
d = treq.get('http://httpbin.org/cookies/set?hello=world')
def _get_jar(resp):
jar = resp.cookies()
print 'The server set our hello cookie to: {0}'.format(jar['hello'])
return treq.get('http://httpbin.org/cookies', cookies=jar)
d.addCallback(_get_jar)
d.addCallback(print_response)
return d
def owner_id(self, request):
auth_headers = {}
auth = request.getHeader('Authorization')
if auth:
auth_headers['Authorization'] = auth
uri = "".join([self._auth_bouncer_url, request.path])
resp = yield treq.get(uri, headers=auth_headers, persistent=False)
yield resp.content()
if resp.code >= 400:
returnValue(None)
x_owner_id = resp.headers.getRawHeaders('X-Owner-Id')
if x_owner_id is None or len(x_owner_id) != 1:
returnValue(None)
returnValue(x_owner_id[0])
def download_file(url, destination_filename):
"""
Returns a deferred!
This downloads a file from a URL and saves it toa file.
:param url:
:param destination_filename:
:return:
"""
destination = open(destination_filename, 'wb')
d = treq.get(url, unbuffered=True)
d.addCallback(treq.collect, destination.write)
d.addBoth(lambda _: destination.close())
return d
def get_comic_by_id(self, comic_id):
"""
Returns the info for the given comic
:param url: Comic ID number
:return: Deferred that fires with a dict of info
"""
self.logger.debug("Getting comic by ID")
if comic_id in self._comic_cache:
return defer.succeed(dict(self._comic_cache[comic_id]))
else:
d = treq.get("http://xkcd.com/%s/info.0.json" % comic_id)
d.addCallbacks(self._get_comic_result, self._get_comic_error,
[comic_id])
return d
def products(self, search=None):
header = {
"MARKET-KEY": self.public_key,
"MARKET-TOKEN": self.token,
'Content-Type': 'application/json'
}
query = {}
if search:
query['keyword'] = search
url = utils.build_url(self.url + "product/v1/allproducts/", query)
logger.debug(url)
resp = yield treq.get(url, headers=header)
product_info = yield treq.json_content(resp)
return product_info
def query():
req = get(
"http://{host}:{port}{path}".format(
host=host, port=port, path=path),
timeout=SOCKET_TIMEOUT_FOR_POLLING,
persistent=False,
)
def failed(failure):
Message.new(message_type=u"acceptance:http_query_failed",
reason=unicode(failure)).write()
return False
req.addCallbacks(content, failed)
return req
def get_connected_servers(self):
if not self.nodeurl:
return None
try:
resp = yield treq.get(self.nodeurl)
except ConnectError:
return None
if resp.code == 200:
html = yield treq.content(resp)
match = re.search("Connected to <span>(.+?)</span>",
html.decode("utf-8"))
if match:
return int(match.group(1))
return None
def getOHLCVHistory(self, ticker, period="day", start_datetime=None, end_datetime=None):
payout, denominated = ticker.split('/')
url = self.oanda_uri + "historical-rates/update"
params = {'date_fmt': 'us',
'start_date': start_datetime.strftime('%Y-%m-%d'),
'end_date': end_datetime.strftime('%Y-%m-%d'),
'period': "daily",
'quote_currency': payout,
'base_currency_0': denominated,
'rate': 0,
'view': 'table',
'display': 'absolute',
'price': 'bid',
'data_range': 'd90'}
headers = {'X-Requested-With': 'XMLHttpRequest',
'X-Prototype-Version': '1.7',
'Referer': 'http://www.oanda.com/currency/historical-rates/'}
try:
response = yield treq.get(url, params=params, headers=headers)
except Exception as e:
pass
content = yield response.content()
parsed = json.loads(content)
data = parsed['widget'][0]['data']
ohlcv_history = {}
for row in data:
date = datetime.fromtimestamp(row[0]/1e3)
price = float(row[1])
epoch = datetime.utcfromtimestamp(0)
open_timestamp = int((date - epoch).total_seconds() * 1e6)
tomorrow_date = date + timedelta(days=1)
close_timestamp = int((tomorrow_date - epoch).total_seconds() * 1e6) - 1
ohlcv = {
'contract': ticker,
'open': price,
'high': price,
'low': price,
'close': price,
'volume': 0,
'vwap': price,
'open_timestamp': open_timestamp,
'close_timestamp': close_timestamp,
'period': period
}
ohlcv_history[open_timestamp] = ohlcv
returnValue(ohlcv_history)
def get(self, key):
response = yield treq.get('{}/v2/keys/{}/{}'.format(
self._select_host(), self._prefix, key))
result = yield response.json()
if 'errorCode' in result:
if result['errorCode'] == 100:
# key not found
returnValue(None)
else:
self.log.warn(
'etcd get(): unprocessed error code {error_code}',
error_code=errorCode)
else:
returnValue(result['node']['value'])
def query_comment_by_hash(self, market_hash):
header = {
"MARKET-KEY": self.public_key,
"MARKET-TOKEN": self.token,
'Content-Type': 'application/json'
}
url = utils.build_url(self.url + "comment/v1/comment/list/",
{'market_hash': market_hash})
logger.debug(url)
resp = yield treq.get(url, headers=header)
logger.debug(resp)
comment_info = yield treq.json_content(resp)
logger.debug('upload file info to market confirm: %s', comment_info)
return comment_info['data']
def query_by_following_seller(self):
header = {
"MARKET-KEY": self.public_key,
"MARKET-TOKEN": self.token,
'Content-Type': 'application/json'
}
url = utils.build_url(self.url +
"product/v1/my_seller/product/search/")
logger.debug(url)
resp = yield treq.get(url, headers=header)
logger.debug(resp)
comment_info = yield treq.json_content(resp)
logger.debug('query by following seller confirm: %s', comment_info)
return comment_info['data']
def query_by_seller(self, public_key):
logger.debug("in query by seller")
logger.debug("seller's public key: %s", self.public_key)
logger.debug("public key used by query: %s", public_key)
url = self.url + 'product/v1/es_product/search/?ordering=-created&offset=0&limit=100&status=0&seller=' + str(
public_key)
header = {
"MARKET-KEY": self.public_key,
"MARKET-TOKEN": self.token,
'Content-Type': 'application/json'
}
resp = yield treq.get(url, headers=header, persistent=False)
confirm_info = yield treq.json_content(resp)
return confirm_info['results']
def test_basic_subresources(self):
"""
A basic WSGI app can be ran, with subresources
"""
temp_reactor = SelectReactor()
r = router.RouterWorkerSession(config=self.config,
reactor=temp_reactor)
# Open the transport
transport = FakeWAMPTransport(r)
r.onOpen(transport)
realm_config = {u"name": u"realm1", u'roles': []}
r.start_router_realm(u"realm1", realm_config)
r.start_router_transport(
"component1", {
u"type": u"web",
u"endpoint": {
u"type": u"tcp",
u"port": 8080
},
u"paths": {
u"/": {
"module": u"crossbar.worker.test.test_router",
"object": u"hello",
"type": u"wsgi"
},
u"json": {
"type": u"json",
"value": {}
}
}
})
# Make a request to the /json endpoint, which is technically a child of
# the WSGI app, but is not served by WSGI.
d = treq.get("http://localhost:8080/json", reactor=temp_reactor)
d.addCallback(treq.content)
d.addCallback(self.assertEqual, b"{}")
d.addCallback(lambda _: temp_reactor.stop())
def escape():
if temp_reactor.running:
temp_reactor.stop()
temp_reactor.callLater(1, escape)
temp_reactor.run()
return d
def ping(self):
"""
Start a timer which will bounce messages off the API gateway on a regular basis and (re)register
endpoints if they're not already registered.
"""
try:
api_ping = '{}://{}:{}/api/1.0.0/ping/{}/None'.format(
self._env.api_protocol, self._env.api_host, self._env.api_port,
self._key)
self._env.logger.info('PING> {}'.format(api_ping))
def check(response):
if response.code == 204:
self._env.logger.info('204 - Registering new routes')
self.register_routes()
elif response.code == 404:
self._env.logger.info('404 - api gateway is not available')
elif response.code != 200:
self._env.logger.error('{} - UNKNOWN ERROR'.format(
response.code))
return response
def log(failure):
"""
Just log the error, a return code of 'False' will be returned elsewhere
:param failure: A treq failure object
"""
return self._env.logger.warning('[ping] {}'.format(
failure.getErrorMessage()))
treq.get(api_ping).addCallback(check).addErrback(log)
except requests.exceptions.ConnectionError as e:
self._env.logger.warning('ping failed for "{}"'.format(api_ping))
self._env.logger.warning('ping return = "{}"'.format(
e.args[0].reason))
self._state = False
def main(reactor):
print('List of tuples')
resp = yield treq.get('https://httpbin.org/get',
params=[('foo', 'bar'), ('baz', 'bax')])
content = yield resp.text()
print(content)
print('Single value dictionary')
resp = yield treq.get('https://httpbin.org/get',
params={
'foo': 'bar',
'baz': 'bax'
})
content = yield resp.text()
print(content)
print('Multi value dictionary')
resp = yield treq.get('https://httpbin.org/get',
params={b'foo': [b'bar', b'baz', b'bax']})
content = yield resp.text()
print(content)
print('Mixed value dictionary')
resp = yield treq.get('https://httpbin.org/get',
params={
'foo': [1, 2, 3],
'bax': b'quux',
b'bar': 'foo'
})
content = yield resp.text()
print(content)
print('Preserved query parameters')
resp = yield treq.get('https://httpbin.org/get?foo=bar',
params={'baz': 'bax'})
content = yield resp.text()
print(content)
def test_json_body(self):
class Api(object):
app = Klein()
@app.route('/')
@json_body
def route(self, req, body):
bodies.append(body)
bodies = []
srv = yield ToyServer.from_test(self, Api().app)
yield treq.get(srv.url, persistent=False, data=json.dumps({'foo': 23}))
self.assertEqual(bodies, [{'foo': 23}])
def _report_authors(data, headers):
print("Commits:")
commits_resp = yield treq.get(data['commits_url'], headers=headers)
commits_data = yield commits_resp.text()
commits = json.loads(commits_data)
authors = set()
for commit in commits:
if commit['author'] is None:
print(" {}: no author!".format(commit['sha']))
else:
author = commit['author']['login']
print(" {}: {}".format(commit['sha'], author))
if author not in ignore_handles:
authors.add(author)
returnValue(authors)
def main(reactor):
print 'List of tuples'
resp = yield treq.get('http://httpbin.org/get',
params=[('foo', 'bar'), ('baz', 'bax')])
content = yield treq.content(resp)
print content
print 'Single value dictionary'
resp = yield treq.get('http://httpbin.org/get',
params={
'foo': 'bar',
'baz': 'bax'
})
content = yield treq.content(resp)
print content
print 'Multi value dictionary'
resp = yield treq.get('http://httpbin.org/get',
params={'foo': ['bar', 'baz', 'bax']})
content = yield treq.content(resp)
print content
print 'Mixed value dictionary'
resp = yield treq.get('http://httpbin.org/get',
params={
'foo': ['bar', 'baz'],
'bax': 'quux'
})
content = yield treq.content(resp)
print content
print 'Preserved query parameters'
resp = yield treq.get('http://httpbin.org/get?foo=bar',
params={'baz': 'bax'})
content = yield treq.content(resp)
print content
def treq_get(dispatcher, intent):
"""
Performer to execute an HTTP GET.
:param dispatcher: The dispatcher used to dispatch this performance.
:param HTTPGet intent: The intent to be performed.
"""
action = startAction(action_type=u"flocker:provision:_effect:treq_get")
with action.context():
Message.log(url=intent.url)
# Do not use persistent HTTP connections, because they will not be
# cleaned up by the end of the test.
d = DeferredContext(get(intent.url, persistent=False))
d.addActionFinish()
return d.result
def request_get(url, load_json=False):
log.msg("sending request to url %s" % url)
headers = {
"user-agent": "Mozilla/5.0 (Linux; U; Android 2.2; en-us; SCH-I800 Build/FROYO) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1"}
dfd = get(url, headers=headers)
def done(response):
if load_json:
dfd = response.json()
else:
dfd = response.content()
return dfd
dfd.addCallback(done)
return dfd
def download(reactor):
parser = ArgumentParser()
parser.add_argument('url')
arguments = parser.parse_args()
response = yield treq.get(arguments.url, timeout=30.0, reactor=reactor)
if response.code != http.OK:
reason = http.RESPONSES[response.code]
raise RuntimeError(f'Failed: {response.code} {reason}')
content = yield response.content()
parsed = feedparser.parse(content)
print(parsed['feed']['title'])
print(parsed['feed']['description'])
print('*** ENTRIES ***')
for entry in parsed['entries']:
print(entry['title'])
def created():
"""
Check the dataset state list for the expected dataset.
"""
request = get(self.base_url + b"/state/datasets", persistent=False)
request.addCallback(content)
def got_body(body):
body = loads(body)
# Current state listing includes bogus metadata
# https://clusterhq.atlassian.net/browse/FLOC-1386
expected_dataset = dataset_properties.copy()
expected_dataset[u"metadata"].clear()
return expected_dataset in body
request.addCallback(got_body)
return request
def fetch_icon(self, url, dest):
agent = None
if self.use_tor:
tor = yield get_tor(reactor)
if not tor:
raise TorError("Could not connect to a running Tor daemon")
agent = tor.web_agent()
resp = yield treq.get(url, agent=agent)
if resp.code == 200:
content = yield treq.content(resp)
log.debug("Received %i bytes", len(content))
with atomic_write(dest, mode="wb", overwrite=True) as f:
f.write(content)
self.got_icon.emit(dest)
else:
log.warning("Error fetching service icon: %i", resp.code)
def go(self):
"""
Attempt to download L{self.url}.
"""
headers = self.kwargs.pop('headers', Headers())
headers.setRawHeaders('user-agent', ['Eridanus IRC bot'])
response = yield treq.get(str(self.url),
timeout=self.timeout,
headers=headers,
*self.args,
**self.kwargs)
data = yield response.content()
if response.code // 100 == 2:
returnValue((data, response.headers))
else:
raise weberror.Error(response.code, response.phrase, data)
def get(self, call, params={}, auth=False):
url = self.endpoint + call
if auth:
headers = self.get_auth(call, params)
else:
headers = None
response = yield treq.get(url, params=params, headers=headers)
content = yield response.content()
result = json.loads(content)
if 'message' in result:
raise BitFinexException(result['message'])
returnValue(result)
def wgetDownload(self, url):
"""
Download `url`
"""
headers = {"User-Agent": ["curl/7.38.0"]}
# TODO: use designated outbound interface
# out_addr = None
# if CowrieConfig.has_option("honeypot", "out_addr"):
# out_addr = (CowrieConfig.get("honeypot", "out_addr"), 0)
deferred = treq.get(url=url,
allow_redirects=True,
headers=headers,
timeout=10)
return deferred
def _make_request(self, method, payload):
"""
Actually make the HTTP request.
:rtype : twisted.internet.defer.Deferred
"""
# Convert unicode strings to utf8-encoded bytestrings (treq doesn't
# seem to be able to encode unicode strings properly)
# Alters payload in-place, but this is a "private" method, and we don't
# reuse is after this call anyway.
payload["client_id"] = "Ultros-Domai.nr"
for k, v in payload.iteritems():
if isinstance(v, unicode):
payload[k] = v.encode("utf8")
deferred = treq.get(self.API_URL + method, params=payload)
deferred.addCallback(self._handle_response)
return deferred
def doUpdate(self, reactor):
"""
Make an HTTP GET to update-url with the new digest
"""
alias = self.parent['alias']
assert alias in DBAlias
connect(**DBHost[alias])
url = self['update-url'] + self._digest
token = usertoken.get(LOCALAPI_EMAIL)
res = yield treq.get(url, headers={'x-token': token})
response = yield res.content()
assert res.code == 200, 'Fail %s\n%s' % (res.code, response)
print 'update success'
def send_request(self, method, endpoint, params=None, data=None):
if params is None:
params = {}
if method.lower() == 'get':
response = yield treq.get(self.host_address + endpoint,
headers=None,
params=params)
content = yield response.text()
elif method.lower() == 'post':
response = yield treq.post(self.host_address + endpoint,
params=params,
data=data)
content = yield response.text()
else:
raise IndexError('Wrong http method')
return content
