def post(self, *args, **kwargs):
print('进入')
http = httpclient.AsyncHTTPClient()
data = yield http.fetch("http://127.0.0.1:8000/log", body={22: 44})
print('完事', data)
self.finish('6666')
def getInfo(url):
http_client = httpclient.AsyncHTTPClient()
http_client.fetch(url, callback=processInfo)
def get_auth_http_client(self):
return httpclient.AsyncHTTPClient()
def __init__(self, *args, **kwargs):
self.timeout = kwargs.pop('timeout', None)
super(HTTPClient, self).__init__(*args, **kwargs)
self.client = httpclient.AsyncHTTPClient()
def download(url):
http_client = httpclient.AsyncHTTPClient()
http_client.fetch(url, callback=doUpdate)
def _FetchGoogleContacts(self):
"""Do GMail specific data gathering and checking.
Queries Google data API for contacts in JSON format.
"""
# Track fetched contacts regardless of rank in order to dedup contacts retrieved from Google.
assert self._identity.refresh_token is not None, self._identity
if self._identity.expires and self._identity.expires < time.time():
yield gen.Task(self._identity.RefreshGoogleAccessToken,
self._client)
logging.info('fetching Google contacts for identity %r...' %
self._identity)
http_client = httpclient.AsyncHTTPClient()
# Google data API uses 1-based start index.
start_index = 1
retries = 0
count = FetchContactsOperation._MAX_FETCH_COUNT
while True:
if retries >= FetchContactsOperation._MAX_FETCH_RETRIES:
raise TooManyRetriesError(
'failed to fetch contacts %d times; aborting' % retries)
logging.info('fetching next %d Google contacts for user %d' %
(count, self._user_id))
url = FetchContactsOperation._GOOGLE_CONTACTS_URL + '?' + \
urllib.urlencode({'max-results': count,
'start-index': start_index,
'alt': 'json'})
response = yield gen.Task(http_client.fetch,
url,
method='GET',
headers={
'Authorization':
'OAuth %s' %
self._identity.access_token,
'GData-Version':
3.0
})
try:
response_dict = www_util.ParseJSONResponse(response)['feed']
except Exception as exc:
logging.warning('failed to fetch Google contacts: %s' % exc)
retries += 1
continue
# Temporarily log additional information to figure out why some responses don't seem to have "entry" fields.
if 'entry' not in response_dict:
logging.warning('Missing entry: %s' %
json.dumps(response_dict, indent=True))
for c_dict in response_dict.get('entry', []):
# Build identities_properties list from all emails/phone numbers associated with this contact.
identities_properties = []
# Process emails first so that if there are any emails, one of them will be first in the
# identities_properties list. This will be *the* identity used for down-level client message
# migration.
for email_info in c_dict.get('gd$email', []):
email = email_info.get('address', None)
if email is not None:
email_type = FetchContactsOperation._GOOGLE_TYPE_LOOKUP.get(
email_info.get('rel', None), None)
identity_properties = (
'Email:' + Identity.CanonicalizeEmail(email),
email_info.get('label', email_type))
if email_info.get('primary', False):
# Insert the primary email address at the head of the list. Older clients will get this
# as the only email address for this contact when they query_contacts.
identities_properties.insert(
0, identity_properties)
else:
identities_properties.append(identity_properties)
for phone_info in c_dict.get('gd$phoneNumber', []):
# See RFC3966: "The tel URI for Telephone Numbers" for more information about this format.
# It should be 'tel:' + E.164 format phone number.
phone = phone_info.get('uri', None)
if phone is not None and phone.startswith(
'tel:+') and Identity.CanCanonicalizePhone(
phone[4:]):
phone_type = FetchContactsOperation._GOOGLE_TYPE_LOOKUP.get(
phone_info.get('rel', None), None)
identities_properties.append(
('Phone:' + Identity.CanonicalizePhone(phone[4:]),
phone_info.get('label', phone_type)))
if len(identities_properties) == 0:
continue
# Normalize name to None if empty.
gd_name = c_dict.get('gd$name', None)
if gd_name is not None:
names = {
'name':
gd_name.get('gd$fullName', {}).get('$t', None),
'given_name':
gd_name.get('gd$givenName', {}).get('$t', None),
'family_name':
gd_name.get('gd$familyName', {}).get('$t', None)
}
else:
names = {
'name': None,
'given_name': None,
'family_name': None
}
fetched_contact = Contact.CreateFromKeywords(
self._user_id,
identities_properties,
self._notify_timestamp,
Contact.GMAIL,
rank=None,
**names)
self._fetched_contacts[
fetched_contact.contact_id] = fetched_contact
# Prepare to fetch next batch.
# Indexes are 1-based, so add 1 to max_index.
if 'openSearch$totalResults' in response_dict:
max_index = int(
response_dict['openSearch$totalResults']['$t']) + 1
else:
max_index = FetchContactsOperation._MAX_GOOGLE_CONTACTS + 1
next_index = int(
response_dict['openSearch$startIndex']['$t']) + len(
response_dict.get('entry', []))
count = min(max_index - next_index,
FetchContactsOperation._MAX_FETCH_COUNT)
if len(
self._fetched_contacts
) < FetchContactsOperation._MAX_GOOGLE_CONTACTS and count > 0:
start_index = next_index
retries = 0
continue
else:
raise gen.Return()
def _DetermineFacebookRankings():
"""Uses The tags from friends and the authors of the
photos are used to determine friend rank for facebook contacts. The
basic algorithm is:
sorted([sum(exp_decay(pc.time) * strength(pc)) for pc in photos])
A 'pc' in is a photo connection. There are three types, ordered by
the 'strength' they impart in the summation equation:
- from: the poster of a photo (strength=1.0)
- tag: another user tagged in the photo (strength=1.0)
- like: a facebook user who 'liked' the photo (strength=0.25)
Exponential decay uses _FACEBOOK_CONNECTION_HALF_LIFE for half life.
The rankings are passed to the provided callback as a dictionary of
identity ('FacebookGraph:<id>') => rank.
"""
logging.info(
'determining facebook contact rankings for identity %r...' %
self._identity)
http_client = httpclient.AsyncHTTPClient()
friends = dict() # facebook id => connection strength
likes = dict()
now = util.GetCurrentTimestamp()
def _ComputeScore(create_iso8601, conn_type):
"""Computes the strength of a photo connection based on the time
that's passed and the connection type.
"""
decay = 0.001 # default is 1/1000th
if create_iso8601:
dt = iso8601.parse_date(create_iso8601)
create_time = calendar.timegm(dt.utctimetuple())
decay = math.exp(
-math.log(2) * (now - create_time) /
FetchContactsOperation._FACEBOOK_CONNECTION_HALF_LIFE)
return decay * FetchContactsOperation._PHOTO_CONNECTION_STRENGTHS[
conn_type]
# Construct the URL that will kick things off.
url = FetchContactsOperation._FACEBOOK_PHOTOS_URL + '?' + \
urllib.urlencode({'access_token': self._identity.access_token,
'format': 'json', 'limit': FetchContactsOperation._MAX_FETCH_COUNT})
while True:
logging.info(
'querying next %d Facebook photos for user %d' %
(FetchContactsOperation._MAX_FETCH_COUNT, self._user_id))
response = yield gen.Task(http_client.fetch, url, method='GET')
response_dict = www_util.ParseJSONResponse(response)
for p_dict in response_dict['data']:
created_time = p_dict.get('created_time', None)
if p_dict.get('from', None) and p_dict['from']['id']:
from_id = p_dict['from']['id']
friends[from_id] = friends.get(from_id, 0.0) + \
_ComputeScore(created_time, 'from')
if p_dict.get('tags', None):
for tag in p_dict['tags']['data']:
if tag.get('id', None) is not None:
friends[tag['id']] = friends.get(tag['id'], 0.0) + \
_ComputeScore(tag.get('created_time', None), 'tag')
if p_dict.get('likes', None):
for like in p_dict['likes']['data']:
if like.get('id', None) is not None:
likes[like['id']] = likes.get(like['id'], 0.0) + \
_ComputeScore(created_time, 'like')
if (len(response_dict['data'])
== FetchContactsOperation._MAX_FETCH_COUNT
and response_dict.has_key('paging')
and response_dict['paging'].has_key('next')):
url = response_dict['paging']['next']
else:
for fb_id in friends.keys():
friends[fb_id] += likes.get(fb_id, 0.0)
ranked_friends = sorted(friends.items(),
key=itemgetter(1),
reverse=True)
logging.info(
'successfully ranked %d Facebook contacts for user %d'
% (len(ranked_friends), self._user_id))
raise gen.Return(dict([('FacebookGraph:%s' % fb_id, rank) for rank, (fb_id, _) in \
izip(xrange(len(ranked_friends)), ranked_friends)]))
def get_async_client():
http_client = httpclient.AsyncHTTPClient()
return http_client
def handler(self, *args):
http = httpclient.AsyncHTTPClient()
http.fetch("http://api.flickr.com/services/feeds/photos_public.gne",
callback=self.on_response)
def __init__(self, host='127.0.0.1', port=8500, scheme='http'):
self.host = host
self.port = port
self.scheme = scheme
self.base_uri = '%s://%s:%s' % (self.scheme, self.host, self.port)
self.client = httpclient.AsyncHTTPClient()
def post(self, *args, **kwargs):
xml_data = self.request.body
dict_data = xmltodict.parse(xml_data)
msg_type = dict_data['xml']['MsgType']
if msg_type == 'text':
content = dict_data['xml']['Content']
req = robot(content)
client = httpclient.AsyncHTTPClient()
response = yield client.fetch(req)
if response.error:
self.send_error(500)
else:
data = json.loads(response.body.decode('utf-8'))
resp_data = {
'xml': {
'ToUserName': dict_data['xml']['FromUserName'],
'FromUserName': dict_data['xml']['ToUserName'],
'CreateTime': int(time.time()),
'MsgType': 'text',
'Content': data["results"][0]["values"]["text"]
}
}
self.write(xmltodict.unparse(resp_data))
elif msg_type == 'image':
media_id = dict_data['xml']['MediaId']
resp_data = {
'xml': {
'ToUserName': dict_data['xml']['FromUserName'],
'FromUserName': dict_data['xml']['ToUserName'],
'CreateTime': int(time.time()),
'MsgType': 'image',
'Image': {
'MediaId': media_id
}
}
}
self.write(xmltodict.unparse(resp_data))
elif msg_type == 'voice':
recognition = dict_data['xml']['Recognition']
media_id = dict_data['xml']['MediaId']
print(recognition)
resp_data = {
'xml': {
'ToUserName': dict_data['xml']['FromUserName'],
'FromUserName': dict_data['xml']['ToUserName'],
'CreateTime': int(time.time()),
'MsgType': 'voice',
'Voice': {
'MediaId': media_id
}
}
}
if recognition == '授权链接':
resp_data = {
'xml': {
'ToUserName':
dict_data['xml']['FromUserName'],
'FromUserName':
dict_data['xml']['ToUserName'],
'CreateTime':
int(time.time()),
'MsgType':
'text',
'Content':
'https://open.weixin.qq.com/connect/oauth2/authorize?appid=%s&redirect_uri=%s&response_type=code&scope=snsapi_userinfo&state=iloveu#wechat_redirect'
% (WECHAT_APPID, quote(REDIRECT_URI))
}
}
self.write(xmltodict.unparse(resp_data))
elif msg_type == 'event':
event_type = dict_data['xml']['Event']
if event_type == 'subscribe':
resp_data = {
'xml': {
'ToUserName': dict_data['xml']['FromUserName'],
'FromUserName': dict_data['xml']['ToUserName'],
'CreateTime': int(time.time()),
'MsgType': 'text',
'Content': 'Hello Python --Tornado'
}
}
event_key = dict_data['xml'].get('EventKey', None)
if event_key:
scend_id = event_key[8:]
resp_data['xml'][
'Content'] = 'The parameter of this qr code is:%s' % scend_id
self.write(xmltodict.unparse(resp_data))
elif event_type == 'SCAN':
event_key = dict_data['xml']['EventKey']
resp_data = {
'xml': {
'ToUserName':
dict_data['xml']['FromUserName'],
'FromUserName':
dict_data['xml']['ToUserName'],
'CreateTime':
int(time.time()),
'MsgType':
'text',
'Content':
'Thank you for your attention. The parameters of your scan are:： %s'
% event_key
}
}
self.write(xmltodict.unparse(resp_data))
elif event_type == 'unsubscribe':
resp_data = {
'xml': {
'ToUserName': dict_data['xml']['FromUserName'],
'FromUserName': dict_data['xml']['ToUserName'],
'CreateTime': int(time.time()),
'MsgType': 'text',
'Content': 'Bey --Tornado'
}
}
self.write(xmltodict.unparse(resp_data))
else: # Other events
resp_data = {
'xml': {
'ToUserName': dict_data['xml']['FromUserName'],
'FromUserName': dict_data['xml']['ToUserName'],
'CreateTime': int(time.time()),
'MsgType': 'text',
'Content': 'The universal response'
}
}
self.write(xmltodict.unparse(resp_data))
else:
resp_data = {
'xml': {
'ToUserName': dict_data['xml']['FromUserName'],
'FromUserName': dict_data['xml']['ToUserName'],
'CreateTime': int(time.time()),
'MsgType': 'text',
'Content': 'The universal response'
}
}
self.write(xmltodict.unparse(resp_data))
def let_the_streaming_begin(io_loop, bootstrap_ip_ports):
#Setup the DHT
global dht
dht = DHT(51414, bootstrap_ip_ports, io_loop = io_loop)
#Setup KTorrent and Its URL Handlers
settings = dict( (k, v.value()) for k,v in options.items() )
application = BTApplication(routes, **settings)
Connection.ioloop = io_loop
Connection.application = application
settings['log_function'] = request_logger
frontend_application = tornado.web.Application(frontend_routes, **settings)
frontend_server = tornado.httpserver.HTTPServer(frontend_application, io_loop=io_loop)
Connection.frontend_server = frontend_server
try:
frontend_server.bind(options.frontend_port, '')
frontend_server.start()
#logging.info('started frontend server')
except:
logging.error('could not start frontend server')
btserver = BTProtocolServer(application, io_loop=io_loop)
btserver.bind(options.port, '')
btserver.start()
logging.info('started btserver')
logging.info('\n\n')
Client.resume()
client = Client.instances[0]
Client.http_client = httpclient.AsyncHTTPClient()
Torrent.client = client
tornado.ioloop.PeriodicCallback( Connection.make_piece_request, 1000 * 1, io_loop=io_loop ).start()
tornado.ioloop.PeriodicCallback( Connection.get_metainfo, 1000 * 1, io_loop=io_loop ).start() # better to make event driven
tornado.ioloop.PeriodicCallback( Client.tick, 1000 * 1, io_loop=io_loop ).start()
tornado.ioloop.PeriodicCallback( client.do_trackers, 1000 * 1, io_loop=io_loop ).start()
tornado.ioloop.PeriodicCallback( client.peer_think, 3000 * 1, io_loop=io_loop ).start()
tornado.ioloop.PeriodicCallback( Connection.cleanup_old_requests, 1000 * 1, io_loop=io_loop ).start()
def got_interrupt_signal(signum=None, frame=None):
logging.info('got quit signal ... saving quick resume')
Client.save_settings()
#Torrent.save_quick_resume()
sys.exit()
signal.signal(signal.SIGINT, got_interrupt_signal)
settings = dict( (k, v.value()) for k,v in options.items() )
dht_frontend_routes = [
('/?', dhttornado.IndexHandler),
]
dht_frontend_application = tornado.web.Application(dht_frontend_routes, **settings)
dht_frontend_server = tornado.httpserver.HTTPServer(dht_frontend_application, io_loop=io_loop)
dht_frontend_server.bind(options.dht_frontend_port, '')
dht_frontend_server.start()
dhttornado.IndexHandler.register_dht(dht)
dht.bootstrap()
dht.start() #This also does io_loop.start()
def add_logs(self):
api_client = httpclient.AsyncHTTPClient()
yield api_client.fetch(self.url,
method='POST',
body=httputil.urlencode(self.data))
def initialize(self, format=None, format_prefix=""):
self.format = format or self.default_format
self.format_prefix = format_prefix
self.http_client = httpclient.AsyncHTTPClient()
def __init__(self, search, i, session):
self.search = search
self.id = i
self.client = httpclient.AsyncHTTPClient()
self.session = session
super(DetailWorker, self).__init__()
def init_handler(self):
self.room = self.options.get('room')
self.key = self.options.get('key')
assert self.room, 'Hipchat room is not defined.'
assert self.key, 'Hipchat key is not defined.'
self.client = hc.AsyncHTTPClient()
async def proxy(self, host, port, proxied_path):
'''
This serverextension handles:
{base_url}/proxy/{port([0-9]+)}/{proxied_path}
{base_url}/proxy/absolute/{port([0-9]+)}/{proxied_path}
{base_url}/{proxy_base}/{proxied_path}
'''
if not self._check_host_whitelist(host):
self.set_status(403)
self.write(
"Host '{host}' is not whitelisted. "
"See https://jupyter-server-proxy.readthedocs.io/en/latest/arbitrary-ports-hosts.html for info."
.format(host=host))
return
if 'Proxy-Connection' in self.request.headers:
del self.request.headers['Proxy-Connection']
self._record_activity()
if self.request.headers.get("Upgrade", "").lower() == 'websocket':
# We wanna websocket!
# jupyterhub/jupyter-server-proxy@36b3214
self.log.info(
"we wanna websocket, but we don't define WebSocketProxyHandler"
)
self.set_status(500)
body = self.request.body
if not body:
if self.request.method == 'POST':
body = b''
else:
body = None
client = httpclient.AsyncHTTPClient()
req = self._build_proxy_request(host, port, proxied_path, body)
try:
response = await client.fetch(req, raise_error=False)
except httpclient.HTTPError as err:
if err.code == 599:
self._record_activity()
self.set_status(599)
self.write(str(err))
return
else:
raise
# record activity at start and end of requests
self._record_activity()
# For all non http errors...
if response.error and type(response.error) is not httpclient.HTTPError:
self.set_status(500)
self.write(str(response.error))
else:
self.set_status(response.code, response.reason)
# clear tornado default header
self._headers = httputil.HTTPHeaders()
for header, v in response.headers.get_all():
if header not in ('Content-Length', 'Transfer-Encoding',
'Content-Encoding', 'Connection'):
# some header appear multiple times, eg 'Set-Cookie'
self.add_header(header, v)
if response.body:
self.write(response.body)
def authenticate_redirect(self):
http = httpclient.AsyncHTTPClient()
http.fetch(
self._oauth_request_token_url(),
self.async_callback(self._on_request_token,
self._OAUTH_AUTHENTICATE_URL, None))
def _FetchFacebookContacts(self):
"""Do Facebook specific data gathering and checking.
Queries Facebook graph API for friend list using the identity's access token.
"""
@gen.coroutine
def _DetermineFacebookRankings():
"""Uses The tags from friends and the authors of the
photos are used to determine friend rank for facebook contacts. The
basic algorithm is:
sorted([sum(exp_decay(pc.time) * strength(pc)) for pc in photos])
A 'pc' in is a photo connection. There are three types, ordered by
the 'strength' they impart in the summation equation:
- from: the poster of a photo (strength=1.0)
- tag: another user tagged in the photo (strength=1.0)
- like: a facebook user who 'liked' the photo (strength=0.25)
Exponential decay uses _FACEBOOK_CONNECTION_HALF_LIFE for half life.
The rankings are passed to the provided callback as a dictionary of
identity ('FacebookGraph:<id>') => rank.
"""
logging.info(
'determining facebook contact rankings for identity %r...' %
self._identity)
http_client = httpclient.AsyncHTTPClient()
friends = dict() # facebook id => connection strength
likes = dict()
now = util.GetCurrentTimestamp()
def _ComputeScore(create_iso8601, conn_type):
"""Computes the strength of a photo connection based on the time
that's passed and the connection type.
"""
decay = 0.001 # default is 1/1000th
if create_iso8601:
dt = iso8601.parse_date(create_iso8601)
create_time = calendar.timegm(dt.utctimetuple())
decay = math.exp(
-math.log(2) * (now - create_time) /
FetchContactsOperation._FACEBOOK_CONNECTION_HALF_LIFE)
return decay * FetchContactsOperation._PHOTO_CONNECTION_STRENGTHS[
conn_type]
# Construct the URL that will kick things off.
url = FetchContactsOperation._FACEBOOK_PHOTOS_URL + '?' + \
urllib.urlencode({'access_token': self._identity.access_token,
'format': 'json', 'limit': FetchContactsOperation._MAX_FETCH_COUNT})
while True:
logging.info(
'querying next %d Facebook photos for user %d' %
(FetchContactsOperation._MAX_FETCH_COUNT, self._user_id))
response = yield gen.Task(http_client.fetch, url, method='GET')
response_dict = www_util.ParseJSONResponse(response)
for p_dict in response_dict['data']:
created_time = p_dict.get('created_time', None)
if p_dict.get('from', None) and p_dict['from']['id']:
from_id = p_dict['from']['id']
friends[from_id] = friends.get(from_id, 0.0) + \
_ComputeScore(created_time, 'from')
if p_dict.get('tags', None):
for tag in p_dict['tags']['data']:
if tag.get('id', None) is not None:
friends[tag['id']] = friends.get(tag['id'], 0.0) + \
_ComputeScore(tag.get('created_time', None), 'tag')
if p_dict.get('likes', None):
for like in p_dict['likes']['data']:
if like.get('id', None) is not None:
likes[like['id']] = likes.get(like['id'], 0.0) + \
_ComputeScore(created_time, 'like')
if (len(response_dict['data'])
== FetchContactsOperation._MAX_FETCH_COUNT
and response_dict.has_key('paging')
and response_dict['paging'].has_key('next')):
url = response_dict['paging']['next']
else:
for fb_id in friends.keys():
friends[fb_id] += likes.get(fb_id, 0.0)
ranked_friends = sorted(friends.items(),
key=itemgetter(1),
reverse=True)
logging.info(
'successfully ranked %d Facebook contacts for user %d'
% (len(ranked_friends), self._user_id))
raise gen.Return(dict([('FacebookGraph:%s' % fb_id, rank) for rank, (fb_id, _) in \
izip(xrange(len(ranked_friends)), ranked_friends)]))
logging.info('fetching Facebook contacts for identity %r...' %
self._identity)
http_client = httpclient.AsyncHTTPClient()
# Track fetched contacts regardless of rank in order to dedup contacts retrieved from Facebook.
rankless_ids = set()
# First get the rankings and then fetch the contacts.
rankings = yield _DetermineFacebookRankings()
url = FetchContactsOperation._FACEBOOK_FRIENDS_URL + '?' + \
urllib.urlencode({'fields': 'first_name,name,last_name',
'access_token': self._identity.access_token,
'format': 'json', 'limit': FetchContactsOperation._MAX_FETCH_COUNT})
retries = 0
while True:
if retries >= FetchContactsOperation._MAX_FETCH_RETRIES:
raise TooManyRetriesError(
'failed to fetch contacts %d times; aborting' % retries)
logging.info(
'fetching next %d Facebook contacts for user %d' %
(FetchContactsOperation._MAX_FETCH_COUNT, self._user_id))
response = yield gen.Task(http_client.fetch, url, method='GET')
try:
response_dict = www_util.ParseJSONResponse(response)
except Exception as exc:
logging.warning('failed to fetch Facebook contacts: %s' % exc)
retries += 1
continue
for c_dict in response_dict['data']:
if c_dict.has_key('id'):
ident = 'FacebookGraph:%s' % c_dict['id']
# Skip contact if name is not present, or is empty.
name = c_dict.get('name', None)
if name:
names = {
'name': name,
'given_name': c_dict.get('first_name', None),
'family_name': c_dict.get('last_name', None)
}
# Check to see if we've already processed an identical contact.
rankless_id = Contact.CalculateContactEncodedDigest(
identities_properties=[(ident, None)], **names)
if rankless_id in rankless_ids:
# Duplicate among fetched contacts. Skip it.
continue
else:
rankless_ids.add(rankless_id)
rank = rankings[ident] if ident in rankings else None
fetched_contact = Contact.CreateFromKeywords(
self._user_id, [(ident, None)],
self._notify_timestamp,
Contact.FACEBOOK,
rank=rank,
**names)
self._fetched_contacts[
fetched_contact.contact_id] = fetched_contact
# Prepare to fetch next batch.
if (len(response_dict['data'])
== FetchContactsOperation._MAX_FETCH_COUNT
and response_dict.has_key('paging')
and response_dict['paging'].has_key('next')):
retries = 0
url = response_dict['paging']['next']
else:
break
def _get_next_id():
response = yield httpclient.AsyncHTTPClient().fetch(DB_USERS)
data = json.loads(response.body.decode('utf-8'))
i = int(data['doc_count']) + 1000
zeros = '0' * (9 - len(str(i)))
raise gen.Return(zeros + str(i))
os.remove('url.txt')
f = open('url.txt', 'a')
urllies = ['']
while n > 0:
url = raw_input("what url would you like to test: ")
req = Request(url)
try:
response = urlopen(req)
except URLError, e:
if hasattr(e, 'reason'):
print(url + ' failed for this reason: ' + e.reason)
elif hasattr(e, 'code'):
print(url + ' failed because of code: ' + e.code)
else:
urllies.append(str(url))
f.write(url + '\n')
n -= 1
urllies.pop(0)
f.close()
print("test started at: " + str(time.time()))
httpc = httpclient.AsyncHTTPClient()
while m > 0:
for url in open('url.txt'):
print(urllies[i] + " at " + str(time.time()) + " responds with:")
i += 1
httpc.fetch(url.strip(), handle_request, method='HEAD')
m -= 1
ioloop.IOLoop.instance().start()
print("test ended at: " + str(time.time()))
def _get_records_count(self):
response = yield httpclient.AsyncHTTPClient().fetch(DB_RECORDS)
dat = json.loads(response.body.decode('utf-8'))
return int(dat['doc_count']) + 1000
id = obj[id_field]
temp = '{ "index" : { "_id" : "' + id + '" } }\n' + json.dumps(
obj) + '\n'
body = body + temp
i = 0
def handle_request(response):
print '%d %s' % (response.code, response.body)
global i
i -= 1
if i == 0:
ioloop.IOLoop.instance().stop()
http_client = httpclient.AsyncHTTPClient()
postHeaders = {"Content-Type": "application/json"}
#for future implementation
for iterator in range(1):
i += 1
print("sending request")
http_client.fetch(request_url,
handle_request,
method='POST',
headers=postHeaders,
body=body)
ioloop.IOLoop.instance().start()
def twitter_request(self,
path,
callback,
access_token=None,
post_args=None,
**args):
"""Fetches the given API path, e.g., "/statuses/user_timeline/btaylor"
The path should not include the format (we automatically append
".json" and parse the JSON output).
If the request is a POST, post_args should be provided. Query
string arguments should be given as keyword arguments.
All the Twitter methods are documented at
http://apiwiki.twitter.com/Twitter-API-Documentation.
Many methods require an OAuth access token which you can obtain
through authorize_redirect() and get_authenticated_user(). The
user returned through that process includes an 'access_token'
attribute that can be used to make authenticated requests via
this method. Example usage::
class MainHandler(tornado.web.RequestHandler,
tornado.auth.TwitterMixin):
@tornado.web.authenticated
@tornado.web.asynchronous
def get(self):
self.twitter_request(
"/statuses/update",
post_args={"status": "Testing Tornado Web Server"},
access_token=user["access_token"],
callback=self.async_callback(self._on_post))
def _on_post(self, new_entry):
if not new_entry:
# Call failed; perhaps missing permission?
self.authorize_redirect()
return
self.finish("Posted a message!")
"""
# Add the OAuth resource request signature if we have credentials
url = "http://api.twitter.com/1" + path + ".json"
if access_token:
all_args = {}
all_args.update(args)
all_args.update(post_args or {})
consumer_token = self._oauth_consumer_token()
method = "POST" if post_args is not None else "GET"
oauth = self._oauth_request_parameters(url,
access_token,
all_args,
method=method)
args.update(oauth)
if args: url += "?" + urllib.urlencode(args)
callback = self.async_callback(self._on_twitter_request, callback)
http = httpclient.AsyncHTTPClient()
if post_args is not None:
http.fetch(url,
method="POST",
body=urllib.urlencode(post_args),
callback=callback)
else:
http.fetch(url, callback=callback)
def checkForUpdates(url):
http_client = httpclient.AsyncHTTPClient()
http_client.fetch(url, callback=checkVersion)
def friendfeed_request(self,
path,
callback,
access_token=None,
post_args=None,
**args):
"""Fetches the given relative API path, e.g., "/bret/friends"
If the request is a POST, post_args should be provided. Query
string arguments should be given as keyword arguments.
All the FriendFeed methods are documented at
http://friendfeed.com/api/documentation.
Many methods require an OAuth access token which you can obtain
through authorize_redirect() and get_authenticated_user(). The
user returned through that process includes an 'access_token'
attribute that can be used to make authenticated requests via
this method. Example usage::
class MainHandler(tornado.web.RequestHandler,
tornado.auth.FriendFeedMixin):
@tornado.web.authenticated
@tornado.web.asynchronous
def get(self):
self.friendfeed_request(
"/entry",
post_args={"body": "Testing Tornado Web Server"},
access_token=self.current_user["access_token"],
callback=self.async_callback(self._on_post))
def _on_post(self, new_entry):
if not new_entry:
# Call failed; perhaps missing permission?
self.authorize_redirect()
return
self.finish("Posted a message!")
"""
# Add the OAuth resource request signature if we have credentials
url = "http://friendfeed-api.com/v2" + path
if access_token:
all_args = {}
all_args.update(args)
all_args.update(post_args or {})
consumer_token = self._oauth_consumer_token()
method = "POST" if post_args is not None else "GET"
oauth = self._oauth_request_parameters(url,
access_token,
all_args,
method=method)
args.update(oauth)
if args: url += "?" + urllib.urlencode(args)
callback = self.async_callback(self._on_friendfeed_request, callback)
http = httpclient.AsyncHTTPClient()
if post_args is not None:
http.fetch(url,
method="POST",
body=urllib.urlencode(post_args),
callback=callback)
else:
http.fetch(url, callback=callback)
def download(url):
http_client = httpclient.AsyncHTTPClient()
http_client.fetch(url, callback=decompress)
def _get_http_client(self):
"""Returns an asynchronous web client object
The object is actually of type SimpleAsyncHTTPClient
"""
return httpclient.AsyncHTTPClient()
def get_query_records(sql, query_limit):
'''
periodic query records function
'''
logging.info('a little brain dead recolection of records')
record_list = []
http_client = httpclient.AsyncHTTPClient()
def handle_record_uuid(response):
'''
Request Handler Record UUID
'''
if response.error:
logging.error(response.error)
else:
logging.info(response.body)
def handle_request(response):
'''
Request Handler
'''
if response.error:
logging.error(response.error)
else:
logging.info(response.body)
result = json.loads(response.body)
request_id = result.get('uuid', None)
if request_id:
request_id = request_id.get('uuid')
http_client.fetch('http://iofun.io/records/{0}'.format(request_id),
headers={"Content-Type": "application/json"},
method='GET',
callback=handle_record_uuid)
try:
# Get SQL database from system settings
# PostgreSQL insert new sip account query
query = '''
SELECT
DISTINCT ON (uniqueid) uniqueid,
calldate as start,
date(calldate) as strdate,
clid as callerid,
src as source,
dst as destination,
dcontext as destination_context,
channel,
dstchannel as destination_channel,
duration,
billsec,
billsec as seconds,
disposition,
checked
FROM cdr
WHERE checked = false
ORDER BY uniqueid DESC
LIMIT {0};
'''.format(query_limit)
result = yield sql.query(query)
logging.warning('Getting {0} rows from PostgreSQL'.format(len(result)))
for x in result:
record_list.append(x)
result.free()
# TODO: Still need to check the follings exceptions with the new queries module.
#except (psycopg2.Warning, psycopg2.Error) as e:
# logging.exception(e)
# raise e
except Exception, e:
logging.exception(e)
raise e
def get(self):
# http://localhost:port/search?idx_job=0
idx_job = self.get_argument('idx_job')
docID_words = pickle.load(
open('RelevanceAnalysis/relevance_jobs/%s.out' % idx_job, 'rb'))
# format of term_relevance_docs cache: ['term'] = ['docID1', 'docID2']
"""
cache_dict = defaultdict(list)
if os.path.exists('RelevanceAnalysis/cache_jobs/cache'):
cache = open('RelevanceAnalysis/cache_jobs/cache',"r")
for line in cache:
key = json.loads(line)[0]
value = json.loads(line)[1]
cache_dict[key] = value
"""
docID_releventDocs = defaultdict(list)
count = 0
for docID in docID_words:
releventDocs = []
storedDocs = []
storedDocs.append(docID)
for i in range(len(docID_words[docID])):
query = docID_words[docID][i][0]
print(query)
"""
if query in cache_dict.keys():
docIDs = cache_dict[query]
for i in range(len(docIDs)):
releventDocs.append((docIDs[i], query))
else:
"""
httpclient.AsyncHTTPClient.configure(None,
defaults={
'connect_timeout':
300,
'request_timeout': 300
})
http_client_index = httpclient.AsyncHTTPClient()
indexServerRequest = yield [
http_client_index.fetch(server + query)
for server in inventory.indexServers
]
docIDs = []
for request in indexServerRequest:
docIDs += json.loads(request.body.decode())["postings"]
docIDs.sort(key=lambda x: -x[1])
num_result = min(MAX_RESULT_PER_WORD, len(docIDs))
docIDs_query = []
releventDocs_per_query = []
for d in docIDs:
if d[0] in storedDocs:
continue
else:
releventDocs_per_query.append((d[0], query))
storedDocs.append(d[0])
# cache_dict[query].append(d[0])
if len(releventDocs_per_query) == num_result:
break
releventDocs += releventDocs_per_query
#append_cache = open('RelevanceAnalysis/cache_jobs/cache',"a")
#append_cache.write(json.dumps((query, cache_dict[query])) + "\n")
count += 1
print("%s has extracted %d docs" % (idx_job, count))
for i in range(len(releventDocs)):
print(releventDocs[i])
docID_releventDocs[docID] = releventDocs
pickle.dump(
docID_releventDocs,
open(
'IndexingRetrieval/relevance_docs/tmpDocs/relevantDocs' +
str(idx_job), 'wb'))
self.finish()
