def send_tags(self, method, info, tags, sk):
"""Sends tags to last.fm. method is one of:
album.addtags, artist.addtags or track.addtags
info_dict is the artist, track and album info
tags is a comma delimited list of no more than 10 tags"""
#All methods require these parameters:
#tags (Required) : A comma delimited list of user supplied tags to apply
#to this album. Accepts a maximum of 10 tags.
#api_key (Required) : A Last.fm API key.
#api_sig (Required)
#sk (Required)
#artist (Required) : The artist name in question
post_values = {
"method": method,
"tags": tags,
"api_key": self.api_key,
"sk": sk,
"artist": info['Artist']
}
#these methods require additional info:
#album.addTags -> album
#track.addTags -> track
if method == "album.addtags":
post_values['album'] = info['Album']
if method == "track.addtags":
post_values['track'] = info['Track']
post_values['api_sig'] = self.create_api_sig(post_values)
conn = HttpRequest(self.url, urllib.urlencode(post_values))
response = conn.connect()
def set_user_image(self):
webservice = webservices.LastfmWebService()
url = "http://ws.audioscrobbler.com/2.0/?method=user.getinfo&user=%s&api_key=%s"
request = HttpRequest(url=url % (self.username, webservice.api_key),
timeout=10)
msg = request.connect(xml=True)
image_url = webservice.parse_xml(msg, "image")
if image_url is None:
return
if not os.path.exists(self.HOME_DIR + os.path.basename(image_url)):
request = HttpRequest(image_url)
request.retrieve(image_url,
self.HOME_DIR + os.path.basename(image_url),
self.tree.get_widget("user_thumb"))
else:
image = gtk.gdk.pixbuf_new_from_file_at_size(
self.HOME_DIR + os.path.basename(image_url), 100, 40)
self.tree.get_widget("user_thumb").set_from_pixbuf(image)
def __init__(self, client, address):
""" Handles the request and sends a response to the client.
Args:
client (socket.socket): The client of the request.
address (tuple(str, int)): The client address and port, for logging purposes.
"""
self.__client = client
self.__address = address
self.__response = HttpResponse()
self.__request = None
self.__close_connection = True
stop_handling_request = False
try:
""" First parses the HTTP request and, if there are hooks to call after parsing them, calls them.
"""
self.__request = HttpRequest(client)
self.__after_parsing()
except StopHandlingRequestException:
""" If there is any reason to stop the regular execution of the request handling, the after parsing hooks
have to raise a `StopHandlingRequestException`. See `after_parsing_request` method for more information.
"""
stop_handling_request = True
except HttpRequestParseErrorException:
""" If the request cannot be parsed, it returns a 400 HTTP error code to the client.
"""
stop_handling_request = True
self.__response.status = 400
if not stop_handling_request:
request_uri = self.__request.request_uri
request_method = self.__request.method
""" Checks if the request has a valid HTTP method, if not, it returns a 400 HTTP error code to the client.
"""
if request_method in [
"GET", "POST", "HEAD", "PUT", "DELETE", "TRACE", "OPTIONS",
"CONNECT", "PATCH"
]:
""" Checks if the request is for the API or the app and handles it accordingly.
"""
if request_uri == self.__API_URI or request_uri.startswith(
self.__API_URI + "/"):
self.__handle_api_request()
else:
self.__handle_app_request()
else:
self.__response.status = 400
self.__end_handling()
else:
self.__end_handling()
def main():
req = HttpRequest()
req.start()
hthread = hue.HueThread(ip="192.168.10.2")
hthread.start()
while True:
# get image
ret, img = cap.read()
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
# face detect
faces = face_cascade.detectMultiScale(gray,
scaleFactor=1.3,
minNeighbors=5)
human_num = len(faces)
# if face detect
if (human_num > 0):
hthread.changeState(hue.MeetingStart())
#time = datetime.now().strftime("%Y/%m/%d %H:%M:%S")
time = datetime.now().isoformat()
#time.microsecond = 0
#time = time.isoformat()
print(time, ' human_num : ', human_num)
req.add(
StatusReq(room=room_Name, timestamp=time, occupied=human_num))
else:
hthread.changeState(hue.MeetingEnd())
# draw rect
for x, y, w, h in faces:
cv2.rectangle(img, (x, y), (x + w, y + h), (255, 0, 0), 2)
#face = img[y: y + h, x: x + w]
#face_gray = gray[y: y + h, x: x + w]
#eyes = eye_cascade.detectMultiScale(face_gray)
#for (ex, ey, ew, eh) in eyes:
# cv2.rectangle(face, (ex, ey), (ex + ew, ey + eh), (0, 255, 0), 2)
# show img
cv2.imshow('video image', img)
# wait key
key = cv2.waitKey(10)
# quit
if key == 27: # ESCキーで終了
break
cap.release()
cv2.destroyAllWindows()
req.stop()
req.join()
def get_user_top_tags(self, username, limit=15):
#method user.getTopTags
#Params
#user (Required) : The user name
#limit (Optional) : Limit the number of tags returned
#api_key (Required) : A Last.fm API key.
encoded_values = urllib.urlencode({
"method": "user.gettoptags",
"user": username,
"limit": limit,
"api_key": self.api_key
})
url = self.url + "?" + encoded_values
conn = HttpRequest(url)
xml_doc = conn.connect(xml=True)
return self.parse_xml_doc(xml_doc, "name")
def get_popular_tags(self, method, info_dict):
"""method is either artist.gettoptags or track.gettoptags"""
#Params
#track (Optional) : The track name in question
#artist (Required) : The artist name in question
#api_key (Required) : A Last.fm API key.
dict = {
"method": method,
"artist": info_dict['Artist'],
"api_key": self.api_key
}
if method == "track.gettoptags":
dict['track'] = info_dict['Track']
encoded_values = urllib.urlencode(dict)
url = self.url + "?" + encoded_values
conn = HttpRequest(url)
xml_doc = conn.connect(xml=True)
return self.parse_xml_doc(xml_doc, "name")
def crawl(self, response):
print(response)
urls = [
'https://list.mgtv.com/3/176--------a1-c2-1--a1-.html?channelId=3',
'https://list.mgtv.com/3/175--------a1-c2-1--a1-.html?channelId=3',
'https://list.mgtv.com/3/177--------a1-c2-1--a1-.html?channelId=3',
'https://list.mgtv.com/3/178--------a1-c2-1--a1-.html?channelId=3',
'https://list.mgtv.com/3/43--------a1-c2-1--a1-.html?channelId=3',
'https://list.mgtv.com/3/44--------a1-c2-1--a1-.html?channelId=3',
'https://list.mgtv.com/2/a1-10--------c2-1---.html?channelId=2',
'https://list.mgtv.com/2/a1-12--------c2-1---.html?channelId=2',
'https://list.mgtv.com/2/a1-11--------c2-1---.html?channelId=2',
'https://list.mgtv.com/2/a1-193--------c2-1---.html?channelId=2',
'https://list.mgtv.com/50/a1-52--------c2-1---.html?channelId=50',
'https://list.mgtv.com/50/a1-53--------c2-1---.html?channelId=50',
'https://list.mgtv.com/1/a1-1--------c2-1---.html?channelId=1',
'https://list.mgtv.com/1/a1-2--------c2-1---.html?channelId=1'
]
for url in urls:
yield HttpRequest(url=url, callback=self.crawl1)
def _send_post(self, post_values):
req = HttpRequest(url=self.submission_url,
data=post_values,
timeout=10)
success, msg = req.connect()
if success:
self.deletion_ids.extend(self.del_ids)
return True
else:
if msg[0] == "BADSESSION":
self.bad_session_count += 1
server_response, message = self.handshake()
if server_response == "OK" and self.bad_session_count <= 3:
return self._send_post(post_values)
else:
self.parent.write_info(_("Error during handshake."))
self.parent.write_info(
_("There was an error sending data to last.fm:") + "\n" +
"\n".join(msg))
return False
def handshake(self):
self.handshake_attempts += 1
self.timestamp = self.create_timestamp()
self.authentication_code = self.create_authentication_code()
self.url = self.base_url + r"/?" + self.encode_url()
req = HttpRequest(url=self.url, timeout=10)
success, response = req.connect()
print req.handshake_response(response[0])
if success:
self.session_id = response[1]
status = response[0]
self.submission_url = response[3]
if status != "OK":
if self.handshake_attempts <= 3:
print "Handshake error attempt %d of 3" % self.handshake_attempts
return self.handshake()
else:
return "FAILED", "Failed to make a handshake with Last.fm"
else:
self.handshake_attempts = 0
msg = req.handshake_response(response[0])
return response[0], msg
def format_crawl(cls):
self = cls(cls.start_urls)
for url in self.start_urls:
yield HttpRequest(url=url, callback=self.crawl)
def __init__(self):
self.httpRequest = HttpRequest()
def get_post_content_and_time(post_url, post_type, post_name, time_last_time):
post_full_url = 'http://www.hi-pda.com/forum/' + post_url
post_headers = {
'Referer': 'http://www.hi-pda.com/forum/',
'Host': 'www.hi-pda.com'
}
logging.info('Get post[%s] by url[%s].' % (post_name, post_url))
#根据url请求帖子内容
post_request = HttpRequest(post_full_url, None, post_headers)
post_request.send_request()
post_resp_content = post_request.get_resp_content()
try:
post_resp_content = post_resp_content.decode('gbk')
except UnicodeDecodeError as e:
logging.error('Decode post response content failed.')
logging.exception(e)
#从帖子内容中解析帖子的摘要
re_pattern_content = re.compile(
r'''<meta name="description" content="(.*)" />''')
result_content = re_pattern_content.search(post_resp_content)
post_content = None
post_update_time = None
post_create_time = None
if result_content is None:
logging.warn('Request failed.')
else:
post_content = result_content.groups()[0]
if post_content is None:
logging.warn('Get post conetent failed.')
else:
#从帖子内容中解析帖子更新时间
re_pattern_update_time = re.compile(r'''于 (.*) 编辑''')
result_update_time = re_pattern_update_time.search(post_content)
if result_update_time is None:
pass
else:
post_update_time = result_update_time.groups()[0]
if post_update_time is None:
#从帖子内容中解析帖子发表时间
re_pattern_create_time = re.compile(r'''<em id=".+">发表于 (.+)</em>''')
result_create_time = re_pattern_create_time.search(post_resp_content)
if result_create_time is None:
logging.warn('Get post time failed.')
else:
post_create_time = result_create_time.groups()[0]
else:
post_create_time = post_update_time
post_create_time_datetime = datetime.strptime(post_create_time,
'%Y-%m-%d %H:%M')
post_create_time_stamp = post_create_time_datetime.timestamp()
post = None
#比较帖子时间和上次爬取时间，如果大于上次爬取时间，则视为本次爬取目标
if (post_create_time_stamp - time_last_time) > 0:
conn = mysql.connector.connect(user=db_user,
password=db_passwd,
database=db_name)
cursor = conn.cursor()
logging.info('post_type:' + post_type)
logging.info('post_name:' + post_name)
logging.info('post_url:' + post_full_url)
logging.info('post_create_time:' + post_create_time)
logging.info('post_content:' + post_content)
post_id = next_id()
post = Post(id=post_id,
post_type=post_type,
post_title=post_name,
post_owner='hipda',
post_content=post_content,
post_link=post_full_url,
post_time=post_create_time)
# post.save()
cursor.execute(
'insert into posts (id, post_type, post_title, post_owner, post_content, post_link, post_time, created_at ) values (%s, %s, %s, %s, %s, %s, %s, %s)',
[
post_id, post_type, post_name, 'hipda', post_content,
post_full_url, post_create_time_stamp, post_create_time_stamp
])
conn.commit()
cursor.close()
conn.close()
time.sleep(1)
else:
logging.info('Post time[%s] is not after last time.' %
post_create_time_datetime)
return post
else:
get_post_time_last_time = datetime.strptime(
get_post_time_last_time, '%Y-%m-%d %H:%M:%S')
get_post_time_last_time_stamp = get_post_time_last_time.timestamp()
logging.info('************last time: %s************' %
get_post_time_last_time)
#using cookieJar & HTTPCookieProcessor to automatically handle cookies
cj = http.cookiejar.CookieJar()
opener = urllib.request.build_opener(
urllib.request.HTTPCookieProcessor(cj))
urllib.request.install_opener(opener)
pda_url = 'http://www.hi-pda.com/'
pda_request = HttpRequest(pda_url)
pda_request.send_request()
pda_resp_content = pda_request.get_resp_content()
formhash_url = 'http://www.hi-pda.com/forum/logging.php?action=login&referer=http%3A//www.hi-pda.com/forum/'
formhash_request = HttpRequest(formhash_url, None,
{'Host': 'www.hi-pda.com'})
formhash_request.send_request()
formhash_resp_content = formhash_request.get_resp_content()
try:
formhash_resp_content = formhash_resp_content.decode('gbk')
except UnicodeDecodeError as e:
logging.error('Decode formhash response content failed.')
logging.exception(e)
# print( formhash_resp_content )
