def crawl(self, parser):
"""
爬取
:param parser:
:return:
"""
html_parser = Html_Parser()
for url in parser['urls']:
response = Html_Downloader.download(url)
if response is not None:
proxy_list = html_parser.parse(response, parser)
if proxy_list is not None:
# 检查爬取到的proxy
count, new = 0, 0
for proxy in proxy_list:
count += 1
proxy_str = '%s:%s' % (proxy['ip'], proxy['port'])
if proxy_str not in self.proxies_set:
self.proxies_set.add(proxy_str)
new += 1
self.sqlhelper.insert(proxy)
self.url_count += 1
logger.info(
'%d/%d -- <%s> 获取%d, 未记录的%d' %
(self.url_count, self.url_total, url, count, new))
else:
self.url_count += 1
logger.warning('%d/%d -- <%s> 解析数据错误' %
(self.url_count, self.url_total, url))
else:
self.url_count += 1
logger.warning('%d/%d -- <%s> 下载页面错误' %
(self.url_count, self.url_total, url))
def producer(url, q):
while True:
i = 0
logger.debug('rtmp: {} read+'.format(url))
video_capture = cv2.VideoCapture(url)
ret_val, image = video_capture.read()
if False is video_capture.isOpened() or False is ret_val:
logger.warning('{} url is: {} {}'.format(url, video_capture.isOpened(), ret_val))
continue
logger.debug('rtmp: {} load finish'.format(url))
while True:
i += 1
ret, frame = video_capture.read()
if not ret:
break
if i % TIMES != 0 or image is None:
continue
if not FX == FY == 1:
try:
logger.debug('{}: {} fps image resize'.format(url, i))
frame = cv2.resize(frame, (0, 0), fx=FX, fy=FY)
except Exception as e:
logger.error(e)
logger.error('image is bad')
break
if q.full():
q.get()
q.put(frame)
logger.info('{} image save to queue {}'.format(i, q.qsize()))
def update_tags(cls, xg_device_token, tags):
try:
r = Session.query(cls).filter(cls.xg_device_token == xg_device_token).first()
r.tags = tags
Session.add(r)
Session.commit()
except sqlalchemy.exc.IntegrityError, e:
logger.warning('msg[update tags error] table[%s] e[%s]' % (__tablename__, e))
Session.rollback()
def _handle(self, data: bytes, source_addr: tuple) -> None:
"""Handle the received data.
Parse received data to DNS message, loop up the requested domain name
in local database or foreign DNS server and send the result back to
the user.
Args:
data: Received data.
source_addr: Source host addr.
"""
message = Message.from_bytes(data)
question = message.Questions[0]
if question is None:
logger.error('{addr}Format error'.format(addr=source_addr))
return
if question.QTYPE == RR_type.A:
rdata = self._cache.get(question.get_QNAME(), 'A')
if rdata is not None:
if rdata != b'\x00\x00\x00\x00':
logger.info('{addr}Found A of {name}'.format(
name=question.get_QNAME(), addr=source_addr))
header = message.Header
header.RA = b'\x01'
header.QDCOUNT = b'\x00\x01'
header.ANCOUNT = b'\x00\x01'
header.QR = b'\x01'
ttl = self._cache.get_ttl(question.get_QNAME()) \
if self._cache.get_ttl(question.get_QNAME()) != -1 else 0
answer = ResourceRecord(b'\xc0\x0c', RR_type.A, RR_class.IN, ttl, 4,
self._cache.get(question.get_QNAME(), 'A'))
response = Message(header, [question], [answer], [], []).get_bytes()
else:
logger.warning('{addr}Blocked {name}'.format(
name=question.get_QNAME(), addr=source_addr))
header = message.Header
header.RA = b'\x01'
header.QDCOUNT = b'\x00\x00'
header.ANCOUNT = b'\x00\x00'
header.QR = b'\x01'
header.RCODE = b'\x03'
response = Message(header, [], [], [], []).get_bytes()
else:
logger.debug('{addr}Forward to remote DNS server: {name}'.format(
name=question.get_QNAME(), addr=source_addr))
response = self._forward(data)
else:
logger.debug('{addr}Forward to remote DNS server: {name}'.format(
name=question.get_QNAME(), addr=source_addr))
response = self._forward(data)
self._socket.sendto(response, source_addr)
def init():
cfg = {}
config = ConfigParser.ConfigParser()
config.read('config/config.ini')
sections = config.sections()
for section in sections:
cfg[section] = {}
options = config.options(section)
for option in options:
try:
cfg[section][option] = config.get(section, option)
except Exception, e:
logger.warning("parse config error, e[%s]" % e)
def update(cls, sid, status, valid=1, identifier=''):
try:
r = Session.query(cls).filter(cls.sid == sid).first()
if r:
r.identifier = identifier
r.status = status
r.valid = valid
Session.add(r)
Session.commit()
return r
except sqlalchemy.exc.IntegrityError, e:
Session.rollback()
logger.warning('msg[update %s error] e[%s]' % (__tablename__, e))
return None
def send_sms(self, request):
logger.info('send sms, request[%s]' % request)
if request.mobile == '' or request.content == '':
logger.warning('param error request[%s]' % request)
return PARAM_ERROR
param = self.__build_param(request)
result = http.request(SEND_SMS_URL, param, 'GET', 10)
if not result:
return HTTP_REQUEST_ERROR
logger.debug('result:%s' % result)
root = None
sid = request.sid
errno = 0
task_id = ''
valid = 0
dbstatus = 0
try:
root = ET.fromstring(result)
status = root.find('returnstatus').text
if status != 'Success':
msg = root.find('message').text
logger.warning('send failed, msg[%s]' % msg)
errno = SEND_FAILED
else:
task_id = root.find('taskID').text
dbstatus = 1
valid = 1
except xml.etree.ElementTree.ParseError:
logger.warning("invalid result[%s] request[%s]" % (result, request))
errno = THIRD_SERVER_ERROR
except Exception, e:
logger.warning('xml parse exception,ult[%s] request[%s]' % (result, request))
def __build_param(self, request):
data = {}
data['action'] = 'send'
data['account'] = self.account
data['password'] = self.password
data['mobile'] = request.mobile
data['content'] = request.content
data['sendTime'] = ''
if hasattr(request, 'send_time') and request.send_time != 0:
t = request.send_time
try:
x = time.localtime(t)
data['sendTime'] = time.strftime("%Y-%m-%d %H:%M:%S", x)
except Exception, e:
logger.warning('time trans exception, t[%s], e[%s]' % (t, e))
def get_proxy_info(index_num, bar_n):
"""
爬取所有代理并存储
:param index_num:
:param bar_n:
:return:
"""
text = "bar {}".format(bar_n)
for page in tqdm(index_num, desc=text, position=bar_n):
gen_response = yield from session.get(
url=config.URL_66IP.format(str(page)))
html_response = yield from gen_response.read()
soup = BeautifulSoup(html_response, "lxml")
tags_aims = soup.select(selector="#main > div > div > table > tr")
tags_aims = tags_aims[1:]
for i in range(len(tags_aims)):
if tags_aims[i]:
info_list = list(tags_aims[i].strings)
proxy_ip = info_list[0] + ":" + info_list[1]
r.lpush(config.PROXY_ALL_66IP_KEY, proxy_ip)
else:
logger.warning("Web page is empty")
def request(url, params, method='GET', timeout=1):
result = None
response = None
if method == 'GET':
url = '%s?%s' % (url, urllib.urlencode(params))
logger.info('request url: %s' % url)
try:
response = urllib2.urlopen(url, timeout=timeout)
result = response.read()
except urllib2.URLError as e:
code = e.code if hasattr(e, 'code') else -1
reason = e.reason if hasattr(e, 'reason') else None
logger.warning('url error, code[%s] reason[%s]' % (code, reason))
except urllib2.HTTPError as e:
code = e.code if hasattr(e, 'code') else -1
logger.warning('http error, code[%s]' % (code))
except Exception, e:
logger.warning("url get exception, url: %s, e: %s" % (url, e))
finally:
code = e.code if hasattr(e, 'code') else -1
reason = e.reason if hasattr(e, 'reason') else None
logger.warning('url error, code[%s] reason[%s]' % (code, reason))
except urllib2.HTTPError as e:
code = e.code if hasattr(e, 'code') else -1
logger.warning('http error, code[%s]' % (code))
except Exception, e:
logger.warning("url get exception, url: %s, e: %s" % (url, e))
finally:
if response:
response.close()
if method == 'POST':
try:
req = urllib2.Request(url, urllib.urlencode(params))
response = urllib2.urlopen(req)
result = response.read()
except urllib2.URLError as e:
code = e.code if hasattr(e, 'code') else -1
reason = e.reason if hasattr(e, 'reason') else None
logger.warning('url error, code[%s] reason[%s]' % (code, reason))
except urllib2.HTTPError as e:
code = e.code if hasattr(e, 'code') else -1
logger.warning('http error, code[%s]' % (code))
except Exception, e:
logger.warning("url get exception, url: %s, e: %s" % (url, e))
finally:
if response:
response.close()
return result
def fread(self, config):
if self.conf.has_option(self.section, config):
return self.conf.get(self.section, config)
else:
logger.warning('{}{} not existed'.format(self.section, config))
return None
def main(yolo, url, CreateBoxEncoder, q):
producer = None
if KAFKA_ON:
ip_port = '{}:{}'.format(KAFKA_IP, KAFKA_PORT)
producer = KafkaProducer(bootstrap_servers=ip_port)
logger.debug('open kafka')
# Definition of the parameters
max_cosine_distance = 0.3
nn_budget = None
nms_max_overlap = 1.0
metric = nn_matching.NearestNeighborDistanceMetric("cosine", max_cosine_distance, nn_budget)
tracker = Tracker(metric)
door = get_door(url)
# init var
center_mass = {}
miss_ids = []
disappear_box = {}
person_list = []
in_house = {}
in_out_door = {"out_door_per": 0, "into_door_per": 0}
only_id = str(uuid.uuid4())
logger.debug('rtmp: {} load finish'.format(url))
last_person_num = 0
last_monitor_people = 0
while True:
t1 = time.time()
if q.empty():
continue
frame = q.get()
image = Image.fromarray(frame[..., ::-1]) # bgr to rgb
boxs, scores_ = yolo.detect_image(image)
t2 = time.time()
# print('5====={}======{}'.format(os.getpid(), round(t2 - t1, 4)))
now = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))
logger.debug("box_num: {}".format(len(boxs)))
features = CreateBoxEncoder.encoder(frame, boxs)
# score to 1.0 here).
# detections = [Detection(bbox, 1.0, feature) for bbox, feature in zip(boxs, features)]
detections = [Detection(bbox, scores_, feature) for bbox, scores_, feature in zip(boxs, scores_, features)]
# Run non-maxima suppression.
boxes = np.array([d.tlwh for d in detections])
scores = np.array([d.confidence for d in detections])
indices = preprocessing.non_max_suppression(boxes, nms_max_overlap, scores)
detections = [detections[i] for i in indices]
# Call the tracker
tracker.predict()
tracker.update(detections)
# 实时人员ID保存
track_id_list = []
high_score_ids = {}
# if len(tracker.tracks) == 0:
# logger.info('sleep {}'.format(SLEEP_TIME))
# sleep(SLEEP_TIME)
# continue
for track in tracker.tracks:
# 当跟踪的目标在未来的20帧未出现,则判断丢失,保存至消失的id中间区
if track.time_since_update == MAX_AGE:
miss_id = str(track.track_id)
miss_ids.append(miss_id)
if not track.is_confirmed() or track.time_since_update > 1:
continue
# 如果人id存在,就把人id的矩形框坐标放进center_mass 否则 创建一个key(人id),value(矩形框坐标)放进center_mass
track_id = str(track.track_id)
bbox = track.to_tlbr()
near_door = is_near_door({track_id: bbox}, door)
if track.score >= 0.92 and not near_door:
high_score_ids[track_id] = [[int(bbox[0]), int(bbox[1]), int(bbox[2]), int(bbox[3])]]
track_id_list.append(track_id)
if track_id in center_mass:
center_ = center_mass.get(track_id)
if len(center_) > 49:
center_.pop(0)
center_.append([int(bbox[0]), int(bbox[1]), int(bbox[2]), int(bbox[3])])
else:
center_mass[track_id] = [[int(bbox[0]), int(bbox[1]), int(bbox[2]), int(bbox[3])]]
logger.info('id:{}, score:{}'.format(track_id, track.score))
for id in miss_ids:
if id in center_mass.keys():
disappear_box[id] = center_mass[id]
del center_mass[id]
miss_ids.clear()
# # 进出门判断
out_or_in(center_mass, door, in_house, disappear_box, in_out_door)
# 相对精准识别人 用来实时传递当前人数
box_score_person = [scores for scores in scores_ if scores > 0.72]
person_sum = in_out_door['into_door_per'] - in_out_door['out_door_per']
# if person_sum <= len(high_score_ids) and not near_door:
if person_sum <= len(high_score_ids):
# 当时精准人数大于进出门之差时 来纠正进门人数 并把出门人数置为0
if person_sum == len(high_score_ids) == 1:
pass
# print('person_sum == len(high_score_ids) == 1')
else:
logger.warning('reset in_out_door person')
in_out_door['out_door_per'] = 0
in_out_door['into_door_per'] = len(high_score_ids)
in_house.update(high_score_ids)
# print('high score:{}'.format(high_score_ids))
logger.warning(
'22222222-id: {} after into of door: {}'.format(in_house.keys(), in_out_door['into_door_per']))
person_sum = len(high_score_ids)
if in_out_door['into_door_per'] == in_out_door['out_door_per'] > 0:
in_out_door['into_door_per'] = in_out_door['out_door_per'] = 0
if len(person_list) > 100:
person_list.pop(0)
person_list.append(person_sum)
# 从url提取摄像头编号
pattern = str(url)[7:].split(r"/")
logger.debug('pattern {}'.format(pattern[VIDEO_CONDE]))
video_id = pattern[VIDEO_CONDE]
logger.info('object tracking cost {}'.format(time.time() - t1))
# 当列表中都是0的时候 重置进出门人数和所有字典参数变量
if person_list.count(0) == len(person_list) == 101:
logger.debug('long time person is 0')
in_out_door['into_door_per'] = 0
in_out_door['out_door_per'] = 0
in_house.clear()
logger.warning('All Clear')
# 当满足条件时候 往前端模块发送人员的信息
if (last_person_num != person_sum or last_monitor_people != len(box_score_person)) and producer:
monitor_people_num = len(box_score_person)
logger.debug("person-sum:{} monitor-people_num:{}".format(person_sum, monitor_people_num))
save_to_kafka(TOPIC_SHOW, now, person_sum, url, producer, video_id, monitor_people_num, only_id)
if last_person_num > 0 and person_sum == 0:
only_id = str(uuid.uuid4())
if last_person_num == 0 and person_sum > 0:
save_to_kafka(TOPIC_NVR, now, person_sum, url, producer, video_id, len(box_score_person), only_id)
# last_time = int(time.time())
last_person_num = person_sum
last_monitor_people = len(box_score_person)
# 当满足条件时候 往NVR模块发送信息
logger.info('url:{} into_door_per: {}'.format(url, in_out_door['into_door_per']))
logger.info('url:{} out_door_per: {}'.format(url, in_out_door['out_door_per']))
logger.info('url:{} in_house: {}'.format(url, in_house))
logger.info('url:{} monitor_people_num: {}'.format(url, len(box_score_person)))
logger.info('url:{} person_sum: {}'.format(url, person_sum))
logger.info('GPU image load cost {}'.format(time.time() - t1))
t3 = time.time()
fps = ((1 / (round(t3 - t1, 4))))
print('6====={}======{}'.format(os.getpid(), round(round(t3 - t1, 4) * 1000, 2)))
logger.debug("fps= %f" % (fps))