def detect_to_turn_car(self):
"""
遇到障碍物,红外避障模块的指示灯亮,端口电平为LOW
未遇到障碍物,红外避障模块的指示灯灭,端口电平为HIGH
Return:
- Block detected: True
- Block not detected: False
"""
left_sensor, right_sensor = self.detect()
logging.debug(
f'Infra red detecting ... \nleft: {left_sensor} right: {right_sensor}'
)
if left_sensor is True and right_sensor is True:
return False
else:
self.motor.aquire()
if left_sensor is True and right_sensor is False:
self.motor.spin_left(force=True) # 右边探测到有障碍物，有信号返回，原地向左转
time.sleep(0.01)
elif right_sensor is True and left_sensor is False:
self.motor.spin_right(force=True) # 左边探测到有障碍物，有信号返回，原地向右转
time.sleep(0.01)
elif right_sensor is False and left_sensor is False:
self.motor.spin_right(force=True) # 当两侧均检测到障碍物时调用固定方向的避障(原地右转)
time.sleep(0.01)
self.motor.free()
self.motor.release()
return True
def crawl_transaction_by_search(self, args):
""" 依据商圈或小区 爬取一页历史成交房源 """
search_key, page = args
url_page = self.base_url + f"chengjiao/pg{page}rs{search_key}/"
content = self.request_fn(url_page)
soup = BeautifulSoup(content, self.bs4_parser)
logging.debug('@crawl_transaction_by_search: {0} - page - {1}: {2}'.format(search_key, page, url_page))
session = DBSession()
for ul_tag in soup.find_all("ul", class_="listContent"):
for item_tag in ul_tag.find_all("li"):
try:
info_dict = self.parse_transaction_content(item_tag)
query = session.query(TransactionInfo).filter(TransactionInfo.id == info_dict['id'])
if query.first():
query.update(info_dict)
else:
session.add(TransactionInfo(**info_dict))
session.commit()
logging.debug('@crawl_transaction_by_search: {0} - page - {1}: {2}'.format(
search_key, page, info_dict))
except Exception as e:
logging.exception('@crawl_transaction_by_search: {0} - page - {1}: {2}'.format(
search_key, page, e))
time.sleep(3)
logging.info('@crawl_transaction_by_search: {0} - page - {1} complete.'.format(search_key, page))
def crawl_sale_by_search(self, args):
""" 根据商圈或社区爬取一页在售房源 """
search_key, page = args
url_page = self.base_url + f"ershoufang/pg{page}rs{search_key}/"
content = self.request_fn(url_page)
soup = BeautifulSoup(content, self.bs4_parser)
logging.debug('@crawl_sale_by_search: {0} - page - {1}: {2}'.format(search_key, page, url_page))
session = DBSession()
for ul_tag in soup.find_all("ul", class_="sellListContent"):
for item_tag in ul_tag.find_all("li"):
try:
info_dict = self.parse_sale_content(item_tag)
logging.debug('@crawl_sale_by_search: {0} - page - {1}: {2}'.format(search_key, page, info_dict))
sale_info = SaleInfo(**info_dict)
if not sale_info.house_id or not sale_info.community_id or not sale_info.district:
continue
session.add(sale_info)
except Exception as e:
session.rollback()
logging.exception('@crawl_sale_by_search: {0} - page - {1}: {2}'.format(search_key, page, e))
time.sleep(3)
session.commit()
session.close()
logging.info('@crawl_sale_by_search: {0} - page - {1} complete.'.format(search_key, page))
def crawl_community_by_district(self, args):
""" 根据区县爬取一页小区信息 """
district, page = args
url_page = self.base_url + f"xiaoqu/{district}/pg{page}/"
content = self.request_fn(url_page)
soup = BeautifulSoup(content, self.bs4_parser)
logging.debug('@crawl_community_by_district: {0} - page - {1}: {2}'.format(district, page, url_page))
session = DBSession()
for ul_tag in soup.find_all("ul", class_="listContent"):
for item_tag in ul_tag.find_all("li"):
try:
info_dict = self.parse_community_content(item_tag)
query = session.query(CommunityInfo).filter(CommunityInfo.id == info_dict['id'])
if query.first():
query.update(info_dict)
else:
session.add(CommunityInfo(**info_dict))
session.commit()
logging.debug('@crawl_community_by_district: {0} - page - {1}: {2}'.format(district, page, info_dict))
except Exception as e:
session.rollback()
logging.exception('@crawl_community_by_district: {0} - page - {1}: {2}'.format(district, page, e))
time.sleep(3)
session.close()
logging.info('@crawl_community_by_district: {0} - page - {1} complete.'.format(district, page))
def __init__(self, name, channel, frequency):
# Initialise the PCA9685 using the default address (0x40).
self.pwm = Adafruit_PCA9685.PCA9685()
# Alternatively specify a different address and/or bus:
#pwm = Adafruit_PCA9685.PCA9685(address=0x41, busnum=2)
# Configure min and max servo pulse lengths
self.servo_min = int(0.035 * 4096.0) # Min pulse length out of 4096
self.servo_max = int(0.1450 * 4096.0) # Max pulse length out of 4096
# Channel settings
self.channel = channel
# Servo settings
self.maxDegrees = 180
self.minDegrees = 0
self.pwm.set_pwm_freq(frequency)
self.name = name
log.debug("Initialized servo {}".format(name))
def callback_handler(call):
logging.debug(f'call.data {type(call.data)} = {call.data}')
view_class, model_class = views.decode_data(call.data)
if call.message.chat.id not in user_sessions:
logging.info(f'Clearing session {call.message.chat.id}')
user_sessions[call.message.chat.id] = {}
if model_class is not None:
user_sessions[call.message.chat.
id][f'{type(model_class).__name__}'] = model_class
view = view_class()
if isinstance(view, views.GetBusSchedule):
if 'Destinations' not in user_sessions[call.message.chat.id]:
view = views.SelectDestination()
elif 'Locations' not in user_sessions[call.message.chat.id]:
view = views.SelectLocation()
elif 'Destinations' in user_sessions[
call.message.chat.id] and 'Locations' in user_sessions[
call.message.chat.id]:
view = views.ShowSheduleResult(user_sessions[call.message.chat.id])
del user_sessions[call.message.chat.id]['Destinations']
del user_sessions[call.message.chat.id]['Locations']
else:
view = views.StartMenu()
bot.edit_message_text(chat_id=call.message.chat.id,
message_id=call.message.message_id,
text=view.get_message_text(),
reply_markup=view.get_keyboard())
def notify(self, message):
try:
bot = telegram.Bot(token=settings.TELEGRAM_TOKEN)
except telegram.error.InvalidToken:
logging.error("Invalid Token. Check your Telegram token configuration.")
return
try:
logging.debug(bot.getMe())
except telegram.error.Unauthorized:
logging.error("Unauthorized. Check your Telegram credentials.")
return
# bot_updates = bot.getUpdates()
# for update in bot_updates:
# logging.debug("USER ID : " + str(update.message.from_user.username))
# logging.debug("CHAT ID : " + str(update.message.chat_id))
# if bot.getChat(16740022).username == "luca_cipi":
# logging.debug("TROVATA")
# if not bot_updates or not bot_updates[-1].message.chat_id:
# logging.error("We need your telegram chat id. Please, send any message to your bot.")
# return
try:
bot.sendMessage(chat_id=settings.TELEGRAM_CHAT_ID,
text=self.device_name + message)
except telegram.TelegramError:
logging.error("An error raised sending the Telegram message. " +
"Please, send a new message to your bot and try again. " +
"This way we check if the chat_id is not updated.")
def _on_release(self, key):
logging.debug(f'Press {key}')
# if self.motor.lock:
# pass
# else:
self.motor.free()
time.sleep(0.1)
def _run(self):
debug = 0
counter = 0
dev_cache = []
momentum = 0
dev_m = 0
cache_len = 0
while self.camera.cap.isOpened():
counter += 1
try:
start = time.time()
deviation = self.camera.analyze()
# logging.debug(f'deviation: {deviation}')
# self._pilot(deviation)
if len(dev_cache) == 0:
avg_deviation = deviation
if abs(deviation - avg_deviation) < 20:
dev_cache.append(deviation)
if len(dev_cache) >= 3:
counter = 0
end = time.time()
delta = end - start
logging.debug(f'analyze 3 times: {delta:.3f}s')
avg_deviation = sum(dev_cache) / len(dev_cache)
dev_m = (0.3 * dev_m + avg_deviation) / 1.3
logging.info(f'deviation momentum: {dev_m}')
self._pilot(dev_m)
except KeyboardInterrupt as e:
counter = 0
dev_cache = []
time.sleep(20)
break
def thresh_process(self, frame_pers_final):
logging.debug(f'denoising {frame_pers_final.shape}')
# threshold
frame_thresh = cv2.inRange(frame_pers_final, 40, 225)
frame_thresh = frame_pers_final
# frame_thresh = cv2.GaussianBlur(frame_thresh, (5, 5), 2)
frame_thresh = cv2.dilate(frame_thresh, (5, 5), iterations=5)
# frame_thresh = cv2.morphologyEx(frame_thresh, cv2.MORPH_CLOSE, (5,5), iterations=10)
# canny
frame_canny = cv2.Canny(frame_pers_final, 40, 225, None, 3, False)
frame_canny = cv2.morphologyEx(frame_canny,
cv2.MORPH_CLOSE, (5, 5),
iterations=5)
# edge
canny_mask_inv = cv2.bitwise_not(frame_canny)
thresh_mask_inv = cv2.bitwise_not(frame_thresh)
mask_inv = canny_mask_inv + thresh_mask_inv
frame_edge = cv2.bitwise_and(frame_canny,
frame_thresh,
mask=cv2.bitwise_not(mask_inv))
# frame_edge = cv2.dilate(frame_edge, (5, 5), iterations=5)
# frame_edge = cv2.morphologyEx(frame_edge, cv2.MORPH_OPEN, (5, 5), iterations=3)
# frame_edge = cv2.GaussianBlur(frame_edge, (5, 5), 5)
_, frame_edge = cv2.threshold(frame_edge, 0, 125, cv2.THRESH_BINARY)
return frame_thresh, frame_canny, frame_edge
def encode_data(view_class: View, model_class: Models = None):
logging.debug(f'view_class {view_class}')
logging.debug(f'model_class {type(model_class)} = {model_class}')
if model_class is not None:
return f'{view_class}|{type(model_class).__name__}|{model_class.value}'
else:
return f'{view_class}|None|None'
def request_data(url, retry=0, auto_proxy=False, delay=0, **kwargs):
"""
Get请求爬取源代码
:param url: 目标网站
:param retry: 是否重试
:param auto_proxy: 是否使用代理ip
:param delay: 延迟时间
:param kwargs: requests.get参数
:return: text
"""
if delay:
time.sleep(delay)
if retry:
sess = requests.Session()
sess.mount('http://', HTTPAdapter(max_retries=retry))
sess.mount('https://', HTTPAdapter(max_retries=retry))
method = functools.partial(sess.request, method='get')
else:
method = requests.get
if auto_proxy:
kwargs.update({'proxies': {'http': 'http://{}'.format(get_proxy())}})
try:
res = method(url=url, headers=get_header(), **kwargs)
if res.status_code == 200:
logging.debug("Request Data - {0} - {1}".format(
res.status_code, url))
return res.text
logging.info("Request Data - {0} - {1}".format(res.status_code, url))
except requests.exceptions.RequestException as e:
logging.error("Request ERROR: {0}, url: {1}".format(e, url))
def recycle(self, value):
if not isinstance(value, bool):
raise TypeError(
f'recycle must be bool. recycle: {type(value)} = "{value}"')
else:
self.recycle = value
logging.debug(f'Ok! recycle = "{value}"')
def get_local_video_size(youtube_id, default=None):
try:
return os.path.getsize(
os.path.join(settings.CONTENT_ROOT, "%s.mp4" % youtube_id))
except Exception as e:
logging.debug(str(e))
return default
def delimiter(self, value):
if not isinstance(value, str):
raise TypeError(
f'delimiter must be str. delimiter: {type(value)} = {value}')
else:
self.delimiter = value
logging.debug(f'Ok! delimiter = "{value}"')
def notify(self, message):
try:
bot = telegram.Bot(token=settings.TELEGRAM_TOKEN)
except telegram.error.InvalidToken:
logging.error(
"Invalid Token. Check your Telegram token configuration.")
return
try:
logging.debug(bot.getMe())
except telegram.error.Unauthorized:
logging.error("Unauthorized. Check your Telegram credentials.")
return
bot_updates = bot.getUpdates()
if not bot_updates or not bot_updates[-1].message.chat_id:
logging.error(
"We need your telegram chat id. Please, send any message to your bot."
)
return
try:
bot.sendMessage(chat_id=bot_updates[-1].message.chat_id,
text=self.device_name + message)
except telegram.TelegramError:
logging.error(
"An error raised sending the Telegram message. " +
"Please, send a new message to your bot and try again. " +
"This way we check if the chat_id is not updated.")
def ignore_first_line(self, value):
if not isinstance(value, bool):
raise TypeError(
f'ignore_first_line must be bool. ignore_first_line: {type(value)} = "{value}"'
)
else:
self.ignoreFirstLine = value
logging.debug(f'Ok! ignore_first_line = "{value}"')
def file_name(self, value):
# todo: Check if the path is valid
if not isinstance(value, str):
raise TypeError(
f'file_name must be str. file_name: {type(value)} = {value}')
else:
self.filename = value
logging.debug(f'Ok! File {value} exists.')
def stop_thread(self, value):
if not isinstance(value, bool):
raise TypeError(
f'stop_thread must be bool. stop_thread: {type(value)} = "{value}"'
)
else:
self.recycle = value
logging.debug(f'Ok! stop_thread = "{value}"')
def quoted_data(self, value):
if not isinstance(value, bool):
raise TypeError(
f'quoted_data must be bool. quoted_data: {type(value)} = "{value}"'
)
else:
self.quotedData = value
logging.debug(f'Ok! quoted_data = "{value}"')
def __init__(self,
name: str = 'BasicElement',
comments: str = '',
is_enable: bool = True):
logging.debug(f'{type(self).__name__} | Init started...')
self.name = name
self.comments = comments
self.is_enable = is_enable
def get_video_counts(topic, videos_path=settings.CONTENT_ROOT, force=False):
""" Uses the (json) topic tree to query the django database for which video files exist
Returns the original topic dictionary, with two properties added to each NON-LEAF node:
* nvideos_known: The # of videos in and under that node, that are known (i.e. in the Khan Academy library)
* nvideos_local: The # of vidoes in and under that node, that were actually downloaded and available locally
And the following property for leaf nodes:
* on_disk
Input Parameters:
* videos_path: the path to video files
"""
if not force and "nvideos_local" in topic:
return (topic, topic["nvideos_local"], topic["nvideos_known"], False)
nvideos_local = 0
nvideos_known = 0
# Can't deal with leaves
assert "children" in topic, "Should not be calling this function on leaves; it's inefficient!"
# Only look for videos if there are more branches
if len(topic["children"]) == 0:
logging.debug("no children: %s" % topic["path"])
elif len(topic["children"]) > 0:
# RECURSIVE CALL:
# The children have children, let them figure things out themselves
# $ASSUMPTION: if first child is a branch, THEY'RE ALL BRANCHES.
# if first child is a leaf, THEY'RE ALL LEAVES
if "children" in topic["children"][0]:
for child in topic["children"]:
if not force and "nvideos_local" in child:
continue
(child, _, _, _) = get_video_counts(topic=child,
videos_path=videos_path)
nvideos_local += child["nvideos_local"]
nvideos_known += child["nvideos_known"]
# BASE CASE:
# All my children are leaves, so we'll query here (a bit more efficient than 1 query per leaf)
else:
videos = get_videos(topic)
for video in videos:
if force or "urls" not in video:
stamp_urls_on_video(video)
nvideos_local += int(bool(
video["urls"])) # add 1 if video["on_disk"]
nvideos_known = len(videos)
changed = topic.get("nvideos_local", -1) != nvideos_local
changed = changed or topic.get("nvideos_known", -1) != nvideos_known
topic["nvideos_local"] = nvideos_local
topic["nvideos_known"] = nvideos_known
topic["available"] = bool(nvideos_local) or bool(
settings.BACKUP_VIDEO_SOURCE)
return (topic, nvideos_local, nvideos_known, changed)
def run(self):
try:
logging.debug('Start smartcar in mode auto_pilot with sensors ...')
GpioMgmt().init_pwm()
self.ultrasonic.init_pwm()
self._run()
except Exception as e:
GpioMgmt().init_pin()
logging.error(str(e))
def file_encoding(self, value):
# todo: Check if the encoding type is valid
if not isinstance(value, str):
raise TypeError(
f'file_encoding must be str. file_encoding: {type(value)} = {value}'
)
else:
self.fileEncoding = value
logging.debug(f'Ok! Encoding = "{value}"')
def stamp_availability_on_topic(topic, videos_path=settings.CONTENT_ROOT, force=True, stamp_urls=True, update_counts_question_mark= None):
""" Uses the (json) topic tree to query the django database for which video files exist
Returns the original topic dictionary, with two properties added to each NON-LEAF node:
* nvideos_known: The # of videos in and under that node, that are known (i.e. in the Khan Academy library)
* nvideos_local: The # of vidoes in and under that node, that were actually downloaded and available locally
And the following property for leaf nodes:
* on_disk
Input Parameters:
* videos_path: the path to video files
"""
if update_counts_question_mark is None:
update_counts_question_mark = do_video_counts_need_update_question_mark()
if not force and "nvideos_local" in topic:
return (topic, topic["nvideos_local"], topic["nvideos_known"], False)
nvideos_local = 0
nvideos_known = 0
# Can't deal with leaves
assert topic["kind"] == "Topic", "Should not be calling this function on leaves; it's inefficient!"
# Only look for videos if there are more branches
if len(topic["children"]) == 0:
logging.debug("no children: %s" % topic["path"])
for child in topic["children"]:
# RECURSIVE CALL:
# The children have children, let them figure things out themselves
if "children" in child:
if not force and "nvideos_local" in child:
continue
stamp_availability_on_topic(topic=child, videos_path=videos_path, force=force, stamp_urls=stamp_urls, update_counts_question_mark=update_counts_question_mark)
nvideos_local += child["nvideos_local"]
nvideos_known += child["nvideos_known"]
# BASE CASE:
# All my children are leaves, so we'll query here (a bit more efficient than 1 query per leaf)
videos = get_videos(topic)
for video in videos:
if force or update_counts_question_mark or "availability" not in video:
stamp_availability_on_video(video, force=force, stamp_urls=stamp_urls, videos_path=videos_path)
nvideos_local += int(video["on_disk"])
nvideos_known += len(videos)
nvideos_available = nvideos_local if not settings.BACKUP_VIDEO_SOURCE else nvideos_known
changed = "nvideos_local" in topic and topic["nvideos_local"] != nvideos_local
changed = changed or ("nvideos_known" in topic and topic["nvideos_known"] != nvideos_known)
topic["nvideos_local"] = nvideos_local
topic["nvideos_known"] = nvideos_known
topic["nvideos_available"] = nvideos_available
topic["available"] = bool(nvideos_local) or bool(settings.BACKUP_VIDEO_SOURCE)
return (topic, nvideos_local, nvideos_known, nvideos_available, changed)
def variable_names(self, value):
# todo: Check if the variable names are valid
if not isinstance(value, str):
raise TypeError(
f'variable_names must be str. variable_names: {type(value)} = "{value}"'
)
else:
self.variableNames = value
logging.debug(f'Ok! variable_names = "{value}"')
def __init__(self, user_session: dict = None):
logging.debug(f'{type(self).__name__} | Init started...')
self._user_session = user_session
self._keyboard = types.InlineKeyboardMarkup()
self._create_keyboard_header()
self._create_keyboard_content()
self._create_keyboard_footer()
self._message_text = self._set_message_text()
logging.debug(f'{type(self).__name__} | Init complited')
def _create_keyboard_content(self):
logging.debug(
f'{type(self).__name__} | Create keyboard content started...')
select_destination_button = types.InlineKeyboardButton(
text="Узнать расписание",
callback_data=encode_data(GetBusSchedule.__name__))
self._keyboard.add(select_destination_button)
logging.debug(
f'{type(self).__name__} | Create keyboard content completed')
def get_video_counts(topic, videos_path=settings.CONTENT_ROOT, force=False):
""" Uses the (json) topic tree to query the django database for which video files exist
Returns the original topic dictionary, with two properties added to each NON-LEAF node:
* nvideos_known: The # of videos in and under that node, that are known (i.e. in the Khan Academy library)
* nvideos_local: The # of vidoes in and under that node, that were actually downloaded and available locally
And the following property for leaf nodes:
* on_disk
Input Parameters:
* videos_path: the path to video files
"""
if not force and "nvideos_local" in topic:
return (topic, topic["nvideos_local"], topic["nvideos_known"], False)
nvideos_local = 0
nvideos_known = 0
# Can't deal with leaves
assert "children" in topic, "Should not be calling this function on leaves; it's inefficient!"
# Only look for videos if there are more branches
if len(topic["children"]) == 0:
logging.debug("no children: %s" % topic["path"])
elif len(topic["children"]) > 0:
# RECURSIVE CALL:
# The children have children, let them figure things out themselves
# $ASSUMPTION: if first child is a branch, THEY'RE ALL BRANCHES.
# if first child is a leaf, THEY'RE ALL LEAVES
if "children" in topic["children"][0]:
for child in topic["children"]:
if not force and "nvideos_local" in child:
continue
(child, _, _, _) = get_video_counts(topic=child, videos_path=videos_path)
nvideos_local += child["nvideos_local"]
nvideos_known += child["nvideos_known"]
# BASE CASE:
# All my children are leaves, so we'll query here (a bit more efficient than 1 query per leaf)
else:
videos = get_videos(topic)
for video in videos:
if force or "urls" not in video:
stamp_urls_on_video(video)
nvideos_local += int(bool(video["urls"])) # add 1 if video["on_disk"]
nvideos_known = len(videos)
changed = topic.get("nvideos_local", -1) != nvideos_local
changed = changed or topic.get("nvideos_known", -1) != nvideos_known
topic["nvideos_local"] = nvideos_local
topic["nvideos_known"] = nvideos_known
topic["available"] = bool(nvideos_local) or bool(settings.BACKUP_VIDEO_SOURCE)
return (topic, nvideos_local, nvideos_known, changed)
def apply_plugin(self, plugin):
logging.debug("%s before apply plugins = %s" %
(self.text, self.plugins_context))
context_plugin = plugin.apply(self)
logging.debug("%s after apply plugins = %s" %
(self.text, self.plugins_context))
for area in context_plugin:
context_plugin[area] = [context_plugin[area]]
self.merge_plugin(copy.deepcopy(context_plugin))
if self.text:
for childview in self.children:
childview.apply_plugin(plugin)
def update_plugin_context(self):
logging.debug("%s update plugin context" % self.text)
#if self.classname == 'MainView' and self.parent:
# self.merge_plugin(self.parent.plugins_context)
#obj = self.parent
#if obj:
# while obj.parent: #do not add index's plugins
# self.merge_plugin(obj.plugins_context)
# obj = obj.parent
self.update_context({'plugins_context': self.plugins_context})
if 'sider' in self.plugins_context.keys():
self.update_context({'has_sider': True})
def run(self):
try:
logging.debug('Start smartcar in mode auto_pilot with camera ...')
GpioMgmt().init_pwm()
# self.ultrasonic.init_pwm()
self._run()
except KeyboardInterrupt as e:
GpioMgmt().release()
logging.info("[+] Exiting")
raise e
except Exception as e:
GpioMgmt().init_pin()
logging.error(str(e))
def get_local_video_size(youtube_id, default=None):
try:
return os.path.getsize(os.path.join(settings.CONTENT_ROOT, "%s.mp4" % youtube_id))
except Exception as e:
logging.debug(str(e))
return default
