def get_plant_info(name: str, page_size: int) -> List[Dict[str, Any]]: try: trefle_token = os.environ["TREFLE_ACCESS_TOKEN"] api = Shamrock(trefle_token, page_size=page_size) except KeyError as e: logger.error( "TREFLE_ACCESS_TOKEN environment variable not set, cannot fetch plants data" ) raise e except Exception as e: logger.error( f"Could not create Trefle API resource due to {type(e)}, args: {e.args}" ) raise e logger.info(f"Fetching data for plant name: {name}") batch = api.search(name) search_results = [] for d in batch: plant = api.plants(d["id"]) search_results.append(plant) return search_results
async def crawl(self): logger.info(f'{self._name}开始爬取...') try: return await self.do_crawl() except Exception as e: logger.exception(f'{self._name}爬取失败:e:{e}') return []
def post_sensors_data() -> Response: """ Endpoint receiving sensors data Db table: CREATE TABLE events.sensors ( uuid UUID DEFAULT uuid_generate_v4 (), created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, measure_type VARCHAR(32) NOT NULL, unit VARCHAR(32) NOT NULL, value NUMERIC NOT NULL ); curl -X POST http://localhost:5000/pi/sensors -H "Content-Type: application/json" -d @tests/resources/post_pi_sensor.json """ try: payload = request.get_json() with get_cursor(db_creds, commit=True) as cur: logger.info(request.data) q = ( f"INSERT INTO events.sensors (measure_type, unit, value) VALUES ('{payload['measure_type']}', " f"'{payload['unit']}', '{payload['value']}')") cur.execute(q) return jsonify(status_code=200, ) except KeyError as e: return jsonify( message=f"Missing key in payload: {type(e)}", args=e.args, status_code=400, error_type="Bad Request", ) except UniqueViolation as e: return jsonify( message=f"Entity already exists: {type(e)}", args=e.args, status_code=409, error_type="Conflict", ) except psycopg2.Error as e: return jsonify( message=f"Psycopg2 driver error: {type(e)}", args=e.args, status_code=500, error_type="Internal Server Error", ) except Exception as e: return jsonify( message=f"Internal Server Error: {type(e)}", args=e.args, status_code=500, error_type="Internal Server Error", )
def run(): logger.info('初始化sqlite数据库...') sqlite_opt.init_db() scheduler = BackgroundScheduler() scheduler.add_job(crawl, 'interval', seconds=SPIDER['crawl_interval']) scheduler.add_job(validator.run, 'interval', seconds=VALIDATOR['validate_interval']) scheduler.add_job(anonymity_validator.run, 'interval', seconds=ANONYMITY_VALIDATOR['interval']) scheduler.start() app.run(host=WEB_SERVER['host'], port=WEB_SERVER['port'])
def add_proxy(self, proxy): session = self._DBSession() session.add(proxy) result = 0 # 提交即保存到数据库: try: session.commit() result = 1 except IntegrityError as e: logger.info(f'ip: {proxy.url} 已存在') finally: # 关闭session: session.close() return result
async def crawl(self): logger.info(f'{self._name}开始爬取...') res = [] for url in self._urls: try: for page in self.get_page_range(): async with aiohttp.ClientSession() as session: async with session.get(self.get_page_url(url, page), headers=HEADERS) as resp: resp.encoding = self.get_encoding() temp = self.do_crawl(await resp.text()) res.extend(temp) await asyncio.sleep(self.get_interval()) except Exception as e: logger.exception(f'{self._name}爬取失败url: {url}, :e:{e}') return res
async def valid_proxy(self, proxy_url, proxy_type): async with aiohttp.ClientSession() as session: try: async with session.get( self.urls[proxy_type], proxy=proxy_url, headers=HEADERS, timeout=ANONYMITY_VALIDATOR['request_timeout'] ) as resp: if resp.status == 200: # 检验其匿名性 r_dict = json.loads(await resp.text()) headers = r_dict.get('headers', '') ip = r_dict.get('origin') proxy_connection = headers.get('Proxy-Connection', None) flag = True if ',' in ip: ips = str.split(ip, ',') first = ips[0] for p in ips: if first != p.lstrip(): proxy_cover = ProxyCoverEnum.TRANSPARENT.value # 透明 flag = False break if flag: if proxy_connection: proxy_cover = ProxyCoverEnum.NORMAL_COVER.value # 普匿 else: proxy_cover = ProxyCoverEnum.HIGH_COVER.value # 高匿 # 更新匿名性 sqlite_opt.update_anonymity(proxy_url, proxy_cover) logger.info( f'验证匿名性成功: url:{proxy_url}, coverValue:{proxy_cover}' ) else: logger.warn( f'验证匿名性失败, proxy_url:{proxy_url}, 返回码:{resp.status}' ) except asyncio.TimeoutError: logger.warn(f'验证匿名性请求超时, proxy_url:{proxy_url}') except ConnectionRefusedError: logger.warn(f'验证匿名性请求被拒绝, proxy_url:{proxy_url}') except Exception as e: # logger.exception(e) logger.warn(f'验证匿名性失败, proxy_url:{proxy_url}, e:{e}')
tasks = [] for spider_name in SPIDER['list']: tasks.append(spider_collection[spider_name].crawl()) # proxies.extend(spider_collection[spider_name].crawl()) loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) results = loop.run_until_complete(asyncio.gather(*tasks)) loop.close() for proxies_list in results: proxies.extend(proxies_list) # proxies = loop.run_until_complete(asyncio.gather(*tasks)) # 持久化 save(proxies) def save(proxies: typing.List[ProxyEntity]): for proxy in proxies: sqlite_opt.add_proxy(proxy) if __name__ == '__main__': logger.info('初始化sqlite数据库...') sqlite_opt.init_db() scheduler = BackgroundScheduler() scheduler.add_job(crawl, 'interval', seconds=SPIDER['crawl_interval']) scheduler.add_job(validator.run, 'interval', seconds=VALIDATOR['validate_interval']) scheduler.start() app.run(host=WEB_SERVER['host'], port=WEB_SERVER['port'])
def run(self): logger.info('开始删除不可用代理') sqlite_opt.remove_all_zero_reliability() logger.info('不可用代理删除完毕')
def spider_register(cls): spider_collection.update({cls.__name__: cls()}) logger.info(f'注册{cls.__name__}') return cls
def identify_plant(): """ Endpoint receiving an image and querying Plant.id to identify the plant species curl -X POST http://localhost:5000/identify_plant -F "file=@./tests/resources/test_flower_img.jpeg" -H "Content-Type: multipart/form-data" """ if "file" not in request.files: return jsonify( message=f"Bad Request: file missing", status_code=400, error_type="Bad Request", ) file = request.files["file"] # if user does not select file, browser also # submit a empty part without filename if file.filename == "" or not file \ or not allowed_file(file.filename, ALLOWED_IMG_EXTENSIONS): return jsonify( message=f"Bad Request: file missing", status_code=400, error_type="Bad Request", ) filename = secure_filename(file.filename) tmp_filename = os.path.join(app.config['UPLOAD_FOLDER'], filename) try: logger.info(f"Saving tmp image file at {tmp_filename}") # Save temp file, clean afterwards file.save(tmp_filename) # Build plant.id request with open(tmp_filename, "rb") as f: images = [base64.b64encode(f.read()).decode("ascii")] plant_id_payload = { "images": images, "modifiers": ["similar_images"], "plant_details": [ "name_authority", "common_names", "url", "wiki_description", "taxonomy" ], } # Return plant.id response response = requests.post("https://api.plant.id/v2/identify", json=plant_id_payload, headers={ "Content-Type": "application/json", "Api-Key": os.environ["PLANT_ID_API_ACCESS_TOKEN"] }).json() return response except Exception as e: return jsonify( message=f"Internal Server Error: {type(e)}", args=e.args, status_code=500, error_type="Internal Server Error", ) finally: if os.path.exists(tmp_filename): logger.info(f"Removing tmp image file at {tmp_filename}") os.remove(tmp_filename)