def get_mobile_phone(request): mongo_instance = MongoDB(MONGO_MOBILEBRAND_DB, MONGO_MOBILEBRAND_COLLECTIONS) try: update_time = request.GET.get("update_time") if update_time: update_time = datetime.strptime(update_time, "%Y-%m-%d %H:%M:%S.%f") else: update_time = datetime.now() - relativedelta(days=1) data_list = [] for data in mongo_instance.getAll(filter={"update_time": {"$gt": update_time}}, fields={"product_name": 1, "for_sale": 1, "brand_name": 1, "product_price": 1, "update_time": 1}): data["_id"] = "0x" + str(data["_id"]) data_list.append(data) except Exception: logger.exception("get_mobile_phone") return HttpResponseBadRequest("Bad Request!") else: return JsonResponse(data_list, safe=False) finally: mongo_instance.close()
class ProxyApi(object): def __init__(self): self.mongo_instance = MongoDB(MONGO_PROXY_DB, MONGO_PROXY_COLLECTIONS) # 线程安全并带连接池 self.scheme_filter_dict = { SchemeType.HTTP: { "$ne": SchemeType.HTTPS }, SchemeType.HTTPS: { "$ne": SchemeType.HTTP }, SchemeType.HTTP_OR_HTTPS: { "$eq": SchemeType.HTTP_OR_HTTPS }, } self.method_filter_dict = { SupportMethod.GET: { "$ne": SupportMethod.POST }, SupportMethod.POST: { "$ne": SupportMethod.GET }, SupportMethod.GET_OR_POST: { "$eq": SupportMethod.GET_OR_POST }, } self.good_quality_dict = { SchemeType.HTTP: { "$gte": len(HTTP_CHECK_URL_LIST) }, SchemeType.HTTPS: { "$gte": len(HTTPS_CHECK_URL_LIST) }, SchemeType.HTTP_OR_HTTPS: { "$gte": len(HTTPS_CHECK_URL_LIST) }, } self.good_response_time_dict = { SchemeType.HTTP: { "$lt": 1, "$gte": 0 }, SchemeType.HTTPS: { "$lt": 3, "$gte": 0 }, SchemeType.HTTP_OR_HTTPS: { "$lt": 1, "$gte": 0 }, } def close(self): self.mongo_instance.close() def get_proxy_from_kuaidaili(self, stable_time=StableTime.MIN_10): try: url = "http://dps.kuaidaili.com/api/getdps/?" \ "orderid=959308673589451&num=50&sep=2&ut=" + str(stable_time) resp = http_get(url) if resp.status_code != 200: raise NoProxyException return resp.text.split() except Exception: from traceback import print_exc print_exc() raise NoProxyException def get_proxy_all(self, location=None, anonymous=AnonymousLevel.MIDDLE, scheme=SchemeType.HTTP, method=SupportMethod.GET): the_filter = { "quality": self.good_quality_dict[scheme], "response_time": self.good_response_time_dict[scheme], # "anonymous_level": {"$lte": anonymous}, "scheme_type": self.scheme_filter_dict[scheme], # "support_method": self.method_filter_dict[method], } # if location: # the_filter["location"] = re_compile(".*" + location + ".*") cursor = self.mongo_instance.getAll(filter=the_filter, fields={ "ip": 1, "port": 1, "_id": 0 }, sort=[("response_time", MONGO_ASCENDING)]) return [item["ip"] + ":" + str(item["port"]) for item in cursor] def get_proxy_one(self, location=None, anonymous=AnonymousLevel.MIDDLE, scheme=SchemeType.HTTP, method=SupportMethod.GET, stable_time=StableTime.MIN_10): good_proxys = self.get_proxy_all(location, anonymous, scheme, method) if good_proxys: return rand_choice(good_proxys) else: raise NoProxyException