def get_page(self, url, page_number): # print url html_text = self.requests_page(url) if not html_text: return soup = BeautifulSoup(html_text) product_list = soup.select("li.item") for index , p in enumerate(product_list): rank = (page_number - 1) * 96 + index + 1 # print rank # print index # print p # data_url = p.select() # print "+++++++++++++++++++++++++++++++++++++++++++++++++++++" data_url = p.select("a.plp_text_link")[0].attrs.get('href') true_data_url = 'http://www.revolve.com/'+data_url print true_data_url find_queue = session.query(CollectionQueue).filter(CollectionQueue.site_name == 'Revolve') \ .filter(CollectionQueue.url == true_data_url).first() print find_queue if not find_queue: cq = CollectionQueue() cq.site_name = self.site_name cq.add_time = int(time.time()) cq.category = self.category cq.secondary_category = self.second_category cq.rank = rank cq.url = true_data_url # cq.img_url = img_url session.add(cq) session.commit() print("added....")
def create_users(argvs): ''' create little_finger access user :param argvs: :return: ''' if '-f' in argvs: user_file = argvs[argvs.index("-f") +1 ] else: print_err("invalid usage, should be:\ncreateusers -f <the new users file>",quit=True) source = yaml_parser(user_file) if source: for key,val in source.items(): print(key,val) obj = models.UserProfile(username=key,password=val.get('password')) if val.get('groups'): groups = session.query(models.Group).filter(models.Group.name.in_(val.get('groups'))).all() if not groups: print_err("none of [%s] exist in group table." % val.get('groups'),quit=True) obj.groups = groups if val.get('bind_hosts'): bind_hosts = common_filters.bind_hosts_filter(val) obj.bind_hosts = bind_hosts #print(obj) session.add(obj) session.commit()
def create_users(argvs): """ create little_finger access user :param argvs: :return: """ if "-f" in argvs: user_file = argvs[argvs.index("-f") + 1] else: print_err("invalid usage, should be:\ncreateusers -f <the new users file>", quit=True) source = yaml_parser(user_file) if source: for key, val in source.items(): print(key, val) obj = models.UserProfile(username=key, password=val.get("password")) if val.get("groups"): groups = session.query(models.Group).filter(models.Group.name.in_(val.get("groups"))).all() if not groups: print_err("none of [%s] exist in group table." % val.get("groups"), quit=True) obj.groups = groups if val.get("bind_hosts"): bind_hosts = common_filters.bind_hosts_filter(val) obj.bind_hosts = bind_hosts # print(obj) session.add(obj) session.commit()
def create_groups(argvs): ''' create groups :param argvs: :return: ''' if '-f' in argvs: group_file = argvs[argvs.index("-f") + 1] else: print_err( "invalid usage, should be:\ncreategroups -f <the new groups file>", quit=True) source = yaml_parser(group_file) if source: for key, val in source.items(): print(key, val) obj = models.Group(name=key) if val.get('bind_hosts'): bind_hosts = common_filters.bind_hosts_filter(val) obj.bind_hosts = bind_hosts if val.get('user_profiles'): user_profiles = common_filters.user_profiles_filter(val) obj.user_profiles = user_profiles session.add(obj) session.commit()
def create_bindhosts(argvs): ''' create bind hosts :param argvs: :return: ''' if '-f' in argvs: bindhosts_file = argvs[argvs.index("-f") + 1] else: print_err( "invalid usage, should be:\ncreate_hosts -f <the new bindhosts file>", quit=True) source = yaml_parser(bindhosts_file) if source: for key, val in source.items(): # print(key,val) host_obj = session.query(models.Host).filter( models.Host.hostname == val.get('hostname')).first() assert host_obj for item in val['remote_users']: print(item) assert item.get('auth_type') if item.get('auth_type') == 'ssh-passwd': remoteuser_obj = session.query(models.RemoteUser).filter( models.RemoteUser.username == item.get('username'), models.RemoteUser.password == item.get( 'password')).first() else: remoteuser_obj = session.query(models.RemoteUser).filter( models.RemoteUser.username == item.get('username'), models.RemoteUser.auth_type == item.get('auth_type'), ).first() if not remoteuser_obj: print_err("RemoteUser obj %s does not exist." % item, quit=True) bindhost_obj = models.BindHost(host_id=host_obj.id, remoteuser_id=remoteuser_obj.id) session.add(bindhost_obj) # for groups this host binds to if source[key].get('groups'): group_objs = session.query(models.Group).filter( models.Group.name.in_( source[key].get('groups'))).all() assert group_objs print('groups:', group_objs) bindhost_obj.groups = group_objs # for user_profiles this host binds to if source[key].get('user_profiles'): userprofile_objs = session.query( models.UserProfile).filter( models.UserProfile.username.in_( source[key].get('user_profiles'))).all() assert userprofile_objs print("userprofiles:", userprofile_objs) bindhost_obj.user_profiles = userprofile_objs # print(bindhost_obj) session.commit()
def create_remoteusers(argvs): ''' create remoteusers :param argvs: :return: ''' if '-f' in argvs: remoteusers_file = argvs[argvs.index("-f") +1 ] else: print_err("invalid usage, should be:\ncreate_remoteusers -f <the new remoteusers file>",quit=True) source = yaml_parser(remoteusers_file) if source: for key,val in source.items(): print(key,val) obj = models.RemoteUser(username=val.get('username'),auth_type=val.get('auth_type'),password=val.get('password')) session.add(obj) session.commit()
def create_hosts(argvs): ''' create hosts :param argvs: :return: ''' if '-f' in argvs: hosts_file = argvs[argvs.index("-f") + 1] else: print_err("invalid usage, should be:\ncreate_hosts -f <the new hosts file>",quit=True) source = yaml_parser(hosts_file) if source: for key,val in source.items(): print(key,val) obj = models.Host(hostname=key,ip_addr=val.get('ip_addr'), port=val.get('port') or 22) session.add(obj) session.commit()
def create_hosts(argvs): """ create hosts :param argvs: :return: """ if "-f" in argvs: hosts_file = argvs[argvs.index("-f") + 1] else: print_err("invalid usage, should be:\ncreate_hosts -f <the new hosts file>", quit=True) source = yaml_parser(hosts_file) if source: for key, val in source.items(): print(key, val) obj = models.Host(hostname=key, ip_addr=val.get("ip_addr"), port=val.get("port") or 22) session.add(obj) session.commit()
def create_remoteusers(argvs): """ create remoteusers :param argvs: :return: """ if "-f" in argvs: remoteusers_file = argvs[argvs.index("-f") + 1] else: print_err("invalid usage, should be:\ncreate_remoteusers -f <the new remoteusers file>", quit=True) source = yaml_parser(remoteusers_file) if source: for key, val in source.items(): print(key, val) obj = models.RemoteUser( username=val.get("username"), auth_type=val.get("auth_type"), password=val.get("password") ) session.add(obj) session.commit()
def get_page(self, url, page_number): # print url html_text = self.requests_page(url) if not html_text: return soup = BeautifulSoup(html_text) product_list = soup.select("div.category-image") # print product_list for index, p in enumerate(product_list): rank = (page_number - 1) * 24 + index + 1 # print rank data_price = p.select("span.price")[-1] data_url = p.select("a")[0].attrs.get('href') true_data_url = 'http://www.lulus.com' + data_url # print true_data_url data_name = p.select("img")[0].attrs.get('alt') # print data_name img_url = p.select("img")[1].attrs.get('data-src') # print img_url if not data_price or not data_name or not true_data_url: print("[Not Found] url:%s" % url) return find_queue = session.query(CollectionQueue).filter(CollectionQueue.site_name == 'Lulus') \ .filter(CollectionQueue.url == true_data_url).first() print find_queue if not find_queue: cq = CollectionQueue() cq.site_name = self.site_name cq.add_time = int(time.time()) cq.category = self.category cq.secondary_category = self.second_category cq.rank = rank cq.url = true_data_url cq.img_url = img_url session.add(cq) session.commit() print("added....")
def get_page(self, url, page_number): # print url html_text = self.requests_page(url) if not html_text: return soup = BeautifulSoup(html_text) product_list = soup.select("div.category-image") # print product_list for index, p in enumerate(product_list): rank = (page_number - 1) * 60 + index + 1 print rank data_price = p.select("span.price")[-1] data_url = p.select("a")[0].attrs.get('href') true_data_url = 'http://www.lulus.com' + data_url # print true_data_url data_name = p.select("img")[0].attrs.get('alt') # print data_name img_url = p.select("img")[1].attrs.get('data-src') print img_url if not data_price or not data_name or not true_data_url: print("[Not Found] url:%s" % url) return find_queue = session.query(CollectionQueue).filter(CollectionQueue.site_name == 'Lulus') \ .filter(CollectionQueue.url == true_data_url).first() print find_queue if not find_queue: cq = CollectionQueue() cq.site_name = self.site_name cq.add_time = int(time.time()) cq.category = self.category cq.secondary_category = self.second_category cq.rank = rank cq.url = true_data_url cq.img_url = img_url session.add(cq) session.commit() print("added....")
def create_groups(argvs): """ create groups :param argvs: :return: """ if "-f" in argvs: group_file = argvs[argvs.index("-f") + 1] else: print_err("invalid usage, should be:\ncreategroups -f <the new groups file>", quit=True) source = yaml_parser(group_file) if source: for key, val in source.items(): print(key, val) obj = models.Group(name=key) if val.get("bind_hosts"): bind_hosts = common_filters.bind_hosts_filter(val) obj.bind_hosts = bind_hosts if val.get("user_profiles"): user_profiles = common_filters.user_profiles_filter(val) obj.user_profiles = user_profiles session.add(obj) session.commit()
goods = Goods() goods.goods_name = name goods.goods_price = price goods.add_time = int(time.time()) goods.brands_name = site_name goods.img_url = down_img_url goods.rank = rank goods.status = 1 goods.goods_url = url_dic.get('url', '') goods.details_img1 = img_1 # goods.details_img2 = img2 # goods.details_img3 = img3 # goods.details_img4 = img4 session.add(goods) session.commit() filter_color = None if color: filter_color = session.query(Filter).filter(Filter.name == color) \ .filter(Filter.fg_id == filter_group_dic.get('color')).first() print filter_color if filter_color: color_goods_filter = GoodsFilter() color_goods_filter.goods_id = goods.goods_id color_goods_filter.filter_id = filter_color.filter_id color_goods_filter.filter_name = filter_color.name color_goods_filter.fg_id = filter_color.fg_id color_goods_filter.filter_group_name = 'color'
def create_bindhosts(argvs): """ create bind hosts :param argvs: :return: """ if "-f" in argvs: bindhosts_file = argvs[argvs.index("-f") + 1] else: print_err("invalid usage, should be:\ncreate_hosts -f <the new bindhosts file>", quit=True) source = yaml_parser(bindhosts_file) if source: for key, val in source.items(): # print(key,val) host_obj = session.query(models.Host).filter(models.Host.hostname == val.get("hostname")).first() assert host_obj for item in val["remote_users"]: print(item) assert item.get("auth_type") if item.get("auth_type") == "ssh-passwd": remoteuser_obj = ( session.query(models.RemoteUser) .filter( models.RemoteUser.username == item.get("username"), models.RemoteUser.password == item.get("password"), ) .first() ) else: remoteuser_obj = ( session.query(models.RemoteUser) .filter( models.RemoteUser.username == item.get("username"), models.RemoteUser.auth_type == item.get("auth_type"), ) .first() ) if not remoteuser_obj: print_err("RemoteUser obj %s does not exist." % item, quit=True) bindhost_obj = models.BindHost(host_id=host_obj.id, remoteuser_id=remoteuser_obj.id) session.add(bindhost_obj) # for groups this host binds to if source[key].get("groups"): group_objs = ( session.query(models.Group).filter(models.Group.name.in_(source[key].get("groups"))).all() ) assert group_objs print("groups:", group_objs) bindhost_obj.groups = group_objs # for user_profiles this host binds to if source[key].get("user_profiles"): userprofile_objs = ( session.query(models.UserProfile) .filter(models.UserProfile.username.in_(source[key].get("user_profiles"))) .all() ) assert userprofile_objs print("userprofiles:", userprofile_objs) bindhost_obj.user_profiles = userprofile_objs # print(bindhost_obj) session.commit()
# img1 = img_src[0] # first pic to show # img2 = img_src[1] # img3 = img_src[2] # img4 = img_src[3] goods = Goods() goods.goods_name = name goods.goods_price = price goods.add_time = int(time.time()) goods.brands_name = site_name goods.rank = rank goods.status = 1 goods.goods_url = url_dic.get('url', '') session.add(goods) session.commit() curr_time = int(time.time()) cq_info = session.query(CollectionQueue).filter(CollectionQueue.cq_id == cq_id).first() if cq_info: cq_info.goods_id = goods.goods_id cq_info.is_collected = 1 cq_info.collected_time = curr_time ch = CollectionHistory() ch.goods_id =goods.goods_id ch.goods_price = price ch.rank = rank ch.collected_time = curr_time session.add(ch)
def get_list(self): r = s.get(self.fav_url, headers=self.headers) # print r.content # print r.headers # print r.url # print r.status_code res_json = r.json() paging = res_json.get('paging', {}) next_url = paging.get('next', '') data_list = res_json.get('data', []) for answer in data_list: question = answer.get('question', {}) title = question.get('title', '') id = int(answer.get('id', '')) url = answer.get('url', '') if title and id > 0 and url: find_queue = session.query(CollectionQueue).filter(CollectionQueue.answer_id == id).first() if not find_queue: self.check_num += 1 web_url = 'http://www.zhihu.com/question/20070065/answer/%s' % id cq = CollectionQueue() cq.fav_id = self.fav_id cq.title = title cq.api_url = url cq.web_url = web_url cq.answer_id = id cq.add_time = int(time.time()) session.add(cq) session.commit() session.close() sqs_body = { 'api_url':url, 'parent_note':self.parent_note, 'answer_id':cq.answer_id } m = Message() m.set_body(json.dumps(sqs_body)) zhihufav_sqs.write(m) else: if find_queue.fav_id == 0: find_queue.fav_id = self.fav_id session.commit() print("[Find Queue] %s" % url) if len(data_list): self.check_page = self.check_page - 1 self.fav_url = next_url if self.force: print("force next url %s" % next_url) return self.get_list() elif self.check_page > 0: print("start next url %s" % next_url) return self.get_list() title = self.title num = self.check_num if num > 0: InstaPushNotify.notify(title, str(num))