Beispiel #1
0
    def get_page(self, url, page_number):
        # print url
        html_text = self.requests_page(url)
        if not html_text:
            return
        soup = BeautifulSoup(html_text)
        product_list = soup.select("li.item")
        for index , p in enumerate(product_list):
            rank = (page_number - 1) * 96 + index + 1
            # print rank
            # print index
            # print p
            # data_url = p.select()
            # print "+++++++++++++++++++++++++++++++++++++++++++++++++++++"
            data_url = p.select("a.plp_text_link")[0].attrs.get('href')
            true_data_url = 'http://www.revolve.com/'+data_url
            print true_data_url

            find_queue = session.query(CollectionQueue).filter(CollectionQueue.site_name == 'Revolve') \
                .filter(CollectionQueue.url == true_data_url).first()
            print find_queue
            if not find_queue:
                cq = CollectionQueue()
                cq.site_name = self.site_name
                cq.add_time = int(time.time())
                cq.category = self.category
                cq.secondary_category = self.second_category
                cq.rank = rank
                cq.url = true_data_url
                # cq.img_url = img_url
                session.add(cq)
                session.commit()
                print("added....")
Beispiel #2
0
def create_users(argvs):
    '''
    create little_finger access user
    :param argvs:
    :return:
    '''
    if '-f' in argvs:
        user_file  = argvs[argvs.index("-f") +1 ]
    else:
        print_err("invalid usage, should be:\ncreateusers -f <the new users file>",quit=True)

    source = yaml_parser(user_file)
    if source:
        for key,val in source.items():
            print(key,val)
            obj = models.UserProfile(username=key,password=val.get('password'))
            if val.get('groups'):
                groups = session.query(models.Group).filter(models.Group.name.in_(val.get('groups'))).all()
                if not groups:
                    print_err("none of [%s] exist in group table." % val.get('groups'),quit=True)
                obj.groups = groups
            if val.get('bind_hosts'):
                bind_hosts = common_filters.bind_hosts_filter(val)
                obj.bind_hosts = bind_hosts
            #print(obj)
            session.add(obj)
        session.commit()
Beispiel #3
0
def create_users(argvs):
    """
    create little_finger access user
    :param argvs:
    :return:
    """
    if "-f" in argvs:
        user_file = argvs[argvs.index("-f") + 1]
    else:
        print_err("invalid usage, should be:\ncreateusers -f <the new users file>", quit=True)

    source = yaml_parser(user_file)
    if source:
        for key, val in source.items():
            print(key, val)
            obj = models.UserProfile(username=key, password=val.get("password"))
            if val.get("groups"):
                groups = session.query(models.Group).filter(models.Group.name.in_(val.get("groups"))).all()
                if not groups:
                    print_err("none of [%s] exist in group table." % val.get("groups"), quit=True)
                obj.groups = groups
            if val.get("bind_hosts"):
                bind_hosts = common_filters.bind_hosts_filter(val)
                obj.bind_hosts = bind_hosts
            # print(obj)
            session.add(obj)
        session.commit()
Beispiel #4
0
def create_groups(argvs):
    '''
    create groups
    :param argvs:
    :return:
    '''
    if '-f' in argvs:
        group_file = argvs[argvs.index("-f") + 1]
    else:
        print_err(
            "invalid usage, should be:\ncreategroups -f <the new groups file>",
            quit=True)
    source = yaml_parser(group_file)
    if source:
        for key, val in source.items():
            print(key, val)
            obj = models.Group(name=key)
            if val.get('bind_hosts'):
                bind_hosts = common_filters.bind_hosts_filter(val)
                obj.bind_hosts = bind_hosts

            if val.get('user_profiles'):
                user_profiles = common_filters.user_profiles_filter(val)
                obj.user_profiles = user_profiles
            session.add(obj)
        session.commit()
Beispiel #5
0
def create_bindhosts(argvs):
    '''
    create bind hosts
    :param argvs:
    :return:
    '''
    if '-f' in argvs:
        bindhosts_file = argvs[argvs.index("-f") + 1]
    else:
        print_err(
            "invalid usage, should be:\ncreate_hosts -f <the new bindhosts file>",
            quit=True)
    source = yaml_parser(bindhosts_file)
    if source:
        for key, val in source.items():
            # print(key,val)
            host_obj = session.query(models.Host).filter(
                models.Host.hostname == val.get('hostname')).first()
            assert host_obj
            for item in val['remote_users']:
                print(item)
                assert item.get('auth_type')
                if item.get('auth_type') == 'ssh-passwd':
                    remoteuser_obj = session.query(models.RemoteUser).filter(
                        models.RemoteUser.username == item.get('username'),
                        models.RemoteUser.password == item.get(
                            'password')).first()
                else:
                    remoteuser_obj = session.query(models.RemoteUser).filter(
                        models.RemoteUser.username == item.get('username'),
                        models.RemoteUser.auth_type == item.get('auth_type'),
                    ).first()
                if not remoteuser_obj:
                    print_err("RemoteUser obj %s does not exist." % item,
                              quit=True)
                bindhost_obj = models.BindHost(host_id=host_obj.id,
                                               remoteuser_id=remoteuser_obj.id)
                session.add(bindhost_obj)
                # for groups this host binds to
                if source[key].get('groups'):
                    group_objs = session.query(models.Group).filter(
                        models.Group.name.in_(
                            source[key].get('groups'))).all()
                    assert group_objs
                    print('groups:', group_objs)
                    bindhost_obj.groups = group_objs
                # for user_profiles this host binds to
                if source[key].get('user_profiles'):
                    userprofile_objs = session.query(
                        models.UserProfile).filter(
                            models.UserProfile.username.in_(
                                source[key].get('user_profiles'))).all()
                    assert userprofile_objs
                    print("userprofiles:", userprofile_objs)
                    bindhost_obj.user_profiles = userprofile_objs
                # print(bindhost_obj)
        session.commit()
Beispiel #6
0
def create_remoteusers(argvs):
    '''
    create remoteusers
    :param argvs:
    :return:
    '''
    if '-f' in argvs:
        remoteusers_file  = argvs[argvs.index("-f") +1 ]
    else:
        print_err("invalid usage, should be:\ncreate_remoteusers -f <the new remoteusers file>",quit=True)
    source = yaml_parser(remoteusers_file)
    if source:
        for key,val in source.items():
            print(key,val)
            obj = models.RemoteUser(username=val.get('username'),auth_type=val.get('auth_type'),password=val.get('password'))
            session.add(obj)
        session.commit()
Beispiel #7
0
def create_hosts(argvs):
    '''
    create hosts
    :param argvs:
    :return:
    '''
    if '-f' in argvs:
        hosts_file  = argvs[argvs.index("-f") + 1]
    else:
        print_err("invalid usage, should be:\ncreate_hosts -f <the new hosts file>",quit=True)
    source = yaml_parser(hosts_file)
    if source:
        for key,val in source.items():
            print(key,val)
            obj = models.Host(hostname=key,ip_addr=val.get('ip_addr'), port=val.get('port') or 22)
            session.add(obj)
        session.commit()
Beispiel #8
0
def create_hosts(argvs):
    """
    create hosts
    :param argvs:
    :return:
    """
    if "-f" in argvs:
        hosts_file = argvs[argvs.index("-f") + 1]
    else:
        print_err("invalid usage, should be:\ncreate_hosts -f <the new hosts file>", quit=True)
    source = yaml_parser(hosts_file)
    if source:
        for key, val in source.items():
            print(key, val)
            obj = models.Host(hostname=key, ip_addr=val.get("ip_addr"), port=val.get("port") or 22)
            session.add(obj)
        session.commit()
Beispiel #9
0
def create_remoteusers(argvs):
    """
    create remoteusers
    :param argvs:
    :return:
    """
    if "-f" in argvs:
        remoteusers_file = argvs[argvs.index("-f") + 1]
    else:
        print_err("invalid usage, should be:\ncreate_remoteusers -f <the new remoteusers file>", quit=True)
    source = yaml_parser(remoteusers_file)
    if source:
        for key, val in source.items():
            print(key, val)
            obj = models.RemoteUser(
                username=val.get("username"), auth_type=val.get("auth_type"), password=val.get("password")
            )
            session.add(obj)
        session.commit()
Beispiel #10
0
    def get_page(self, url, page_number):
        # print url
        html_text = self.requests_page(url)
        if not html_text:
            return
        soup = BeautifulSoup(html_text)
        product_list = soup.select("div.category-image")
        # print product_list
        for index, p in enumerate(product_list):
            rank = (page_number - 1) * 24 + index + 1
            # print rank
            data_price = p.select("span.price")[-1]
            data_url = p.select("a")[0].attrs.get('href')
            true_data_url = 'http://www.lulus.com' + data_url
            # print true_data_url
            data_name = p.select("img")[0].attrs.get('alt')
            # print data_name
            img_url = p.select("img")[1].attrs.get('data-src')
            # print img_url

            if not data_price or not data_name or not true_data_url:
                print("[Not Found] url:%s" % url)
                return

            find_queue = session.query(CollectionQueue).filter(CollectionQueue.site_name == 'Lulus') \
                .filter(CollectionQueue.url == true_data_url).first()
            print find_queue
            if not find_queue:
                cq = CollectionQueue()
                cq.site_name = self.site_name
                cq.add_time = int(time.time())
                cq.category = self.category
                cq.secondary_category = self.second_category
                cq.rank = rank
                cq.url = true_data_url
                cq.img_url = img_url
                session.add(cq)
                session.commit()
                print("added....")
Beispiel #11
0
    def get_page(self, url, page_number):
        # print url
        html_text = self.requests_page(url)
        if not html_text:
            return
        soup = BeautifulSoup(html_text)
        product_list = soup.select("div.category-image")
        # print product_list
        for index, p in enumerate(product_list):
            rank = (page_number - 1) * 60 + index + 1
            print rank
            data_price = p.select("span.price")[-1]
            data_url = p.select("a")[0].attrs.get('href')
            true_data_url = 'http://www.lulus.com' + data_url
            # print true_data_url
            data_name = p.select("img")[0].attrs.get('alt')
            # print data_name
            img_url = p.select("img")[1].attrs.get('data-src')
            print img_url

            if not data_price or not data_name or not true_data_url:
                print("[Not Found] url:%s" % url)
                return

            find_queue = session.query(CollectionQueue).filter(CollectionQueue.site_name == 'Lulus') \
                .filter(CollectionQueue.url == true_data_url).first()
            print find_queue
            if not find_queue:
                cq = CollectionQueue()
                cq.site_name = self.site_name
                cq.add_time = int(time.time())
                cq.category = self.category
                cq.secondary_category = self.second_category
                cq.rank = rank
                cq.url = true_data_url
                cq.img_url = img_url
                session.add(cq)
                session.commit()
                print("added....")
Beispiel #12
0
def create_groups(argvs):
    """
    create groups
    :param argvs:
    :return:
    """
    if "-f" in argvs:
        group_file = argvs[argvs.index("-f") + 1]
    else:
        print_err("invalid usage, should be:\ncreategroups -f <the new groups file>", quit=True)
    source = yaml_parser(group_file)
    if source:
        for key, val in source.items():
            print(key, val)
            obj = models.Group(name=key)
            if val.get("bind_hosts"):
                bind_hosts = common_filters.bind_hosts_filter(val)
                obj.bind_hosts = bind_hosts

            if val.get("user_profiles"):
                user_profiles = common_filters.user_profiles_filter(val)
                obj.user_profiles = user_profiles
            session.add(obj)
        session.commit()
Beispiel #13
0
        goods = Goods()
        goods.goods_name = name
        goods.goods_price = price
        goods.add_time = int(time.time())
        goods.brands_name = site_name
        goods.img_url = down_img_url
        goods.rank = rank
        goods.status = 1
        goods.goods_url = url_dic.get('url', '')
        goods.details_img1 = img_1
        # goods.details_img2 = img2
        # goods.details_img3 = img3
        # goods.details_img4 = img4

        session.add(goods)
        session.commit()

        filter_color = None
        if color:
            filter_color = session.query(Filter).filter(Filter.name == color) \
                .filter(Filter.fg_id == filter_group_dic.get('color')).first()
        print filter_color

        if filter_color:
            color_goods_filter = GoodsFilter()
            color_goods_filter.goods_id = goods.goods_id
            color_goods_filter.filter_id = filter_color.filter_id
            color_goods_filter.filter_name = filter_color.name
            color_goods_filter.fg_id = filter_color.fg_id
            color_goods_filter.filter_group_name = 'color'
Beispiel #14
0
def create_bindhosts(argvs):
    """
    create bind hosts
    :param argvs:
    :return:
    """
    if "-f" in argvs:
        bindhosts_file = argvs[argvs.index("-f") + 1]
    else:
        print_err("invalid usage, should be:\ncreate_hosts -f <the new bindhosts file>", quit=True)
    source = yaml_parser(bindhosts_file)
    if source:
        for key, val in source.items():
            # print(key,val)
            host_obj = session.query(models.Host).filter(models.Host.hostname == val.get("hostname")).first()
            assert host_obj
            for item in val["remote_users"]:
                print(item)
                assert item.get("auth_type")
                if item.get("auth_type") == "ssh-passwd":
                    remoteuser_obj = (
                        session.query(models.RemoteUser)
                        .filter(
                            models.RemoteUser.username == item.get("username"),
                            models.RemoteUser.password == item.get("password"),
                        )
                        .first()
                    )
                else:
                    remoteuser_obj = (
                        session.query(models.RemoteUser)
                        .filter(
                            models.RemoteUser.username == item.get("username"),
                            models.RemoteUser.auth_type == item.get("auth_type"),
                        )
                        .first()
                    )
                if not remoteuser_obj:
                    print_err("RemoteUser obj %s does not exist." % item, quit=True)
                bindhost_obj = models.BindHost(host_id=host_obj.id, remoteuser_id=remoteuser_obj.id)
                session.add(bindhost_obj)
                # for groups this host binds to
                if source[key].get("groups"):
                    group_objs = (
                        session.query(models.Group).filter(models.Group.name.in_(source[key].get("groups"))).all()
                    )
                    assert group_objs
                    print("groups:", group_objs)
                    bindhost_obj.groups = group_objs
                # for user_profiles this host binds to
                if source[key].get("user_profiles"):
                    userprofile_objs = (
                        session.query(models.UserProfile)
                        .filter(models.UserProfile.username.in_(source[key].get("user_profiles")))
                        .all()
                    )
                    assert userprofile_objs
                    print("userprofiles:", userprofile_objs)
                    bindhost_obj.user_profiles = userprofile_objs
                # print(bindhost_obj)
        session.commit()
Beispiel #15
0
        goods = Goods()
        goods.goods_name = name
        goods.goods_price = price
        goods.add_time = int(time.time())
        goods.brands_name = site_name
        goods.img_url = down_img_url
        goods.rank = rank
        goods.status = 1
        goods.goods_url = url_dic.get('url', '')
        goods.details_img1 = img_1
        # goods.details_img2 = img2
        # goods.details_img3 = img3
        # goods.details_img4 = img4

        session.add(goods)
        session.commit()

        filter_color = None
        if color:
            filter_color = session.query(Filter).filter(Filter.name == color) \
                .filter(Filter.fg_id == filter_group_dic.get('color')).first()
        print filter_color

        if filter_color:
            color_goods_filter = GoodsFilter()
            color_goods_filter.goods_id = goods.goods_id
            color_goods_filter.filter_id = filter_color.filter_id
            color_goods_filter.filter_name = filter_color.name
            color_goods_filter.fg_id = filter_color.fg_id
            color_goods_filter.filter_group_name = 'color'
Beispiel #16
0
        # img1 = img_src[0]  # first pic to show
        # img2 = img_src[1]
        # img3 = img_src[2]
        # img4 = img_src[3]


        goods = Goods()
        goods.goods_name = name
        goods.goods_price = price
        goods.add_time = int(time.time())
        goods.brands_name = site_name
        goods.rank = rank
        goods.status = 1
        goods.goods_url = url_dic.get('url', '')

        session.add(goods)
        session.commit()

        curr_time = int(time.time())
        cq_info = session.query(CollectionQueue).filter(CollectionQueue.cq_id == cq_id).first()
        if cq_info:
            cq_info.goods_id = goods.goods_id
            cq_info.is_collected = 1
            cq_info.collected_time = curr_time

        ch = CollectionHistory()
        ch.goods_id =goods.goods_id
        ch.goods_price = price
        ch.rank = rank
        ch.collected_time = curr_time
        session.add(ch)
Beispiel #17
0
    def get_list(self):
        r = s.get(self.fav_url, headers=self.headers)
        # print r.content
        # print r.headers
        # print r.url
        # print r.status_code
        res_json = r.json()
        paging = res_json.get('paging', {})
        next_url = paging.get('next', '')
        data_list = res_json.get('data', [])

        for answer in data_list:
            question = answer.get('question', {})
            title = question.get('title', '')
            id = int(answer.get('id', ''))
            url = answer.get('url', '')
            if title and id > 0 and url:
                find_queue = session.query(CollectionQueue).filter(CollectionQueue.answer_id == id).first()
                if not find_queue:
                    self.check_num += 1
                    web_url = 'http://www.zhihu.com/question/20070065/answer/%s' % id
                    cq = CollectionQueue()
                    cq.fav_id = self.fav_id
                    cq.title = title
                    cq.api_url = url
                    cq.web_url = web_url
                    cq.answer_id = id
                    cq.add_time = int(time.time())
                    session.add(cq)
                    session.commit()
                    session.close()

                    sqs_body = {
                        'api_url':url,
                        'parent_note':self.parent_note,
                        'answer_id':cq.answer_id
                    }
                    m = Message()
                    m.set_body(json.dumps(sqs_body))
                    zhihufav_sqs.write(m)

                else:
                    if find_queue.fav_id == 0:
                        find_queue.fav_id = self.fav_id
                        session.commit()
                    print("[Find Queue] %s" % url)


        if len(data_list):
            self.check_page = self.check_page - 1
            self.fav_url = next_url
            if self.force:
                print("force next url %s" % next_url)
                return self.get_list()

            elif self.check_page > 0:
                print("start next url %s" % next_url)
                return self.get_list()


        title = self.title
        num = self.check_num
        if num > 0:
            InstaPushNotify.notify(title, str(num))