def get_page(self, url, page_number):
# print url
html_text = self.requests_page(url)
if not html_text:
return
soup = BeautifulSoup(html_text)
product_list = soup.select("li.item")
for index , p in enumerate(product_list):
rank = (page_number - 1) * 96 + index + 1
# print rank
# print index
# print p
# data_url = p.select()
# print "+++++++++++++++++++++++++++++++++++++++++++++++++++++"
data_url = p.select("a.plp_text_link")[0].attrs.get('href')
true_data_url = 'http://www.revolve.com/'+data_url
print true_data_url
find_queue = session.query(CollectionQueue).filter(CollectionQueue.site_name == 'Revolve') \
.filter(CollectionQueue.url == true_data_url).first()
print find_queue
if not find_queue:
cq = CollectionQueue()
cq.site_name = self.site_name
cq.add_time = int(time.time())
cq.category = self.category
cq.secondary_category = self.second_category
cq.rank = rank
cq.url = true_data_url
# cq.img_url = img_url
session.add(cq)
session.commit()
print("added....")
def create_users(argvs):
'''
create little_finger access user
:param argvs:
:return:
'''
if '-f' in argvs:
user_file = argvs[argvs.index("-f") +1 ]
else:
print_err("invalid usage, should be:\ncreateusers -f <the new users file>",quit=True)
source = yaml_parser(user_file)
if source:
for key,val in source.items():
print(key,val)
obj = models.UserProfile(username=key,password=val.get('password'))
if val.get('groups'):
groups = session.query(models.Group).filter(models.Group.name.in_(val.get('groups'))).all()
if not groups:
print_err("none of [%s] exist in group table." % val.get('groups'),quit=True)
obj.groups = groups
if val.get('bind_hosts'):
bind_hosts = common_filters.bind_hosts_filter(val)
obj.bind_hosts = bind_hosts
#print(obj)
session.add(obj)
session.commit()
def create_users(argvs):
"""
create little_finger access user
:param argvs:
:return:
"""
if "-f" in argvs:
user_file = argvs[argvs.index("-f") + 1]
else:
print_err("invalid usage, should be:\ncreateusers -f <the new users file>", quit=True)
source = yaml_parser(user_file)
if source:
for key, val in source.items():
print(key, val)
obj = models.UserProfile(username=key, password=val.get("password"))
if val.get("groups"):
groups = session.query(models.Group).filter(models.Group.name.in_(val.get("groups"))).all()
if not groups:
print_err("none of [%s] exist in group table." % val.get("groups"), quit=True)
obj.groups = groups
if val.get("bind_hosts"):
bind_hosts = common_filters.bind_hosts_filter(val)
obj.bind_hosts = bind_hosts
# print(obj)
session.add(obj)
session.commit()
def create_groups(argvs):
'''
create groups
:param argvs:
:return:
'''
if '-f' in argvs:
group_file = argvs[argvs.index("-f") + 1]
else:
print_err(
"invalid usage, should be:\ncreategroups -f <the new groups file>",
quit=True)
source = yaml_parser(group_file)
if source:
for key, val in source.items():
print(key, val)
obj = models.Group(name=key)
if val.get('bind_hosts'):
bind_hosts = common_filters.bind_hosts_filter(val)
obj.bind_hosts = bind_hosts
if val.get('user_profiles'):
user_profiles = common_filters.user_profiles_filter(val)
obj.user_profiles = user_profiles
session.add(obj)
session.commit()
def create_bindhosts(argvs):
'''
create bind hosts
:param argvs:
:return:
'''
if '-f' in argvs:
bindhosts_file = argvs[argvs.index("-f") + 1]
else:
print_err(
"invalid usage, should be:\ncreate_hosts -f <the new bindhosts file>",
quit=True)
source = yaml_parser(bindhosts_file)
if source:
for key, val in source.items():
# print(key,val)
host_obj = session.query(models.Host).filter(
models.Host.hostname == val.get('hostname')).first()
assert host_obj
for item in val['remote_users']:
print(item)
assert item.get('auth_type')
if item.get('auth_type') == 'ssh-passwd':
remoteuser_obj = session.query(models.RemoteUser).filter(
models.RemoteUser.username == item.get('username'),
models.RemoteUser.password == item.get(
'password')).first()
else:
remoteuser_obj = session.query(models.RemoteUser).filter(
models.RemoteUser.username == item.get('username'),
models.RemoteUser.auth_type == item.get('auth_type'),
).first()
if not remoteuser_obj:
print_err("RemoteUser obj %s does not exist." % item,
quit=True)
bindhost_obj = models.BindHost(host_id=host_obj.id,
remoteuser_id=remoteuser_obj.id)
session.add(bindhost_obj)
# for groups this host binds to
if source[key].get('groups'):
group_objs = session.query(models.Group).filter(
models.Group.name.in_(
source[key].get('groups'))).all()
assert group_objs
print('groups:', group_objs)
bindhost_obj.groups = group_objs
# for user_profiles this host binds to
if source[key].get('user_profiles'):
userprofile_objs = session.query(
models.UserProfile).filter(
models.UserProfile.username.in_(
source[key].get('user_profiles'))).all()
assert userprofile_objs
print("userprofiles:", userprofile_objs)
bindhost_obj.user_profiles = userprofile_objs
# print(bindhost_obj)
session.commit()
def create_remoteusers(argvs):
'''
create remoteusers
:param argvs:
:return:
'''
if '-f' in argvs:
remoteusers_file = argvs[argvs.index("-f") +1 ]
else:
print_err("invalid usage, should be:\ncreate_remoteusers -f <the new remoteusers file>",quit=True)
source = yaml_parser(remoteusers_file)
if source:
for key,val in source.items():
print(key,val)
obj = models.RemoteUser(username=val.get('username'),auth_type=val.get('auth_type'),password=val.get('password'))
session.add(obj)
session.commit()
def create_hosts(argvs):
'''
create hosts
:param argvs:
:return:
'''
if '-f' in argvs:
hosts_file = argvs[argvs.index("-f") + 1]
else:
print_err("invalid usage, should be:\ncreate_hosts -f <the new hosts file>",quit=True)
source = yaml_parser(hosts_file)
if source:
for key,val in source.items():
print(key,val)
obj = models.Host(hostname=key,ip_addr=val.get('ip_addr'), port=val.get('port') or 22)
session.add(obj)
session.commit()
def create_hosts(argvs):
"""
create hosts
:param argvs:
:return:
"""
if "-f" in argvs:
hosts_file = argvs[argvs.index("-f") + 1]
else:
print_err("invalid usage, should be:\ncreate_hosts -f <the new hosts file>", quit=True)
source = yaml_parser(hosts_file)
if source:
for key, val in source.items():
print(key, val)
obj = models.Host(hostname=key, ip_addr=val.get("ip_addr"), port=val.get("port") or 22)
session.add(obj)
session.commit()
def create_remoteusers(argvs):
"""
create remoteusers
:param argvs:
:return:
"""
if "-f" in argvs:
remoteusers_file = argvs[argvs.index("-f") + 1]
else:
print_err("invalid usage, should be:\ncreate_remoteusers -f <the new remoteusers file>", quit=True)
source = yaml_parser(remoteusers_file)
if source:
for key, val in source.items():
print(key, val)
obj = models.RemoteUser(
username=val.get("username"), auth_type=val.get("auth_type"), password=val.get("password")
)
session.add(obj)
session.commit()
def get_page(self, url, page_number):
# print url
html_text = self.requests_page(url)
if not html_text:
return
soup = BeautifulSoup(html_text)
product_list = soup.select("div.category-image")
# print product_list
for index, p in enumerate(product_list):
rank = (page_number - 1) * 24 + index + 1
# print rank
data_price = p.select("span.price")[-1]
data_url = p.select("a")[0].attrs.get('href')
true_data_url = 'http://www.lulus.com' + data_url
# print true_data_url
data_name = p.select("img")[0].attrs.get('alt')
# print data_name
img_url = p.select("img")[1].attrs.get('data-src')
# print img_url
if not data_price or not data_name or not true_data_url:
print("[Not Found] url:%s" % url)
return
find_queue = session.query(CollectionQueue).filter(CollectionQueue.site_name == 'Lulus') \
.filter(CollectionQueue.url == true_data_url).first()
print find_queue
if not find_queue:
cq = CollectionQueue()
cq.site_name = self.site_name
cq.add_time = int(time.time())
cq.category = self.category
cq.secondary_category = self.second_category
cq.rank = rank
cq.url = true_data_url
cq.img_url = img_url
session.add(cq)
session.commit()
print("added....")
def get_page(self, url, page_number):
# print url
html_text = self.requests_page(url)
if not html_text:
return
soup = BeautifulSoup(html_text)
product_list = soup.select("div.category-image")
# print product_list
for index, p in enumerate(product_list):
rank = (page_number - 1) * 60 + index + 1
print rank
data_price = p.select("span.price")[-1]
data_url = p.select("a")[0].attrs.get('href')
true_data_url = 'http://www.lulus.com' + data_url
# print true_data_url
data_name = p.select("img")[0].attrs.get('alt')
# print data_name
img_url = p.select("img")[1].attrs.get('data-src')
print img_url
if not data_price or not data_name or not true_data_url:
print("[Not Found] url:%s" % url)
return
find_queue = session.query(CollectionQueue).filter(CollectionQueue.site_name == 'Lulus') \
.filter(CollectionQueue.url == true_data_url).first()
print find_queue
if not find_queue:
cq = CollectionQueue()
cq.site_name = self.site_name
cq.add_time = int(time.time())
cq.category = self.category
cq.secondary_category = self.second_category
cq.rank = rank
cq.url = true_data_url
cq.img_url = img_url
session.add(cq)
session.commit()
print("added....")
def create_groups(argvs):
"""
create groups
:param argvs:
:return:
"""
if "-f" in argvs:
group_file = argvs[argvs.index("-f") + 1]
else:
print_err("invalid usage, should be:\ncreategroups -f <the new groups file>", quit=True)
source = yaml_parser(group_file)
if source:
for key, val in source.items():
print(key, val)
obj = models.Group(name=key)
if val.get("bind_hosts"):
bind_hosts = common_filters.bind_hosts_filter(val)
obj.bind_hosts = bind_hosts
if val.get("user_profiles"):
user_profiles = common_filters.user_profiles_filter(val)
obj.user_profiles = user_profiles
session.add(obj)
session.commit()
goods = Goods()
goods.goods_name = name
goods.goods_price = price
goods.add_time = int(time.time())
goods.brands_name = site_name
goods.img_url = down_img_url
goods.rank = rank
goods.status = 1
goods.goods_url = url_dic.get('url', '')
goods.details_img1 = img_1
# goods.details_img2 = img2
# goods.details_img3 = img3
# goods.details_img4 = img4
session.add(goods)
session.commit()
filter_color = None
if color:
filter_color = session.query(Filter).filter(Filter.name == color) \
.filter(Filter.fg_id == filter_group_dic.get('color')).first()
print filter_color
if filter_color:
color_goods_filter = GoodsFilter()
color_goods_filter.goods_id = goods.goods_id
color_goods_filter.filter_id = filter_color.filter_id
color_goods_filter.filter_name = filter_color.name
color_goods_filter.fg_id = filter_color.fg_id
color_goods_filter.filter_group_name = 'color'
def create_bindhosts(argvs):
"""
create bind hosts
:param argvs:
:return:
"""
if "-f" in argvs:
bindhosts_file = argvs[argvs.index("-f") + 1]
else:
print_err("invalid usage, should be:\ncreate_hosts -f <the new bindhosts file>", quit=True)
source = yaml_parser(bindhosts_file)
if source:
for key, val in source.items():
# print(key,val)
host_obj = session.query(models.Host).filter(models.Host.hostname == val.get("hostname")).first()
assert host_obj
for item in val["remote_users"]:
print(item)
assert item.get("auth_type")
if item.get("auth_type") == "ssh-passwd":
remoteuser_obj = (
session.query(models.RemoteUser)
.filter(
models.RemoteUser.username == item.get("username"),
models.RemoteUser.password == item.get("password"),
)
.first()
)
else:
remoteuser_obj = (
session.query(models.RemoteUser)
.filter(
models.RemoteUser.username == item.get("username"),
models.RemoteUser.auth_type == item.get("auth_type"),
)
.first()
)
if not remoteuser_obj:
print_err("RemoteUser obj %s does not exist." % item, quit=True)
bindhost_obj = models.BindHost(host_id=host_obj.id, remoteuser_id=remoteuser_obj.id)
session.add(bindhost_obj)
# for groups this host binds to
if source[key].get("groups"):
group_objs = (
session.query(models.Group).filter(models.Group.name.in_(source[key].get("groups"))).all()
)
assert group_objs
print("groups:", group_objs)
bindhost_obj.groups = group_objs
# for user_profiles this host binds to
if source[key].get("user_profiles"):
userprofile_objs = (
session.query(models.UserProfile)
.filter(models.UserProfile.username.in_(source[key].get("user_profiles")))
.all()
)
assert userprofile_objs
print("userprofiles:", userprofile_objs)
bindhost_obj.user_profiles = userprofile_objs
# print(bindhost_obj)
session.commit()
goods = Goods()
goods.goods_name = name
goods.goods_price = price
goods.add_time = int(time.time())
goods.brands_name = site_name
goods.img_url = down_img_url
goods.rank = rank
goods.status = 1
goods.goods_url = url_dic.get('url', '')
goods.details_img1 = img_1
# goods.details_img2 = img2
# goods.details_img3 = img3
# goods.details_img4 = img4
session.add(goods)
session.commit()
filter_color = None
if color:
filter_color = session.query(Filter).filter(Filter.name == color) \
.filter(Filter.fg_id == filter_group_dic.get('color')).first()
print filter_color
if filter_color:
color_goods_filter = GoodsFilter()
color_goods_filter.goods_id = goods.goods_id
color_goods_filter.filter_id = filter_color.filter_id
color_goods_filter.filter_name = filter_color.name
color_goods_filter.fg_id = filter_color.fg_id
color_goods_filter.filter_group_name = 'color'
# img1 = img_src[0] # first pic to show
# img2 = img_src[1]
# img3 = img_src[2]
# img4 = img_src[3]
goods = Goods()
goods.goods_name = name
goods.goods_price = price
goods.add_time = int(time.time())
goods.brands_name = site_name
goods.rank = rank
goods.status = 1
goods.goods_url = url_dic.get('url', '')
session.add(goods)
session.commit()
curr_time = int(time.time())
cq_info = session.query(CollectionQueue).filter(CollectionQueue.cq_id == cq_id).first()
if cq_info:
cq_info.goods_id = goods.goods_id
cq_info.is_collected = 1
cq_info.collected_time = curr_time
ch = CollectionHistory()
ch.goods_id =goods.goods_id
ch.goods_price = price
ch.rank = rank
ch.collected_time = curr_time
session.add(ch)
def get_list(self):
r = s.get(self.fav_url, headers=self.headers)
# print r.content
# print r.headers
# print r.url
# print r.status_code
res_json = r.json()
paging = res_json.get('paging', {})
next_url = paging.get('next', '')
data_list = res_json.get('data', [])
for answer in data_list:
question = answer.get('question', {})
title = question.get('title', '')
id = int(answer.get('id', ''))
url = answer.get('url', '')
if title and id > 0 and url:
find_queue = session.query(CollectionQueue).filter(CollectionQueue.answer_id == id).first()
if not find_queue:
self.check_num += 1
web_url = 'http://www.zhihu.com/question/20070065/answer/%s' % id
cq = CollectionQueue()
cq.fav_id = self.fav_id
cq.title = title
cq.api_url = url
cq.web_url = web_url
cq.answer_id = id
cq.add_time = int(time.time())
session.add(cq)
session.commit()
session.close()
sqs_body = {
'api_url':url,
'parent_note':self.parent_note,
'answer_id':cq.answer_id
}
m = Message()
m.set_body(json.dumps(sqs_body))
zhihufav_sqs.write(m)
else:
if find_queue.fav_id == 0:
find_queue.fav_id = self.fav_id
session.commit()
print("[Find Queue] %s" % url)
if len(data_list):
self.check_page = self.check_page - 1
self.fav_url = next_url
if self.force:
print("force next url %s" % next_url)
return self.get_list()
elif self.check_page > 0:
print("start next url %s" % next_url)
return self.get_list()
title = self.title
num = self.check_num
if num > 0:
InstaPushNotify.notify(title, str(num))