def parse_category(url, doc, res=''):
'''
desc:
用来解析面包屑, 分析出具体商品所属的cat层
'''
save_list = []
_sum_url = doc.xpath('//*[@id="root-nav"]/div/div/span[1]/a[2]/@href')[0]
_cat_list = parse_qs(urlparse(_sum_url).query,
keep_blank_values=False).get('cat')[0]
cat_list = [int(x) for x in _cat_list.split(',')]
assert len(cat_list) == 3
l1_name = doc.xpath(
'//*[@id="root-nav"]/div/div/strong/a')[0].text_content()
l1_url = doc.xpath('//*[@id="root-nav"]/div/div/strong/a/@href')[0]
l2_url = doc.xpath('//*[@id="root-nav"]/div/div/span[1]/a[1]/@href')[0]
l2_name = doc.xpath(
'//*[@id="root-nav"]/div/div/span[1]/a[1]')[0].text_content()
l3_url = doc.xpath('//*[@id="root-nav"]/div/div/span[1]/a[2]/@href')[0]
l3_name = doc.xpath('//*[@id="root-nav"]/div/div/span[1]/a[2]/text()')[0]
save_list.append({
"cat_id": cat_list[0],
"cat_name": l1_name,
"cat_url": l1_url,
"level": '1'
})
save_list.append({
"cat_id": cat_list[1],
"cat_name": l2_name,
"cat_url": l2_url,
"level": '2'
})
save_list.append({
"cat_id": cat_list[2],
"cat_name": l3_name,
"cat_url": l3_url,
"level": '3'
})
for _d in save_list:
try:
_d['cat_name'] = _d['cat_name'].encode('utf-8')
_d['cat_url'] = perfect_href(_d['cat_url'])
Category.create(**_d)
except IntegrityError:
logger.info('category faild repeat --- cat_id: %s , url:%s' %
(_d['cat_id'], url))
res = {
'top_id': cat_list[0],
'second_id': cat_list[1],
'third_id': cat_list[2],
'top_name': l1_name.encode('utf-8'),
'second_name': l2_name.encode('utf-8'),
'third_name': l3_name.encode('utf-8'),
}
return res
def test_create_stock_no_duplicate(self):
"""Makes sure stock don't have the same stock_id when creating."""
Category.create(category_id='001', description='Testing Stock')
create_stock('001', 'Testing stock', 1, '001', 9.99)
from peewee import IntegrityError
with self.assertRaises(IntegrityError):
create_stock('001', 'Testing stock', 1, '001', 9.99)
q = Stock.delete().where(Stock.stock_id == '001')
q.execute()
t = Category.delete().where(Category.category_id == '001')
t.execute()
def post(self):
name = self.get_argument('name', None)
slug = self.get_argument('slug', None)
q = Category.select().where(Category.name == name)
if q.count() > 0:
self.flash('cateegory exists!')
self.render('admin/category/add.html')
return
else:
Category.create(name=name, slug=slug)
self.redirect('/admin/category')
def post(self):
name = self.get_argument('name', None)
slug = self.get_argument('slug', None)
q = Category.select().where(Category.name == name)
if q.count() > 0:
self.flash('cateegory exists!')
self.render('admin/category/add.html')
return
else:
Category.create(name=name, slug=slug)
self.redirect('/admin/category')
def test_create_stock(self):
"""Make sure it create a stock."""
Category.create(category_id='001', description='Testing Stock')
create_stock('001', 'Testing stock', 1, '001', 9.99)
s = Stock.select().where(Stock.stock_id == '001')
t = Stock.delete().where(Stock.stock_id == '001')
t.execute()
t = Category.delete().where(Category.category_id == '001')
t.execute()
self.assertEqual(str(s), ("<class 'models.Stock'> SELECT `t1`.`id`,"
" `t1`.`stock_id`, `t1`.`description`,"
" `t1`.`quantity`, `t1`.`category_id`,"
" `t1`.`price` FROM `stock` AS t1"
" WHERE (`t1`.`stock_id` = %s) ['001']"))
def test_create_outgoing(self):
"""Create an outgoing record."""
Category.create(category_id='001', description='Testing Stock')
Project.create(project_id='001', project_description="Testing")
Role.create(role_name='Admin')
User.create(first_name='Jay', second_name='Palm', phone='9783978223', role='Admin', username='******',
password='******', email='*****@*****.**')
create_stock('001', 'Testing stock', 1, '001', 9.99)
create_outgoing_stock(stock='001', project_id='001', date="2015-07-22", quantity='7', username='******')
OutgoingStock.get(OutgoingStock.stock == '001').delete_instance()
Stock.get(Stock.category == '001').delete_instance()
User.get(User.username == 'JayPalm').delete_instance()
Role.get(Role.role_name == 'Admin').delete_instance()
Project.get(Project.project_id == '001').delete_instance()
Category.get(Category.category_id == '001').delete_instance()
def test_create_incoming(self):
"""Create an incoming record."""
Category.create(category_id='001', description='Testing Stock')
create_stock('001', 'Testing stock', 1, '001', 9.99)
create_incoming_stock(stock="001", date="2015-07-22", quantity=13, price=13.99)
p = IncomingStock.select().where(IncomingStock.stock == '001')
q = IncomingStock.delete().where(IncomingStock.stock == '001')
q.execute()
s = Stock.select().where(Stock.stock_id == '001')
t = Stock.delete().where(Stock.stock_id == '001')
t.execute()
t = Category.delete().where(Category.category_id == '001')
t.execute()
self.assertEqual(str(p), ("<class 'models.IncomingStock'> SELECT `t1`.`id`, `t1`.`stock_id`, "
"`t1`.`date`, `t1`.`price`, `t1`.`cost` FROM `incomingstock` AS t1 WHERE "
"(`t1`.`stock_id` = %s) ['001']"))
def create(name):
try:
user = user_session.current_user()
category = Category.create(name=name, user=user)
category.save()
return f'Category {category.name} created successfully!'
except peewee.IntegrityError:
return f'Category {name} already exist!'
def add_category():
if request.method == 'GET':
return render_template("add_category.html")
name = request.form.get('category', '').replace(' ', '_')
try:
is_valid_category(name)
category = Category.create(name)
except RuntimeError, e:
return render_template("add_category.html", category=name, error=e.message)
def category_add():
if request.method == 'GET':
all_categories = Category.select()
template = env.get_template('post/category_add.html')
return template.render(categories=all_categories)
if request.method == 'POST':
new_category = Category.create(category_name=post_get('category_name'))
app.flash(u'Нова категорія була успішно додана')
redirect('/category/add')
def create_db():
logging.info("Creating SQLite database %s..." % DB_FILE)
# Define database
db = SqliteDatabase(DB_FILE, threadlocals=True)
# Connect to database
db.connect()
# Create tables
Category.create_table()
Feed.create_table()
Post.create_table()
Image.create_table()
# Create default Unsorted category
Category.create(name='Unsorted', comment='Uncategorised feeds', order=1)
logging.info("Database created.")
def create_db():
logging.info("Creating SQLite database %s..." % DB_FILE)
# Define database
db = SqliteDatabase(DB_FILE, threadlocals=True)
# Connect to database
db.connect()
# Create tables
Category.create_table()
Feed.create_table()
Post.create_table()
Image.create_table()
# Create default Unsorted category
Category.create(name='Unsorted', comment='Uncategorised feeds', order=1)
logging.info("Database created.")
def add_categories():
new_categories = {}
categories_input = flask.request.form.get("categories-input", "", type=str)
for name in categories_input.split(";"):
name = name.strip()
if not name:
break
if not Category.select().where(Category.name == name).exists():
category = Category.create(name=name)
new_categories[str(category.id)] = category.name
return flask.jsonify(**new_categories)
def create(category):
id = category.get('id') if category.get('id') else str(uuid.uuid4())
print(id)
if not validate_uuid(id, 4):
return uuid_notvalidate("Category", "id")
while True:
existing_cate = Category.query.filter(Category.id == id).one_or_none()
if existing_cate is None:
category['id'] = id
break
else:
id = str(uuid.uuid4())
cate = Category.create(**category)
return jsonify(cate.to_dict()), 201
def load_defaults():
logging.info("Loading default feed entries into database %s..." % DB_FILE)
# Open defaults file
with open(DEFAULTS_FILE, 'r') as f:
rows = f.readlines()
f.close()
# Define database
db = SqliteDatabase(DB_FILE, threadlocals=True)
# Connect to database
db.connect()
# Iterate over default feeds list
# PSV format name|url|category
for row in rows:
(name, url, category) = row.split('|')
category = category.strip()
# Update Category table
c = Category.create(name=category, comment='Default category', order=1)
# Get Category insert id
cid = c.id
# Update Feeds table
f = Feed.create(name=name,
version='',
url=url,
category=cid,
favicon='',
comment='Default feed',
description='Default feed')
# Get Feed insert id
fid = f.id
# Get favicon for this Feed
# returns path to local favicon file, or None
# write to current feed record
logging.info("Getting favicon for %s" % f.url)
f.favicon = getFavicon(fid)
logging.info("Got favicon %s" % f.favicon)
# Save entry to feeds table
f.save()
logging.info("Default feeds loaded.")
def post(self):
form = CategoryForm()
if request.method == 'POST':
if form.validate_on_submit():
try:
category = Category.create()
form.populate_obj(category)
category.save()
flash(gettext('Category succesfully created'))
return util.redirect_json_or_html(url_for('CategoriesView:index'), 'category')
except:
flash(gettext('Error while creating the category'), 'error')
else:
flash(gettext('Invalid submission, please check the message below'), 'error')
return render_template('admin/categories/add.html',
title=gettext('Create Category'),
form=form)
def load_defaults():
logging.info("Loading default feed entries into database %s..." % DB_FILE)
# Open defaults file
with open(DEFAULTS_FILE, 'r') as f:
rows = f.readlines()
f.close()
# Define database
db = SqliteDatabase(DB_FILE, threadlocals=True)
# Connect to database
db.connect()
# Iterate over default feeds list
# PSV format name|url|category
for row in rows:
(name, url, category) = row.split('|')
category = category.strip()
# Update Category table
c = Category.create(name=category, comment='Default category', order=1)
# Get Category insert id
cid = c.id
# Update Feeds table
f = Feed.create(name=name, version='', url=url, category=cid, favicon='', comment='Default feed', description='Default feed')
# Get Feed insert id
fid = f.id
# Get favicon for this Feed
# returns path to local favicon file, or None
# write to current feed record
logging.info("Getting favicon for %s" % f.url)
f.favicon = getFavicon(fid)
logging.info("Got favicon %s" % f.favicon)
# Save entry to feeds table
f.save()
logging.info("Default feeds loaded.")
def add_category():
# Start with no errors or fields
errors = None
fields = None
# Store user id from session to associate category with user
user_id = session['user_id']
if request.method == 'POST':
fields = {'name': request.form['name']}
# Validate form submission by checking no empty fields
errors = check_no_blanks(fields=fields)
if not errors:
# Check that category name does not already exist
# TODO: this check needs to check by case insensitive
if Category.get_by_name(fields['name']):
errors['name_exists'] = True
else:
category = Category.create(name=fields['name'],
user_id=user_id)
flash('Category created')
return redirect(url_for('view_category', id=category.id))
return render_template('add_category.html', fields=fields, errors=errors)
def web_scrape():
for u in URLS:
if u['type'] == 'winwin':
url = u['url']
headers = {'User-Agent': USER_AGENT}
response = requests.get(url, headers=headers)
soup = BeautifulSoup(response.text, "html.parser")
category = soup.find('div', attrs={'class': 'category-title'})
category_name = category.text.replace('/', '-').replace('\u0161', 's').replace('\u010d', 'c').replace('\u017e', 'z').strip()
cat = Category.get_category_by_name(category_name)
if cat:
pass
else:
cat = Category.create(name=category_name)
for li in soup.findAll('li', attrs={'class': 'item'}):
title = li.find('span', attrs={'itemprop': 'name'})
comments = li.find('div', attrs={'class': 'ratings'}).find('span', attrs={'class': 'text-info'})
if comments:
count = int(comments.text)
else:
count = 0
special_price = li.find('p', attrs={'class': 'special-price'})
if special_price:
price = special_price.find('span', attrs={'class': 'price'})
else:
price = li.find('span', attrs={'class': 'price'})
url = li.find('a', href=True, attrs={'class': 'product-image'})
model = title.text.replace('/', '-').replace(' ', '-').replace(',', '-').replace('*', '-').replace('.', '-').replace('"', '-').replace('\xa0', '-').replace('\u0161', 's').replace('\u010d', 'c').replace('\u017e', 'z')
product = Product.get_product_by_title(model)
if product:
pass
else:
Product.create(title=model,
comments=count, price=int(price.text[:-4].replace('.', "")),
url=url.get('href'), category=cat)
def insert_categories():
Category.create(name='Liquidos', description='liquidos')
Category.create(name='Embutidos', description='embutidos')
Category.create(name='Snacks', description='snacks')
Category.create(name='Aderezos', description='aderezos')
Category.create(name='Carnes', description='carnes')
def insert_data(table_name):
if table_name == 'users':
user = User.create(username='******',
age=30,
password='******',
email='*****@*****.**')
user = User.create(username='******',
age=35,
password='******',
email='*****@*****.**')
user = User.create(username='******',
age=60,
password='******',
email='*****@*****.**')
user = User.create(username='******',
age=68,
password='******',
email='*****@*****.**')
user = User.create(username='******',
age=23,
password='******',
email='*****@*****.**')
user.save()
elif table_name == 'stores':
store = Store.create(user_id=1,
name='La esquina',
address='Fuentes del Molino #10')
store = Store.create(user_id=1,
name='Don Simon',
address='Fuentes del Molino #12')
store = Store.create(user_id=1,
name='Carmelita',
address='Fuentes del Molino #13')
store = Store.create(user_id=4,
name='Wallis',
address='Fuentes del Molino #14')
store = Store.create(user_id=5,
name='Don perse',
address='Fuentes del Molino #16')
store.save()
elif table_name == 'products':
product = Product.create(store_id=1,
name='Pan',
description='Pan Integral',
price=5.5,
stock=10)
product = Product.create(store_id=1,
name='Leche',
description='Baja en grasas',
price=15.5,
stock=24)
product = Product.create(store_id=1,
name='Jamon',
description='Pavo',
price=45.5,
stock=10)
product = Product.create(store_id=2,
name='Soda',
description='Dieta',
price=10.5,
stock=10)
product = Product.create(store_id=2,
name='Fritura',
description='Churros',
price=9.5,
stock=10)
product = Product.create(store_id=2,
name='Salsa',
description='Habanero',
price=11.5,
stock=10)
product.save()
elif table_name == 'categories':
category = Category.create(name='Liquidos', description='liquidos')
category = Category.create(name='Embutidos', description='embutidos')
category = Category.create(name='Snacks', description='snacks')
category = Category.create(name='Aderezos', description='aderezos')
category = Category.create(name='Carnes', description='carnes')
category.save()
elif table_name == 'product_categories':
ProductCategory.create(category_id=1, product_id=2)
ProductCategory.create(category_id=1, product_id=4)
ProductCategory.create(category_id=2, product_id=3)
ProductCategory.create(category_id=3, product_id=5)
ProductCategory.create(category_id=4, product_id=6)
ProductCategory.create(category_id=5, product_id=3)
Testing OPML import
(c) 2015 KyubiSystems: www.kyubi.co.uk
"""
from opml import Opml
from models import Category, Feed, IntegrityError
o = Opml.OpmlReader('./opml/testing.xml')
o.parseOpml()
# Show retrieved data structures
print(o.version)
# Save categories to DB, skip invalid or duplicate feeds
for c in o.categories:
cat = Category.create(name=c)
try:
cat.save()
except IntegrityError:
pass
# Iterate over feeds found
for f in o.feeds:
print('------------')
print(f)
# Get corresponding Category id
cat_id = Category.get(Category.name == f['category']).id
if o.version == "1.0":
def create_category(category_id, description):
"""Takes arguments and create a new record in category table."""
Category.create(category_id=category_id, description=description)
Testing OPML import
(c) 2015-2020 KyubiSystems: www.kyubi.co.uk
"""
from opml import Opml
from models import Category, Feed, IntegrityError
o = Opml.OpmlReader('./opml/testing.xml')
o.parseOpml()
# Show retrieved data structures
print(o.version)
# Save categories to DB, skip invalid or duplicate feeds
for c in o.categories:
cat = Category.create(name=c)
try:
cat.save()
except IntegrityError:
pass
# Iterate over feeds found
for f in o.feeds:
print('------------')
print(f)
# Get corresponding Category id
cat_id = Category.get(Category.name == f['category']).id
if o.version == "1.0":
def opml_parse():
UPLOAD_FOLDER = os.path.realpath('.') + '/static/uploads'
file = request.files['file']
if file and allowed_file(file.filename):
opml_filename = str(uuid.uuid4()) + '.xml' # use UUID as unique uploaded filename root
opml_path = os.path.join(UPLOAD_FOLDER, opml_filename)
file.save(opml_path)
print('OPML uploaded OK!')
# run Opml parser on uploaded file
o = Opml.OpmlReader(opml_path)
o.parseOpml()
print('OPML parsed OK!')
# Save categories to DB, skip invalid or duplicate feeds
for c in o.categories:
try:
cat = Category.create(name=c)
cat.save()
except IntegrityError:
pass
print('Categories added to DB!')
# Iterate over feeds found
for f in o.feeds:
print('------------')
print(f)
# Get corresponding Category id
cat_id = Category.get(Category.name == f['category']).id
if o.version == "1.0":
# Add feed from OPML version 1.0
# TODO: Exception handling
feed = Feed.create(name=f['text'], category=cat_id, version=f['type'], url=f['url'])
elif o.version == "1.1" or o.version == "2.0":
# Add feed from OPML version 1.1
# TODO: Exception handling
feed = Feed.create(name=f['title'], category=cat_id, version=f['type'], comment=f['text'],
description=f['description'], url=f['xmlUrl'])
else:
continue
# Add feed to DB, skip invalid or duplicate feeds
try:
feed.save()
except IntegrityError:
pass
print('Feeds added to DB!')
# return send_from_directory(UPLOAD_FOLDER, opml_filename) # Test returning uploaded OPML file
return redirect(url_for('index'), code=302)
return "<h1>Oops, something went wrong here...</h1>file extension is " + os.path.splitext(file.filename)[1]
def insert_categories():
Category.create(name='Alimento', description='Alimento')
Category.create(name='Enlatado', description='Enlatado')
Category.create(name='Juguete', description='Juguete')
Category.create(name='Vacuna', description='Vacuna')
# coding:utf-8
import csv
# 读取岗位族群分类信息
from models import Category
with open('04.csv', 'r') as csvfile:
reader = csv.DictReader(csvfile)
rows = [row for row in reader]
# 抓取数据时是深度优先
for row in rows:
if not row['parent_code']:
c = Category.create(name=row['name'], code=row['code'])
else:
parent = Category.select().where(Category.code == row['parent_code']).get()
Category.create(name=row['name'], code=row['code'], parent=parent)
def opml_parse():
UPLOAD_FOLDER = os.path.realpath('.') + '/static/uploads'
file = request.files['file']
if file and allowed_file(file.filename):
opml_filename = str(uuid.uuid4()) + '.xml' # use UUID as unique uploaded filename root
opml_path = os.path.join(UPLOAD_FOLDER, opml_filename)
file.save(opml_path)
print('OPML uploaded OK!')
# run Opml parser on uploaded file
o = Opml.OpmlReader(opml_path)
o.parseOpml()
print('OPML parsed OK!')
# Save categories to DB, skip invalid or duplicate feeds
for c in o.categories:
try:
cat = Category.create(name=c)
cat.save()
except IntegrityError:
pass
print('Categories added to DB!')
# Iterate over feeds found
for f in o.feeds:
print('------------')
print(f)
# Get corresponding Category id
cat_id = Category.get(Category.name == f['category']).id
if o.version == "1.0":
# Add feed from OPML version 1.0
# TODO: Exception handling
feed = Feed.create(name=f['text'], category=cat_id, version=f['type'], url=f['url'])
elif o.version == "1.1" or o.version == "2.0":
# Add feed from OPML version 1.1
# TODO: Exception handling
feed = Feed.create(name=f['title'], category=cat_id, version=f['type'],
comment=f['text'], description=f['description'], url=f['xmlUrl'])
else:
continue
# Add feed to DB, skip invalid or duplicate feeds
try:
feed.save()
except IntegrityError:
pass
print('Feeds added to DB!')
# return send_from_directory(UPLOAD_FOLDER, opml_filename)
# Test returning uploaded OPML file
return redirect(url_for('index'), code=302)
return "<h1>Oops, something went wrong here...</h1>file extension is " + os.path.splitext(file.filename)[1]
