def process_staged_urls():
"""Query download all staged URLs, Update Catalog and Document"""
engine = db_connect()
create_tables(engine)
Session = sessionmaker(bind=engine)
session = Session()
# for event in session.query(EventStage).all():
# copy_event_from_stage(event)
for url_record in session.query(UrlStage).all():
# print(url_record.url)
place_record = session.query(Place). \
filter(Place.ocd_division_id == url_record.ocd_division_id).first()
event_record = session.query(Event). \
filter(Event.ocd_division_id == url_record.ocd_division_id,
Event.record_date == url_record.event_date,
Event.name == url_record.event).first()
print(f'place id: {place_record.id}\n event_id:{event_record.id}')
catalog_entry = session.query(Catalog). \
filter(Catalog.url_hash == url_record.url_hash).first()
# Document already exists in catalog
if catalog_entry:
catalog_id = catalog_entry.id
print(f'catalog_id---------{catalog_id}')
document = map_document(
url_record, place_record.id, event_record.id, catalog_id)
save_record(document)
print("existing in catalog adding reference to document")
else:
print("Does not exist")
# Download and save document
catalog = Catalog(
url=url_record.url,
url_hash=url_record.url_hash,
location='placeholder',
filename=f'{url_record.url_hash}.pdf'
)
doc = Media(url_record)
# download
result = doc.gather()
# Add to doc catalog
if result:
catalog.location = result
catalog_id = save_record(catalog)
# Add document reference
document = map_document(
url_record, place_record.id, event_record.id, catalog_id)
doc_id = save_record(document)
print(f'Added {url_record.url_hash} doc_id: {doc_id}')
def catalogs():
"""
GET: Shows all the catalogs available.
POST: Creates a new catalog.
:return: template (if GET) and a redirect to catalogs on POST.
"""
if request.method == "GET":
db_session = DBSession()
catalogs_all = db_session.query(Catalog, User).join(Catalog.user)
catalogs_count = catalogs_all.count()
db_session.close()
return render_template('catalogs/catalogs.html',
tuple=catalogs_all,
count=catalogs_count)
elif request.method == "POST":
if not is_signed_in():
flash(Markup(MUST_SIGN_IN))
return redirect(request.referrer)
db_session = DBSession()
name = request.form['name']
name = name.strip()
if not name:
flash(EMPTY_FORM)
return redirect(request.referrer)
email = session['idinfo']['email']
user = db_session.query(User).filter_by(email=email).first()
catalog = Catalog(name=name, user_id=user.id)
db_session.add(catalog)
db_session.commit()
db_session.close()
return redirect(url_for('catalogs'))
def save(self):
catalog = Catalog(name=self.cleaned_data['name'])
parent = self.cleaned_data.get('parent', None)
if parent:
catalog.parent = Catalog.get(parent)
return catalog
def newcatalog():
if 'username' not in login_session:
return redirect('/login')
if request.method == 'POST':
newCatalog = Catalog(
name=request.form['name'], user_id=login_session['user_id'])
session.add(newCatalog)
flash('!New Item %s added to catalog' % newCatalog.name)
session.commit()
return redirect(url_for('showcatalog'))
else:
return render_template('newcatalog.html')
def create_catalog():
"""Create a new catalog for the current user."""
user = session.query(User).filter_by(email=auth_session['email']).first()
if request.method == 'POST':
catalog = Catalog(name=request.form['name'],
description=request.form['description'],
user_id=user.id)
session.add(catalog)
session.commit()
return redirect(url_for('index_catalog'))
else:
catalogs = session.query(Catalog).filter_by(user_id=user.id).limit(5)
return render_template('catalog/create.html', catalogs=catalogs)
def process_item(self, item, spider):
"""Save deals in the database.
This method is called for every item pipeline component.
"""
session = self.Session()
song = Catalog(**item)
try:
session.add(song)
session.commit()
except:
session.rollback()
raise
finally:
session.close()
return item
def populate_db():
with app.app_context():
session = db.session()
db.metadata.create_all(db.engine)
products = Product.query.all()
for product in products:
part_number = product.part_number
manufacturer = product.manufacturer
sku = make_sku(part_number, manufacturer)
cost = product.primary_cost
res_price = make_res_price(cost)
com_price = make_com_price(cost)
shipping = get_shipping(product.weight)
leadtime = get_leadtime(manufacturer)
quantity = get_quantity()
fba = get_fba()
instock = get_instock()
if get_item(sku) is not None:
continue
try:
item = Catalog(
sku=sku,
part_number=product.part_number,
cost=cost,
res_price=res_price,
com_price=com_price,
shipping=shipping,
leadtime=leadtime,
quantity=quantity,
manufacturer=product.manufacturer,
#asin = asin,
upc=product.upc,
fba=fba,
instock=instock,
#listing_asin = asin,
product_id=product.id)
db.session.add(item)
db.session.commit()
except:
print('Unable to add to db')
print(sku, res_price, cost, shipping)
def new_catalog():
"""Create a new catalog"""
if request.method == 'POST':
name = clean_data(request.form['name'])
if Catalog.query.filter(Catalog.name == name).first():
flash('Catalog name already exist, please choose another one')
return redirect(url_for('new_catalog'))
user_id = session.get('user_id')
new_catalog = Catalog(name=name, user_id=user_id)
db_session.add(new_catalog)
try:
db_session.commit()
except:
flash('Something went wrong, please try again')
return redirect(url_for('new_catalog'))
flash("new catalog {} had successfully added".format(name))
print("new catalog {} had successfully added".format(name))
return redirect(url_for('console'))
else:
catalogs = Catalog.query.all()
return render_template('new_catalog.html', catalogs=catalogs)
typing = telegram.ChatAction.TYPING
# keyboards
main_kbd_user = [[texts.catalog_btn_user, texts.cart_btn_user],
[texts.orders_btn_user, texts.info_btn_user]]
# inline keyboard for catalog
catalog_ikbd = [[ikb(texts.prev_btn, callback_data="<"), ikb(texts.next_btn, callback_data=">")],
[ikb(texts.show_img_btn, callback_data="img")], [ikb(texts.to_cart_btn, callback_data="to_cart")]]
with open('data.json', 'r', encoding='utf8') as fp:
catalog = Catalog(json.load(fp, object_pairs_hook=OrderedDict))
engine = create_engine('postgresql://%s:%s@%s:%s/%s' % (db_username, db_password, db_host, db_port, db_name))
Session = sessionmaker(bind=engine)
session = Session()
Base.metadata.create_all(engine)
# matplotlib font settings
font = {'family': 'DejaVu Serif', 'weight': 'normal', 'size': 24} # avail_font_names = [f.name for f in matplotlib.font_manager.fontManager.ttflist]
rc('font', **font)
def kbd(k):
return telegram.ReplyKeyboardMarkup(k, one_time_keyboard=True, resize_keyboard=True)
# revert all of them back to the last commit by calling # session.rollback() session = DBSession() # Create dummy user if you are not login yet # User1 = User(name="Robo Barista", email="*****@*****.**", # picture='https://pbs.twimg.com/profile_images/2671170543/ # 18debd694829ed78203a5a36dd364160_400x400.png') # session.add(User1) # session.commit() # Query first user if you are already login User1 = session.query(User).order_by(asc(User.created_at)).first() # Catalog for Soccer catalog1 = Catalog(user_id=1, name="Soccer") session.add(catalog1) session.commit() # Catalog for Basketball catalog1 = Catalog(user_id=1, name="Basketball") session.add(catalog1) session.commit() # Catalog for Thyme for Baseball catalog1 = Catalog(user_id=1, name="Baseball") session.add(catalog1) session.commit()
from sqldatabase.parser import Parser
class ProductsDB:
def __init__(self, base, db_url):
engine = create_engine(db_url)
base.metadata.create_all(engine)
session_db = sessionmaker(bind=engine)
self.__session = session_db()
@property
def session(self):
return self.__session
if __name__ == '__main__':
bd_url = 'sqlite:///catalogs.sqlite'
db = ProductsDB(Base, bd_url)
parser = Parser()
list_cat = []
for i in parser.catalogs[0]:
cat = Catalog(i['parent_group_name'], i['parent_group_code'])
list_cat.append(cat)
db.session.add(cat)
for i in parser.products:
list = []
list.append(list_cat[i['Catalog_code']])
db.session.add(Products(i['name'], i['plu'], i['Price'], list))
db.session.commit()
session = DBSession()
# Create dummy user
User1 = User(
Uname="John Doe",
email="*****@*****.**",
picture=
'https://lc-imageresizer-live-s.legocdn.com/resize?width=744&imageUrl=https%3a%2f%2fwww.lego.com%2fr%2fwww%2fr%2fportals%2f-%2fmedia%2fthemes%2fworlds%2ffrontpage%2fcta%2fcta-minifig-05.png%3fl.r%3d-1845865519'
)
session.add(User1)
session.commit()
#Catalog entry for Lego City
catalog1 = Catalog(
user_id=1,
Cname="City",
catalog_image=
"https://sh-s7-live-s.legocdn.com/is/image/LEGOMKTG/city%2D%2D201606%2D%2Dgl%2D%2Dlogo?$CatListLogo$"
)
session.add(catalog1)
session.commit()
#Catalog entry for Lego Star Wars
catalog2 = Catalog(
user_id=1,
Cname="Star Wars",
catalog_image=
"https://sh-s7-live-s.legocdn.com/is/image/LEGOMKTG/star-wars-black--201606--gl--logo?$CatListLogo$"
)
session.add(catalog2)
