def compare(days_count : int):
TODAY_PRICE = 0
PAST_PRICE = 1
days = [day.date for day in Price.select().distinct(Price.date)] # Get all days
if days_count >= len(days):
print(f"Parser has only {len(days)} day(s) info")
return
elif days_count <= 0:
print(f"Error with days_count <= 0")
return
else:
for product in Product.select().execute():
today_prices = Price.select().where((Price.date == date.today()) & (Price.product == product.id)).order_by(Price.shop).execute()
past_prices = Price.select().where((Price.date == days[-days_count-1]) & (Price.product == product.id)).order_by(Price.shop).execute()
print('-------')
for prices in zip(today_prices, past_prices):
print("Price for {:s} has {:s} by {:.2f} in shop '{:s}' for {:s} user".format(
prices[TODAY_PRICE].product.group,
'increased' if prices[TODAY_PRICE].price > prices[PAST_PRICE].price
else 'decreased' if prices[TODAY_PRICE].price < prices[PAST_PRICE].price else 'not changed',
prices[TODAY_PRICE].price - prices[PAST_PRICE].price
if prices[TODAY_PRICE].price >= prices[PAST_PRICE].price
else prices[PAST_PRICE].price - prices[TODAY_PRICE].price,
prices[TODAY_PRICE].shop,
'authorized' if prices[TODAY_PRICE].authorized
else 'not authorized'))
def update_predictions():
predictions = Prediction.select().where((Prediction.actual == 0))
for p in predictions:
# Monday = 0, Sunday = 6
# If Friday or Saturday or Sunday then search for Monday
if 4 <= p.time.weekday() <= 6:
for i in [0, 1, 2, 3, 4, 5, 6]:
price = Price.select(Price.current).where(
(Price.time > p.time) & (fn.WEEKDAY(Price.time) == i)
& (Price.company == p.company)).order_by(
Price.id.desc()).limit(1)
if len(price) > 0 or (p.time +
timedelta(days=i)) > datetime.now():
break
else:
for i in [1, 2, 3, 4, 5, 6]:
weekday = (p.time.weekday() + i) % 7
price = Price.select(
Price.current,
Price.time).where((Price.time > p.time)
& (fn.WEEKDAY(Price.time) == weekday)
& (Price.company == p.company)).order_by(
Price.id.desc()).limit(1)
if len(price) > 0 or p.time + timedelta(
days=i) > datetime.now():
break
price = price[0] if len(price) > 0 else None
if price is not None:
p.actual = price.current
p.actual_price_time = price.time
p.save()
def createDB(): print "Creating the initial db" Company.create_table() NewsSource.create_table() News.create_table() Price.create_table() OpinionAPI.create_table() OpinionAPIResponse.create_table()
def process_item(self, item, spider):
# 写入小区房价
if isinstance(item, FangtianxiaRentItem):
db_house = self.session.query(House).filter(
House.house_name == item['name']).filter(
House.city_name == item['city_name']).first()
if db_house:
if db_house.property_corp == u'暂无数据':
db_house.property_corp = item['property_corp']
if db_house.developers == u'暂无数据':
db_house.developers = item['developers']
if db_house.building_date == u'暂无数据':
db_house.building_date = item['building_date']
if db_house.building_date == u'暂无数据':
db_house.building_date = item['building_date']
# 需要添加判断, 以防同一个价格数据插入多次
p = Price(price=item['price'],
origin=item['origin'],
months=item['months'],
crawl_time=item['crawl_date'],
uid=db_house.id)
db_house.price.append(p)
else:
house = House(house_name=item['name'],
city_name=item['city_name'],
url=item['url'],
latitude=item['latitude'],
longitude=item['longitude'],
address=item['location'],
building_date=item['building_date'],
building_type=item['building_type'],
property_corp=item['property_corp'],
developers=item['developers'],
district=item['district'])
price = Price(price=item['price'],
origin=item['origin'],
months=item['months'],
crawl_time=item['crawl_date'])
house.price.append(price)
self.session.add(house)
try:
self.session.commit()
except Exception, e:
print e
self.session.rollback()
return item
def prices_management():
if request.method == "POST":
form = CreatePriceForm(request.form)
if form.validate():
price = Price(name=form.name.data, price=form.price.data)
price.put()
else:
form = CreatePriceForm()
existing_prices = Price.query().fetch(100)
return render_template("admin/prices.html",
existing_prices=existing_prices,
form=form)
def prices_management():
if request.method == "POST":
form = CreatePriceForm(request.form)
if form.validate():
price = Price(name=form.name.data, price=form.price.data)
price.put()
else:
form = CreatePriceForm()
existing_prices = Price.query().fetch(100)
return render_template("admin/prices.html",
existing_prices=existing_prices,
form=form)
def list_stock(jwt):
req = request.get_json()
try:
exchange = req['exchange']
price = req['price']
name = req['name']
code = req['code']
stock = Stock(code=code, name=name, market_id=exchange)
stock.insert()
price = Price(code=code, price=price, timestamp=datetime.now())
price.insert()
db.session.close()
except:
abort(400)
return jsonify({"success": True})
def price_update(self, request):
"""Update a price"""
entity = Price.price_update(request)
if entity:
return entity
else:
raise endpoints.NotFoundException()
def parse_price():
request_green = requests.Session()
request_evroopt = requests.Session()
authorized_request_green = green_parser.authorize(requests.Session())
authorized_request_evroopt = e_parser.authorize(requests.Session())
if date.today() in [price.date for price in Price.select().execute()]:
return
else:
products = Product.select().order_by(Product.id).execute()
bar = Bar('Parse prices', max=len(products))
for product in products:
with conn.atomic() as trn:
try:
main_parser(green_parser, request_green, product.id, product.link_g)
main_parser(green_parser, authorized_request_green, product.id, product.link_g)
main_parser(e_parser, request_evroopt, product.id, product.link_e)
main_parser(e_parser, authorized_request_evroopt, product.id, product.link_e)
except AttributeError:
trn.rollback()
finally:
bar.next()
bar.finish()
def stream_prices_to_db(self):
while True:
try:
prices = self.get_current_prices()
current_ts = int(round(time.time() * 1000))
for coin, price in prices.items():
inserted_at = current_ts
coin_type = coin # BTC or ADA
price_usd = price
price = Price(inserted_at=inserted_at,
coin_type=coin_type,
price_usd=price_usd)
self.session.add(price)
self.session.commit()
self.session.close()
time.sleep(60)
except (IncompleteRead, ProtocolError, AttributeError) as e:
# Oh well, reconnect and keep trucking
print(f"An exception occurred during crypto price call: {e}\n")
continue
except Exception as e:
print(
f"An unexpected exception occurred during crypto price call: {e}\n"
)
continue
except KeyboardInterrupt as e:
print(f"Stopping the stream... closing the session...")
self.session.close()
print(f"Good bye!")
break
def bookxcess(book, debug=False):
obj = Book.get_by_isbn(book['isbn13'], defaults=book)
prices = Price.find_for_book(obj, 'bookxcess')
if prices:
return prices
else:
return []
def my_scrapper():
#Retrieve the newest price for each stock
query = db.session.query(Price).distinct(Price.code).order_by(
Price.code, Price.id.desc()).all()
values = {x.code: x.price for x in query}
# Generate new price every 20 seconds
# The new price is generated using random walk model
# Insert a new record using current time and new price
for record in values:
current_time = datetime.now()
value = values[record]
new_value = value + random.randn() * 3
new_price = Price(code=record, price=new_value, timestamp=current_time)
new_price.insert()
db.session.close()
db.session.close()
def run():
loop = asyncio.get_event_loop()
# # 获取数据库连接池
task = aiomysql.create_pool(**DATABASES)
pool = loop.run_until_complete(asyncio.gather(task))
Price.pool = pool[0]
# 获取任务
tasks = loop.run_until_complete(asyncio.gather(Price.get_tasks()))
print(tasks)
# 生成任务
my_func_list = (Price(name).save() for name in tasks)
loop.run_until_complete(asyncio.gather(*my_func_list))
def add_precio():
body = request.get_json()
precio1 = Price(market_name=body["market_name"],
price=body["price"],
id_ingrediente=body["id_ingrediente"])
db.session.add(precio1)
db.session.commit()
return "ok", 200
def price_delete(self, request):
"""Delete a price"""
entity = Price.get_by_id(request.id)
if entity:
entity.key.delete()
return BaseResponse(messages="")
else:
raise endpoints.NotFoundException()
def get_analytics(date_from, date_to, ticker_code):
trades = Price.select(Price.stock, Price.date,
(Price.open - Price.close).alias("delta_open_close"),
(Price.high -
Price.low).alias("delta_high_low")).where(
(Price.stock == ticker_code)
& (Price.date.between(date_from, date_to)))
return trades
def get_price(price_name):
"Looks up the current price for the given cost code."
price = Price.query(Price.name == price_name).get()
if not price:
raise ValueError("Price %s not found." % price_name)
return price.price
def get_price(price_name):
"Looks up the current price for the given cost code."
price = Price.query(Price.name == price_name).get()
if not price:
raise ValueError("Price %s not found." % price_name)
return price.price
def ticker():
last_price = Price.last()
if last_price is not None:
return jsonify(btcnok={
'buy': str(round(last_price.btcnok(rate=settings['BUY_RATE']), 2)),
'sell': str(round(last_price.btcnok(rate=settings['SELL_RATE']), 2)),
})
else:
abort(503)
def get_products(self):
"""
function parses the products xml file
and returns list of products
"""
lst = []
for product in self.products.findall('product'):
id = product.find('id').text
name = product.find('name').text
dispensary_id = product.find('dispensary_id').text
dispensary_name = product.find('dispensary_name').text
canabis_brand = product.find('canabis_brand').text
canabis_strain = product.find('canabis_strain').text
category = product.find('category').text
subcategory = product.find('subcategory').text
thc_level = product.find('thc_level').text
cbd_level = product.find('cbd_level').text
cbn_level = product.find('cbn_level').text
thc_level_type = product.find('thc_level_type').text
cbd_level_type = product.find('cbd_level_type').text
cbn_level_type = product.find('cbn_level_type').text
description = product.find('description').text
created_at = product.find('created_at').text
updated_at = product.find('updated_at').text
prices = []
urls = []
images = []
for child in product:
if child.tag == 'prices':
for cost in child.findall('cost'):
prices.append(Price(cost.attrib['unit'], cost.text))
if child.tag == 'urls':
admin = child.find('admin').text
public = child.find('public').text
urls.append(UrlInfo(admin, public))
if child.tag == 'images':
for image in child.findall('image'):
images.append(Image(
image.attrib['main'],
image.text,
))
lst.append(
Product(id, name, dispensary_id, dispensary_name,
canabis_brand, canabis_strain, category, subcategory,
thc_level, cbd_level, cbn_level, thc_level_type,
cbd_level_type, cbn_level_type, prices, urls, images,
description, created_at, updated_at))
return lst
def update_product(self, request):
'''
Update a product
'''
product = Product.get_by_urlsafe_key(request.productKey)
if not product:
message = 'No product with the key "%s" exists.' % request.productKey
raise endpoints.NotFoundException(message)
user = User.get_by_urlsafe_key(request.userKey)
if not user:
message = 'No user with the key "%s" exists.' % request.userKey
raise endpoints.NotFoundException(message)
category = None
if request.category:
category = ProductCategory.get_by_urlsafe_key(request.category)
price = None
if request.price:
price = request.price
currency = CurrencyUnit.get_by_urlsafe_key(price.currencyKey)
location = None
if price.location:
location = PhysicalLocation.load(lat=price.location.lat,
lon=price.location.lon)
price = Price.create(user=user,
value=price.value,
currency=currency,
location=location)
manufacturer = None
if request.manufacturer:
manufacturer = Manufacturer.load(name=request.manufacturer,
user=user)
specs = []
if request.specs:
for spec in request.specs:
specification = Specification.get_by_urlsafe_key(spec.key)
unit = Unit.get_by_urlsafe_key(spec.unit)
specs.append([specification, spec.value, unit])
product.update(user=user,
name=request.name,
category=category,
manufacturer=manufacturer,
price=price,
specs=specs)
return message_types.VoidMessage()
def add_quote():
data = request.get_json()['quote']
Quote.query.filter_by(id=data['id']).update(
dict(phone=None, skype=None, email=None, validity=data['validity']))
quote = Quote.query.filter_by(id=data['id']).first()
quote_data = {
'id': quote.id,
'name': quote.supplier.name,
'expiration': quote.validity,
'info': quote.info,
'phone': quote.phone,
'email': quote.email,
'skype': quote.skype
}
db.session.commit()
order_list = []
for o in data['orders']:
order_data = {
'grade': o['grade'],
'quantity': o['quantity'],
'maxSulphur': o['maxSulphur'],
'unit': o['unit'],
'specs': o['spec'],
'comments': o['comments'],
'price': o['price'],
'terms': o['terms'],
'physical': o['physical'],
'delivery': o['delivery']
}
order = Order.query.filter_by(deal_id=quote.deal_id,
grade=order_data['grade']).first()
price = Price(order_data['price'], order_data['terms'],
order_data['physical'], order_data['delivery'], quote,
order)
db.session.add(price)
db.session.commit()
order_list.append(order_data)
quote_data['orders'] = order_list
d = Deal.query.filter_by(id=quote.deal_id).first()
deal = {
'id': d.id,
'port': d.port,
'vessel': d.vessel,
'imo': d.imo,
'loa': d.loa,
'grossTonnage': d.grossTonnage,
'buyer': d.buyer,
'orderedBy': d.orderedBy,
'eta': d.eta,
'etd': d.etd,
}
response = jsonify(deal)
response.headers.add('Access-Control-Allow-Origin', '*')
return response
def parse_prices():
url = "https://ru.investing.com/indices/mcx-components"
headers = {
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/73.0.3683.86 YaBrowser/19.4.0.2397 Yowser/2.5 Safari/537.36 "
}
r = requests.get(url=url, headers=headers)
sel = Selector(text=r.text)
companies_info = sel.css('table#cr1 > tbody').xpath(
'./tr[contains(@id,"pair")]')
for company in companies_info:
name_el = company.xpath('./td[2]/a')
name = name_el.xpath('./text()').extract_first()
current_price = company.xpath(
'./td[contains(@class,"last")]/text()').extract_first()
current_price = Formatter.format_price(current_price)
high_price = company.xpath(
'./td[contains(@class,"high")]/text()').extract_first()
high_price = Formatter.format_price(high_price)
low_price = company.xpath(
'./td[contains(@class,"low")]/text()').extract_first()
low_price = Formatter.format_price(low_price)
volume = company.xpath(
'./td[contains(@class,"turnover")]/text()').extract_first()
volume = Formatter.format_volume(volume)
update_time = int(
company.xpath(
'./td[contains(@class,"time")]/@data-value').extract_first())
p = Price()
p.current = current_price
p.high = high_price
p.low = low_price
p.volume = volume
p.time = datetime.fromtimestamp(update_time)
company = Company.select().where(Company.name == name).limit(1)
if len(company) > 0:
p.company = company[0].id
p.save()
else:
continue
def ticker():
last_price = Price.last()
if last_price is not None:
return jsonify(
btcnok={
'buy':
str(round(last_price.btcnok(rate=settings['BUY_RATE']), 2)),
'sell':
str(round(last_price.btcnok(rate=settings['SELL_RATE']), 2)),
})
else:
abort(503)
def new_album():
"""
Add a new album
"""
form = PriceForm(request.form)
if request.method == 'POST' and form.validate():
# save the album
price = Price()
save_changes(price, form, new=True)
flash('Price created successfully!')
return redirect('/')
return render_template('new_album.html', form=form)
def add_cafe_price(cafeid=None):
form = PriceForm()
if cafeid:
cafe = Cafe.query.filter_by(id=cafeid).first_or_404()
form.cafeid.choices = [(cafe.id, cafe.name)]
form.cafeid.data = cafe.id
else:
cafes = Cafe.query.all()
if not cafes:
flash("There are no existing cafes. Would you like to make one instead?", "warning")
return redirect(url_for("home"))
form.cafeid.choices = [(cafe.id, cafe.name) for cafe in cafes]
if request.method == "GET":
return render_template("priceform.html", cafe=cafe, form=form, formtype="Add", current_user=current_user)
if request.method == "POST" and form.validate_on_submit():
cafeid = form.data["cafeid"]
if Price.query.filter_by(cafeid=cafeid).filter_by(size=form.data["size"]).first():
flash("There is already a price for this size", "danger")
return redirect(url_for("add_cafe_price", cafeid=cafeid))
price = Price(cafe.id)
if cafeid:
price.cafeid = cafeid
else:
price.cafeid = form.data["cafeid"]
price.size = form.data["size"]
price.amount = form.data["amount"]
db.session.add(price)
db.session.commit()
write_to_events("created", "price", price.id)
flash("Price added to cafe '%s'" % cafe.name, "success")
return redirect(url_for("view_cafe", cafeid=cafeid))
else:
for field, errors in form.errors.items():
flash("Error in %s: %s" % (field, "; ".join(errors)), "danger")
return render_template("priceform.html", cafe=cafe, form=form, formtype="Add", current_user=current_user)
def handle_precio():
preciosTemp = Price()
# get all the people
precios_query = preciosTemp.query.all()
# get only the ones named "Joe"
# receta_query = recetasTemp.query.filter_by(name='pollo')
# map the results and your list of people its now inside all_people variable
all_precios = list(map(lambda x: x.serializePrecios(), precios_query))
return jsonify(all_precios), 200
def GetPrices(prediction_id):
prices = Price.query(Price.prediction_id == ndb.Key(
urlsafe=prediction_id)).order(-Price.date).fetch(30)
print(str(prices))
prices = [{
'price': p.value,
'date': {
'year': p.date.year,
'month': p.date.month,
'day': p.date.day
}
} for p in prices]
return prices
def update_price(self, product):
self.logger.info('[UPDATE_PRICES] Updating price for {}'.format(product.name))
product_data = Walcart.product(product.id)
current_price = product_data.get('salePrice')
stock = product_data.get('stock')
old_price = None if not product.prices else product.prices[-1].price
self.logger.info('\t[UPDATE_PRICES] Old Price: {}\tCurrent Price: {}'.format(old_price, current_price))
if old_price != current_price:
self.logger.info('\t\t[UPDATE_PRICES] Price has been updated successfully.')
product.prices.append(Price(price=current_price, stock=stock))
res = self.pub.call({'method': 'save', 'resource': serialize(product)})
if not res.get('success'):
self.logger.critical('[UPDATE_PRICES] {}'.format(res['message']))
else:
self.logger.info('\t\t[UPDATE_PRICES] Price has not changed. No update necessary.')
return (product, False)
def process_item(self, item, spider):
"""Save deals in the database.
This method is called for every item pipeline component.
"""
session = self.Session()
deal = Price(**item)
try:
session.add(deal)
session.commit()
except:
session.rollback()
raise
finally:
session.close()
return item
def home():
now = settings['TIMEZONE'].fromutc(datetime.utcnow())
last_price = Price.last()
if last_price is not None:
current_price = {
'buy': last_price.btcnok(rate=settings['BUY_RATE']),
'sell': last_price.btcnok(rate=settings['SELL_RATE']),
}
else:
current_price = None
context = {
'current_price': current_price,
'price_history': json.dumps(get_price_history(app)),
'now': now,
}
return render_template('home.html', **context)
def process_item(self, item, spider):
""" Save item in the database. """
session = self.Session()
deal = Price(**item)
try:
session.add(deal)
session.commit()
except Exception as ex:
logger.error(ex)
logger.error(traceback.format_exc())
session.rollback()
raise
finally:
session.close()
return item
def home():
now = settings['TIMEZONE'].fromutc(datetime.utcnow())
last_price = Price.last()
if last_price is not None:
current_price = {
'buy': last_price.btcnok(rate=settings['BUY_RATE']),
'sell': last_price.btcnok(rate=settings['SELL_RATE']),
}
else:
current_price = None
context = {
'current_price': current_price,
'price_history': json.dumps(get_price_history(app)),
'now': now,
}
return render_template('home.html', **context)
def main():
Session = sessionmaker(bind=get_engine())
session = Session()
response = requests.get('https://api.coinmarketcap.com/v1/ticker/')
for data in response.json():
price_usd = data['price_usd']
updated = data['last_updated']
crypt_currency_id = data['id']
price = Price(price_usd=price_usd,
updated=datetime.datetime.fromtimestamp(int(updated)),
crypt_currency_id=crypt_currency_id)
session.add(price)
session.flush()
session.commit()
def make_product(product_data):
product = Product(
id = product_data.get('itemId'),
upc = product_data.get('upc'),
name = product_data.get('name'),
thumbnail_img = product_data.get('thumbnailImage'),
med_img = product_data.get('mediumImage'),
lg_img = product_data.get('largeImage'),
short_descr = product_data.get('shortDescription'),
long_descr = product_data.get('longDescription'),
msrp = product_data.get('msrp'),
add_to_cart_url = product_data.get('addToCartUrl'),
url = product_data.get('productUrl'),
)
price = product_data.get('salePrice')
if price:
product.prices.append(Price(price=price, stock=product_data.get('stock')))
return product
def new_product():
if request.method == 'GET':
return render_template('newproduct.html')
else:
try:
data = request.get_json()
prices = []
for price in data['prices']:
options = []
for option_id in price['options']:
current_option = data['options'][option_id]
options.append(
Option(option_id=option_id,
option_type=current_option['type'],
option_text=current_option['text'],
option_value=current_option['value']).save())
current = Price(
valid_from=datetime.utcnow(),
valid_to=datetime.utcnow(),
currency=price['currency'],
original_price=price['originalPrice'],
discounted_price=price['discountedPrice'],
discount_rate=price['discountRate'],
# stock=price['stock'],
# is_active=price['isActive'],
options=options)
prices.append(current)
product = Product(title=data['title'],
description=data['description'],
image_url=data['imageURL'],
prices=prices).save()
return jsonify({'created': str(product.id), 'success': True})
except Exception as e:
traceback.print_exc()
return make_response('An error occured while adding product.', 400)
return jsonify({'hello': 'world'})
def update_event_details(event, event_form):
details = event_form.collectEventDetails()
details["authorizations"] = [Authorization.query.filter_by(name=auth).first()
for auth in details["authorizations"]]
details["platforms"] = [Platform.query.filter_by(name=plat).first()
for plat in details["platforms"]]
details["resources"] = [Resource.query.filter_by(name=res).first()
for res in details["resources"]]
details["tags"] = [Tag.query.filter_by(name=tag).first()
for tag in details["tags"]]
details["prices"] = [Price(name=price['name'],
description=price['description'],
value=price['price'],
availability=price['availability'][0])
for price in details['prices']]
if event_form.image_file.data:
details["image_file"] = event_form.image_file.data.filename
for attribute in details:
try:
getattr(event, attribute)
except AttributeError:
pass
setattr(event, attribute, details[attribute])
def prices(self, request):
"""Create new price"""
entity = Price.put_from_message(request)
return entity
def prices_show(self, request):
"""Show all prices"""
entity = Price.get_prices(request)
return entity
numbers = re.findall("[0-9]{1,4}/?[A-Za-z]?", cleaned_street)
if len(numbers) > 0:
street_number = numbers[0]
else:
street_number = ""
# cleaned address
cleaned_street = cleaned_street.split(',')[0].split('-')[0]
# generate address
address = Address(province=row['Province'],
city=row['City'],
street=cleaned_street,
number=str(street_number),
cap=int(row['CAP']),
location=location)
# generate price
price = Price(row['Price'])
# generate contact
contact = Contact(phone=row['Phone'],
email=row['Email'],
website=row['Website'])
#generate facility
facility = Facility(type=row['Type'],
price=price,
contact=contact,
ranking=1,
address=address,
calendar=None)
data.append(facility.toJSON())
#print(data)
print("2. Exporting assured json data")
ds_out_path = '../../dataset/Formal Modeling/data/accomodations_assured.json'
