def process_item(self, item, spider):
# 写入小区房价
if isinstance(item, FangtianxiaRentItem):
db_house = self.session.query(House).filter(
House.house_name == item['name']).filter(
House.city_name == item['city_name']).first()
if db_house:
if db_house.property_corp == u'暂无数据':
db_house.property_corp = item['property_corp']
if db_house.developers == u'暂无数据':
db_house.developers = item['developers']
if db_house.building_date == u'暂无数据':
db_house.building_date = item['building_date']
if db_house.building_date == u'暂无数据':
db_house.building_date = item['building_date']
# 需要添加判断, 以防同一个价格数据插入多次
p = Price(price=item['price'],
origin=item['origin'],
months=item['months'],
crawl_time=item['crawl_date'],
uid=db_house.id)
db_house.price.append(p)
else:
house = House(house_name=item['name'],
city_name=item['city_name'],
url=item['url'],
latitude=item['latitude'],
longitude=item['longitude'],
address=item['location'],
building_date=item['building_date'],
building_type=item['building_type'],
property_corp=item['property_corp'],
developers=item['developers'],
district=item['district'])
price = Price(price=item['price'],
origin=item['origin'],
months=item['months'],
crawl_time=item['crawl_date'])
house.price.append(price)
self.session.add(house)
try:
self.session.commit()
except Exception, e:
print e
self.session.rollback()
return item
def add_cafe_price(cafeid=None):
form = PriceForm()
if cafeid:
cafe = Cafe.query.filter_by(id=cafeid).first_or_404()
form.cafeid.choices = [(cafe.id, cafe.name)]
form.cafeid.data = cafe.id
else:
cafes = Cafe.query.all()
if not cafes:
flash("There are no existing cafes. Would you like to make one instead?", "warning")
return redirect(url_for("home"))
form.cafeid.choices = [(cafe.id, cafe.name) for cafe in cafes]
if request.method == "GET":
return render_template("priceform.html", cafe=cafe, form=form, formtype="Add", current_user=current_user)
if request.method == "POST" and form.validate_on_submit():
cafeid = form.data["cafeid"]
if Price.query.filter_by(cafeid=cafeid).filter_by(size=form.data["size"]).first():
flash("There is already a price for this size", "danger")
return redirect(url_for("add_cafe_price", cafeid=cafeid))
price = Price(cafe.id)
if cafeid:
price.cafeid = cafeid
else:
price.cafeid = form.data["cafeid"]
price.size = form.data["size"]
price.amount = form.data["amount"]
db.session.add(price)
db.session.commit()
write_to_events("created", "price", price.id)
flash("Price added to cafe '%s'" % cafe.name, "success")
return redirect(url_for("view_cafe", cafeid=cafeid))
else:
for field, errors in form.errors.items():
flash("Error in %s: %s" % (field, "; ".join(errors)), "danger")
return render_template("priceform.html", cafe=cafe, form=form, formtype="Add", current_user=current_user)
def stream_prices_to_db(self):
while True:
try:
prices = self.get_current_prices()
current_ts = int(round(time.time() * 1000))
for coin, price in prices.items():
inserted_at = current_ts
coin_type = coin # BTC or ADA
price_usd = price
price = Price(inserted_at=inserted_at,
coin_type=coin_type,
price_usd=price_usd)
self.session.add(price)
self.session.commit()
self.session.close()
time.sleep(60)
except (IncompleteRead, ProtocolError, AttributeError) as e:
# Oh well, reconnect and keep trucking
print(f"An exception occurred during crypto price call: {e}\n")
continue
except Exception as e:
print(
f"An unexpected exception occurred during crypto price call: {e}\n"
)
continue
except KeyboardInterrupt as e:
print(f"Stopping the stream... closing the session...")
self.session.close()
print(f"Good bye!")
break
def add_precio():
body = request.get_json()
precio1 = Price(market_name=body["market_name"],
price=body["price"],
id_ingrediente=body["id_ingrediente"])
db.session.add(precio1)
db.session.commit()
return "ok", 200
def get_products(self):
"""
function parses the products xml file
and returns list of products
"""
lst = []
for product in self.products.findall('product'):
id = product.find('id').text
name = product.find('name').text
dispensary_id = product.find('dispensary_id').text
dispensary_name = product.find('dispensary_name').text
canabis_brand = product.find('canabis_brand').text
canabis_strain = product.find('canabis_strain').text
category = product.find('category').text
subcategory = product.find('subcategory').text
thc_level = product.find('thc_level').text
cbd_level = product.find('cbd_level').text
cbn_level = product.find('cbn_level').text
thc_level_type = product.find('thc_level_type').text
cbd_level_type = product.find('cbd_level_type').text
cbn_level_type = product.find('cbn_level_type').text
description = product.find('description').text
created_at = product.find('created_at').text
updated_at = product.find('updated_at').text
prices = []
urls = []
images = []
for child in product:
if child.tag == 'prices':
for cost in child.findall('cost'):
prices.append(Price(cost.attrib['unit'], cost.text))
if child.tag == 'urls':
admin = child.find('admin').text
public = child.find('public').text
urls.append(UrlInfo(admin, public))
if child.tag == 'images':
for image in child.findall('image'):
images.append(Image(
image.attrib['main'],
image.text,
))
lst.append(
Product(id, name, dispensary_id, dispensary_name,
canabis_brand, canabis_strain, category, subcategory,
thc_level, cbd_level, cbn_level, thc_level_type,
cbd_level_type, cbn_level_type, prices, urls, images,
description, created_at, updated_at))
return lst
def add_quote():
data = request.get_json()['quote']
Quote.query.filter_by(id=data['id']).update(
dict(phone=None, skype=None, email=None, validity=data['validity']))
quote = Quote.query.filter_by(id=data['id']).first()
quote_data = {
'id': quote.id,
'name': quote.supplier.name,
'expiration': quote.validity,
'info': quote.info,
'phone': quote.phone,
'email': quote.email,
'skype': quote.skype
}
db.session.commit()
order_list = []
for o in data['orders']:
order_data = {
'grade': o['grade'],
'quantity': o['quantity'],
'maxSulphur': o['maxSulphur'],
'unit': o['unit'],
'specs': o['spec'],
'comments': o['comments'],
'price': o['price'],
'terms': o['terms'],
'physical': o['physical'],
'delivery': o['delivery']
}
order = Order.query.filter_by(deal_id=quote.deal_id,
grade=order_data['grade']).first()
price = Price(order_data['price'], order_data['terms'],
order_data['physical'], order_data['delivery'], quote,
order)
db.session.add(price)
db.session.commit()
order_list.append(order_data)
quote_data['orders'] = order_list
d = Deal.query.filter_by(id=quote.deal_id).first()
deal = {
'id': d.id,
'port': d.port,
'vessel': d.vessel,
'imo': d.imo,
'loa': d.loa,
'grossTonnage': d.grossTonnage,
'buyer': d.buyer,
'orderedBy': d.orderedBy,
'eta': d.eta,
'etd': d.etd,
}
response = jsonify(deal)
response.headers.add('Access-Control-Allow-Origin', '*')
return response
def new_album():
"""
Add a new album
"""
form = PriceForm(request.form)
if request.method == 'POST' and form.validate():
# save the album
price = Price()
save_changes(price, form, new=True)
flash('Price created successfully!')
return redirect('/')
return render_template('new_album.html', form=form)
def handle_precio():
preciosTemp = Price()
# get all the people
precios_query = preciosTemp.query.all()
# get only the ones named "Joe"
# receta_query = recetasTemp.query.filter_by(name='pollo')
# map the results and your list of people its now inside all_people variable
all_precios = list(map(lambda x: x.serializePrecios(), precios_query))
return jsonify(all_precios), 200
def prices_management():
if request.method == "POST":
form = CreatePriceForm(request.form)
if form.validate():
price = Price(name=form.name.data, price=form.price.data)
price.put()
else:
form = CreatePriceForm()
existing_prices = Price.query().fetch(100)
return render_template("admin/prices.html",
existing_prices=existing_prices,
form=form)
def list_stock(jwt):
req = request.get_json()
try:
exchange = req['exchange']
price = req['price']
name = req['name']
code = req['code']
stock = Stock(code=code, name=name, market_id=exchange)
stock.insert()
price = Price(code=code, price=price, timestamp=datetime.now())
price.insert()
db.session.close()
except:
abort(400)
return jsonify({"success": True})
def run():
loop = asyncio.get_event_loop()
# # 获取数据库连接池
task = aiomysql.create_pool(**DATABASES)
pool = loop.run_until_complete(asyncio.gather(task))
Price.pool = pool[0]
# 获取任务
tasks = loop.run_until_complete(asyncio.gather(Price.get_tasks()))
print(tasks)
# 生成任务
my_func_list = (Price(name).save() for name in tasks)
loop.run_until_complete(asyncio.gather(*my_func_list))
def my_scrapper():
#Retrieve the newest price for each stock
query = db.session.query(Price).distinct(Price.code).order_by(
Price.code, Price.id.desc()).all()
values = {x.code: x.price for x in query}
# Generate new price every 20 seconds
# The new price is generated using random walk model
# Insert a new record using current time and new price
for record in values:
current_time = datetime.now()
value = values[record]
new_value = value + random.randn() * 3
new_price = Price(code=record, price=new_value, timestamp=current_time)
new_price.insert()
db.session.close()
db.session.close()
def update_price(self, product):
self.logger.info('[UPDATE_PRICES] Updating price for {}'.format(product.name))
product_data = Walcart.product(product.id)
current_price = product_data.get('salePrice')
stock = product_data.get('stock')
old_price = None if not product.prices else product.prices[-1].price
self.logger.info('\t[UPDATE_PRICES] Old Price: {}\tCurrent Price: {}'.format(old_price, current_price))
if old_price != current_price:
self.logger.info('\t\t[UPDATE_PRICES] Price has been updated successfully.')
product.prices.append(Price(price=current_price, stock=stock))
res = self.pub.call({'method': 'save', 'resource': serialize(product)})
if not res.get('success'):
self.logger.critical('[UPDATE_PRICES] {}'.format(res['message']))
else:
self.logger.info('\t\t[UPDATE_PRICES] Price has not changed. No update necessary.')
return (product, False)
def process_item(self, item, spider):
"""Save deals in the database.
This method is called for every item pipeline component.
"""
session = self.Session()
deal = Price(**item)
try:
session.add(deal)
session.commit()
except:
session.rollback()
raise
finally:
session.close()
return item
def process_item(self, item, spider):
""" Save item in the database. """
session = self.Session()
deal = Price(**item)
try:
session.add(deal)
session.commit()
except Exception as ex:
logger.error(ex)
logger.error(traceback.format_exc())
session.rollback()
raise
finally:
session.close()
return item
def make_product(product_data):
product = Product(
id = product_data.get('itemId'),
upc = product_data.get('upc'),
name = product_data.get('name'),
thumbnail_img = product_data.get('thumbnailImage'),
med_img = product_data.get('mediumImage'),
lg_img = product_data.get('largeImage'),
short_descr = product_data.get('shortDescription'),
long_descr = product_data.get('longDescription'),
msrp = product_data.get('msrp'),
add_to_cart_url = product_data.get('addToCartUrl'),
url = product_data.get('productUrl'),
)
price = product_data.get('salePrice')
if price:
product.prices.append(Price(price=price, stock=product_data.get('stock')))
return product
def main():
Session = sessionmaker(bind=get_engine())
session = Session()
response = requests.get('https://api.coinmarketcap.com/v1/ticker/')
for data in response.json():
price_usd = data['price_usd']
updated = data['last_updated']
crypt_currency_id = data['id']
price = Price(price_usd=price_usd,
updated=datetime.datetime.fromtimestamp(int(updated)),
crypt_currency_id=crypt_currency_id)
session.add(price)
session.flush()
session.commit()
def parse_prices():
url = "https://ru.investing.com/indices/mcx-components"
headers = {
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/73.0.3683.86 YaBrowser/19.4.0.2397 Yowser/2.5 Safari/537.36 "
}
r = requests.get(url=url, headers=headers)
sel = Selector(text=r.text)
companies_info = sel.css('table#cr1 > tbody').xpath(
'./tr[contains(@id,"pair")]')
for company in companies_info:
name_el = company.xpath('./td[2]/a')
name = name_el.xpath('./text()').extract_first()
current_price = company.xpath(
'./td[contains(@class,"last")]/text()').extract_first()
current_price = Formatter.format_price(current_price)
high_price = company.xpath(
'./td[contains(@class,"high")]/text()').extract_first()
high_price = Formatter.format_price(high_price)
low_price = company.xpath(
'./td[contains(@class,"low")]/text()').extract_first()
low_price = Formatter.format_price(low_price)
volume = company.xpath(
'./td[contains(@class,"turnover")]/text()').extract_first()
volume = Formatter.format_volume(volume)
update_time = int(
company.xpath(
'./td[contains(@class,"time")]/@data-value').extract_first())
p = Price()
p.current = current_price
p.high = high_price
p.low = low_price
p.volume = volume
p.time = datetime.fromtimestamp(update_time)
company = Company.select().where(Company.name == name).limit(1)
if len(company) > 0:
p.company = company[0].id
p.save()
else:
continue
def save_prices(element_list, stock_code):
for element in element_list:
price = Price()
price.stock = stock_code
price_date = datetime.date(datetime.strptime(element[0], '%m/%d/%Y'))
price.date = price_date
price.open = element[1]
price.high = element[2]
price.low = element[3]
price.close = element[4]
volume = element[5].replace(',', '')
if volume == '':
volume = 0
price.volume = volume
try:
price.save()
logger.info(
f'Price object: {price.stock} in date: {price.date} saved in db'
)
except Exception as exception:
logger.error(
f'{price.code} in date: {price.date} is not saved: {exception}'
)
def update_event_details(event, event_form):
details = event_form.collectEventDetails()
details["authorizations"] = [Authorization.query.filter_by(name=auth).first()
for auth in details["authorizations"]]
details["platforms"] = [Platform.query.filter_by(name=plat).first()
for plat in details["platforms"]]
details["resources"] = [Resource.query.filter_by(name=res).first()
for res in details["resources"]]
details["tags"] = [Tag.query.filter_by(name=tag).first()
for tag in details["tags"]]
details["prices"] = [Price(name=price['name'],
description=price['description'],
value=price['price'],
availability=price['availability'][0])
for price in details['prices']]
if event_form.image_file.data:
details["image_file"] = event_form.image_file.data.filename
for attribute in details:
try:
getattr(event, attribute)
except AttributeError:
pass
setattr(event, attribute, details[attribute])
def new_product():
if request.method == 'GET':
return render_template('newproduct.html')
else:
try:
data = request.get_json()
prices = []
for price in data['prices']:
options = []
for option_id in price['options']:
current_option = data['options'][option_id]
options.append(
Option(option_id=option_id,
option_type=current_option['type'],
option_text=current_option['text'],
option_value=current_option['value']).save())
current = Price(
valid_from=datetime.utcnow(),
valid_to=datetime.utcnow(),
currency=price['currency'],
original_price=price['originalPrice'],
discounted_price=price['discountedPrice'],
discount_rate=price['discountRate'],
# stock=price['stock'],
# is_active=price['isActive'],
options=options)
prices.append(current)
product = Product(title=data['title'],
description=data['description'],
image_url=data['imageURL'],
prices=prices).save()
return jsonify({'created': str(product.id), 'success': True})
except Exception as e:
traceback.print_exc()
return make_response('An error occured while adding product.', 400)
return jsonify({'hello': 'world'})
from app import create_app
from models import Price, Stock, db
#p = subprocess.Popen([sys.executable, 'price_generator.py'])
app = create_app()
with app.app_context():
#Retrieve the newest price for each stock
query = db.session.query(Price).distinct(Price.code).order_by(Price.code,Price.id.desc()).all()
values = {x.code:x.price for x in query}
# Generate new price every 10 seconds
# The new price is generated using random walk model
# Insert a new record using current time and new price
starttime = time.time()
while True:
# Check if there is any insertion or deletion on table stock
# If so, retrieve the last traded price for each stock again
if len(values) != db.session.query(Stock).count():
query = db.session.query(Price).distinct(Price.code).order_by(Price.code,Price.id.desc()).all()
values = {x.code:x.price for x in query}
for record in values:
current_time = datetime.now()
value = values[record]
new_value = value + random.randn() * 3
new_price = Price(code = record, price = new_value, timestamp=current_time)
new_price.insert()
values[record] = new_value
time.sleep(10.0 - ((time.time() - starttime) % 10.0))
numbers = re.findall("[0-9]{1,4}/?[A-Za-z]?", cleaned_street)
if len(numbers) > 0:
street_number = numbers[0]
else:
street_number = ""
# cleaned address
cleaned_street = cleaned_street.split(',')[0].split('-')[0]
# generate address
address = Address(province=row['Province'],
city=row['City'],
street=cleaned_street,
number=str(street_number),
cap=int(row['CAP']),
location=location)
# generate price
price = Price(row['Price'])
# generate contact
contact = Contact(phone=row['Phone'],
email=row['Email'],
website=row['Website'])
#generate facility
facility = Facility(type=row['Type'],
price=price,
contact=contact,
ranking=1,
address=address,
calendar=None)
data.append(facility.toJSON())
#print(data)
print("2. Exporting assured json data")
ds_out_path = '../../dataset/Formal Modeling/data/accomodations_assured.json'
query = json.loads(response.content)
payload = query['payload']
tkADD = payload[0]['address']
tkNAME = payload[0]['name']
tkSYMBOL = payload[0]['symbol']
tkPRICE = payload[0]['priceUSD']
print(tkADD, tkPRICE, tkNAME, tkSYMBOL)
db_string = 'postgresql://*****:*****@localhost/practiceDataDB'
db = models.create_engine(db_string, echo=False)
base = models.declarative_base()
base.metadata.create_all(db)
db_Session = sessionmaker(db)
db_session = db_Session()
result = Price(address=tkADD,
name=tkNAME,
symbol=tkSYMBOL,
price=tkPRICE)
db_session.add(result)
db_session.commit()
print('entry added')
db_session.close()
if choice2.lower() == 'historical':
# https://docs.amberdata.io/reference#get-historical-token-price for questions on querystring
tokenAddress2 = str(input('input hash of token you want to look up? '))
url = "https://web3api.io/api/v2/market/tokens/prices/" + tokenAddress2 + "/historical"
querystring = {"timeInterval": "d", "timeFormat": "ms"}
headers = {"x-api-key": secrets.api_key}
def populateTemplate(self, path):
with open(path) as json_file:
data = json.load(json_file)
self.event_type = EventType._class
self.title = data.get('title', '')
self.description = data.get('description', '')
self.host_email = data.get('instructorEmail', '')
self.host_name = data.get('instructorName', '')
self.location = data.get('location', '')
self.registration_limit = data.get('registrationLimit', '')
value = data.get('classDate', '')
if self.isNotBlank(value):
self.classDate = parse(value, fuzzy=True)
value = data.get('startTime', '')
if self.isNotBlank(value):
self.start_time = parse(value, fuzzy=True)
value = data.get('stopTime', '')
if self.isNotBlank(value):
self.end_time = parse(value, fuzzy=True)
self.duration = self.end_time - self.start_time
self.start_time = self.start_time.time()
self.min_age = data.get('minimumAge', '')
self.max_age = data.get('maximumAge', '')
self.event_type = EventType._class
self.image_file = 'default.jpg'
# linked tables
self.price_list = data.get('prices', [])
taglist = data.get('tags', [])
self.req_auths = []
self.calendar_resources = []
for tags in taglist:
if tags == "Required auth's":
self.req_auths = taglist[tags]
elif tags == "Resources":
for r in taglist[tags]:
if r[0:11] == 'Makerspace ':
r = r[11:]
self.calendar_resources.append(r)
self.platforms = Platform.query.all()
self.authorizations = [
Authorization.query.filter_by(name=auth).first()
for auth in self.req_auths
]
self.resources = [
Resource.query.filter_by(name=res).first()
for res in self.calendar_resources
]
self.prices = [
Price(name=price['name'],
description=price['description'],
value=price['price'],
availability=price['availability'][0]
if price['availability'] else "")
for price in self.price_list
]
self.tags = [Tag.query.filter_by(name=path.split('/')[-2]).first()]
if not self.tags[0]:
self.tags = [Tag(name=path.split('/')[-2])]
def send_quote():
order_list = []
count = 0
if request.method == 'POST':
data = request.form
supplier = Supplier.query.filter_by(id=data['supplier_id']).first()
deal = Deal.query.filter_by(id=data['deal_id']).first()
orders = Order.query.filter_by(deal_id=deal.id)
quote = {
'name': supplier.name,
'expiration': data['expiration'],
'info': data['info'],
'phone': data['phone'],
'email': data['email'],
'skype': data['skype']
}
Quote.query.filter_by(deal_id=deal.id, supplier_id=supplier.id).update(
dict(validity=quote['expiration'],
email=quote['email'],
phone=quote['phone'],
skype=quote['skype'],
info=quote['info']))
new_quote = Quote.query.filter_by(deal_id=deal.id,
supplier_id=supplier.id).first()
db.session.commit()
for o in orders:
order_data = {
'grade': data['grade%s' % count],
'quantity': data['quantity%s' % count],
'maxSulphur': data['maxSulphur%s' % count],
'unit': data['unit%s' % count],
'specifications': data['spec%s' % count],
'comments': data['comments%s' % count],
'price': data['price%s' % count],
'terms': data['terms%s' % count],
'delivery': data['delivery%s' % count],
'physical': data['physical%s' % count]
}
order = Order.query.filter_by(deal_id=deal.id,
grade=order_data['grade']).first()
price = Price(order_data['price'], order_data['terms'],
order_data['physical'], order_data['delivery'],
new_quote, order)
db.session.add(price)
db.session.commit()
order_list.append(order_data)
count += 1
quote['orders'] = order_list
pusher_client.trigger('test_channel', 'my_event', quote)
return render_template('success_quote.html', deal=deal)
else:
deal_id = request.args['deal_id']
supplier_id = request.args['supplier_id']
deal = Deal.query.filter_by(id=deal_id).first()
orders = Order.query.filter_by(deal_id=deal.id)
for order in orders:
order = {
'grade': order.grade,
'quantity': order.quantity,
'unit': order.unit,
'maxSulphur': order.maxSulphur,
'spec': order.spec,
'comments': order.comments
}
order['number'] = count
order_list.append(order)
count += 1
return render_template('quote_form.html',
deal_id=deal_id,
supplier_id=supplier_id,
deal=deal,
orders=order_list)
