def compare(days_count : int): TODAY_PRICE = 0 PAST_PRICE = 1 days = [day.date for day in Price.select().distinct(Price.date)] # Get all days if days_count >= len(days): print(f"Parser has only {len(days)} day(s) info") return elif days_count <= 0: print(f"Error with days_count <= 0") return else: for product in Product.select().execute(): today_prices = Price.select().where((Price.date == date.today()) & (Price.product == product.id)).order_by(Price.shop).execute() past_prices = Price.select().where((Price.date == days[-days_count-1]) & (Price.product == product.id)).order_by(Price.shop).execute() print('-------') for prices in zip(today_prices, past_prices): print("Price for {:s} has {:s} by {:.2f} in shop '{:s}' for {:s} user".format( prices[TODAY_PRICE].product.group, 'increased' if prices[TODAY_PRICE].price > prices[PAST_PRICE].price else 'decreased' if prices[TODAY_PRICE].price < prices[PAST_PRICE].price else 'not changed', prices[TODAY_PRICE].price - prices[PAST_PRICE].price if prices[TODAY_PRICE].price >= prices[PAST_PRICE].price else prices[PAST_PRICE].price - prices[TODAY_PRICE].price, prices[TODAY_PRICE].shop, 'authorized' if prices[TODAY_PRICE].authorized else 'not authorized'))
def update_predictions(): predictions = Prediction.select().where((Prediction.actual == 0)) for p in predictions: # Monday = 0, Sunday = 6 # If Friday or Saturday or Sunday then search for Monday if 4 <= p.time.weekday() <= 6: for i in [0, 1, 2, 3, 4, 5, 6]: price = Price.select(Price.current).where( (Price.time > p.time) & (fn.WEEKDAY(Price.time) == i) & (Price.company == p.company)).order_by( Price.id.desc()).limit(1) if len(price) > 0 or (p.time + timedelta(days=i)) > datetime.now(): break else: for i in [1, 2, 3, 4, 5, 6]: weekday = (p.time.weekday() + i) % 7 price = Price.select( Price.current, Price.time).where((Price.time > p.time) & (fn.WEEKDAY(Price.time) == weekday) & (Price.company == p.company)).order_by( Price.id.desc()).limit(1) if len(price) > 0 or p.time + timedelta( days=i) > datetime.now(): break price = price[0] if len(price) > 0 else None if price is not None: p.actual = price.current p.actual_price_time = price.time p.save()
def createDB(): print "Creating the initial db" Company.create_table() NewsSource.create_table() News.create_table() Price.create_table() OpinionAPI.create_table() OpinionAPIResponse.create_table()
def process_item(self, item, spider): # 写入小区房价 if isinstance(item, FangtianxiaRentItem): db_house = self.session.query(House).filter( House.house_name == item['name']).filter( House.city_name == item['city_name']).first() if db_house: if db_house.property_corp == u'暂无数据': db_house.property_corp = item['property_corp'] if db_house.developers == u'暂无数据': db_house.developers = item['developers'] if db_house.building_date == u'暂无数据': db_house.building_date = item['building_date'] if db_house.building_date == u'暂无数据': db_house.building_date = item['building_date'] # 需要添加判断, 以防同一个价格数据插入多次 p = Price(price=item['price'], origin=item['origin'], months=item['months'], crawl_time=item['crawl_date'], uid=db_house.id) db_house.price.append(p) else: house = House(house_name=item['name'], city_name=item['city_name'], url=item['url'], latitude=item['latitude'], longitude=item['longitude'], address=item['location'], building_date=item['building_date'], building_type=item['building_type'], property_corp=item['property_corp'], developers=item['developers'], district=item['district']) price = Price(price=item['price'], origin=item['origin'], months=item['months'], crawl_time=item['crawl_date']) house.price.append(price) self.session.add(house) try: self.session.commit() except Exception, e: print e self.session.rollback() return item
def prices_management(): if request.method == "POST": form = CreatePriceForm(request.form) if form.validate(): price = Price(name=form.name.data, price=form.price.data) price.put() else: form = CreatePriceForm() existing_prices = Price.query().fetch(100) return render_template("admin/prices.html", existing_prices=existing_prices, form=form)
def list_stock(jwt): req = request.get_json() try: exchange = req['exchange'] price = req['price'] name = req['name'] code = req['code'] stock = Stock(code=code, name=name, market_id=exchange) stock.insert() price = Price(code=code, price=price, timestamp=datetime.now()) price.insert() db.session.close() except: abort(400) return jsonify({"success": True})
def price_update(self, request): """Update a price""" entity = Price.price_update(request) if entity: return entity else: raise endpoints.NotFoundException()
def parse_price(): request_green = requests.Session() request_evroopt = requests.Session() authorized_request_green = green_parser.authorize(requests.Session()) authorized_request_evroopt = e_parser.authorize(requests.Session()) if date.today() in [price.date for price in Price.select().execute()]: return else: products = Product.select().order_by(Product.id).execute() bar = Bar('Parse prices', max=len(products)) for product in products: with conn.atomic() as trn: try: main_parser(green_parser, request_green, product.id, product.link_g) main_parser(green_parser, authorized_request_green, product.id, product.link_g) main_parser(e_parser, request_evroopt, product.id, product.link_e) main_parser(e_parser, authorized_request_evroopt, product.id, product.link_e) except AttributeError: trn.rollback() finally: bar.next() bar.finish()
def stream_prices_to_db(self): while True: try: prices = self.get_current_prices() current_ts = int(round(time.time() * 1000)) for coin, price in prices.items(): inserted_at = current_ts coin_type = coin # BTC or ADA price_usd = price price = Price(inserted_at=inserted_at, coin_type=coin_type, price_usd=price_usd) self.session.add(price) self.session.commit() self.session.close() time.sleep(60) except (IncompleteRead, ProtocolError, AttributeError) as e: # Oh well, reconnect and keep trucking print(f"An exception occurred during crypto price call: {e}\n") continue except Exception as e: print( f"An unexpected exception occurred during crypto price call: {e}\n" ) continue except KeyboardInterrupt as e: print(f"Stopping the stream... closing the session...") self.session.close() print(f"Good bye!") break
def bookxcess(book, debug=False): obj = Book.get_by_isbn(book['isbn13'], defaults=book) prices = Price.find_for_book(obj, 'bookxcess') if prices: return prices else: return []
def my_scrapper(): #Retrieve the newest price for each stock query = db.session.query(Price).distinct(Price.code).order_by( Price.code, Price.id.desc()).all() values = {x.code: x.price for x in query} # Generate new price every 20 seconds # The new price is generated using random walk model # Insert a new record using current time and new price for record in values: current_time = datetime.now() value = values[record] new_value = value + random.randn() * 3 new_price = Price(code=record, price=new_value, timestamp=current_time) new_price.insert() db.session.close() db.session.close()
def run(): loop = asyncio.get_event_loop() # # 获取数据库连接池 task = aiomysql.create_pool(**DATABASES) pool = loop.run_until_complete(asyncio.gather(task)) Price.pool = pool[0] # 获取任务 tasks = loop.run_until_complete(asyncio.gather(Price.get_tasks())) print(tasks) # 生成任务 my_func_list = (Price(name).save() for name in tasks) loop.run_until_complete(asyncio.gather(*my_func_list))
def add_precio(): body = request.get_json() precio1 = Price(market_name=body["market_name"], price=body["price"], id_ingrediente=body["id_ingrediente"]) db.session.add(precio1) db.session.commit() return "ok", 200
def price_delete(self, request): """Delete a price""" entity = Price.get_by_id(request.id) if entity: entity.key.delete() return BaseResponse(messages="") else: raise endpoints.NotFoundException()
def get_analytics(date_from, date_to, ticker_code): trades = Price.select(Price.stock, Price.date, (Price.open - Price.close).alias("delta_open_close"), (Price.high - Price.low).alias("delta_high_low")).where( (Price.stock == ticker_code) & (Price.date.between(date_from, date_to))) return trades
def get_price(price_name): "Looks up the current price for the given cost code." price = Price.query(Price.name == price_name).get() if not price: raise ValueError("Price %s not found." % price_name) return price.price
def ticker(): last_price = Price.last() if last_price is not None: return jsonify(btcnok={ 'buy': str(round(last_price.btcnok(rate=settings['BUY_RATE']), 2)), 'sell': str(round(last_price.btcnok(rate=settings['SELL_RATE']), 2)), }) else: abort(503)
def get_products(self): """ function parses the products xml file and returns list of products """ lst = [] for product in self.products.findall('product'): id = product.find('id').text name = product.find('name').text dispensary_id = product.find('dispensary_id').text dispensary_name = product.find('dispensary_name').text canabis_brand = product.find('canabis_brand').text canabis_strain = product.find('canabis_strain').text category = product.find('category').text subcategory = product.find('subcategory').text thc_level = product.find('thc_level').text cbd_level = product.find('cbd_level').text cbn_level = product.find('cbn_level').text thc_level_type = product.find('thc_level_type').text cbd_level_type = product.find('cbd_level_type').text cbn_level_type = product.find('cbn_level_type').text description = product.find('description').text created_at = product.find('created_at').text updated_at = product.find('updated_at').text prices = [] urls = [] images = [] for child in product: if child.tag == 'prices': for cost in child.findall('cost'): prices.append(Price(cost.attrib['unit'], cost.text)) if child.tag == 'urls': admin = child.find('admin').text public = child.find('public').text urls.append(UrlInfo(admin, public)) if child.tag == 'images': for image in child.findall('image'): images.append(Image( image.attrib['main'], image.text, )) lst.append( Product(id, name, dispensary_id, dispensary_name, canabis_brand, canabis_strain, category, subcategory, thc_level, cbd_level, cbn_level, thc_level_type, cbd_level_type, cbn_level_type, prices, urls, images, description, created_at, updated_at)) return lst
def update_product(self, request): ''' Update a product ''' product = Product.get_by_urlsafe_key(request.productKey) if not product: message = 'No product with the key "%s" exists.' % request.productKey raise endpoints.NotFoundException(message) user = User.get_by_urlsafe_key(request.userKey) if not user: message = 'No user with the key "%s" exists.' % request.userKey raise endpoints.NotFoundException(message) category = None if request.category: category = ProductCategory.get_by_urlsafe_key(request.category) price = None if request.price: price = request.price currency = CurrencyUnit.get_by_urlsafe_key(price.currencyKey) location = None if price.location: location = PhysicalLocation.load(lat=price.location.lat, lon=price.location.lon) price = Price.create(user=user, value=price.value, currency=currency, location=location) manufacturer = None if request.manufacturer: manufacturer = Manufacturer.load(name=request.manufacturer, user=user) specs = [] if request.specs: for spec in request.specs: specification = Specification.get_by_urlsafe_key(spec.key) unit = Unit.get_by_urlsafe_key(spec.unit) specs.append([specification, spec.value, unit]) product.update(user=user, name=request.name, category=category, manufacturer=manufacturer, price=price, specs=specs) return message_types.VoidMessage()
def add_quote(): data = request.get_json()['quote'] Quote.query.filter_by(id=data['id']).update( dict(phone=None, skype=None, email=None, validity=data['validity'])) quote = Quote.query.filter_by(id=data['id']).first() quote_data = { 'id': quote.id, 'name': quote.supplier.name, 'expiration': quote.validity, 'info': quote.info, 'phone': quote.phone, 'email': quote.email, 'skype': quote.skype } db.session.commit() order_list = [] for o in data['orders']: order_data = { 'grade': o['grade'], 'quantity': o['quantity'], 'maxSulphur': o['maxSulphur'], 'unit': o['unit'], 'specs': o['spec'], 'comments': o['comments'], 'price': o['price'], 'terms': o['terms'], 'physical': o['physical'], 'delivery': o['delivery'] } order = Order.query.filter_by(deal_id=quote.deal_id, grade=order_data['grade']).first() price = Price(order_data['price'], order_data['terms'], order_data['physical'], order_data['delivery'], quote, order) db.session.add(price) db.session.commit() order_list.append(order_data) quote_data['orders'] = order_list d = Deal.query.filter_by(id=quote.deal_id).first() deal = { 'id': d.id, 'port': d.port, 'vessel': d.vessel, 'imo': d.imo, 'loa': d.loa, 'grossTonnage': d.grossTonnage, 'buyer': d.buyer, 'orderedBy': d.orderedBy, 'eta': d.eta, 'etd': d.etd, } response = jsonify(deal) response.headers.add('Access-Control-Allow-Origin', '*') return response
def parse_prices(): url = "https://ru.investing.com/indices/mcx-components" headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) " "Chrome/73.0.3683.86 YaBrowser/19.4.0.2397 Yowser/2.5 Safari/537.36 " } r = requests.get(url=url, headers=headers) sel = Selector(text=r.text) companies_info = sel.css('table#cr1 > tbody').xpath( './tr[contains(@id,"pair")]') for company in companies_info: name_el = company.xpath('./td[2]/a') name = name_el.xpath('./text()').extract_first() current_price = company.xpath( './td[contains(@class,"last")]/text()').extract_first() current_price = Formatter.format_price(current_price) high_price = company.xpath( './td[contains(@class,"high")]/text()').extract_first() high_price = Formatter.format_price(high_price) low_price = company.xpath( './td[contains(@class,"low")]/text()').extract_first() low_price = Formatter.format_price(low_price) volume = company.xpath( './td[contains(@class,"turnover")]/text()').extract_first() volume = Formatter.format_volume(volume) update_time = int( company.xpath( './td[contains(@class,"time")]/@data-value').extract_first()) p = Price() p.current = current_price p.high = high_price p.low = low_price p.volume = volume p.time = datetime.fromtimestamp(update_time) company = Company.select().where(Company.name == name).limit(1) if len(company) > 0: p.company = company[0].id p.save() else: continue
def ticker(): last_price = Price.last() if last_price is not None: return jsonify( btcnok={ 'buy': str(round(last_price.btcnok(rate=settings['BUY_RATE']), 2)), 'sell': str(round(last_price.btcnok(rate=settings['SELL_RATE']), 2)), }) else: abort(503)
def new_album(): """ Add a new album """ form = PriceForm(request.form) if request.method == 'POST' and form.validate(): # save the album price = Price() save_changes(price, form, new=True) flash('Price created successfully!') return redirect('/') return render_template('new_album.html', form=form)
def add_cafe_price(cafeid=None): form = PriceForm() if cafeid: cafe = Cafe.query.filter_by(id=cafeid).first_or_404() form.cafeid.choices = [(cafe.id, cafe.name)] form.cafeid.data = cafe.id else: cafes = Cafe.query.all() if not cafes: flash("There are no existing cafes. Would you like to make one instead?", "warning") return redirect(url_for("home")) form.cafeid.choices = [(cafe.id, cafe.name) for cafe in cafes] if request.method == "GET": return render_template("priceform.html", cafe=cafe, form=form, formtype="Add", current_user=current_user) if request.method == "POST" and form.validate_on_submit(): cafeid = form.data["cafeid"] if Price.query.filter_by(cafeid=cafeid).filter_by(size=form.data["size"]).first(): flash("There is already a price for this size", "danger") return redirect(url_for("add_cafe_price", cafeid=cafeid)) price = Price(cafe.id) if cafeid: price.cafeid = cafeid else: price.cafeid = form.data["cafeid"] price.size = form.data["size"] price.amount = form.data["amount"] db.session.add(price) db.session.commit() write_to_events("created", "price", price.id) flash("Price added to cafe '%s'" % cafe.name, "success") return redirect(url_for("view_cafe", cafeid=cafeid)) else: for field, errors in form.errors.items(): flash("Error in %s: %s" % (field, "; ".join(errors)), "danger") return render_template("priceform.html", cafe=cafe, form=form, formtype="Add", current_user=current_user)
def handle_precio(): preciosTemp = Price() # get all the people precios_query = preciosTemp.query.all() # get only the ones named "Joe" # receta_query = recetasTemp.query.filter_by(name='pollo') # map the results and your list of people its now inside all_people variable all_precios = list(map(lambda x: x.serializePrecios(), precios_query)) return jsonify(all_precios), 200
def GetPrices(prediction_id): prices = Price.query(Price.prediction_id == ndb.Key( urlsafe=prediction_id)).order(-Price.date).fetch(30) print(str(prices)) prices = [{ 'price': p.value, 'date': { 'year': p.date.year, 'month': p.date.month, 'day': p.date.day } } for p in prices] return prices
def update_price(self, product): self.logger.info('[UPDATE_PRICES] Updating price for {}'.format(product.name)) product_data = Walcart.product(product.id) current_price = product_data.get('salePrice') stock = product_data.get('stock') old_price = None if not product.prices else product.prices[-1].price self.logger.info('\t[UPDATE_PRICES] Old Price: {}\tCurrent Price: {}'.format(old_price, current_price)) if old_price != current_price: self.logger.info('\t\t[UPDATE_PRICES] Price has been updated successfully.') product.prices.append(Price(price=current_price, stock=stock)) res = self.pub.call({'method': 'save', 'resource': serialize(product)}) if not res.get('success'): self.logger.critical('[UPDATE_PRICES] {}'.format(res['message'])) else: self.logger.info('\t\t[UPDATE_PRICES] Price has not changed. No update necessary.') return (product, False)
def process_item(self, item, spider): """Save deals in the database. This method is called for every item pipeline component. """ session = self.Session() deal = Price(**item) try: session.add(deal) session.commit() except: session.rollback() raise finally: session.close() return item
def home(): now = settings['TIMEZONE'].fromutc(datetime.utcnow()) last_price = Price.last() if last_price is not None: current_price = { 'buy': last_price.btcnok(rate=settings['BUY_RATE']), 'sell': last_price.btcnok(rate=settings['SELL_RATE']), } else: current_price = None context = { 'current_price': current_price, 'price_history': json.dumps(get_price_history(app)), 'now': now, } return render_template('home.html', **context)
def process_item(self, item, spider): """ Save item in the database. """ session = self.Session() deal = Price(**item) try: session.add(deal) session.commit() except Exception as ex: logger.error(ex) logger.error(traceback.format_exc()) session.rollback() raise finally: session.close() return item
def main(): Session = sessionmaker(bind=get_engine()) session = Session() response = requests.get('https://api.coinmarketcap.com/v1/ticker/') for data in response.json(): price_usd = data['price_usd'] updated = data['last_updated'] crypt_currency_id = data['id'] price = Price(price_usd=price_usd, updated=datetime.datetime.fromtimestamp(int(updated)), crypt_currency_id=crypt_currency_id) session.add(price) session.flush() session.commit()
def make_product(product_data): product = Product( id = product_data.get('itemId'), upc = product_data.get('upc'), name = product_data.get('name'), thumbnail_img = product_data.get('thumbnailImage'), med_img = product_data.get('mediumImage'), lg_img = product_data.get('largeImage'), short_descr = product_data.get('shortDescription'), long_descr = product_data.get('longDescription'), msrp = product_data.get('msrp'), add_to_cart_url = product_data.get('addToCartUrl'), url = product_data.get('productUrl'), ) price = product_data.get('salePrice') if price: product.prices.append(Price(price=price, stock=product_data.get('stock'))) return product
def new_product(): if request.method == 'GET': return render_template('newproduct.html') else: try: data = request.get_json() prices = [] for price in data['prices']: options = [] for option_id in price['options']: current_option = data['options'][option_id] options.append( Option(option_id=option_id, option_type=current_option['type'], option_text=current_option['text'], option_value=current_option['value']).save()) current = Price( valid_from=datetime.utcnow(), valid_to=datetime.utcnow(), currency=price['currency'], original_price=price['originalPrice'], discounted_price=price['discountedPrice'], discount_rate=price['discountRate'], # stock=price['stock'], # is_active=price['isActive'], options=options) prices.append(current) product = Product(title=data['title'], description=data['description'], image_url=data['imageURL'], prices=prices).save() return jsonify({'created': str(product.id), 'success': True}) except Exception as e: traceback.print_exc() return make_response('An error occured while adding product.', 400) return jsonify({'hello': 'world'})
def update_event_details(event, event_form): details = event_form.collectEventDetails() details["authorizations"] = [Authorization.query.filter_by(name=auth).first() for auth in details["authorizations"]] details["platforms"] = [Platform.query.filter_by(name=plat).first() for plat in details["platforms"]] details["resources"] = [Resource.query.filter_by(name=res).first() for res in details["resources"]] details["tags"] = [Tag.query.filter_by(name=tag).first() for tag in details["tags"]] details["prices"] = [Price(name=price['name'], description=price['description'], value=price['price'], availability=price['availability'][0]) for price in details['prices']] if event_form.image_file.data: details["image_file"] = event_form.image_file.data.filename for attribute in details: try: getattr(event, attribute) except AttributeError: pass setattr(event, attribute, details[attribute])
def prices(self, request): """Create new price""" entity = Price.put_from_message(request) return entity
def prices_show(self, request): """Show all prices""" entity = Price.get_prices(request) return entity
numbers = re.findall("[0-9]{1,4}/?[A-Za-z]?", cleaned_street) if len(numbers) > 0: street_number = numbers[0] else: street_number = "" # cleaned address cleaned_street = cleaned_street.split(',')[0].split('-')[0] # generate address address = Address(province=row['Province'], city=row['City'], street=cleaned_street, number=str(street_number), cap=int(row['CAP']), location=location) # generate price price = Price(row['Price']) # generate contact contact = Contact(phone=row['Phone'], email=row['Email'], website=row['Website']) #generate facility facility = Facility(type=row['Type'], price=price, contact=contact, ranking=1, address=address, calendar=None) data.append(facility.toJSON()) #print(data) print("2. Exporting assured json data") ds_out_path = '../../dataset/Formal Modeling/data/accomodations_assured.json'