def load_data(request):
data = requests.get(DEFAULT_DATA_URL).json()
for i in data:
query = Market(date=datetime.datetime.strptime(i.get('Date'),
'%d-%b-%Y'),
open=i.get('Open'),
close=i.get('Close'),
high=i.get('High'),
low=i.get('Low'),
shares_traded=i.get('Shares Traded'),
turnover=i.get('Turnover (Rs. Cr)'))
query.save()
dump = json.dumps({'success': True, 'results': 'Data loaded successfully'})
return HttpResponse(dump, content_type='application/json')
def save_market_to_db(msg_data: dict):
"""
- Получить запись текущего юзера (запрос в базу по имени и взять одну запись,
если исключение что нет записи, то создать юзера нового)
- Получить все записи маркетов текущего юзера
-
:return:
"""
print('----now------')
print(msg_data)
session = Session()
user = get_or_create(session, User, user_name=msg_data['user']['user_name'])
user.free_slots = int(msg_data['market']['free_slots'])
user.telegram_id = int(msg_data['user']['telegram_id'])
user.name = str(msg_data['user']['name'])
all_markets_user = session.query(Market).filter_by(user=user).all()
# удалить все магазины текущего юзера из базы
for market in all_markets_user:
session.delete(market)
session.commit()
res_dict = msg_data['market']['resources']
current_resources = [get_or_create(session, Resource, name=resource) for resource in res_dict.keys()]
current_markers = [Market(user=user,
resource=res,
count=res_dict[res.name],
last_update=msg_data['user']['last_update']) for res in current_resources]
session.add_all(current_markers)
session.commit()
def seed_database():
'''add in Drugs, Countries, and a few markets'''
drugs = []
for d in mock_data.drugs:
q = Drug.query.filter_by(name=d).first()
if q is None:
drug = Drug(name=d)
drugs.append(drug)
db.session.add_all(drugs)
db.session.commit()
countries = []
for c in mock_data.all_countries:
q = Country.query.filter_by(name=c[0]).first()
if q is None:
country = Country(name=c[0], c2=c[1])
countries.append(country)
db.session.add_all(countries)
db.session.commit()
ms = ["Rechem", "DN1", "DN2"]
markets = []
for m in ms:
q = Market.query.filter_by(name=m).first()
if q is None:
market = Market(name=m)
markets.append(market)
db.session.add_all(markets)
db.session.commit()
return '', 204
def marketregister(request):
if request.method == 'GET':
return render(request, 'app/marketregister.html', {})
else:
result_dict = {}
Name = request.POST['name']
Address = request.POST['address']
User_id = request.POST['user_id']
PW = request.POST['passwd']
CPW = request.POST['confirm']
BIRTH = request.POST['birth']
RRN = request.POST['rrn']
residentnum = BIRTH + '-' + RRN
alluser = Market.objects.all()
list = []
for i in range(0, len(alluser)):
a = alluser.values('residentnum')[i]['residentnum']
list.append(a)
if PW != CPW:
result_dict['result'] = 'Password does not match'
return JsonResponse(result_dict)
elif Name == '' or Address == '' or User_id == '' or PW == '' or BIRTH == '' or RRN == '':
result_dict['result'] = 'Please pill out the form'
return JsonResponse(result_dict)
elif residentnum in list:
result_dict['result'] = 'Resident Number cannot be used'
return JsonResponse(result_dict)
else:
try:
Market.objects.get(user_id=User_id)
result_dict['result'] = 'ID cannot be used'
except Market.DoesNotExist:
market = Market(name=Name, address=Address, user_id=User_id, passwd=PW, passconfirm=CPW,
residentnum=residentnum)
market.c_date = timezone.now()
market.save()
result_dict['result'] = 'success'
return JsonResponse(result_dict)
def import_rechem_data():
c = sqlite3.connect('rechem_listings.db')
with open('rechem_drug_titles', 'r') as f:
s = f.read()
drug_title_dict = ast.literal_eval(s)
rechem_pages = pd.read_sql_query("SELECT * FROM Listings", c)
rechem_listings = pd.read_sql_query("SELECT * FROM Listing_index", c)
for i, row in rechem_listings.iterrows():
rechem_listings.loc[i, 'drug'] = drug_title_dict[row['title']]
m = Market.query.filter_by(name="rechem_real").first()
if not m:
m = Market(name="rechem_real")
db.session.add(m)
db.session.commit()
print("market added")
listings = []
for i, row in rechem_listings.iterrows():
drug = Drug.query.filter_by(name=row['drug']).first()
if not drug:
db.session.add(Drug(name=row['drug']))
db.session.commit()
#check if listing for this drug exists. if not, add it.
listing = Listing.query.filter_by(drug=drug).first()
if not listing:
listings.append(Listing(url=row['url'], market=m, drug=drug))
if listings:
db.session.add_all(listings)
db.session.commit()
print("Listings added")
pages = []
for i, row in rechem_pages.iterrows():
listing = rechem_listings[rechem_listings['id'] == row['listing_index_id']]
listing_drug = listing['drug'].item()
listing_name = listing['title'].item()
drug = Drug.query.filter_by(name=listing_drug).first()
listing = Listing.query.filter_by(drug=drug, market=m).first()
# check if a page exsits for this time/listing. If not, add it.
timestamp = datetime.strptime(row['scraped_time'], "%Y-%m-%d %H:%M:%S")
page = Page.query.filter_by(listing=listing, timestamp=timestamp).first()
if not page:
pages.append(Page(html=row['page_text'], timestamp=timestamp, name=listing_name, listing=listing))
if pages:
db.session.add_all(pages)
db.session.commit()
print("Pages added")
return ('', 204)
def create_market():
name = request.form['name']
market = Market.query.filter_by(name=name).first()
if market:
flash('{} is already a market. Use a different name.'.format(name))
else:
market = Market(name=name)
db.session.add(market)
db.session.commit()
markets = Market.query.all()
# TODO: may be better to return ('', 204) for this. We may want to delete through this route in the future on
# TODO: a different page and not return the table. Create a separate route to get markets and render table
return render_template('_data_summary_table.html', markets=markets)
def setUp(self):
self.app = myapp
self.app.config.update(
TESTING = True,
SQLALCHEMY_ECHO = False,
SQLALCHEMY_DATABASE_URI = 'sqlite://',
)
self.client = self.app.test_client()
# The cleanup and schema creation
db.session.close()
db.drop_all()
db.create_all()
# The test fixtures (Ideally this would sit under a separate fixtures modules)
testData = [
(
'Cricket',
'Australia Vs Ireland',
datetime(2018, 11, 23, 10, 10, 10),
[('Australia', 1.01), ('Ireland', 1.01)]
),
(
'Football',
'Spain Vs Germany',
datetime(2018, 11, 22, 10, 10, 10),
[('Spain', 1.01), ('Germany', 1.01)]
),
(
'Football',
'Portugal Vs Italy',
datetime(2018, 11, 21, 10, 10, 10),
[('Portugal', 1.01), ('Italy', 1.01)]
)
]
for sportName, eventName, eventTime, selections in testData:
sport = getOrCreate(db.session, Sport, name=sportName)
event = Event(
name=eventName,
startTime=eventTime,
sport=sport
)
market = Market(name='Winner', event=event)
for selName, odds in selections:
db.session.add(Selection(name=selName, odds=odds, market=market))
db.session.commit()
def update_market(game):
"""Get market prices"""
supremacy = Supremacy(game.game_id, game.game_host)
result = supremacy.market()
orders = result["asks"][1] + result["bids"][1]
market = Market()
market.game_id = game.id
market.datetime = datetime.now()
db.session.add(market)
prices = {}
for resource in orders:
if resource[1]:
lowest_order = resource[1][0]
price = Price()
price.value = lowest_order["limit"]
price.buy = lowest_order["buy"]
price.resource_id = lowest_order["resourceType"]
market.prices.append(price)
prices[price.resource_id] = price
for order_json in resource[1]:
player = game.players.filter(
Player.player_id == order_json["playerID"]).first()
order = Order()
order.order_id = order_json["orderID"]
order.amount = order_json["amount"]
order.buy = order_json["buy"]
order.limit = order_json["limit"]
order.resource_id = order_json["resourceType"]
market.orders.append(order)
if player is not None:
player.orders.append(order)
db.session.add(order)
db.session.commit()
prev_market = market.previous
if prev_market:
prev_prices = prev_market.price_list
if prev_prices:
for resource, price in prices.items():
if prev_prices[resource]:
price.previous_id = prev_prices[resource].id
for resource, price in prices.items():
prev_price = price.previous
if prev_price:
prev_prev_price = prev_price.previous
if prev_prev_price:
if prev_prev_price.value == prev_price.value and \
prev_price.value == price.value:
price.previous_id = prev_prev_price.id
db.session.commit()
db.session.delete(prev_price)
db.session.commit()
def sf4fefffdsf():
c = sqlite3.connect('app/rechem_listings.db')
countries = pd.read_sql_query("SELECT * FROM country", c)
new_countries = []
for i, row in countries.iterrows():
country = Country(name=row['name'], c2=row['c2'])
if not Country.query.filter_by(name=country.name).first():
new_countries.append(country)
db.session.add_all(new_countries)
db.session.commit()
drugs = pd.read_sql_query("SELECT * FROM drug", c)
new_drugs = []
for i, row in drugs.iterrows():
drug = Drug(name=row['name'])
if not Drug.query.filter_by(name=drug.name).first():
new_drugs.append(drug)
db.session.add_all(new_drugs)
db.session.commit()
markets = pd.read_sql_query("SELECT * FROM market", c)
new_markets = []
for i, row in markets.iterrows():
market = Market(name=row['name'])
if not Market.query.filter_by(name=market.name).first():
new_markets.append(market)
db.session.add_all(new_markets)
db.session.commit()
listings = pd.read_sql_query("SELECT * FROM listing", c)
rechem_listings = listings[listings['market_id'] == '4']
new_listings = []
for i, row in rechem_listings.iterrows():
market_name = markets[markets['id'] == int(
row['market_id'])]['name'].item()
new_market_id = Market.query.filter_by(name=market_name).first().id
drug_name = drugs[drugs['id'] == int(row['drug_id'])]['name'].item()
new_drug_id = Drug.query.filter_by(name=drug_name).first().id
# TODO: this is required for sqlite, but may or may not work with sqlalchemy
time_format = "%Y-%m-%d %H:%M:%S.%f"
timestamp = datetime.strptime(row['timestamp'], time_format)
listing = Listing(url=row['url'],
seller=None,
timestamp=timestamp,
market_id=new_market_id,
drug_id=new_drug_id,
origin_id=None)
if not Listing.query.filter_by(url=listing.url).first():
new_listings.append(listing)
db.session.add_all(new_listings)
db.session.commit()
# Get all pages with a listing id that is from the rechem market
pages = pd.read_sql_query("SELECT * FROM page", c)
rechem_pages = pages[pages['listing_id'].isin(rechem_listings['id'])]
new_pages = []
for i, row in rechem_pages.iterrows():
listing_url = listings[listings['id'] == int(
row['listing_id'])]['url'].item()
new_listing_id = Listing.query.filter_by(url=listing_url).first().id
# TODO: this is required for sqlite, but may or may not work with sqlalchemy
time_format = "%Y-%m-%d %H:%M:%S.%f"
timestamp = datetime.strptime(row['timestamp'], time_format)
page = Page(name=row['name'],
html=row['html'],
timestamp=timestamp,
listing_id=new_listing_id)
if not Page.query.filter_by(listing_id=page.listing_id,
timestamp=page.timestamp).first():
new_pages.append(page)
else:
print("page already found:")
db.session.add_all(new_pages)
db.session.commit()
return "data successfully added"
def messageHandler():
"""Each message is expected to contain the full data for that event (match)
Message types handled by this API are as below:
- NewEvent: A complete new sporting event is being created.
- UpdateOdds: There is an update for the odds field (all the other fields
remain unchanged)
Url for POST: http://localhost:5000/api/match/
@return Response: Appropriate response object with data as appropriate
"""
content = request.json
messageType = content['message_type']
event = content['event']
# We assume there is only one market i.e. Winning, hence the [0]
market = event['markets'][0]
# Block that handles creation of a new match record (asumes a valid sport
# is available and supplied in this event data)
if messageType == 'NewEvent':
# Check if sport id is valid
sport = Sport.query.filter_by(id=event['sport']['id']).first_or_404()
newEvent = Event(
name=event['name'],
startTime=datetime.strptime(
event['startTime'],
'%Y-%m-%d %H:%M:%S'
),
sport=sport
)
# Create the market, assuming an instance of market is unique to an event
newMarket = Market(name=market['name'], event=newEvent)
# Create the initial set of selections w/o restricting collection length
for selection in market['selections']:
newSelection = Selection(
name=selection['name'],
odds=selection['odds'],
market=newMarket
)
db.session.add(newSelection)
db.session.commit()
myapp.logger.info('New event %s created', newEvent.name)
return Response('Event created', 201)
# Block that handles update odds (only) request, ensures no other fields are updated
elif messageType == 'UpdateOdds':
# Check if event id is valid
selEvent = Event.query.filter_by(id=event['id']).first_or_404()
myapp.logger.info('Selected event is %s', selEvent.name)
# Now that we know event is valid, we find the market and retrieve its selections
selMarket = Market.query.filter_by(eventId=selEvent.id).first_or_404()
newOddsMap = dict((sel['id'], sel['odds']) for sel in market['selections'])
for selection in selMarket.selections:
newOdds = newOddsMap.get(selection.id)
# Check if None to get 0.0 pass in as valid
if newOdds is not None and newOdds != selection.odds:
selection.odds = newOdds
myapp.logger.info(
'Selection %s updated with odds %f',
selection.name,
selection.odds
)
db.session.commit()
return Response('', 204)
myapp.logger.error('Message type %s is not valid', messageType)
return Response('Bad request', 400)
def import_rechem():
con = sqlite3.connect('app/rechem_listings.db')
con.row_factory = sqlite3.Row # returns sqlite3 query result as dict rather than tuple
c = con.cursor()
c.execute("SELECT * FROM country")
countries = c.fetchall()
new_countries = []
for row in countries:
country = Country(name=row['name'], c2=row['c2'])
if not Country.query.filter_by(name=country.name).first():
new_countries.append(country)
db.session.add_all(new_countries)
db.session.commit()
c.execute("SELECT * FROM drug")
drugs = c.fetchall()
new_drugs = []
for row in drugs:
drug = Drug(name=row['name'])
if not Drug.query.filter_by(name=drug.name).first():
new_drugs.append(drug)
db.session.add_all(new_drugs)
db.session.commit()
c.execute("SELECT * FROM market")
markets = c.fetchall()
new_markets = []
for row in markets:
market = Market(name=row['name'])
if not Market.query.filter_by(name=market.name).first():
new_markets.append(market)
db.session.add_all(new_markets)
db.session.commit()
c.execute("SELECT * FROM listing")
listings = c.fetchall()
rechem_listings = [
listing for listing in listings if listing['market_id'] == '4'
]
rechem_ids = [d['id'] for d in rechem_listings]
new_listings = []
for row in rechem_listings:
new_market_id = Market.query.filter_by(name="Rechem").first().id
drug_name = [d for d in drugs
if d['id'] == int(row['drug_id'])][0]['name']
new_drug_id = Drug.query.filter_by(name=drug_name).first().id
# TODO: this is required for sqlite, but may or may not work with sqlalchemy
time_format = "%Y-%m-%d %H:%M:%S.%f"
timestamp = datetime.strptime(row['timestamp'], time_format)
listing = Listing(url=row['url'],
seller=None,
timestamp=timestamp,
market_id=new_market_id,
drug_id=new_drug_id,
origin_id=None)
if not Listing.query.filter_by(url=listing.url).first():
new_listings.append(listing)
db.session.add_all(new_listings)
db.session.commit()
c.execute("SELECT * FROM page")
pages = c.fetchall()
rechem_pages = [page for page in pages if page['listing_id'] in rechem_ids]
new_pages = []
for row in rechem_pages:
listing_url = [
d for d in listings if d['id'] == int(row['listing_id'])
][0]['url']
new_listing_id = Listing.query.filter_by(url=listing_url).first().id
# TODO: this is required for sqlite, but may or may not work with sqlalchemy
time_format = "%Y-%m-%d %H:%M:%S.%f"
timestamp = datetime.strptime(row['timestamp'], time_format)
page = Page(name=row['name'],
html=row['html'],
timestamp=timestamp,
listing_id=new_listing_id)
if not Page.query.filter_by(listing_id=page.listing_id,
timestamp=page.timestamp).first():
new_pages.append(page)
else:
print("page already found:")
db.session.add_all(new_pages)
db.session.commit()
return "", 204