def get(self, result_id):
"""Handles """
result = ResultModel.get_by_id(int(result_id))
date = result.timestamp
data = []
site = result.site_name
result = result.merchants
lines = result.split(r'\n')
for line in lines:
items = line.split(r'\t')
if len(items) == 6:
site = 'apple' # This is needed for 'www.bestbuy.com'
data.append(items)
results = ResultModel.query().order(-ResultModel.timestamp).fetch()
values = {
'site_names': URLS,
'results': results,
'merchants': data,
'date': date,
'site': site
}
# self.session.add_flash('Some message', level='error')
self.render('index.html', values)
def save_result(self, merchants_data):
"""Accept list with strings"""
merchants = r'\n'.join(x for x in merchants_data)
result = ResultModel(merchants=merchants, site_name=self.site_name)
result.put()
result_id = result.key.id()
result_id = '|'.join([str(result_id), str(result.timestamp)])
self.update_site_results(result_id=result_id, site_name=self.site_name)
return result_id
async def div(params: ParamsModel):
try:
res = ResultModel(result=(params.a / params.b))
except ZeroDivisionError:
raise HTTPException(404, "error division by zero")
await db.add(OperationModel(operation=div.__name__, params=params, result=res))
return res
def post(self, result_id):
"""Delete a results object"""
if self.logged_in:
result = ResultModel.get_by_id(int(result_id))
# Should delete result from site model
site = result.site_name
print site
q = "SELECT * FROM SitesModel WHERE site_name = '%s'" % site
data_entry = ndb.gql(q).fetch()[0]
print data_entry
results = data_entry.results.split('/')
for result in results:
if str(result_id) in result:
results.remove(result)
data_entry.results = '/'.join(results)
data_entry.put()
try:
result.key.delete()
self.session.add_flash(u'result %s successfully deleted.' % result_id, level='success')
except CapabilityDisabledError:
self.session.add_flash(u'App Engine Datastore is currently in read-only mode.', level='error')
return self.redirect('/results')
else:
self.redirect('/login')
def download_file(result_id):
result = ResultModel.find_by_id(result_id)
model = DesignModel.find_by_id(result.model_id)
if model and (model.username == session["username"]
or session["admin"] == True):
try:
#Set custom mimetype and as_attmt values
custom_mimetypes = {
'.txt': 'text/plain',
'.json': 'text/plain',
'.csv': 'text/plain',
}
file_ext = os.path.basename(result.objname)[-4:]
mimetype = custom_mimetypes.get(file_ext, None)
#text files will open in new tab
as_attachment = False if mimetype in ['text/plain'] else True
uploads = os.path.join(current_app.config['UPLOADED_FILES_DEST'],
os.path.dirname(result.objname))
return send_from_directory(directory=uploads,
filename=os.path.basename(
result.objname),
attachment_filename=result.name,
mimetype=mimetype,
as_attachment=as_attachment)
except:
traceback.print_exc()
return "File not found!"
else:
flash('Unauthorized: Unable to download model id {}'.format(model_id),
'danger')
return redirect(url_for(".index", model_id=model.id))
def get(self):
"""Response all malls from last job"""
start_time = datetime.datetime.now()
date = start_time.strftime('%Y-%m-%d %H:%M:%S')
results = ResultModel.query().order(-ResultModel.timestamp).fetch()
last_results = results[-10:]
data_entries = last_results
sites = OrderedDict([[x, '-'] for x in URLS])
headers = OrderedDict([[x, '-'] for x in URLS])
data = dict()
for entry in data_entries:
date_scraped = entry.timestamp
scraped_from = entry.site_name
# Table header
headers[scraped_from] = ('\n'.join([scraped_from, date_scraped.strftime('%Y-%m-%d %H:%M:%S')]))
vendors = entry.merchants
vendors = vendors.split(r'\n')
for vendor in vendors:
result = vendor.split(r'\t')
name = result[0]
try:
rate = result[1]
except ValueError:
rate = ' '
try: # If this vendor is listed
rates = data[name]
except KeyError:
rates = sites
rates[scraped_from] = rate
data[name] = rates
# for item in data:
# print item
# print '----------'
# costs = data[item]
# for d in costs:
# cost = get_data_from_html(costs[d])
# if cost == u' ':
# pass
# else:
# print d, cost
# print '==================='
# print headers
values = {
'site_names': URLS,
'date': date,
'data': data,
'site': ''
}
return self.render('all_malls.html', values)
def get(self):
"""Handles GET /index and /"""
results = ResultModel.query().order(-ResultModel.timestamp).fetch()
values = {
'site_names': URLS,
'results': results
}
# self.session.add_flash('Some message', level='error')
self.render('index.html', values)
def test_result(result_id):
"""Main page"""
result = ResultModel.get_by_id(int(result_id))
date = result.timestamp
data = []
site = result.site_name
result = result.merchants
lines = result.split(r'\n')
for line in lines:
items = line.split(r'\t')
if len(items) == 6:
site = 'apple' # This is needed for 'www.bestbuy.com'
data.append(items)
results = ResultModel.query().order(-ResultModel.timestamp).fetch()
return render_template('test.html', site_names=URLS, results=results, merchants=data, date=date, site=site)
def show_result(result_id):
"""List all scraped data"""
result = ResultModel.get_by_id(result_id)
result = result.merchants
data = result.split(r'\n')
merchants = dict()
for string in data:
data = string.split(r'\t')
merchants[data[0]] = data[1]
return render_template('list_merchants.html', site_names=URLS, merchants=merchants)
def list_results():
"""List all scraped data"""
user = users.get_current_user()
print dir(user)
user_name = user.email()
if user_name is u'*****@*****.**':
results = ResultModel.query().order(-ResultModel.timestamp).fetch()
return render_template('list_data.html', site_names=URLS, results=results, name=user_name)
else:
sites_list = SitesModel.query()
return render_template('sites.html', site_names=URLS, sites=sites_list, name=user_name)
def delete_result(result_id):
result = ResultModel.find_by_id(result_id)
model = DesignModel.find_by_id(result.model_id)
if result and model and model.username == session["username"]:
result.delete_from_db()
try:
uploads = os.path.join(current_app.config['UPLOADED_FILES_DEST'])
os.remove(os.path.join(uploads, result.objname))
except:
traceback.print_exc()
flash('Result File deletion failed', 'danger')
else:
flash('Unauthorized: Unable to delete result id {}'.format(result_id),
'danger')
return redirect(url_for(".index", model_id=model.id))
def get_data(self, result_id):
"""Get data from DB by id
Return dictionary with 'name': 'rate'
"""
result = ResultModel.get_by_id(int(result_id))
try:
result = result.merchants
except AttributeError:
return False
data = result.split(r'\n')
merchants = dict()
for item in data:
res = item.split(r'\t')
name = res[0]
rate = res[1].split(' ')[0]
merchants[name] = rate
return merchants
def get(self):
"""Page to delete some results"""
if self.logged_in:
name = self.current_user.name.encode('utf8')
if name == 'Дмитрий Брач' or name == 'Jasper Moy':
results = ResultModel.query().order(-ResultModel.timestamp).fetch()
values = {'user': self.current_user,
'site_names': URLS,
'results': results
}
# self.session.add_flash('Some message', level='error')
self.render('admin.html', values)
else:
self.redirect('/logout')
return
else:
self.redirect('/login')
def by_time():
"""Response up to 5 last 5 results"""
site = ''
if request.method == 'POST':
start_time = request.form['start_time'] # {0}
# start_time = datetime.datetime(start_time).strptime()
else:
start_time = datetime.datetime.now()
# end_time = start_time - datetime.timedelta(days=5) # {1}
# q = "SELECT * FROM ResultModel WHERE timestamp < DATETIME({0}) AND timestamp > DATE({1})".format(
# start_time, end_time)
date = start_time.strftime('%Y-%m-%d %H:%M:%S')
# q = "SELECT * FROM ResultModel WHERE timestamp <= DATETIME('%s')" % date
# data_entries = ndb.gql(q).fetch()
# print(q)
results = ResultModel.query().order(-ResultModel.timestamp).fetch()
return render_template('test.html', site_names=URLS, results=results, site=site)
def delete_result(result_id):
"""Delete an results object"""
result = ResultModel.get_by_id(result_id)
# Should delete result from site model
site = result.site_name
q = "SELECT * FROM ResultModel WHERE site_name = '%s'" % site
data_entry = ndb.gql(q).fetch()
results = data_entry.results.split('/')
for result in results:
if str(result_id) in result:
results.remove(result)
data_entry.results = '/'.join(results)
data_entry.put()
try:
result.key.delete()
flash(u'result %s successfully deleted.' % result_id, 'success')
return redirect(url_for('list_results'))
except CapabilityDisabledError:
flash(u'App Engine Datastore is currently in read-only mode.', 'info')
return redirect(url_for('list_results'))
def download_file(result_id):
result = ResultModel.find_by_id(result_id)
model = DesignModel.find_by_id(result.model_id)
if model and session["admin"] == True:
try:
uploads = os.path.join(current_app.config['UPLOADED_FILES_DEST'],
os.path.dirname(result.objname))
return send_from_directory(directory=uploads,
filename=os.path.basename(
result.objname),
attachment_filename=result.name,
as_attachment=True)
except:
traceback.print_exc()
return "File not found!"
else:
flash('Unauthorized: Unable to download model id {}'.format(model_id),
'danger')
return redirect(url_for(".index"))
async def result():
if not (TSK_ID_LIST and RES_DICT) or len(TSK_ID_LIST) != len(RES_DICT):
return ResultModel(ret=1)
msg = "".join([RES_DICT[tsk_id] for tsk_id in TSK_ID_LIST])
return ResultModel(ret=0, message=msg)
def test():
"""Test the new look"""
results = ResultModel.query().order(-ResultModel.timestamp).fetch()
return render_template('test.html', site_names=URLS, results=results)
from decimal import Decimal
import boto3
import moto
from models import ResultModel
with moto.mock_dynamodb2():
region = 'eu-west-1'
ResultModel.Meta.region = region
ResultModel.create_table(wait=True)
# Data
result_map = {
'semester 1': 3.75,
'semester 2': 3.17,
'semester 3': 3.90,
'semester 4': 3.67
}
result_list = [3.75, 3.17, 3.90, 3.67]
# Insert as Map with PynamoDB
result1 = ResultModel(result=result_map)
result1.save()
assert ResultModel.count() == 1
result1_id = result1.id
print(result1_id)
# Retrieve the data from inserted as list
result1_retr = ResultModel.get(result1_id)
assert result1_retr.result.attribute_values == result_map
def index():
results = ResultModel.find_all()
return render_template("results/indexall.html",
results=result_list_schema.dump(results))
async def mul(params: ParamsModel):
res = ResultModel(result=(params.a * params.b))
await db.add(OperationModel(operation=mul.__name__, params=params, result=res))
return res
