def get(self, result_id): """Handles """ result = ResultModel.get_by_id(int(result_id)) date = result.timestamp data = [] site = result.site_name result = result.merchants lines = result.split(r'\n') for line in lines: items = line.split(r'\t') if len(items) == 6: site = 'apple' # This is needed for 'www.bestbuy.com' data.append(items) results = ResultModel.query().order(-ResultModel.timestamp).fetch() values = { 'site_names': URLS, 'results': results, 'merchants': data, 'date': date, 'site': site } # self.session.add_flash('Some message', level='error') self.render('index.html', values)
def get(self): """Response all malls from last job""" start_time = datetime.datetime.now() date = start_time.strftime('%Y-%m-%d %H:%M:%S') results = ResultModel.query().order(-ResultModel.timestamp).fetch() last_results = results[-10:] data_entries = last_results sites = OrderedDict([[x, '-'] for x in URLS]) headers = OrderedDict([[x, '-'] for x in URLS]) data = dict() for entry in data_entries: date_scraped = entry.timestamp scraped_from = entry.site_name # Table header headers[scraped_from] = ('\n'.join([scraped_from, date_scraped.strftime('%Y-%m-%d %H:%M:%S')])) vendors = entry.merchants vendors = vendors.split(r'\n') for vendor in vendors: result = vendor.split(r'\t') name = result[0] try: rate = result[1] except ValueError: rate = ' ' try: # If this vendor is listed rates = data[name] except KeyError: rates = sites rates[scraped_from] = rate data[name] = rates # for item in data: # print item # print '----------' # costs = data[item] # for d in costs: # cost = get_data_from_html(costs[d]) # if cost == u' ': # pass # else: # print d, cost # print '===================' # print headers values = { 'site_names': URLS, 'date': date, 'data': data, 'site': '' } return self.render('all_malls.html', values)
def get(self): """Handles GET /index and /""" results = ResultModel.query().order(-ResultModel.timestamp).fetch() values = { 'site_names': URLS, 'results': results } # self.session.add_flash('Some message', level='error') self.render('index.html', values)
def list_results(): """List all scraped data""" user = users.get_current_user() print dir(user) user_name = user.email() if user_name is u'*****@*****.**': results = ResultModel.query().order(-ResultModel.timestamp).fetch() return render_template('list_data.html', site_names=URLS, results=results, name=user_name) else: sites_list = SitesModel.query() return render_template('sites.html', site_names=URLS, sites=sites_list, name=user_name)
def get(self): """Page to delete some results""" if self.logged_in: name = self.current_user.name.encode('utf8') if name == 'Дмитрий Брач' or name == 'Jasper Moy': results = ResultModel.query().order(-ResultModel.timestamp).fetch() values = {'user': self.current_user, 'site_names': URLS, 'results': results } # self.session.add_flash('Some message', level='error') self.render('admin.html', values) else: self.redirect('/logout') return else: self.redirect('/login')
def test_result(result_id): """Main page""" result = ResultModel.get_by_id(int(result_id)) date = result.timestamp data = [] site = result.site_name result = result.merchants lines = result.split(r'\n') for line in lines: items = line.split(r'\t') if len(items) == 6: site = 'apple' # This is needed for 'www.bestbuy.com' data.append(items) results = ResultModel.query().order(-ResultModel.timestamp).fetch() return render_template('test.html', site_names=URLS, results=results, merchants=data, date=date, site=site)
def by_time(): """Response up to 5 last 5 results""" site = '' if request.method == 'POST': start_time = request.form['start_time'] # {0} # start_time = datetime.datetime(start_time).strptime() else: start_time = datetime.datetime.now() # end_time = start_time - datetime.timedelta(days=5) # {1} # q = "SELECT * FROM ResultModel WHERE timestamp < DATETIME({0}) AND timestamp > DATE({1})".format( # start_time, end_time) date = start_time.strftime('%Y-%m-%d %H:%M:%S') # q = "SELECT * FROM ResultModel WHERE timestamp <= DATETIME('%s')" % date # data_entries = ndb.gql(q).fetch() # print(q) results = ResultModel.query().order(-ResultModel.timestamp).fetch() return render_template('test.html', site_names=URLS, results=results, site=site)
def test(): """Test the new look""" results = ResultModel.query().order(-ResultModel.timestamp).fetch() return render_template('test.html', site_names=URLS, results=results)