def update_site_results(result_id, site_name):
try:
site = SitesModel.query().filter(SitesModel.site_name == site_name)
site = site.fetch()[0]
results = '/'.join([str(site.results), str(result_id)])
site.results = results
site.put()
except Exception:
site = SitesModel(key=site_key(site_name), site_name=site_name, results=result_id)
site.put()
def get(self):
if self.logged_in:
sites = SitesModel.query().order().fetch()
changed_sites = OrderedDict([[x, ' '] for x in URLS])
for site in sites:
results = site.results
lasts = results.split('/')
if len(lasts) < 2:
# Only one result
continue
last = lasts[-1].split('|')[0] # [1] - timestamp
last_result = self.get_data(last)
i = -2
while True:
prev = lasts[i].split('|')[0]
prev_result = self.get_data(prev)
if prev_result:
break
i -= 1
# Now we have IDs
changes = self.compare_data(last_result, prev_result)
if len(changes) > 0:
changed_sites[site.site_name] = changes
# Mail results
from mailer.mail_send import SendStatistics
stat = False
for val in changed_sites.itervalues():
if val is not ' ':
stat = True
break
if stat:
result = ''
for k in changed_sites.iterkeys():
changes = changed_sites[k]
changed_cost = ''
for change in changes:
change = ' '.join(get_data_from_html(change).split('/$'))
if change:
changed_cost = '; '.join([change, changed_cost])
if changed_cost:
line = ' '.join([k, changed_cost])
result = '\n'.join([result, line])
stats = SendStatistics()
stats.post(data=result)
values = {'user': self.current_user,
'site_names': URLS,
'sites': changed_sites,
'site': ''
}
# self.session.add_flash('Some message', level='error')
self.render('index.html', values)
else:
self.redirect('/login')
def list_results():
"""List all scraped data"""
user = users.get_current_user()
print dir(user)
user_name = user.email()
if user_name is u'*****@*****.**':
results = ResultModel.query().order(-ResultModel.timestamp).fetch()
return render_template('list_data.html', site_names=URLS, results=results, name=user_name)
else:
sites_list = SitesModel.query()
return render_template('sites.html', site_names=URLS, sites=sites_list, name=user_name)
def check_modification():
def get_data(result_id):
"""Get data from DB by id
Return dictionary with 'name': 'rate'
"""
result = ResultModel.get_by_id(int(result_id))
try:
result = result.merchants
except AttributeError:
return False
data = result.split(r'\n')
merchants = dict()
for item in data:
res = item.split(r'\t')
name = res[0]
rate = res[1].split(' ')[0]
merchants[name] = rate
return merchants
def compare_data(last, prev):
"""Receive two dictionary with 'name': 'rate'
If some of them is changed should alert?
"""
list_of_changes = []
for name in last:
last_rate = last[name]
try:
prev_rate = prev[name]
if last_rate != prev_rate:
changed = name + ' ' + prev_rate + '/' + last_rate
list_of_changes.append(changed)
except KeyError:
changed = name + ' ' + ' ' + '/' + last_rate
list_of_changes.append(changed)
return list_of_changes
sites = SitesModel.query().order().fetch()
changed_sites = OrderedDict([[x, ''] for x in URLS])
for site in sites:
results = site.results
lasts = results.split('/')
if len(lasts) < 2:
# Only one result
continue
last = lasts[-1].split('|')[0] # [1] - timestamp
last_result = get_data(last)
i = -2
while True:
prev = lasts[i].split('|')[0]
prev_result = get_data(prev)
if prev_result:
break
i -= 1
# Now we have IDs
changes = compare_data(last_result, prev_result)
if len(changes) > 0:
changed_sites[site.site_name] = changes
# Mail results
stat = False
for val in changed_sites.itervalues():
if val is not '':
stat = True
break
if stat:
send_mail(changed_sites)
return render_template('changes.html', site_names=URLS,
sites=changed_sites,
site='')
def sites():
"""List of sites to show"""
sites_list = SitesModel.query()
return render_template('sites.html', site_names=URLS, sites=sites_list)
