def create_metric_textfiles(traffic_to_plot,metrics_to_plot):
"""Put together CSVs of dates and traffic."""
if traffic_to_plot:
start = traffic_to_plot[0].date
stop = start
for t in traffic_to_plot:
d = t.date
if d < start:
start = d
if d > stop:
stop = d
x = start
date_range = [start]
while x != stop:
x += datetime.timedelta(days=1)
date_range.append(x)
else:
date_range = []
debug.onscreen('WARNING: No traffic to plot. Did you put any in the database?')
#Timeplot is designed to take in CSV text files, so build one as a string for each metric:
metric_textfiles = {}
for m in metrics_to_plot:
metric_textfile_strlist = []
append = metric_textfile_strlist.append
for d in date_range:
sd = str(d)
for t in traffic_to_plot:
if t.date == d:
if t.metric == m:
append('%s%s%s' % (sd,',',str(t.count)))
metric_textfiles[m.id] = '\\n'.join(metric_textfile_strlist)
return metric_textfiles
def scan_url(self, url):
USER_AGENT = 'OPMS/1.0 (Ubuntu 10.04; Virtual Server) Django/1.4.0'
request = urllib2.Request(url)
request.add_header('User-Agent', USER_AGENT)
opener = urllib2.build_opener()
time_of_request = datetime.datetime.utcnow()
start = time.time()
request = opener.open(request)
ttfb = time.time() - start
output = request.read()
status = 200 # Presumed because there would be an error otherwise
ttlb = time.time() - start
debug.onscreen("{0}:{1} - TTFB={2} - TTLB={3}".format(time_of_request, url, ttfb, ttlb))
return status, time_of_request, ttfb, ttlb
def event_add(request,event=None, error='', message=''):
"""Adds a new event to the database. Optionally, it may replace the event instead."""
categories = Category.objects.all()
error_fields=[]
try:
widget = bool(request.POST['widget'])
except:
widget = False
if widget == True:
url = request.POST['url']
detail = request.POST['description']
title = request.POST['title']
try:
timestamp = request.POST['timestamp']
datetimestamp = parse(timestamp)
except:
datetimestamp = datetime.datetime.now()
debug.onscreen('WARNING: Widget returned datetime we couldn\'t process. Defaulting to today.')
debug.onscreen('Autocompleting form from widget... ' + url + str(timestamp) + title)
default_event = Event(date=datetimestamp.date(), title=title, detail=detail, category=Category.objects.filter(description='Found on the internet')[0], user_email='')
else:
default_event = Event(date=datetime.date.today(), title='', detail='', category=Category.objects.filter(description='Events')[0], user_email='')
try:
added = bool(request.POST['add'])
except:
added = False
try:
action = request.POST['action']
except:
action = 'add'
if added == True:
try:
new_date = parse(request.POST['date'], dayfirst=True)
except:
error += ' Date invalid or not specified.'
try:
new_detail = request.POST['detail']
if new_detail == '':
error += ' Event text is blank.'
except:
error += ' No event text provided.'
try:
new_title = request.POST['title']
if new_title == '':
error += ' Title is blank.'
except:
error += ' No event title provided.'
try:
new_category = Category.objects.filter(pk=int(request.POST['category_id']))[0] #The [0] is OK since the fact that category_id is a primary key ensures that the array has only length 1.
except:
error += ' Category invalid or nonexistent.'
try:
new_user_email = request.POST['user_email']
if new_user_email == '':
error += ' You haven\'t provided your e-mail address.'
except:
error += ' No user e-mail address provided.'
if error == '':
new_event = Event(date=new_date, title=new_title, detail=new_detail, category=new_category, user_email=new_user_email)
try:
new_event.full_clean()
try:
new_event.save()
message += 'Your event was added to the database.'
except:
error += 'Failed to access the database.'
except ValidationError as ve:
for k in ve.message_dict.keys():
error_fields.append(k)
for m in ve.message_dict[k]:
error += m + ' '
default_event = new_event
if action == 'saveandaddanother' or action == 'add' or error != '':
return render_to_response('feedback/event_add.html',
{'categories': categories,
'error': error,
'added': added,
'message': message,
'error_fields': error_fields,
'event': default_event},
context_instance=RequestContext(request))
elif action == 'save':
return index(request, error=error, message=message)
else:
error += 'Invalid submit action requested.'
return render_to_response('feedback/event_add.html',
{'categories': categories,
'error': error,
'added': added,
'message': message,
'error_fields': error_fields,
'event': default_event},
context_instance=RequestContext(request))
def index(request, error='', message='', tag=None, tag_id=None, comment_id=None, event_id=None, metric_id=None):
"""Display a report on either a given tag, or a default set of comments, events and metrics if a tag is not specified."""
if tag:
metrics_to_plot = Metric.objects.filter(tags=tag)
else:
metrics_to_plot = Metric.objects.filter(source='appleweekly')
traffic_to_plot = []
for metric in metrics_to_plot:
metric_traffic = list(Traffic.objects.filter(metric=metric))
if metric_traffic:
traffic_to_plot.append(metric_traffic)
chart = False
for m in metrics_to_plot:
if m.source == 'appleweekly':
try:
#Import Apple weekly summary metrics, but just for one-time use - don't save in db.
append = traffic_to_plot.append #Avoid re-calling the .append function in the middle of all those loops.
for w in AppleWeeklySummary.merged.all():
for field in AppleWeeklySummary._meta._fields(): #This grabs a list of field objects from the model specified as part of the stats app
if field.verbose_name == m.appleweeklyfield: #Verbose name is specified as ("verbose_name") in stats/models/apple_summary.py
append(Traffic(date=w.week_beginning, count=w.__dict__[field.name], metric=m))
except:
debug.onscreen('WARNING: Can\'t find any Apple summary data. Have you imported it?')
elif m.source == 'itu-collection-chart':
try:
#Add the first chartrecord of the day to traffic_to_plot
dates = []
chartrecords = ItuCollectionChartScan.objects.filter(itucollection=m.itucollection).order_by('date')
for chartrecord in chartrecords:
if chartrecord.date.date() not in dates:
dates.append(chartrecord.date.date())
for date in dates:
chartrecords_day = []
for chartrecord in chartrecords:
if chartrecord.date.date() == date:
chartrecords_day.append(chartrecord)
traffic_to_plot.append(
Traffic(date=date, count=(-1 * chartrecords_day[0].position), metric=m))
chart = True
except:
error += 'Failed to process traffic for an itu-collection-chart.'
elif m.source == 'itu-item-chart':
try:
#Add the first chartrecord of the day to traffic_to_plot
dates = []
chartrecords = ItuItemChartScan.objects.filter(ituitem=m.ituitem).order_by('date')
for chartrecord in chartrecords:
if chartrecord.date.date() not in dates:
dates.append(chartrecord.date.date())
for date in dates:
chartrecords_day = []
for chartrecord in chartrecords:
if chartrecord.date.date() == date:
chartrecords_day.append(chartrecord)
traffic_to_plot.append(
Traffic(date=date, count=(-1 * chartrecords_day[0].position), metric=m))
chart = True
except:
error += 'Failed to process traffic for an itu-item-chart.'
elif m.source =='itu-#tc':
try:
dates_processed = []
for tc_scan in ItuScanLog.objects.filter(mode=2).order_by('time'):
date = tc_scan.time.date()
if date not in dates_processed:
dates_processed.append(date)
tc_count = ItuCollectionChartScan.objects.filter(scanlog=tc_scan,
itucollection__institution=m.ituinstitution).count()
traffic_to_plot.append(Traffic(date=date, count=tc_count, metric=m))
except:
error += 'Failed to process traffic for the # of collections in the top 200.'
elif m.source =='itu-#ti':
try:
dates_processed = []
for ti_scan in ItuScanLog.objects.filter(mode=3).order_by('time'):
date = ti_scan.time.date()
if date not in dates_processed:
dates_processed.append(date)
ti_count = ItuItemChartScan.objects.filter(scanlog=ti_scan,
ituitem__institution=m.ituinstitution).count()
traffic_to_plot.append(Traffic(date=date, count=ti_count, metric=m))
except:
error += 'Failed to process traffic for the # of collections in the top 200.'
#NOTE: We do not need to handle the temporal range of comments and events since this is done automatically by Timeplot.
from_itunes_u = Category.objects.get(description='From iTunes U')
#Create comments in the feedback database if they don't already exist.
for itu_comment in ItuComment.objects.filter(ituinstitution__name = 'Oxford University'):
comment = Comment(
date=itu_comment.date,
time=datetime.time(0,0,0),
source=itu_comment.itucollectionhistorical.name + ' - comment by ' + itu_comment.source,
detail=itu_comment.detail,
user_email='*****@*****.**',
moderated=True,
category=from_itunes_u,
itu_source=itu_comment
)
if Comment.objects.filter(detail=itu_comment.detail).count() > 0:
pass
else:
comment.save()
if tag:
comments_to_plot = Comment.objects.filter(moderated=True,tags=tag)
events_to_plot = Event.objects.filter(moderated=True,tags=tag)
else:
comments_to_plot = Comment.objects.filter(moderated=True)
events_to_plot = Event.objects.filter(moderated=True)
categories_to_plot = []
for category in comments_to_plot.values_list('category').distinct():
categories_to_plot.append(Category.objects.get(id=category[0]))
for category in events_to_plot.values_list('category').distinct():
if Category.objects.get(id=category[0]) not in categories_to_plot:
categories_to_plot.append(Category.objects.get(id=category[0]))
return render_to_response('feedback/index.html', {
'metrics_to_plot': metrics_to_plot,
'metric_textfiles': create_metric_textfiles(traffic_to_plot,metrics_to_plot),
'categories_to_plot': categories_to_plot,
'comments_to_plot': comments_to_plot,
'events': events_to_plot,
'chart': chart,
'error': error,
'message': message,
'tag': tag, 'tag_id': tag_id, 'tags': Tag.objects.all(), 'comment_id': comment_id, 'event_id': event_id, 'metric_id': metric_id,
}, context_instance=RequestContext(request))
def handle_label(self, path, **options):
verbosity = int(options.get('verbosity', 0))
if verbosity > 1:
debug.DEBUG = True
# # Some basic checking
# if not filename.endswith('.txt'):
# raise CommandError("This is not a text (.txt) log file.\n\n")
# Scan directory for files, compare them to names in the existing LogFile list. Import the first X new files.
found_files_list = self._list_files(path)
found_files_list.sort() # Trust the naming conventions to put a sortable date on them
import_file_limit = 21
if len(found_files_list) < import_file_limit:
import_file_limit = len(found_files_list)
# Have this only work on the one file when debug is switched on
if debug.DEBUG:
import_file_limit = 1
print "{0} files have been found. Importing up to {1} of them now.".format(
len(found_files_list),
import_file_limit
)
for filename in found_files_list:
if import_file_limit > 0:
# Reset statistics
self.import_stats['filename'] = filename
self.import_stats['line_counter'] = 0
self.import_stats['line_count'] = 0
self.import_stats['import_starttime'] = datetime.datetime.now(pytz.utc)
self.import_stats['import_startline'] = int(options.get('start_at_line', 1))
# This only needs setting/getting the once per call of this function
logfile_obj, created = self._logfile(filename, 'itu-raw')
if not created:
err_string = "This file has already been imported: ({0})".format(filename)
debug.onscreen(err_string)
continue
import_file_limit -= 1
print "Import of [{0}] started at {1:%Y-%m-%d %H:%M:%S}\n".format(
filename,
datetime.datetime.now(pytz.utc)
)
# Create an error log per import file
debug.errorlog_start(filename + '_import-error.log')
# Send the file off to be parsed
self._parsefile(logfile_obj)
# Final stats output at end of file
try:
self.import_stats['import_duration'] = float((datetime.datetime.now(pytz.utc) - self.import_stats.get('import_starttime')).seconds)
self.import_stats['import_rate'] = float(self.import_stats.get('line_counter')-self.import_stats.get('import_startline')) /\
self.import_stats['import_duration']
except ZeroDivisionError:
self.import_stats['import_rate'] = 0
# Write the error cache to disk
debug.errorlog_stop()
print """
Import finished at {0:%Y-%m-%d %H:%M:%S}
{1:d} Lines parsed over {2:.1f} seconds
Giving a rate of {3:.3f} lines/sec
""".format(
datetime.datetime.now(pytz.utc),
self.import_stats.get('line_counter'),
self.import_stats.get('import_duration'),
self.import_stats.get('import_rate')
)
return None
def _parseline(self, entrydict, logfile_obj):
# # Build the log entry dictionary
# arle = {
# "logfile" : logfile_obj,
# "artist_id" : long(entrydict.get("artist_id")),
# "itunes_id" : long(entrydict.get("itunes_id")),
# "action_type" : self._action_type_validation(entrydict.get("action_type")),
# "title" : entrydict.get("title","Unknown"),
# "url" : entrydict.get("url",""),
# "episode_id" : long(entrydict.get("episode_id",0)),
# "episode_title" : entrydict.get("episode_title",None),
# "episode_type" : entrydict.get("episode_type",None),
# "storefront" : int(entrydict.get("storefront",0)),
# "user_agent" : self._user_agent(entrydict.get("useragent","")),
# "ipaddress" : self._ip_to_domainname(entrydict.get("ip_address",None)),
# "timestamp" : self._parse_timestamp(entrydict.get("timestamp")),
# "user_id" : entrydict.get("user_id","")
# }
debug.onscreen("_parseline():entrydict=" + str(entrydict))
# Build the log entry dictionary
arle = AppleRawLogEntry()
arle.logfile = logfile_obj
try:
arle.artist_id = long(entrydict.get("artist_id"))
except ValueError:
arle.artist_id = -1
except TypeError:
arle.artist_id = -1
try:
arle.itunes_id = long(entrydict.get("itunes_id"))
except ValueError:
arle.itunes_id = -1
except TypeError:
arle.itunes_id = -1
arle.action_type = self._action_type_validation(entrydict.get("action_type"))
arle.title = entrydict.get("title","Unknown")
arle.url = entrydict.get("url","")
try:
arle.episode_id = long(entrydict.get("episode_id"))
except ValueError:
arle.episode_id = None
except TypeError:
arle.episode_id = None
arle.episode_title = entrydict.get("episode_title",None)
arle.episode_type = entrydict.get("episode_type",None)
arle.storefront = self._storefront(entrydict.get("storefront","0"))
arle.user_agent = self._user_agent(entrydict.get("useragent",""))
arle.ipaddress = self._ip_to_domainname(entrydict.get("ip_address",None))
arle.timestamp = self._parse_timestamp(entrydict.get("timestamp"))
arle.user_id = entrydict.get("user_id","")
arle.save(force_insert=True)
# Add to the daily summary dictionary
if self.summary.get("date", None) is None:
self.summary["date"] = "{0:%Y-%m-%d}".format(arle.timestamp)
if arle.action_type == "AutoDownload":
self.summary["auto_download"] = int(self.summary.get("auto_download",0) + 1)
elif arle.action_type == "Browse":
self.summary["browse"] = int(self.summary.get("browse",0) + 1)
elif arle.action_type == "Download":
self.summary["download"] = int(self.summary.get("download",0) + 1)
elif arle.action_type == "DownloadAll":
self.summary["download_all"] = int(self.summary.get("download_all",0) + 1)
elif arle.action_type == "Stream":
self.summary["stream"] = int(self.summary.get("stream",0) + 1)
elif arle.action_type == "Subscribe":
self.summary["subscribe"] = int(self.summary.get("subscribe",0) + 1)
elif arle.action_type == "SubscriptionEnclosure":
self.summary["subscription_enclosure"] = int(self.summary.get("subscription_enclosure",0) + 1)
else:
self.summary["unknown"] = int(self.summary.get("unknown",0) + 1)
return None
