def test_handle_no_policelog(self):
"""
handle non-existing police log
issue 44
2006-03-17"""
if USE_WEB:
issue_num = 44
findpolicelog.scrape_and_populate_db(issue_num, issue_num)
try:
police_log = PoliceLog.objects.get(
issue_number__exact=issue_num)
except PoliceLog.DoesNotExist:
#
# couldn't retrieve db entry
self.assertEquals(0, 1)
else:
#
# issue exists, but no police log
self.assertEquals(police_log.issue_exists, True)
#
# police log url is blank
self.assert_(len(police_log.police_log_url) == 0)
self.assertEquals(police_log.pub_date,
datetime.date(2006, 3, 17))
def test_find_same_issue_twice(self):
"""
try scraping same issue twice, check results in db"""
if USE_WEB:
issue_num = 39
#
# scrape same issue twice
findpolicelog.scrape_and_populate_db(issue_num, issue_num)
findpolicelog.scrape_and_populate_db(issue_num, issue_num)
#
# should only be one entry in the db
self.assertEquals(1, len(PoliceLog.objects.all()))
def test_find_two_issues(self):
"""
find police log in 2 issues"""
if USE_WEB:
issue_num = 39
#
# scrape same issue twice
findpolicelog.scrape_and_populate_db(issue_num, issue_num + 1)
#
# should be 2 entries in the db
issue_list = PoliceLog.objects.all()
self.assertEquals(2, len(issue_list))
def test_find_single_log(self):
"""
get single issue's policelog info, populate db"""
if USE_WEB:
issue_num = 39
findpolicelog.scrape_and_populate_db(issue_num, issue_num)
try:
police_log = PoliceLog.objects.get(
issue_number__exact=issue_num)
except PoliceLog.DoesNotExist:
#
# couldn't retrieve db entry
self.assertEquals(0, 1)
else:
self.assertEquals(police_log.issue_exists, True)
self.assertEquals(police_log.pub_date,
datetime.date(2006, 2, 10))
url = 'http://www.mv-voice.com/story.php?story_id=985'
self.assertEquals(police_log.police_log_url, url)
def test_nonexistent_issue0(self):
"""
user asks for issue 0"""
##################3
# mvv website chokes on issue 0
issue_num = 0
#
#
# expect zero entries
try:
findpolicelog.scrape_and_populate_db(issue_num, issue_num)
police_log = PoliceLog.objects.get(issue_number__exact=issue_num)
#
# expect exception
except PoliceLog.DoesNotExist:
pass
else:
self.assert_(1)
def test_download(self):
"""
try to download existing url"""
if USE_WEB:
issue_num = 39
#
findpolicelog.scrape_and_populate_db(issue_num, issue_num)
#
# expect 1 entry
try:
police_log = PoliceLog.objects.get(
issue_number__exact=issue_num)
#
# expect exception
except PoliceLog.DoesNotExist:
self.assert_(0)
#
# download url
fname = findpolicelog.download_police_log(issue_num, TEST_DATA_OUT)
#
# file was downloaded?
target_filename = findpolicelog.create_download_filename(issue_num)
self.assert_(
os.path.exists(os.sep.join([TEST_DATA_OUT, target_filename])))
#
# delete file
try:
os.remove(target_filename)
except:
pass
def test_nonexistent_issue1(self):
"""
user asks for issue 1"""
if USE_WEB:
######################3
# issue 1 doesn't exist either
issue_num = 1
#
findpolicelog.scrape_and_populate_db(issue_num, issue_num)
#
# expect 1 entry with no pub date
try:
police_log = PoliceLog.objects.get(
issue_number__exact=issue_num)
#
# expect exception
except PoliceLog.DoesNotExist:
self.assert_(0)
self.assertEquals(police_log.issue_exists, False)
self.assertEquals(police_log.pub_date, None)
def list_issues(request):
"""List all Voice issues in the database
"""
my_logger.debug('request.method=(%s)' % request.method)
my_logger.debug('request.POST=(%s)' % request.POST)
#
# default status message
message='Hello!\n'
if (request.method != 'POST'):
#
# method will be 'GET' if we didn't get here via form submit
pass
else:
try:
start_issue=int(request.POST['start_index'])
stop_issue=int(request.POST['stop_index'])
except ValueError:
#
# invalid strings passed as start_index and stop_index
start_issue=0;
stop_issue=0;
#
# identify which button was pressed
k=request.POST.keys()
my_logger.debug(k)
if 'find' in k:
#
# user wants to find logs
message = message + ('find (%d..%d)\n' % (start_issue,stop_issue))
my_logger.debug('find police logs')
try:
findpolicelog.scrape_and_populate_db(start_issue,stop_issue)
except PoliceLog.DoesNotExist:
message=message + 'Illegal index range (%d,%d)\n' % (start_issue,stop_issue)
elif 'download' in k:
#
# user wants to download police log
message = message + ('download (%d..%d)\n' % (start_issue,stop_issue))
my_logger.debug('download police logs')
findpolicelog.download_police_logs(start_issue,stop_issue)
elif 'parse' in k:
#
# user wants to parse downloaded log
message = message + ('parse (%d..%d)' % (start_issue,stop_issue))
parsepolicelog.parse_log_and_populate_db(start_issue,stop_issue)
else:
message = message + 'Unidentified button\n'
#
# list issues in db
issue_list = PoliceLog.objects.all().order_by('-issue_number')
template=loader.get_template('mvvscrape/list_issues.html')
# determine whether associated crime reports exist
# iterate through issue_list
# for each issue, locate if any corresponding crimeporets in database
# if so, then modify issue object
for index in range(0,len(issue_list)):
#
# any crime rpeorts associated with this issue number?
issue_number = issue_list[index].issue_number
crimereport_list = CrimeReport.objects.filter(policelog__issue_number__exact=issue_number)
#
# bind a new attribute to the issue called "has_crime_reports"
if len(crimereport_list):
issue_list[index].has_crime_reports=str(issue_list[index].issue_number)
else:
issue_list[index].has_crime_reports=None
my_logger.debug('issue[%d]: %d %s' % (index,
issue_list[index].issue_number,
issue_list[index].has_crime_reports))
#
# build context dictionary
context = Context({'issue_list': issue_list,'message':message})
return HttpResponse(template.render(context))