def test_handle_no_policelog(self): """ handle non-existing police log issue 44 2006-03-17""" if USE_WEB: issue_num = 44 findpolicelog.scrape_and_populate_db(issue_num, issue_num) try: police_log = PoliceLog.objects.get( issue_number__exact=issue_num) except PoliceLog.DoesNotExist: # # couldn't retrieve db entry self.assertEquals(0, 1) else: # # issue exists, but no police log self.assertEquals(police_log.issue_exists, True) # # police log url is blank self.assert_(len(police_log.police_log_url) == 0) self.assertEquals(police_log.pub_date, datetime.date(2006, 3, 17))
def test_find_same_issue_twice(self): """ try scraping same issue twice, check results in db""" if USE_WEB: issue_num = 39 # # scrape same issue twice findpolicelog.scrape_and_populate_db(issue_num, issue_num) findpolicelog.scrape_and_populate_db(issue_num, issue_num) # # should only be one entry in the db self.assertEquals(1, len(PoliceLog.objects.all()))
def test_find_two_issues(self): """ find police log in 2 issues""" if USE_WEB: issue_num = 39 # # scrape same issue twice findpolicelog.scrape_and_populate_db(issue_num, issue_num + 1) # # should be 2 entries in the db issue_list = PoliceLog.objects.all() self.assertEquals(2, len(issue_list))
def test_find_single_log(self): """ get single issue's policelog info, populate db""" if USE_WEB: issue_num = 39 findpolicelog.scrape_and_populate_db(issue_num, issue_num) try: police_log = PoliceLog.objects.get( issue_number__exact=issue_num) except PoliceLog.DoesNotExist: # # couldn't retrieve db entry self.assertEquals(0, 1) else: self.assertEquals(police_log.issue_exists, True) self.assertEquals(police_log.pub_date, datetime.date(2006, 2, 10)) url = 'http://www.mv-voice.com/story.php?story_id=985' self.assertEquals(police_log.police_log_url, url)
def test_nonexistent_issue0(self): """ user asks for issue 0""" ##################3 # mvv website chokes on issue 0 issue_num = 0 # # # expect zero entries try: findpolicelog.scrape_and_populate_db(issue_num, issue_num) police_log = PoliceLog.objects.get(issue_number__exact=issue_num) # # expect exception except PoliceLog.DoesNotExist: pass else: self.assert_(1)
def test_download(self): """ try to download existing url""" if USE_WEB: issue_num = 39 # findpolicelog.scrape_and_populate_db(issue_num, issue_num) # # expect 1 entry try: police_log = PoliceLog.objects.get( issue_number__exact=issue_num) # # expect exception except PoliceLog.DoesNotExist: self.assert_(0) # # download url fname = findpolicelog.download_police_log(issue_num, TEST_DATA_OUT) # # file was downloaded? target_filename = findpolicelog.create_download_filename(issue_num) self.assert_( os.path.exists(os.sep.join([TEST_DATA_OUT, target_filename]))) # # delete file try: os.remove(target_filename) except: pass
def test_nonexistent_issue1(self): """ user asks for issue 1""" if USE_WEB: ######################3 # issue 1 doesn't exist either issue_num = 1 # findpolicelog.scrape_and_populate_db(issue_num, issue_num) # # expect 1 entry with no pub date try: police_log = PoliceLog.objects.get( issue_number__exact=issue_num) # # expect exception except PoliceLog.DoesNotExist: self.assert_(0) self.assertEquals(police_log.issue_exists, False) self.assertEquals(police_log.pub_date, None)
def list_issues(request): """List all Voice issues in the database """ my_logger.debug('request.method=(%s)' % request.method) my_logger.debug('request.POST=(%s)' % request.POST) # # default status message message='Hello!\n' if (request.method != 'POST'): # # method will be 'GET' if we didn't get here via form submit pass else: try: start_issue=int(request.POST['start_index']) stop_issue=int(request.POST['stop_index']) except ValueError: # # invalid strings passed as start_index and stop_index start_issue=0; stop_issue=0; # # identify which button was pressed k=request.POST.keys() my_logger.debug(k) if 'find' in k: # # user wants to find logs message = message + ('find (%d..%d)\n' % (start_issue,stop_issue)) my_logger.debug('find police logs') try: findpolicelog.scrape_and_populate_db(start_issue,stop_issue) except PoliceLog.DoesNotExist: message=message + 'Illegal index range (%d,%d)\n' % (start_issue,stop_issue) elif 'download' in k: # # user wants to download police log message = message + ('download (%d..%d)\n' % (start_issue,stop_issue)) my_logger.debug('download police logs') findpolicelog.download_police_logs(start_issue,stop_issue) elif 'parse' in k: # # user wants to parse downloaded log message = message + ('parse (%d..%d)' % (start_issue,stop_issue)) parsepolicelog.parse_log_and_populate_db(start_issue,stop_issue) else: message = message + 'Unidentified button\n' # # list issues in db issue_list = PoliceLog.objects.all().order_by('-issue_number') template=loader.get_template('mvvscrape/list_issues.html') # determine whether associated crime reports exist # iterate through issue_list # for each issue, locate if any corresponding crimeporets in database # if so, then modify issue object for index in range(0,len(issue_list)): # # any crime rpeorts associated with this issue number? issue_number = issue_list[index].issue_number crimereport_list = CrimeReport.objects.filter(policelog__issue_number__exact=issue_number) # # bind a new attribute to the issue called "has_crime_reports" if len(crimereport_list): issue_list[index].has_crime_reports=str(issue_list[index].issue_number) else: issue_list[index].has_crime_reports=None my_logger.debug('issue[%d]: %d %s' % (index, issue_list[index].issue_number, issue_list[index].has_crime_reports)) # # build context dictionary context = Context({'issue_list': issue_list,'message':message}) return HttpResponse(template.render(context))