def main(): myConfig = SkemaPackConfig( SkemaPackConfig_stdin() ) myHtmlGetter = HtmlGetter.htmlGetter() htmlResponse = myHtmlGetter.getSkemaWithPost(myConfig.get("SkemaScraper","TeacherId"),\ myConfig.get("SkemaScraper","FirstWeek"),\ myConfig.get("SkemaScraper","LastWeek")) response = htmlResponse.read() htmlScraper = SdeSkemaScraper(DateFormat = myConfig.get("SkemaScraper","Dateformat")) htmlScraper.feed(response) try: myGmailOutput = GmailOutput_API(myConfig.get("gmail","username"),myConfig.get("gmail","Password")) except : #TODO : GmailOutput_API should raise a common exception when gdata throws gdata.service.BasAuthentication print ( "login to gmail failed - please check username and password") exit() for Appointment in htmlScraper.Appointments: myGmailOutput.addAppointment(Appointment) return 0
def testKnownText( self ): """ SdeSkemaScraper : use known data to get known result """ parser = SdeSkemaScraper() parser.feed( SimpleSkemaData ) parser.close() self.assertEqual(len( parser.Appointments ), len(SimpleSkemaDataResult)) self.assertEqual(parser.Appointments , SimpleSkemaDataResult)
def testDateAndDateFromHours(self): """ SdeSkemaScraper : Compare 'date' with date from 'hours' """ parser = SdeSkemaScraper(DateFormat = "%d-%m-%Y") parser.feed( WebPageFromChrome43_43_19lektioner ) parser.close() for app in parser.Appointments: self.assertEqual( app['Hours'][0].date(), app['Date'] ) pass
def testWebPageInput5weeks(self): """ SdeSkemaScraper : use webpage source from webgetter - 5 weeks """ parser = SdeSkemaScraper(DateFormat = "%d-%m-%Y") parser.feed( WebPageFromHtmlGetter43_48_72lektioner ) parser.close() i = len(parser.Appointments) self.assertEqual(i,72) pass
def testWebPageInput1week_c(self): """ SdeSkemaScraper : use webpage source from chrome - 1 week """ parser = SdeSkemaScraper(DateFormat = "%d-%m-%Y") parser.feed( WebPageFromChrome43_43_19lektioner ) parser.close() i = len(parser.Appointments) self.assertEqual(i,19) pass
def testKnownTextBadChars( self ): """ SdeSkemaScraper : use known data to get known result (including øæå) """ parser = SdeSkemaScraper() parser.feed( SkemaDataBadChars ) parser.close() self.assertEqual(len( parser.Appointments ), len(SkemaDataBadCharsResult)) for i in range(0, len( parser.Appointments ) ): self.assertEqual(parser.Appointments[i] , SkemaDataBadCharsResult[i])
def testSdeSkemaIterator(self): """ SdeSkemaScraper : test iterator functionality """ parser = SdeSkemaScraper() parser.feed(SimpleSkemaData) parser.close() i = 0 for entry in SdeSkema( SimpleSkemaData ): self.assertEqual( entry, parser.Appointments[i] ) i = i+1
def testGetSkemaWithPost_1(self): ''' HtmlGetter : do a POST 1 week''' myLoader = loadWeb.htmlGetter() try: self._htmlResponse = myLoader.getSkemaWithPost(3735, 43, 43, 2010) except IOError: self.fail("IOError. No internet?") parser = SdeSkemaScraper(DateFormat = "%d-%m-%Y") parser.feed( self._htmlResponse.read() ) parser.close() i = len(parser.Appointments) self.assertEqual(i,19) pass
def testGetSkemaWithPost_2(self): ''' HtmlGetter : do a POST 5 weeks ''' # TODO : can we change this into a static test? As it is now it requires net-access and fails if changes are made myLoader = loadWeb.htmlGetter() try: self._htmlResponse = myLoader.getSkemaWithPost(3735, 43, 48, 2010) except IOError: self.fail("IOError. No internet?") parser = SdeSkemaScraper(DateFormat = "%d-%m-%Y") parser.feed( self._htmlResponse.read() ) parser.close() i = len(parser.Appointments) self.assertEqual(i,75) pass
def main(): myConfig = SkemaPackConfig( SkemaPackConfig_stdin() ) myHtmlGetter = HtmlGetter.htmlGetter() htmlResponse = myHtmlGetter.getSkemaWithPost(myConfig.get("SkemaScraper","TeacherId"),\ myConfig.get("SkemaScraper","FirstWeek"),\ myConfig.get("SkemaScraper","LastWeek")) response = htmlResponse.read() htmlScraper = SdeSkemaScraper(DateFormat = myConfig.get("SkemaScraper","Dateformat")) htmlScraper.feed(response) for Appointment in htmlScraper.Appointments: print Appointment
def testGetSkemaWithPostWithSpecifiedYear(self): ''' HtmlGetter : do a POST 1 week with the year specified''' myLoader = loadWeb.htmlGetter() try: self._htmlResponse = myLoader.getSkemaWithPost(3735, 04, 04, 2011) except IOError: self.fail("IOError. No internet?") parser = SdeSkemaScraper(DateFormat = "%d-%m-%Y") parser.feed( self._htmlResponse.read() ) parser.close() fp = open( Filename_pfl_2011_04, "r" ) Filecontent = fp.read() fp.close() parser2 = SdeSkemaScraper(DateFormat = "%d-%m-%Y") parser2.feed( Filecontent ) parser2.close() self.assertEqual(parser.Appointments, parser2.Appointments) pass
def ImportSdeSkema( config, ConfigSet = "SkemaScraper" ): ''' Config parameters: TeacherId, FirstWeek, LastWeek, Year, InputDateformat, Infile (if set, read from file) ''' # read data from file or net if config.has_option(ConfigSet, "Infile"): Data = open(config.get(ConfigSet, "Infile")).read() else: myLoader = loadWeb.htmlGetter( config.get(ConfigSet, "Site") ) Data = myLoader.getSkemaWithPost(config.get(ConfigSet, "TeacherId"), config.get(ConfigSet, "FirstWeek"), config.get(ConfigSet, "LastWeek"), config.get(ConfigSet, "Year")).read() # and process data parser = SdeSkemaScraper( config.get(ConfigSet, "InputDateformat"), Teacher = config.get(ConfigSet, "Teacher" ) ) parser.feed( Data ) parser.close() # TODO: Decide on name for this, Appointments or Events return parser.Appointments
def testBogusTextData( self ): """ SdeSkemaScraper : use non-HTML text in parser """ parser = SdeSkemaScraper() parser.feed("ThisHasNothingToDoWithHtml") parser.close() self.assertEqual(parser.Appointments , [])
def testEmptyData( self ): """ SdeSkemaScraper : use empty text in parser """ parser = SdeSkemaScraper() parser.feed("") parser.close() self.assertEqual(parser.Appointments , [])