def testKnownText( self ):
""" SdeSkemaScraper : use known data to get known result """
parser = SdeSkemaScraper()
parser.feed( SimpleSkemaData )
parser.close()
self.assertEqual(len( parser.Appointments ), len(SimpleSkemaDataResult))
self.assertEqual(parser.Appointments , SimpleSkemaDataResult)
def main():
myConfig = SkemaPackConfig( SkemaPackConfig_stdin() )
myHtmlGetter = HtmlGetter.htmlGetter()
htmlResponse = myHtmlGetter.getSkemaWithPost(myConfig.get("SkemaScraper","TeacherId"),\
myConfig.get("SkemaScraper","FirstWeek"),\
myConfig.get("SkemaScraper","LastWeek"))
response = htmlResponse.read()
htmlScraper = SdeSkemaScraper(DateFormat = myConfig.get("SkemaScraper","Dateformat"))
htmlScraper.feed(response)
try:
myGmailOutput = GmailOutput_API(myConfig.get("gmail","username"),myConfig.get("gmail","Password"))
except :
#TODO : GmailOutput_API should raise a common exception when gdata throws gdata.service.BasAuthentication
print ( "login to gmail failed - please check username and password")
exit()
for Appointment in htmlScraper.Appointments:
myGmailOutput.addAppointment(Appointment)
return 0
def testDateAndDateFromHours(self):
""" SdeSkemaScraper : Compare 'date' with date from 'hours' """
parser = SdeSkemaScraper(DateFormat = "%d-%m-%Y")
parser.feed( WebPageFromChrome43_43_19lektioner )
parser.close()
for app in parser.Appointments:
self.assertEqual( app['Hours'][0].date(), app['Date'] )
pass
def testWebPageInput5weeks(self):
""" SdeSkemaScraper : use webpage source from webgetter - 5 weeks """
parser = SdeSkemaScraper(DateFormat = "%d-%m-%Y")
parser.feed( WebPageFromHtmlGetter43_48_72lektioner )
parser.close()
i = len(parser.Appointments)
self.assertEqual(i,72)
pass
def testWebPageInput1week_c(self):
""" SdeSkemaScraper : use webpage source from chrome - 1 week """
parser = SdeSkemaScraper(DateFormat = "%d-%m-%Y")
parser.feed( WebPageFromChrome43_43_19lektioner )
parser.close()
i = len(parser.Appointments)
self.assertEqual(i,19)
pass
def testKnownTextBadChars( self ):
""" SdeSkemaScraper : use known data to get known result (including øæå) """
parser = SdeSkemaScraper()
parser.feed( SkemaDataBadChars )
parser.close()
self.assertEqual(len( parser.Appointments ), len(SkemaDataBadCharsResult))
for i in range(0, len( parser.Appointments ) ):
self.assertEqual(parser.Appointments[i] , SkemaDataBadCharsResult[i])
def testSdeSkemaIterator(self):
""" SdeSkemaScraper : test iterator functionality """
parser = SdeSkemaScraper()
parser.feed(SimpleSkemaData)
parser.close()
i = 0
for entry in SdeSkema( SimpleSkemaData ):
self.assertEqual( entry, parser.Appointments[i] )
i = i+1
def testGetSkemaWithPost_1(self):
''' HtmlGetter : do a POST 1 week'''
myLoader = loadWeb.htmlGetter()
try:
self._htmlResponse = myLoader.getSkemaWithPost(3735, 43, 43, 2010)
except IOError:
self.fail("IOError. No internet?")
parser = SdeSkemaScraper(DateFormat = "%d-%m-%Y")
parser.feed( self._htmlResponse.read() )
parser.close()
i = len(parser.Appointments)
self.assertEqual(i,19)
pass
def testGetSkemaWithPost_2(self):
''' HtmlGetter : do a POST 5 weeks '''
# TODO : can we change this into a static test? As it is now it requires net-access and fails if changes are made
myLoader = loadWeb.htmlGetter()
try:
self._htmlResponse = myLoader.getSkemaWithPost(3735, 43, 48, 2010)
except IOError:
self.fail("IOError. No internet?")
parser = SdeSkemaScraper(DateFormat = "%d-%m-%Y")
parser.feed( self._htmlResponse.read() )
parser.close()
i = len(parser.Appointments)
self.assertEqual(i,75)
pass
def main():
myConfig = SkemaPackConfig( SkemaPackConfig_stdin() )
myHtmlGetter = HtmlGetter.htmlGetter()
htmlResponse = myHtmlGetter.getSkemaWithPost(myConfig.get("SkemaScraper","TeacherId"),\
myConfig.get("SkemaScraper","FirstWeek"),\
myConfig.get("SkemaScraper","LastWeek"))
response = htmlResponse.read()
htmlScraper = SdeSkemaScraper(DateFormat = myConfig.get("SkemaScraper","Dateformat"))
htmlScraper.feed(response)
for Appointment in htmlScraper.Appointments:
print Appointment
def testGetSkemaWithPostWithSpecifiedYear(self):
''' HtmlGetter : do a POST 1 week with the year specified'''
myLoader = loadWeb.htmlGetter()
try:
self._htmlResponse = myLoader.getSkemaWithPost(3735, 04, 04, 2011)
except IOError:
self.fail("IOError. No internet?")
parser = SdeSkemaScraper(DateFormat = "%d-%m-%Y")
parser.feed( self._htmlResponse.read() )
parser.close()
fp = open( Filename_pfl_2011_04, "r" )
Filecontent = fp.read()
fp.close()
parser2 = SdeSkemaScraper(DateFormat = "%d-%m-%Y")
parser2.feed( Filecontent )
parser2.close()
self.assertEqual(parser.Appointments, parser2.Appointments)
pass
def ImportSdeSkema( config, ConfigSet = "SkemaScraper" ):
''' Config parameters: TeacherId, FirstWeek, LastWeek, Year, InputDateformat,
Infile (if set, read from file)
'''
# read data from file or net
if config.has_option(ConfigSet, "Infile"):
Data = open(config.get(ConfigSet, "Infile")).read()
else:
myLoader = loadWeb.htmlGetter( config.get(ConfigSet, "Site") )
Data = myLoader.getSkemaWithPost(config.get(ConfigSet, "TeacherId"),
config.get(ConfigSet, "FirstWeek"),
config.get(ConfigSet, "LastWeek"),
config.get(ConfigSet, "Year")).read()
# and process data
parser = SdeSkemaScraper( config.get(ConfigSet, "InputDateformat"),
Teacher = config.get(ConfigSet, "Teacher" ) )
parser.feed( Data )
parser.close()
# TODO: Decide on name for this, Appointments or Events
return parser.Appointments
def testNonTextData( self ):
""" SdeSkemaScraper : use integer instead of text in parser """
parser = SdeSkemaScraper()
self.assertRaises(TypeError, parser.feed, 1)
parser.close()
def testBogusTextData( self ):
""" SdeSkemaScraper : use non-HTML text in parser """
parser = SdeSkemaScraper()
parser.feed("ThisHasNothingToDoWithHtml")
parser.close()
self.assertEqual(parser.Appointments , [])
def testEmptyData( self ):
""" SdeSkemaScraper : use empty text in parser """
parser = SdeSkemaScraper()
parser.feed("")
parser.close()
self.assertEqual(parser.Appointments , [])
