def test_sortHostnames(self):
parser = URLParser()
hostnames = {"ccc.com": 2, "bbb.com": 3, "aaa.com": 2}
sortedNames = parser.sortHostnames(hostnames)
self.assertEquals(sortedNames[0], ('bbb.com', 3))
self.assertEquals(sortedNames[1], ('aaa.com', 2))
self.assertEquals(sortedNames[2], ('ccc.com', 2))
def test_parseURLs(self):
parser = URLParser()
hostnames = parser.parseURLs("urls.txt")
self.assertEqual(len(hostnames), 5)
self.assertEqual(hostnames["twitter.com"], 3)
self.assertEqual(hostnames["abcnews.go.com"], 2)
self.assertEqual(hostnames["google.co.uk"], 1)
self.assertEqual(hostnames["newsfeed.time.com"], 1)
self.assertEqual(hostnames["world.time.com"], 1)
def __init__(self, pType, lines, frame, order=True): Thread.__init__(self) self.pType = pType self.lines = lines self.parser = URLParser() self.frame = frame self.order = order self.start() #start automatically
class unittest_URLParser(unittest.TestCase):
def setUp(self, logFile='log/URLParser.log'):
self.logger = Logger(logFile, 'DEBUG', 'Sub')
self.logger = self.logger.getLogger()
self.parser1 = URLParser('am://test.cmc.ec.gc.ca:24901')
self.parser2 = URLParser('wmo://192.108.62.1:24901')
self.parser3 = URLParser('file://localhost//apps/px/operator')
self.parser4 = URLParser(
'amqp://*****:*****@test.cmc.ec.gc.ca//data')
self.parser5 = URLParser('amis://test.cmc.ec.gc.ca:24901')
self.parser6 = URLParser('ftp://*****:*****@192.108.62.1//data')
def test_URLParser(self):
self.assertEqual(
self.parser1.parse(),
('am', None, None, None, 'test.cmc.ec.gc.ca', '24901'))
self.assertEqual(
self.parser1.join(self.parser1.protocol, self.parser1.path,
self.parser1.user, self.parser1.passwd,
self.parser1.host, self.parser1.port),
'am://test.cmc.ec.gc.ca:24901')
self.assertEqual(self.parser2.parse(),
('wmo', None, None, None, '192.108.62.1', '24901'))
self.assertEqual(
self.parser3.parse(),
('file', '//apps/px/operator', None, None, 'localhost', None))
self.assertEqual(
self.parser4.parse(),
('amqp', '//data', 'guest', 'guestpw', 'test.cmc.ec.gc.ca', None))
self.assertEqual(
self.parser4.join(self.parser4.protocol, self.parser4.path,
self.parser4.user, self.parser4.passwd,
self.parser4.host, self.parser4.port),
'amqp://*****:*****@test.cmc.ec.gc.ca//data')
self.assertEqual(
self.parser5.parse(),
('amis', None, None, None, 'test.cmc.ec.gc.ca', '24901'))
self.assertEqual(
self.parser6.parse(),
('ftp', '//data', 'guest', 'guestpw', '192.108.62.1', None))
self.assertEqual(
self.parser6.join(self.parser6.protocol, self.parser6.path,
self.parser6.user, self.parser6.passwd,
self.parser6.host, self.parser6.port),
'ftp://*****:*****@192.108.62.1//data')
def setUp(self,logFile='log/URLParser.log'):
self.logger = Logger(logFile, 'DEBUG', 'Sub')
self.logger = self.logger.getLogger()
self.parser1 = URLParser('am://test.cmc.ec.gc.ca:24901')
self.parser2 = URLParser('wmo://192.108.62.1:24901')
self.parser3 = URLParser('file://localhost//apps/px/operator')
self.parser4 = URLParser('amqp://*****:*****@test.cmc.ec.gc.ca//data')
self.parser5 = URLParser('amis://test.cmc.ec.gc.ca:24901')
self.parser6 = URLParser('ftp://*****:*****@192.108.62.1//data')
class BatotoThread(Thread): def __init__(self, pType, lines, frame, order=True): Thread.__init__(self) self.pType = pType self.lines = lines self.parser = URLParser() self.frame = frame self.order = order self.start() #start automatically def run(self): wx.CallAfter(self.frame.SetLocked, True) if self.pType == 0: for line in self.lines: if self.order: self.ParseFirstThread(line) else: self.ParseLastThread(line) elif self.pType == 1: self.ParseLastThread(self.lines) else: self.ParseFirstThread(self.lines) wx.CallAfter(self.frame.SetLocked, False) def ParseFirstThread(self, line): self.ParseLine(line) wx.CallAfter(self.frame.UiClear, False) def ParseLastThread(self, line): self.ParseLine(line); wx.CallAfter(self.frame.UiClear, True) def ParseLine(self, line): if self.parser.testURL(line): global HOME_DIR self.parser.downloadFromURL(line, HOME_DIR, self.frame)
class unittest_URLParser(unittest.TestCase):
def setUp(self,logFile='log/URLParser.log'):
self.logger = Logger(logFile, 'DEBUG', 'Sub')
self.logger = self.logger.getLogger()
self.parser1 = URLParser('am://test.cmc.ec.gc.ca:24901')
self.parser2 = URLParser('wmo://192.108.62.1:24901')
self.parser3 = URLParser('file://localhost//apps/px/operator')
self.parser4 = URLParser('amqp://*****:*****@test.cmc.ec.gc.ca//data')
self.parser5 = URLParser('amis://test.cmc.ec.gc.ca:24901')
self.parser6 = URLParser('ftp://*****:*****@192.108.62.1//data')
def test_URLParser(self):
self.assertEqual(self.parser1.parse(), ('am', None, None, None, 'test.cmc.ec.gc.ca', '24901'))
self.assertEqual(self.parser1.join(self.parser1.protocol, self.parser1.path, self.parser1.user, self.parser1.passwd, self.parser1.host, self.parser1.port),'am://test.cmc.ec.gc.ca:24901')
self.assertEqual(self.parser2.parse(), ('wmo', None, None, None, '192.108.62.1', '24901'))
self.assertEqual(self.parser3.parse(), ('file', '//apps/px/operator', None, None, 'localhost', None))
self.assertEqual(self.parser4.parse(), ('amqp', '//data', 'guest', 'guestpw', 'test.cmc.ec.gc.ca', None))
self.assertEqual(self.parser4.join(self.parser4.protocol, self.parser4.path, self.parser4.user, self.parser4.passwd, self.parser4.host, self.parser4.port),'amqp://*****:*****@test.cmc.ec.gc.ca//data')
self.assertEqual(self.parser5.parse(), ('amis', None, None, None, 'test.cmc.ec.gc.ca', '24901'))
self.assertEqual(self.parser6.parse(), ('ftp', '//data', 'guest', 'guestpw', '192.108.62.1', None))
self.assertEqual(self.parser6.join(self.parser6.protocol, self.parser6.path, self.parser6.user, self.parser6.passwd, self.parser6.host, self.parser6.port),'ftp://*****:*****@192.108.62.1//data')
def readConfigFile(self, filePath, currentDir, currentFileOption):
def isTrue(s):
if s == 'True' or s == 'true' or s == 'yes' or s == 'on' or \
s == 'Yes' or s == 'YES' or s == 'TRUE' or s == 'ON' or \
s == '1' or s == 'On' :
return True
else:
return False
def stringToOctal(string):
if len(string) != 3:
return 0644
else:
return int(string[0]) * 64 + int(string[1]) * 8 + int(
string[2])
try:
config = open(filePath, 'r')
except:
(type, value, tb) = sys.exc_info()
print("Type: %s, Value: %s" % (type, value))
return
for line in config.readlines():
words = line.split()
if (len(words) >= 2 and not re.compile('^[ \t]*#').search(line)):
try:
if words[0] == 'accept':
cmask = re.compile(words[1])
cFileOption = currentFileOption
if len(words) > 2: cFileOption = words[2]
self.masks.append(
(words[1], currentDir, cFileOption, cmask, True))
elif words[0] == 'reject':
cmask = re.compile(words[1])
self.masks.append((words[1], currentDir,
currentFileOption, cmask, False))
elif words[0] == 'imask':
self.masks_deprecated.append(
(words[1], currentDir, currentFileOption))
elif words[0] == 'emask':
self.masks_deprecated.append((words[1], ))
elif words[0] == 'directory':
currentDir = words[1]
elif words[0] == 'filename':
currentFileOption = words[1]
elif words[0] == 'include':
fileName = PXPaths.TX_CONF + words[1]
self.readConfigFile(fileName, currentDir,
currentFileOption)
elif words[0] == 'destination':
self.url = words[1]
urlParser = URLParser(words[1])
(self.protocol, currentDir, self.user, self.passwd,
self.host, self.port) = urlParser.parse()
if len(words) > 2:
currentFileOption = words[2]
elif words[0] == 'validation':
self.validation = isTrue(words[1])
elif words[0] == 'purgeAlias':
self.purgeAliases[words[1]] = words[2]
elif words[0] == 'purge':
try:
self.purge = self.purgeAliases[words[1]]
self.parsePurgeInstructions(
self.purgeAliases[words[1]])
except:
self.purge = words[1]
self.parsePurgeInstructions(words[1])
elif words[0] == 'noduplicates':
self.nodups = isTrue(words[1])
elif words[0] == 'patternMatching':
self.patternMatching = isTrue(words[1])
elif words[0] == 'keepAlive':
self.keepAlive = isTrue(words[1])
elif words[0] == 'mtime':
self.mtime = int(words[1])
elif words[0] == 'sorter':
self.sorter = words[1]
elif words[0] == 'type':
self.type = words[1]
elif words[0] == 'protocol':
self.protocol = words[1]
elif words[0] == 'maxLength':
self.maxLength = int(words[1])
elif words[0] == 'host':
self.host = words[1]
elif words[0] == 'port':
self.port = int(words[1])
elif words[0] == 'user':
self.user = words[1]
elif words[0] == 'password':
self.passwd = words[1]
elif words[0] == 'ssh_keyfile':
self.ssh_keyfile = words[1]
elif words[0] == 'batch':
self.batch = int(words[1])
elif words[0] == 'kbytes_ps':
self.kbytes_ps = int(words[1]) + 0.0
elif words[0] == 'cache_size':
self.cache_size = int(words[1])
elif words[0] == 'debug' and isTrue(words[1]):
self.debug = True
elif words[0] == 'timeout':
self.timeout = int(words[1])
elif words[0] == 'chmod':
self.chmod = int(words[1])
elif words[0] == 'timeout_send':
self.timeout_send = int(words[1])
elif words[0] == 'lock':
self.lock = words[1]
elif words[0] == 'ftp_mode':
self.ftp_mode = words[1]
elif words[0] == 'dir_pattern':
self.dir_pattern = isTrue(words[1])
elif words[0] == 'binary':
self.binary = isTrue(words[1])
elif words[0] == 'dir_mkdir':
self.dir_mkdir = isTrue(words[1])
elif words[0] == 'destfn_script':
self.destfn_execfile = words[1]
elif words[0] == 'dx_script':
self.dx_execfile = words[1]
elif words[0] == 'fx_script':
self.fx_execfile = words[1]
elif words[0] == 'send_script':
self.send_execfile = words[1]
elif words[0] == 'am_dest_thread':
if self.am_dest_thread == None:
self.am_dest_thread = {}
self.am_dest_thread[words[1]] = int(words[2])
elif words[0] == 'logrotate':
if words[1].isdigit():
self.logger.setBackupCount(int(words[1]))
# AMQP
elif words[0] == 'exchange_content':
self.exchange_content = words[1]
elif words[0] == 'exchange_key':
self.exchange_key = words[1]
elif words[0] == 'exchange_name':
self.exchange_name = words[1]
elif words[0] == 'exchange_realm':
self.exchange_realm = words[1]
elif words[0] == 'exchange_type':
if words[1] in [
'fanout', 'direct', 'topic', 'headers'
]:
self.exchange_type = words[1]
else:
self.logger.error("Problem with exchange_type %s" %
words[1])
except:
self.logger.error(
"Problem with this line (%s) in configuration file of client %s"
% (words, self.name))
#if not self.validation:
# self.sorter = 'None' # Must be a string because eval will be subsequently applied to this
config.close()
def readConfig(self,filePath):
def isTrue(s):
if s == 'True' or s == 'true' or s == 'yes' or s == 'on' or \
s == 'Yes' or s == 'YES' or s == 'TRUE' or s == 'ON' or \
s == '1' or s == 'On' :
return True
else:
return False
try:
config = open(filePath, 'r')
except:
(type, value, tb) = sys.exc_info()
print("Type: %s, Value: %s" % (type, value))
return
# current dir and filename could eventually be used
# for file renaming and perhaps file move (like a special receiver/dispatcher)
currentDir = '.' # just to preserve consistency with client : unused in source for now
currentFileOption = 'WHATFN' # just to preserve consistency with client : unused in source for now
currentTransformation = 'GIFFY' # Default transformation for tmasks
currentLST = None # a list consisting of one directory followed one or more file patterns
for line in config.readlines():
words = line.split()
if (len(words) >= 2 and not re.compile('^[ \t]*#').search(line)):
try:
if words[0] == 'extension':
if len(words[1].split(':')) != 5:
self.logger.error("Extension (%s) for source %s has wrong number of fields" % (words[1], self.name))
else:
self.extension = ':' + words[1]
self.extension = self.extension.replace('-NAME',self.name)
elif words[0] == 'arrival_extension':
if len(words[1].split(':')) != 5:
self.logger.error("arrival_extension (%s) for source %s has wrong number of fields" % (words[1], self.name))
else:
self.arrival_extension = ':' + words[1]
self.arrival_extension = self.arrival_extension.replace('-NAME',self.name)
elif words[0] == 'accept':
cmask = re.compile(words[1])
self.masks.append((words[1], currentDir, currentFileOption,cmask,True))
elif words[0] == 'reject':
cmask = re.compile(words[1])
self.masks.append((words[1], currentDir, currentFileOption,cmask,False))
elif words[0] == 'routemask': self.routemask = isTrue(words[1])
elif words[0] == 'routing_version': self.routing_version = int(words[1])
elif words[0] == 'noduplicates': self.nodups = isTrue(words[1])
elif words[0] == 'imask': self.masks_deprecated.append((words[1], currentDir, currentFileOption))
elif words[0] == 'emask': self.masks_deprecated.append((words[1],))
elif words[0] == 'timask': self.tmasks.append((words[1], currentTransformation))
elif words[0] == 'temask': self.tmasks.append((words[1],))
elif words[0] == 'transformation': currentTransformation = words[1]
elif words[0] == 'batch': self.batch = int(words[1])
elif words[0] == 'cache_size': self.cache_size = int(words[1])
elif words[0] == 'bulletin_type': self.bulletin_type = words[1]
elif words[0] == 'type': self.type = words[1]
elif words[0] == 'port': self.port = int(words[1])
elif words[0] == 'AddSMHeader' and isTrue(words[1]): self.addSMHeader = True
elif words[0] == 'addStationInFilename' : self.addStationInFilename = isTrue(words[1])
elif words[0] == 'patternMatching': self.patternMatching = isTrue(words[1])
elif words[0] == 'clientsPatternMatching': self.clientsPatternMatching = isTrue(words[1])
elif words[0] == 'validation' and isTrue(words[1]): self.validation = True
elif words[0] == 'keepAlive': self.keepAlive = isTrue(words[1])
elif words[0] == 'debug' and isTrue(words[1]): self.debug = True
elif words[0] == 'mtime': self.mtime = int(words[1])
elif words[0] == 'sorter': self.sorter = words[1]
elif words[0] == 'header': self.headers.append(words[1])
elif words[0] == 'hours': self.issue_hours.append(words[1])
elif words[0] == 'primary': self.issue_primary.append(words[1])
elif words[0] == 'cycle': self.issue_cycle.append(words[1])
elif words[0] == 'feed': self.feeds.append(words[1])
elif words[0] == 'routingTable': self.routingTable = words[1]
elif words[0] == 'fx_script': self.fx_execfile = words[1]
elif words[0] == 'lx_script': self.lx_execfile = words[1]
elif words[0] == 'pull_script': self.pull_execfile = words[1]
elif words[0] == 'arrival':
if self.mapEnteteDelai == None : self.mapEnteteDelai = {}
self.mapEnteteDelai[words[1]] = (int(words[2]), int(words[3]))
elif words[0] == 'logrotate':
if words[1].isdigit():
self.logger.setBackupCount(int(words[1]))
# options for pull
elif words[0] == 'directory':
currentDir = words[1]
currentLST = []
# permit directory duplications but warn
for lst in self.pulls :
if lst[0] == currentDir :
currentLST = lst
break
if len(currentLST) != 0 :
self.logger.warning("This directory appears twice %s" % currentDir)
self.logger.warning("Please correct your config")
continue
# normal directory addition
currentLST.append( currentDir )
self.pulls.append( currentLST )
elif words[0] == 'get':
currentFilePattern = words[1]
currentLST.append(currentFilePattern)
elif words[0] == 'destination':
self.url = words[1]
urlParser = URLParser(words[1])
(self.protocol, currentDir, self.user, self.passwd, self.host, self.port) = urlParser.parse()
if len(words) > 2:
currentFileOption = words[2]
currentLST = []
currentLST.append( currentDir )
self.pulls.append( currentLST )
elif words[0] == 'protocol': self.protocol = words[1]
elif words[0] == 'host': self.host = words[1]
elif words[0] == 'user': self.user = words[1]
elif words[0] == 'password': self.passwd = words[1]
elif words[0] == 'ssh_keyfile': self.ssh_keyfile = words[1]
elif words[0] == 'timeout_get': self.timeout_get = int(words[1])
elif words[0] == 'ftp_mode': self.ftp_mode = words[1]
elif words[0] == 'pull_sleep': self.pull_sleep = int(words[1])
elif words[0] == 'pull_wait': self.pull_wait = int(words[1])
elif words[0] == 'delete': self.delete = isTrue(words[1])
elif words[0] == 'pull_prefix': self.pull_prefix = words[1]
# AMQP
elif words[0] == 'exchange_key': self.exchange_key = words[1]
elif words[0] == 'exchange_name': self.exchange_name = words[1]
elif words[0] == 'exchange_realm': self.exchange_realm = words[1]
elif words[0] == 'exchange_type':
if words[1] in ['fanout','direct','topic','headers'] :
self.exchange_type = words[1]
else :
self.logger.error("Problem with exchange_type %s" % words[1])
# options for collector
if self.type == 'collector' :
if words[0] == 'aaxx' : self.aaxx = words[1].split(',')
if words[0] == 'metar' : self.metar = words[1].split(',')
elif words[0] == 'taf' : self.taf = words[1].split(',')
elif words[0] == 'history': self.history = int(words[1])
elif words[0] == 'future' : self.future = int(words[1])
elif words[0] == 'issue' :
if words[1] == 'all' :
lst = []
lst.append(words[1])
self.issue_hours.append(lst)
else :
lst = words[1].split(",")
self.issue_hours.append( lst )
self.issue_primary.append( int(words[2]) )
self.issue_cycle.append( int(words[3]) )
except:
self.logger.error("Problem with this line (%s) in configuration file of source %s" % (words, self.name))
config.close()
if len(self.masks) > 0 : self.patternMatching = True
if len(self.masks_deprecated) > 0 : self.patternMatching = True
self.logger.debug("Configuration file of source %s has been read" % (self.name))
def readConfig(self, filePath):
def isTrue(s):
if s == 'True' or s == 'true' or s == 'yes' or s == 'on' or \
s == 'Yes' or s == 'YES' or s == 'TRUE' or s == 'ON' or \
s == '1' or s == 'On' :
return True
else:
return False
try:
config = open(filePath, 'r')
except:
(type, value, tb) = sys.exc_info()
print("Type: %s, Value: %s" % (type, value))
return
# current dir and filename could eventually be used
# for file renaming and perhaps file move (like a special receiver/dispatcher)
currentDir = '.' # just to preserve consistency with client : unused in source for now
currentFileOption = 'WHATFN' # just to preserve consistency with client : unused in source for now
currentTransformation = 'GIFFY' # Default transformation for tmasks
currentLST = None # a list consisting of one directory followed one or more file patterns
for line in config.readlines():
words = line.split()
if (len(words) >= 2 and not re.compile('^[ \t]*#').search(line)):
try:
if words[0] == 'extension':
if len(words[1].split(':')) != 5:
self.logger.error(
"Extension (%s) for source %s has wrong number of fields"
% (words[1], self.name))
else:
self.extension = ':' + words[1]
self.extension = self.extension.replace(
'-NAME', self.name)
elif words[0] == 'arrival_extension':
if len(words[1].split(':')) != 5:
self.logger.error(
"arrival_extension (%s) for source %s has wrong number of fields"
% (words[1], self.name))
else:
self.arrival_extension = ':' + words[1]
self.arrival_extension = self.arrival_extension.replace(
'-NAME', self.name)
elif words[0] == 'accept':
cmask = re.compile(words[1])
self.masks.append((words[1], currentDir,
currentFileOption, cmask, True))
elif words[0] == 'reject':
cmask = re.compile(words[1])
self.masks.append((words[1], currentDir,
currentFileOption, cmask, False))
elif words[0] == 'routemask':
self.routemask = isTrue(words[1])
elif words[0] == 'routing_version':
self.routing_version = int(words[1])
elif words[0] == 'noduplicates':
self.nodups = isTrue(words[1])
elif words[0] == 'imask':
self.masks_deprecated.append(
(words[1], currentDir, currentFileOption))
elif words[0] == 'emask':
self.masks_deprecated.append((words[1], ))
elif words[0] == 'timask':
self.tmasks.append((words[1], currentTransformation))
elif words[0] == 'temask':
self.tmasks.append((words[1], ))
elif words[0] == 'transformation':
currentTransformation = words[1]
elif words[0] == 'batch':
self.batch = int(words[1])
elif words[0] == 'cache_size':
self.cache_size = int(words[1])
elif words[0] == 'bulletin_type':
self.bulletin_type = words[1]
elif words[0] == 'type':
self.type = words[1]
elif words[0] == 'port':
self.port = int(words[1])
elif words[0] == 'AddSMHeader' and isTrue(words[1]):
self.addSMHeader = True
elif words[0] == 'addStationInFilename':
self.addStationInFilename = isTrue(words[1])
elif words[0] == 'patternMatching':
self.patternMatching = isTrue(words[1])
elif words[0] == 'clientsPatternMatching':
self.clientsPatternMatching = isTrue(words[1])
elif words[0] == 'validation' and isTrue(words[1]):
self.validation = True
elif words[0] == 'keepAlive':
self.keepAlive = isTrue(words[1])
elif words[0] == 'debug' and isTrue(words[1]):
self.debug = True
elif words[0] == 'mtime':
self.mtime = int(words[1])
elif words[0] == 'sorter':
self.sorter = words[1]
elif words[0] == 'header':
self.headers.append(words[1])
elif words[0] == 'hours':
self.issue_hours.append(words[1])
elif words[0] == 'primary':
self.issue_primary.append(words[1])
elif words[0] == 'cycle':
self.issue_cycle.append(words[1])
elif words[0] == 'feed':
self.feeds.append(words[1])
elif words[0] == 'routingTable':
self.routingTable = words[1]
elif words[0] == 'fx_script':
self.fx_execfile = words[1]
elif words[0] == 'lx_script':
self.lx_execfile = words[1]
elif words[0] == 'pull_script':
self.pull_execfile = words[1]
elif words[0] == 'vip':
self.vip = words[1]
elif words[0] == 'arrival':
if self.mapEnteteDelai == None:
self.mapEnteteDelai = {}
self.mapEnteteDelai[words[1]] = (int(words[2]),
int(words[3]))
elif words[0] == 'logrotate':
if words[1].isdigit():
self.logger.setBackupCount(int(words[1]))
# options for pull
elif words[0] == 'directory':
currentDir = words[1]
currentLST = []
# permit directory duplications but warn
for lst in self.pulls:
if lst[0] == currentDir:
currentLST = lst
break
if len(currentLST) != 0:
self.logger.warning(
"This directory appears twice %s" % currentDir)
self.logger.warning("Please correct your config")
continue
# normal directory addition
currentLST.append(currentDir)
self.pulls.append(currentLST)
elif words[0] == 'get':
currentFilePattern = words[1]
currentLST.append(currentFilePattern)
elif words[0] == 'destination':
self.url = words[1]
urlParser = URLParser(words[1])
(self.protocol, currentDir, self.user, self.passwd,
self.host, self.port) = urlParser.parse()
if len(words) > 2:
currentFileOption = words[2]
currentLST = []
currentLST.append(currentDir)
self.pulls.append(currentLST)
elif words[0] == 'protocol':
self.protocol = words[1]
elif words[0] == 'host':
self.host = words[1]
elif words[0] == 'user':
self.user = words[1]
elif words[0] == 'password':
self.passwd = words[1]
elif words[0] == 'ssh_keyfile':
self.ssh_keyfile = words[1]
elif words[0] == 'timeout_get':
self.timeout_get = int(words[1])
elif words[0] == 'ftp_mode':
self.ftp_mode = words[1]
elif words[0] == 'pull_sleep':
self.pull_sleep = int(words[1])
elif words[0] == 'pull_wait':
self.pull_wait = int(words[1])
elif words[0] == 'delete':
self.delete = isTrue(words[1])
elif words[0] == 'pull_prefix':
self.pull_prefix = words[1]
# AMQP
elif words[0] == 'exchange_key':
self.exchange_key = words[1]
elif words[0] == 'exchange_name':
self.exchange_name = words[1]
elif words[0] == 'exchange_realm':
self.exchange_realm = words[1]
elif words[0] == 'exchange_type':
if words[1] in [
'fanout', 'direct', 'topic', 'headers'
]:
self.exchange_type = words[1]
else:
self.logger.error("Problem with exchange_type %s" %
words[1])
# options for collector
if self.type == 'collector':
if words[0] == 'aaxx': self.aaxx = words[1].split(',')
if words[0] == 'metar':
self.metar = words[1].split(',')
elif words[0] == 'taf':
self.taf = words[1].split(',')
elif words[0] == 'history':
self.history = int(words[1])
elif words[0] == 'future':
self.future = int(words[1])
elif words[0] == 'issue':
if words[1] == 'all':
lst = []
lst.append(words[1])
self.issue_hours.append(lst)
else:
lst = words[1].split(",")
self.issue_hours.append(lst)
self.issue_primary.append(int(words[2]))
self.issue_cycle.append(int(words[3]))
except:
self.logger.error(
"Problem with this line (%s) in configuration file of source %s"
% (words, self.name))
config.close()
if len(self.masks) > 0: self.patternMatching = True
if len(self.masks_deprecated) > 0: self.patternMatching = True
self.logger.debug("Configuration file of source %s has been read" %
(self.name))
def setUp(self):
self.parser = URLParser()
class TestURLParser(unittest.TestCase):
def setUp(self):
self.parser = URLParser()
def testDownload(self):
my_url = "http://eapeyton.com"
with Downloader() as dler:
dler.uncache(my_url)
webpage = dler.download(my_url)
self.assertEqual(webpage[:15], """<!DOCTYPE html>""")
def testDownloadCache(self):
my_url = "http://eapeyton.com"
with Downloader() as dler:
dler.uncache(my_url)
dler.cache[my_url] = "Be cool Honey Bunny!"
webpage = dler.download("http://eapeyton.com")
self.assertEqual(webpage, "Be cool Honey Bunny!")
def testComposeLeagueURL(self):
url = self.parser.composeURL(leagueId=1015919)
self.assertEqual(url, "http://games.espn.go.com/ffl/leagueoffice?leagueId=1015919")
url = self.parser.composeURL(leagueId=5555555)
self.assertEqual(url, "http://games.espn.go.com/ffl/leagueoffice?leagueId=5555555")
def testComposeTeamURL(self):
url = self.parser.composeURL(teamId=4,leagueId=1015919)
self.assertEqual(url,"http://games.espn.go.com/ffl/clubhouse?leagueId=1015919&teamId=4")
url = self.parser.composeURL(leagueId=5555555, teamId=12)
self.assertEqual(url,"http://games.espn.go.com/ffl/clubhouse?leagueId=5555555&teamId=12")
def testRegexTeam(self):
self.parser.parse("http://games.espn.go.com/ffl/clubhouse?leagueId=1015919&teamId=1&seasonId=2013")
self.assertEqual(self.parser.league.id, 1015919)
self.assertEqual(self.parser.team.id, 1)
self.assertIn(2013, self.parser.league.seasons)
def testRegexSeason(self):
self.parser.parse("http://games.espn.go.com/ffl/leagueoffice?leagueId=1015919&seasonId=2012")
self.assertEqual(self.parser.league.id, 1015919)
self.assertEqual(self.parser.season.year, 2012)
def testParseNoYear(self):
self.parser.parse("http://games.espn.go.com/ffl/leagueoffice?leagueId=1020911")
self.assertEqual(self.parser.league.id, 1020911)
self.assertEqual(self.parser.season.year, 2013)
def testParsePlayers(self):
self.parser.parse("http://games.espn.go.com/ffl/clubhouse?leagueId=1015919&teamId=1&seasonId=2012")
team = self.parser.team
self.assertEqual(team.id, 1)
self.assertHasSequoyahPlayers(team)
def assertHasSequoyahPlayers(self, team):
self.assertHasPlayer(team, 11237, "Matt Ryan")
self.assertHasPlayer(team, 11289, "Ray Rice")
self.assertHasPlayer(team, 14885, "Doug Martin")
self.assertHasPlayer(team, 9705, "Brandon Marshall")
self.assertHasPlayer(team, 10447, "Calvin Johnson")
self.assertHasPlayer(team, 13232, "Jimmy Graham")
self.assertHasPlayer(team, 2578, "Reggie Wayne")
self.assertHasPlayer(team, 60015, "Dolphins D/ST")
self.assertHasPlayer(team, 3504, "Shayne Graham")
self.assertHasPlayer(team, 11238, "Darren McFadden")
self.assertHasPlayer(team, 13204, "Ryan Matthews")
self.assertHasPlayer(team, 60007, "Broncos D/ST")
self.assertHasPlayer(team, 11439, "Pierre Garcon")
self.assertHasPlayer(team, 9613, "DeAngelo Willioms")
def assertHasPlayer(self,team,id,name):
player = Player(id,name)
self.assertIn(player, team.players)
def testParseSeason(self):
self.parser.parse("http://games.espn.go.com/ffl/leagueoffice?leagueId=408631&seasonId=2013")
season = self.parser.season
self.assertEqual(season.year, 2013)
self.assertHasTeam(season, 1, "Nick Burke")
self.assertHasTeam(season, 2, "Jeff Siegel")
self.assertHasTeam(season, 3, "Nick Macie")
self.assertHasTeam(season, 4, "D Duncanson")
self.assertHasTeam(season, 5, "Chris Nachmias")
self.assertHasTeam(season, 6, "Chris Thorne")
self.assertHasTeam(season, 7, "ryan davis")
self.assertHasTeam(season, 8, "Vishnu Iyengar")
self.assertHasTeam(season, 9, "vishaak ravi")
self.assertHasTeam(season, 10, "Eric Peyton")
def testParseSeason2(self):
self.parser.parse("http://games.espn.go.com/ffl/leagueoffice?leagueId=1015919&seasonId=2012")
season = self.parser.season
self.assertHasTeam(season, 1, "Eric Peyton")
team = season.teams[1]
self.assertHasSequoyahPlayers(team)
def testParsePlayers2(self):
self.parser.parse("http://games.espn.go.com/ffl/clubhouse?leagueId=1015919&teamId=1&seasonId=2012")
season = self.parser.season
self.assertEqual(season.year, 2012)
self.assertEqual(season.league.id, 1015919)
self.assertHasSequoyahPlayers(self.parser.team)
self.assertHasTeam(season, 1, "Eric Peyton")
self.assertHasTeam(season, 2, "Eddie Peyton")
self.assertHasTeam(season, 3, "Mala Morjaria")
self.assertHasTeam(season, 10, "Rikin Marfatia")
def assertHasTeam(self,season,id,owner):
team = Team(id, owner=owner)
self.assertIn(team, season.teams.values())
@unittest.skip("Impossible right now because dicts are serialized in no particular order...")
def testParseToJSON(self):
self.maxDiff = None
team = self.parser.parse_team("http://games.espn.go.com/ffl/clubhouse?leagueId=1015919&teamId=1&seasonId=2012")
key = open('TeamJSONKey.json','r')
self.assertEqual(team.to_JSON(), key.read())
key.close()
def readConfigFile(self,filePath,currentDir,currentFileOption):
def isTrue(s):
if s == 'True' or s == 'true' or s == 'yes' or s == 'on' or \
s == 'Yes' or s == 'YES' or s == 'TRUE' or s == 'ON' or \
s == '1' or s == 'On' :
return True
else:
return False
def stringToOctal(string):
if len(string) != 3:
return 0644
else:
return int(string[0])*64 + int(string[1])*8 + int(string[2])
try:
config = open(filePath, 'r')
except:
(type, value, tb) = sys.exc_info()
print("Type: %s, Value: %s" % (type, value))
return
for line in config.readlines():
words = line.split()
if (len(words) >= 2 and not re.compile('^[ \t]*#').search(line)):
try:
if words[0] == 'accept':
cmask = re.compile(words[1])
cFileOption = currentFileOption
if len(words) > 2: cFileOption = words[2]
self.masks.append((words[1], currentDir, cFileOption, cmask, True))
elif words[0] == 'reject':
cmask = re.compile(words[1])
self.masks.append((words[1], currentDir, currentFileOption, cmask, False))
elif words[0] == 'imask': self.masks_deprecated.append((words[1], currentDir, currentFileOption))
elif words[0] == 'emask': self.masks_deprecated.append((words[1],))
elif words[0] == 'directory': currentDir = words[1]
elif words[0] == 'filename': currentFileOption = words[1]
elif words[0] == 'include':
fileName = PXPaths.TX_CONF + words[1]
self.readConfigFile(fileName,currentDir,currentFileOption)
elif words[0] == 'destination':
self.url = words[1]
urlParser = URLParser(words[1])
(self.protocol, currentDir, self.user, self.passwd, self.host, self.port) = urlParser.parse()
if len(words) > 2:
currentFileOption = words[2]
elif words[0] == 'validation': self.validation = isTrue(words[1])
elif words[0] == 'purgeAlias':
self.purgeAliases[words[1]] = words[2]
elif words[0] == 'purge':
try:
self.purge = self.purgeAliases[words[1]]
self.parsePurgeInstructions(self.purgeAliases[words[1]])
except:
self.purge = words[1]
self.parsePurgeInstructions(words[1])
elif words[0] == 'noduplicates': self.nodups = isTrue(words[1])
elif words[0] == 'patternMatching': self.patternMatching = isTrue(words[1])
elif words[0] == 'keepAlive': self.keepAlive = isTrue(words[1])
elif words[0] == 'mtime': self.mtime = int(words[1])
elif words[0] == 'sorter': self.sorter = words[1]
elif words[0] == 'type': self.type = words[1]
elif words[0] == 'protocol': self.protocol = words[1]
elif words[0] == 'maxLength': self.maxLength = int(words[1])
elif words[0] == 'host': self.host = words[1]
elif words[0] == 'port': self.port = int(words[1])
elif words[0] == 'user': self.user = words[1]
elif words[0] == 'password': self.passwd = words[1]
elif words[0] == 'ssh_keyfile': self.ssh_keyfile = words[1]
elif words[0] == 'batch': self.batch = int(words[1])
elif words[0] == 'kbytes_ps': self.kbytes_ps = int(words[1]) + 0.0
elif words[0] == 'cache_size': self.cache_size = int(words[1])
elif words[0] == 'debug' and isTrue(words[1]): self.debug = True
elif words[0] == 'timeout': self.timeout = int(words[1])
elif words[0] == 'chmod': self.chmod = int(words[1])
elif words[0] == 'timeout_send': self.timeout_send = int(words[1])
elif words[0] == 'lock': self.lock = words[1]
elif words[0] == 'ftp_mode': self.ftp_mode = words[1]
elif words[0] == 'dir_pattern': self.dir_pattern = isTrue(words[1])
elif words[0] == 'binary': self.binary = isTrue(words[1])
elif words[0] == 'dir_mkdir': self.dir_mkdir = isTrue(words[1])
elif words[0] == 'destfn_script': self.destfn_execfile = words[1]
elif words[0] == 'dx_script': self.dx_execfile = words[1]
elif words[0] == 'fx_script': self.fx_execfile = words[1]
elif words[0] == 'send_script': self.send_execfile = words[1]
elif words[0] == 'am_dest_thread':
if self.am_dest_thread == None : self.am_dest_thread = {}
self.am_dest_thread[words[1]] = int(words[2])
elif words[0] == 'logrotate':
if words[1].isdigit():
self.logger.setBackupCount(int(words[1]))
# AMQP
elif words[0] == 'exchange_content': self.exchange_content = words[1]
elif words[0] == 'exchange_key': self.exchange_key = words[1]
elif words[0] == 'exchange_name': self.exchange_name = words[1]
elif words[0] == 'exchange_realm': self.exchange_realm = words[1]
elif words[0] == 'exchange_type':
if words[1] in ['fanout','direct','topic','headers'] :
self.exchange_type = words[1]
else :
self.logger.error("Problem with exchange_type %s" % words[1])
except:
self.logger.error("Problem with this line (%s) in configuration file of client %s" % (words, self.name))
#if not self.validation:
# self.sorter = 'None' # Must be a string because eval will be subsequently applied to this
config.close()
import argparse
from URLParser import URLParser
parser = argparse.ArgumentParser(
description="Parse hostnames from a text file.")
parser.add_argument('file', help="A file containing hostnames")
args = parser.parse_args()
parser = URLParser()
hostnames = parser.parseURLs(args.file)
for hostname, count in parser.sortHostnames(hostnames):
print(count, hostname)
def setUp(self, logFile='log/URLParser.log'):
self.logger = Logger(logFile, 'DEBUG', 'Sub')
self.logger = self.logger.getLogger()
self.parser1 = URLParser('am://test.cmc.ec.gc.ca:24901')
self.parser2 = URLParser('wmo://192.108.62.1:24901')
self.parser3 = URLParser('file://localhost//apps/px/operator')
self.parser4 = URLParser(
'amqp://*****:*****@test.cmc.ec.gc.ca//data')
self.parser5 = URLParser('amis://test.cmc.ec.gc.ca:24901')
self.parser6 = URLParser('ftp://*****:*****@192.108.62.1//data')
