def updatelist(filename):
topsites = wordlist('http://www.google.com/adplanner/static/top1000/', '<a href="http://(.*?)/"target')
f = fsemaphore(filename, 'wb')
f.lockex()
f.write('\n'.join(topsites))
f.close()
return topsites
def test_simple_wordlist_resource_gz(self):
self.assertListEqual(
['', '3', '4', '5'],
wordlist('file://%s' %
resource_filename('tests.resources', 'wordlist.txt.gz'),
strip='^1',
ignore='^2'))
def __getitem__(self, key):
section, option = self._get_option_value(key, True)
value = self.get(section, option)
value = self._parse_value(value)
if option == 'wordlist':
value = wordlist(value)
return value
def readlist(filename):
f = fsemaphore(filename)
f.locksh()
data = wordlist('file://%s' % filename)
f.close()
return dict(
map(lambda x: (int(x[0]), x[1]), map(lambda x: x.split(','), data)))
def test_callable_decompressor(self):
self.assertEqual(
['1', '2', '3', '4', '5'],
wordlist('file://%s' %
resource_filename('tests.resources', 'wordlist.txt.gz'),
decompressor=lambda d: zlib.decompress(
d, 16 + zlib.MAX_WBITS)))
def updatelist(filename):
topsites = wordlist('http://www.google.com/adplanner/static/top1000/',
'<a href="http://(.*?)/"target')
f = fsemaphore(filename, 'wb')
f.lockex()
f.write('\n'.join(topsites))
f.close()
return topsites
def updatelist(filename):
f = fsemaphore(filename, 'wb')
f.lockex()
categories = wordlist('http://sitereview.cwfservice.net/catdesc.jsp?referrer=k9webprotection&host=%3Clocalserver%3E&port=%3Clocalport%3E', '>(\d+) - (.+?)<', ignore=None)
for c in categories:
f.write('%s\n' % ','.join(c))
f.close()
return dict(map(lambda x: (int(x[0]),x[1]), categories))
def updatelist(filename):
topsites = []
f = fsemaphore(filename, 'wb')
f.lockex()
for i in xrange(20):
page = wordlist('http://www.alexa.com/topsites/global;%d' % i, '<a href="/siteinfo/(.+)?">')
topsites += page
f.write('\n'.join(page))
f.write('\n')
f.close()
return topsites
def updatelist(filename):
topsites = []
f = fsemaphore(filename, 'wb')
f.lockex()
for i in xrange(0,20):
page = wordlist('http://www.alexa.com/topsites/global;%d' % i, 'topsites-label">(.*?)</')
topsites += page
f.write('\n'.join(page))
f.write('\n')
f.close()
return topsites
def updatelist(filename):
topsites = []
f = fsemaphore(filename, 'wb')
f.lockex()
for i in xrange(20):
page = wordlist('http://www.alexa.com/topsites/global;%d' % i,
'<a href="/siteinfo/(.+)?">')
topsites += page
f.write('\n'.join(page))
f.write('\n')
f.close()
return topsites
def updatelist(filename):
f = fsemaphore(filename, 'wb')
f.lockex()
categories = wordlist(
'http://sitereview.cwfservice.net/catdesc.jsp?referrer=k9webprotection&host=%3Clocalserver%3E&port=%3Clocalport%3E',
'>(\d+) - (.+?)<',
ignore=None)
for c in categories:
f.write('%s\n' % ','.join(c))
f.close()
return dict(map(lambda x: (int(x[0]), x[1]), categories))
def updatelist(filename):
f = fsemaphore(filename, 'wb')
f.lockex()
ouis = dict(
map(
lambda x: split(r'\s+\(base 16\)\s+', x),
wordlist('http://standards.ieee.org/develop/regauth/oui/oui.txt',
r'([\d\w]{6}\s+\(base 16\)\s+\w.+)\n')))
for o in ouis:
f.write('%s\n' % ','.join([o, ouis[o]]))
f.close()
return ouis
def updatelist(filename):
f = fsemaphore(filename, 'wb')
f.lockex()
ouis = dict(
map(
lambda x: split(r'\s+\(base 16\)\s+', x),
wordlist('http://standards.ieee.org/develop/regauth/oui/oui.txt', r'([\d\w]{6}\s+\(base 16\)\s+\w.+)\n')
)
)
for o in ouis:
f.write('%s\n' % ','.join([o, ouis[o]]))
f.close()
return ouis
def updatelist(filename):
d = None
with fsemaphore(filename, 'wb') as f:
f.lockex()
try:
categories = wordlist(
'http://sitereview.bluecoat.com/rest/categoryList?alpha=true',
loads
)
d = dict([('%02x' % c['num'], c['name']) for c in categories])
f.write(dumps(d))
except Exception, e:
f.close()
os.unlink(tmpfile)
raise e
def readlist(filename):
f = fsemaphore(filename)
f.locksh()
data = wordlist('file://%s' % filename)
f.close()
return data
def readlist(filename):
f = fsemaphore(filename)
f.locksh()
data = wordlist('file://%s' % filename)
f.close()
return data
def readlist(filename):
with fsemaphore(filename) as f:
f.locksh()
data = wordlist('file://%s' % filename, loads)
return data
def test_wordlist_object_return(self):
self.assertEqual(1, wordlist(1))
def test_callable_match(self):
self.assertEqual(
['1', '2', '3', '4', '5'],
wordlist('file://%s' %
resource_filename('tests.resources', 'wordlist.txt'),
match=lambda d: d.decode('utf8').split('\n')))
def test_wordlist_empty_string(self):
self.assertEqual([], wordlist(''))
def readlist(filename):
f = fsemaphore(filename)
f.locksh()
data = wordlist('file://%s' % filename)
f.close()
return dict(map(lambda x: (int(x[0]),x[1]), map(lambda x: x.split(','), data)))
def test_simple_wordlist_resource(self):
self.assertListEqual(
['1', '2', '3', '4', '5'],
wordlist('file://%s' %
resource_filename('tests.resources', 'wordlist.txt')))
