def test_real_path_relative(self):
"""test relative path
Args:
None
Return
None
"""
root = os.path.dirname(os.path.realpath("."))
test_path1 = "./test/relative/path"
test_path2 = "test/relative/path"
result1 = utils.realpath(test_path1)
result2 = utils.realpath(test_path2)
self.assertEqual(result1, os.path.join(root, test_path1))
self.assertEqual(result2, os.path.join(root, test_path2))
def test_real_path_relative(self):
"""test relative path
Args:
None
Return
None
"""
root = os.path.dirname(os.path.realpath("."))
test_path1 = "./test/relative/path"
test_path2 = "test/relative/path"
result1 = utils.realpath(test_path1)
result2 = utils.realpath(test_path2)
self.assertEqual(result1, os.path.join(root, test_path1))
self.assertEqual(result2, os.path.join(root, test_path2))
def handle(self, task):
"""Fetching routine
Args:
task Task object
Return:
None
"""
history_lock = lock.Lock.retrive_lock()
history_lock.acquire()
if task.url in self.__history:
logging.info("Url has been fetched: {url}".format(url=task.url))
history_lock.release()
return
self.__history.append(task.url)
history_lock.release()
if task.depth > self.__max_dep:
raise SpiderException(
"Not a valid task: {task}".format(task=str(task)))
time.sleep(self.__frequency)
fetched_page = page.Page(task.url, 5)
try:
fetched_page.hydrate()
except urllib2.HTTPError as e:
logging.error("HTTP ERROR {url}: {error}".format(url=task.url,
error=str(e)))
return
except urllib2.URLError as e:
logging.error("Url ERROR {url}: {error}".format(url=task.url,
error=str(e)))
return
if task.depth < self.__max_dep:
self.__add_task(fetched_page, task.depth)
imgs = fetched_page.get_resource_url(self.__image_suffix)
if len(imgs) == 0:
return
path = os.path.join(utils.realpath(self.__output_dir),
utils.url2path(fetched_page.url))
try:
if not os.path.isfile(path):
output_file = open(path, "w")
else:
output_file = open(path, "a")
except IOError as e:
logging.error("Can't open file {path}: {error}".format(path=path,
error=e[1]))
return
for img in imgs:
url = self.__fix_up_url(img, fetched_page)
output_file.write(url + "\n")
output_file.close()
def test_real_path_absolute(self):
"""test absolute path
Args:
None
Return
None
"""
root = os.path.dirname(os.path.realpath("."))
test_path = "/home/users/shechenglu/test/absolute/path"
result3 = utils.realpath(test_path)
self.assertEqual(result3, "/home/users/shechenglu/test/absolute/path")
def test_real_path_absolute(self):
"""test absolute path
Args:
None
Return
None
"""
root = os.path.dirname(os.path.realpath("."))
test_path = "/home/users/shechenglu/test/absolute/path"
result3 = utils.realpath(test_path)
self.assertEqual(result3, "/home/users/shechenglu/test/absolute/path")
def handle(self, task):
"""Fetching routine
Args:
task Task object
Return:
None
"""
history_lock = lock.Lock.retrive_lock()
history_lock.acquire()
if task.url in self.__history:
logging.info("Url has been fetched: {url}".format(url=task.url))
history_lock.release()
return
self.__history.append(task.url)
history_lock.release()
if task.depth > self.__max_dep:
raise SpiderException("Not a valid task: {task}".format(task=str(task)))
time.sleep(self.__frequency)
fetched_page = page.Page(task.url, 5)
try:
fetched_page.hydrate()
except urllib2.HTTPError as e:
logging.error("HTTP ERROR {url}: {error}"
.format(url=task.url, error=str(e)))
return
except urllib2.URLError as e:
logging.error("Url ERROR {url}: {error}"
.format(url=task.url, error=str(e)))
return
if task.depth < self.__max_dep:
self.__add_task(fetched_page, task.depth)
imgs = fetched_page.get_resource_url(self.__image_suffix)
if len(imgs) == 0:
return
path = os.path.join(utils.realpath(self.__output_dir),
utils.url2path(fetched_page.url))
try:
if not os.path.isfile(path):
output_file = open(path, "w")
else:
output_file = open(path, "a")
except IOError as e:
logging.error("Can't open file {path}: {error}"
.format(path=path, error=e[1]))
return
for img in imgs:
url = self.__fix_up_url(img, fetched_page)
output_file.write(url + "\n")
output_file.close()
def add_seed_tasks(url_seeds_file):
"""Add the 0 depth urls to the task queue
Args:
None
Return:
None
"""
if not os.path.isfile(utils.realpath(url_seeds_file)):
logging.error("Invalid seeds file, quit")
sys.exit(1)
with open(url_seeds_file, "r") as seeds_file:
for url in seeds_file:
fetch_task = task.Task(url, 0)
taskqueue.TaskQueue.retrive_queue().put((0, fetch_task), True, 10)
def add_seed_tasks(url_seeds_file):
"""Add the 0 depth urls to the task queue
Args:
None
Return:
None
"""
if not os.path.isfile(utils.realpath(url_seeds_file)):
logging.error("Invalid seeds file, quit")
sys.exit(1)
with open(url_seeds_file, "r") as seeds_file:
for url in seeds_file:
fetch_task = task.Task(url, 0)
taskqueue.TaskQueue.retrive_queue().put((0, fetch_task), True, 10)
def __unicode__(self):
un = os.uname()
debinfo = u''
shellpath = utils.realpath('/bin/sh')
init = utils.get_init_system()
locinfo = []
langsetting = os.environ.get('LANG', 'C')
allsetting = os.environ.get('LC_ALL', '')
for setting in ('LANG', 'LC_CTYPE'):
if setting == 'LANG':
env = langsetting
else:
env = '%s (charmap=%s)' % (os.environ.get(setting, langsetting), commands.getoutput("locale charmap"))
if allsetting and env:
env = "%s (ignored: LC_ALL set to %s)" % (env, allsetting)
else:
env = allsetting or env
locinfo.append('%s=%s' % (setting, env))
locinfo = ', '.join(locinfo)
ph = getattr(self, 'pseudoheaders', None)
if ph:
headers = u'\n'.join(ph)+u'\n'
else:
headers = u''
version = getattr(self, 'version', None)
if version:
headers += u'Version: %s\n' % version
body = getattr(self, 'body', u'')
body = body.decode('utf8')
# add NEWBIELINE only if it's less than advanced and the package is not
# one of the specials (f.e. those with a dedicated function) also
# thinking about those systems that don't have 'specials' dict
if self.mode < utils.MODE_ADVANCED and self.package not in \
debbugs.SYSTEMS[self.system].get('specials', {}).keys():
body = utils.NEWBIELINE+u'\n\n'+body
elif not body:
body = u'\n'
if self.issource:
reportto = 'Source'
else:
reportto = 'Package'
if not self.followup:
for (attr, name) in dict(severity='Severity',
justification='Justification',
tags='Tags',
filename='File').iteritems():
a = getattr(self, attr, None)
if a:
headers += u'%s: %s\n' % (name, a)
report = u"%s: %s\n%s\n" % (reportto, self.package, headers)
else:
report = "Followup-For: Bug #%d\n%s: %s\n%s\n" % (
self.followup, reportto, self.package, headers)
infofunc = debbugs.SYSTEMS[self.system].get('infofunc', debbugs.generic_infofunc)
if infofunc:
debinfo += infofunc()
if un[0] == 'GNU':
# Use uname -v on Hurd
uname_string = un[3]
else:
kern = un[0]
if kern.startswith('GNU/'):
kern = kern[4:]
uname_string = '%s %s' % (kern, un[2])
if kern == 'Linux':
kinfo = []
if 'SMP' in un[3]:
cores = utils.get_cpu_cores()
if cores > 1:
kinfo += ['SMP w/%d CPU cores' % cores]
elif cores == 1:
kinfo += ['SMP w/1 CPU core']
if 'PREEMPT' in un[3]:
kinfo += ['PREEMPT']
if kinfo:
uname_string = '%s (%s)' % (uname_string, '; '.join(kinfo))
if uname_string:
debinfo += u'Kernel: %s\n' % uname_string
if locinfo:
debinfo += u'Locale: %s\n' % locinfo
if shellpath != '/bin/sh':
debinfo += u'Shell: /bin/sh linked to %s\n' % shellpath
if init:
debinfo += u'Init: %s\n' % init
# Don't include system info for certain packages
if self.sysinfo:
report = u"%s%s%s\n-- System Information:\n%s" % (report, body, self.incfiles, debinfo)
else:
report = u"%s%s%s" % (report, body, self.incfiles)
if hasattr(self, 'depinfo'):
report += self.depinfo
if hasattr(self, 'confinfo'):
report += self.confinfo
# add debsums output to the bug report
if self.debsumsoutput:
report += u"\n-- debsums errors found:\n%s\n" % self.debsumsoutput
return report
def test_realpath_sym(self):
self.assertEqual(realpath(self.linkpath, "/etc"), "../tmp")
def test_realpath_rel4(self):
self.assertEqual(realpath("a/b", "/etc"), "../tmp/a/b")
def test_realpath_rel3(self):
self.assertEqual(realpath("..", "/etc"), "..")
def test_realpath_rel2(self):
self.assertEqual(realpath(".", "/etc"), "../tmp")
def test_realpath_rel_norm(self):
self.assertEqual(realpath("test/../../../../", "/"), ".")
def test_realpath_rel(self):
self.assertEqual(realpath("test/", "/"), "tmp/test")
def test_realpath_abs(self):
self.assertEqual(realpath("test"), "/tmp/test")
