def test_real_path_relative(self): """test relative path Args: None Return None """ root = os.path.dirname(os.path.realpath(".")) test_path1 = "./test/relative/path" test_path2 = "test/relative/path" result1 = utils.realpath(test_path1) result2 = utils.realpath(test_path2) self.assertEqual(result1, os.path.join(root, test_path1)) self.assertEqual(result2, os.path.join(root, test_path2))
def handle(self, task): """Fetching routine Args: task Task object Return: None """ history_lock = lock.Lock.retrive_lock() history_lock.acquire() if task.url in self.__history: logging.info("Url has been fetched: {url}".format(url=task.url)) history_lock.release() return self.__history.append(task.url) history_lock.release() if task.depth > self.__max_dep: raise SpiderException( "Not a valid task: {task}".format(task=str(task))) time.sleep(self.__frequency) fetched_page = page.Page(task.url, 5) try: fetched_page.hydrate() except urllib2.HTTPError as e: logging.error("HTTP ERROR {url}: {error}".format(url=task.url, error=str(e))) return except urllib2.URLError as e: logging.error("Url ERROR {url}: {error}".format(url=task.url, error=str(e))) return if task.depth < self.__max_dep: self.__add_task(fetched_page, task.depth) imgs = fetched_page.get_resource_url(self.__image_suffix) if len(imgs) == 0: return path = os.path.join(utils.realpath(self.__output_dir), utils.url2path(fetched_page.url)) try: if not os.path.isfile(path): output_file = open(path, "w") else: output_file = open(path, "a") except IOError as e: logging.error("Can't open file {path}: {error}".format(path=path, error=e[1])) return for img in imgs: url = self.__fix_up_url(img, fetched_page) output_file.write(url + "\n") output_file.close()
def test_real_path_absolute(self): """test absolute path Args: None Return None """ root = os.path.dirname(os.path.realpath(".")) test_path = "/home/users/shechenglu/test/absolute/path" result3 = utils.realpath(test_path) self.assertEqual(result3, "/home/users/shechenglu/test/absolute/path")
def handle(self, task): """Fetching routine Args: task Task object Return: None """ history_lock = lock.Lock.retrive_lock() history_lock.acquire() if task.url in self.__history: logging.info("Url has been fetched: {url}".format(url=task.url)) history_lock.release() return self.__history.append(task.url) history_lock.release() if task.depth > self.__max_dep: raise SpiderException("Not a valid task: {task}".format(task=str(task))) time.sleep(self.__frequency) fetched_page = page.Page(task.url, 5) try: fetched_page.hydrate() except urllib2.HTTPError as e: logging.error("HTTP ERROR {url}: {error}" .format(url=task.url, error=str(e))) return except urllib2.URLError as e: logging.error("Url ERROR {url}: {error}" .format(url=task.url, error=str(e))) return if task.depth < self.__max_dep: self.__add_task(fetched_page, task.depth) imgs = fetched_page.get_resource_url(self.__image_suffix) if len(imgs) == 0: return path = os.path.join(utils.realpath(self.__output_dir), utils.url2path(fetched_page.url)) try: if not os.path.isfile(path): output_file = open(path, "w") else: output_file = open(path, "a") except IOError as e: logging.error("Can't open file {path}: {error}" .format(path=path, error=e[1])) return for img in imgs: url = self.__fix_up_url(img, fetched_page) output_file.write(url + "\n") output_file.close()
def add_seed_tasks(url_seeds_file): """Add the 0 depth urls to the task queue Args: None Return: None """ if not os.path.isfile(utils.realpath(url_seeds_file)): logging.error("Invalid seeds file, quit") sys.exit(1) with open(url_seeds_file, "r") as seeds_file: for url in seeds_file: fetch_task = task.Task(url, 0) taskqueue.TaskQueue.retrive_queue().put((0, fetch_task), True, 10)
def __unicode__(self): un = os.uname() debinfo = u'' shellpath = utils.realpath('/bin/sh') init = utils.get_init_system() locinfo = [] langsetting = os.environ.get('LANG', 'C') allsetting = os.environ.get('LC_ALL', '') for setting in ('LANG', 'LC_CTYPE'): if setting == 'LANG': env = langsetting else: env = '%s (charmap=%s)' % (os.environ.get(setting, langsetting), commands.getoutput("locale charmap")) if allsetting and env: env = "%s (ignored: LC_ALL set to %s)" % (env, allsetting) else: env = allsetting or env locinfo.append('%s=%s' % (setting, env)) locinfo = ', '.join(locinfo) ph = getattr(self, 'pseudoheaders', None) if ph: headers = u'\n'.join(ph)+u'\n' else: headers = u'' version = getattr(self, 'version', None) if version: headers += u'Version: %s\n' % version body = getattr(self, 'body', u'') body = body.decode('utf8') # add NEWBIELINE only if it's less than advanced and the package is not # one of the specials (f.e. those with a dedicated function) also # thinking about those systems that don't have 'specials' dict if self.mode < utils.MODE_ADVANCED and self.package not in \ debbugs.SYSTEMS[self.system].get('specials', {}).keys(): body = utils.NEWBIELINE+u'\n\n'+body elif not body: body = u'\n' if self.issource: reportto = 'Source' else: reportto = 'Package' if not self.followup: for (attr, name) in dict(severity='Severity', justification='Justification', tags='Tags', filename='File').iteritems(): a = getattr(self, attr, None) if a: headers += u'%s: %s\n' % (name, a) report = u"%s: %s\n%s\n" % (reportto, self.package, headers) else: report = "Followup-For: Bug #%d\n%s: %s\n%s\n" % ( self.followup, reportto, self.package, headers) infofunc = debbugs.SYSTEMS[self.system].get('infofunc', debbugs.generic_infofunc) if infofunc: debinfo += infofunc() if un[0] == 'GNU': # Use uname -v on Hurd uname_string = un[3] else: kern = un[0] if kern.startswith('GNU/'): kern = kern[4:] uname_string = '%s %s' % (kern, un[2]) if kern == 'Linux': kinfo = [] if 'SMP' in un[3]: cores = utils.get_cpu_cores() if cores > 1: kinfo += ['SMP w/%d CPU cores' % cores] elif cores == 1: kinfo += ['SMP w/1 CPU core'] if 'PREEMPT' in un[3]: kinfo += ['PREEMPT'] if kinfo: uname_string = '%s (%s)' % (uname_string, '; '.join(kinfo)) if uname_string: debinfo += u'Kernel: %s\n' % uname_string if locinfo: debinfo += u'Locale: %s\n' % locinfo if shellpath != '/bin/sh': debinfo += u'Shell: /bin/sh linked to %s\n' % shellpath if init: debinfo += u'Init: %s\n' % init # Don't include system info for certain packages if self.sysinfo: report = u"%s%s%s\n-- System Information:\n%s" % (report, body, self.incfiles, debinfo) else: report = u"%s%s%s" % (report, body, self.incfiles) if hasattr(self, 'depinfo'): report += self.depinfo if hasattr(self, 'confinfo'): report += self.confinfo # add debsums output to the bug report if self.debsumsoutput: report += u"\n-- debsums errors found:\n%s\n" % self.debsumsoutput return report
def test_realpath_sym(self): self.assertEqual(realpath(self.linkpath, "/etc"), "../tmp")
def test_realpath_rel4(self): self.assertEqual(realpath("a/b", "/etc"), "../tmp/a/b")
def test_realpath_rel3(self): self.assertEqual(realpath("..", "/etc"), "..")
def test_realpath_rel2(self): self.assertEqual(realpath(".", "/etc"), "../tmp")
def test_realpath_rel_norm(self): self.assertEqual(realpath("test/../../../../", "/"), ".")
def test_realpath_rel(self): self.assertEqual(realpath("test/", "/"), "tmp/test")
def test_realpath_abs(self): self.assertEqual(realpath("test"), "/tmp/test")