def get_filelists(self): files = {} for uri in self.link_filelists: protocol, server, path = urlparse.urlparse(uri)[:3] rdfdir = os.path.dirname(path) if not self.quiet: self.log.write_stdout("Retrieving file list: %s" % uri) try: gzuri = utils.normalize_uri(uri + ".gz") file = self.download_file(gzuri, TYPE=2) except: file = self.download_file(uri, TYPE=2) pass fd = open(file) lines = fd.readlines() fd.close() baseuri = os.path.dirname(uri) if not self.bracket_regexp.match(lines[0]): raise ex.format_error, "Wrong format in filelist: %s" % uri for line in lines: if self.bracket_regexp.match(line): pkg = string.replace(line, "[", "]") pkg = string.replace(pkg, "]", "") pkg = os.path.join(rdfdir, pkg) pkg = urlparse.urlunparse((protocol, server, pkg, "", "", "")) files[pkg] = [] else: files[pkg].append(string.strip(line)) return files
def get_remote_info(self): force_old_style = False files = {} resources = {} if not self.link_list: return (files, resources) self.link_filelists = [] self.link_resourcelists = [] for baseuri in self.link_list: fileuri = utils.normalize_uri(baseuri + FILELISTNAME) self.link_filelists.append(fileuri) resuri = utils.normalize_uri(baseuri + RESOURCESFILENAME) try: newresources = self.get_resources(resuri) newresources.update(resources) resources = newresources except Exception, e: force_old_style = True reslisturi = utils.normalize_uri(baseuri + RESOURCELIST_FILENAME) self.link_resourcelists.append(reslisturi) pass
def get_resources(self, uri): resources = {} protocol, server, path = urlparse.urlparse(uri)[:3] baseuri = os.path.dirname(uri) if not server: baseuri = os.path.join("../..", baseuri) if not self.quiet: self.log.write_stdout("Retrieving resources: %s" % uri) try: gzuri = utils.normalize_uri(uri + ".gz") retr_file = self.download_file(gzuri, TYPE=2) except: try: retr_file = self.download_file(uri, TYPE=2) except: if not quiet: message = "No %s found, %s scheduled for download." % (RESOURCESFILENAME, RESOURCELIST_FILENAME) self.log.write_stdout(message) raise pass resources = self.reader.read_resources(retr_file, baseuri) return resources