def downloadReadingList(list, orig_config, callback, use_cache=True, re_read=True): from planet import logger import config try: import urllib2, StringIO from planet.spider import filename # list cache file name cache_filename = filename(config.cache_lists_directory(), list) # retrieve list options (e.g., etag, last-modified) from cache options = {} # add original options for key in orig_config.options(list): options[key] = orig_config.get(list, key) try: if use_cache: cached_config = ConfigParser() cached_config.read(cache_filename) for option in cached_config.options(list): options[option] = cached_config.get(list,option) except: pass cached_config = ConfigParser() cached_config.add_section(list) for key, value in options.items(): cached_config.set(list, key, value) # read list curdir=getattr(os.path, 'curdir', '.') if sys.platform.find('win') < 0: base = urljoin('file:', os.path.abspath(curdir)) else: path = os.path.abspath(os.path.curdir) base = urljoin('file:///', path.replace(':','|').replace('\\','/')) request = urllib2.Request(urljoin(base + '/', list)) if options.has_key("etag"): request.add_header('If-None-Match', options['etag']) if options.has_key("last-modified"): request.add_header('If-Modified-Since', options['last-modified']) response = urllib2.urlopen(request) if response.headers.has_key('etag'): cached_config.set(list, 'etag', response.headers['etag']) if response.headers.has_key('last-modified'): cached_config.set(list, 'last-modified', response.headers['last-modified']) # convert to config.ini data = StringIO.StringIO(response.read()) if callback: callback(data, cached_config) # write to cache if use_cache: cache = open(cache_filename, 'w') cached_config.write(cache) cache.close() # re-parse and proceed logger.debug("Using %s readinglist", list) if re_read: if use_cache: orig_config.read(cache_filename) else: cdata = StringIO.StringIO() cached_config.write(cdata) cdata.seek(0) orig_config.readfp(cdata) except: try: if re_read: if use_cache: if not orig_config.read(cache_filename): raise Exception() else: cdata = StringIO.StringIO() cached_config.write(cdata) cdata.seek(0) orig_config.readfp(cdata) logger.info("Using cached %s readinglist", list) except: logger.exception("Unable to read %s readinglist", list)
def load(config_files): """ initialize and load a configuration""" global parser parser = ConfigParser() parser.read(config_files) import config, planet from planet import opml, foaf, csv_config log = planet.logger if not log: log = planet.getLogger(config.log_level(),config.log_format()) # Theme support theme = config.output_theme() if theme: for path in ("", os.path.join(sys.path[0],'themes')): theme_dir = os.path.join(path,theme) theme_file = os.path.join(theme_dir,'config.ini') if os.path.exists(theme_file): # initial search list for theme directories dirs = config.template_directories() if theme_dir not in dirs: dirs.append(theme_dir) if not hasattr(config_files, 'append'): config_files = [config_files] for config_file in config_files: if os.path.dirname(config_file) not in dirs: dirs.append(os.path.dirname(config_file)) # read in the theme parser = ConfigParser() parser.read(theme_file) bom = config.bill_of_materials() # complete search list for theme directories dirs += [os.path.join(theme_dir,dir) for dir in config.template_directories() if dir not in dirs] # merge configurations, allowing current one to override theme template_files = config.template_files() parser.set('Planet','template_files','') parser.read(config_files) for file in config.bill_of_materials(): if not file in bom: bom.append(file) parser.set('Planet', 'bill_of_materials', ' '.join(bom)) parser.set('Planet', 'template_directories', ' '.join(dirs)) parser.set('Planet', 'template_files', ' '.join(template_files + config.template_files())) break else: log.error('Unable to find theme %s', theme) # Filter support dirs = config.filter_directories() filter_dir = os.path.join(sys.path[0],'filters') if filter_dir not in dirs and os.path.exists(filter_dir): parser.set('Planet', 'filter_directories', ' '.join(dirs+[filter_dir])) # Reading list support reading_lists = config.reading_lists() if reading_lists: if not os.path.exists(config.cache_lists_directory()): os.makedirs(config.cache_lists_directory()) def data2config(data, cached_config): if content_type(list).find('opml')>=0: opml.opml2config(data, cached_config) elif content_type(list).find('foaf')>=0: foaf.foaf2config(data, cached_config) elif content_type(list).find('csv')>=0: csv_config.csv2config(data, cached_config) elif content_type(list).find('config')>=0: cached_config.readfp(data) else: from planet import shell import StringIO cached_config.readfp(StringIO.StringIO(shell.run( content_type(list), data.getvalue(), mode="filter"))) if cached_config.sections() in [[], [list]]: raise Exception for list in reading_lists: downloadReadingList(list, parser, data2config)
def load(config_file): """ initialize and load a configuration""" global parser parser = ConfigParser() parser.read(config_file) import config, planet from planet import opml, foaf, csv_config log = planet.logger if not log: log = planet.getLogger(config.log_level(),config.log_format()) # Theme support theme = config.output_theme() if theme: for path in ("", os.path.join(sys.path[0],'themes')): theme_dir = os.path.join(path,theme) theme_file = os.path.join(theme_dir,'config.ini') if os.path.exists(theme_file): # initial search list for theme directories dirs = config.template_directories() if theme_dir not in dirs: dirs.append(theme_dir) if os.path.dirname(config_file) not in dirs: dirs.append(os.path.dirname(config_file)) # read in the theme parser = ConfigParser() parser.read(theme_file) bom = config.bill_of_materials() # complete search list for theme directories dirs += [os.path.join(theme_dir,dir) for dir in config.template_directories() if dir not in dirs] # merge configurations, allowing current one to override theme template_files = config.template_files() parser.set('Planet','template_files','') parser.read(config_file) for file in config.bill_of_materials(): if not file in bom: bom.append(file) parser.set('Planet', 'bill_of_materials', ' '.join(bom)) parser.set('Planet', 'template_directories', ' '.join(dirs)) parser.set('Planet', 'template_files', ' '.join(template_files + config.template_files())) break else: log.error('Unable to find theme %s', theme) # Filter support dirs = config.filter_directories() filter_dir = os.path.join(sys.path[0],'filters') if filter_dir not in dirs and os.path.exists(filter_dir): parser.set('Planet', 'filter_directories', ' '.join(dirs+[filter_dir])) # Reading list support reading_lists = config.reading_lists() if reading_lists: if not os.path.exists(config.cache_lists_directory()): os.makedirs(config.cache_lists_directory()) def data2config(data, cached_config): if content_type(list).find('opml')>=0: opml.opml2config(data, cached_config) elif content_type(list).find('foaf')>=0: foaf.foaf2config(data, cached_config) elif content_type(list).find('csv')>=0: csv_config.csv2config(data, cached_config) elif content_type(list).find('config')>=0: cached_config.readfp(data) else: from planet import shell import StringIO cached_config.readfp(StringIO.StringIO(shell.run( content_type(list), data.getvalue(), mode="filter"))) if cached_config.sections() in [[], [list]]: raise Exception for list in reading_lists: downloadReadingList(list, parser, data2config)