def parse(buildout_filename, nocache=False): sys.stderr.write("Parsing buildout files:") if nocache: sys.stderr.write("\n(not using caches)") base_relative = find_relative(buildout_filename) session = requests_session(nocache=nocache) version_sections = _extract_versions_section( session, buildout_filename, version_sections=OrderedDict(), relative=base_relative) sys.stderr.write("\nparsing finished.\n") pkgs = {} for name in version_sections: for pkg in version_sections[name]: if pkg not in pkgs: pkgs[pkg] = OrderedDict() for pkgname in pkgs: pkg = pkgs[pkgname] for name in version_sections: if pkgname in version_sections[name]: pkg[name] = version_sections[name][pkgname] return pkgs
def get(pkginfo, buildout): filepath = TRACKINGFILENAME relative = find_relative(buildout) if relative: filepath = os.path.join(relative, TRACKINGFILENAME) if not os.path.exists(filepath): # We are not used as a buildout extension, so this file is # not available. return sys.stderr.write( '\nRead tracking information from buildout extension: \n' '- {0}\n'.format( filepath ) ) try: with open(filepath, 'r') as fp: pkginfo['tracking'] = json.load(fp) except (IOError, ValueError) as e: sys.stderr.write(' - ' + str(e) + '\n') return delta = time.time() - pkginfo['tracking']['generated'] days = int(delta // (60*60*24)) hours = int(delta // (60*60) - days*60*60) minutes = int(delta // (60) - days*60*60 - hours*60) seconds = delta % 60 sys.stderr.write( '- age of gathered data: {0:d}d {1:d}h {2:d}m {3:2.3f}s\n'.format( days, hours, minutes, seconds, ) )
def parse(buildout_filename, nocache=False): sys.stderr.write("Parsing buildout files:") if nocache: sys.stderr.write("\n(not using caches)") base_relative, buildout_filename = find_relative(buildout_filename) session = requests_session(nocache=nocache) version_sections, annotations = _extract_versions_section( session, buildout_filename, relative=base_relative) sys.stderr.write("\nparsing finished.\n") pkgs = {} for name in version_sections: for pkg in version_sections[name]: if pkg not in pkgs: pkgs[pkg] = OrderedDict() for pkgname in pkgs: pkg = pkgs[pkgname] for name in version_sections.keys(): if pkgname in version_sections.get(name, {}): pkg[name] = {"v": version_sections[name][pkgname], "a": ""} for name in annotations.keys(): if pkgname in annotations.get(name, {}): if name in pkg: pkg[name]["a"] = annotations[name][pkgname] else: pkg[name] = {"v": None, "a": annotations[name][pkgname]} return pkgs
def parse(buildout_filename, nocache=False): sys.stderr.write("Parsing buildout files:") if nocache: sys.stderr.write("\n(not using caches)") base_relative = find_relative(buildout_filename) session = requests_session(nocache=nocache) version_sections, annotations = _extract_versions_section(session, buildout_filename, relative=base_relative) sys.stderr.write("\nparsing finished.\n") pkgs = {} for name in version_sections: for pkg in version_sections[name]: if pkg not in pkgs: pkgs[pkg] = OrderedDict() for pkgname in pkgs: pkg = pkgs[pkgname] for name in version_sections.keys(): if pkgname in version_sections.get(name, {}): pkg[name] = {"v": version_sections[name][pkgname], "a": ""} for name in annotations.keys(): if pkgname in annotations.get(name, {}): if name in pkg: pkg[name]["a"] = annotations[name][pkgname] else: pkg[name] = {"v": None, "a": annotations[name][pkgname]} return pkgs
def get(pkginfo, buildout): filepath = TRACKINGFILENAME relative = find_relative(buildout) if relative: filepath = os.path.join(relative, TRACKINGFILENAME) if not os.path.exists(filepath): # We are not used as a buildout extension, so this file is # not available. return sys.stderr.write( '\nRead tracking information from buildout extension: \n' '- {0}\n'.format( filepath ) ) try: with open(filepath, 'r') as fp: pkginfo['tracking'] = json.load(fp) except (IOError, ValueError) as e: sys.stderr.write(' - ' + str(e) + '\n') return delta = time.time() - pkginfo['tracking']['generated'] days = int(delta // (60 * 60 * 24)) hours = int(delta // (60 * 60) - days * 60 * 60) minutes = int(delta // (60) - days * 60 * 60 - hours * 60) seconds = delta % 60 sys.stderr.write( '- age of gathered data: {0:d}d {1:d}h {2:d}m {3:2.3f}s\n'.format( days, hours, minutes, seconds, ) )
def _extract_versions_section(session, filename, version_sections=None, annotations=None, relative=None): if version_sections is None: version_sections = OrderedDict() if annotations is None: annotations = OrderedDict() sys.stderr.write("\n- {0}".format(filename)) if ( relative is not None and "://" not in filename and not filename.startswith("/") and not filename.startswith(relative) ): filename = relative + "/" + filename config = ConfigParser() if os.path.isfile(filename): config.read(filename) else: resp = session.get(filename) config.readfp(StringIO(resp.text)) if resp.from_cache: sys.stderr.write("\n from cache") elif resp.status_code != 200: sys.stderr.write("\n ERROR {0:d}".format(resp.status_code)) else: sys.stderr.write("\n fresh from server") # first read own versions section if config.has_section("versions"): version_sections[filename] = OrderedDict(config.items("versions")) sys.stderr.write("\n {0:d} entries in versions section.".format(len(version_sections[filename]))) if config.has_section("versionannotations"): annotations[filename] = OrderedDict(config.items("versionannotations")) sys.stderr.write("\n {0:d} entries in annotations section.".format(len(annotations[filename]))) try: extends = config.get("buildout", "extends").strip() except (NoSectionError, NoOptionError): return version_sections for extend in reversed(extends.splitlines()): extend = extend.strip() if not extend: continue sub_relative = find_relative(extend) or relative _extract_versions_section(session, extend, version_sections, annotations, sub_relative) return version_sections, annotations
def _extract_versions_section(session, filename, version_sections=None, relative=None): sys.stderr.write('\n- {0}'.format(filename)) if (relative is not None and "://" not in filename and not filename.startswith('/') and not filename.startswith(relative)): filename = relative + '/' + filename config = ConfigParser() if os.path.isfile(filename): config.read(filename) else: resp = session.get(filename) config.readfp(StringIO(resp.text)) if resp.from_cache: sys.stderr.write('\n from cache') elif resp.status_code != 200: sys.stderr.write('\n ERROR {0:d}'.format(resp.status_code)) else: sys.stderr.write('\n fresh from server') # first read own versions section if config.has_section('versions'): version_sections[filename] = OrderedDict(config.items('versions')) sys.stderr.write('\n {0:d} entries in versions section.'.format( len(version_sections[filename]))) try: extends = config.get('buildout', 'extends').strip() except (NoSectionError, NoOptionError): return version_sections for extend in reversed(extends.splitlines()): extend = extend.strip() if not extend: continue sub_relative = find_relative(extend) or relative _extract_versions_section(session, extend, version_sections, sub_relative) return version_sections
def parse(buildout_filename, nocache=False): sys.stderr.write("Parsing buildout files:") if nocache: sys.stderr.write("\n(not using caches)") base_relative = find_relative(buildout_filename) session = requests_session(nocache=nocache) version_sections = _extract_versions_section( session, buildout_filename, version_sections=OrderedDict(), relative=base_relative ) sys.stderr.write("\nparsing finished.\n") pkgs = {} for name in version_sections: for pkg in version_sections[name]: if pkg not in pkgs: pkgs[pkg] = OrderedDict() for pkgname in pkgs: pkg = pkgs[pkgname] for name in version_sections: if pkgname in version_sections[name]: pkg[name] = version_sections[name][pkgname] return pkgs
def _extract_versions_section( # NOQA: C901 session, filename, base_dir=None, version_sections=None, annotations=None, relative=None, version_section_name=None, versionannotation_section_name="versionannotations", ): if base_dir is None: base_dir = os.path.dirname(os.path.abspath(filename)) if version_sections is None: version_sections = OrderedDict() if annotations is None: annotations = OrderedDict() if "://" not in filename: if relative and "://" in relative: # relative to url! filename = "{0}/{1}".format(relative, filename) else: if relative: if filename.startswith(relative + "/"): filename = filename[len(relative + "/"):] filename = os.path.join(base_dir, relative, filename) else: filename = os.path.join(base_dir, filename) sys.stderr.write("\n- {0}".format(filename)) try: with nostdout(): buildout = Buildout(filename, []) # Use zc.buildout parser except UserError: buildout = {"buildout": {}} config = ConfigParser() if os.path.isfile(filename): config.read(filename) elif "://" in filename: resp = session.get(filename) config.readfp(StringIO(resp.text)) if resp.from_cache: sys.stderr.write("\n from cache") elif resp.status_code != 200: sys.stderr.write("\n ERROR {0:d}".format(resp.status_code)) else: sys.stderr.write("\n fresh from server") else: raise ValueError("{0} does not exist!".format(filename)) # first read own versions section current_version_section_name = buildout["buildout"].get( "versions", "versions") if version_section_name is None: # initial name version_section_name = current_version_section_name elif version_section_name != current_version_section_name: # name changed, not sure if this works as expected! - jensens sys.stderr.write( "\nName of [versions] (versions = versions) has changed." '\nGlobal versions section name: "{gname}"' '\nVersions pinned under that new Section namespace "{nname}"' " will be ignored.".format( gname=version_section_name, nname=buildout["buildout"].get("versions"))) if filename.startswith(base_dir): key_name = filename[len(base_dir) + 1:] else: key_name = filename if config.has_section(version_section_name): version_sections[key_name] = OrderedDict( config.items(version_section_name)) sys.stderr.write("\n {0:d} entries in versions section.".format( len(version_sections[key_name]))) # read versionannotations versionannotation_section_name = buildout["buildout"].get( "versionannotations", versionannotation_section_name) if config.has_section(versionannotation_section_name): annotations[key_name] = OrderedDict( config.items(versionannotation_section_name)) sys.stderr.write("\n {0:d} entries in annotations section.".format( len(annotations[key_name]))) try: extends = config.get("buildout", "extends").strip() except (NoSectionError, NoOptionError): return version_sections, annotations for extend in reversed(extends.splitlines()): extend = extend.strip() if not extend: continue sub_relative, extend = find_relative(extend, relative) _extract_versions_section( session, extend, base_dir, version_sections, annotations, sub_relative, version_section_name, versionannotation_section_name, ) return version_sections, annotations