def add_package(self, path, deltas, repo_uri): package = pisi.package.Package(path, "r") md = package.get_metadata() md.package.packageSize = os.path.getsize(path) md.package.packageHash = util.sha1_file(path) if ctx.config.options and ctx.config.options.absolute_urls: md.package.packageURI = os.path.realpath(path) else: # create relative path by default # TODO: in the future well do all of this with purl/pfile/&helpers # really? heheh -- future exa md.package.packageURI = util.removepathprefix(repo_uri, path) # check package semantics errs = md.errors() if md.errors(): ctx.ui.error(_("Package %s: metadata corrupt, skipping...") % md.package.name) ctx.ui.error(unicode(Error(*errs))) else: # No need to carry these with index (#3965) md.package.files = None md.package.additionalFiles = None if md.package.name in deltas: for delta_path in deltas[md.package.name]: delta = metadata.Delta() delta.packageURI = util.removepathprefix(repo_uri, delta_path) delta.packageSize = os.path.getsize(delta_path) delta.packageHash = util.sha1_file(delta_path) name, buildFrom, buildTo = util.parse_delta_package_name(delta_path) delta.buildFrom = buildFrom md.package.deltaPackages.append(delta) self.packages.append(md.package)
def add_package(self, path, deltas, repo_uri): package = pisi.package.Package(path, 'r') md = package.get_metadata() md.package.packageSize = os.path.getsize(path) md.package.packageHash = util.sha1_file(path) if ctx.config.options and ctx.config.options.absolute_urls: md.package.packageURI = os.path.realpath(path) else: # create relative path by default # TODO: in the future well do all of this with purl/pfile/&helpers # really? heheh -- future exa md.package.packageURI = util.removepathprefix(repo_uri, path) # check package semantics errs = md.errors() if md.errors(): ctx.ui.error(_('Package %s: metadata corrupt, skipping...') % md.package.name) ctx.ui.error(unicode(Error(*errs))) else: # No need to carry these with index (#3965) md.package.files = None md.package.additionalFiles = None if md.package.name in deltas: for delta_path in deltas[md.package.name]: delta = metadata.Delta() delta.packageURI = util.removepathprefix(repo_uri, delta_path) delta.packageSize = os.path.getsize(delta_path) delta.packageHash = util.sha1_file(delta_path) name, buildFrom, buildTo = util.parse_delta_package_name(delta_path) delta.buildFrom = buildFrom md.package.deltaPackages.append(delta) self.packages.append(md.package)
def gen_files_xml(self, package): """Generetes files.xml using the path definitions in specfile and generated files by the build system.""" files = Files() install_dir = self.bctx.pkg_install_dir() collisions = check_path_collision(package, self.spec.packages) if collisions: raise Error(_('Path collisions detected')) d = {} for pinfo in package.paths: path = install_dir + pinfo.pathname for fpath, fhash in util.get_file_hashes(path, collisions, install_dir): frpath = util.removepathprefix(install_dir, fpath) # relative path ftype = get_file_type(frpath, package.paths) try: # broken links can cause problem fsize = str(os.path.getsize(fpath)) except OSError: fsize = "0" d[frpath] = FileInfo(frpath, ftype, fsize, fhash) for (p, fileinfo) in d.iteritems(): files.append(fileinfo) files_xml_path = os.path.join(self.bctx.pkg_dir(), ctx.const.files_xml) files.write(files_xml_path) self.files = files
def add_path(path): # add the files under material path for fpath, fhash in util.get_file_hashes(path, collisions, install_dir): if ( ctx.get_option("create_static") and fpath.endswith(ctx.const.ar_file_suffix) and not package.name.endswith(ctx.const.static_name_suffix) and util.is_ar_file(fpath) ): # if this is an ar file, and this package is not a static package, # don't include this file into the package. continue frpath = util.removepathprefix(install_dir, fpath) # relative path ftype, permanent = get_file_type(frpath, package.files) fsize = long(util.dir_size(fpath)) if not os.path.islink(fpath): st = os.stat(fpath) else: st = os.lstat(fpath) fileinfo = pisi.files.FileInfo( path=frpath, type=ftype, permanent=permanent, size=fsize, hash=fhash, uid=str(st.st_uid), gid=str(st.st_gid), mode=oct(stat.S_IMODE(st.st_mode)), ) files.append(fileinfo) if stat.S_IMODE(st.st_mode) & stat.S_ISUID: ctx.ui.warning(_("/%s has suid bit set") % frpath)
def unpack_file_cond(self, pred, target_dir, archive_root = ''): """Unpack/Extract files according to predicate function pred: filename -> bool unpacks stuff into target_dir and only extracts files from archive_root, treating it as the archive root""" zip_obj = self.zip_obj for info in zip_obj.infolist(): if pred(info.filename): # check if condition holds # below code removes that, so we find it here is_dir = info.filename.endswith('/') # calculate output file name if archive_root == '': outpath = info.filename else: # change archive_root if util.subpath(archive_root, info.filename): outpath = util.removepathprefix(archive_root, info.filename) else: continue # don't extract if not under ofile = os.path.join(target_dir, outpath) if is_dir: # this is a directory if not os.path.isdir(ofile): os.makedirs(ofile) perm = info.external_attr perm &= 0xFFFF0000 perm >>= 16 perm |= 0x00000100 os.chmod(ofile, perm) continue # check that output dir is present util.check_dir(os.path.dirname(ofile)) # remove output file we might be overwriting. # (also check for islink? for broken symlinks...) if os.path.isfile(ofile) or os.path.islink(ofile): os.remove(ofile) if info.external_attr == self.symmagic: if os.path.isdir(ofile): shutil.rmtree(ofile) # a rare case, the file used to be a dir, now it is a symlink! target = zip_obj.read(info.filename) os.symlink(target, ofile) else: perm = info.external_attr perm &= 0x08FF0000 perm >>= 16 perm |= 0x00000100 if sys.version_info[:2] < (2, 6): zip_obj.decompressToFile(info.filename, ofile) else: info.filename = outpath zip_obj.extract(info, target_dir) os.chmod(ofile, perm)
def add_package(self, path, repo_uri): package = Package(path, 'r') # extract control files util.clean_dir(ctx.config.install_dir()) package.extract_PISI_files(ctx.config.install_dir()) md = metadata.MetaData() md.read(os.path.join(ctx.config.install_dir(), ctx.const.metadata_xml)) md.package.packageSize = os.path.getsize(path) if ctx.config.options and ctx.config.options.absolute_uris: # FIXME: the name "absolute_uris" does not seem to fit below :/ md.package.packageURI = os.path.realpath(path) else: # create relative path by default # TODO: in the future well do all of this with purl/pfile/&helpers # really? heheh -- future exa md.package.packageURI = util.removepathprefix(repo_uri, path) # check package semantics errs = md.errors() if md.errors(): ctx.ui.error( _('Package %s: metadata corrupt, skipping...') % md.package.name) ctx.ui.error(unicode(Error(*errs))) else: self.packages.append(md.package)
def add_path(path): # add the files under material path for fpath, fhash in util.get_file_hashes(path, collisions, install_dir): if ctx.get_option('create_static') \ and fpath.endswith(ctx.const.ar_file_suffix) \ and not package.name.endswith(ctx.const.static_name_suffix) \ and util.is_ar_file(fpath): # if this is an ar file, and this package is not a static package, # don't include this file into the package. continue frpath = util.removepathprefix(install_dir, fpath) # relative path ftype, permanent = get_file_type(frpath, package.files) fsize = long(util.dir_size(fpath)) if not os.path.islink(fpath): st = os.stat(fpath) else: st = os.lstat(fpath) d[frpath] = pisi.files.FileInfo(path=frpath, type=ftype, permanent=permanent, size=fsize, hash=fhash, uid=str(st.st_uid), gid=str(st.st_gid), mode=oct( stat.S_IMODE(st.st_mode))) if stat.S_IMODE(st.st_mode) & stat.S_ISUID: ctx.ui.warning(_("/%s has suid bit set") % frpath)
def unpack_file_cond(self, pred, target_dir, archive_root=''): """Unpack/Extract files according to predicate function pred: filename -> bool unpacks stuff into target_dir and only extracts files from archive_root, treating it as the archive root""" zip_obj = self.zip_obj for info in zip_obj.infolist(): if pred(info.filename): # check if condition holds # below code removes that, so we find it here is_dir = info.filename.endswith('/') # calculate output file name if archive_root == '': outpath = info.filename else: # change archive_root if util.subpath(archive_root, info.filename): outpath = util.removepathprefix(archive_root, info.filename) else: continue # don't extract if not under ofile = os.path.join(target_dir, outpath) if is_dir: # this is a directory if not os.path.isdir(ofile): os.makedirs(ofile) perm = info.external_attr perm &= 0xFFFF0000 perm >>= 16 perm |= 0x00000100 os.chmod(ofile, perm) continue # check that output dir is present util.ensure_dirs(os.path.dirname(ofile)) # remove output file we might be overwriting. # (also check for islink? for broken symlinks...) if os.path.isfile(ofile) or os.path.islink(ofile): os.remove(ofile) if info.external_attr == self.symmagic: if os.path.isdir(ofile): # A rare case, the file used to be a dir, # now it is a symlink! shutil.rmtree(ofile) target = zip_obj.read(info.filename) os.symlink(target, ofile) else: perm = info.external_attr perm &= 0x08FF0000 perm >>= 16 perm |= 0x00000100 info.filename = outpath zip_obj.extract(info, target_dir) os.chmod(ofile, perm)
def unpack_file_cond(self, pred, target_dir, archive_root = ''): """Unpack/Extract files according to predicate function pred: filename -> bool unpacks stuff into target_dir and only extracts files from archive_root, treating it as the archive root""" zip_obj = self.zip_obj for info in zip_obj.infolist(): if pred(info.filename): # check if condition holds # below code removes that, so we find it here is_dir = info.filename.endswith('/') # calculate output file name if archive_root == '': outpath = info.filename else: # change archive_root if util.subpath(archive_root, info.filename): outpath = util.removepathprefix(archive_root, info.filename) else: continue # don't extract if not under ofile = os.path.join(target_dir, outpath) if is_dir: # this is a directory d = os.path.join(target_dir, outpath) if not os.path.isdir(d): os.makedirs(d) perm = info.external_attr perm &= 0xFFFF0000 perm >>= 16 perm |= 0x00000100 os.chmod(d, perm) continue # check that output dir is present util.check_dir(os.path.dirname(ofile)) # remove output file we might be overwriting. # (also check for islink? for broken symlinks...) if os.path.exists(ofile) or os.path.islink(ofile): os.remove(ofile) if info.external_attr == self.symmagic: target = zip_obj.read(info.filename) os.symlink(target, ofile) else: perm = info.external_attr perm &= 0x08FF0000 perm >>= 16 perm |= 0x00000100 buff = open (ofile, 'wb') file_content = zip_obj.read(info.filename) buff.write(file_content) buff.close() os.chmod(ofile, perm)
def add_package(self, path, deltas, repo_uri): package = pisi.package.Package(path, 'r') md = package.get_metadata() md.package.packageSize = long(os.path.getsize(path)) md.package.packageHash = util.sha1_file(path) if ctx.config.options and ctx.config.options.absolute_urls: md.package.packageURI = os.path.realpath(path) else: # create relative path by default # TODO: in the future well do all of this with purl/pfile/&helpers # really? heheh -- future exa md.package.packageURI = util.removepathprefix(repo_uri, path) # check package semantics errs = md.errors() if md.errors(): ctx.ui.error(_('Package %s: metadata corrupt, skipping...') % md.package.name) ctx.ui.error(unicode(Error(*errs))) else: # No need to carry these with index (#3965) md.package.files = None md.package.additionalFiles = None if md.package.name in deltas: name, version, release, distro_id, arch = \ util.split_package_filename(path) for delta_path in deltas[md.package.name]: src_release, dst_release, delta_distro_id, delta_arch = \ util.split_delta_package_filename(delta_path)[1:] # Add only delta to latest build of the package if dst_release != md.package.release or \ (delta_distro_id, delta_arch) != (distro_id, arch): continue delta = metadata.Delta() delta.packageURI = util.removepathprefix(repo_uri, delta_path) delta.packageSize = long(os.path.getsize(delta_path)) delta.packageHash = util.sha1_file(delta_path) delta.releaseFrom = src_release md.package.deltaPackages.append(delta) self.packages.append(md.package)
def add_path(path): # add the files under material path for fpath, fhash in util.get_file_hashes(path, collisions, install_dir): frpath = util.removepathprefix(install_dir, fpath) # relative path ftype = get_file_type(frpath, package.files) try: # broken links and empty dirs can cause problem fsize = os.path.getsize(fpath) except OSError: fsize = None d[frpath] = FileInfo(path=frpath, type=ftype, size=fsize, hash=fhash)
def add_source(self, path, repo_uri): ctx.ui.info(_('Adding %s to source index') % path) sf = specfile.SpecFile() sf.read(path) errs = sf.errors() if sf.errors(): ctx.ui.error(_('SpecFile in %s is corrupt') % path) ctx.ui.error(str(Error(*errs))) else: if ctx.config.options and ctx.config.options.absolute_uris: sf.source.sourceURI = os.path.realpath(path) else: # create relative path by default sf.source.sourceURI = util.removepathprefix(repo_uri, path) self.sources.append(sf.source)
def add_path(path): # add the files under material path for fpath, fhash in util.get_file_hashes(path, collisions, install_dir): if ctx.get_option('create_static') \ and fpath.endswith(ctx.const.ar_file_suffix) \ and not package.name.endswith(ctx.const.static_name_suffix) \ and util.is_ar_file(fpath): # if this is an ar file, and this package is not a static package, # don't include this file into the package. continue frpath = util.removepathprefix(install_dir, fpath) # relative path ftype, permanent = get_file_type(frpath, package.files, install_dir) fsize = util.dir_size(fpath) d[frpath] = FileInfo(path=frpath, type=ftype, permanent=permanent, size=fsize, hash=fhash)
def strip_debug_action(filepath, fileinfo, install_dir, ag): excludelist = tuple(ag.get("NoStrip", [])) # real path in .pisi package path = '/' + util.removepathprefix(install_dir, filepath) if path.startswith(excludelist): return outputpath = util.join_path(os.path.dirname(install_dir), ctx.const.debug_dir_suffix, ctx.const.debug_files_suffix, path) if util.strip_file(filepath, fileinfo, outputpath): ctx.ui.debug("%s [%s]" % (path, "stripped"))
def strip_debug_action(filepath, fileinfo, install_dir, ag): excludelist = tuple(ag.get("NoStrip", [])) # real path in .pisi package path = "/" + util.removepathprefix(install_dir, filepath) if path.startswith(excludelist): return outputpath = util.join_path( os.path.dirname(install_dir), ctx.const.debug_dir_suffix, ctx.const.debug_files_suffix, path ) if util.strip_file(filepath, fileinfo, outputpath): ctx.ui.debug("%s [%s]" % (path, "stripped"))
def add_spec(self, path, repo_uri): import pisi.operations.build ctx.ui.info(_('Adding %s to source index') % path) #TODO: may use try/except to handle this builder = pisi.operations.build.Builder(path) #ctx.ui.error(_('SpecFile in %s is corrupt, skipping...') % path) #ctx.ui.error(str(Error(*errs))) builder.fetch_component() sf = builder.spec if ctx.config.options and ctx.config.options.absolute_urls: sf.source.sourceURI = os.path.realpath(path) else: # create relative path by default sf.source.sourceURI = util.removepathprefix(repo_uri, path) # check component self.specs.append(sf)
def strip_debug_action(filepath, fileinfo, install_dir, ag): excludelist = [] if not ag.has_key('NoStrip') else ag['NoStrip'] outputpath = util.join_path(os.path.dirname(install_dir), ctx.const.debug_dir_suffix, ctx.const.debug_files_suffix, util.remove_prefix(install_dir, filepath)) # real path in .pisi package p = '/' + util.removepathprefix(install_dir, filepath) strip = True for exclude in excludelist: if p.startswith(exclude): strip = False if strip: if util.strip_file(filepath, fileinfo, outputpath): ctx.ui.debug("%s [%s]" % (p, "stripped"))
def add_package(self, path, repo_uri): package = Package(path, 'r') md = package.get_metadata() md.package.packageSize = os.path.getsize(path) md.package.packageHash = util.sha1_file(path) if ctx.config.options and ctx.config.options.absolute_urls: md.package.packageURI = os.path.realpath(path) else: # create relative path by default # TODO: in the future well do all of this with purl/pfile/&helpers # really? heheh -- future exa md.package.packageURI = util.removepathprefix(repo_uri, path) # check package semantics errs = md.errors() if md.errors(): ctx.ui.error(_('Package %s: metadata corrupt, skipping...') % md.package.name) ctx.ui.error(unicode(Error(*errs))) else: self.packages.append(md.package)
def add_spec(params): try: path, repo_uri = params #TODO: may use try/except to handle this builder = pisi.operations.build.Builder(path) builder.fetch_component() sf = builder.spec if ctx.config.options and ctx.config.options.absolute_urls: sf.source.sourceURI = os.path.realpath(path) else: sf.source.sourceURI = util.removepathprefix(repo_uri, path) ctx.ui.info("%-80.80s\r" % (_('Adding %s to source index') % path), noln=False if ctx.config.get_option("verbose") else True) return sf except KeyboardInterrupt: # Multiprocessing hack, see add_package method for explanation raise Exception
def add_spec(params): try: path, repo_uri = params #TODO: may use try/except to handle this builder = pisi.operations.build.Builder(path) builder.fetch_component() sf = builder.spec if ctx.config.options and ctx.config.options.absolute_urls: sf.source.sourceURI = os.path.realpath(path) else: sf.source.sourceURI = util.removepathprefix(repo_uri, path) ctx.ui.info("%-80.80s\r" % (_('Adding %s to source index') % path), noln = True) return sf except KeyboardInterrupt: # Multiprocessing hack, see add_package method for explanation raise Exception
def add_spec(params): try: path, repo_uri = params ctx.ui.info(_("Adding %s to source index") % path) # TODO: may use try/except to handle this builder = pisi.operations.build.Builder(path) # ctx.ui.error(_('SpecFile in %s is corrupt, skipping...') % path) # ctx.ui.error(str(Error(*errs))) builder.fetch_component() sf = builder.spec if ctx.config.options and ctx.config.options.absolute_urls: sf.source.sourceURI = os.path.realpath(path) else: # create relative path by default sf.source.sourceURI = util.removepathprefix(repo_uri, path) # check component return sf except KeyboardInterrupt: # Multiprocessing hack, see add_package method for explanation raise Exception
def add_package(self, path, repo_uri): package = Package(path, 'r') # extract control files util.clean_dir(ctx.config.install_dir()) package.extract_PISI_files(ctx.config.install_dir()) md = metadata.MetaData() md.read(os.path.join(ctx.config.install_dir(), ctx.const.metadata_xml)) if ctx.config.options and ctx.config.options.absolute_uris: md.package.packageURI = os.path.realpath(path) else: # create relative path by default # TODO: in the future we'll do all of this with purl/pfile/&helpers # After that, we'll remove the ugly repo_uri parameter from this # function. md.package.packageURI = util.removepathprefix(repo_uri, path) # check package semantics if md.has_errors(): ctx.ui.error(_('Package %s: metadata corrupt') % md.package.name) else: self.packages.append(md.package)
def add_path(path): # add the files under material path for fpath, fhash in util.get_file_hashes(path, collisions, install_dir): if ctx.get_option('create_static') \ and fpath.endswith(ctx.const.ar_file_suffix) \ and not package.name.endswith(ctx.const.static_name_suffix) \ and util.is_ar_file(fpath): # if this is an ar file, and this package is not a static package, # don't include this file into the package. continue frpath = util.removepathprefix(install_dir, fpath) # relative path ftype, permanent = get_file_type(frpath, package.files, install_dir) fsize = long(util.dir_size(fpath)) d[frpath] = FileInfo(path=frpath, type=ftype, permanent=permanent, size=fsize, hash=fhash)
def add_package(self, path, repo_uri): package = Package(path, 'r') # extract control files util.clean_dir(ctx.config.install_dir()) package.extract_PISI_files(ctx.config.install_dir()) md = metadata.MetaData() md.read(os.path.join(ctx.config.install_dir(), ctx.const.metadata_xml)) if ctx.config.options and ctx.config.options.absolute_uris: # FIXME: the name "absolute_uris" does not seem to fit below :/ md.package.packageURI = os.path.realpath(path) else: # create relative path by default # TODO: in the future well do all of this with purl/pfile/&helpers # really? heheh -- future exa md.package.packageURI = util.removepathprefix(repo_uri, path) # check package semantics errs = md.errors() if md.errors(): ctx.ui.error(_('Package %s: metadata corrupt') % md.package.name) ctx.ui.error(str(Error(*errs))) else: self.packages.append(md.package)
def add_package(params): try: path, deltas, repo_uri = params ctx.ui.info("%-80.80s\r" % (_("Adding package to index: %s") % os.path.basename(path)), noln=True) package = pisi.package.Package(path, "r") md = package.get_metadata() md.package.packageSize = long(os.path.getsize(path)) md.package.packageHash = util.sha1_file(path) if ctx.config.options and ctx.config.options.absolute_urls: md.package.packageURI = os.path.realpath(path) else: md.package.packageURI = util.removepathprefix(repo_uri, path) # check package semantics errs = md.errors() if md.errors(): ctx.ui.info("") ctx.ui.error(_("Package %s: metadata corrupt, skipping...") % md.package.name) ctx.ui.error(unicode(Error(*errs))) else: # No need to carry these with index (#3965) md.package.files = None md.package.additionalFiles = None if md.package.name in deltas: name, version, release, distro_id, arch = util.split_package_filename(path) for delta_path in deltas[md.package.name]: src_release, dst_release, delta_distro_id, delta_arch = util.split_delta_package_filename( delta_path )[1:] # Add only delta to latest build of the package if dst_release != md.package.release or (delta_distro_id, delta_arch) != (distro_id, arch): continue delta = metadata.Delta() delta.packageURI = util.removepathprefix(repo_uri, delta_path) delta.packageSize = long(os.path.getsize(delta_path)) delta.packageHash = util.sha1_file(delta_path) delta.releaseFrom = src_release md.package.deltaPackages.append(delta) return md.package except KeyboardInterrupt: # Handle KeyboardInterrupt exception to prevent ugly backtrace of all # worker processes and propagate the exception to main process. # # Probably it's better to use just 'raise' here, but multiprocessing # module has some bugs about that: (python#8296, python#9205 and # python#9207 ) # # For now, worker processes do not propagate exceptions other than # Exception (like KeyboardInterrupt), so we have to manually propagate # KeyboardInterrupt exception as an Exception. raise Exception
def add_package(params): try: path, deltas, repo_uri = params ctx.ui.info( "%-80.80s\r" % (_('Adding package to index: %s') % os.path.basename(path)), noln=True) package = pisi.package.Package(path, 'r') md = package.get_metadata() md.package.packageSize = long(os.path.getsize(path)) md.package.packageHash = util.sha1_file(path) if ctx.config.options and ctx.config.options.absolute_urls: md.package.packageURI = os.path.realpath(path) else: md.package.packageURI = util.removepathprefix(repo_uri, path) # check package semantics errs = md.errors() if md.errors(): ctx.ui.info("") ctx.ui.error( _('Package %s: metadata corrupt, skipping...') % md.package.name) ctx.ui.error(unicode(Error(*errs))) else: # No need to carry these with index (#3965) md.package.files = None md.package.additionalFiles = None if md.package.name in deltas: name, version, release, distro_id, arch = \ util.split_package_filename(path) for delta_path in deltas[md.package.name]: src_release, dst_release, delta_distro_id, delta_arch = \ util.split_delta_package_filename(delta_path)[1:] # Add only delta to latest build of the package if dst_release != md.package.release or \ (delta_distro_id, delta_arch) != (distro_id, arch): continue delta = metadata.Delta() delta.packageURI = util.removepathprefix( repo_uri, delta_path) delta.packageSize = long(os.path.getsize(delta_path)) delta.packageHash = util.sha1_file(delta_path) delta.releaseFrom = src_release md.package.deltaPackages.append(delta) return md.package except KeyboardInterrupt: # Handle KeyboardInterrupt exception to prevent ugly backtrace of all # worker processes and propagate the exception to main process. # # Probably it's better to use just 'raise' here, but multiprocessing # module has some bugs about that: (python#8296, python#9205 and # python#9207 ) # # For now, worker processes do not propagate exceptions other than # Exception (like KeyboardInterrupt), so we have to manually propagate # KeyboardInterrupt exception as an Exception. raise Exception