def varfunc(varname, origvalue, op, newlines): if varname == 'SRC_URI': if not origvalue.startswith('npm://'): src_uri = origvalue.split() changed = False deplist = {} for dep, depver in optdeps.items(): depdata = self.get_npm_data(dep, depver, d) if self.check_npm_optional_dependency(depdata): deplist[dep] = depdata for dep, depver in devdeps.items(): depdata = self.get_npm_data(dep, depver, d) if self.check_npm_optional_dependency(depdata): deplist[dep] = depdata for dep, depver in deps.items(): depdata = self.get_npm_data(dep, depver, d) deplist[dep] = depdata for dep, depdata in deplist.items(): version = depdata.get('version', None) if version: url = 'npm://registry.npmjs.org;name=%s;version=%s;subdir=node_modules/%s' % (dep, version, dep) scriptutils.fetch_uri(d, url, srctree) src_uri.append(url) changed = True if changed: return src_uri, None, -1, True return origvalue, None, 0, True
def varfunc(varname, origvalue, op, newlines): if varname == 'SRC_URI': if not origvalue.startswith('npm://'): src_uri = origvalue.split() changed = False deplist = {} for dep, depver in optdeps.items(): depdata = self.get_npm_data(dep, depver, d) if self.check_npm_optional_dependency(depdata): deplist[dep] = depdata for dep, depver in devdeps.items(): depdata = self.get_npm_data(dep, depver, d) if self.check_npm_optional_dependency(depdata): deplist[dep] = depdata for dep, depver in deps.items(): depdata = self.get_npm_data(dep, depver, d) deplist[dep] = depdata for dep, depdata in deplist.items(): version = depdata.get('version', None) if version: url = 'npm://registry.npmjs.org;name=%s;version=%s;subdir=node_modules/%s' % ( dep, version, dep) scriptutils.fetch_uri(d, url, srctree) src_uri.append(url) changed = True if changed: return src_uri, None, -1, True return origvalue, None, 0, True
def varfunc(varname, origvalue, op, newlines): if varname == 'SRC_URI': if not origvalue.startswith('npm://'): src_uri = origvalue.split() changed = False for dep, depdata in deps.items(): version = self.get_node_version(dep, depdata, d) if version: url = 'npm://registry.npmjs.org;name=%s;version=%s;subdir=node_modules/%s' % (dep, version, dep) scriptutils.fetch_uri(d, url, srctree) src_uri.append(url) changed = True if changed: return src_uri, None, -1, True return origvalue, None, 0, True
def create_recipe(args): import bb.process import tempfile import shutil import oe.recipeutils pkgarch = "" if args.machine: pkgarch = "${MACHINE_ARCH}" extravalues = {} checksums = (None, None) tempsrc = '' source = args.source srcsubdir = '' srcrev = '${AUTOREV}' if os.path.isfile(source): source = 'file://%s' % os.path.abspath(source) if scriptutils.is_src_url(source): # Fetch a URL fetchuri = reformat_git_uri(urldefrag(source)[0]) if args.binary: # Assume the archive contains the directory structure verbatim # so we need to extract to a subdirectory fetchuri += ';subdir=${BP}' srcuri = fetchuri rev_re = re.compile(';rev=([^;]+)') res = rev_re.search(srcuri) if res: srcrev = res.group(1) srcuri = rev_re.sub('', srcuri) tempsrc = tempfile.mkdtemp(prefix='recipetool-') srctree = tempsrc d = bb.data.createCopy(tinfoil.config_data) if fetchuri.startswith('npm://'): # Check if npm is available npm_bindir = check_npm(tinfoil, args.devtool) d.prependVar('PATH', '%s:' % npm_bindir) logger.info('Fetching %s...' % srcuri) try: checksums = scriptutils.fetch_uri(d, fetchuri, srctree, srcrev) except bb.fetch2.BBFetchException as e: logger.error(str(e).rstrip()) sys.exit(1) dirlist = os.listdir(srctree) if 'git.indirectionsymlink' in dirlist: dirlist.remove('git.indirectionsymlink') if len(dirlist) == 1: singleitem = os.path.join(srctree, dirlist[0]) if os.path.isdir(singleitem): # We unpacked a single directory, so we should use that srcsubdir = dirlist[0] srctree = os.path.join(srctree, srcsubdir) else: with open(singleitem, 'r', errors='surrogateescape') as f: if '<html' in f.read(100).lower(): logger.error('Fetching "%s" returned a single HTML page - check the URL is correct and functional' % fetchuri) sys.exit(1) if os.path.exists(os.path.join(srctree, '.gitmodules')) and srcuri.startswith('git://'): srcuri = 'gitsm://' + srcuri[6:] logger.info('Fetching submodules...') bb.process.run('git submodule update --init --recursive', cwd=srctree) if is_package(fetchuri): tmpfdir = tempfile.mkdtemp(prefix='recipetool-') try: pkgfile = None try: fileuri = fetchuri + ';unpack=0' scriptutils.fetch_uri(tinfoil.config_data, fileuri, tmpfdir, srcrev) for root, _, files in os.walk(tmpfdir): for f in files: pkgfile = os.path.join(root, f) break except bb.fetch2.BBFetchException as e: logger.warn('Second fetch to get metadata failed: %s' % str(e).rstrip()) if pkgfile: if pkgfile.endswith(('.deb', '.ipk')): stdout, _ = bb.process.run('ar x %s' % pkgfile, cwd=tmpfdir) stdout, _ = bb.process.run('tar xf control.tar.gz', cwd=tmpfdir) values = convert_debian(tmpfdir) extravalues.update(values) elif pkgfile.endswith(('.rpm', '.srpm')): stdout, _ = bb.process.run('rpm -qp --xml %s > pkginfo.xml' % pkgfile, cwd=tmpfdir) values = convert_rpm_xml(os.path.join(tmpfdir, 'pkginfo.xml')) extravalues.update(values) finally: shutil.rmtree(tmpfdir) else: # Assume we're pointing to an existing source tree if args.extract_to: logger.error('--extract-to cannot be specified if source is a directory') sys.exit(1) if not os.path.isdir(source): logger.error('Invalid source directory %s' % source) sys.exit(1) srctree = source srcuri = '' if os.path.exists(os.path.join(srctree, '.git')): # Try to get upstream repo location from origin remote try: stdout, _ = bb.process.run('git remote -v', cwd=srctree, shell=True) except bb.process.ExecutionError as e: stdout = None if stdout: for line in stdout.splitlines(): splitline = line.split() if len(splitline) > 1: if splitline[0] == 'origin' and scriptutils.is_src_url(splitline[1]): srcuri = reformat_git_uri(splitline[1]) srcsubdir = 'git' break if args.src_subdir: srcsubdir = os.path.join(srcsubdir, args.src_subdir) srctree_use = os.path.join(srctree, args.src_subdir) else: srctree_use = srctree if args.outfile and os.path.isdir(args.outfile): outfile = None outdir = args.outfile else: outfile = args.outfile outdir = None if outfile and outfile != '-': if os.path.exists(outfile): logger.error('Output file %s already exists' % outfile) sys.exit(1) lines_before = [] lines_after = [] lines_before.append('# Recipe created by %s' % os.path.basename(sys.argv[0])) lines_before.append('# This is the basis of a recipe and may need further editing in order to be fully functional.') lines_before.append('# (Feel free to remove these comments when editing.)') # We need a blank line here so that patch_recipe_lines can rewind before the LICENSE comments lines_before.append('') handled = [] licvalues = handle_license_vars(srctree_use, lines_before, handled, extravalues, tinfoil.config_data) classes = [] # FIXME This is kind of a hack, we probably ought to be using bitbake to do this pn = None pv = None if outfile: recipefn = os.path.splitext(os.path.basename(outfile))[0] fnsplit = recipefn.split('_') if len(fnsplit) > 1: pn = fnsplit[0] pv = fnsplit[1] else: pn = recipefn if args.version: pv = args.version if args.name: pn = args.name if args.name.endswith('-native'): if args.also_native: logger.error('--also-native cannot be specified for a recipe named *-native (*-native denotes a recipe that is already only for native) - either remove the -native suffix from the name or drop --also-native') sys.exit(1) classes.append('native') elif args.name.startswith('nativesdk-'): if args.also_native: logger.error('--also-native cannot be specified for a recipe named nativesdk-* (nativesdk-* denotes a recipe that is already only for nativesdk)') sys.exit(1) classes.append('nativesdk') if pv and pv not in 'git svn hg'.split(): realpv = pv else: realpv = None if srcuri and not realpv or not pn: name_pn, name_pv = determine_from_url(srcuri) if name_pn and not pn: pn = name_pn if name_pv and not realpv: realpv = name_pv if not srcuri: lines_before.append('# No information for SRC_URI yet (only an external source tree was specified)') lines_before.append('SRC_URI = "%s"' % srcuri) (md5value, sha256value) = checksums if md5value: lines_before.append('SRC_URI[md5sum] = "%s"' % md5value) if sha256value: lines_before.append('SRC_URI[sha256sum] = "%s"' % sha256value) if srcuri and supports_srcrev(srcuri): lines_before.append('') lines_before.append('# Modify these as desired') lines_before.append('PV = "%s+git${SRCPV}"' % (realpv or '1.0')) if not args.autorev and srcrev == '${AUTOREV}': if os.path.exists(os.path.join(srctree, '.git')): (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srctree) srcrev = stdout.rstrip() lines_before.append('SRCREV = "%s"' % srcrev) lines_before.append('') if srcsubdir and not args.binary: # (for binary packages we explicitly specify subdir= when fetching to # match the default value of S, so we don't need to set it in that case) lines_before.append('S = "${WORKDIR}/%s"' % srcsubdir) lines_before.append('') if pkgarch: lines_after.append('PACKAGE_ARCH = "%s"' % pkgarch) lines_after.append('') if args.binary: lines_after.append('INSANE_SKIP_${PN} += "already-stripped"') lines_after.append('') if args.fetch_dev: extravalues['fetchdev'] = True else: extravalues['fetchdev'] = None # Find all plugins that want to register handlers logger.debug('Loading recipe handlers') raw_handlers = [] for plugin in plugins: if hasattr(plugin, 'register_recipe_handlers'): plugin.register_recipe_handlers(raw_handlers) # Sort handlers by priority handlers = [] for i, handler in enumerate(raw_handlers): if isinstance(handler, tuple): handlers.append((handler[0], handler[1], i)) else: handlers.append((handler, 0, i)) handlers.sort(key=lambda item: (item[1], -item[2]), reverse=True) for handler, priority, _ in handlers: logger.debug('Handler: %s (priority %d)' % (handler.__class__.__name__, priority)) setattr(handler, '_devtool', args.devtool) handlers = [item[0] for item in handlers] # Apply the handlers if args.binary: classes.append('bin_package') handled.append('buildsystem') for handler in handlers: handler.process(srctree_use, classes, lines_before, lines_after, handled, extravalues) extrafiles = extravalues.pop('extrafiles', {}) extra_pn = extravalues.pop('PN', None) extra_pv = extravalues.pop('PV', None) if extra_pv and not realpv: realpv = extra_pv if not validate_pv(realpv): realpv = None else: realpv = realpv.lower().split()[0] if '_' in realpv: realpv = realpv.replace('_', '-') if extra_pn and not pn: pn = extra_pn if pn.startswith('GNU '): pn = pn[4:] if ' ' in pn: # Probably a descriptive identifier rather than a proper name pn = None else: pn = pn.lower() if '_' in pn: pn = pn.replace('_', '-') if not outfile: if not pn: log_error_cond('Unable to determine short program name from source tree - please specify name with -N/--name or output file name with -o/--outfile', args.devtool) # devtool looks for this specific exit code, so don't change it sys.exit(15) else: if srcuri and srcuri.startswith(('gitsm://', 'git://', 'hg://', 'svn://')): suffix = srcuri.split(':', 1)[0] if suffix == 'gitsm': suffix = 'git' outfile = '%s_%s.bb' % (pn, suffix) elif realpv: outfile = '%s_%s.bb' % (pn, realpv) else: outfile = '%s.bb' % pn if outdir: outfile = os.path.join(outdir, outfile) # We need to check this again if os.path.exists(outfile): logger.error('Output file %s already exists' % outfile) sys.exit(1) # Move any extra files the plugins created to a directory next to the recipe if extrafiles: if outfile == '-': extraoutdir = pn else: extraoutdir = os.path.join(os.path.dirname(outfile), pn) bb.utils.mkdirhier(extraoutdir) for destfn, extrafile in extrafiles.items(): shutil.move(extrafile, os.path.join(extraoutdir, destfn)) lines = lines_before lines_before = [] skipblank = True for line in lines: if skipblank: skipblank = False if not line: continue if line.startswith('S = '): if realpv and pv not in 'git svn hg'.split(): line = line.replace(realpv, '${PV}') if pn: line = line.replace(pn, '${BPN}') if line == 'S = "${WORKDIR}/${BPN}-${PV}"': skipblank = True continue elif line.startswith('SRC_URI = '): if realpv: line = line.replace(realpv, '${PV}') elif line.startswith('PV = '): if realpv: line = re.sub('"[^+]*\+', '"%s+' % realpv, line) lines_before.append(line) if args.also_native: lines = lines_after lines_after = [] bbclassextend = None for line in lines: if line.startswith('BBCLASSEXTEND ='): splitval = line.split('"') if len(splitval) > 1: bbclassextend = splitval[1].split() if not 'native' in bbclassextend: bbclassextend.insert(0, 'native') line = 'BBCLASSEXTEND = "%s"' % ' '.join(bbclassextend) lines_after.append(line) if not bbclassextend: lines_after.append('BBCLASSEXTEND = "native"') postinst = ("postinst", extravalues.pop('postinst', None)) postrm = ("postrm", extravalues.pop('postrm', None)) preinst = ("preinst", extravalues.pop('preinst', None)) prerm = ("prerm", extravalues.pop('prerm', None)) funcs = [postinst, postrm, preinst, prerm] for func in funcs: if func[1]: RecipeHandler.genfunction(lines_after, 'pkg_%s_${PN}' % func[0], func[1]) outlines = [] outlines.extend(lines_before) if classes: if outlines[-1] and not outlines[-1].startswith('#'): outlines.append('') outlines.append('inherit %s' % ' '.join(classes)) outlines.append('') outlines.extend(lines_after) if extravalues: if 'LICENSE' in extravalues and not licvalues: # Don't blow away 'CLOSED' value that comments say we set del extravalues['LICENSE'] _, outlines = oe.recipeutils.patch_recipe_lines(outlines, extravalues, trailing_newline=False) if args.extract_to: scriptutils.git_convert_standalone_clone(srctree) if os.path.isdir(args.extract_to): # If the directory exists we'll move the temp dir into it instead of # its contents - of course, we could try to always move its contents # but that is a pain if there are symlinks; the simplest solution is # to just remove it first os.rmdir(args.extract_to) shutil.move(srctree, args.extract_to) if tempsrc == srctree: tempsrc = None log_info_cond('Source extracted to %s' % args.extract_to, args.devtool) if outfile == '-': sys.stdout.write('\n'.join(outlines) + '\n') else: with open(outfile, 'w') as f: lastline = None for line in outlines: if not lastline and not line: # Skip extra blank lines continue f.write('%s\n' % line) lastline = line log_info_cond('Recipe %s has been created; further editing may be required to make it fully functional' % outfile, args.devtool) if tempsrc: if args.keep_temp: logger.info('Preserving temporary directory %s' % tempsrc) else: shutil.rmtree(tempsrc) return 0
def create_recipe(args): import bb.process import tempfile import shutil pkgarch = "" if args.machine: pkgarch = "${MACHINE_ARCH}" checksums = (None, None) tempsrc = '' srcsubdir = '' srcrev = '${AUTOREV}' if '://' in args.source: # Fetch a URL fetchuri = urlparse.urldefrag(args.source)[0] if args.binary: # Assume the archive contains the directory structure verbatim # so we need to extract to a subdirectory fetchuri += ';subdir=%s' % os.path.splitext( os.path.basename(urlparse.urlsplit(fetchuri).path))[0] srcuri = fetchuri rev_re = re.compile(';rev=([^;]+)') res = rev_re.search(srcuri) if res: srcrev = res.group(1) srcuri = rev_re.sub('', srcuri) tempsrc = tempfile.mkdtemp(prefix='recipetool-') srctree = tempsrc logger.info('Fetching %s...' % srcuri) checksums = scriptutils.fetch_uri(tinfoil.config_data, fetchuri, srctree, srcrev) dirlist = os.listdir(srctree) if 'git.indirectionsymlink' in dirlist: dirlist.remove('git.indirectionsymlink') if len(dirlist) == 1 and os.path.isdir( os.path.join(srctree, dirlist[0])): # We unpacked a single directory, so we should use that srcsubdir = dirlist[0] srctree = os.path.join(srctree, srcsubdir) else: # Assume we're pointing to an existing source tree if args.extract_to: logger.error( '--extract-to cannot be specified if source is a directory') sys.exit(1) if not os.path.isdir(args.source): logger.error('Invalid source directory %s' % args.source) sys.exit(1) srcuri = '' srctree = args.source outfile = args.outfile if outfile and outfile != '-': if os.path.exists(outfile): logger.error('Output file %s already exists' % outfile) sys.exit(1) lines_before = [] lines_after = [] lines_before.append('# Recipe created by %s' % os.path.basename(sys.argv[0])) lines_before.append( '# This is the basis of a recipe and may need further editing in order to be fully functional.' ) lines_before.append('# (Feel free to remove these comments when editing.)') lines_before.append('#') licvalues = guess_license(srctree) lic_files_chksum = [] if licvalues: licenses = [] for licvalue in licvalues: if not licvalue[0] in licenses: licenses.append(licvalue[0]) lic_files_chksum.append('file://%s;md5=%s' % (licvalue[1], licvalue[2])) lines_before.append( '# WARNING: the following LICENSE and LIC_FILES_CHKSUM values are best guesses - it is' ) lines_before.append( '# your responsibility to verify that the values are complete and correct.' ) if len(licvalues) > 1: lines_before.append('#') lines_before.append( '# NOTE: multiple licenses have been detected; if that is correct you should separate' ) lines_before.append( '# these in the LICENSE value using & if the multiple licenses all apply, or | if there' ) lines_before.append( '# is a choice between the multiple licenses. If in doubt, check the accompanying' ) lines_before.append( '# documentation to determine which situation is applicable.') else: lines_before.append( '# Unable to find any files that looked like license statements. Check the accompanying' ) lines_before.append( '# documentation and source headers and set LICENSE and LIC_FILES_CHKSUM accordingly.' ) lines_before.append('#') lines_before.append( '# NOTE: LICENSE is being set to "CLOSED" to allow you to at least start building - if' ) lines_before.append( '# this is not accurate with respect to the licensing of the software being built (it' ) lines_before.append( '# will not be in most cases) you must specify the correct value before using this' ) lines_before.append( '# recipe for anything other than initial testing/development!') licenses = ['CLOSED'] lines_before.append('LICENSE = "%s"' % ' '.join(licenses)) lines_before.append('LIC_FILES_CHKSUM = "%s"' % ' \\\n '.join(lic_files_chksum)) lines_before.append('') # FIXME This is kind of a hack, we probably ought to be using bitbake to do this # we'd also want a way to automatically set outfile based upon auto-detecting these values from the source if possible recipefn = os.path.splitext(os.path.basename(outfile))[0] fnsplit = recipefn.split('_') if len(fnsplit) > 1: pn = fnsplit[0] pv = fnsplit[1] else: pn = recipefn pv = None if args.version: pv = args.version if pv and pv not in 'git svn hg'.split(): realpv = pv else: realpv = None if srcuri: if realpv: srcuri = srcuri.replace(realpv, '${PV}') else: lines_before.append( '# No information for SRC_URI yet (only an external source tree was specified)' ) lines_before.append('SRC_URI = "%s"' % srcuri) (md5value, sha256value) = checksums if md5value: lines_before.append('SRC_URI[md5sum] = "%s"' % md5value) if sha256value: lines_before.append('SRC_URI[sha256sum] = "%s"' % sha256value) if srcuri and supports_srcrev(srcuri): lines_before.append('') lines_before.append('# Modify these as desired') lines_before.append('PV = "%s+git${SRCPV}"' % (realpv or '1.0')) lines_before.append('SRCREV = "%s"' % srcrev) lines_before.append('') if srcsubdir and pv: if srcsubdir == "%s-%s" % (pn, pv): # This would be the default, so we don't need to set S in the recipe srcsubdir = '' if srcsubdir: if pv and pv not in 'git svn hg'.split(): srcsubdir = srcsubdir.replace(pv, '${PV}') lines_before.append('S = "${WORKDIR}/%s"' % srcsubdir) lines_before.append('') if pkgarch: lines_after.append('PACKAGE_ARCH = "%s"' % pkgarch) lines_after.append('') if args.binary: lines_after.append('INSANE_SKIP_${PN} += "already-stripped"') lines_after.append('') # Find all plugins that want to register handlers handlers = [] for plugin in plugins: if hasattr(plugin, 'register_recipe_handlers'): plugin.register_recipe_handlers(handlers) # Apply the handlers classes = [] handled = [] if args.binary: classes.append('bin_package') handled.append('buildsystem') for handler in handlers: handler.process(srctree, classes, lines_before, lines_after, handled) outlines = [] outlines.extend(lines_before) if classes: outlines.append('inherit %s' % ' '.join(classes)) outlines.append('') outlines.extend(lines_after) if args.extract_to: scriptutils.git_convert_standalone_clone(srctree) if os.path.isdir(args.extract_to): # If the directory exists we'll move the temp dir into it instead of # its contents - of course, we could try to always move its contents # but that is a pain if there are symlinks; the simplest solution is # to just remove it first os.rmdir(args.extract_to) shutil.move(srctree, args.extract_to) logger.info('Source extracted to %s' % args.extract_to) if outfile == '-': sys.stdout.write('\n'.join(outlines) + '\n') else: with open(outfile, 'w') as f: f.write('\n'.join(outlines) + '\n') logger.info( 'Recipe %s has been created; further editing may be required to make it fully functional' % outfile) if tempsrc: shutil.rmtree(tempsrc) return 0
def _extract_new_source(newpv, srctree, no_patch, srcrev, branch, keep_temp, tinfoil, rd): """Extract sources of a recipe with a new version""" def __run(cmd): """Simple wrapper which calls _run with srctree as cwd""" return _run(cmd, srctree) crd = rd.createCopy() pv = crd.getVar('PV') crd.setVar('PV', newpv) tmpsrctree = None uri, rev = _get_uri(crd) if srcrev: rev = srcrev if uri.startswith('git://'): __run('git fetch') __run('git checkout %s' % rev) __run('git tag -f devtool-base-new') md5 = None sha256 = None else: __run('git checkout devtool-base -b devtool-%s' % newpv) tmpdir = tempfile.mkdtemp(prefix='devtool') try: md5, sha256 = scriptutils.fetch_uri(tinfoil.config_data, uri, tmpdir, rev) except bb.fetch2.FetchError as e: raise DevtoolError(e) tmpsrctree = _get_srctree(tmpdir) srctree = os.path.abspath(srctree) # Delete all sources so we ensure no stray files are left over for item in os.listdir(srctree): if item in ['.git', 'oe-local-files']: continue itempath = os.path.join(srctree, item) if os.path.isdir(itempath): shutil.rmtree(itempath) else: os.remove(itempath) # Copy in new ones _copy_source_code(tmpsrctree, srctree) (stdout, _) = __run('git ls-files --modified --others --exclude-standard') for f in stdout.splitlines(): __run('git add "%s"' % f) useroptions = [] oe.patch.GitApplyTree.gitCommandUserOptions(useroptions, d=rd) __run( 'git %s commit -q -m "Commit of upstream changes at version %s" --allow-empty' % (' '.join(useroptions), newpv)) __run('git tag -f devtool-base-%s' % newpv) (stdout, _) = __run('git rev-parse HEAD') rev = stdout.rstrip() if no_patch: patches = oe.recipeutils.get_recipe_patches(crd) if len(patches): logger.warn( 'By user choice, the following patches will NOT be applied') for patch in patches: logger.warn("%s" % os.path.basename(patch)) else: __run('git checkout devtool-patched -b %s' % branch) skiptag = False try: __run('git rebase %s' % rev) except bb.process.ExecutionError as e: skiptag = True if 'conflict' in e.stdout: logger.warn( 'Command \'%s\' failed:\n%s\n\nYou will need to resolve conflicts in order to complete the upgrade.' % (e.command, e.stdout.rstrip())) else: logger.warn('Command \'%s\' failed:\n%s' % (e.command, e.stdout)) if not skiptag: if uri.startswith('git://'): suffix = 'new' else: suffix = newpv __run('git tag -f devtool-patched-%s' % suffix) if tmpsrctree: if keep_temp: logger.info('Preserving temporary directory %s' % tmpsrctree) else: shutil.rmtree(tmpsrctree) return (rev, md5, sha256)
def create_recipe(args): import bb.process import tempfile import shutil pkgarch = "" if args.machine: pkgarch = "${MACHINE_ARCH}" checksums = (None, None) tempsrc = '' srcsubdir = '' srcrev = '${AUTOREV}' if '://' in args.source: # Fetch a URL fetchuri = reformat_git_uri(urlparse.urldefrag(args.source)[0]) if args.binary: # Assume the archive contains the directory structure verbatim # so we need to extract to a subdirectory fetchuri += ';subdir=%s' % os.path.splitext(os.path.basename(urlparse.urlsplit(fetchuri).path))[0] srcuri = fetchuri rev_re = re.compile(';rev=([^;]+)') res = rev_re.search(srcuri) if res: srcrev = res.group(1) srcuri = rev_re.sub('', srcuri) tempsrc = tempfile.mkdtemp(prefix='recipetool-') srctree = tempsrc if fetchuri.startswith('npm://'): # Check if npm is available npm = bb.utils.which(tinfoil.config_data.getVar('PATH', True), 'npm') if not npm: logger.error('npm:// URL requested but npm is not available - you need to either build nodejs-native or install npm using your package manager') sys.exit(1) logger.info('Fetching %s...' % srcuri) try: checksums = scriptutils.fetch_uri(tinfoil.config_data, fetchuri, srctree, srcrev) except bb.fetch2.BBFetchException as e: logger.error(str(e).rstrip()) sys.exit(1) dirlist = os.listdir(srctree) if 'git.indirectionsymlink' in dirlist: dirlist.remove('git.indirectionsymlink') if len(dirlist) == 1: singleitem = os.path.join(srctree, dirlist[0]) if os.path.isdir(singleitem): # We unpacked a single directory, so we should use that srcsubdir = dirlist[0] srctree = os.path.join(srctree, srcsubdir) else: with open(singleitem, 'r') as f: if '<html' in f.read(100).lower(): logger.error('Fetching "%s" returned a single HTML page - check the URL is correct and functional' % fetchuri) sys.exit(1) else: # Assume we're pointing to an existing source tree if args.extract_to: logger.error('--extract-to cannot be specified if source is a directory') sys.exit(1) if not os.path.isdir(args.source): logger.error('Invalid source directory %s' % args.source) sys.exit(1) srctree = args.source srcuri = '' if os.path.exists(os.path.join(srctree, '.git')): # Try to get upstream repo location from origin remote try: stdout, _ = bb.process.run('git remote -v', cwd=srctree, shell=True) except bb.process.ExecutionError as e: stdout = None if stdout: for line in stdout.splitlines(): splitline = line.split() if len(splitline) > 1: if splitline[0] == 'origin' and '://' in splitline[1]: srcuri = reformat_git_uri(splitline[1]) srcsubdir = 'git' break if args.src_subdir: srcsubdir = os.path.join(srcsubdir, args.src_subdir) srctree_use = os.path.join(srctree, args.src_subdir) else: srctree_use = srctree if args.outfile and os.path.isdir(args.outfile): outfile = None outdir = args.outfile else: outfile = args.outfile outdir = None if outfile and outfile != '-': if os.path.exists(outfile): logger.error('Output file %s already exists' % outfile) sys.exit(1) lines_before = [] lines_after = [] lines_before.append('# Recipe created by %s' % os.path.basename(sys.argv[0])) lines_before.append('# This is the basis of a recipe and may need further editing in order to be fully functional.') lines_before.append('# (Feel free to remove these comments when editing.)') lines_before.append('#') licvalues = guess_license(srctree_use) lic_files_chksum = [] if licvalues: licenses = [] for licvalue in licvalues: if not licvalue[0] in licenses: licenses.append(licvalue[0]) lic_files_chksum.append('file://%s;md5=%s' % (licvalue[1], licvalue[2])) lines_before.append('# WARNING: the following LICENSE and LIC_FILES_CHKSUM values are best guesses - it is') lines_before.append('# your responsibility to verify that the values are complete and correct.') if len(licvalues) > 1: lines_before.append('#') lines_before.append('# NOTE: multiple licenses have been detected; if that is correct you should separate') lines_before.append('# these in the LICENSE value using & if the multiple licenses all apply, or | if there') lines_before.append('# is a choice between the multiple licenses. If in doubt, check the accompanying') lines_before.append('# documentation to determine which situation is applicable.') else: lines_before.append('# Unable to find any files that looked like license statements. Check the accompanying') lines_before.append('# documentation and source headers and set LICENSE and LIC_FILES_CHKSUM accordingly.') lines_before.append('#') lines_before.append('# NOTE: LICENSE is being set to "CLOSED" to allow you to at least start building - if') lines_before.append('# this is not accurate with respect to the licensing of the software being built (it') lines_before.append('# will not be in most cases) you must specify the correct value before using this') lines_before.append('# recipe for anything other than initial testing/development!') licenses = ['CLOSED'] lines_before.append('LICENSE = "%s"' % ' '.join(licenses)) lines_before.append('LIC_FILES_CHKSUM = "%s"' % ' \\\n '.join(lic_files_chksum)) lines_before.append('') classes = [] # FIXME This is kind of a hack, we probably ought to be using bitbake to do this pn = None pv = None if outfile: recipefn = os.path.splitext(os.path.basename(outfile))[0] fnsplit = recipefn.split('_') if len(fnsplit) > 1: pn = fnsplit[0] pv = fnsplit[1] else: pn = recipefn if args.version: pv = args.version if args.name: pn = args.name if args.name.endswith('-native'): if args.also_native: logger.error('--also-native cannot be specified for a recipe named *-native (*-native denotes a recipe that is already only for native) - either remove the -native suffix from the name or drop --also-native') sys.exit(1) classes.append('native') elif args.name.startswith('nativesdk-'): if args.also_native: logger.error('--also-native cannot be specified for a recipe named nativesdk-* (nativesdk-* denotes a recipe that is already only for nativesdk)') sys.exit(1) classes.append('nativesdk') if pv and pv not in 'git svn hg'.split(): realpv = pv else: realpv = None if srcuri and not realpv or not pn: name_pn, name_pv = determine_from_url(srcuri) if name_pn and not pn: pn = name_pn if name_pv and not realpv: realpv = name_pv if not srcuri: lines_before.append('# No information for SRC_URI yet (only an external source tree was specified)') lines_before.append('SRC_URI = "%s"' % srcuri) (md5value, sha256value) = checksums if md5value: lines_before.append('SRC_URI[md5sum] = "%s"' % md5value) if sha256value: lines_before.append('SRC_URI[sha256sum] = "%s"' % sha256value) if srcuri and supports_srcrev(srcuri): lines_before.append('') lines_before.append('# Modify these as desired') lines_before.append('PV = "%s+git${SRCPV}"' % (realpv or '1.0')) lines_before.append('SRCREV = "%s"' % srcrev) lines_before.append('') if srcsubdir: lines_before.append('S = "${WORKDIR}/%s"' % srcsubdir) lines_before.append('') if pkgarch: lines_after.append('PACKAGE_ARCH = "%s"' % pkgarch) lines_after.append('') if args.binary: lines_after.append('INSANE_SKIP_${PN} += "already-stripped"') lines_after.append('') # Find all plugins that want to register handlers logger.debug('Loading recipe handlers') raw_handlers = [] for plugin in plugins: if hasattr(plugin, 'register_recipe_handlers'): plugin.register_recipe_handlers(raw_handlers) # Sort handlers by priority handlers = [] for i, handler in enumerate(raw_handlers): if isinstance(handler, tuple): handlers.append((handler[0], handler[1], i)) else: handlers.append((handler, 0, i)) handlers.sort(key=lambda item: (item[1], -item[2]), reverse=True) for handler, priority, _ in handlers: logger.debug('Handler: %s (priority %d)' % (handler.__class__.__name__, priority)) handlers = [item[0] for item in handlers] # Apply the handlers handled = [] handled.append(('license', licvalues)) if args.binary: classes.append('bin_package') handled.append('buildsystem') extravalues = {} for handler in handlers: handler.process(srctree_use, classes, lines_before, lines_after, handled, extravalues) extrafiles = extravalues.pop('extrafiles', {}) if not realpv: realpv = extravalues.get('PV', None) if realpv: if not validate_pv(realpv): realpv = None else: realpv = realpv.lower().split()[0] if '_' in realpv: realpv = realpv.replace('_', '-') if not pn: pn = extravalues.get('PN', None) if pn: if pn.startswith('GNU '): pn = pn[4:] if ' ' in pn: # Probably a descriptive identifier rather than a proper name pn = None else: pn = pn.lower() if '_' in pn: pn = pn.replace('_', '-') if not outfile: if not pn: logger.error('Unable to determine short program name from source tree - please specify name with -N/--name or output file name with -o/--outfile') # devtool looks for this specific exit code, so don't change it sys.exit(15) else: if srcuri and srcuri.startswith(('git://', 'hg://', 'svn://')): outfile = '%s_%s.bb' % (pn, srcuri.split(':', 1)[0]) elif realpv: outfile = '%s_%s.bb' % (pn, realpv) else: outfile = '%s.bb' % pn if outdir: outfile = os.path.join(outdir, outfile) # We need to check this again if os.path.exists(outfile): logger.error('Output file %s already exists' % outfile) sys.exit(1) # Move any extra files the plugins created to a directory next to the recipe if extrafiles: if outfile == '-': extraoutdir = pn else: extraoutdir = os.path.join(os.path.dirname(outfile), pn) bb.utils.mkdirhier(extraoutdir) for destfn, extrafile in extrafiles.iteritems(): shutil.move(extrafile, os.path.join(extraoutdir, destfn)) lines = lines_before lines_before = [] skipblank = True for line in lines: if skipblank: skipblank = False if not line: continue if line.startswith('S = '): if realpv and pv not in 'git svn hg'.split(): line = line.replace(realpv, '${PV}') if pn: line = line.replace(pn, '${BPN}') if line == 'S = "${WORKDIR}/${BPN}-${PV}"': skipblank = True continue elif line.startswith('SRC_URI = '): if realpv: line = line.replace(realpv, '${PV}') elif line.startswith('PV = '): if realpv: line = re.sub('"[^+]*\+', '"%s+' % realpv, line) lines_before.append(line) if args.also_native: lines = lines_after lines_after = [] bbclassextend = None for line in lines: if line.startswith('BBCLASSEXTEND ='): splitval = line.split('"') if len(splitval) > 1: bbclassextend = splitval[1].split() if not 'native' in bbclassextend: bbclassextend.insert(0, 'native') line = 'BBCLASSEXTEND = "%s"' % ' '.join(bbclassextend) lines_after.append(line) if not bbclassextend: lines_after.append('BBCLASSEXTEND = "native"') outlines = [] outlines.extend(lines_before) if classes: if outlines[-1] and not outlines[-1].startswith('#'): outlines.append('') outlines.append('inherit %s' % ' '.join(classes)) outlines.append('') outlines.extend(lines_after) if args.extract_to: scriptutils.git_convert_standalone_clone(srctree) if os.path.isdir(args.extract_to): # If the directory exists we'll move the temp dir into it instead of # its contents - of course, we could try to always move its contents # but that is a pain if there are symlinks; the simplest solution is # to just remove it first os.rmdir(args.extract_to) shutil.move(srctree, args.extract_to) if tempsrc == srctree: tempsrc = None logger.info('Source extracted to %s' % args.extract_to) if outfile == '-': sys.stdout.write('\n'.join(outlines) + '\n') else: with open(outfile, 'w') as f: f.write('\n'.join(outlines) + '\n') logger.info('Recipe %s has been created; further editing may be required to make it fully functional' % outfile) if tempsrc: shutil.rmtree(tempsrc) return 0
def _extract_new_source(newpv, srctree, no_patch, srcrev, branch, keep_temp, tinfoil, rd): """Extract sources of a recipe with a new version""" def __run(cmd): """Simple wrapper which calls _run with srctree as cwd""" return _run(cmd, srctree) crd = rd.createCopy() pv = crd.getVar('PV', True) crd.setVar('PV', newpv) tmpsrctree = None uri, rev = _get_uri(crd) if srcrev: rev = srcrev if uri.startswith('git://'): __run('git checkout %s' % rev) __run('git tag -f devtool-base-new') md5 = None sha256 = None else: __run('git checkout -b devtool-%s' % newpv) tmpdir = tempfile.mkdtemp(prefix='devtool') try: md5, sha256 = scriptutils.fetch_uri(tinfoil.config_data, uri, tmpdir, rev) except bb.fetch2.FetchError as e: raise DevtoolError(e) tmpsrctree = _get_srctree(tmpdir) scrtree = os.path.abspath(srctree) _copy_source_code(tmpsrctree, srctree) (stdout,_) = __run('git ls-files --modified --others --exclude-standard') for f in stdout.splitlines(): __run('git add "%s"' % f) __run('git commit -q -m "Commit of upstream changes at version %s" --allow-empty' % newpv) __run('git tag -f devtool-base-%s' % newpv) (stdout, _) = __run('git rev-parse HEAD') rev = stdout.rstrip() if no_patch: patches = oe.recipeutils.get_recipe_patches(crd) if len(patches): logger.warn('By user choice, the following patches will NOT be applied') for patch in patches: logger.warn("%s" % os.path.basename(patch)) else: try: __run('git checkout devtool-patched -b %s' % branch) __run('git rebase %s' % rev) if uri.startswith('git://'): suffix = 'new' else: suffix = newpv __run('git tag -f devtool-patched-%s' % suffix) except bb.process.ExecutionError as e: logger.warn('Command \'%s\' failed:\n%s' % (e.command, e.stdout)) if tmpsrctree: if keep_temp: logger.info('Preserving temporary directory %s' % tmpsrctree) else: shutil.rmtree(tmpsrctree) return (rev, md5, sha256)
def create_recipe(args): import bb.process import tempfile import shutil pkgarch = "" if args.machine: pkgarch = "${MACHINE_ARCH}" checksums = (None, None) tempsrc = '' srcsubdir = '' srcrev = '${AUTOREV}' if '://' in args.source: # Fetch a URL fetchuri = urlparse.urldefrag(args.source)[0] if args.binary: # Assume the archive contains the directory structure verbatim # so we need to extract to a subdirectory fetchuri += ';subdir=%s' % os.path.splitext( os.path.basename(urlparse.urlsplit(fetchuri).path))[0] git_re = re.compile('(https?)://([^;]+\.git)(;.*)?') res = git_re.match(fetchuri) if res: # Need to switch the URI around so that the git fetcher is used fetchuri = 'git://%s;protocol=%s%s' % (res.group(2), res.group(1), res.group(3) or '') srcuri = fetchuri rev_re = re.compile(';rev=([^;]+)') res = rev_re.search(srcuri) if res: srcrev = res.group(1) srcuri = rev_re.sub('', srcuri) tempsrc = tempfile.mkdtemp(prefix='recipetool-') srctree = tempsrc logger.info('Fetching %s...' % srcuri) try: checksums = scriptutils.fetch_uri(tinfoil.config_data, fetchuri, srctree, srcrev) except bb.fetch2.FetchError: # Error already printed sys.exit(1) dirlist = os.listdir(srctree) if 'git.indirectionsymlink' in dirlist: dirlist.remove('git.indirectionsymlink') if len(dirlist) == 1: singleitem = os.path.join(srctree, dirlist[0]) if os.path.isdir(singleitem): # We unpacked a single directory, so we should use that srcsubdir = dirlist[0] srctree = os.path.join(srctree, srcsubdir) else: with open(singleitem, 'r') as f: if '<html' in f.read(100).lower(): logger.error( 'Fetching "%s" returned a single HTML page - check the URL is correct and functional' % fetchuri) sys.exit(1) else: # Assume we're pointing to an existing source tree if args.extract_to: logger.error( '--extract-to cannot be specified if source is a directory') sys.exit(1) if not os.path.isdir(args.source): logger.error('Invalid source directory %s' % args.source) sys.exit(1) srcuri = '' srctree = args.source if args.outfile and os.path.isdir(args.outfile): outfile = None outdir = args.outfile else: outfile = args.outfile outdir = None if outfile and outfile != '-': if os.path.exists(outfile): logger.error('Output file %s already exists' % outfile) sys.exit(1) lines_before = [] lines_after = [] lines_before.append('# Recipe created by %s' % os.path.basename(sys.argv[0])) lines_before.append( '# This is the basis of a recipe and may need further editing in order to be fully functional.' ) lines_before.append('# (Feel free to remove these comments when editing.)') lines_before.append('#') licvalues = guess_license(srctree) lic_files_chksum = [] if licvalues: licenses = [] for licvalue in licvalues: if not licvalue[0] in licenses: licenses.append(licvalue[0]) lic_files_chksum.append('file://%s;md5=%s' % (licvalue[1], licvalue[2])) lines_before.append( '# WARNING: the following LICENSE and LIC_FILES_CHKSUM values are best guesses - it is' ) lines_before.append( '# your responsibility to verify that the values are complete and correct.' ) if len(licvalues) > 1: lines_before.append('#') lines_before.append( '# NOTE: multiple licenses have been detected; if that is correct you should separate' ) lines_before.append( '# these in the LICENSE value using & if the multiple licenses all apply, or | if there' ) lines_before.append( '# is a choice between the multiple licenses. If in doubt, check the accompanying' ) lines_before.append( '# documentation to determine which situation is applicable.') else: lines_before.append( '# Unable to find any files that looked like license statements. Check the accompanying' ) lines_before.append( '# documentation and source headers and set LICENSE and LIC_FILES_CHKSUM accordingly.' ) lines_before.append('#') lines_before.append( '# NOTE: LICENSE is being set to "CLOSED" to allow you to at least start building - if' ) lines_before.append( '# this is not accurate with respect to the licensing of the software being built (it' ) lines_before.append( '# will not be in most cases) you must specify the correct value before using this' ) lines_before.append( '# recipe for anything other than initial testing/development!') licenses = ['CLOSED'] lines_before.append('LICENSE = "%s"' % ' '.join(licenses)) lines_before.append('LIC_FILES_CHKSUM = "%s"' % ' \\\n '.join(lic_files_chksum)) lines_before.append('') classes = [] # FIXME This is kind of a hack, we probably ought to be using bitbake to do this pn = None pv = None if outfile: recipefn = os.path.splitext(os.path.basename(outfile))[0] fnsplit = recipefn.split('_') if len(fnsplit) > 1: pn = fnsplit[0] pv = fnsplit[1] else: pn = recipefn if args.version: pv = args.version if args.name: pn = args.name if args.name.endswith('-native'): if args.also_native: logger.error( '--also-native cannot be specified for a recipe named *-native (*-native denotes a recipe that is already only for native) - either remove the -native suffix from the name or drop --also-native' ) sys.exit(1) classes.append('native') elif args.name.startswith('nativesdk-'): if args.also_native: logger.error( '--also-native cannot be specified for a recipe named nativesdk-* (nativesdk-* denotes a recipe that is already only for nativesdk)' ) sys.exit(1) classes.append('nativesdk') if pv and pv not in 'git svn hg'.split(): realpv = pv else: realpv = None if srcuri and not realpv or not pn: parseres = urlparse.urlparse(srcuri) if parseres.path: srcfile = os.path.basename(parseres.path) name_pn, name_pv = determine_from_filename(srcfile) logger.debug( 'Determined from filename: name = "%s", version = "%s"' % (name_pn, name_pv)) if name_pn and not pn: pn = name_pn if name_pv and not realpv: realpv = name_pv if not srcuri: lines_before.append( '# No information for SRC_URI yet (only an external source tree was specified)' ) lines_before.append('SRC_URI = "%s"' % srcuri) (md5value, sha256value) = checksums if md5value: lines_before.append('SRC_URI[md5sum] = "%s"' % md5value) if sha256value: lines_before.append('SRC_URI[sha256sum] = "%s"' % sha256value) if srcuri and supports_srcrev(srcuri): lines_before.append('') lines_before.append('# Modify these as desired') lines_before.append('PV = "%s+git${SRCPV}"' % (realpv or '1.0')) lines_before.append('SRCREV = "%s"' % srcrev) lines_before.append('') if srcsubdir: lines_before.append('S = "${WORKDIR}/%s"' % srcsubdir) lines_before.append('') if pkgarch: lines_after.append('PACKAGE_ARCH = "%s"' % pkgarch) lines_after.append('') if args.binary: lines_after.append('INSANE_SKIP_${PN} += "already-stripped"') lines_after.append('') # Find all plugins that want to register handlers logger.debug('Loading recipe handlers') raw_handlers = [] for plugin in plugins: if hasattr(plugin, 'register_recipe_handlers'): plugin.register_recipe_handlers(raw_handlers) # Sort handlers by priority handlers = [] for i, handler in enumerate(raw_handlers): if isinstance(handler, tuple): handlers.append((handler[0], handler[1], i)) else: handlers.append((handler, 0, i)) handlers.sort(key=lambda item: (item[1], -item[2]), reverse=True) for handler, priority, _ in handlers: logger.debug('Handler: %s (priority %d)' % (handler.__class__.__name__, priority)) handlers = [item[0] for item in handlers] # Apply the handlers handled = [] if args.binary: classes.append('bin_package') handled.append('buildsystem') extravalues = {} for handler in handlers: handler.process(srctree, classes, lines_before, lines_after, handled, extravalues) if not realpv: realpv = extravalues.get('PV', None) if realpv: if not validate_pv(realpv): realpv = None else: realpv = realpv.lower().split()[0] if '_' in realpv: realpv = realpv.replace('_', '-') if not pn: pn = extravalues.get('PN', None) if pn: if pn.startswith('GNU '): pn = pn[4:] if ' ' in pn: # Probably a descriptive identifier rather than a proper name pn = None else: pn = pn.lower() if '_' in pn: pn = pn.replace('_', '-') if not outfile: if not pn: logger.error( 'Unable to determine short program name from source tree - please specify name with -N/--name or output file name with -o/--outfile' ) # devtool looks for this specific exit code, so don't change it sys.exit(15) else: if srcuri and srcuri.startswith(('git://', 'hg://', 'svn://')): outfile = '%s_%s.bb' % (pn, srcuri.split(':', 1)[0]) elif realpv: outfile = '%s_%s.bb' % (pn, realpv) else: outfile = '%s.bb' % pn if outdir: outfile = os.path.join(outdir, outfile) # We need to check this again if os.path.exists(outfile): logger.error('Output file %s already exists' % outfile) sys.exit(1) lines = lines_before lines_before = [] skipblank = True for line in lines: if skipblank: skipblank = False if not line: continue if line.startswith('S = '): if realpv and pv not in 'git svn hg'.split(): line = line.replace(realpv, '${PV}') if pn: line = line.replace(pn, '${BPN}') if line == 'S = "${WORKDIR}/${BPN}-${PV}"': skipblank = True continue elif line.startswith('SRC_URI = '): if realpv: line = line.replace(realpv, '${PV}') elif line.startswith('PV = '): if realpv: line = re.sub('"[^+]*\+', '"%s+' % realpv, line) lines_before.append(line) if args.also_native: lines = lines_after lines_after = [] bbclassextend = None for line in lines: if line.startswith('BBCLASSEXTEND ='): splitval = line.split('"') if len(splitval) > 1: bbclassextend = splitval[1].split() if not 'native' in bbclassextend: bbclassextend.insert(0, 'native') line = 'BBCLASSEXTEND = "%s"' % ' '.join(bbclassextend) lines_after.append(line) if not bbclassextend: lines_after.append('BBCLASSEXTEND = "native"') outlines = [] outlines.extend(lines_before) if classes: outlines.append('inherit %s' % ' '.join(classes)) outlines.append('') outlines.extend(lines_after) if args.extract_to: scriptutils.git_convert_standalone_clone(srctree) if os.path.isdir(args.extract_to): # If the directory exists we'll move the temp dir into it instead of # its contents - of course, we could try to always move its contents # but that is a pain if there are symlinks; the simplest solution is # to just remove it first os.rmdir(args.extract_to) shutil.move(srctree, args.extract_to) if tempsrc == srctree: tempsrc = None logger.info('Source extracted to %s' % args.extract_to) if outfile == '-': sys.stdout.write('\n'.join(outlines) + '\n') else: with open(outfile, 'w') as f: f.write('\n'.join(outlines) + '\n') logger.info( 'Recipe %s has been created; further editing may be required to make it fully functional' % outfile) if tempsrc: shutil.rmtree(tempsrc) return 0
def _extract_new_source(newpv, srctree, no_patch, srcrev, branch, keep_temp, tinfoil, rd): """Extract sources of a recipe with a new version""" def __run(cmd): """Simple wrapper which calls _run with srctree as cwd""" return _run(cmd, srctree) crd = rd.createCopy() pv = crd.getVar('PV', True) crd.setVar('PV', newpv) tmpsrctree = None uri, rev = _get_uri(crd) if srcrev: rev = srcrev if uri.startswith('git://'): __run('git fetch') __run('git checkout %s' % rev) __run('git tag -f devtool-base-new') md5 = None sha256 = None else: __run('git checkout -b devtool-%s' % newpv) tmpdir = tempfile.mkdtemp(prefix='devtool') try: md5, sha256 = scriptutils.fetch_uri(tinfoil.config_data, uri, tmpdir, rev) except bb.fetch2.FetchError as e: raise DevtoolError(e) tmpsrctree = _get_srctree(tmpdir) scrtree = os.path.abspath(srctree) _copy_source_code(tmpsrctree, srctree) (stdout, _) = __run('git ls-files --modified --others --exclude-standard') for f in stdout.splitlines(): __run('git add "%s"' % f) __run( 'git commit -q -m "Commit of upstream changes at version %s" --allow-empty' % newpv) __run('git tag -f devtool-base-%s' % newpv) (stdout, _) = __run('git rev-parse HEAD') rev = stdout.rstrip() if no_patch: patches = oe.recipeutils.get_recipe_patches(crd) if len(patches): logger.warn( 'By user choice, the following patches will NOT be applied') for patch in patches: logger.warn("%s" % os.path.basename(patch)) else: try: __run('git checkout devtool-patched -b %s' % branch) __run('git rebase %s' % rev) if uri.startswith('git://'): suffix = 'new' else: suffix = newpv __run('git tag -f devtool-patched-%s' % suffix) except bb.process.ExecutionError as e: logger.warn('Command \'%s\' failed:\n%s' % (e.command, e.stdout)) if tmpsrctree: if keep_temp: logger.info('Preserving temporary directory %s' % tmpsrctree) else: shutil.rmtree(tmpsrctree) return (rev, md5, sha256)
def _extract_new_source(newpv, srctree, no_patch, srcrev, branch, keep_temp, tinfoil, rd): """Extract sources of a recipe with a new version""" def __run(cmd): """Simple wrapper which calls _run with srctree as cwd""" return _run(cmd, srctree) crd = rd.createCopy() pv = crd.getVar('PV', True) crd.setVar('PV', newpv) tmpsrctree = None uri, rev = _get_uri(crd) if srcrev: rev = srcrev if uri.startswith('git://'): __run('git fetch') __run('git checkout %s' % rev) __run('git tag -f devtool-base-new') md5 = None sha256 = None else: __run('git checkout devtool-base -b devtool-%s' % newpv) tmpdir = tempfile.mkdtemp(prefix='devtool') try: md5, sha256 = scriptutils.fetch_uri(tinfoil.config_data, uri, tmpdir, rev) except bb.fetch2.FetchError as e: raise DevtoolError(e) tmpsrctree = _get_srctree(tmpdir) srctree = os.path.abspath(srctree) # Delete all sources so we ensure no stray files are left over for item in os.listdir(srctree): if item in ['.git', 'oe-local-files']: continue itempath = os.path.join(srctree, item) if os.path.isdir(itempath): shutil.rmtree(itempath) else: os.remove(itempath) # Copy in new ones _copy_source_code(tmpsrctree, srctree) (stdout,_) = __run('git ls-files --modified --others --exclude-standard') for f in stdout.splitlines(): __run('git add "%s"' % f) __run('git commit -q -m "Commit of upstream changes at version %s" --allow-empty' % newpv) __run('git tag -f devtool-base-%s' % newpv) (stdout, _) = __run('git rev-parse HEAD') rev = stdout.rstrip() if no_patch: patches = oe.recipeutils.get_recipe_patches(crd) if len(patches): logger.warn('By user choice, the following patches will NOT be applied') for patch in patches: logger.warn("%s" % os.path.basename(patch)) else: __run('git checkout devtool-patched -b %s' % branch) skiptag = False try: __run('git rebase %s' % rev) except bb.process.ExecutionError as e: skiptag = True if 'conflict' in e.stdout: logger.warn('Command \'%s\' failed:\n%s\n\nYou will need to resolve conflicts in order to complete the upgrade.' % (e.command, e.stdout.rstrip())) else: logger.warn('Command \'%s\' failed:\n%s' % (e.command, e.stdout)) if not skiptag: if uri.startswith('git://'): suffix = 'new' else: suffix = newpv __run('git tag -f devtool-patched-%s' % suffix) if tmpsrctree: if keep_temp: logger.info('Preserving temporary directory %s' % tmpsrctree) else: shutil.rmtree(tmpsrctree) return (rev, md5, sha256)
def create_recipe(args): import bb.process import tempfile import shutil pkgarch = "" if args.machine: pkgarch = "${MACHINE_ARCH}" checksums = (None, None) tempsrc = '' srcsubdir = '' srcrev = '${AUTOREV}' if '://' in args.source: # Fetch a URL fetchuri = urlparse.urldefrag(args.source)[0] if args.binary: # Assume the archive contains the directory structure verbatim # so we need to extract to a subdirectory fetchuri += ';subdir=%s' % os.path.splitext(os.path.basename(urlparse.urlsplit(fetchuri).path))[0] srcuri = fetchuri rev_re = re.compile(';rev=([^;]+)') res = rev_re.search(srcuri) if res: srcrev = res.group(1) srcuri = rev_re.sub('', srcuri) tempsrc = tempfile.mkdtemp(prefix='recipetool-') srctree = tempsrc logger.info('Fetching %s...' % srcuri) checksums = scriptutils.fetch_uri(tinfoil.config_data, fetchuri, srctree, srcrev) dirlist = os.listdir(srctree) if 'git.indirectionsymlink' in dirlist: dirlist.remove('git.indirectionsymlink') if len(dirlist) == 1 and os.path.isdir(os.path.join(srctree, dirlist[0])): # We unpacked a single directory, so we should use that srcsubdir = dirlist[0] srctree = os.path.join(srctree, srcsubdir) else: # Assume we're pointing to an existing source tree if args.extract_to: logger.error('--extract-to cannot be specified if source is a directory') sys.exit(1) if not os.path.isdir(args.source): logger.error('Invalid source directory %s' % args.source) sys.exit(1) srcuri = '' srctree = args.source outfile = args.outfile if outfile and outfile != '-': if os.path.exists(outfile): logger.error('Output file %s already exists' % outfile) sys.exit(1) lines_before = [] lines_after = [] lines_before.append('# Recipe created by %s' % os.path.basename(sys.argv[0])) lines_before.append('# This is the basis of a recipe and may need further editing in order to be fully functional.') lines_before.append('# (Feel free to remove these comments when editing.)') lines_before.append('#') licvalues = guess_license(srctree) lic_files_chksum = [] if licvalues: licenses = [] for licvalue in licvalues: if not licvalue[0] in licenses: licenses.append(licvalue[0]) lic_files_chksum.append('file://%s;md5=%s' % (licvalue[1], licvalue[2])) lines_before.append('# WARNING: the following LICENSE and LIC_FILES_CHKSUM values are best guesses - it is') lines_before.append('# your responsibility to verify that the values are complete and correct.') if len(licvalues) > 1: lines_before.append('#') lines_before.append('# NOTE: multiple licenses have been detected; if that is correct you should separate') lines_before.append('# these in the LICENSE value using & if the multiple licenses all apply, or | if there') lines_before.append('# is a choice between the multiple licenses. If in doubt, check the accompanying') lines_before.append('# documentation to determine which situation is applicable.') else: lines_before.append('# Unable to find any files that looked like license statements. Check the accompanying') lines_before.append('# documentation and source headers and set LICENSE and LIC_FILES_CHKSUM accordingly.') lines_before.append('#') lines_before.append('# NOTE: LICENSE is being set to "CLOSED" to allow you to at least start building - if') lines_before.append('# this is not accurate with respect to the licensing of the software being built (it') lines_before.append('# will not be in most cases) you must specify the correct value before using this') lines_before.append('# recipe for anything other than initial testing/development!') licenses = ['CLOSED'] lines_before.append('LICENSE = "%s"' % ' '.join(licenses)) lines_before.append('LIC_FILES_CHKSUM = "%s"' % ' \\\n '.join(lic_files_chksum)) lines_before.append('') # FIXME This is kind of a hack, we probably ought to be using bitbake to do this # we'd also want a way to automatically set outfile based upon auto-detecting these values from the source if possible recipefn = os.path.splitext(os.path.basename(outfile))[0] fnsplit = recipefn.split('_') if len(fnsplit) > 1: pn = fnsplit[0] pv = fnsplit[1] else: pn = recipefn pv = None if args.version: pv = args.version if pv and pv not in 'git svn hg'.split(): realpv = pv else: realpv = None if srcuri: if realpv: srcuri = srcuri.replace(realpv, '${PV}') else: lines_before.append('# No information for SRC_URI yet (only an external source tree was specified)') lines_before.append('SRC_URI = "%s"' % srcuri) (md5value, sha256value) = checksums if md5value: lines_before.append('SRC_URI[md5sum] = "%s"' % md5value) if sha256value: lines_before.append('SRC_URI[sha256sum] = "%s"' % sha256value) if srcuri and supports_srcrev(srcuri): lines_before.append('') lines_before.append('# Modify these as desired') lines_before.append('PV = "%s+git${SRCPV}"' % (realpv or '1.0')) lines_before.append('SRCREV = "%s"' % srcrev) lines_before.append('') if srcsubdir and pv: if srcsubdir == "%s-%s" % (pn, pv): # This would be the default, so we don't need to set S in the recipe srcsubdir = '' if srcsubdir: if pv and pv not in 'git svn hg'.split(): srcsubdir = srcsubdir.replace(pv, '${PV}') lines_before.append('S = "${WORKDIR}/%s"' % srcsubdir) lines_before.append('') if pkgarch: lines_after.append('PACKAGE_ARCH = "%s"' % pkgarch) lines_after.append('') if args.binary: lines_after.append('INSANE_SKIP_${PN} += "already-stripped"') lines_after.append('') # Find all plugins that want to register handlers handlers = [] for plugin in plugins: if hasattr(plugin, 'register_recipe_handlers'): plugin.register_recipe_handlers(handlers) # Apply the handlers classes = [] handled = [] if args.binary: classes.append('bin_package') handled.append('buildsystem') for handler in handlers: handler.process(srctree, classes, lines_before, lines_after, handled) outlines = [] outlines.extend(lines_before) if classes: outlines.append('inherit %s' % ' '.join(classes)) outlines.append('') outlines.extend(lines_after) if args.extract_to: scriptutils.git_convert_standalone_clone(srctree) if os.path.isdir(args.extract_to): # If the directory exists we'll move the temp dir into it instead of # its contents - of course, we could try to always move its contents # but that is a pain if there are symlinks; the simplest solution is # to just remove it first os.rmdir(args.extract_to) shutil.move(srctree, args.extract_to) logger.info('Source extracted to %s' % args.extract_to) if outfile == '-': sys.stdout.write('\n'.join(outlines) + '\n') else: with open(outfile, 'w') as f: f.write('\n'.join(outlines) + '\n') logger.info('Recipe %s has been created; further editing may be required to make it fully functional' % outfile) if tempsrc: shutil.rmtree(tempsrc) return 0