def create_recipe(args): import bb.process import tempfile import shutil import oe.recipeutils pkgarch = "" if args.machine: pkgarch = "${MACHINE_ARCH}" extravalues = {} checksums = (None, None) tempsrc = '' source = args.source srcsubdir = '' srcrev = '${AUTOREV}' if os.path.isfile(source): source = 'file://%s' % os.path.abspath(source) if scriptutils.is_src_url(source): # Fetch a URL fetchuri = reformat_git_uri(urldefrag(source)[0]) if args.binary: # Assume the archive contains the directory structure verbatim # so we need to extract to a subdirectory fetchuri += ';subdir=${BP}' srcuri = fetchuri rev_re = re.compile(';rev=([^;]+)') res = rev_re.search(srcuri) if res: srcrev = res.group(1) srcuri = rev_re.sub('', srcuri) tempsrc = tempfile.mkdtemp(prefix='recipetool-') srctree = tempsrc d = bb.data.createCopy(tinfoil.config_data) if fetchuri.startswith('npm://'): # Check if npm is available npm_bindir = check_npm(tinfoil, args.devtool) d.prependVar('PATH', '%s:' % npm_bindir) logger.info('Fetching %s...' % srcuri) try: checksums = scriptutils.fetch_uri(d, fetchuri, srctree, srcrev) except bb.fetch2.BBFetchException as e: logger.error(str(e).rstrip()) sys.exit(1) dirlist = os.listdir(srctree) if 'git.indirectionsymlink' in dirlist: dirlist.remove('git.indirectionsymlink') if len(dirlist) == 1: singleitem = os.path.join(srctree, dirlist[0]) if os.path.isdir(singleitem): # We unpacked a single directory, so we should use that srcsubdir = dirlist[0] srctree = os.path.join(srctree, srcsubdir) else: with open(singleitem, 'r', errors='surrogateescape') as f: if '<html' in f.read(100).lower(): logger.error('Fetching "%s" returned a single HTML page - check the URL is correct and functional' % fetchuri) sys.exit(1) if os.path.exists(os.path.join(srctree, '.gitmodules')) and srcuri.startswith('git://'): srcuri = 'gitsm://' + srcuri[6:] logger.info('Fetching submodules...') bb.process.run('git submodule update --init --recursive', cwd=srctree) if is_package(fetchuri): tmpfdir = tempfile.mkdtemp(prefix='recipetool-') try: pkgfile = None try: fileuri = fetchuri + ';unpack=0' scriptutils.fetch_uri(tinfoil.config_data, fileuri, tmpfdir, srcrev) for root, _, files in os.walk(tmpfdir): for f in files: pkgfile = os.path.join(root, f) break except bb.fetch2.BBFetchException as e: logger.warn('Second fetch to get metadata failed: %s' % str(e).rstrip()) if pkgfile: if pkgfile.endswith(('.deb', '.ipk')): stdout, _ = bb.process.run('ar x %s' % pkgfile, cwd=tmpfdir) stdout, _ = bb.process.run('tar xf control.tar.gz', cwd=tmpfdir) values = convert_debian(tmpfdir) extravalues.update(values) elif pkgfile.endswith(('.rpm', '.srpm')): stdout, _ = bb.process.run('rpm -qp --xml %s > pkginfo.xml' % pkgfile, cwd=tmpfdir) values = convert_rpm_xml(os.path.join(tmpfdir, 'pkginfo.xml')) extravalues.update(values) finally: shutil.rmtree(tmpfdir) else: # Assume we're pointing to an existing source tree if args.extract_to: logger.error('--extract-to cannot be specified if source is a directory') sys.exit(1) if not os.path.isdir(source): logger.error('Invalid source directory %s' % source) sys.exit(1) srctree = source srcuri = '' if os.path.exists(os.path.join(srctree, '.git')): # Try to get upstream repo location from origin remote try: stdout, _ = bb.process.run('git remote -v', cwd=srctree, shell=True) except bb.process.ExecutionError as e: stdout = None if stdout: for line in stdout.splitlines(): splitline = line.split() if len(splitline) > 1: if splitline[0] == 'origin' and scriptutils.is_src_url(splitline[1]): srcuri = reformat_git_uri(splitline[1]) srcsubdir = 'git' break if args.src_subdir: srcsubdir = os.path.join(srcsubdir, args.src_subdir) srctree_use = os.path.join(srctree, args.src_subdir) else: srctree_use = srctree if args.outfile and os.path.isdir(args.outfile): outfile = None outdir = args.outfile else: outfile = args.outfile outdir = None if outfile and outfile != '-': if os.path.exists(outfile): logger.error('Output file %s already exists' % outfile) sys.exit(1) lines_before = [] lines_after = [] lines_before.append('# Recipe created by %s' % os.path.basename(sys.argv[0])) lines_before.append('# This is the basis of a recipe and may need further editing in order to be fully functional.') lines_before.append('# (Feel free to remove these comments when editing.)') # We need a blank line here so that patch_recipe_lines can rewind before the LICENSE comments lines_before.append('') handled = [] licvalues = handle_license_vars(srctree_use, lines_before, handled, extravalues, tinfoil.config_data) classes = [] # FIXME This is kind of a hack, we probably ought to be using bitbake to do this pn = None pv = None if outfile: recipefn = os.path.splitext(os.path.basename(outfile))[0] fnsplit = recipefn.split('_') if len(fnsplit) > 1: pn = fnsplit[0] pv = fnsplit[1] else: pn = recipefn if args.version: pv = args.version if args.name: pn = args.name if args.name.endswith('-native'): if args.also_native: logger.error('--also-native cannot be specified for a recipe named *-native (*-native denotes a recipe that is already only for native) - either remove the -native suffix from the name or drop --also-native') sys.exit(1) classes.append('native') elif args.name.startswith('nativesdk-'): if args.also_native: logger.error('--also-native cannot be specified for a recipe named nativesdk-* (nativesdk-* denotes a recipe that is already only for nativesdk)') sys.exit(1) classes.append('nativesdk') if pv and pv not in 'git svn hg'.split(): realpv = pv else: realpv = None if srcuri and not realpv or not pn: name_pn, name_pv = determine_from_url(srcuri) if name_pn and not pn: pn = name_pn if name_pv and not realpv: realpv = name_pv if not srcuri: lines_before.append('# No information for SRC_URI yet (only an external source tree was specified)') lines_before.append('SRC_URI = "%s"' % srcuri) (md5value, sha256value) = checksums if md5value: lines_before.append('SRC_URI[md5sum] = "%s"' % md5value) if sha256value: lines_before.append('SRC_URI[sha256sum] = "%s"' % sha256value) if srcuri and supports_srcrev(srcuri): lines_before.append('') lines_before.append('# Modify these as desired') lines_before.append('PV = "%s+git${SRCPV}"' % (realpv or '1.0')) if not args.autorev and srcrev == '${AUTOREV}': if os.path.exists(os.path.join(srctree, '.git')): (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srctree) srcrev = stdout.rstrip() lines_before.append('SRCREV = "%s"' % srcrev) lines_before.append('') if srcsubdir and not args.binary: # (for binary packages we explicitly specify subdir= when fetching to # match the default value of S, so we don't need to set it in that case) lines_before.append('S = "${WORKDIR}/%s"' % srcsubdir) lines_before.append('') if pkgarch: lines_after.append('PACKAGE_ARCH = "%s"' % pkgarch) lines_after.append('') if args.binary: lines_after.append('INSANE_SKIP_${PN} += "already-stripped"') lines_after.append('') if args.fetch_dev: extravalues['fetchdev'] = True else: extravalues['fetchdev'] = None # Find all plugins that want to register handlers logger.debug('Loading recipe handlers') raw_handlers = [] for plugin in plugins: if hasattr(plugin, 'register_recipe_handlers'): plugin.register_recipe_handlers(raw_handlers) # Sort handlers by priority handlers = [] for i, handler in enumerate(raw_handlers): if isinstance(handler, tuple): handlers.append((handler[0], handler[1], i)) else: handlers.append((handler, 0, i)) handlers.sort(key=lambda item: (item[1], -item[2]), reverse=True) for handler, priority, _ in handlers: logger.debug('Handler: %s (priority %d)' % (handler.__class__.__name__, priority)) setattr(handler, '_devtool', args.devtool) handlers = [item[0] for item in handlers] # Apply the handlers if args.binary: classes.append('bin_package') handled.append('buildsystem') for handler in handlers: handler.process(srctree_use, classes, lines_before, lines_after, handled, extravalues) extrafiles = extravalues.pop('extrafiles', {}) extra_pn = extravalues.pop('PN', None) extra_pv = extravalues.pop('PV', None) if extra_pv and not realpv: realpv = extra_pv if not validate_pv(realpv): realpv = None else: realpv = realpv.lower().split()[0] if '_' in realpv: realpv = realpv.replace('_', '-') if extra_pn and not pn: pn = extra_pn if pn.startswith('GNU '): pn = pn[4:] if ' ' in pn: # Probably a descriptive identifier rather than a proper name pn = None else: pn = pn.lower() if '_' in pn: pn = pn.replace('_', '-') if not outfile: if not pn: log_error_cond('Unable to determine short program name from source tree - please specify name with -N/--name or output file name with -o/--outfile', args.devtool) # devtool looks for this specific exit code, so don't change it sys.exit(15) else: if srcuri and srcuri.startswith(('gitsm://', 'git://', 'hg://', 'svn://')): suffix = srcuri.split(':', 1)[0] if suffix == 'gitsm': suffix = 'git' outfile = '%s_%s.bb' % (pn, suffix) elif realpv: outfile = '%s_%s.bb' % (pn, realpv) else: outfile = '%s.bb' % pn if outdir: outfile = os.path.join(outdir, outfile) # We need to check this again if os.path.exists(outfile): logger.error('Output file %s already exists' % outfile) sys.exit(1) # Move any extra files the plugins created to a directory next to the recipe if extrafiles: if outfile == '-': extraoutdir = pn else: extraoutdir = os.path.join(os.path.dirname(outfile), pn) bb.utils.mkdirhier(extraoutdir) for destfn, extrafile in extrafiles.items(): shutil.move(extrafile, os.path.join(extraoutdir, destfn)) lines = lines_before lines_before = [] skipblank = True for line in lines: if skipblank: skipblank = False if not line: continue if line.startswith('S = '): if realpv and pv not in 'git svn hg'.split(): line = line.replace(realpv, '${PV}') if pn: line = line.replace(pn, '${BPN}') if line == 'S = "${WORKDIR}/${BPN}-${PV}"': skipblank = True continue elif line.startswith('SRC_URI = '): if realpv: line = line.replace(realpv, '${PV}') elif line.startswith('PV = '): if realpv: line = re.sub('"[^+]*\+', '"%s+' % realpv, line) lines_before.append(line) if args.also_native: lines = lines_after lines_after = [] bbclassextend = None for line in lines: if line.startswith('BBCLASSEXTEND ='): splitval = line.split('"') if len(splitval) > 1: bbclassextend = splitval[1].split() if not 'native' in bbclassextend: bbclassextend.insert(0, 'native') line = 'BBCLASSEXTEND = "%s"' % ' '.join(bbclassextend) lines_after.append(line) if not bbclassextend: lines_after.append('BBCLASSEXTEND = "native"') postinst = ("postinst", extravalues.pop('postinst', None)) postrm = ("postrm", extravalues.pop('postrm', None)) preinst = ("preinst", extravalues.pop('preinst', None)) prerm = ("prerm", extravalues.pop('prerm', None)) funcs = [postinst, postrm, preinst, prerm] for func in funcs: if func[1]: RecipeHandler.genfunction(lines_after, 'pkg_%s_${PN}' % func[0], func[1]) outlines = [] outlines.extend(lines_before) if classes: if outlines[-1] and not outlines[-1].startswith('#'): outlines.append('') outlines.append('inherit %s' % ' '.join(classes)) outlines.append('') outlines.extend(lines_after) if extravalues: if 'LICENSE' in extravalues and not licvalues: # Don't blow away 'CLOSED' value that comments say we set del extravalues['LICENSE'] _, outlines = oe.recipeutils.patch_recipe_lines(outlines, extravalues, trailing_newline=False) if args.extract_to: scriptutils.git_convert_standalone_clone(srctree) if os.path.isdir(args.extract_to): # If the directory exists we'll move the temp dir into it instead of # its contents - of course, we could try to always move its contents # but that is a pain if there are symlinks; the simplest solution is # to just remove it first os.rmdir(args.extract_to) shutil.move(srctree, args.extract_to) if tempsrc == srctree: tempsrc = None log_info_cond('Source extracted to %s' % args.extract_to, args.devtool) if outfile == '-': sys.stdout.write('\n'.join(outlines) + '\n') else: with open(outfile, 'w') as f: lastline = None for line in outlines: if not lastline and not line: # Skip extra blank lines continue f.write('%s\n' % line) lastline = line log_info_cond('Recipe %s has been created; further editing may be required to make it fully functional' % outfile, args.devtool) if tempsrc: if args.keep_temp: logger.info('Preserving temporary directory %s' % tempsrc) else: shutil.rmtree(tempsrc) return 0
def create_recipe(args): import bb.process import tempfile import shutil pkgarch = "" if args.machine: pkgarch = "${MACHINE_ARCH}" checksums = (None, None) tempsrc = '' srcsubdir = '' srcrev = '${AUTOREV}' if '://' in args.source: # Fetch a URL srcuri = args.source rev_re = re.compile(';rev=([^;]+)') res = rev_re.search(srcuri) if res: srcrev = res.group(1) srcuri = rev_re.sub('', srcuri) tempsrc = tempfile.mkdtemp(prefix='recipetool-') srctree = tempsrc logger.info('Fetching %s...' % srcuri) checksums = fetch_source(args.source, srctree, srcrev) dirlist = os.listdir(srctree) if 'git.indirectionsymlink' in dirlist: dirlist.remove('git.indirectionsymlink') if len(dirlist) == 1 and os.path.isdir(os.path.join(srctree, dirlist[0])): # We unpacked a single directory, so we should use that srcsubdir = dirlist[0] srctree = os.path.join(srctree, srcsubdir) else: # Assume we're pointing to an existing source tree if args.extract_to: logger.error('--extract-to cannot be specified if source is a directory') sys.exit(1) if not os.path.isdir(args.source): logger.error('Invalid source directory %s' % args.source) sys.exit(1) srcuri = '' srctree = args.source outfile = args.outfile if outfile and outfile != '-': if os.path.exists(outfile): logger.error('Output file %s already exists' % outfile) sys.exit(1) lines_before = [] lines_after = [] lines_before.append('# Recipe created by %s' % os.path.basename(sys.argv[0])) lines_before.append('# This is the basis of a recipe and may need further editing in order to be fully functional.') lines_before.append('# (Feel free to remove these comments when editing.)') lines_before.append('#') licvalues = guess_license(srctree) lic_files_chksum = [] if licvalues: licenses = [] for licvalue in licvalues: if not licvalue[0] in licenses: licenses.append(licvalue[0]) lic_files_chksum.append('file://%s;md5=%s' % (licvalue[1], licvalue[2])) lines_before.append('# WARNING: the following LICENSE and LIC_FILES_CHKSUM values are best guesses - it is') lines_before.append('# your responsibility to verify that the values are complete and correct.') if len(licvalues) > 1: lines_before.append('#') lines_before.append('# NOTE: multiple licenses have been detected; if that is correct you should separate') lines_before.append('# these in the LICENSE value using & if the multiple licenses all apply, or | if there') lines_before.append('# is a choice between the multiple licenses. If in doubt, check the accompanying') lines_before.append('# documentation to determine which situation is applicable.') else: lines_before.append('# Unable to find any files that looked like license statements. Check the accompanying') lines_before.append('# documentation and source headers and set LICENSE and LIC_FILES_CHKSUM accordingly.') lines_before.append('#') lines_before.append('# NOTE: LICENSE is being set to "CLOSED" to allow you to at least start building - if') lines_before.append('# this is not accurate with respect to the licensing of the software being built (it') lines_before.append('# will not be in most cases) you must specify the correct value before using this') lines_before.append('# recipe for anything other than initial testing/development!') licenses = ['CLOSED'] lines_before.append('LICENSE = "%s"' % ' '.join(licenses)) lines_before.append('LIC_FILES_CHKSUM = "%s"' % ' \\\n '.join(lic_files_chksum)) lines_before.append('') # FIXME This is kind of a hack, we probably ought to be using bitbake to do this # we'd also want a way to automatically set outfile based upon auto-detecting these values from the source if possible recipefn = os.path.splitext(os.path.basename(outfile))[0] fnsplit = recipefn.split('_') if len(fnsplit) > 1: pn = fnsplit[0] pv = fnsplit[1] else: pn = recipefn pv = None if args.version: pv = args.version if pv and pv not in 'git svn hg'.split(): realpv = pv else: realpv = None if srcuri: if realpv: srcuri = srcuri.replace(realpv, '${PV}') else: lines_before.append('# No information for SRC_URI yet (only an external source tree was specified)') lines_before.append('SRC_URI = "%s"' % srcuri) (md5value, sha256value) = checksums if md5value: lines_before.append('SRC_URI[md5sum] = "%s"' % md5value) if sha256value: lines_before.append('SRC_URI[sha256sum] = "%s"' % sha256value) if srcuri and supports_srcrev(srcuri): lines_before.append('') lines_before.append('# Modify these as desired') lines_before.append('PV = "%s+git${SRCPV}"' % (realpv or '1.0')) lines_before.append('SRCREV = "%s"' % srcrev) lines_before.append('') if srcsubdir and pv: if srcsubdir == "%s-%s" % (pn, pv): # This would be the default, so we don't need to set S in the recipe srcsubdir = '' if srcsubdir: if pv and pv not in 'git svn hg'.split(): srcsubdir = srcsubdir.replace(pv, '${PV}') lines_before.append('S = "${WORKDIR}/%s"' % srcsubdir) lines_before.append('') if pkgarch: lines_after.append('PACKAGE_ARCH = "%s"' % pkgarch) lines_after.append('') # Find all plugins that want to register handlers handlers = [] for plugin in plugins: if hasattr(plugin, 'register_recipe_handlers'): plugin.register_recipe_handlers(handlers) # Apply the handlers classes = [] handled = [] for handler in handlers: handler.process(srctree, classes, lines_before, lines_after, handled) outlines = [] outlines.extend(lines_before) if classes: outlines.append('inherit %s' % ' '.join(classes)) outlines.append('') outlines.extend(lines_after) if args.extract_to: scriptutils.git_convert_standalone_clone(srctree) shutil.move(srctree, args.extract_to) logger.info('Source extracted to %s' % args.extract_to) if outfile == '-': sys.stdout.write('\n'.join(outlines) + '\n') else: with open(outfile, 'w') as f: f.write('\n'.join(outlines) + '\n') logger.info('Recipe %s has been created; further editing may be required to make it fully functional' % outfile) if tempsrc: shutil.rmtree(tempsrc) return 0
def _extract_source(srctree, keep_temp, devbranch, d): import bb.event def eventfilter(name, handler, event, d): if name == 'base_eventhandler': return True else: return False if hasattr(bb.event, 'set_eventfilter'): bb.event.set_eventfilter(eventfilter) pn = d.getVar('PN', True) if not _check_compatible_recipe(pn, d): return None if os.path.exists(srctree): if not os.path.isdir(srctree): logger.error("output path %s exists and is not a directory" % srctree) return None elif os.listdir(srctree): logger.error("output path %s already exists and is non-empty" % srctree) return None # Prepare for shutil.move later on bb.utils.mkdirhier(srctree) os.rmdir(srctree) initial_rev = None tempdir = tempfile.mkdtemp(prefix='devtool') try: crd = d.createCopy() # Make a subdir so we guard against WORKDIR==S workdir = os.path.join(tempdir, 'workdir') crd.setVar('WORKDIR', workdir) crd.setVar('T', os.path.join(tempdir, 'temp')) if not crd.getVar('S', True).startswith(workdir): # Usually a shared workdir recipe (kernel, gcc) # Try to set a reasonable default if bb.data.inherits_class('kernel', d): crd.setVar('S', '${WORKDIR}/source') else: crd.setVar('S', '${WORKDIR}/${BP}') if bb.data.inherits_class('kernel', d): # We don't want to move the source to STAGING_KERNEL_DIR here crd.setVar('STAGING_KERNEL_DIR', '${S}') # FIXME: This is very awkward. Unfortunately it's not currently easy to properly # execute tasks outside of bitbake itself, until then this has to suffice if we # are to handle e.g. linux-yocto's extra tasks executed = [] def exec_task_func(func, report): if not func in executed: deps = crd.getVarFlag(func, 'deps') if deps: for taskdepfunc in deps: exec_task_func(taskdepfunc, True) if report: logger.info('Executing %s...' % func) fn = d.getVar('FILE', True) localdata = bb.build._task_data(fn, func, crd) bb.build.exec_func(func, localdata) executed.append(func) logger.info('Fetching %s...' % pn) exec_task_func('do_fetch', False) logger.info('Unpacking...') exec_task_func('do_unpack', False) srcsubdir = crd.getVar('S', True) patchsubdir = srcsubdir if srcsubdir != workdir and os.path.dirname(srcsubdir) != workdir: # Handle if S is set to a subdirectory of the source srcsubdir = os.path.join( workdir, os.path.relpath(srcsubdir, workdir).split(os.sep)[0]) scriptutils.git_convert_standalone_clone(srcsubdir) patchdir = os.path.join(patchsubdir, 'patches') haspatches = False if os.path.exists(patchdir): if os.listdir(patchdir): haspatches = True else: os.rmdir(patchdir) if bb.data.inherits_class('kernel-yocto', d): (stdout, _) = bb.process.run('git --git-dir="%s" rev-parse HEAD' % crd.expand('${WORKDIR}/git'), cwd=srcsubdir) initial_rev = stdout.rstrip() else: if not os.listdir(srcsubdir): logger.error( "no source unpacked to S, perhaps the %s recipe doesn't use any source?" % pn) return None if not os.path.exists(os.path.join(srcsubdir, '.git')): bb.process.run('git init', cwd=srcsubdir) bb.process.run('git add .', cwd=srcsubdir) bb.process.run( 'git commit -q -m "Initial commit from upstream at version %s"' % crd.getVar('PV', True), cwd=srcsubdir) (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srcsubdir) initial_rev = stdout.rstrip() bb.process.run('git checkout -b %s' % devbranch, cwd=srcsubdir) bb.process.run('git tag -f devtool-base', cwd=srcsubdir) crd.setVar('PATCHTOOL', 'git') logger.info('Patching...') exec_task_func('do_patch', False) bb.process.run('git tag -f devtool-patched', cwd=srcsubdir) if os.path.exists(patchdir): shutil.rmtree(patchdir) if haspatches: bb.process.run('git checkout patches', cwd=srcsubdir) shutil.move(srcsubdir, srctree) logger.info('Source tree extracted to %s' % srctree) finally: if keep_temp: logger.info('Preserving temporary directory %s' % tempdir) else: shutil.rmtree(tempdir) return initial_rev
def create_recipe(args): import bb.process import tempfile import shutil pkgarch = "" if args.machine: pkgarch = "${MACHINE_ARCH}" checksums = (None, None) tempsrc = '' srcsubdir = '' srcrev = '${AUTOREV}' if '://' in args.source: # Fetch a URL fetchuri = urlparse.urldefrag(args.source)[0] if args.binary: # Assume the archive contains the directory structure verbatim # so we need to extract to a subdirectory fetchuri += ';subdir=%s' % os.path.splitext( os.path.basename(urlparse.urlsplit(fetchuri).path))[0] srcuri = fetchuri rev_re = re.compile(';rev=([^;]+)') res = rev_re.search(srcuri) if res: srcrev = res.group(1) srcuri = rev_re.sub('', srcuri) tempsrc = tempfile.mkdtemp(prefix='recipetool-') srctree = tempsrc logger.info('Fetching %s...' % srcuri) checksums = scriptutils.fetch_uri(tinfoil.config_data, fetchuri, srctree, srcrev) dirlist = os.listdir(srctree) if 'git.indirectionsymlink' in dirlist: dirlist.remove('git.indirectionsymlink') if len(dirlist) == 1 and os.path.isdir( os.path.join(srctree, dirlist[0])): # We unpacked a single directory, so we should use that srcsubdir = dirlist[0] srctree = os.path.join(srctree, srcsubdir) else: # Assume we're pointing to an existing source tree if args.extract_to: logger.error( '--extract-to cannot be specified if source is a directory') sys.exit(1) if not os.path.isdir(args.source): logger.error('Invalid source directory %s' % args.source) sys.exit(1) srcuri = '' srctree = args.source outfile = args.outfile if outfile and outfile != '-': if os.path.exists(outfile): logger.error('Output file %s already exists' % outfile) sys.exit(1) lines_before = [] lines_after = [] lines_before.append('# Recipe created by %s' % os.path.basename(sys.argv[0])) lines_before.append( '# This is the basis of a recipe and may need further editing in order to be fully functional.' ) lines_before.append('# (Feel free to remove these comments when editing.)') lines_before.append('#') licvalues = guess_license(srctree) lic_files_chksum = [] if licvalues: licenses = [] for licvalue in licvalues: if not licvalue[0] in licenses: licenses.append(licvalue[0]) lic_files_chksum.append('file://%s;md5=%s' % (licvalue[1], licvalue[2])) lines_before.append( '# WARNING: the following LICENSE and LIC_FILES_CHKSUM values are best guesses - it is' ) lines_before.append( '# your responsibility to verify that the values are complete and correct.' ) if len(licvalues) > 1: lines_before.append('#') lines_before.append( '# NOTE: multiple licenses have been detected; if that is correct you should separate' ) lines_before.append( '# these in the LICENSE value using & if the multiple licenses all apply, or | if there' ) lines_before.append( '# is a choice between the multiple licenses. If in doubt, check the accompanying' ) lines_before.append( '# documentation to determine which situation is applicable.') else: lines_before.append( '# Unable to find any files that looked like license statements. Check the accompanying' ) lines_before.append( '# documentation and source headers and set LICENSE and LIC_FILES_CHKSUM accordingly.' ) lines_before.append('#') lines_before.append( '# NOTE: LICENSE is being set to "CLOSED" to allow you to at least start building - if' ) lines_before.append( '# this is not accurate with respect to the licensing of the software being built (it' ) lines_before.append( '# will not be in most cases) you must specify the correct value before using this' ) lines_before.append( '# recipe for anything other than initial testing/development!') licenses = ['CLOSED'] lines_before.append('LICENSE = "%s"' % ' '.join(licenses)) lines_before.append('LIC_FILES_CHKSUM = "%s"' % ' \\\n '.join(lic_files_chksum)) lines_before.append('') # FIXME This is kind of a hack, we probably ought to be using bitbake to do this # we'd also want a way to automatically set outfile based upon auto-detecting these values from the source if possible recipefn = os.path.splitext(os.path.basename(outfile))[0] fnsplit = recipefn.split('_') if len(fnsplit) > 1: pn = fnsplit[0] pv = fnsplit[1] else: pn = recipefn pv = None if args.version: pv = args.version if pv and pv not in 'git svn hg'.split(): realpv = pv else: realpv = None if srcuri: if realpv: srcuri = srcuri.replace(realpv, '${PV}') else: lines_before.append( '# No information for SRC_URI yet (only an external source tree was specified)' ) lines_before.append('SRC_URI = "%s"' % srcuri) (md5value, sha256value) = checksums if md5value: lines_before.append('SRC_URI[md5sum] = "%s"' % md5value) if sha256value: lines_before.append('SRC_URI[sha256sum] = "%s"' % sha256value) if srcuri and supports_srcrev(srcuri): lines_before.append('') lines_before.append('# Modify these as desired') lines_before.append('PV = "%s+git${SRCPV}"' % (realpv or '1.0')) lines_before.append('SRCREV = "%s"' % srcrev) lines_before.append('') if srcsubdir and pv: if srcsubdir == "%s-%s" % (pn, pv): # This would be the default, so we don't need to set S in the recipe srcsubdir = '' if srcsubdir: if pv and pv not in 'git svn hg'.split(): srcsubdir = srcsubdir.replace(pv, '${PV}') lines_before.append('S = "${WORKDIR}/%s"' % srcsubdir) lines_before.append('') if pkgarch: lines_after.append('PACKAGE_ARCH = "%s"' % pkgarch) lines_after.append('') if args.binary: lines_after.append('INSANE_SKIP_${PN} += "already-stripped"') lines_after.append('') # Find all plugins that want to register handlers handlers = [] for plugin in plugins: if hasattr(plugin, 'register_recipe_handlers'): plugin.register_recipe_handlers(handlers) # Apply the handlers classes = [] handled = [] if args.binary: classes.append('bin_package') handled.append('buildsystem') for handler in handlers: handler.process(srctree, classes, lines_before, lines_after, handled) outlines = [] outlines.extend(lines_before) if classes: outlines.append('inherit %s' % ' '.join(classes)) outlines.append('') outlines.extend(lines_after) if args.extract_to: scriptutils.git_convert_standalone_clone(srctree) if os.path.isdir(args.extract_to): # If the directory exists we'll move the temp dir into it instead of # its contents - of course, we could try to always move its contents # but that is a pain if there are symlinks; the simplest solution is # to just remove it first os.rmdir(args.extract_to) shutil.move(srctree, args.extract_to) logger.info('Source extracted to %s' % args.extract_to) if outfile == '-': sys.stdout.write('\n'.join(outlines) + '\n') else: with open(outfile, 'w') as f: f.write('\n'.join(outlines) + '\n') logger.info( 'Recipe %s has been created; further editing may be required to make it fully functional' % outfile) if tempsrc: shutil.rmtree(tempsrc) return 0
def create_recipe(args): import bb.process import tempfile import shutil pkgarch = "" if args.machine: pkgarch = "${MACHINE_ARCH}" checksums = (None, None) tempsrc = '' srcsubdir = '' srcrev = '${AUTOREV}' if '://' in args.source: # Fetch a URL fetchuri = reformat_git_uri(urlparse.urldefrag(args.source)[0]) if args.binary: # Assume the archive contains the directory structure verbatim # so we need to extract to a subdirectory fetchuri += ';subdir=%s' % os.path.splitext(os.path.basename(urlparse.urlsplit(fetchuri).path))[0] srcuri = fetchuri rev_re = re.compile(';rev=([^;]+)') res = rev_re.search(srcuri) if res: srcrev = res.group(1) srcuri = rev_re.sub('', srcuri) tempsrc = tempfile.mkdtemp(prefix='recipetool-') srctree = tempsrc if fetchuri.startswith('npm://'): # Check if npm is available npm = bb.utils.which(tinfoil.config_data.getVar('PATH', True), 'npm') if not npm: logger.error('npm:// URL requested but npm is not available - you need to either build nodejs-native or install npm using your package manager') sys.exit(1) logger.info('Fetching %s...' % srcuri) try: checksums = scriptutils.fetch_uri(tinfoil.config_data, fetchuri, srctree, srcrev) except bb.fetch2.BBFetchException as e: logger.error(str(e).rstrip()) sys.exit(1) dirlist = os.listdir(srctree) if 'git.indirectionsymlink' in dirlist: dirlist.remove('git.indirectionsymlink') if len(dirlist) == 1: singleitem = os.path.join(srctree, dirlist[0]) if os.path.isdir(singleitem): # We unpacked a single directory, so we should use that srcsubdir = dirlist[0] srctree = os.path.join(srctree, srcsubdir) else: with open(singleitem, 'r') as f: if '<html' in f.read(100).lower(): logger.error('Fetching "%s" returned a single HTML page - check the URL is correct and functional' % fetchuri) sys.exit(1) else: # Assume we're pointing to an existing source tree if args.extract_to: logger.error('--extract-to cannot be specified if source is a directory') sys.exit(1) if not os.path.isdir(args.source): logger.error('Invalid source directory %s' % args.source) sys.exit(1) srctree = args.source srcuri = '' if os.path.exists(os.path.join(srctree, '.git')): # Try to get upstream repo location from origin remote try: stdout, _ = bb.process.run('git remote -v', cwd=srctree, shell=True) except bb.process.ExecutionError as e: stdout = None if stdout: for line in stdout.splitlines(): splitline = line.split() if len(splitline) > 1: if splitline[0] == 'origin' and '://' in splitline[1]: srcuri = reformat_git_uri(splitline[1]) srcsubdir = 'git' break if args.src_subdir: srcsubdir = os.path.join(srcsubdir, args.src_subdir) srctree_use = os.path.join(srctree, args.src_subdir) else: srctree_use = srctree if args.outfile and os.path.isdir(args.outfile): outfile = None outdir = args.outfile else: outfile = args.outfile outdir = None if outfile and outfile != '-': if os.path.exists(outfile): logger.error('Output file %s already exists' % outfile) sys.exit(1) lines_before = [] lines_after = [] lines_before.append('# Recipe created by %s' % os.path.basename(sys.argv[0])) lines_before.append('# This is the basis of a recipe and may need further editing in order to be fully functional.') lines_before.append('# (Feel free to remove these comments when editing.)') lines_before.append('#') licvalues = guess_license(srctree_use) lic_files_chksum = [] if licvalues: licenses = [] for licvalue in licvalues: if not licvalue[0] in licenses: licenses.append(licvalue[0]) lic_files_chksum.append('file://%s;md5=%s' % (licvalue[1], licvalue[2])) lines_before.append('# WARNING: the following LICENSE and LIC_FILES_CHKSUM values are best guesses - it is') lines_before.append('# your responsibility to verify that the values are complete and correct.') if len(licvalues) > 1: lines_before.append('#') lines_before.append('# NOTE: multiple licenses have been detected; if that is correct you should separate') lines_before.append('# these in the LICENSE value using & if the multiple licenses all apply, or | if there') lines_before.append('# is a choice between the multiple licenses. If in doubt, check the accompanying') lines_before.append('# documentation to determine which situation is applicable.') else: lines_before.append('# Unable to find any files that looked like license statements. Check the accompanying') lines_before.append('# documentation and source headers and set LICENSE and LIC_FILES_CHKSUM accordingly.') lines_before.append('#') lines_before.append('# NOTE: LICENSE is being set to "CLOSED" to allow you to at least start building - if') lines_before.append('# this is not accurate with respect to the licensing of the software being built (it') lines_before.append('# will not be in most cases) you must specify the correct value before using this') lines_before.append('# recipe for anything other than initial testing/development!') licenses = ['CLOSED'] lines_before.append('LICENSE = "%s"' % ' '.join(licenses)) lines_before.append('LIC_FILES_CHKSUM = "%s"' % ' \\\n '.join(lic_files_chksum)) lines_before.append('') classes = [] # FIXME This is kind of a hack, we probably ought to be using bitbake to do this pn = None pv = None if outfile: recipefn = os.path.splitext(os.path.basename(outfile))[0] fnsplit = recipefn.split('_') if len(fnsplit) > 1: pn = fnsplit[0] pv = fnsplit[1] else: pn = recipefn if args.version: pv = args.version if args.name: pn = args.name if args.name.endswith('-native'): if args.also_native: logger.error('--also-native cannot be specified for a recipe named *-native (*-native denotes a recipe that is already only for native) - either remove the -native suffix from the name or drop --also-native') sys.exit(1) classes.append('native') elif args.name.startswith('nativesdk-'): if args.also_native: logger.error('--also-native cannot be specified for a recipe named nativesdk-* (nativesdk-* denotes a recipe that is already only for nativesdk)') sys.exit(1) classes.append('nativesdk') if pv and pv not in 'git svn hg'.split(): realpv = pv else: realpv = None if srcuri and not realpv or not pn: name_pn, name_pv = determine_from_url(srcuri) if name_pn and not pn: pn = name_pn if name_pv and not realpv: realpv = name_pv if not srcuri: lines_before.append('# No information for SRC_URI yet (only an external source tree was specified)') lines_before.append('SRC_URI = "%s"' % srcuri) (md5value, sha256value) = checksums if md5value: lines_before.append('SRC_URI[md5sum] = "%s"' % md5value) if sha256value: lines_before.append('SRC_URI[sha256sum] = "%s"' % sha256value) if srcuri and supports_srcrev(srcuri): lines_before.append('') lines_before.append('# Modify these as desired') lines_before.append('PV = "%s+git${SRCPV}"' % (realpv or '1.0')) lines_before.append('SRCREV = "%s"' % srcrev) lines_before.append('') if srcsubdir: lines_before.append('S = "${WORKDIR}/%s"' % srcsubdir) lines_before.append('') if pkgarch: lines_after.append('PACKAGE_ARCH = "%s"' % pkgarch) lines_after.append('') if args.binary: lines_after.append('INSANE_SKIP_${PN} += "already-stripped"') lines_after.append('') # Find all plugins that want to register handlers logger.debug('Loading recipe handlers') raw_handlers = [] for plugin in plugins: if hasattr(plugin, 'register_recipe_handlers'): plugin.register_recipe_handlers(raw_handlers) # Sort handlers by priority handlers = [] for i, handler in enumerate(raw_handlers): if isinstance(handler, tuple): handlers.append((handler[0], handler[1], i)) else: handlers.append((handler, 0, i)) handlers.sort(key=lambda item: (item[1], -item[2]), reverse=True) for handler, priority, _ in handlers: logger.debug('Handler: %s (priority %d)' % (handler.__class__.__name__, priority)) handlers = [item[0] for item in handlers] # Apply the handlers handled = [] handled.append(('license', licvalues)) if args.binary: classes.append('bin_package') handled.append('buildsystem') extravalues = {} for handler in handlers: handler.process(srctree_use, classes, lines_before, lines_after, handled, extravalues) extrafiles = extravalues.pop('extrafiles', {}) if not realpv: realpv = extravalues.get('PV', None) if realpv: if not validate_pv(realpv): realpv = None else: realpv = realpv.lower().split()[0] if '_' in realpv: realpv = realpv.replace('_', '-') if not pn: pn = extravalues.get('PN', None) if pn: if pn.startswith('GNU '): pn = pn[4:] if ' ' in pn: # Probably a descriptive identifier rather than a proper name pn = None else: pn = pn.lower() if '_' in pn: pn = pn.replace('_', '-') if not outfile: if not pn: logger.error('Unable to determine short program name from source tree - please specify name with -N/--name or output file name with -o/--outfile') # devtool looks for this specific exit code, so don't change it sys.exit(15) else: if srcuri and srcuri.startswith(('git://', 'hg://', 'svn://')): outfile = '%s_%s.bb' % (pn, srcuri.split(':', 1)[0]) elif realpv: outfile = '%s_%s.bb' % (pn, realpv) else: outfile = '%s.bb' % pn if outdir: outfile = os.path.join(outdir, outfile) # We need to check this again if os.path.exists(outfile): logger.error('Output file %s already exists' % outfile) sys.exit(1) # Move any extra files the plugins created to a directory next to the recipe if extrafiles: if outfile == '-': extraoutdir = pn else: extraoutdir = os.path.join(os.path.dirname(outfile), pn) bb.utils.mkdirhier(extraoutdir) for destfn, extrafile in extrafiles.iteritems(): shutil.move(extrafile, os.path.join(extraoutdir, destfn)) lines = lines_before lines_before = [] skipblank = True for line in lines: if skipblank: skipblank = False if not line: continue if line.startswith('S = '): if realpv and pv not in 'git svn hg'.split(): line = line.replace(realpv, '${PV}') if pn: line = line.replace(pn, '${BPN}') if line == 'S = "${WORKDIR}/${BPN}-${PV}"': skipblank = True continue elif line.startswith('SRC_URI = '): if realpv: line = line.replace(realpv, '${PV}') elif line.startswith('PV = '): if realpv: line = re.sub('"[^+]*\+', '"%s+' % realpv, line) lines_before.append(line) if args.also_native: lines = lines_after lines_after = [] bbclassextend = None for line in lines: if line.startswith('BBCLASSEXTEND ='): splitval = line.split('"') if len(splitval) > 1: bbclassextend = splitval[1].split() if not 'native' in bbclassextend: bbclassextend.insert(0, 'native') line = 'BBCLASSEXTEND = "%s"' % ' '.join(bbclassextend) lines_after.append(line) if not bbclassextend: lines_after.append('BBCLASSEXTEND = "native"') outlines = [] outlines.extend(lines_before) if classes: if outlines[-1] and not outlines[-1].startswith('#'): outlines.append('') outlines.append('inherit %s' % ' '.join(classes)) outlines.append('') outlines.extend(lines_after) if args.extract_to: scriptutils.git_convert_standalone_clone(srctree) if os.path.isdir(args.extract_to): # If the directory exists we'll move the temp dir into it instead of # its contents - of course, we could try to always move its contents # but that is a pain if there are symlinks; the simplest solution is # to just remove it first os.rmdir(args.extract_to) shutil.move(srctree, args.extract_to) if tempsrc == srctree: tempsrc = None logger.info('Source extracted to %s' % args.extract_to) if outfile == '-': sys.stdout.write('\n'.join(outlines) + '\n') else: with open(outfile, 'w') as f: f.write('\n'.join(outlines) + '\n') logger.info('Recipe %s has been created; further editing may be required to make it fully functional' % outfile) if tempsrc: shutil.rmtree(tempsrc) return 0
def _extract_source(srctree, keep_temp, devbranch, d): """Extract sources of a recipe""" import bb.event import oe.recipeutils def eventfilter(name, handler, event, d): """Bitbake event filter for devtool extract operation""" if name == 'base_eventhandler': return True else: return False if hasattr(bb.event, 'set_eventfilter'): bb.event.set_eventfilter(eventfilter) pn = d.getVar('PN', True) if not _check_compatible_recipe(pn, d): return None if os.path.exists(srctree): if not os.path.isdir(srctree): logger.error("output path %s exists and is not a directory" % srctree) return None elif os.listdir(srctree): logger.error("output path %s already exists and is non-empty" % srctree) return None # Prepare for shutil.move later on bb.utils.mkdirhier(srctree) os.rmdir(srctree) # We don't want notes to be printed, they are too verbose origlevel = bb.logger.getEffectiveLevel() if logger.getEffectiveLevel() > logging.DEBUG: bb.logger.setLevel(logging.WARNING) initial_rev = None tempdir = tempfile.mkdtemp(prefix='devtool') try: crd = d.createCopy() # Make a subdir so we guard against WORKDIR==S workdir = os.path.join(tempdir, 'workdir') crd.setVar('WORKDIR', workdir) crd.setVar('T', os.path.join(tempdir, 'temp')) if not crd.getVar('S', True).startswith(workdir): # Usually a shared workdir recipe (kernel, gcc) # Try to set a reasonable default if bb.data.inherits_class('kernel', d): crd.setVar('S', '${WORKDIR}/source') else: crd.setVar('S', '${WORKDIR}/${BP}') if bb.data.inherits_class('kernel', d): # We don't want to move the source to STAGING_KERNEL_DIR here crd.setVar('STAGING_KERNEL_DIR', '${S}') # FIXME: This is very awkward. Unfortunately it's not currently easy to properly # execute tasks outside of bitbake itself, until then this has to suffice if we # are to handle e.g. linux-yocto's extra tasks executed = [] def exec_task_func(func, report): """Run specific bitbake task for a recipe""" if not func in executed: deps = crd.getVarFlag(func, 'deps') if deps: for taskdepfunc in deps: exec_task_func(taskdepfunc, True) if report: logger.info('Executing %s...' % func) fn = d.getVar('FILE', True) localdata = bb.build._task_data(fn, func, crd) bb.build.exec_func(func, localdata) executed.append(func) logger.info('Fetching %s...' % pn) exec_task_func('do_fetch', False) logger.info('Unpacking...') exec_task_func('do_unpack', False) srcsubdir = crd.getVar('S', True) if srcsubdir == workdir: # Find non-patch sources that were "unpacked" to srctree directory recipe_patches = [os.path.basename(patch) for patch in oe.recipeutils.get_recipe_patches(crd)] src_files = [fname for fname in _ls_tree(workdir) if os.path.basename(fname) not in recipe_patches] # Force separate S so that patch files can be left out from srctree srcsubdir = tempfile.mkdtemp(dir=workdir) crd.setVar('S', srcsubdir) # Move source files to S for path in src_files: tgt_dir = os.path.join(srcsubdir, os.path.dirname(path)) bb.utils.mkdirhier(tgt_dir) shutil.move(os.path.join(workdir, path), tgt_dir) elif os.path.dirname(srcsubdir) != workdir: # Handle if S is set to a subdirectory of the source srcsubdir = os.path.join(workdir, os.path.relpath(srcsubdir, workdir).split(os.sep)[0]) scriptutils.git_convert_standalone_clone(srcsubdir) patchdir = os.path.join(srcsubdir, 'patches') haspatches = False if os.path.exists(patchdir): if os.listdir(patchdir): haspatches = True else: os.rmdir(patchdir) if bb.data.inherits_class('kernel-yocto', d): (stdout, _) = bb.process.run('git --git-dir="%s" rev-parse HEAD' % crd.expand('${WORKDIR}/git'), cwd=srcsubdir) initial_rev = stdout.rstrip() else: if not os.listdir(srcsubdir): logger.error("no source unpacked to S, perhaps the %s recipe doesn't use any source?" % pn) return None if not os.path.exists(os.path.join(srcsubdir, '.git')): bb.process.run('git init', cwd=srcsubdir) bb.process.run('git add .', cwd=srcsubdir) bb.process.run('git commit -q -m "Initial commit from upstream at version %s"' % crd.getVar('PV', True), cwd=srcsubdir) (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srcsubdir) initial_rev = stdout.rstrip() bb.process.run('git checkout -b %s' % devbranch, cwd=srcsubdir) bb.process.run('git tag -f devtool-base', cwd=srcsubdir) crd.setVar('PATCHTOOL', 'git') logger.info('Patching...') exec_task_func('do_patch', False) bb.process.run('git tag -f devtool-patched', cwd=srcsubdir) if os.path.exists(patchdir): shutil.rmtree(patchdir) if haspatches: bb.process.run('git checkout patches', cwd=srcsubdir) shutil.move(srcsubdir, srctree) logger.info('Source tree extracted to %s' % srctree) finally: bb.logger.setLevel(origlevel) if keep_temp: logger.info('Preserving temporary directory %s' % tempdir) else: shutil.rmtree(tempdir) return initial_rev
def _extract_source(srctree, keep_temp, devbranch, d): """Extract sources of a recipe""" import bb.event import oe.recipeutils def eventfilter(name, handler, event, d): """Bitbake event filter for devtool extract operation""" if name == 'base_eventhandler': return True else: return False if hasattr(bb.event, 'set_eventfilter'): bb.event.set_eventfilter(eventfilter) pn = d.getVar('PN', True) _check_compatible_recipe(pn, d) if os.path.exists(srctree): if not os.path.isdir(srctree): raise DevtoolError("output path %s exists and is not a directory" % srctree) elif os.listdir(srctree): raise DevtoolError("output path %s already exists and is " "non-empty" % srctree) # Prepare for shutil.move later on bb.utils.mkdirhier(srctree) os.rmdir(srctree) # We don't want notes to be printed, they are too verbose origlevel = bb.logger.getEffectiveLevel() if logger.getEffectiveLevel() > logging.DEBUG: bb.logger.setLevel(logging.WARNING) initial_rev = None tempdir = tempfile.mkdtemp(prefix='devtool') try: crd = d.createCopy() # Make a subdir so we guard against WORKDIR==S workdir = os.path.join(tempdir, 'workdir') crd.setVar('WORKDIR', workdir) crd.setVar('T', os.path.join(tempdir, 'temp')) if not crd.getVar('S', True).startswith(workdir): # Usually a shared workdir recipe (kernel, gcc) # Try to set a reasonable default if bb.data.inherits_class('kernel', d): crd.setVar('S', '${WORKDIR}/source') else: crd.setVar('S', '${WORKDIR}/${BP}') if bb.data.inherits_class('kernel', d): # We don't want to move the source to STAGING_KERNEL_DIR here crd.setVar('STAGING_KERNEL_DIR', '${S}') task_executor = BbTaskExecutor(crd) crd.setVar('EXTERNALSRC_forcevariable', '') logger.info('Fetching %s...' % pn) task_executor.exec_func('do_fetch', False) logger.info('Unpacking...') task_executor.exec_func('do_unpack', False) srcsubdir = crd.getVar('S', True) if srcsubdir == workdir: # Find non-patch sources that were "unpacked" to srctree directory recipe_patches = [ os.path.basename(patch) for patch in oe.recipeutils.get_recipe_patches(crd) ] src_files = [ fname for fname in _ls_tree(workdir) if os.path.basename(fname) not in recipe_patches ] # Force separate S so that patch files can be left out from srctree srcsubdir = tempfile.mkdtemp(dir=workdir) crd.setVar('S', srcsubdir) # Move source files to S for path in src_files: tgt_dir = os.path.join(srcsubdir, os.path.dirname(path)) bb.utils.mkdirhier(tgt_dir) shutil.move(os.path.join(workdir, path), tgt_dir) elif os.path.dirname(srcsubdir) != workdir: # Handle if S is set to a subdirectory of the source srcsubdir = os.path.join( workdir, os.path.relpath(srcsubdir, workdir).split(os.sep)[0]) scriptutils.git_convert_standalone_clone(srcsubdir) patchdir = os.path.join(srcsubdir, 'patches') haspatches = False if os.path.exists(patchdir): if os.listdir(patchdir): haspatches = True else: os.rmdir(patchdir) if bb.data.inherits_class('kernel-yocto', d): (stdout, _) = bb.process.run('git --git-dir="%s" rev-parse HEAD' % crd.expand('${WORKDIR}/git'), cwd=srcsubdir) initial_rev = stdout.rstrip() else: if not os.listdir(srcsubdir): raise DevtoolError("no source unpacked to S, perhaps the %s " "recipe doesn't use any source?" % pn) if not os.path.exists(os.path.join(srcsubdir, '.git')): bb.process.run('git init', cwd=srcsubdir) bb.process.run('git add .', cwd=srcsubdir) bb.process.run( 'git commit -q -m "Initial commit from upstream at version %s"' % crd.getVar('PV', True), cwd=srcsubdir) (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srcsubdir) initial_rev = stdout.rstrip() bb.process.run('git checkout -b %s' % devbranch, cwd=srcsubdir) bb.process.run('git tag -f devtool-base', cwd=srcsubdir) crd.setVar('PATCHTOOL', 'git') logger.info('Patching...') task_executor.exec_func('do_patch', False) bb.process.run('git tag -f devtool-patched', cwd=srcsubdir) if os.path.exists(patchdir): shutil.rmtree(patchdir) if haspatches: bb.process.run('git checkout patches', cwd=srcsubdir) shutil.move(srcsubdir, srctree) logger.info('Source tree extracted to %s' % srctree) finally: bb.logger.setLevel(origlevel) if keep_temp: logger.info('Preserving temporary directory %s' % tempdir) else: shutil.rmtree(tempdir) return initial_rev
def create_recipe(args): import bb.process import tempfile import shutil pkgarch = "" if args.machine: pkgarch = "${MACHINE_ARCH}" checksums = (None, None) tempsrc = '' srcsubdir = '' srcrev = '${AUTOREV}' if '://' in args.source: # Fetch a URL fetchuri = urlparse.urldefrag(args.source)[0] if args.binary: # Assume the archive contains the directory structure verbatim # so we need to extract to a subdirectory fetchuri += ';subdir=%s' % os.path.splitext( os.path.basename(urlparse.urlsplit(fetchuri).path))[0] git_re = re.compile('(https?)://([^;]+\.git)(;.*)?') res = git_re.match(fetchuri) if res: # Need to switch the URI around so that the git fetcher is used fetchuri = 'git://%s;protocol=%s%s' % (res.group(2), res.group(1), res.group(3) or '') srcuri = fetchuri rev_re = re.compile(';rev=([^;]+)') res = rev_re.search(srcuri) if res: srcrev = res.group(1) srcuri = rev_re.sub('', srcuri) tempsrc = tempfile.mkdtemp(prefix='recipetool-') srctree = tempsrc logger.info('Fetching %s...' % srcuri) try: checksums = scriptutils.fetch_uri(tinfoil.config_data, fetchuri, srctree, srcrev) except bb.fetch2.FetchError: # Error already printed sys.exit(1) dirlist = os.listdir(srctree) if 'git.indirectionsymlink' in dirlist: dirlist.remove('git.indirectionsymlink') if len(dirlist) == 1: singleitem = os.path.join(srctree, dirlist[0]) if os.path.isdir(singleitem): # We unpacked a single directory, so we should use that srcsubdir = dirlist[0] srctree = os.path.join(srctree, srcsubdir) else: with open(singleitem, 'r') as f: if '<html' in f.read(100).lower(): logger.error( 'Fetching "%s" returned a single HTML page - check the URL is correct and functional' % fetchuri) sys.exit(1) else: # Assume we're pointing to an existing source tree if args.extract_to: logger.error( '--extract-to cannot be specified if source is a directory') sys.exit(1) if not os.path.isdir(args.source): logger.error('Invalid source directory %s' % args.source) sys.exit(1) srcuri = '' srctree = args.source if args.outfile and os.path.isdir(args.outfile): outfile = None outdir = args.outfile else: outfile = args.outfile outdir = None if outfile and outfile != '-': if os.path.exists(outfile): logger.error('Output file %s already exists' % outfile) sys.exit(1) lines_before = [] lines_after = [] lines_before.append('# Recipe created by %s' % os.path.basename(sys.argv[0])) lines_before.append( '# This is the basis of a recipe and may need further editing in order to be fully functional.' ) lines_before.append('# (Feel free to remove these comments when editing.)') lines_before.append('#') licvalues = guess_license(srctree) lic_files_chksum = [] if licvalues: licenses = [] for licvalue in licvalues: if not licvalue[0] in licenses: licenses.append(licvalue[0]) lic_files_chksum.append('file://%s;md5=%s' % (licvalue[1], licvalue[2])) lines_before.append( '# WARNING: the following LICENSE and LIC_FILES_CHKSUM values are best guesses - it is' ) lines_before.append( '# your responsibility to verify that the values are complete and correct.' ) if len(licvalues) > 1: lines_before.append('#') lines_before.append( '# NOTE: multiple licenses have been detected; if that is correct you should separate' ) lines_before.append( '# these in the LICENSE value using & if the multiple licenses all apply, or | if there' ) lines_before.append( '# is a choice between the multiple licenses. If in doubt, check the accompanying' ) lines_before.append( '# documentation to determine which situation is applicable.') else: lines_before.append( '# Unable to find any files that looked like license statements. Check the accompanying' ) lines_before.append( '# documentation and source headers and set LICENSE and LIC_FILES_CHKSUM accordingly.' ) lines_before.append('#') lines_before.append( '# NOTE: LICENSE is being set to "CLOSED" to allow you to at least start building - if' ) lines_before.append( '# this is not accurate with respect to the licensing of the software being built (it' ) lines_before.append( '# will not be in most cases) you must specify the correct value before using this' ) lines_before.append( '# recipe for anything other than initial testing/development!') licenses = ['CLOSED'] lines_before.append('LICENSE = "%s"' % ' '.join(licenses)) lines_before.append('LIC_FILES_CHKSUM = "%s"' % ' \\\n '.join(lic_files_chksum)) lines_before.append('') classes = [] # FIXME This is kind of a hack, we probably ought to be using bitbake to do this pn = None pv = None if outfile: recipefn = os.path.splitext(os.path.basename(outfile))[0] fnsplit = recipefn.split('_') if len(fnsplit) > 1: pn = fnsplit[0] pv = fnsplit[1] else: pn = recipefn if args.version: pv = args.version if args.name: pn = args.name if args.name.endswith('-native'): if args.also_native: logger.error( '--also-native cannot be specified for a recipe named *-native (*-native denotes a recipe that is already only for native) - either remove the -native suffix from the name or drop --also-native' ) sys.exit(1) classes.append('native') elif args.name.startswith('nativesdk-'): if args.also_native: logger.error( '--also-native cannot be specified for a recipe named nativesdk-* (nativesdk-* denotes a recipe that is already only for nativesdk)' ) sys.exit(1) classes.append('nativesdk') if pv and pv not in 'git svn hg'.split(): realpv = pv else: realpv = None if srcuri and not realpv or not pn: parseres = urlparse.urlparse(srcuri) if parseres.path: srcfile = os.path.basename(parseres.path) name_pn, name_pv = determine_from_filename(srcfile) logger.debug( 'Determined from filename: name = "%s", version = "%s"' % (name_pn, name_pv)) if name_pn and not pn: pn = name_pn if name_pv and not realpv: realpv = name_pv if not srcuri: lines_before.append( '# No information for SRC_URI yet (only an external source tree was specified)' ) lines_before.append('SRC_URI = "%s"' % srcuri) (md5value, sha256value) = checksums if md5value: lines_before.append('SRC_URI[md5sum] = "%s"' % md5value) if sha256value: lines_before.append('SRC_URI[sha256sum] = "%s"' % sha256value) if srcuri and supports_srcrev(srcuri): lines_before.append('') lines_before.append('# Modify these as desired') lines_before.append('PV = "%s+git${SRCPV}"' % (realpv or '1.0')) lines_before.append('SRCREV = "%s"' % srcrev) lines_before.append('') if srcsubdir: lines_before.append('S = "${WORKDIR}/%s"' % srcsubdir) lines_before.append('') if pkgarch: lines_after.append('PACKAGE_ARCH = "%s"' % pkgarch) lines_after.append('') if args.binary: lines_after.append('INSANE_SKIP_${PN} += "already-stripped"') lines_after.append('') # Find all plugins that want to register handlers logger.debug('Loading recipe handlers') raw_handlers = [] for plugin in plugins: if hasattr(plugin, 'register_recipe_handlers'): plugin.register_recipe_handlers(raw_handlers) # Sort handlers by priority handlers = [] for i, handler in enumerate(raw_handlers): if isinstance(handler, tuple): handlers.append((handler[0], handler[1], i)) else: handlers.append((handler, 0, i)) handlers.sort(key=lambda item: (item[1], -item[2]), reverse=True) for handler, priority, _ in handlers: logger.debug('Handler: %s (priority %d)' % (handler.__class__.__name__, priority)) handlers = [item[0] for item in handlers] # Apply the handlers handled = [] if args.binary: classes.append('bin_package') handled.append('buildsystem') extravalues = {} for handler in handlers: handler.process(srctree, classes, lines_before, lines_after, handled, extravalues) if not realpv: realpv = extravalues.get('PV', None) if realpv: if not validate_pv(realpv): realpv = None else: realpv = realpv.lower().split()[0] if '_' in realpv: realpv = realpv.replace('_', '-') if not pn: pn = extravalues.get('PN', None) if pn: if pn.startswith('GNU '): pn = pn[4:] if ' ' in pn: # Probably a descriptive identifier rather than a proper name pn = None else: pn = pn.lower() if '_' in pn: pn = pn.replace('_', '-') if not outfile: if not pn: logger.error( 'Unable to determine short program name from source tree - please specify name with -N/--name or output file name with -o/--outfile' ) # devtool looks for this specific exit code, so don't change it sys.exit(15) else: if srcuri and srcuri.startswith(('git://', 'hg://', 'svn://')): outfile = '%s_%s.bb' % (pn, srcuri.split(':', 1)[0]) elif realpv: outfile = '%s_%s.bb' % (pn, realpv) else: outfile = '%s.bb' % pn if outdir: outfile = os.path.join(outdir, outfile) # We need to check this again if os.path.exists(outfile): logger.error('Output file %s already exists' % outfile) sys.exit(1) lines = lines_before lines_before = [] skipblank = True for line in lines: if skipblank: skipblank = False if not line: continue if line.startswith('S = '): if realpv and pv not in 'git svn hg'.split(): line = line.replace(realpv, '${PV}') if pn: line = line.replace(pn, '${BPN}') if line == 'S = "${WORKDIR}/${BPN}-${PV}"': skipblank = True continue elif line.startswith('SRC_URI = '): if realpv: line = line.replace(realpv, '${PV}') elif line.startswith('PV = '): if realpv: line = re.sub('"[^+]*\+', '"%s+' % realpv, line) lines_before.append(line) if args.also_native: lines = lines_after lines_after = [] bbclassextend = None for line in lines: if line.startswith('BBCLASSEXTEND ='): splitval = line.split('"') if len(splitval) > 1: bbclassextend = splitval[1].split() if not 'native' in bbclassextend: bbclassextend.insert(0, 'native') line = 'BBCLASSEXTEND = "%s"' % ' '.join(bbclassextend) lines_after.append(line) if not bbclassextend: lines_after.append('BBCLASSEXTEND = "native"') outlines = [] outlines.extend(lines_before) if classes: outlines.append('inherit %s' % ' '.join(classes)) outlines.append('') outlines.extend(lines_after) if args.extract_to: scriptutils.git_convert_standalone_clone(srctree) if os.path.isdir(args.extract_to): # If the directory exists we'll move the temp dir into it instead of # its contents - of course, we could try to always move its contents # but that is a pain if there are symlinks; the simplest solution is # to just remove it first os.rmdir(args.extract_to) shutil.move(srctree, args.extract_to) if tempsrc == srctree: tempsrc = None logger.info('Source extracted to %s' % args.extract_to) if outfile == '-': sys.stdout.write('\n'.join(outlines) + '\n') else: with open(outfile, 'w') as f: f.write('\n'.join(outlines) + '\n') logger.info( 'Recipe %s has been created; further editing may be required to make it fully functional' % outfile) if tempsrc: shutil.rmtree(tempsrc) return 0
def _extract_source(srctree, keep_temp, devbranch, d): """Extract sources of a recipe""" import bb.event import oe.recipeutils def eventfilter(name, handler, event, d): """Bitbake event filter for devtool extract operation""" if name == 'base_eventhandler': return True else: return False if hasattr(bb.event, 'set_eventfilter'): bb.event.set_eventfilter(eventfilter) pn = d.getVar('PN', True) _check_compatible_recipe(pn, d) if os.path.exists(srctree): if not os.path.isdir(srctree): raise DevtoolError("output path %s exists and is not a directory" % srctree) elif os.listdir(srctree): raise DevtoolError("output path %s already exists and is " "non-empty" % srctree) if 'noexec' in (d.getVarFlags('do_unpack', False) or []): raise DevtoolError("The %s recipe has do_unpack disabled, unable to " "extract source" % pn) # Prepare for shutil.move later on bb.utils.mkdirhier(srctree) os.rmdir(srctree) # We don't want notes to be printed, they are too verbose origlevel = bb.logger.getEffectiveLevel() if logger.getEffectiveLevel() > logging.DEBUG: bb.logger.setLevel(logging.WARNING) initial_rev = None tempdir = tempfile.mkdtemp(prefix='devtool') try: crd = d.createCopy() # Make a subdir so we guard against WORKDIR==S workdir = os.path.join(tempdir, 'workdir') crd.setVar('WORKDIR', workdir) crd.setVar('T', os.path.join(tempdir, 'temp')) if not crd.getVar('S', True).startswith(workdir): # Usually a shared workdir recipe (kernel, gcc) # Try to set a reasonable default if bb.data.inherits_class('kernel', d): crd.setVar('S', '${WORKDIR}/source') else: crd.setVar('S', '${WORKDIR}/%s' % os.path.basename(d.getVar('S', True))) if bb.data.inherits_class('kernel', d): # We don't want to move the source to STAGING_KERNEL_DIR here crd.setVar('STAGING_KERNEL_DIR', '${S}') task_executor = BbTaskExecutor(crd) crd.setVar('EXTERNALSRC_forcevariable', '') logger.info('Fetching %s...' % pn) task_executor.exec_func('do_fetch', False) logger.info('Unpacking...') task_executor.exec_func('do_unpack', False) if bb.data.inherits_class('kernel-yocto', d): # Extra step for kernel to populate the source directory logger.info('Doing kernel checkout...') task_executor.exec_func('do_kernel_checkout', False) srcsubdir = crd.getVar('S', True) # Move local source files into separate subdir recipe_patches = [os.path.basename(patch) for patch in oe.recipeutils.get_recipe_patches(crd)] local_files = oe.recipeutils.get_recipe_local_files(crd) local_files = [fname for fname in local_files if os.path.exists(os.path.join(workdir, fname))] if local_files: for fname in local_files: _move_file(os.path.join(workdir, fname), os.path.join(tempdir, 'oe-local-files', fname)) with open(os.path.join(tempdir, 'oe-local-files', '.gitignore'), 'w') as f: f.write('# Ignore local files, by default. Remove this file ' 'if you want to commit the directory to Git\n*\n') if srcsubdir == workdir: # Find non-patch non-local sources that were "unpacked" to srctree # directory src_files = [fname for fname in _ls_tree(workdir) if os.path.basename(fname) not in recipe_patches] # Force separate S so that patch files can be left out from srctree srcsubdir = tempfile.mkdtemp(dir=workdir) crd.setVar('S', srcsubdir) # Move source files to S for path in src_files: _move_file(os.path.join(workdir, path), os.path.join(srcsubdir, path)) elif os.path.dirname(srcsubdir) != workdir: # Handle if S is set to a subdirectory of the source srcsubdir = os.path.join(workdir, os.path.relpath(srcsubdir, workdir).split(os.sep)[0]) scriptutils.git_convert_standalone_clone(srcsubdir) patchdir = os.path.join(srcsubdir, 'patches') haspatches = False if os.path.exists(patchdir): if os.listdir(patchdir): haspatches = True else: os.rmdir(patchdir) # Make sure that srcsubdir exists bb.utils.mkdirhier(srcsubdir) if not os.path.exists(srcsubdir) or not os.listdir(srcsubdir): logger.warning("no source unpacked to S, either the %s recipe " "doesn't use any source or the correct source " "directory could not be determined" % pn) setup_git_repo(srcsubdir, crd.getVar('PV', True), devbranch) (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srcsubdir) initial_rev = stdout.rstrip() crd.setVar('PATCHTOOL', 'git') logger.info('Patching...') task_executor.exec_func('do_patch', False) bb.process.run('git tag -f devtool-patched', cwd=srcsubdir) if os.path.exists(patchdir): shutil.rmtree(patchdir) if haspatches: bb.process.run('git checkout patches', cwd=srcsubdir) # Move oe-local-files directory to srctree if os.path.exists(os.path.join(tempdir, 'oe-local-files')): logger.info('Adding local source files to srctree...') shutil.move(os.path.join(tempdir, 'oe-local-files'), srcsubdir) shutil.move(srcsubdir, srctree) finally: bb.logger.setLevel(origlevel) if keep_temp: logger.info('Preserving temporary directory %s' % tempdir) else: shutil.rmtree(tempdir) return initial_rev