def create_recipe(args):
import bb.process
import tempfile
import shutil
import oe.recipeutils
pkgarch = ""
if args.machine:
pkgarch = "${MACHINE_ARCH}"
extravalues = {}
checksums = (None, None)
tempsrc = ''
source = args.source
srcsubdir = ''
srcrev = '${AUTOREV}'
if os.path.isfile(source):
source = 'file://%s' % os.path.abspath(source)
if scriptutils.is_src_url(source):
# Fetch a URL
fetchuri = reformat_git_uri(urldefrag(source)[0])
if args.binary:
# Assume the archive contains the directory structure verbatim
# so we need to extract to a subdirectory
fetchuri += ';subdir=${BP}'
srcuri = fetchuri
rev_re = re.compile(';rev=([^;]+)')
res = rev_re.search(srcuri)
if res:
srcrev = res.group(1)
srcuri = rev_re.sub('', srcuri)
tempsrc = tempfile.mkdtemp(prefix='recipetool-')
srctree = tempsrc
d = bb.data.createCopy(tinfoil.config_data)
if fetchuri.startswith('npm://'):
# Check if npm is available
npm_bindir = check_npm(tinfoil, args.devtool)
d.prependVar('PATH', '%s:' % npm_bindir)
logger.info('Fetching %s...' % srcuri)
try:
checksums = scriptutils.fetch_uri(d, fetchuri, srctree, srcrev)
except bb.fetch2.BBFetchException as e:
logger.error(str(e).rstrip())
sys.exit(1)
dirlist = os.listdir(srctree)
if 'git.indirectionsymlink' in dirlist:
dirlist.remove('git.indirectionsymlink')
if len(dirlist) == 1:
singleitem = os.path.join(srctree, dirlist[0])
if os.path.isdir(singleitem):
# We unpacked a single directory, so we should use that
srcsubdir = dirlist[0]
srctree = os.path.join(srctree, srcsubdir)
else:
with open(singleitem, 'r', errors='surrogateescape') as f:
if '<html' in f.read(100).lower():
logger.error('Fetching "%s" returned a single HTML page - check the URL is correct and functional' % fetchuri)
sys.exit(1)
if os.path.exists(os.path.join(srctree, '.gitmodules')) and srcuri.startswith('git://'):
srcuri = 'gitsm://' + srcuri[6:]
logger.info('Fetching submodules...')
bb.process.run('git submodule update --init --recursive', cwd=srctree)
if is_package(fetchuri):
tmpfdir = tempfile.mkdtemp(prefix='recipetool-')
try:
pkgfile = None
try:
fileuri = fetchuri + ';unpack=0'
scriptutils.fetch_uri(tinfoil.config_data, fileuri, tmpfdir, srcrev)
for root, _, files in os.walk(tmpfdir):
for f in files:
pkgfile = os.path.join(root, f)
break
except bb.fetch2.BBFetchException as e:
logger.warn('Second fetch to get metadata failed: %s' % str(e).rstrip())
if pkgfile:
if pkgfile.endswith(('.deb', '.ipk')):
stdout, _ = bb.process.run('ar x %s' % pkgfile, cwd=tmpfdir)
stdout, _ = bb.process.run('tar xf control.tar.gz', cwd=tmpfdir)
values = convert_debian(tmpfdir)
extravalues.update(values)
elif pkgfile.endswith(('.rpm', '.srpm')):
stdout, _ = bb.process.run('rpm -qp --xml %s > pkginfo.xml' % pkgfile, cwd=tmpfdir)
values = convert_rpm_xml(os.path.join(tmpfdir, 'pkginfo.xml'))
extravalues.update(values)
finally:
shutil.rmtree(tmpfdir)
else:
# Assume we're pointing to an existing source tree
if args.extract_to:
logger.error('--extract-to cannot be specified if source is a directory')
sys.exit(1)
if not os.path.isdir(source):
logger.error('Invalid source directory %s' % source)
sys.exit(1)
srctree = source
srcuri = ''
if os.path.exists(os.path.join(srctree, '.git')):
# Try to get upstream repo location from origin remote
try:
stdout, _ = bb.process.run('git remote -v', cwd=srctree, shell=True)
except bb.process.ExecutionError as e:
stdout = None
if stdout:
for line in stdout.splitlines():
splitline = line.split()
if len(splitline) > 1:
if splitline[0] == 'origin' and scriptutils.is_src_url(splitline[1]):
srcuri = reformat_git_uri(splitline[1])
srcsubdir = 'git'
break
if args.src_subdir:
srcsubdir = os.path.join(srcsubdir, args.src_subdir)
srctree_use = os.path.join(srctree, args.src_subdir)
else:
srctree_use = srctree
if args.outfile and os.path.isdir(args.outfile):
outfile = None
outdir = args.outfile
else:
outfile = args.outfile
outdir = None
if outfile and outfile != '-':
if os.path.exists(outfile):
logger.error('Output file %s already exists' % outfile)
sys.exit(1)
lines_before = []
lines_after = []
lines_before.append('# Recipe created by %s' % os.path.basename(sys.argv[0]))
lines_before.append('# This is the basis of a recipe and may need further editing in order to be fully functional.')
lines_before.append('# (Feel free to remove these comments when editing.)')
# We need a blank line here so that patch_recipe_lines can rewind before the LICENSE comments
lines_before.append('')
handled = []
licvalues = handle_license_vars(srctree_use, lines_before, handled, extravalues, tinfoil.config_data)
classes = []
# FIXME This is kind of a hack, we probably ought to be using bitbake to do this
pn = None
pv = None
if outfile:
recipefn = os.path.splitext(os.path.basename(outfile))[0]
fnsplit = recipefn.split('_')
if len(fnsplit) > 1:
pn = fnsplit[0]
pv = fnsplit[1]
else:
pn = recipefn
if args.version:
pv = args.version
if args.name:
pn = args.name
if args.name.endswith('-native'):
if args.also_native:
logger.error('--also-native cannot be specified for a recipe named *-native (*-native denotes a recipe that is already only for native) - either remove the -native suffix from the name or drop --also-native')
sys.exit(1)
classes.append('native')
elif args.name.startswith('nativesdk-'):
if args.also_native:
logger.error('--also-native cannot be specified for a recipe named nativesdk-* (nativesdk-* denotes a recipe that is already only for nativesdk)')
sys.exit(1)
classes.append('nativesdk')
if pv and pv not in 'git svn hg'.split():
realpv = pv
else:
realpv = None
if srcuri and not realpv or not pn:
name_pn, name_pv = determine_from_url(srcuri)
if name_pn and not pn:
pn = name_pn
if name_pv and not realpv:
realpv = name_pv
if not srcuri:
lines_before.append('# No information for SRC_URI yet (only an external source tree was specified)')
lines_before.append('SRC_URI = "%s"' % srcuri)
(md5value, sha256value) = checksums
if md5value:
lines_before.append('SRC_URI[md5sum] = "%s"' % md5value)
if sha256value:
lines_before.append('SRC_URI[sha256sum] = "%s"' % sha256value)
if srcuri and supports_srcrev(srcuri):
lines_before.append('')
lines_before.append('# Modify these as desired')
lines_before.append('PV = "%s+git${SRCPV}"' % (realpv or '1.0'))
if not args.autorev and srcrev == '${AUTOREV}':
if os.path.exists(os.path.join(srctree, '.git')):
(stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srctree)
srcrev = stdout.rstrip()
lines_before.append('SRCREV = "%s"' % srcrev)
lines_before.append('')
if srcsubdir and not args.binary:
# (for binary packages we explicitly specify subdir= when fetching to
# match the default value of S, so we don't need to set it in that case)
lines_before.append('S = "${WORKDIR}/%s"' % srcsubdir)
lines_before.append('')
if pkgarch:
lines_after.append('PACKAGE_ARCH = "%s"' % pkgarch)
lines_after.append('')
if args.binary:
lines_after.append('INSANE_SKIP_${PN} += "already-stripped"')
lines_after.append('')
if args.fetch_dev:
extravalues['fetchdev'] = True
else:
extravalues['fetchdev'] = None
# Find all plugins that want to register handlers
logger.debug('Loading recipe handlers')
raw_handlers = []
for plugin in plugins:
if hasattr(plugin, 'register_recipe_handlers'):
plugin.register_recipe_handlers(raw_handlers)
# Sort handlers by priority
handlers = []
for i, handler in enumerate(raw_handlers):
if isinstance(handler, tuple):
handlers.append((handler[0], handler[1], i))
else:
handlers.append((handler, 0, i))
handlers.sort(key=lambda item: (item[1], -item[2]), reverse=True)
for handler, priority, _ in handlers:
logger.debug('Handler: %s (priority %d)' % (handler.__class__.__name__, priority))
setattr(handler, '_devtool', args.devtool)
handlers = [item[0] for item in handlers]
# Apply the handlers
if args.binary:
classes.append('bin_package')
handled.append('buildsystem')
for handler in handlers:
handler.process(srctree_use, classes, lines_before, lines_after, handled, extravalues)
extrafiles = extravalues.pop('extrafiles', {})
extra_pn = extravalues.pop('PN', None)
extra_pv = extravalues.pop('PV', None)
if extra_pv and not realpv:
realpv = extra_pv
if not validate_pv(realpv):
realpv = None
else:
realpv = realpv.lower().split()[0]
if '_' in realpv:
realpv = realpv.replace('_', '-')
if extra_pn and not pn:
pn = extra_pn
if pn.startswith('GNU '):
pn = pn[4:]
if ' ' in pn:
# Probably a descriptive identifier rather than a proper name
pn = None
else:
pn = pn.lower()
if '_' in pn:
pn = pn.replace('_', '-')
if not outfile:
if not pn:
log_error_cond('Unable to determine short program name from source tree - please specify name with -N/--name or output file name with -o/--outfile', args.devtool)
# devtool looks for this specific exit code, so don't change it
sys.exit(15)
else:
if srcuri and srcuri.startswith(('gitsm://', 'git://', 'hg://', 'svn://')):
suffix = srcuri.split(':', 1)[0]
if suffix == 'gitsm':
suffix = 'git'
outfile = '%s_%s.bb' % (pn, suffix)
elif realpv:
outfile = '%s_%s.bb' % (pn, realpv)
else:
outfile = '%s.bb' % pn
if outdir:
outfile = os.path.join(outdir, outfile)
# We need to check this again
if os.path.exists(outfile):
logger.error('Output file %s already exists' % outfile)
sys.exit(1)
# Move any extra files the plugins created to a directory next to the recipe
if extrafiles:
if outfile == '-':
extraoutdir = pn
else:
extraoutdir = os.path.join(os.path.dirname(outfile), pn)
bb.utils.mkdirhier(extraoutdir)
for destfn, extrafile in extrafiles.items():
shutil.move(extrafile, os.path.join(extraoutdir, destfn))
lines = lines_before
lines_before = []
skipblank = True
for line in lines:
if skipblank:
skipblank = False
if not line:
continue
if line.startswith('S = '):
if realpv and pv not in 'git svn hg'.split():
line = line.replace(realpv, '${PV}')
if pn:
line = line.replace(pn, '${BPN}')
if line == 'S = "${WORKDIR}/${BPN}-${PV}"':
skipblank = True
continue
elif line.startswith('SRC_URI = '):
if realpv:
line = line.replace(realpv, '${PV}')
elif line.startswith('PV = '):
if realpv:
line = re.sub('"[^+]*\+', '"%s+' % realpv, line)
lines_before.append(line)
if args.also_native:
lines = lines_after
lines_after = []
bbclassextend = None
for line in lines:
if line.startswith('BBCLASSEXTEND ='):
splitval = line.split('"')
if len(splitval) > 1:
bbclassextend = splitval[1].split()
if not 'native' in bbclassextend:
bbclassextend.insert(0, 'native')
line = 'BBCLASSEXTEND = "%s"' % ' '.join(bbclassextend)
lines_after.append(line)
if not bbclassextend:
lines_after.append('BBCLASSEXTEND = "native"')
postinst = ("postinst", extravalues.pop('postinst', None))
postrm = ("postrm", extravalues.pop('postrm', None))
preinst = ("preinst", extravalues.pop('preinst', None))
prerm = ("prerm", extravalues.pop('prerm', None))
funcs = [postinst, postrm, preinst, prerm]
for func in funcs:
if func[1]:
RecipeHandler.genfunction(lines_after, 'pkg_%s_${PN}' % func[0], func[1])
outlines = []
outlines.extend(lines_before)
if classes:
if outlines[-1] and not outlines[-1].startswith('#'):
outlines.append('')
outlines.append('inherit %s' % ' '.join(classes))
outlines.append('')
outlines.extend(lines_after)
if extravalues:
if 'LICENSE' in extravalues and not licvalues:
# Don't blow away 'CLOSED' value that comments say we set
del extravalues['LICENSE']
_, outlines = oe.recipeutils.patch_recipe_lines(outlines, extravalues, trailing_newline=False)
if args.extract_to:
scriptutils.git_convert_standalone_clone(srctree)
if os.path.isdir(args.extract_to):
# If the directory exists we'll move the temp dir into it instead of
# its contents - of course, we could try to always move its contents
# but that is a pain if there are symlinks; the simplest solution is
# to just remove it first
os.rmdir(args.extract_to)
shutil.move(srctree, args.extract_to)
if tempsrc == srctree:
tempsrc = None
log_info_cond('Source extracted to %s' % args.extract_to, args.devtool)
if outfile == '-':
sys.stdout.write('\n'.join(outlines) + '\n')
else:
with open(outfile, 'w') as f:
lastline = None
for line in outlines:
if not lastline and not line:
# Skip extra blank lines
continue
f.write('%s\n' % line)
lastline = line
log_info_cond('Recipe %s has been created; further editing may be required to make it fully functional' % outfile, args.devtool)
if tempsrc:
if args.keep_temp:
logger.info('Preserving temporary directory %s' % tempsrc)
else:
shutil.rmtree(tempsrc)
return 0
def create_recipe(args):
import bb.process
import tempfile
import shutil
pkgarch = ""
if args.machine:
pkgarch = "${MACHINE_ARCH}"
checksums = (None, None)
tempsrc = ''
srcsubdir = ''
srcrev = '${AUTOREV}'
if '://' in args.source:
# Fetch a URL
srcuri = args.source
rev_re = re.compile(';rev=([^;]+)')
res = rev_re.search(srcuri)
if res:
srcrev = res.group(1)
srcuri = rev_re.sub('', srcuri)
tempsrc = tempfile.mkdtemp(prefix='recipetool-')
srctree = tempsrc
logger.info('Fetching %s...' % srcuri)
checksums = fetch_source(args.source, srctree, srcrev)
dirlist = os.listdir(srctree)
if 'git.indirectionsymlink' in dirlist:
dirlist.remove('git.indirectionsymlink')
if len(dirlist) == 1 and os.path.isdir(os.path.join(srctree, dirlist[0])):
# We unpacked a single directory, so we should use that
srcsubdir = dirlist[0]
srctree = os.path.join(srctree, srcsubdir)
else:
# Assume we're pointing to an existing source tree
if args.extract_to:
logger.error('--extract-to cannot be specified if source is a directory')
sys.exit(1)
if not os.path.isdir(args.source):
logger.error('Invalid source directory %s' % args.source)
sys.exit(1)
srcuri = ''
srctree = args.source
outfile = args.outfile
if outfile and outfile != '-':
if os.path.exists(outfile):
logger.error('Output file %s already exists' % outfile)
sys.exit(1)
lines_before = []
lines_after = []
lines_before.append('# Recipe created by %s' % os.path.basename(sys.argv[0]))
lines_before.append('# This is the basis of a recipe and may need further editing in order to be fully functional.')
lines_before.append('# (Feel free to remove these comments when editing.)')
lines_before.append('#')
licvalues = guess_license(srctree)
lic_files_chksum = []
if licvalues:
licenses = []
for licvalue in licvalues:
if not licvalue[0] in licenses:
licenses.append(licvalue[0])
lic_files_chksum.append('file://%s;md5=%s' % (licvalue[1], licvalue[2]))
lines_before.append('# WARNING: the following LICENSE and LIC_FILES_CHKSUM values are best guesses - it is')
lines_before.append('# your responsibility to verify that the values are complete and correct.')
if len(licvalues) > 1:
lines_before.append('#')
lines_before.append('# NOTE: multiple licenses have been detected; if that is correct you should separate')
lines_before.append('# these in the LICENSE value using & if the multiple licenses all apply, or | if there')
lines_before.append('# is a choice between the multiple licenses. If in doubt, check the accompanying')
lines_before.append('# documentation to determine which situation is applicable.')
else:
lines_before.append('# Unable to find any files that looked like license statements. Check the accompanying')
lines_before.append('# documentation and source headers and set LICENSE and LIC_FILES_CHKSUM accordingly.')
lines_before.append('#')
lines_before.append('# NOTE: LICENSE is being set to "CLOSED" to allow you to at least start building - if')
lines_before.append('# this is not accurate with respect to the licensing of the software being built (it')
lines_before.append('# will not be in most cases) you must specify the correct value before using this')
lines_before.append('# recipe for anything other than initial testing/development!')
licenses = ['CLOSED']
lines_before.append('LICENSE = "%s"' % ' '.join(licenses))
lines_before.append('LIC_FILES_CHKSUM = "%s"' % ' \\\n '.join(lic_files_chksum))
lines_before.append('')
# FIXME This is kind of a hack, we probably ought to be using bitbake to do this
# we'd also want a way to automatically set outfile based upon auto-detecting these values from the source if possible
recipefn = os.path.splitext(os.path.basename(outfile))[0]
fnsplit = recipefn.split('_')
if len(fnsplit) > 1:
pn = fnsplit[0]
pv = fnsplit[1]
else:
pn = recipefn
pv = None
if args.version:
pv = args.version
if pv and pv not in 'git svn hg'.split():
realpv = pv
else:
realpv = None
if srcuri:
if realpv:
srcuri = srcuri.replace(realpv, '${PV}')
else:
lines_before.append('# No information for SRC_URI yet (only an external source tree was specified)')
lines_before.append('SRC_URI = "%s"' % srcuri)
(md5value, sha256value) = checksums
if md5value:
lines_before.append('SRC_URI[md5sum] = "%s"' % md5value)
if sha256value:
lines_before.append('SRC_URI[sha256sum] = "%s"' % sha256value)
if srcuri and supports_srcrev(srcuri):
lines_before.append('')
lines_before.append('# Modify these as desired')
lines_before.append('PV = "%s+git${SRCPV}"' % (realpv or '1.0'))
lines_before.append('SRCREV = "%s"' % srcrev)
lines_before.append('')
if srcsubdir and pv:
if srcsubdir == "%s-%s" % (pn, pv):
# This would be the default, so we don't need to set S in the recipe
srcsubdir = ''
if srcsubdir:
if pv and pv not in 'git svn hg'.split():
srcsubdir = srcsubdir.replace(pv, '${PV}')
lines_before.append('S = "${WORKDIR}/%s"' % srcsubdir)
lines_before.append('')
if pkgarch:
lines_after.append('PACKAGE_ARCH = "%s"' % pkgarch)
lines_after.append('')
# Find all plugins that want to register handlers
handlers = []
for plugin in plugins:
if hasattr(plugin, 'register_recipe_handlers'):
plugin.register_recipe_handlers(handlers)
# Apply the handlers
classes = []
handled = []
for handler in handlers:
handler.process(srctree, classes, lines_before, lines_after, handled)
outlines = []
outlines.extend(lines_before)
if classes:
outlines.append('inherit %s' % ' '.join(classes))
outlines.append('')
outlines.extend(lines_after)
if args.extract_to:
scriptutils.git_convert_standalone_clone(srctree)
shutil.move(srctree, args.extract_to)
logger.info('Source extracted to %s' % args.extract_to)
if outfile == '-':
sys.stdout.write('\n'.join(outlines) + '\n')
else:
with open(outfile, 'w') as f:
f.write('\n'.join(outlines) + '\n')
logger.info('Recipe %s has been created; further editing may be required to make it fully functional' % outfile)
if tempsrc:
shutil.rmtree(tempsrc)
return 0
def _extract_source(srctree, keep_temp, devbranch, d):
import bb.event
def eventfilter(name, handler, event, d):
if name == 'base_eventhandler':
return True
else:
return False
if hasattr(bb.event, 'set_eventfilter'):
bb.event.set_eventfilter(eventfilter)
pn = d.getVar('PN', True)
if not _check_compatible_recipe(pn, d):
return None
if os.path.exists(srctree):
if not os.path.isdir(srctree):
logger.error("output path %s exists and is not a directory" %
srctree)
return None
elif os.listdir(srctree):
logger.error("output path %s already exists and is non-empty" %
srctree)
return None
# Prepare for shutil.move later on
bb.utils.mkdirhier(srctree)
os.rmdir(srctree)
initial_rev = None
tempdir = tempfile.mkdtemp(prefix='devtool')
try:
crd = d.createCopy()
# Make a subdir so we guard against WORKDIR==S
workdir = os.path.join(tempdir, 'workdir')
crd.setVar('WORKDIR', workdir)
crd.setVar('T', os.path.join(tempdir, 'temp'))
if not crd.getVar('S', True).startswith(workdir):
# Usually a shared workdir recipe (kernel, gcc)
# Try to set a reasonable default
if bb.data.inherits_class('kernel', d):
crd.setVar('S', '${WORKDIR}/source')
else:
crd.setVar('S', '${WORKDIR}/${BP}')
if bb.data.inherits_class('kernel', d):
# We don't want to move the source to STAGING_KERNEL_DIR here
crd.setVar('STAGING_KERNEL_DIR', '${S}')
# FIXME: This is very awkward. Unfortunately it's not currently easy to properly
# execute tasks outside of bitbake itself, until then this has to suffice if we
# are to handle e.g. linux-yocto's extra tasks
executed = []
def exec_task_func(func, report):
if not func in executed:
deps = crd.getVarFlag(func, 'deps')
if deps:
for taskdepfunc in deps:
exec_task_func(taskdepfunc, True)
if report:
logger.info('Executing %s...' % func)
fn = d.getVar('FILE', True)
localdata = bb.build._task_data(fn, func, crd)
bb.build.exec_func(func, localdata)
executed.append(func)
logger.info('Fetching %s...' % pn)
exec_task_func('do_fetch', False)
logger.info('Unpacking...')
exec_task_func('do_unpack', False)
srcsubdir = crd.getVar('S', True)
patchsubdir = srcsubdir
if srcsubdir != workdir and os.path.dirname(srcsubdir) != workdir:
# Handle if S is set to a subdirectory of the source
srcsubdir = os.path.join(
workdir,
os.path.relpath(srcsubdir, workdir).split(os.sep)[0])
scriptutils.git_convert_standalone_clone(srcsubdir)
patchdir = os.path.join(patchsubdir, 'patches')
haspatches = False
if os.path.exists(patchdir):
if os.listdir(patchdir):
haspatches = True
else:
os.rmdir(patchdir)
if bb.data.inherits_class('kernel-yocto', d):
(stdout, _) = bb.process.run('git --git-dir="%s" rev-parse HEAD' %
crd.expand('${WORKDIR}/git'),
cwd=srcsubdir)
initial_rev = stdout.rstrip()
else:
if not os.listdir(srcsubdir):
logger.error(
"no source unpacked to S, perhaps the %s recipe doesn't use any source?"
% pn)
return None
if not os.path.exists(os.path.join(srcsubdir, '.git')):
bb.process.run('git init', cwd=srcsubdir)
bb.process.run('git add .', cwd=srcsubdir)
bb.process.run(
'git commit -q -m "Initial commit from upstream at version %s"'
% crd.getVar('PV', True),
cwd=srcsubdir)
(stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srcsubdir)
initial_rev = stdout.rstrip()
bb.process.run('git checkout -b %s' % devbranch, cwd=srcsubdir)
bb.process.run('git tag -f devtool-base', cwd=srcsubdir)
crd.setVar('PATCHTOOL', 'git')
logger.info('Patching...')
exec_task_func('do_patch', False)
bb.process.run('git tag -f devtool-patched', cwd=srcsubdir)
if os.path.exists(patchdir):
shutil.rmtree(patchdir)
if haspatches:
bb.process.run('git checkout patches', cwd=srcsubdir)
shutil.move(srcsubdir, srctree)
logger.info('Source tree extracted to %s' % srctree)
finally:
if keep_temp:
logger.info('Preserving temporary directory %s' % tempdir)
else:
shutil.rmtree(tempdir)
return initial_rev
def create_recipe(args):
import bb.process
import tempfile
import shutil
pkgarch = ""
if args.machine:
pkgarch = "${MACHINE_ARCH}"
checksums = (None, None)
tempsrc = ''
srcsubdir = ''
srcrev = '${AUTOREV}'
if '://' in args.source:
# Fetch a URL
fetchuri = urlparse.urldefrag(args.source)[0]
if args.binary:
# Assume the archive contains the directory structure verbatim
# so we need to extract to a subdirectory
fetchuri += ';subdir=%s' % os.path.splitext(
os.path.basename(urlparse.urlsplit(fetchuri).path))[0]
srcuri = fetchuri
rev_re = re.compile(';rev=([^;]+)')
res = rev_re.search(srcuri)
if res:
srcrev = res.group(1)
srcuri = rev_re.sub('', srcuri)
tempsrc = tempfile.mkdtemp(prefix='recipetool-')
srctree = tempsrc
logger.info('Fetching %s...' % srcuri)
checksums = scriptutils.fetch_uri(tinfoil.config_data, fetchuri,
srctree, srcrev)
dirlist = os.listdir(srctree)
if 'git.indirectionsymlink' in dirlist:
dirlist.remove('git.indirectionsymlink')
if len(dirlist) == 1 and os.path.isdir(
os.path.join(srctree, dirlist[0])):
# We unpacked a single directory, so we should use that
srcsubdir = dirlist[0]
srctree = os.path.join(srctree, srcsubdir)
else:
# Assume we're pointing to an existing source tree
if args.extract_to:
logger.error(
'--extract-to cannot be specified if source is a directory')
sys.exit(1)
if not os.path.isdir(args.source):
logger.error('Invalid source directory %s' % args.source)
sys.exit(1)
srcuri = ''
srctree = args.source
outfile = args.outfile
if outfile and outfile != '-':
if os.path.exists(outfile):
logger.error('Output file %s already exists' % outfile)
sys.exit(1)
lines_before = []
lines_after = []
lines_before.append('# Recipe created by %s' %
os.path.basename(sys.argv[0]))
lines_before.append(
'# This is the basis of a recipe and may need further editing in order to be fully functional.'
)
lines_before.append('# (Feel free to remove these comments when editing.)')
lines_before.append('#')
licvalues = guess_license(srctree)
lic_files_chksum = []
if licvalues:
licenses = []
for licvalue in licvalues:
if not licvalue[0] in licenses:
licenses.append(licvalue[0])
lic_files_chksum.append('file://%s;md5=%s' %
(licvalue[1], licvalue[2]))
lines_before.append(
'# WARNING: the following LICENSE and LIC_FILES_CHKSUM values are best guesses - it is'
)
lines_before.append(
'# your responsibility to verify that the values are complete and correct.'
)
if len(licvalues) > 1:
lines_before.append('#')
lines_before.append(
'# NOTE: multiple licenses have been detected; if that is correct you should separate'
)
lines_before.append(
'# these in the LICENSE value using & if the multiple licenses all apply, or | if there'
)
lines_before.append(
'# is a choice between the multiple licenses. If in doubt, check the accompanying'
)
lines_before.append(
'# documentation to determine which situation is applicable.')
else:
lines_before.append(
'# Unable to find any files that looked like license statements. Check the accompanying'
)
lines_before.append(
'# documentation and source headers and set LICENSE and LIC_FILES_CHKSUM accordingly.'
)
lines_before.append('#')
lines_before.append(
'# NOTE: LICENSE is being set to "CLOSED" to allow you to at least start building - if'
)
lines_before.append(
'# this is not accurate with respect to the licensing of the software being built (it'
)
lines_before.append(
'# will not be in most cases) you must specify the correct value before using this'
)
lines_before.append(
'# recipe for anything other than initial testing/development!')
licenses = ['CLOSED']
lines_before.append('LICENSE = "%s"' % ' '.join(licenses))
lines_before.append('LIC_FILES_CHKSUM = "%s"' %
' \\\n '.join(lic_files_chksum))
lines_before.append('')
# FIXME This is kind of a hack, we probably ought to be using bitbake to do this
# we'd also want a way to automatically set outfile based upon auto-detecting these values from the source if possible
recipefn = os.path.splitext(os.path.basename(outfile))[0]
fnsplit = recipefn.split('_')
if len(fnsplit) > 1:
pn = fnsplit[0]
pv = fnsplit[1]
else:
pn = recipefn
pv = None
if args.version:
pv = args.version
if pv and pv not in 'git svn hg'.split():
realpv = pv
else:
realpv = None
if srcuri:
if realpv:
srcuri = srcuri.replace(realpv, '${PV}')
else:
lines_before.append(
'# No information for SRC_URI yet (only an external source tree was specified)'
)
lines_before.append('SRC_URI = "%s"' % srcuri)
(md5value, sha256value) = checksums
if md5value:
lines_before.append('SRC_URI[md5sum] = "%s"' % md5value)
if sha256value:
lines_before.append('SRC_URI[sha256sum] = "%s"' % sha256value)
if srcuri and supports_srcrev(srcuri):
lines_before.append('')
lines_before.append('# Modify these as desired')
lines_before.append('PV = "%s+git${SRCPV}"' % (realpv or '1.0'))
lines_before.append('SRCREV = "%s"' % srcrev)
lines_before.append('')
if srcsubdir and pv:
if srcsubdir == "%s-%s" % (pn, pv):
# This would be the default, so we don't need to set S in the recipe
srcsubdir = ''
if srcsubdir:
if pv and pv not in 'git svn hg'.split():
srcsubdir = srcsubdir.replace(pv, '${PV}')
lines_before.append('S = "${WORKDIR}/%s"' % srcsubdir)
lines_before.append('')
if pkgarch:
lines_after.append('PACKAGE_ARCH = "%s"' % pkgarch)
lines_after.append('')
if args.binary:
lines_after.append('INSANE_SKIP_${PN} += "already-stripped"')
lines_after.append('')
# Find all plugins that want to register handlers
handlers = []
for plugin in plugins:
if hasattr(plugin, 'register_recipe_handlers'):
plugin.register_recipe_handlers(handlers)
# Apply the handlers
classes = []
handled = []
if args.binary:
classes.append('bin_package')
handled.append('buildsystem')
for handler in handlers:
handler.process(srctree, classes, lines_before, lines_after, handled)
outlines = []
outlines.extend(lines_before)
if classes:
outlines.append('inherit %s' % ' '.join(classes))
outlines.append('')
outlines.extend(lines_after)
if args.extract_to:
scriptutils.git_convert_standalone_clone(srctree)
if os.path.isdir(args.extract_to):
# If the directory exists we'll move the temp dir into it instead of
# its contents - of course, we could try to always move its contents
# but that is a pain if there are symlinks; the simplest solution is
# to just remove it first
os.rmdir(args.extract_to)
shutil.move(srctree, args.extract_to)
logger.info('Source extracted to %s' % args.extract_to)
if outfile == '-':
sys.stdout.write('\n'.join(outlines) + '\n')
else:
with open(outfile, 'w') as f:
f.write('\n'.join(outlines) + '\n')
logger.info(
'Recipe %s has been created; further editing may be required to make it fully functional'
% outfile)
if tempsrc:
shutil.rmtree(tempsrc)
return 0
def create_recipe(args):
import bb.process
import tempfile
import shutil
pkgarch = ""
if args.machine:
pkgarch = "${MACHINE_ARCH}"
checksums = (None, None)
tempsrc = ''
srcsubdir = ''
srcrev = '${AUTOREV}'
if '://' in args.source:
# Fetch a URL
fetchuri = reformat_git_uri(urlparse.urldefrag(args.source)[0])
if args.binary:
# Assume the archive contains the directory structure verbatim
# so we need to extract to a subdirectory
fetchuri += ';subdir=%s' % os.path.splitext(os.path.basename(urlparse.urlsplit(fetchuri).path))[0]
srcuri = fetchuri
rev_re = re.compile(';rev=([^;]+)')
res = rev_re.search(srcuri)
if res:
srcrev = res.group(1)
srcuri = rev_re.sub('', srcuri)
tempsrc = tempfile.mkdtemp(prefix='recipetool-')
srctree = tempsrc
if fetchuri.startswith('npm://'):
# Check if npm is available
npm = bb.utils.which(tinfoil.config_data.getVar('PATH', True), 'npm')
if not npm:
logger.error('npm:// URL requested but npm is not available - you need to either build nodejs-native or install npm using your package manager')
sys.exit(1)
logger.info('Fetching %s...' % srcuri)
try:
checksums = scriptutils.fetch_uri(tinfoil.config_data, fetchuri, srctree, srcrev)
except bb.fetch2.BBFetchException as e:
logger.error(str(e).rstrip())
sys.exit(1)
dirlist = os.listdir(srctree)
if 'git.indirectionsymlink' in dirlist:
dirlist.remove('git.indirectionsymlink')
if len(dirlist) == 1:
singleitem = os.path.join(srctree, dirlist[0])
if os.path.isdir(singleitem):
# We unpacked a single directory, so we should use that
srcsubdir = dirlist[0]
srctree = os.path.join(srctree, srcsubdir)
else:
with open(singleitem, 'r') as f:
if '<html' in f.read(100).lower():
logger.error('Fetching "%s" returned a single HTML page - check the URL is correct and functional' % fetchuri)
sys.exit(1)
else:
# Assume we're pointing to an existing source tree
if args.extract_to:
logger.error('--extract-to cannot be specified if source is a directory')
sys.exit(1)
if not os.path.isdir(args.source):
logger.error('Invalid source directory %s' % args.source)
sys.exit(1)
srctree = args.source
srcuri = ''
if os.path.exists(os.path.join(srctree, '.git')):
# Try to get upstream repo location from origin remote
try:
stdout, _ = bb.process.run('git remote -v', cwd=srctree, shell=True)
except bb.process.ExecutionError as e:
stdout = None
if stdout:
for line in stdout.splitlines():
splitline = line.split()
if len(splitline) > 1:
if splitline[0] == 'origin' and '://' in splitline[1]:
srcuri = reformat_git_uri(splitline[1])
srcsubdir = 'git'
break
if args.src_subdir:
srcsubdir = os.path.join(srcsubdir, args.src_subdir)
srctree_use = os.path.join(srctree, args.src_subdir)
else:
srctree_use = srctree
if args.outfile and os.path.isdir(args.outfile):
outfile = None
outdir = args.outfile
else:
outfile = args.outfile
outdir = None
if outfile and outfile != '-':
if os.path.exists(outfile):
logger.error('Output file %s already exists' % outfile)
sys.exit(1)
lines_before = []
lines_after = []
lines_before.append('# Recipe created by %s' % os.path.basename(sys.argv[0]))
lines_before.append('# This is the basis of a recipe and may need further editing in order to be fully functional.')
lines_before.append('# (Feel free to remove these comments when editing.)')
lines_before.append('#')
licvalues = guess_license(srctree_use)
lic_files_chksum = []
if licvalues:
licenses = []
for licvalue in licvalues:
if not licvalue[0] in licenses:
licenses.append(licvalue[0])
lic_files_chksum.append('file://%s;md5=%s' % (licvalue[1], licvalue[2]))
lines_before.append('# WARNING: the following LICENSE and LIC_FILES_CHKSUM values are best guesses - it is')
lines_before.append('# your responsibility to verify that the values are complete and correct.')
if len(licvalues) > 1:
lines_before.append('#')
lines_before.append('# NOTE: multiple licenses have been detected; if that is correct you should separate')
lines_before.append('# these in the LICENSE value using & if the multiple licenses all apply, or | if there')
lines_before.append('# is a choice between the multiple licenses. If in doubt, check the accompanying')
lines_before.append('# documentation to determine which situation is applicable.')
else:
lines_before.append('# Unable to find any files that looked like license statements. Check the accompanying')
lines_before.append('# documentation and source headers and set LICENSE and LIC_FILES_CHKSUM accordingly.')
lines_before.append('#')
lines_before.append('# NOTE: LICENSE is being set to "CLOSED" to allow you to at least start building - if')
lines_before.append('# this is not accurate with respect to the licensing of the software being built (it')
lines_before.append('# will not be in most cases) you must specify the correct value before using this')
lines_before.append('# recipe for anything other than initial testing/development!')
licenses = ['CLOSED']
lines_before.append('LICENSE = "%s"' % ' '.join(licenses))
lines_before.append('LIC_FILES_CHKSUM = "%s"' % ' \\\n '.join(lic_files_chksum))
lines_before.append('')
classes = []
# FIXME This is kind of a hack, we probably ought to be using bitbake to do this
pn = None
pv = None
if outfile:
recipefn = os.path.splitext(os.path.basename(outfile))[0]
fnsplit = recipefn.split('_')
if len(fnsplit) > 1:
pn = fnsplit[0]
pv = fnsplit[1]
else:
pn = recipefn
if args.version:
pv = args.version
if args.name:
pn = args.name
if args.name.endswith('-native'):
if args.also_native:
logger.error('--also-native cannot be specified for a recipe named *-native (*-native denotes a recipe that is already only for native) - either remove the -native suffix from the name or drop --also-native')
sys.exit(1)
classes.append('native')
elif args.name.startswith('nativesdk-'):
if args.also_native:
logger.error('--also-native cannot be specified for a recipe named nativesdk-* (nativesdk-* denotes a recipe that is already only for nativesdk)')
sys.exit(1)
classes.append('nativesdk')
if pv and pv not in 'git svn hg'.split():
realpv = pv
else:
realpv = None
if srcuri and not realpv or not pn:
name_pn, name_pv = determine_from_url(srcuri)
if name_pn and not pn:
pn = name_pn
if name_pv and not realpv:
realpv = name_pv
if not srcuri:
lines_before.append('# No information for SRC_URI yet (only an external source tree was specified)')
lines_before.append('SRC_URI = "%s"' % srcuri)
(md5value, sha256value) = checksums
if md5value:
lines_before.append('SRC_URI[md5sum] = "%s"' % md5value)
if sha256value:
lines_before.append('SRC_URI[sha256sum] = "%s"' % sha256value)
if srcuri and supports_srcrev(srcuri):
lines_before.append('')
lines_before.append('# Modify these as desired')
lines_before.append('PV = "%s+git${SRCPV}"' % (realpv or '1.0'))
lines_before.append('SRCREV = "%s"' % srcrev)
lines_before.append('')
if srcsubdir:
lines_before.append('S = "${WORKDIR}/%s"' % srcsubdir)
lines_before.append('')
if pkgarch:
lines_after.append('PACKAGE_ARCH = "%s"' % pkgarch)
lines_after.append('')
if args.binary:
lines_after.append('INSANE_SKIP_${PN} += "already-stripped"')
lines_after.append('')
# Find all plugins that want to register handlers
logger.debug('Loading recipe handlers')
raw_handlers = []
for plugin in plugins:
if hasattr(plugin, 'register_recipe_handlers'):
plugin.register_recipe_handlers(raw_handlers)
# Sort handlers by priority
handlers = []
for i, handler in enumerate(raw_handlers):
if isinstance(handler, tuple):
handlers.append((handler[0], handler[1], i))
else:
handlers.append((handler, 0, i))
handlers.sort(key=lambda item: (item[1], -item[2]), reverse=True)
for handler, priority, _ in handlers:
logger.debug('Handler: %s (priority %d)' % (handler.__class__.__name__, priority))
handlers = [item[0] for item in handlers]
# Apply the handlers
handled = []
handled.append(('license', licvalues))
if args.binary:
classes.append('bin_package')
handled.append('buildsystem')
extravalues = {}
for handler in handlers:
handler.process(srctree_use, classes, lines_before, lines_after, handled, extravalues)
extrafiles = extravalues.pop('extrafiles', {})
if not realpv:
realpv = extravalues.get('PV', None)
if realpv:
if not validate_pv(realpv):
realpv = None
else:
realpv = realpv.lower().split()[0]
if '_' in realpv:
realpv = realpv.replace('_', '-')
if not pn:
pn = extravalues.get('PN', None)
if pn:
if pn.startswith('GNU '):
pn = pn[4:]
if ' ' in pn:
# Probably a descriptive identifier rather than a proper name
pn = None
else:
pn = pn.lower()
if '_' in pn:
pn = pn.replace('_', '-')
if not outfile:
if not pn:
logger.error('Unable to determine short program name from source tree - please specify name with -N/--name or output file name with -o/--outfile')
# devtool looks for this specific exit code, so don't change it
sys.exit(15)
else:
if srcuri and srcuri.startswith(('git://', 'hg://', 'svn://')):
outfile = '%s_%s.bb' % (pn, srcuri.split(':', 1)[0])
elif realpv:
outfile = '%s_%s.bb' % (pn, realpv)
else:
outfile = '%s.bb' % pn
if outdir:
outfile = os.path.join(outdir, outfile)
# We need to check this again
if os.path.exists(outfile):
logger.error('Output file %s already exists' % outfile)
sys.exit(1)
# Move any extra files the plugins created to a directory next to the recipe
if extrafiles:
if outfile == '-':
extraoutdir = pn
else:
extraoutdir = os.path.join(os.path.dirname(outfile), pn)
bb.utils.mkdirhier(extraoutdir)
for destfn, extrafile in extrafiles.iteritems():
shutil.move(extrafile, os.path.join(extraoutdir, destfn))
lines = lines_before
lines_before = []
skipblank = True
for line in lines:
if skipblank:
skipblank = False
if not line:
continue
if line.startswith('S = '):
if realpv and pv not in 'git svn hg'.split():
line = line.replace(realpv, '${PV}')
if pn:
line = line.replace(pn, '${BPN}')
if line == 'S = "${WORKDIR}/${BPN}-${PV}"':
skipblank = True
continue
elif line.startswith('SRC_URI = '):
if realpv:
line = line.replace(realpv, '${PV}')
elif line.startswith('PV = '):
if realpv:
line = re.sub('"[^+]*\+', '"%s+' % realpv, line)
lines_before.append(line)
if args.also_native:
lines = lines_after
lines_after = []
bbclassextend = None
for line in lines:
if line.startswith('BBCLASSEXTEND ='):
splitval = line.split('"')
if len(splitval) > 1:
bbclassextend = splitval[1].split()
if not 'native' in bbclassextend:
bbclassextend.insert(0, 'native')
line = 'BBCLASSEXTEND = "%s"' % ' '.join(bbclassextend)
lines_after.append(line)
if not bbclassextend:
lines_after.append('BBCLASSEXTEND = "native"')
outlines = []
outlines.extend(lines_before)
if classes:
if outlines[-1] and not outlines[-1].startswith('#'):
outlines.append('')
outlines.append('inherit %s' % ' '.join(classes))
outlines.append('')
outlines.extend(lines_after)
if args.extract_to:
scriptutils.git_convert_standalone_clone(srctree)
if os.path.isdir(args.extract_to):
# If the directory exists we'll move the temp dir into it instead of
# its contents - of course, we could try to always move its contents
# but that is a pain if there are symlinks; the simplest solution is
# to just remove it first
os.rmdir(args.extract_to)
shutil.move(srctree, args.extract_to)
if tempsrc == srctree:
tempsrc = None
logger.info('Source extracted to %s' % args.extract_to)
if outfile == '-':
sys.stdout.write('\n'.join(outlines) + '\n')
else:
with open(outfile, 'w') as f:
f.write('\n'.join(outlines) + '\n')
logger.info('Recipe %s has been created; further editing may be required to make it fully functional' % outfile)
if tempsrc:
shutil.rmtree(tempsrc)
return 0
def _extract_source(srctree, keep_temp, devbranch, d):
"""Extract sources of a recipe"""
import bb.event
import oe.recipeutils
def eventfilter(name, handler, event, d):
"""Bitbake event filter for devtool extract operation"""
if name == 'base_eventhandler':
return True
else:
return False
if hasattr(bb.event, 'set_eventfilter'):
bb.event.set_eventfilter(eventfilter)
pn = d.getVar('PN', True)
if not _check_compatible_recipe(pn, d):
return None
if os.path.exists(srctree):
if not os.path.isdir(srctree):
logger.error("output path %s exists and is not a directory" % srctree)
return None
elif os.listdir(srctree):
logger.error("output path %s already exists and is non-empty" % srctree)
return None
# Prepare for shutil.move later on
bb.utils.mkdirhier(srctree)
os.rmdir(srctree)
# We don't want notes to be printed, they are too verbose
origlevel = bb.logger.getEffectiveLevel()
if logger.getEffectiveLevel() > logging.DEBUG:
bb.logger.setLevel(logging.WARNING)
initial_rev = None
tempdir = tempfile.mkdtemp(prefix='devtool')
try:
crd = d.createCopy()
# Make a subdir so we guard against WORKDIR==S
workdir = os.path.join(tempdir, 'workdir')
crd.setVar('WORKDIR', workdir)
crd.setVar('T', os.path.join(tempdir, 'temp'))
if not crd.getVar('S', True).startswith(workdir):
# Usually a shared workdir recipe (kernel, gcc)
# Try to set a reasonable default
if bb.data.inherits_class('kernel', d):
crd.setVar('S', '${WORKDIR}/source')
else:
crd.setVar('S', '${WORKDIR}/${BP}')
if bb.data.inherits_class('kernel', d):
# We don't want to move the source to STAGING_KERNEL_DIR here
crd.setVar('STAGING_KERNEL_DIR', '${S}')
# FIXME: This is very awkward. Unfortunately it's not currently easy to properly
# execute tasks outside of bitbake itself, until then this has to suffice if we
# are to handle e.g. linux-yocto's extra tasks
executed = []
def exec_task_func(func, report):
"""Run specific bitbake task for a recipe"""
if not func in executed:
deps = crd.getVarFlag(func, 'deps')
if deps:
for taskdepfunc in deps:
exec_task_func(taskdepfunc, True)
if report:
logger.info('Executing %s...' % func)
fn = d.getVar('FILE', True)
localdata = bb.build._task_data(fn, func, crd)
bb.build.exec_func(func, localdata)
executed.append(func)
logger.info('Fetching %s...' % pn)
exec_task_func('do_fetch', False)
logger.info('Unpacking...')
exec_task_func('do_unpack', False)
srcsubdir = crd.getVar('S', True)
if srcsubdir == workdir:
# Find non-patch sources that were "unpacked" to srctree directory
recipe_patches = [os.path.basename(patch) for patch in
oe.recipeutils.get_recipe_patches(crd)]
src_files = [fname for fname in _ls_tree(workdir) if
os.path.basename(fname) not in recipe_patches]
# Force separate S so that patch files can be left out from srctree
srcsubdir = tempfile.mkdtemp(dir=workdir)
crd.setVar('S', srcsubdir)
# Move source files to S
for path in src_files:
tgt_dir = os.path.join(srcsubdir, os.path.dirname(path))
bb.utils.mkdirhier(tgt_dir)
shutil.move(os.path.join(workdir, path), tgt_dir)
elif os.path.dirname(srcsubdir) != workdir:
# Handle if S is set to a subdirectory of the source
srcsubdir = os.path.join(workdir, os.path.relpath(srcsubdir, workdir).split(os.sep)[0])
scriptutils.git_convert_standalone_clone(srcsubdir)
patchdir = os.path.join(srcsubdir, 'patches')
haspatches = False
if os.path.exists(patchdir):
if os.listdir(patchdir):
haspatches = True
else:
os.rmdir(patchdir)
if bb.data.inherits_class('kernel-yocto', d):
(stdout, _) = bb.process.run('git --git-dir="%s" rev-parse HEAD' % crd.expand('${WORKDIR}/git'), cwd=srcsubdir)
initial_rev = stdout.rstrip()
else:
if not os.listdir(srcsubdir):
logger.error("no source unpacked to S, perhaps the %s recipe doesn't use any source?" % pn)
return None
if not os.path.exists(os.path.join(srcsubdir, '.git')):
bb.process.run('git init', cwd=srcsubdir)
bb.process.run('git add .', cwd=srcsubdir)
bb.process.run('git commit -q -m "Initial commit from upstream at version %s"' % crd.getVar('PV', True), cwd=srcsubdir)
(stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srcsubdir)
initial_rev = stdout.rstrip()
bb.process.run('git checkout -b %s' % devbranch, cwd=srcsubdir)
bb.process.run('git tag -f devtool-base', cwd=srcsubdir)
crd.setVar('PATCHTOOL', 'git')
logger.info('Patching...')
exec_task_func('do_patch', False)
bb.process.run('git tag -f devtool-patched', cwd=srcsubdir)
if os.path.exists(patchdir):
shutil.rmtree(patchdir)
if haspatches:
bb.process.run('git checkout patches', cwd=srcsubdir)
shutil.move(srcsubdir, srctree)
logger.info('Source tree extracted to %s' % srctree)
finally:
bb.logger.setLevel(origlevel)
if keep_temp:
logger.info('Preserving temporary directory %s' % tempdir)
else:
shutil.rmtree(tempdir)
return initial_rev
def _extract_source(srctree, keep_temp, devbranch, d):
"""Extract sources of a recipe"""
import bb.event
import oe.recipeutils
def eventfilter(name, handler, event, d):
"""Bitbake event filter for devtool extract operation"""
if name == 'base_eventhandler':
return True
else:
return False
if hasattr(bb.event, 'set_eventfilter'):
bb.event.set_eventfilter(eventfilter)
pn = d.getVar('PN', True)
_check_compatible_recipe(pn, d)
if os.path.exists(srctree):
if not os.path.isdir(srctree):
raise DevtoolError("output path %s exists and is not a directory" %
srctree)
elif os.listdir(srctree):
raise DevtoolError("output path %s already exists and is "
"non-empty" % srctree)
# Prepare for shutil.move later on
bb.utils.mkdirhier(srctree)
os.rmdir(srctree)
# We don't want notes to be printed, they are too verbose
origlevel = bb.logger.getEffectiveLevel()
if logger.getEffectiveLevel() > logging.DEBUG:
bb.logger.setLevel(logging.WARNING)
initial_rev = None
tempdir = tempfile.mkdtemp(prefix='devtool')
try:
crd = d.createCopy()
# Make a subdir so we guard against WORKDIR==S
workdir = os.path.join(tempdir, 'workdir')
crd.setVar('WORKDIR', workdir)
crd.setVar('T', os.path.join(tempdir, 'temp'))
if not crd.getVar('S', True).startswith(workdir):
# Usually a shared workdir recipe (kernel, gcc)
# Try to set a reasonable default
if bb.data.inherits_class('kernel', d):
crd.setVar('S', '${WORKDIR}/source')
else:
crd.setVar('S', '${WORKDIR}/${BP}')
if bb.data.inherits_class('kernel', d):
# We don't want to move the source to STAGING_KERNEL_DIR here
crd.setVar('STAGING_KERNEL_DIR', '${S}')
task_executor = BbTaskExecutor(crd)
crd.setVar('EXTERNALSRC_forcevariable', '')
logger.info('Fetching %s...' % pn)
task_executor.exec_func('do_fetch', False)
logger.info('Unpacking...')
task_executor.exec_func('do_unpack', False)
srcsubdir = crd.getVar('S', True)
if srcsubdir == workdir:
# Find non-patch sources that were "unpacked" to srctree directory
recipe_patches = [
os.path.basename(patch)
for patch in oe.recipeutils.get_recipe_patches(crd)
]
src_files = [
fname for fname in _ls_tree(workdir)
if os.path.basename(fname) not in recipe_patches
]
# Force separate S so that patch files can be left out from srctree
srcsubdir = tempfile.mkdtemp(dir=workdir)
crd.setVar('S', srcsubdir)
# Move source files to S
for path in src_files:
tgt_dir = os.path.join(srcsubdir, os.path.dirname(path))
bb.utils.mkdirhier(tgt_dir)
shutil.move(os.path.join(workdir, path), tgt_dir)
elif os.path.dirname(srcsubdir) != workdir:
# Handle if S is set to a subdirectory of the source
srcsubdir = os.path.join(
workdir,
os.path.relpath(srcsubdir, workdir).split(os.sep)[0])
scriptutils.git_convert_standalone_clone(srcsubdir)
patchdir = os.path.join(srcsubdir, 'patches')
haspatches = False
if os.path.exists(patchdir):
if os.listdir(patchdir):
haspatches = True
else:
os.rmdir(patchdir)
if bb.data.inherits_class('kernel-yocto', d):
(stdout, _) = bb.process.run('git --git-dir="%s" rev-parse HEAD' %
crd.expand('${WORKDIR}/git'),
cwd=srcsubdir)
initial_rev = stdout.rstrip()
else:
if not os.listdir(srcsubdir):
raise DevtoolError("no source unpacked to S, perhaps the %s "
"recipe doesn't use any source?" % pn)
if not os.path.exists(os.path.join(srcsubdir, '.git')):
bb.process.run('git init', cwd=srcsubdir)
bb.process.run('git add .', cwd=srcsubdir)
bb.process.run(
'git commit -q -m "Initial commit from upstream at version %s"'
% crd.getVar('PV', True),
cwd=srcsubdir)
(stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srcsubdir)
initial_rev = stdout.rstrip()
bb.process.run('git checkout -b %s' % devbranch, cwd=srcsubdir)
bb.process.run('git tag -f devtool-base', cwd=srcsubdir)
crd.setVar('PATCHTOOL', 'git')
logger.info('Patching...')
task_executor.exec_func('do_patch', False)
bb.process.run('git tag -f devtool-patched', cwd=srcsubdir)
if os.path.exists(patchdir):
shutil.rmtree(patchdir)
if haspatches:
bb.process.run('git checkout patches', cwd=srcsubdir)
shutil.move(srcsubdir, srctree)
logger.info('Source tree extracted to %s' % srctree)
finally:
bb.logger.setLevel(origlevel)
if keep_temp:
logger.info('Preserving temporary directory %s' % tempdir)
else:
shutil.rmtree(tempdir)
return initial_rev
def create_recipe(args):
import bb.process
import tempfile
import shutil
pkgarch = ""
if args.machine:
pkgarch = "${MACHINE_ARCH}"
checksums = (None, None)
tempsrc = ''
srcsubdir = ''
srcrev = '${AUTOREV}'
if '://' in args.source:
# Fetch a URL
fetchuri = urlparse.urldefrag(args.source)[0]
if args.binary:
# Assume the archive contains the directory structure verbatim
# so we need to extract to a subdirectory
fetchuri += ';subdir=%s' % os.path.splitext(
os.path.basename(urlparse.urlsplit(fetchuri).path))[0]
git_re = re.compile('(https?)://([^;]+\.git)(;.*)?')
res = git_re.match(fetchuri)
if res:
# Need to switch the URI around so that the git fetcher is used
fetchuri = 'git://%s;protocol=%s%s' % (res.group(2), res.group(1),
res.group(3) or '')
srcuri = fetchuri
rev_re = re.compile(';rev=([^;]+)')
res = rev_re.search(srcuri)
if res:
srcrev = res.group(1)
srcuri = rev_re.sub('', srcuri)
tempsrc = tempfile.mkdtemp(prefix='recipetool-')
srctree = tempsrc
logger.info('Fetching %s...' % srcuri)
try:
checksums = scriptutils.fetch_uri(tinfoil.config_data, fetchuri,
srctree, srcrev)
except bb.fetch2.FetchError:
# Error already printed
sys.exit(1)
dirlist = os.listdir(srctree)
if 'git.indirectionsymlink' in dirlist:
dirlist.remove('git.indirectionsymlink')
if len(dirlist) == 1:
singleitem = os.path.join(srctree, dirlist[0])
if os.path.isdir(singleitem):
# We unpacked a single directory, so we should use that
srcsubdir = dirlist[0]
srctree = os.path.join(srctree, srcsubdir)
else:
with open(singleitem, 'r') as f:
if '<html' in f.read(100).lower():
logger.error(
'Fetching "%s" returned a single HTML page - check the URL is correct and functional'
% fetchuri)
sys.exit(1)
else:
# Assume we're pointing to an existing source tree
if args.extract_to:
logger.error(
'--extract-to cannot be specified if source is a directory')
sys.exit(1)
if not os.path.isdir(args.source):
logger.error('Invalid source directory %s' % args.source)
sys.exit(1)
srcuri = ''
srctree = args.source
if args.outfile and os.path.isdir(args.outfile):
outfile = None
outdir = args.outfile
else:
outfile = args.outfile
outdir = None
if outfile and outfile != '-':
if os.path.exists(outfile):
logger.error('Output file %s already exists' % outfile)
sys.exit(1)
lines_before = []
lines_after = []
lines_before.append('# Recipe created by %s' %
os.path.basename(sys.argv[0]))
lines_before.append(
'# This is the basis of a recipe and may need further editing in order to be fully functional.'
)
lines_before.append('# (Feel free to remove these comments when editing.)')
lines_before.append('#')
licvalues = guess_license(srctree)
lic_files_chksum = []
if licvalues:
licenses = []
for licvalue in licvalues:
if not licvalue[0] in licenses:
licenses.append(licvalue[0])
lic_files_chksum.append('file://%s;md5=%s' %
(licvalue[1], licvalue[2]))
lines_before.append(
'# WARNING: the following LICENSE and LIC_FILES_CHKSUM values are best guesses - it is'
)
lines_before.append(
'# your responsibility to verify that the values are complete and correct.'
)
if len(licvalues) > 1:
lines_before.append('#')
lines_before.append(
'# NOTE: multiple licenses have been detected; if that is correct you should separate'
)
lines_before.append(
'# these in the LICENSE value using & if the multiple licenses all apply, or | if there'
)
lines_before.append(
'# is a choice between the multiple licenses. If in doubt, check the accompanying'
)
lines_before.append(
'# documentation to determine which situation is applicable.')
else:
lines_before.append(
'# Unable to find any files that looked like license statements. Check the accompanying'
)
lines_before.append(
'# documentation and source headers and set LICENSE and LIC_FILES_CHKSUM accordingly.'
)
lines_before.append('#')
lines_before.append(
'# NOTE: LICENSE is being set to "CLOSED" to allow you to at least start building - if'
)
lines_before.append(
'# this is not accurate with respect to the licensing of the software being built (it'
)
lines_before.append(
'# will not be in most cases) you must specify the correct value before using this'
)
lines_before.append(
'# recipe for anything other than initial testing/development!')
licenses = ['CLOSED']
lines_before.append('LICENSE = "%s"' % ' '.join(licenses))
lines_before.append('LIC_FILES_CHKSUM = "%s"' %
' \\\n '.join(lic_files_chksum))
lines_before.append('')
classes = []
# FIXME This is kind of a hack, we probably ought to be using bitbake to do this
pn = None
pv = None
if outfile:
recipefn = os.path.splitext(os.path.basename(outfile))[0]
fnsplit = recipefn.split('_')
if len(fnsplit) > 1:
pn = fnsplit[0]
pv = fnsplit[1]
else:
pn = recipefn
if args.version:
pv = args.version
if args.name:
pn = args.name
if args.name.endswith('-native'):
if args.also_native:
logger.error(
'--also-native cannot be specified for a recipe named *-native (*-native denotes a recipe that is already only for native) - either remove the -native suffix from the name or drop --also-native'
)
sys.exit(1)
classes.append('native')
elif args.name.startswith('nativesdk-'):
if args.also_native:
logger.error(
'--also-native cannot be specified for a recipe named nativesdk-* (nativesdk-* denotes a recipe that is already only for nativesdk)'
)
sys.exit(1)
classes.append('nativesdk')
if pv and pv not in 'git svn hg'.split():
realpv = pv
else:
realpv = None
if srcuri and not realpv or not pn:
parseres = urlparse.urlparse(srcuri)
if parseres.path:
srcfile = os.path.basename(parseres.path)
name_pn, name_pv = determine_from_filename(srcfile)
logger.debug(
'Determined from filename: name = "%s", version = "%s"' %
(name_pn, name_pv))
if name_pn and not pn:
pn = name_pn
if name_pv and not realpv:
realpv = name_pv
if not srcuri:
lines_before.append(
'# No information for SRC_URI yet (only an external source tree was specified)'
)
lines_before.append('SRC_URI = "%s"' % srcuri)
(md5value, sha256value) = checksums
if md5value:
lines_before.append('SRC_URI[md5sum] = "%s"' % md5value)
if sha256value:
lines_before.append('SRC_URI[sha256sum] = "%s"' % sha256value)
if srcuri and supports_srcrev(srcuri):
lines_before.append('')
lines_before.append('# Modify these as desired')
lines_before.append('PV = "%s+git${SRCPV}"' % (realpv or '1.0'))
lines_before.append('SRCREV = "%s"' % srcrev)
lines_before.append('')
if srcsubdir:
lines_before.append('S = "${WORKDIR}/%s"' % srcsubdir)
lines_before.append('')
if pkgarch:
lines_after.append('PACKAGE_ARCH = "%s"' % pkgarch)
lines_after.append('')
if args.binary:
lines_after.append('INSANE_SKIP_${PN} += "already-stripped"')
lines_after.append('')
# Find all plugins that want to register handlers
logger.debug('Loading recipe handlers')
raw_handlers = []
for plugin in plugins:
if hasattr(plugin, 'register_recipe_handlers'):
plugin.register_recipe_handlers(raw_handlers)
# Sort handlers by priority
handlers = []
for i, handler in enumerate(raw_handlers):
if isinstance(handler, tuple):
handlers.append((handler[0], handler[1], i))
else:
handlers.append((handler, 0, i))
handlers.sort(key=lambda item: (item[1], -item[2]), reverse=True)
for handler, priority, _ in handlers:
logger.debug('Handler: %s (priority %d)' %
(handler.__class__.__name__, priority))
handlers = [item[0] for item in handlers]
# Apply the handlers
handled = []
if args.binary:
classes.append('bin_package')
handled.append('buildsystem')
extravalues = {}
for handler in handlers:
handler.process(srctree, classes, lines_before, lines_after, handled,
extravalues)
if not realpv:
realpv = extravalues.get('PV', None)
if realpv:
if not validate_pv(realpv):
realpv = None
else:
realpv = realpv.lower().split()[0]
if '_' in realpv:
realpv = realpv.replace('_', '-')
if not pn:
pn = extravalues.get('PN', None)
if pn:
if pn.startswith('GNU '):
pn = pn[4:]
if ' ' in pn:
# Probably a descriptive identifier rather than a proper name
pn = None
else:
pn = pn.lower()
if '_' in pn:
pn = pn.replace('_', '-')
if not outfile:
if not pn:
logger.error(
'Unable to determine short program name from source tree - please specify name with -N/--name or output file name with -o/--outfile'
)
# devtool looks for this specific exit code, so don't change it
sys.exit(15)
else:
if srcuri and srcuri.startswith(('git://', 'hg://', 'svn://')):
outfile = '%s_%s.bb' % (pn, srcuri.split(':', 1)[0])
elif realpv:
outfile = '%s_%s.bb' % (pn, realpv)
else:
outfile = '%s.bb' % pn
if outdir:
outfile = os.path.join(outdir, outfile)
# We need to check this again
if os.path.exists(outfile):
logger.error('Output file %s already exists' % outfile)
sys.exit(1)
lines = lines_before
lines_before = []
skipblank = True
for line in lines:
if skipblank:
skipblank = False
if not line:
continue
if line.startswith('S = '):
if realpv and pv not in 'git svn hg'.split():
line = line.replace(realpv, '${PV}')
if pn:
line = line.replace(pn, '${BPN}')
if line == 'S = "${WORKDIR}/${BPN}-${PV}"':
skipblank = True
continue
elif line.startswith('SRC_URI = '):
if realpv:
line = line.replace(realpv, '${PV}')
elif line.startswith('PV = '):
if realpv:
line = re.sub('"[^+]*\+', '"%s+' % realpv, line)
lines_before.append(line)
if args.also_native:
lines = lines_after
lines_after = []
bbclassextend = None
for line in lines:
if line.startswith('BBCLASSEXTEND ='):
splitval = line.split('"')
if len(splitval) > 1:
bbclassextend = splitval[1].split()
if not 'native' in bbclassextend:
bbclassextend.insert(0, 'native')
line = 'BBCLASSEXTEND = "%s"' % ' '.join(bbclassextend)
lines_after.append(line)
if not bbclassextend:
lines_after.append('BBCLASSEXTEND = "native"')
outlines = []
outlines.extend(lines_before)
if classes:
outlines.append('inherit %s' % ' '.join(classes))
outlines.append('')
outlines.extend(lines_after)
if args.extract_to:
scriptutils.git_convert_standalone_clone(srctree)
if os.path.isdir(args.extract_to):
# If the directory exists we'll move the temp dir into it instead of
# its contents - of course, we could try to always move its contents
# but that is a pain if there are symlinks; the simplest solution is
# to just remove it first
os.rmdir(args.extract_to)
shutil.move(srctree, args.extract_to)
if tempsrc == srctree:
tempsrc = None
logger.info('Source extracted to %s' % args.extract_to)
if outfile == '-':
sys.stdout.write('\n'.join(outlines) + '\n')
else:
with open(outfile, 'w') as f:
f.write('\n'.join(outlines) + '\n')
logger.info(
'Recipe %s has been created; further editing may be required to make it fully functional'
% outfile)
if tempsrc:
shutil.rmtree(tempsrc)
return 0
def _extract_source(srctree, keep_temp, devbranch, d):
"""Extract sources of a recipe"""
import bb.event
import oe.recipeutils
def eventfilter(name, handler, event, d):
"""Bitbake event filter for devtool extract operation"""
if name == 'base_eventhandler':
return True
else:
return False
if hasattr(bb.event, 'set_eventfilter'):
bb.event.set_eventfilter(eventfilter)
pn = d.getVar('PN', True)
_check_compatible_recipe(pn, d)
if os.path.exists(srctree):
if not os.path.isdir(srctree):
raise DevtoolError("output path %s exists and is not a directory" %
srctree)
elif os.listdir(srctree):
raise DevtoolError("output path %s already exists and is "
"non-empty" % srctree)
if 'noexec' in (d.getVarFlags('do_unpack', False) or []):
raise DevtoolError("The %s recipe has do_unpack disabled, unable to "
"extract source" % pn)
# Prepare for shutil.move later on
bb.utils.mkdirhier(srctree)
os.rmdir(srctree)
# We don't want notes to be printed, they are too verbose
origlevel = bb.logger.getEffectiveLevel()
if logger.getEffectiveLevel() > logging.DEBUG:
bb.logger.setLevel(logging.WARNING)
initial_rev = None
tempdir = tempfile.mkdtemp(prefix='devtool')
try:
crd = d.createCopy()
# Make a subdir so we guard against WORKDIR==S
workdir = os.path.join(tempdir, 'workdir')
crd.setVar('WORKDIR', workdir)
crd.setVar('T', os.path.join(tempdir, 'temp'))
if not crd.getVar('S', True).startswith(workdir):
# Usually a shared workdir recipe (kernel, gcc)
# Try to set a reasonable default
if bb.data.inherits_class('kernel', d):
crd.setVar('S', '${WORKDIR}/source')
else:
crd.setVar('S', '${WORKDIR}/%s' % os.path.basename(d.getVar('S', True)))
if bb.data.inherits_class('kernel', d):
# We don't want to move the source to STAGING_KERNEL_DIR here
crd.setVar('STAGING_KERNEL_DIR', '${S}')
task_executor = BbTaskExecutor(crd)
crd.setVar('EXTERNALSRC_forcevariable', '')
logger.info('Fetching %s...' % pn)
task_executor.exec_func('do_fetch', False)
logger.info('Unpacking...')
task_executor.exec_func('do_unpack', False)
if bb.data.inherits_class('kernel-yocto', d):
# Extra step for kernel to populate the source directory
logger.info('Doing kernel checkout...')
task_executor.exec_func('do_kernel_checkout', False)
srcsubdir = crd.getVar('S', True)
# Move local source files into separate subdir
recipe_patches = [os.path.basename(patch) for patch in
oe.recipeutils.get_recipe_patches(crd)]
local_files = oe.recipeutils.get_recipe_local_files(crd)
local_files = [fname for fname in local_files if
os.path.exists(os.path.join(workdir, fname))]
if local_files:
for fname in local_files:
_move_file(os.path.join(workdir, fname),
os.path.join(tempdir, 'oe-local-files', fname))
with open(os.path.join(tempdir, 'oe-local-files', '.gitignore'),
'w') as f:
f.write('# Ignore local files, by default. Remove this file '
'if you want to commit the directory to Git\n*\n')
if srcsubdir == workdir:
# Find non-patch non-local sources that were "unpacked" to srctree
# directory
src_files = [fname for fname in _ls_tree(workdir) if
os.path.basename(fname) not in recipe_patches]
# Force separate S so that patch files can be left out from srctree
srcsubdir = tempfile.mkdtemp(dir=workdir)
crd.setVar('S', srcsubdir)
# Move source files to S
for path in src_files:
_move_file(os.path.join(workdir, path),
os.path.join(srcsubdir, path))
elif os.path.dirname(srcsubdir) != workdir:
# Handle if S is set to a subdirectory of the source
srcsubdir = os.path.join(workdir, os.path.relpath(srcsubdir, workdir).split(os.sep)[0])
scriptutils.git_convert_standalone_clone(srcsubdir)
patchdir = os.path.join(srcsubdir, 'patches')
haspatches = False
if os.path.exists(patchdir):
if os.listdir(patchdir):
haspatches = True
else:
os.rmdir(patchdir)
# Make sure that srcsubdir exists
bb.utils.mkdirhier(srcsubdir)
if not os.path.exists(srcsubdir) or not os.listdir(srcsubdir):
logger.warning("no source unpacked to S, either the %s recipe "
"doesn't use any source or the correct source "
"directory could not be determined" % pn)
setup_git_repo(srcsubdir, crd.getVar('PV', True), devbranch)
(stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srcsubdir)
initial_rev = stdout.rstrip()
crd.setVar('PATCHTOOL', 'git')
logger.info('Patching...')
task_executor.exec_func('do_patch', False)
bb.process.run('git tag -f devtool-patched', cwd=srcsubdir)
if os.path.exists(patchdir):
shutil.rmtree(patchdir)
if haspatches:
bb.process.run('git checkout patches', cwd=srcsubdir)
# Move oe-local-files directory to srctree
if os.path.exists(os.path.join(tempdir, 'oe-local-files')):
logger.info('Adding local source files to srctree...')
shutil.move(os.path.join(tempdir, 'oe-local-files'), srcsubdir)
shutil.move(srcsubdir, srctree)
finally:
bb.logger.setLevel(origlevel)
if keep_temp:
logger.info('Preserving temporary directory %s' % tempdir)
else:
shutil.rmtree(tempdir)
return initial_rev