def varfunc(varname, origvalue, op, newlines): if varname == 'SRC_URI': if not origvalue.startswith('npm://'): src_uri = origvalue.split() deplist = {} for dep, depver in optdeps.items(): depdata = self.get_npm_data(dep, depver, d) if self.check_npm_optional_dependency(depdata): deplist[dep] = depdata for dep, depver in devdeps.items(): depdata = self.get_npm_data(dep, depver, d) if self.check_npm_optional_dependency(depdata): deplist[dep] = depdata for dep, depver in deps.items(): depdata = self.get_npm_data(dep, depver, d) deplist[dep] = depdata extra_urls = [] for dep, depdata in deplist.items(): version = depdata.get('version', None) if version: url = 'npm://registry.npmjs.org;name=%s;version=%s;subdir=node_modules/%s' % (dep, version, dep) extra_urls.append(url) if extra_urls: scriptutils.fetch_url(tinfoil, ' '.join(extra_urls), None, srctree, logger) src_uri.extend(extra_urls) return src_uri, None, -1, True return origvalue, None, 0, True
def _extract_new_source(newpv, srctree, no_patch, srcrev, srcbranch, branch, keep_temp, tinfoil, rd): """Extract sources of a recipe with a new version""" def __run(cmd): """Simple wrapper which calls _run with srctree as cwd""" return _run(cmd, srctree) crd = rd.createCopy() pv = crd.getVar('PV') crd.setVar('PV', newpv) tmpsrctree = None uri, rev = _get_uri(crd) if srcrev: rev = srcrev if uri.startswith('git://'): __run('git fetch') __run('git checkout %s' % rev) __run('git tag -f devtool-base-new') md5 = None sha256 = None _, _, _, _, _, params = bb.fetch2.decodeurl(uri) srcsubdir_rel = params.get('destsuffix', 'git') if not srcbranch: check_branch, check_branch_err = __run( 'git branch -r --contains %s' % srcrev) get_branch = [x.strip() for x in check_branch.splitlines()] # Remove HEAD reference point and drop remote prefix get_branch = [ x.split('/', 1)[1] for x in get_branch if not x.startswith('origin/HEAD') ] if 'master' in get_branch: # If it is master, we do not need to append 'branch=master' as this is default. # Even with the case where get_branch has multiple objects, if 'master' is one # of them, we should default take from 'master' srcbranch = '' elif len(get_branch) == 1: # If 'master' isn't in get_branch and get_branch contains only ONE object, then store result into 'srcbranch' srcbranch = get_branch[0] else: # If get_branch contains more than one objects, then display error and exit. mbrch = '\n ' + '\n '.join(get_branch) raise DevtoolError( 'Revision %s was found on multiple branches: %s\nPlease provide the correct branch in the devtool command with "--srcbranch" or "-B" option.' % (srcrev, mbrch)) else: __run('git checkout devtool-base -b devtool-%s' % newpv) tmpdir = tempfile.mkdtemp(prefix='devtool') try: checksums, ftmpdir = scriptutils.fetch_url(tinfoil, uri, rev, tmpdir, logger, preserve_tmp=keep_temp) except scriptutils.FetchUrlFailure as e: raise DevtoolError(e) if ftmpdir and keep_temp: logger.info('Fetch temp directory is %s' % ftmpdir) md5 = checksums['md5sum'] sha256 = checksums['sha256sum'] tmpsrctree = _get_srctree(tmpdir) srctree = os.path.abspath(srctree) srcsubdir_rel = os.path.relpath(tmpsrctree, tmpdir) # Delete all sources so we ensure no stray files are left over for item in os.listdir(srctree): if item in ['.git', 'oe-local-files']: continue itempath = os.path.join(srctree, item) if os.path.isdir(itempath): shutil.rmtree(itempath) else: os.remove(itempath) # Copy in new ones _copy_source_code(tmpsrctree, srctree) (stdout, _) = __run('git ls-files --modified --others --exclude-standard') filelist = stdout.splitlines() pbar = bb.ui.knotty.BBProgress('Adding changed files', len(filelist)) pbar.start() batchsize = 100 for i in range(0, len(filelist), batchsize): batch = filelist[i:i + batchsize] __run('git add -A %s' % ' '.join(['"%s"' % item for item in batch])) pbar.update(i) pbar.finish() useroptions = [] oe.patch.GitApplyTree.gitCommandUserOptions(useroptions, d=rd) __run( 'git %s commit -q -m "Commit of upstream changes at version %s" --allow-empty' % (' '.join(useroptions), newpv)) __run('git tag -f devtool-base-%s' % newpv) (stdout, _) = __run('git rev-parse HEAD') rev = stdout.rstrip() if no_patch: patches = oe.recipeutils.get_recipe_patches(crd) if patches: logger.warning( 'By user choice, the following patches will NOT be applied to the new source tree:\n %s' % '\n '.join([os.path.basename(patch) for patch in patches])) else: __run('git checkout devtool-patched -b %s' % branch) skiptag = False try: __run('git rebase %s' % rev) except bb.process.ExecutionError as e: skiptag = True if 'conflict' in e.stdout: logger.warning( 'Command \'%s\' failed:\n%s\n\nYou will need to resolve conflicts in order to complete the upgrade.' % (e.command, e.stdout.rstrip())) else: logger.warning('Command \'%s\' failed:\n%s' % (e.command, e.stdout)) if not skiptag: if uri.startswith('git://'): suffix = 'new' else: suffix = newpv __run('git tag -f devtool-patched-%s' % suffix) if tmpsrctree: if keep_temp: logger.info('Preserving temporary directory %s' % tmpsrctree) else: shutil.rmtree(tmpsrctree) return (rev, md5, sha256, srcbranch, srcsubdir_rel)
def _extract_new_source(newpv, srctree, no_patch, srcrev, srcbranch, branch, keep_temp, tinfoil, rd): """Extract sources of a recipe with a new version""" def __run(cmd): """Simple wrapper which calls _run with srctree as cwd""" return _run(cmd, srctree) crd = rd.createCopy() pv = crd.getVar('PV') crd.setVar('PV', newpv) tmpsrctree = None uri, rev = _get_uri(crd) if srcrev: rev = srcrev if uri.startswith('git://'): __run('git fetch') __run('git checkout %s' % rev) __run('git tag -f devtool-base-new') md5 = None sha256 = None _, _, _, _, _, params = bb.fetch2.decodeurl(uri) srcsubdir_rel = params.get('destsuffix', 'git') if not srcbranch: check_branch, check_branch_err = __run('git branch -r --contains %s' % srcrev) get_branch = [x.strip() for x in check_branch.splitlines()] # Remove HEAD reference point and drop remote prefix get_branch = [x.split('/', 1)[1] for x in get_branch if not x.startswith('origin/HEAD')] if 'master' in get_branch: # If it is master, we do not need to append 'branch=master' as this is default. # Even with the case where get_branch has multiple objects, if 'master' is one # of them, we should default take from 'master' srcbranch = '' elif len(get_branch) == 1: # If 'master' isn't in get_branch and get_branch contains only ONE object, then store result into 'srcbranch' srcbranch = get_branch[0] else: # If get_branch contains more than one objects, then display error and exit. mbrch = '\n ' + '\n '.join(get_branch) raise DevtoolError('Revision %s was found on multiple branches: %s\nPlease provide the correct branch in the devtool command with "--srcbranch" or "-B" option.' % (srcrev, mbrch)) else: __run('git checkout devtool-base -b devtool-%s' % newpv) tmpdir = tempfile.mkdtemp(prefix='devtool') try: checksums, ftmpdir = scriptutils.fetch_url(tinfoil, uri, rev, tmpdir, logger, preserve_tmp=keep_temp) except scriptutils.FetchUrlFailure as e: raise DevtoolError(e) if ftmpdir and keep_temp: logger.info('Fetch temp directory is %s' % ftmpdir) md5 = checksums['md5sum'] sha256 = checksums['sha256sum'] tmpsrctree = _get_srctree(tmpdir) srctree = os.path.abspath(srctree) srcsubdir_rel = os.path.relpath(tmpsrctree, tmpdir) # Delete all sources so we ensure no stray files are left over for item in os.listdir(srctree): if item in ['.git', 'oe-local-files']: continue itempath = os.path.join(srctree, item) if os.path.isdir(itempath): shutil.rmtree(itempath) else: os.remove(itempath) # Copy in new ones _copy_source_code(tmpsrctree, srctree) (stdout,_) = __run('git ls-files --modified --others --exclude-standard') filelist = stdout.splitlines() pbar = bb.ui.knotty.BBProgress('Adding changed files', len(filelist)) pbar.start() batchsize = 100 for i in range(0, len(filelist), batchsize): batch = filelist[i:i+batchsize] __run('git add -A %s' % ' '.join(['"%s"' % item for item in batch])) pbar.update(i) pbar.finish() useroptions = [] oe.patch.GitApplyTree.gitCommandUserOptions(useroptions, d=rd) __run('git %s commit -q -m "Commit of upstream changes at version %s" --allow-empty' % (' '.join(useroptions), newpv)) __run('git tag -f devtool-base-%s' % newpv) (stdout, _) = __run('git rev-parse HEAD') rev = stdout.rstrip() if no_patch: patches = oe.recipeutils.get_recipe_patches(crd) if patches: logger.warning('By user choice, the following patches will NOT be applied to the new source tree:\n %s' % '\n '.join([os.path.basename(patch) for patch in patches])) else: __run('git checkout devtool-patched -b %s' % branch) skiptag = False try: __run('git rebase %s' % rev) except bb.process.ExecutionError as e: skiptag = True if 'conflict' in e.stdout: logger.warning('Command \'%s\' failed:\n%s\n\nYou will need to resolve conflicts in order to complete the upgrade.' % (e.command, e.stdout.rstrip())) else: logger.warning('Command \'%s\' failed:\n%s' % (e.command, e.stdout)) if not skiptag: if uri.startswith('git://'): suffix = 'new' else: suffix = newpv __run('git tag -f devtool-patched-%s' % suffix) if tmpsrctree: if keep_temp: logger.info('Preserving temporary directory %s' % tmpsrctree) else: shutil.rmtree(tmpsrctree) return (rev, md5, sha256, srcbranch, srcsubdir_rel)
def _extract_new_source(newpv, srctree, no_patch, srcrev, branch, keep_temp, tinfoil, rd): """Extract sources of a recipe with a new version""" def __run(cmd): """Simple wrapper which calls _run with srctree as cwd""" return _run(cmd, srctree) crd = rd.createCopy() pv = crd.getVar('PV') crd.setVar('PV', newpv) tmpsrctree = None uri, rev = _get_uri(crd) if srcrev: rev = srcrev if uri.startswith('git://'): __run('git fetch') __run('git checkout %s' % rev) __run('git tag -f devtool-base-new') md5 = None sha256 = None else: __run('git checkout devtool-base -b devtool-%s' % newpv) tmpdir = tempfile.mkdtemp(prefix='devtool') try: checksums, ftmpdir = scriptutils.fetch_url(tinfoil, uri, rev, tmpdir, logger, preserve_tmp=keep_temp) except scriptutils.FetchUrlFailure as e: raise DevtoolError(e) if ftmpdir and keep_temp: logger.info('Fetch temp directory is %s' % ftmpdir) md5 = checksums['md5sum'] sha256 = checksums['sha256sum'] tmpsrctree = _get_srctree(tmpdir) srctree = os.path.abspath(srctree) # Delete all sources so we ensure no stray files are left over for item in os.listdir(srctree): if item in ['.git', 'oe-local-files']: continue itempath = os.path.join(srctree, item) if os.path.isdir(itempath): shutil.rmtree(itempath) else: os.remove(itempath) # Copy in new ones _copy_source_code(tmpsrctree, srctree) (stdout,_) = __run('git ls-files --modified --others --exclude-standard') for f in stdout.splitlines(): __run('git add "%s"' % f) useroptions = [] oe.patch.GitApplyTree.gitCommandUserOptions(useroptions, d=rd) __run('git %s commit -q -m "Commit of upstream changes at version %s" --allow-empty' % (' '.join(useroptions), newpv)) __run('git tag -f devtool-base-%s' % newpv) (stdout, _) = __run('git rev-parse HEAD') rev = stdout.rstrip() if no_patch: patches = oe.recipeutils.get_recipe_patches(crd) if len(patches): logger.warn('By user choice, the following patches will NOT be applied') for patch in patches: logger.warn("%s" % os.path.basename(patch)) else: __run('git checkout devtool-patched -b %s' % branch) skiptag = False try: __run('git rebase %s' % rev) except bb.process.ExecutionError as e: skiptag = True if 'conflict' in e.stdout: logger.warn('Command \'%s\' failed:\n%s\n\nYou will need to resolve conflicts in order to complete the upgrade.' % (e.command, e.stdout.rstrip())) else: logger.warn('Command \'%s\' failed:\n%s' % (e.command, e.stdout)) if not skiptag: if uri.startswith('git://'): suffix = 'new' else: suffix = newpv __run('git tag -f devtool-patched-%s' % suffix) if tmpsrctree: if keep_temp: logger.info('Preserving temporary directory %s' % tmpsrctree) else: shutil.rmtree(tmpsrctree) return (rev, md5, sha256)