def git_branch(_dst, _orig_branch, _branch): logger.info('Branching %s: %s -> %s' % (_dst, _orig_branch, _branch)) _cmd = ['git', 'fetch', '.', '%s:%s' % (_orig_branch, _branch)] if force: _cmd.append('-f') utils_setup.run_cmd(_cmd, cwd=_dst) work_list.append('%s %s' % (_dst, _branch))
def commit_files(self): logger.debug('Starting') # List of all files that may change due to config filelist = [ 'README', 'default.xml', '.gitignore', '.gitconfig', ] # If we are mirroring, skip all of these... if self.mirror != True: filelist.append('layers/local') filelist.append('.templateconf') filelist.append('config/bblayers.conf.sample') filelist.append('config/conf-notes.txt') filelist.append('config/local.conf.sample') if os.path.exists('config/site.conf.sample'): filelist.append('config/site.conf.sample') # Add log dir if it contains files if os.listdir('config/log'): filelist.append('config/log') # git init if not os.path.exists(self.project_dir + '/.git'): cmd = [self.tools['git'], 'init', self.project_dir] if self.quiet == self.default_repo_quiet: cmd.append(self.quiet) utils_setup.run_cmd(cmd, environment=self.env, cwd=self.conf_dir) # Add self.install_dir as a submodule if it is in self.project_dir if self.install_dir.startswith(self.project_dir + '/'): logger.debug('Add %s as a submodule' % self.install_dir) cmd = [self.tools['git'], 'submodule', 'add', \ './' + os.path.relpath(self.install_dir, self.project_dir)] utils_setup.run_cmd(cmd, environment=self.env, cwd=self.project_dir) filelist.append(self.install_dir) filelist.append('.gitmodules') # git add manifest. (Since these files are new, always try to add them) cmd = [self.tools['git'], 'add', '--'] + filelist utils_setup.run_cmd(cmd, environment=self.env, cwd=self.project_dir) try: cmd = [self.tools['git'], 'diff-index', '--quiet', 'HEAD', '--'] + filelist utils_setup.run_cmd(cmd, log=2, environment=self.env, cwd=self.project_dir, stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL) except: logger.plain('Updated project configuration') # Command failed -- so self.default_xml changed... cmd = [self.tools['git'], 'commit', '-m', 'Configuration change - %s' % (self.setup_args), '--'] + filelist utils_setup.run_cmd(cmd, environment=self.env, cwd=self.project_dir) logger.debug('Done')
def update_mirror(_dst_mirror): cmd = ['git', 'add', '-A', '.'] utils_setup.run_cmd(cmd, cwd=_dst_mirror) logger.debug('Updating mirror-index') cmd = ['git', 'commit', '-m', 'Updated index - Flatten Mirror'] try: utils_setup.run_cmd(cmd, cwd=_dst_mirror) except: # Nothing changed... pass
def call_repo_sync(self, args): logger.debug('Starting') repo = self.tools['repo'] cmd = args cmd.insert(0, repo) cmd.insert(1, 'sync') if self.force_sync: cmd.append(self.force_sync) log_it = 1 if self.repo_verbose is not True and self.quiet == self.default_repo_quiet: cmd.append(self.quiet) log_it = 0 utils_setup.run_cmd(cmd, environment=self.env, log=log_it) logger.debug('Done')
def push_or_copy(_layer, _src, _dst, _branch=None): if subset_folders: if _layer not in subset_folders: logger.critical("Layer %s (%s) not in SUBSET_FOLDERS" % (_layer, _src)) raise elif subset_folders[_layer] == "[SKIP]": return else: dstdir = os.path.dirname(_dst) dstbase = os.path.basename(_dst) _dst = os.path.join(dstdir, subset_folders[_layer], dstbase) if not os.path.exists(_src): _src += '.git' if not os.path.exists(_src): logger.critical("Unable to find %s!" % _src) raise # Make output consistent if not strip_git and not _dst.endswith('.git'): _dst += '.git' if strip_git and _dst.endswith('.git'): _dst = _dst[:-4] if not git_push or not _branch: logger.plain('cp %s -> %s' % (_src, _dst)) shutil.copytree(_src, _dst, symlinks=True, ignore_dangling_symlinks=True) else: logger.plain('push %s -> %s (%s)' % (_src, _dst, _branch)) if os.path.exists(_dst): logger.critical('Destination %s already exists!' % _dst) raise os.makedirs(_dst, exist_ok=True) # New bare repo _cmd = [ 'git', 'init', '--bare' ] utils_setup.run_cmd(_cmd, cwd=_dst) # Push just the one branch _cmd = [ 'git', 'push', os.path.abspath(_dst), _branch ] utils_setup.run_cmd(_cmd, cwd=_src)
def call_repo_init(self, args): logger.debug('Starting') repo = self.tools['repo'] directory = os.path.join(self.project_dir, self.check_repo_install_dir) if os.path.exists(directory): logger.info('Done: detected repo init already run since %s exists' % directory) return cmd = args cmd.insert(0, repo) cmd.insert(1, 'init') if self.depth: cmd.append(self.depth) log_it = 1 if self.repo_verbose is not True and self.quiet == self.default_repo_quiet: cmd.append(self.quiet) log_it = 0 try: utils_setup.run_cmd(cmd, environment=self.env, log=log_it) except Exception as e: raise e logger.debug('Done')
def get_mirror_dir(_layer, _dst): if subset_folders: if _layer in subset_folders: if subset_folders[_layer] == "[SKIP]": return None dstdir = os.path.dirname(_dst) dstbase = os.path.basename(_dst) _dst = os.path.join(dstdir, subset_folders[_layer], dstbase) else: logger.error("Layer %s not in SUBSET_FOLDERS" % _layer) # Use default directory # If the target directory does not exist, create it if not os.path.exists(_dst): cmd = ['git', 'init', _dst] utils_setup.run_cmd(cmd) if not branch or branch == "": logger.critical('Where did the branch val go!? %s' % branch) raise cmd = ['git', 'checkout', '-b', branch] try: utils_setup.run_cmd(cmd, cwd=_dst) except: # if we failed, then simply try to switch branches cmd = ['git', 'checkout', branch] utils_setup.run_cmd(cmd, cwd=_dst) return _dst
def write_local_mirror_index(self, setup, mirror_index_path): import subprocess import utils_setup # We need access to the sortRestApi function... from layer_index import Layer_Index li = Layer_Index() # We want to move to a generic named branch, now that we've done the fixups. try: cmd = [setup.tools['git'], 'checkout', '--orphan', setup.base_branch ] utils_setup.run_cmd(cmd, environment=setup.env, cwd=mirror_index_path, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) except Exception: cmd = [setup.tools['git'], 'checkout', setup.base_branch ] utils_setup.run_cmd(cmd, log=2, environment=setup.env, cwd=mirror_index_path) cmd = [setup.tools['git'], 'reset', '--hard' ] utils_setup.run_cmd(cmd, log=2, environment=setup.env, cwd=mirror_index_path) # Remove obsolete entries only for (dirpath, _, filenames) in os.walk(mirror_index_path): if dirpath.endswith('/.git') or '/.git/' in dirpath: continue for filename in filenames: if filename not in self.indexes and filename not in self.xmls: logger.debug('ws mirror-index remove obsolete %s' % os.path.join(dirpath, filename)) os.remove(os.path.join(dirpath, filename)) for entry in self.indexes: logger.debug('Writing windshare index %s...' % entry) fpath = os.path.join(mirror_index_path, entry) json.dump(li.sortRestApi(self.indexes[entry]), open(fpath, 'wt'), indent=4) for entry in self.xmls: logger.debug('Writing windshare xml %s...' % entry) os.makedirs(os.path.join(mirror_index_path, 'xml'), exist_ok=True) fpath = os.path.join(mirror_index_path, 'xml', entry) with open(fpath, 'wt') as fout: for _line in self.xmls[entry]: fout.write(_line + '\n') cmd = [setup.tools['git'], 'add', '-A', '.'] utils_setup.run_cmd(cmd, log=2, environment=setup.env, cwd=mirror_index_path) try: cmd = [setup.tools['git'], 'diff-index', '--quiet', 'HEAD', '--'] utils_setup.run_cmd(cmd, environment=setup.env, cwd=mirror_index_path, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) except Exception: # We expect to fail to this code logger.debug('Updating windshare mirror-index') cmd = [setup.tools['git'], 'commit', '-m', 'Updated index - %s' % (setup.setup_args)] utils_setup.run_cmd(cmd, log=2, environment=setup.env, cwd=mirror_index_path)
def update_mirror_index(self): logger.debug('Starting') path = os.path.join(self.project_dir, 'mirror-index') logger.plain('Exporting mirror-index %s...' % (path)) if not os.path.exists(path): cmd = [self.tools['git'], 'init', path] if self.quiet == self.default_repo_quiet: cmd.append(self.quiet) utils_setup.run_cmd(cmd, environment=self.env, cwd=self.project_dir) cmd = [self.tools['git'], 'checkout', '-b', self.base_branch] ret = subprocess.Popen(cmd, env=self.env, cwd=path, close_fds=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) ret.wait() if (ret.returncode != 0): # if we failed, then simply try to switch branches cmd = [self.tools['git'], 'checkout', self.base_branch] ret = subprocess.Popen(cmd, env=self.env, cwd=path, close_fds=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) ret.wait() # Make sure the directory is empty, use -f to ignore failures for (dirpath, dirnames, filenames) in os.walk(path): if dirpath.endswith('/.git') or path + '/.git' in dirpath: continue for filename in filenames: os.remove(os.path.join(dirpath, filename)) # Construct a list of all layers we've downloaded, by url, including sublayers not activated url_cache = {} for (lindex, layerBranch) in self.requiredlayers + self.recommendedlayers: for layer in self.index.find_layer(lindex, id=layerBranch['layer']): vcs_url = layer['vcs_url'] if not vcs_url in url_cache: url_cache[vcs_url] = [] url_cache[vcs_url].append((lindex, layerBranch['branch'])) # Serialize the information for each of the layers (and their sublayers) for vcs_url in url_cache: for (lindex, branchid) in url_cache[vcs_url]: for layer in lindex['layerItems']: if layer['vcs_url'] in url_cache: for lb in self.index.getLayerBranch(lindex, branchid=branchid, layerItem=layer): self.index.serialize_index( lindex, os.path.join(path, lindex['CFG']['DESCRIPTION']), split=True, layerBranches=[lb], IncludeCFG=True, mirror=True, base_url=self.base_url) name = layer['name'] destdir = os.path.join(path, 'xml') srcfile = os.path.join(self.xml_dir, '%s.inc' % (name)) if os.path.exists(srcfile): os.makedirs(destdir, exist_ok=True) shutil.copy(srcfile, destdir) srcfile = os.path.join(self.xml_dir, '%s.xml' % (name)) if os.path.exists(srcfile): os.makedirs(destdir, exist_ok=True) shutil.copy(srcfile, destdir) # Special processing for the openembedded-core layer if name == 'openembedded-core': srcfile = os.path.join(self.xml_dir, 'bitbake.inc') if os.path.exists(srcfile): os.makedirs(destdir, exist_ok=True) shutil.copy(srcfile, destdir) srcfile = os.path.join(self.xml_dir, 'bitbake.xml') if os.path.exists(srcfile): os.makedirs(destdir, exist_ok=True) shutil.copy(srcfile, destdir) # git add file. cmd = [self.tools['git'], 'add', '-A', '.'] utils_setup.run_cmd(cmd, environment=self.env, cwd=path) cmd = [self.tools['git'], 'diff-index', '--quiet', 'HEAD', '--'] ret = subprocess.Popen(cmd, env=self.env, cwd=path, close_fds=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) ret.wait() if (ret.returncode != 0): logger.debug('Updating mirror-index') cmd = [ self.tools['git'], 'commit', '-m', 'Updated index - %s' % (self.setup_args) ] utils_setup.run_cmd(cmd, environment=self.env, cwd=path) logger.debug('Done')
def load_layer_index(self): # Load Layer_Index replace = [] replace = replace + settings.REPLACE replace = replace + [ ('#INSTALL_DIR#', self.install_dir), ('#BASE_URL#', self.base_url), ('#BASE_BRANCH#', self.base_branch), ] # See if there is a mirror index available from the BASE_URL mirror_index_path = None mirror_index = os.path.join(self.conf_dir, 'mirror-index') cmd = [ self.tools['git'], 'ls-remote', self.base_url + '/mirror-index', self.base_branch ] ret = subprocess.Popen(cmd, env=self.env, cwd=self.project_dir, close_fds=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) ret.wait() if (ret.returncode == 0): logger.plain('Loading the mirror index from %s (%s)...' % (self.base_url + '/mirror-index', self.base_branch)) # This MIGHT be a valid mirror.. if not os.path.exists(mirror_index): os.makedirs(mirror_index) cmd = [self.tools['git'], 'init'] utils_setup.run_cmd(cmd, environment=self.env, cwd=mirror_index) cmd = [ self.tools['git'], 'fetch', '-f', '-n', '-u', self.base_url + '/mirror-index', self.base_branch + ':' + self.base_branch ] ret = subprocess.Popen(cmd, env=self.env, cwd=mirror_index, close_fds=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) ret.wait() if (ret.returncode == 0): logger.debug('Found mirrored index.') cmd = [self.tools['git'], 'checkout', self.base_branch] utils_setup.run_cmd(cmd, environment=self.env, cwd=mirror_index) cmd = [self.tools['git'], 'reset', '--hard'] utils_setup.run_cmd(cmd, environment=self.env, cwd=mirror_index) mirror_index_path = mirror_index # Mirror also has a copy of the associated XML bits self.xml_dir = os.path.join(mirror_index, 'xml') self.index = Layer_Index(indexcfg=settings.INDEXES, base_branch=self.base_branch, replace=replace, mirror=mirror_index_path)
def load_mirror_index(self, remote_mirror): # See if there is a mirror index available from the BASE_URL mirror_index = os.path.join(self.conf_dir, 'mirror-index') try: cmd = [self.tools['git'], 'ls-remote', remote_mirror, self.base_branch] utils_setup.run_cmd(cmd, log=2, environment=self.env, cwd=self.project_dir, stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL) except: try: remote_mirror += "/.git" cmd = [self.tools['git'], 'ls-remote', remote_mirror, self.base_branch] utils_setup.run_cmd(cmd, log=2, environment=self.env, cwd=self.project_dir, stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL) except: # No mirror, return return None logger.plain('Loading the mirror index from %s (%s)...' % (remote_mirror, self.base_branch)) # This MIGHT be a valid mirror.. if not os.path.exists(mirror_index): os.makedirs(mirror_index) cmd = [self.tools['git'], 'init' ] utils_setup.run_cmd(cmd, log=2, environment=self.env, cwd=mirror_index) try: cmd = [self.tools['git'], 'fetch', '-f', '-n', '-u', remote_mirror, self.base_branch + ':' + self.base_branch] utils_setup.run_cmd(cmd, log=2, environment=self.env, cwd=mirror_index) except: # Could not fetch, return return None logger.debug('Found mirrored index.') cmd = [self.tools['git'], 'checkout', self.base_branch ] utils_setup.run_cmd(cmd, log=2, environment=self.env, cwd=mirror_index) cmd = [self.tools['git'], 'reset', '--hard' ] utils_setup.run_cmd(cmd, log=2, environment=self.env, cwd=mirror_index) return mirror_index
# Branch the setup program.... git_branch(_dst=dst, _orig_branch=branch, _branch=dest_branch) completed.append(dst) # Transform and export the mirror index git_branch(mirror_path, branch, dest_branch) completed.append(mirror_path) index = Layer_Index(indexcfg=settings.INDEXES, base_branch=branch, replace=settings.REPLACE, mirror=mirror_path) cmd = ['git', 'checkout', dest_branch] utils_setup.run_cmd(cmd, cwd=mirror_path) logger.info('Loading default.xml') tree = ET.parse('default.xml') root = tree.getroot() logger.info('Branching based on default.xml') default_revision = None base_url = None for child in root: if child.tag == 'remote': if 'fetch' in child.attrib: base_url = child.attrib['fetch'] if child.tag == 'default': if 'revision' in child.attrib: