def generate_patches(tinfoil, fetchdir, changeset, outputdir): import oe.recipeutils tmpoutdir = tempfile.mkdtemp(dir=outputdir) last_layer = None patchname = '' patches = [] outfile = None try: for change in changeset.recipechange_set.all().order_by('recipe__layerbranch'): fields = change.changed_fields(mapped=True) if fields: layerbranch = change.recipe.layerbranch layer = layerbranch.layer if last_layer != layer: patchname = "%s.patch" % layer.name patches.append(patchname) layerfetchdir = os.path.join(fetchdir, layer.get_fetch_dir()) utils.checkout_layer_branch(layerbranch, layerfetchdir) layerdir = os.path.join(layerfetchdir, layerbranch.vcs_subdir) config_data_copy = recipeparse.setup_layer(tinfoil.config_data, fetchdir, layerdir, layer, layerbranch, logger) if outfile: outfile.close() outfile = open(os.path.join(tmpoutdir, patchname), 'w') last_layer = layer recipefile = str(os.path.join(layerfetchdir, layerbranch.vcs_subdir, change.recipe.filepath, change.recipe.filename)) patchdatalist = oe.recipeutils.patch_recipe(config_data_copy, recipefile, fields, patch=True, relpath=layerfetchdir) for patchdata in patchdatalist: for line in patchdata: outfile.write(line) finally: if outfile: outfile.close() # If we have more than one patch, tar it up, otherwise just return the single patch file ret = None if len(patches) > 1: (tmptarfd, tmptarname) = tempfile.mkstemp('.tar.gz', 'bulkchange-', outputdir) tmptarfile = os.fdopen(tmptarfd, "wb") tar = tarfile.open(None, "w:gz", tmptarfile) try: for patch in patches: patchfn = os.path.join(tmpoutdir, patch) tar.add(patchfn, arcname=patch) finally: tar.close() ret = tmptarname elif len(patches) == 1: (tmppatchfd, tmppatchname) = tempfile.mkstemp('.patch', 'bulkchange-', outputdir) tmppatchfile = os.fdopen(tmppatchfd, "wb") try: with open(os.path.join(tmpoutdir, patches[0]), "rb") as patchfile: shutil.copyfileobj(patchfile, tmppatchfile) finally: tmppatchfile.close() ret = tmppatchname shutil.rmtree(tmpoutdir) return ret
def generate_patches(tinfoil, fetchdir, changeset, outputdir): tmpoutdir = tempfile.mkdtemp(dir=outputdir) last_layer = None patchname = '' patches = [] outfile = None try: for change in changeset.recipechange_set.all().order_by('recipe__layerbranch'): fields = change.changed_fields(mapped=True) if fields: layerbranch = change.recipe.layerbranch layer = layerbranch.layer if last_layer != layer: patchname = "%s.patch" % layer.name patches.append(patchname) layerfetchdir = os.path.join(fetchdir, layer.get_fetch_dir()) recipeparse.checkout_layer_branch(layerbranch, layerfetchdir) layerdir = os.path.join(layerfetchdir, layerbranch.vcs_subdir) config_data_copy = recipeparse.setup_layer(tinfoil.config_data, fetchdir, layerdir, layer, layerbranch) if outfile: outfile.close() outfile = open(os.path.join(tmpoutdir, patchname), 'w') last_layer = layer recipefile = str(os.path.join(layerfetchdir, layerbranch.vcs_subdir, change.recipe.filepath, change.recipe.filename)) varlist = list(set(fields.keys() + meta_vars)) varfiles = recipeparse.get_var_files(recipefile, varlist, config_data_copy) filevars = localise_file_vars(recipefile, varfiles, fields.keys()) for f, fvars in filevars.items(): filefields = dict((k, fields[k]) for k in fvars) patch = patch_recipe(f, layerfetchdir, filefields) for line in patch: outfile.write(line) finally: if outfile: outfile.close() # If we have more than one patch, tar it up, otherwise just return the single patch file ret = None if len(patches) > 1: (tmptarfd, tmptarname) = tempfile.mkstemp('.tar.gz', 'bulkchange-', outputdir) tmptarfile = os.fdopen(tmptarfd, "w") tar = tarfile.open(None, "w:gz", tmptarfile) for patch in patches: patchfn = os.path.join(tmpoutdir, patch) tar.add(patchfn) tar.close() ret = tmptarname elif len(patches) == 1: (tmppatchfd, tmppatchname) = tempfile.mkstemp('.patch', 'bulkchange-', outputdir) tmppatchfile = os.fdopen(tmppatchfd, "w") with open(os.path.join(tmpoutdir, patches[0]), "rb") as patchfile: shutil.copyfileobj(patchfile, tmppatchfile) tmppatchfile.close() ret = tmppatchname shutil.rmtree(tmpoutdir) return ret
def main(): if LooseVersion(git.__version__) < '0.3.1': logger.error( "Version of GitPython is too old, please install GitPython (python-git) 0.3.1 or later in order to use this script" ) sys.exit(1) parser = optparse.OptionParser(usage=""" %prog [options]""") parser.add_option("-b", "--branch", help="Specify branch to update", action="store", dest="branch", default='master') parser.add_option("-l", "--layer", help="Layer to update", action="store", dest="layer") parser.add_option( "-r", "--reload", help="Reload recipe data instead of updating since last update", action="store_true", dest="reload") parser.add_option( "", "--fullreload", help="Discard existing recipe data and fetch it from scratch", action="store_true", dest="fullreload") parser.add_option("-n", "--dry-run", help="Don't write any data back to the database", action="store_true", dest="dryrun") parser.add_option("", "--nocheckout", help="Don't check out branches", action="store_true", dest="nocheckout") parser.add_option("", "--stop-on-error", help="Stop on first parsing error", action="store_true", default=False, dest="stop_on_error") parser.add_option("-i", "--initial", help="Print initial values parsed from layer.conf only", action="store_true") parser.add_option("-d", "--debug", help="Enable debug output", action="store_const", const=logging.DEBUG, dest="loglevel", default=logging.INFO) parser.add_option("-q", "--quiet", help="Hide all output except error messages", action="store_const", const=logging.ERROR, dest="loglevel") parser.add_option( "", "--keep-temp", help="Preserve temporary directory at the end instead of deleting it", action="store_true") options, args = parser.parse_args(sys.argv) if len(args) > 1: logger.error('unexpected argument "%s"' % args[1]) parser.print_help() sys.exit(1) if options.fullreload: options.reload = True utils.setup_django() import settings from layerindex.models import LayerItem, LayerBranch, Recipe, RecipeFileDependency, Machine, Distro, BBAppend, BBClass, IncFile from django.db import transaction logger.setLevel(options.loglevel) branch = utils.get_branch(options.branch) if not branch: logger.error("Specified branch %s is not valid" % options.branch) sys.exit(1) fetchdir = settings.LAYER_FETCH_DIR if not fetchdir: logger.error("Please set LAYER_FETCH_DIR in settings.py") sys.exit(1) bitbakepath = os.path.join(fetchdir, 'bitbake') layer = utils.get_layer(options.layer) urldir = layer.get_fetch_dir() repodir = os.path.join(fetchdir, urldir) layerbranch = layer.get_layerbranch(options.branch) branchname = options.branch branchdesc = options.branch if layerbranch: if layerbranch.actual_branch: branchname = layerbranch.actual_branch branchdesc = "%s (%s)" % (options.branch, branchname) # Collect repo info repo = git.Repo(repodir) if repo.bare: logger.error('Repository %s is bare, not supported' % repodir) sys.exit(1) topcommit = repo.commit('origin/%s' % branchname) if options.nocheckout: topcommit = repo.commit('HEAD') tinfoil = None tempdir = None try: with transaction.atomic(): newbranch = False if not layerbranch: # LayerBranch doesn't exist for this branch, create it newbranch = True layerbranch = LayerBranch() layerbranch.layer = layer layerbranch.branch = branch layerbranch_source = layer.get_layerbranch(branch) if not layerbranch_source: layerbranch_source = layer.get_layerbranch(None) if layerbranch_source: layerbranch.vcs_subdir = layerbranch_source.vcs_subdir layerbranch.save() if layerbranch_source: for maintainer in layerbranch_source.layermaintainer_set.all( ): maintainer.pk = None maintainer.id = None maintainer.layerbranch = layerbranch maintainer.save() if layerbranch.vcs_subdir and not options.nocheckout: # Find latest commit in subdirectory # A bit odd to do it this way but apparently there's no other way in the GitPython API topcommit = next( repo.iter_commits('origin/%s' % branchname, paths=layerbranch.vcs_subdir), None) layerdir = os.path.join(repodir, layerbranch.vcs_subdir) layerdir_start = os.path.normpath(layerdir) + os.sep layerrecipes = Recipe.objects.filter(layerbranch=layerbranch) layermachines = Machine.objects.filter(layerbranch=layerbranch) layerdistros = Distro.objects.filter(layerbranch=layerbranch) layerappends = BBAppend.objects.filter(layerbranch=layerbranch) layerclasses = BBClass.objects.filter(layerbranch=layerbranch) layerincfiles = IncFile.objects.filter(layerbranch=layerbranch) if layerbranch.vcs_last_rev != topcommit.hexsha or options.reload or options.initial: # Check out appropriate branch if not options.nocheckout: utils.checkout_layer_branch(layerbranch, repodir, logger=logger) logger.info("Collecting data for layer %s on branch %s" % (layer.name, branchdesc)) try: (tinfoil, tempdir) = recipeparse.init_parser( settings, branch, bitbakepath, nocheckout=options.nocheckout, logger=logger) except recipeparse.RecipeParseError as e: logger.error(str(e)) sys.exit(1) logger.debug('Using temp directory %s' % tempdir) # Clear the default value of SUMMARY so that we can use DESCRIPTION instead if it hasn't been set tinfoil.config_data.setVar('SUMMARY', '') # Clear the default value of DESCRIPTION so that we can see where it's not set tinfoil.config_data.setVar('DESCRIPTION', '') # Clear the default value of HOMEPAGE ('unknown') tinfoil.config_data.setVar('HOMEPAGE', '') # Set a blank value for LICENSE so that it doesn't cause the parser to die (e.g. with meta-ti - # why won't they just fix that?!) tinfoil.config_data.setVar('LICENSE', '') layerconfparser = layerconfparse.LayerConfParse( logger=logger, tinfoil=tinfoil) layer_config_data = layerconfparser.parse_layer(layerdir) if not layer_config_data: logger.info( "Skipping update of layer %s for branch %s - conf/layer.conf may have parse issues" % (layer.name, branchdesc)) layerconfparser.shutdown() sys.exit(1) utils.set_layerbranch_collection_version(layerbranch, layer_config_data, logger=logger) if options.initial: # Use print() rather than logger.info() since "-q" makes it print nothing. for i in [ "BBFILE_COLLECTIONS", "LAYERVERSION", "LAYERDEPENDS", "LAYERRECOMMENDS" ]: print( '%s = "%s"' % (i, utils.get_layer_var( layer_config_data, i, logger))) sys.exit(0) # Set up for recording patch info utils.setup_core_layer_sys_path(settings, branch.name) skip_patches = False try: import oe.recipeutils except ImportError: logger.warn( 'Failed to find lib/oe/recipeutils.py in layers - patch information will not be collected' ) skip_patches = True utils.add_dependencies(layerbranch, layer_config_data, logger=logger) utils.add_recommends(layerbranch, layer_config_data, logger=logger) layerbranch.save() try: config_data_copy = recipeparse.setup_layer( tinfoil.config_data, fetchdir, layerdir, layer, layerbranch, logger) except recipeparse.RecipeParseError as e: logger.error(str(e)) sys.exit(1) if layerbranch.vcs_last_rev and not options.reload: try: diff = repo.commit( layerbranch.vcs_last_rev).diff(topcommit) except Exception as e: logger.warn( "Unable to get diff from last commit hash for layer %s - falling back to slow update: %s" % (layer.name, str(e))) diff = None else: diff = None # We handle recipes specially to try to preserve the same id # when recipe upgrades happen (so that if a user bookmarks a # recipe page it remains valid) layerrecipes_delete = [] layerrecipes_add = [] # Check if any paths should be ignored because there are layers within this layer removedirs = [] for root, dirs, files in os.walk(layerdir): for diritem in dirs: if os.path.exists( os.path.join(root, diritem, 'conf', 'layer.conf')): removedirs.append( os.path.join(root, diritem) + os.sep) if diff: # Apply git changes to existing recipe list if layerbranch.vcs_subdir: subdir_start = os.path.normpath( layerbranch.vcs_subdir) + os.sep else: subdir_start = "" updatedrecipes = set() dirtyrecipes = set() other_deletes = [] other_adds = [] for diffitem in diff.iter_change_type('R'): oldpath = diffitem.a_blob.path newpath = diffitem.b_blob.path skip = False for removedir in removedirs: # FIXME what about files moved into removedirs? if oldpath.startswith(removedir): skip = True break if skip: continue if oldpath.startswith(subdir_start): if not newpath.startswith(subdir_start): logger.debug( "Treating rename of %s to %s as a delete since new path is outside layer" % (oldpath, newpath)) other_deletes.append(diffitem) continue (oldtypename, oldfilepath, oldfilename) = recipeparse.detect_file_type( oldpath, subdir_start) (newtypename, newfilepath, newfilename) = recipeparse.detect_file_type( newpath, subdir_start) if oldtypename != newtypename: # This is most likely to be a .inc file renamed to a .bb - and since # there may be another recipe deleted at the same time we probably want # to consider that, so just treat it as a delete and an add logger.debug( "Treating rename of %s to %s as a delete and add (since type changed)" % (oldpath, newpath)) other_deletes.append(diffitem) other_adds.append(diffitem) elif oldtypename == 'recipe': results = layerrecipes.filter( filepath=oldfilepath).filter( filename=oldfilename) if len(results): recipe = results[0] logger.debug("Rename recipe %s to %s" % (recipe, newpath)) recipe.filepath = newfilepath recipe.filename = newfilename recipe.save() update_recipe_file( tinfoil, config_data_copy, os.path.join(layerdir, newfilepath), recipe, layerdir_start, repodir, options.stop_on_error, skip_patches) updatedrecipes.add( os.path.join(oldfilepath, oldfilename)) updatedrecipes.add( os.path.join(newfilepath, newfilename)) else: logger.warn( "Renamed recipe %s could not be found" % oldpath) other_adds.append(diffitem) elif oldtypename == 'bbappend': results = layerappends.filter( filepath=oldfilepath).filter( filename=oldfilename) if len(results): logger.debug( "Rename bbappend %s to %s" % (results[0], os.path.join(newfilepath, newfilename))) results[0].filepath = newfilepath results[0].filename = newfilename results[0].save() else: logger.warn( "Renamed bbappend %s could not be found" % oldpath) other_adds.append(diffitem) elif oldtypename == 'machine': results = layermachines.filter( name=oldfilename) if len(results): logger.debug("Rename machine %s to %s" % (results[0], newfilename)) results[0].name = newfilename results[0].save() else: logger.warn( "Renamed machine %s could not be found" % oldpath) other_adds.append(diffitem) elif oldtypename == 'distro': results = layerdistros.filter(name=oldfilename) if len(results): logger.debug("Rename distro %s to %s" % (results[0], newfilename)) results[0].name = newfilename results[0].save() else: logger.warn( "Renamed distro %s could not be found" % oldpath) other_adds.append(diffitem) elif oldtypename == 'bbclass': results = layerclasses.filter(name=oldfilename) if len(results): logger.debug("Rename class %s to %s" % (results[0], newfilename)) results[0].name = newfilename results[0].save() else: logger.warn( "Renamed class %s could not be found" % oldpath) other_adds.append(diffitem) elif oldtypename == 'incfile': results = layerincfiles.filter( path=os.path.join(oldfilepath, oldfilename)) if len(results): logger.debug("Rename inc file %s to %s" % (results[0], newfilename)) results[0].name = newfilename results[0].save() else: logger.warn( "Renamed inc file %s could not be found" % oldpath) other_adds.append(diffitem) deps = RecipeFileDependency.objects.filter( layerbranch=layerbranch).filter(path=oldpath) for dep in deps: dirtyrecipes.add(dep.recipe) for diffitem in itertools.chain(diff.iter_change_type('D'), other_deletes): path = diffitem.a_blob.path if path.startswith(subdir_start): skip = False for removedir in removedirs: if path.startswith(removedir): skip = True break if skip: continue (typename, filepath, filename) = recipeparse.detect_file_type( path, subdir_start) if typename == 'recipe': values = layerrecipes.filter( filepath=filepath).filter( filename=filename).values( 'id', 'filepath', 'filename', 'pn') if len(values): layerrecipes_delete.append(values[0]) logger.debug("Mark %s for deletion" % values[0]) updatedrecipes.add( os.path.join(values[0]['filepath'], values[0]['filename'])) else: logger.warn( "Deleted recipe %s could not be found" % path) elif typename == 'bbappend': layerappends.filter(filepath=filepath).filter( filename=filename).delete() elif typename == 'machine': layermachines.filter(name=filename).delete() elif typename == 'distro': layerdistros.filter(name=filename).delete() elif typename == 'bbclass': layerclasses.filter(name=filename).delete() elif typename == 'incfile': layerincfiles.filter(path=os.path.join( filepath, filename)).delete() for diffitem in itertools.chain(diff.iter_change_type('A'), other_adds): path = diffitem.b_blob.path if path.startswith(subdir_start): skip = False for removedir in removedirs: if path.startswith(removedir): skip = True break if skip: continue (typename, filepath, filename) = recipeparse.detect_file_type( path, subdir_start) if typename == 'recipe': layerrecipes_add.append( os.path.join(repodir, path)) logger.debug("Mark %s for addition" % path) updatedrecipes.add( os.path.join(filepath, filename)) elif typename == 'bbappend': append = BBAppend() append.layerbranch = layerbranch append.filename = filename append.filepath = filepath append.save() elif typename == 'machine': machine = Machine() machine.layerbranch = layerbranch machine.name = filename update_machine_conf_file( os.path.join(repodir, path), machine) machine.save() elif typename == 'distro': distro = Distro() distro.layerbranch = layerbranch distro.name = filename update_distro_conf_file( os.path.join(repodir, path), distro, config_data_copy) distro.save() elif typename == 'bbclass': bbclass = BBClass() bbclass.layerbranch = layerbranch bbclass.name = filename bbclass.save() elif typename == 'incfile': incfile = IncFile() incfile.layerbranch = layerbranch incfile.path = os.path.join(filepath, filename) incfile.save() for diffitem in diff.iter_change_type('M'): path = diffitem.b_blob.path if path.startswith(subdir_start): skip = False for removedir in removedirs: if path.startswith(removedir): skip = True break if skip: continue (typename, filepath, filename) = recipeparse.detect_file_type( path, subdir_start) if typename == 'recipe': logger.debug("Mark %s for update" % path) results = layerrecipes.filter( filepath=filepath).filter( filename=filename)[:1] if results: recipe = results[0] update_recipe_file( tinfoil, config_data_copy, os.path.join(layerdir, filepath), recipe, layerdir_start, repodir, options.stop_on_error, skip_patches) recipe.save() updatedrecipes.add(recipe.full_path()) elif typename == 'machine': results = layermachines.filter(name=filename) if results: machine = results[0] update_machine_conf_file( os.path.join(repodir, path), machine) machine.save() elif typename == 'distro': results = layerdistros.filter(name=filename) if results: distro = results[0] update_distro_conf_file( os.path.join(repodir, path), distro, config_data_copy) distro.save() deps = RecipeFileDependency.objects.filter( layerbranch=layerbranch).filter(path=path) for dep in deps: dirtyrecipes.add(dep.recipe) for recipe in dirtyrecipes: if not recipe.full_path() in updatedrecipes: update_recipe_file( tinfoil, config_data_copy, os.path.join(layerdir, recipe.filepath), recipe, layerdir_start, repodir, options.stop_on_error, skip_patches) else: # Collect recipe data from scratch layerrecipe_fns = [] if options.fullreload: layerrecipes.delete() else: # First, check which recipes still exist layerrecipe_values = layerrecipes.values( 'id', 'filepath', 'filename', 'pn') for v in layerrecipe_values: if v['filepath'].startswith('../'): # FIXME: These recipes were present due to a bug (not handling renames # to paths outside the layer) - this can be removed at some point in the future preserve = False else: root = os.path.join(layerdir, v['filepath']) fullpath = os.path.join(root, v['filename']) if os.path.exists(fullpath): preserve = True for removedir in removedirs: if fullpath.startswith(removedir): preserve = False break else: preserve = False if preserve: # Recipe still exists, update it results = layerrecipes.filter(id=v['id'])[:1] recipe = results[0] update_recipe_file(tinfoil, config_data_copy, root, recipe, layerdir_start, repodir, options.stop_on_error, skip_patches) else: # Recipe no longer exists, mark it for later on layerrecipes_delete.append(v) layerrecipe_fns.append(fullpath) layermachines.delete() layerdistros.delete() layerappends.delete() layerclasses.delete() for root, dirs, files in os.walk(layerdir): if '.git' in dirs: dirs.remove('.git') for diritem in dirs[:]: fullpath = os.path.join(root, diritem) + os.sep if fullpath in removedirs: dirs.remove(diritem) for f in files: fullpath = os.path.join(root, f) (typename, _, filename) = recipeparse.detect_file_type( fullpath, layerdir_start) if typename == 'recipe': if fullpath not in layerrecipe_fns: layerrecipes_add.append(fullpath) elif typename == 'bbappend': append = BBAppend() append.layerbranch = layerbranch append.filename = f append.filepath = os.path.relpath( root, layerdir) append.save() elif typename == 'machine': machine = Machine() machine.layerbranch = layerbranch machine.name = filename update_machine_conf_file(fullpath, machine) machine.save() elif typename == 'distro': distro = Distro() distro.layerbranch = layerbranch distro.name = filename update_distro_conf_file( fullpath, distro, config_data_copy) distro.save() elif typename == 'bbclass': bbclass = BBClass() bbclass.layerbranch = layerbranch bbclass.name = filename bbclass.save() elif typename == 'incfile': incfile = IncFile() incfile.layerbranch = layerbranch incfile.path = os.path.relpath( fullpath, layerdir) incfile.save() for added in layerrecipes_add: # This is good enough without actually parsing the file (pn, pv) = split_recipe_fn(added) oldid = -1 for deleted in layerrecipes_delete: if deleted['pn'] == pn: oldid = deleted['id'] layerrecipes_delete.remove(deleted) break if oldid > -1: # Reclaim a record we would have deleted results = Recipe.objects.filter(id=oldid)[:1] recipe = results[0] logger.debug("Reclaim %s for %s %s" % (recipe, pn, pv)) else: # Create new record logger.debug("Add new recipe %s" % added) recipe = Recipe() recipe.layerbranch = layerbranch recipe.filename = os.path.basename(added) root = os.path.dirname(added) recipe.filepath = os.path.relpath(root, layerdir) update_recipe_file(tinfoil, config_data_copy, root, recipe, layerdir_start, repodir, options.stop_on_error, skip_patches) recipe.save() for deleted in layerrecipes_delete: logger.debug("Delete %s" % deleted) results = Recipe.objects.filter(id=deleted['id'])[:1] recipe = results[0] recipe.delete() # Save repo info layerbranch.vcs_last_rev = topcommit.hexsha layerbranch.vcs_last_commit = datetime.fromtimestamp( topcommit.committed_date) else: logger.info("Layer %s is already up-to-date for branch %s" % (layer.name, branchdesc)) layerbranch.vcs_last_fetch = datetime.now() layerbranch.save() if options.dryrun: raise DryRunRollbackException() except KeyboardInterrupt: logger.warn("Update interrupted, changes to %s rolled back" % layer.name) sys.exit(254) except SystemExit: raise except DryRunRollbackException: pass except: import traceback logger.error(traceback.format_exc().rstrip()) sys.exit(1) finally: if tinfoil and (LooseVersion(bb.__version__) > LooseVersion("1.27")): tinfoil.shutdown() if tempdir: if options.keep_temp: logger.debug('Preserving temp directory %s' % tempdir) else: logger.debug('Deleting temp directory') utils.rmtree_force(tempdir) sys.exit(0)
def main(): parser = optparse.OptionParser(usage=""" %prog [options] <bitbakepath> <oeclassicpath>""") parser.add_option("-b", "--branch", help="Specify branch to import into", action="store", dest="branch", default='oe-classic') parser.add_option("-l", "--layer", help="Specify layer to import into", action="store", dest="layer", default='oe-classic') parser.add_option("-n", "--dry-run", help="Don't write any data back to the database", action="store_true", dest="dryrun") parser.add_option("-d", "--debug", help="Enable debug output", action="store_const", const=logging.DEBUG, dest="loglevel", default=logging.INFO) parser.add_option("-q", "--quiet", help="Hide all output except error messages", action="store_const", const=logging.ERROR, dest="loglevel") options, args = parser.parse_args(sys.argv) if len(args) < 3: logger.error('You must specify bitbakepath and oeclassicpath') parser.print_help() sys.exit(1) if len(args) > 3: logger.error('unexpected argument "%s"' % args[3]) parser.print_help() sys.exit(1) utils.setup_django() import settings from layerindex.models import LayerItem, LayerBranch, Recipe, ClassicRecipe, Machine, BBAppend, BBClass from django.db import transaction logger.setLevel(options.loglevel) branch = utils.get_branch(options.branch) if not branch: logger.error("Specified branch %s is not valid" % options.branch) sys.exit(1) res = list(LayerItem.objects.filter(name=options.layer)[:1]) if res: layer = res[0] else: layer = LayerItem() layer.name = options.layer layer.status = 'P' layer.layer_type = 'M' layer.summary = 'OE-Classic' layer.description = 'OpenEmbedded-Classic' layer.vcs_url = 'git://git.openembedded.org/openembedded' layer.vcs_web_url = 'http://cgit.openembedded.org/openembedded' layer.vcs_web_tree_base_url = 'http://cgit.openembedded.org/openembedded/tree/%path%' layer.vcs_web_file_base_url = 'http://cgit.openembedded.org/openembedded/tree/%path%' layer.comparison = True layer.save() layerbranch = layer.get_layerbranch(options.branch) if not layerbranch: # LayerBranch doesn't exist for this branch, create it layerbranch = LayerBranch() layerbranch.layer = layer layerbranch.branch = branch layerbranch.save() fetchdir = settings.LAYER_FETCH_DIR if not fetchdir: logger.error("Please set LAYER_FETCH_DIR in settings.py") sys.exit(1) if not os.path.exists(fetchdir): os.makedirs(fetchdir) fetchedrepos = [] failedrepos = [] bitbakepath = args[1] oeclassicpath = args[2] confparentdir = os.path.abspath( os.path.join(os.path.dirname(os.path.abspath(__file__)), '../../oe-classic')) os.environ['BBPATH'] = str("%s:%s" % (confparentdir, oeclassicpath)) try: (tinfoil, tempdir) = recipeparse.init_parser(settings, branch, bitbakepath, nocheckout=True, classic=True, logger=logger) except recipeparse.RecipeParseError as e: logger.error(str(e)) sys.exit(1) # Clear the default value of SUMMARY so that we can use DESCRIPTION instead if it hasn't been set tinfoil.config_data.setVar('SUMMARY', '') # Clear the default value of DESCRIPTION so that we can see where it's not set tinfoil.config_data.setVar('DESCRIPTION', '') # Clear the default value of HOMEPAGE ('unknown') tinfoil.config_data.setVar('HOMEPAGE', '') try: with transaction.atomic(): layerdir_start = os.path.normpath(oeclassicpath) + os.sep layerrecipes = Recipe.objects.filter(layerbranch=layerbranch) layermachines = Machine.objects.filter(layerbranch=layerbranch) layerdistros = Distro.objects.filter(layerbranch=layerbranch) layerappends = BBAppend.objects.filter(layerbranch=layerbranch) layerclasses = BBClass.objects.filter(layerbranch=layerbranch) try: config_data_copy = recipeparse.setup_layer( tinfoil.config_data, fetchdir, oeclassicpath, layer, layerbranch, logger) except recipeparse.RecipeParseError as e: logger.error(str(e)) sys.exit(1) layerrecipes.delete() layermachines.delete() layerdistros.delete() layerappends.delete() layerclasses.delete() for root, dirs, files in os.walk(oeclassicpath): if '.git' in dirs: dirs.remove('.git') for f in files: fullpath = os.path.join(root, f) (typename, filepath, filename) = recipeparse.detect_file_type( fullpath, layerdir_start) if typename == 'recipe': recipe = ClassicRecipe() recipe.layerbranch = layerbranch recipe.filename = filename recipe.filepath = filepath update_recipe_file(tinfoil, config_data_copy, root, recipe, layerdir_start, oeclassicpath) recipe.save() layerbranch.vcs_last_fetch = datetime.now() layerbranch.save() if options.dryrun: raise DryRunRollbackException() except DryRunRollbackException: pass except: import traceback traceback.print_exc() finally: tinfoil.shutdown() shutil.rmtree(tempdir) sys.exit(0)
def main(): parser = optparse.OptionParser( usage = """ %prog [options] <bitbakepath> <oeclassicpath>""") parser.add_option("-b", "--branch", help = "Specify branch to import into", action="store", dest="branch", default='oe-classic') parser.add_option("-l", "--layer", help = "Specify layer to import into", action="store", dest="layer", default='oe-classic') parser.add_option("-n", "--dry-run", help = "Don't write any data back to the database", action="store_true", dest="dryrun") parser.add_option("-d", "--debug", help = "Enable debug output", action="store_const", const=logging.DEBUG, dest="loglevel", default=logging.INFO) parser.add_option("-q", "--quiet", help = "Hide all output except error messages", action="store_const", const=logging.ERROR, dest="loglevel") options, args = parser.parse_args(sys.argv) if len(args) < 3: logger.error('You must specify bitbakepath and oeclassicpath') parser.print_help() sys.exit(1) if len(args) > 3: logger.error('unexpected argument "%s"' % args[3]) parser.print_help() sys.exit(1) utils.setup_django() import settings from layerindex.models import LayerItem, LayerBranch, Recipe, ClassicRecipe, Machine, BBAppend, BBClass from django.db import transaction logger.setLevel(options.loglevel) branch = utils.get_branch(options.branch) if not branch: logger.error("Specified branch %s is not valid" % options.branch) sys.exit(1) res = list(LayerItem.objects.filter(name=options.layer)[:1]) if res: layer = res[0] else: layer = LayerItem() layer.name = options.layer layer.status = 'P' layer.layer_type = 'M' layer.summary = 'OE-Classic' layer.description = 'OpenEmbedded-Classic' layer.vcs_url = 'git://git.openembedded.org/openembedded' layer.vcs_web_url = 'http://cgit.openembedded.org/cgit.cgi/openembedded' layer.vcs_web_tree_base_url = 'http://cgit.openembedded.org/cgit.cgi/openembedded/tree/%path%' layer.vcs_web_file_base_url = 'http://cgit.openembedded.org/cgit.cgi/openembedded/tree/%path%' layer.classic = True layer.save() layerbranch = layer.get_layerbranch(options.branch) if not layerbranch: # LayerBranch doesn't exist for this branch, create it layerbranch = LayerBranch() layerbranch.layer = layer layerbranch.branch = branch layerbranch.save() fetchdir = settings.LAYER_FETCH_DIR if not fetchdir: logger.error("Please set LAYER_FETCH_DIR in settings.py") sys.exit(1) if not os.path.exists(fetchdir): os.makedirs(fetchdir) fetchedrepos = [] failedrepos = [] bitbakepath = args[1] oeclassicpath = args[2] confparentdir = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '../../oe-classic')) os.environ['BBPATH'] = str("%s:%s" % (confparentdir, oeclassicpath)) try: (tinfoil, tempdir) = recipeparse.init_parser(settings, branch, bitbakepath, nocheckout=True, classic=True, logger=logger) except recipeparse.RecipeParseError as e: logger.error(str(e)) sys.exit(1) # Clear the default value of SUMMARY so that we can use DESCRIPTION instead if it hasn't been set tinfoil.config_data.setVar('SUMMARY', '') # Clear the default value of DESCRIPTION so that we can see where it's not set tinfoil.config_data.setVar('DESCRIPTION', '') # Clear the default value of HOMEPAGE ('unknown') tinfoil.config_data.setVar('HOMEPAGE', '') transaction.enter_transaction_management() transaction.managed(True) try: layerdir_start = os.path.normpath(oeclassicpath) + os.sep layerrecipes = Recipe.objects.filter(layerbranch=layerbranch) layermachines = Machine.objects.filter(layerbranch=layerbranch) layerappends = BBAppend.objects.filter(layerbranch=layerbranch) layerclasses = BBClass.objects.filter(layerbranch=layerbranch) try: config_data_copy = recipeparse.setup_layer(tinfoil.config_data, fetchdir, oeclassicpath, layer, layerbranch) except recipeparse.RecipeParseError as e: logger.error(str(e)) transaction.rollback() sys.exit(1) layerrecipes.delete() layermachines.delete() layerappends.delete() layerclasses.delete() for root, dirs, files in os.walk(oeclassicpath): if '.git' in dirs: dirs.remove('.git') for f in files: fullpath = os.path.join(root, f) (typename, filepath, filename) = recipeparse.detect_file_type(fullpath, layerdir_start) if typename == 'recipe': recipe = ClassicRecipe() recipe.layerbranch = layerbranch recipe.filename = filename recipe.filepath = filepath update_recipe_file(config_data_copy, root, recipe, layerdir_start, oeclassicpath) recipe.save() layerbranch.vcs_last_fetch = datetime.now() layerbranch.save() if options.dryrun: transaction.rollback() else: transaction.commit() except: import traceback traceback.print_exc() transaction.rollback() finally: transaction.leave_transaction_management() shutil.rmtree(tempdir) sys.exit(0)
def main(): if LooseVersion(git.__version__) < '0.3.1': logger.error("Version of GitPython is too old, please install GitPython (python-git) 0.3.1 or later in order to use this script") sys.exit(1) parser = optparse.OptionParser( usage = """ %prog [options]""") parser.add_option("-b", "--branch", help = "Specify branch to update", action="store", dest="branch", default='master') parser.add_option("-l", "--layer", help = "Specify layers to update (use commas to separate multiple). Default is all published layers.", action="store", dest="layers") parser.add_option("-r", "--reload", help = "Reload recipe data instead of updating since last update", action="store_true", dest="reload") parser.add_option("", "--fullreload", help = "Discard existing recipe data and fetch it from scratch", action="store_true", dest="fullreload") parser.add_option("-n", "--dry-run", help = "Don't write any data back to the database", action="store_true", dest="dryrun") parser.add_option("-x", "--nofetch", help = "Don't fetch repositories", action="store_true", dest="nofetch") parser.add_option("", "--nocheckout", help = "Don't check out branches", action="store_true", dest="nocheckout") parser.add_option("-d", "--debug", help = "Enable debug output", action="store_const", const=logging.DEBUG, dest="loglevel", default=logging.INFO) parser.add_option("-q", "--quiet", help = "Hide all output except error messages", action="store_const", const=logging.ERROR, dest="loglevel") options, args = parser.parse_args(sys.argv) if len(args) > 1: logger.error('unexpected argument "%s"' % args[1]) parser.print_help() sys.exit(1) if options.fullreload: options.reload = True utils.setup_django() import settings from layerindex.models import LayerItem, LayerBranch, Recipe, RecipeFileDependency, Machine, BBAppend, BBClass from django.db import transaction logger.setLevel(options.loglevel) branch = utils.get_branch(options.branch) if not branch: logger.error("Specified branch %s is not valid" % options.branch) sys.exit(1) fetchdir = settings.LAYER_FETCH_DIR if not fetchdir: logger.error("Please set LAYER_FETCH_DIR in settings.py") sys.exit(1) if options.layers: layerquery = LayerItem.objects.filter(classic=False).filter(name__in=options.layers.split(',')) if layerquery.count() == 0: logger.error('No layers matching specified query "%s"' % options.layers) sys.exit(1) else: layerquery = LayerItem.objects.filter(classic=False).filter(status='P') if layerquery.count() == 0: logger.info("No published layers to update") sys.exit(1) if not os.path.exists(fetchdir): os.makedirs(fetchdir) fetchedrepos = [] failedrepos = [] lockfn = os.path.join(fetchdir, "layerindex.lock") lockfile = utils.lock_file(lockfn) if not lockfile: logger.error("Layer index lock timeout expired") sys.exit(1) try: bitbakepath = os.path.join(fetchdir, 'bitbake') if not options.nofetch: # Fetch latest metadata from repositories for layer in layerquery: # Handle multiple layers in a single repo urldir = layer.get_fetch_dir() repodir = os.path.join(fetchdir, urldir) if not (layer.vcs_url in fetchedrepos or layer.vcs_url in failedrepos): logger.info("Fetching remote repository %s" % layer.vcs_url) out = None try: if not os.path.exists(repodir): out = utils.runcmd("git clone %s %s" % (layer.vcs_url, urldir), fetchdir, logger=logger) else: out = utils.runcmd("git fetch", repodir, logger=logger) except Exception as e: logger.error("Fetch of layer %s failed: %s" % (layer.name, str(e))) failedrepos.append(layer.vcs_url) continue fetchedrepos.append(layer.vcs_url) if not fetchedrepos: logger.error("No repositories could be fetched, exiting") sys.exit(1) logger.info("Fetching bitbake from remote repository %s" % settings.BITBAKE_REPO_URL) if not os.path.exists(bitbakepath): out = utils.runcmd("git clone %s %s" % (settings.BITBAKE_REPO_URL, 'bitbake'), fetchdir, logger=logger) else: out = utils.runcmd("git fetch", bitbakepath, logger=logger) try: (tinfoil, tempdir) = recipeparse.init_parser(settings, branch, bitbakepath, nocheckout=options.nocheckout, logger=logger) except recipeparse.RecipeParseError as e: logger.error(str(e)) sys.exit(1) # Clear the default value of SUMMARY so that we can use DESCRIPTION instead if it hasn't been set tinfoil.config_data.setVar('SUMMARY', '') # Clear the default value of DESCRIPTION so that we can see where it's not set tinfoil.config_data.setVar('DESCRIPTION', '') # Clear the default value of HOMEPAGE ('unknown') tinfoil.config_data.setVar('HOMEPAGE', '') # Set a blank value for LICENSE so that it doesn't cause the parser to die (e.g. with meta-ti - # why won't they just fix that?!) tinfoil.config_data.setVar('LICENSE', '') # Process and extract data from each layer for layer in layerquery: transaction.enter_transaction_management() transaction.managed(True) try: urldir = layer.get_fetch_dir() repodir = os.path.join(fetchdir, urldir) if layer.vcs_url in failedrepos: logger.info("Skipping update of layer %s as fetch of repository %s failed" % (layer.name, layer.vcs_url)) transaction.rollback() continue layerbranch = layer.get_layerbranch(options.branch) branchname = options.branch branchdesc = options.branch if layerbranch: if layerbranch.actual_branch: branchname = layerbranch.actual_branch branchdesc = "%s (%s)" % (options.branch, branchname) # Collect repo info repo = git.Repo(repodir) assert repo.bare == False try: if options.nocheckout: topcommit = repo.commit('HEAD') else: topcommit = repo.commit('origin/%s' % branchname) except: if layerbranch: logger.error("Failed update of layer %s - branch %s no longer exists" % (layer.name, branchdesc)) else: logger.info("Skipping update of layer %s - branch %s doesn't exist" % (layer.name, branchdesc)) transaction.rollback() continue newbranch = False if not layerbranch: # LayerBranch doesn't exist for this branch, create it newbranch = True layerbranch = LayerBranch() layerbranch.layer = layer layerbranch.branch = branch layerbranch_source = layer.get_layerbranch('master') if not layerbranch_source: layerbranch_source = layer.get_layerbranch(None) if layerbranch_source: layerbranch.vcs_subdir = layerbranch_source.vcs_subdir layerbranch.save() if layerbranch_source: for maintainer in layerbranch_source.layermaintainer_set.all(): maintainer.pk = None maintainer.id = None maintainer.layerbranch = layerbranch maintainer.save() for dep in layerbranch_source.dependencies_set.all(): dep.pk = None dep.id = None dep.layerbranch = layerbranch dep.save() if layerbranch.vcs_subdir and not options.nocheckout: # Find latest commit in subdirectory # A bit odd to do it this way but apparently there's no other way in the GitPython API topcommit = next(repo.iter_commits('origin/%s' % branchname, paths=layerbranch.vcs_subdir), None) if not topcommit: # This will error out if the directory is completely invalid or had never existed at this point # If it previously existed but has since been deleted, you will get the revision where it was # deleted - so we need to handle that case separately later if newbranch: logger.info("Skipping update of layer %s for branch %s - subdirectory %s does not exist on this branch" % (layer.name, branchdesc, layerbranch.vcs_subdir)) elif layerbranch.vcs_subdir: logger.error("Subdirectory for layer %s does not exist on branch %s - if this is legitimate, the layer branch record should be deleted" % (layer.name, branchdesc)) else: logger.error("Failed to get last revision for layer %s on branch %s" % (layer.name, branchdesc)) transaction.rollback() continue layerdir = os.path.join(repodir, layerbranch.vcs_subdir) layerdir_start = os.path.normpath(layerdir) + os.sep layerrecipes = Recipe.objects.filter(layerbranch=layerbranch) layermachines = Machine.objects.filter(layerbranch=layerbranch) layerappends = BBAppend.objects.filter(layerbranch=layerbranch) layerclasses = BBClass.objects.filter(layerbranch=layerbranch) if layerbranch.vcs_last_rev != topcommit.hexsha or options.reload: # Check out appropriate branch if not options.nocheckout: out = utils.runcmd("git checkout origin/%s" % branchname, repodir, logger=logger) out = utils.runcmd("git clean -f -x", repodir, logger=logger) if layerbranch.vcs_subdir and not os.path.exists(layerdir): if newbranch: logger.info("Skipping update of layer %s for branch %s - subdirectory %s does not exist on this branch" % (layer.name, branchdesc, layerbranch.vcs_subdir)) else: logger.error("Subdirectory for layer %s does not exist on branch %s - if this is legitimate, the layer branch record should be deleted" % (layer.name, branchdesc)) transaction.rollback() continue if not os.path.exists(os.path.join(layerdir, 'conf/layer.conf')): logger.error("conf/layer.conf not found for layer %s - is subdirectory set correctly?" % layer.name) transaction.rollback() continue logger.info("Collecting data for layer %s on branch %s" % (layer.name, branchdesc)) try: config_data_copy = recipeparse.setup_layer(tinfoil.config_data, fetchdir, layerdir, layer, layerbranch) except recipeparse.RecipeParseError as e: logger.error(str(e)) transaction.rollback() continue if layerbranch.vcs_last_rev and not options.reload: try: diff = repo.commit(layerbranch.vcs_last_rev).diff(topcommit) except Exception as e: logger.warn("Unable to get diff from last commit hash for layer %s - falling back to slow update: %s" % (layer.name, str(e))) diff = None else: diff = None # We handle recipes specially to try to preserve the same id # when recipe upgrades happen (so that if a user bookmarks a # recipe page it remains valid) layerrecipes_delete = [] layerrecipes_add = [] # Check if any paths should be ignored because there are layers within this layer removedirs = [] for root, dirs, files in os.walk(layerdir): for d in dirs: if os.path.exists(os.path.join(root, d, 'conf', 'layer.conf')): removedirs.append(os.path.join(root, d) + os.sep) if diff: # Apply git changes to existing recipe list if layerbranch.vcs_subdir: subdir_start = os.path.normpath(layerbranch.vcs_subdir) + os.sep else: subdir_start = "" updatedrecipes = set() for d in diff.iter_change_type('D'): path = d.a_blob.path if path.startswith(subdir_start): skip = False for d in removedirs: if path.startswith(d): skip = True break if skip: continue (typename, filepath, filename) = recipeparse.detect_file_type(path, subdir_start) if typename == 'recipe': values = layerrecipes.filter(filepath=filepath).filter(filename=filename).values('id', 'filepath', 'filename', 'pn') if len(values): layerrecipes_delete.append(values[0]) logger.debug("Mark %s for deletion" % values[0]) updatedrecipes.add(os.path.join(values[0]['filepath'], values[0]['filename'])) else: logger.warn("Deleted recipe %s could not be found" % path) elif typename == 'bbappend': layerappends.filter(filepath=filepath).filter(filename=filename).delete() elif typename == 'machine': layermachines.filter(name=filename).delete() elif typename == 'bbclass': layerclasses.filter(name=filename).delete() for d in diff.iter_change_type('A'): path = d.b_blob.path if path.startswith(subdir_start): skip = False for d in removedirs: if path.startswith(d): skip = True break if skip: continue (typename, filepath, filename) = recipeparse.detect_file_type(path, subdir_start) if typename == 'recipe': layerrecipes_add.append(os.path.join(repodir, path)) logger.debug("Mark %s for addition" % path) updatedrecipes.add(os.path.join(filepath, filename)) elif typename == 'bbappend': append = BBAppend() append.layerbranch = layerbranch append.filename = filename append.filepath = filepath append.save() elif typename == 'machine': machine = Machine() machine.layerbranch = layerbranch machine.name = filename update_machine_conf_file(os.path.join(repodir, path), machine) machine.save() elif typename == 'bbclass': bbclass = BBClass() bbclass.layerbranch = layerbranch bbclass.name = filename bbclass.save() dirtyrecipes = set() for d in diff.iter_change_type('M'): path = d.a_blob.path if path.startswith(subdir_start): skip = False for d in removedirs: if path.startswith(d): skip = True break if skip: continue (typename, filepath, filename) = recipeparse.detect_file_type(path, subdir_start) if typename == 'recipe': logger.debug("Mark %s for update" % path) results = layerrecipes.filter(filepath=filepath).filter(filename=filename)[:1] if results: recipe = results[0] update_recipe_file(config_data_copy, os.path.join(layerdir, filepath), recipe, layerdir_start, repodir) recipe.save() updatedrecipes.add(recipe.full_path()) elif typename == 'machine': results = layermachines.filter(name=filename) if results: machine = results[0] update_machine_conf_file(os.path.join(repodir, path), machine) machine.save() deps = RecipeFileDependency.objects.filter(layerbranch=layerbranch).filter(path=path) for dep in deps: dirtyrecipes.add(dep.recipe) for recipe in dirtyrecipes: if not recipe.full_path() in updatedrecipes: update_recipe_file(config_data_copy, os.path.join(layerdir, recipe.filepath), recipe, layerdir_start, repodir) else: # Collect recipe data from scratch layerrecipe_fns = [] if options.fullreload: layerrecipes.delete() else: # First, check which recipes still exist layerrecipe_values = layerrecipes.values('id', 'filepath', 'filename', 'pn') for v in layerrecipe_values: root = os.path.join(layerdir, v['filepath']) fullpath = os.path.join(root, v['filename']) preserve = True if os.path.exists(fullpath): for d in removedirs: if fullpath.startswith(d): preserve = False break else: preserve = False if preserve: # Recipe still exists, update it results = layerrecipes.filter(id=v['id'])[:1] recipe = results[0] update_recipe_file(config_data_copy, root, recipe, layerdir_start, repodir) else: # Recipe no longer exists, mark it for later on layerrecipes_delete.append(v) layerrecipe_fns.append(fullpath) layermachines.delete() layerappends.delete() layerclasses.delete() for root, dirs, files in os.walk(layerdir): if '.git' in dirs: dirs.remove('.git') for d in dirs[:]: fullpath = os.path.join(root, d) + os.sep if fullpath in removedirs: dirs.remove(d) for f in files: fullpath = os.path.join(root, f) (typename, _, filename) = recipeparse.detect_file_type(fullpath, layerdir_start) if typename == 'recipe': if fullpath not in layerrecipe_fns: layerrecipes_add.append(fullpath) elif typename == 'bbappend': append = BBAppend() append.layerbranch = layerbranch append.filename = f append.filepath = os.path.relpath(root, layerdir) append.save() elif typename == 'machine': machine = Machine() machine.layerbranch = layerbranch machine.name = filename update_machine_conf_file(fullpath, machine) machine.save() elif typename == 'bbclass': bbclass = BBClass() bbclass.layerbranch = layerbranch bbclass.name = filename bbclass.save() for added in layerrecipes_add: # This is good enough without actually parsing the file (pn, pv) = split_recipe_fn(added) oldid = -1 for deleted in layerrecipes_delete: if deleted['pn'] == pn: oldid = deleted['id'] layerrecipes_delete.remove(deleted) break if oldid > -1: # Reclaim a record we would have deleted results = Recipe.objects.filter(id=oldid)[:1] recipe = results[0] logger.debug("Reclaim %s for %s %s" % (recipe, pn, pv)) else: # Create new record logger.debug("Add new recipe %s" % added) recipe = Recipe() recipe.layerbranch = layerbranch recipe.filename = os.path.basename(added) root = os.path.dirname(added) recipe.filepath = os.path.relpath(root, layerdir) update_recipe_file(config_data_copy, root, recipe, layerdir_start, repodir) recipe.save() for deleted in layerrecipes_delete: logger.debug("Delete %s" % deleted) results = Recipe.objects.filter(id=deleted['id'])[:1] recipe = results[0] recipe.delete() # Save repo info layerbranch.vcs_last_rev = topcommit.hexsha layerbranch.vcs_last_commit = datetime.fromtimestamp(topcommit.committed_date) else: logger.info("Layer %s is already up-to-date for branch %s" % (layer.name, branchdesc)) layerbranch.vcs_last_fetch = datetime.now() layerbranch.save() if options.dryrun: transaction.rollback() else: transaction.commit() # Slightly hacky way of avoiding memory leaks bb.event.ui_queue = [] bb.parse.parse_py.BBHandler.cached_statements = {} bb.codeparser.codeparsercache = bb.codeparser.CodeParserCache() if hasattr(bb.codeparser, 'codecache'): bb.codeparser.codecache = bb.codeparser.SetCache() bb.fetch._checksum_cache = bb.checksum.FileChecksumCache() bb.fetch.urldata_cache = {} bb.fetch.saved_headrevs = {} bb.parse.__pkgsplit_cache__={} bb.parse.__mtime_cache = {} bb.parse.init_parser(tinfoil.config_data) except KeyboardInterrupt: transaction.rollback() logger.warn("Update interrupted, changes to %s rolled back" % layer.name) break except: import traceback traceback.print_exc() transaction.rollback() finally: transaction.leave_transaction_management() finally: utils.unlock_file(lockfile) shutil.rmtree(tempdir) sys.exit(0)