import logging import subprocess import urllib.request import json import datetime from django.utils import timezone sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), '..'))) import utils from layerconfparse import LayerConfParse class DryRunRollbackException(Exception): pass logger = utils.logger_create('LayerIndexImport') iso8601_date_re = re.compile('^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}') def datetime_hook(jsdict): for key, value in jsdict.items(): if isinstance(value, str) and iso8601_date_re.match(value): jsdict[key] = timezone.make_naive(datetime.datetime.strptime(value, "%Y-%m-%dT%H:%M:%S%z")) return jsdict def main(): parser = argparse.ArgumentParser(description="Layer index import utility. Imports layer information from another layer index instance using the REST API. WARNING: this will overwrite data in your database, use with caution!") parser.add_argument('url', help='Layer index URL to fetch from') parser.add_argument('-b', '--branch', action='store', help='Restrict to import a specific branch only (separate multiple branches with commas)')
import logging from datetime import datetime import re import tempfile import shutil import errno from distutils.version import LooseVersion import itertools import utils import recipeparse import layerconfparse import warnings warnings.filterwarnings("ignore", category=DeprecationWarning) logger = utils.logger_create('LayerIndexUpdate') # Ensure PythonGit is installed (buildhistory_analysis needs it) try: import git except ImportError: logger.error( "Please install PythonGit 0.3.1 or later in order to use this script") sys.exit(1) class DryRunRollbackException(Exception): pass def check_machine_conf(path, subdir_start):
# Copyright (C) 2013 Intel Corporation # Author: Paul Eggleton <*****@*****.**> # # Licensed under the MIT license, see COPYING.MIT for details import sys import os.path sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), '..'))) import optparse import re import utils import logging logger = utils.logger_create('LayerIndexClassicUpdate') class DryRunRollbackException(Exception): pass def main(): parser = optparse.OptionParser( usage = """ %prog [options]""") parser.add_option("-b", "--branch", help = "Specify branch to import into", action="store", dest="branch", default='oe-classic') parser.add_option("-l", "--layer",
import os import argparse import subprocess import logging import urllib.request import json import shutil import tempfile import tarfile sys.path.insert( 0, os.path.realpath(os.path.join(os.path.dirname(__file__), '..'))) import utils logger = utils.logger_create('ClearImport') def import_derivative(args, tempdir): def extract_tar(tarballfn): with tarfile.open(tarballfn, 'r') as tar: if not utils.check_tar_contents(tar): logger.error('Invalid source tarball') return None tar.extractall(tempdir) for entry in os.listdir(tempdir): pth = os.path.join(tempdir, entry) if os.path.isdir(pth): return pth logger.error('No directory found after extracting source tarball') return None
class ImportProject: logger = utils.logger_create('ProjectIndexImport') def find_layers(self, path): self.logger.debug("finding layer..") result = [] for root, _, files in os.walk(path, followlinks=True): for _ in fnmatch.filter(files, 'layer.conf'): if not root.endswith('conf'): continue self.logger.debug("Found %s" % root) result.append(root) return result def main(self): parser = optparse.OptionParser(usage=""" %prog [options] [directory]""") parser.add_option("-d", "--debug", help="Enable debug output", action="store_const", const=logging.DEBUG, dest="loglevel", default=logging.INFO) parser.add_option("-n", "--dry-run", help="Don't write any data back to the database", action="store_true", dest="dryrun") self.options, args = parser.parse_args(sys.argv) self.logger.setLevel(self.options.loglevel) if len(args) == 1: print("Please provide a directory.") sys.exit(1) install_dir = args[1] lc_list = self.find_layers(install_dir) core_layer = self.add_core(lc_list) if core_layer: lc_list.remove(core_layer) for layer in lc_list: self.add_layer(layer) def add_layer(self, layer): self.logger.debug("Processing layer %s" % layer) try: git_dir = utils.runcmd("git rev-parse --show-toplevel", destdir=layer, logger=self.logger) except Exception as e: self.logger.error( "Cannot get root dir for layer %s: %s - Skipping." % (layer, str(e))) return 1 layer_name = layer.split('/')[-2] layer_subdir = None if os.path.basename(git_dir) != layer_name: layer_subdir = layer_name layer_name = self.get_layer_name(layer) for i in [1, 2, 3]: remote = utils.runcmd("git remote", destdir=git_dir, logger=self.logger) if not remote: self.logger.warning("Cannot find remote git for %s" % layer_name) return 1 try: git_url = utils.runcmd("git config --get remote.%s.url" % remote, destdir=git_dir, logger=self.logger) except Exception as e: self.logger.info( "Cannot get remote.%s.url for git dir %s: %s" % (remote, git_dir, str(e))) if not os.path.exists(git_url): # Assume this is remote. self.logger.debug("Found git url = %s" % git_url) remote_branch = utils.runcmd( "git rev-parse --abbrev-ref --symbolic-full-name @\{u\}", destdir=git_dir, logger=self.logger) if remote_branch.startswith(remote): actual_branch = remote_branch[len(remote) + 1:] break self.logger.debug("Iterating to find git url into %s" % git_dir) git_dir = git_url if not git_url: self.logger.warning("Cannot find layer %s git url" % layer) return 1 cmd = ['import_layer.py'] if self.options.loglevel == logging.DEBUG: cmd.append("-d") if layer_subdir: cmd.append("-s") cmd.append(layer_subdir) if actual_branch: cmd.append("-a") cmd.append(actual_branch) cmd.append(git_url) cmd.append(layer_name) prefix = "Calling" if self.options.dryrun: prefix = "Would Call" self.logger.info("%s import_layer.main with %s for dir %s" % (prefix, str(cmd), layer)) sys.argv = cmd if not self.options.dryrun: try: import_layer.main() except SystemExit as see: return see.code return 0 def get_layer_name(self, layerconfdir): layer_name = layerconfdir.split('/')[-2] self.logger.debug('getting layer %s' % layerconfdir) layer_conf = os.path.join(layerconfdir, 'layer.conf') if os.path.isfile(layer_conf): with open(layer_conf) as conf: for line in conf: if 'BBLAYERS_LAYERINDEX_NAME' in line: layer_name = line.split('=')[1].strip(' "\n') return layer_name def add_core(self, layers): utils.setup_django() core = None import settings for layer in layers: layer_name = self.get_layer_name(layer) if layer_name == settings.CORE_LAYER_NAME: if self.add_layer(layer): self.logger.info('Failed to add core layer\n') core = layer self.update() break return core def update(self): update_py = os.path.realpath( os.path.join(os.path.dirname(__file__), '../update.py')) cmd = [update_py] if self.options.loglevel == logging.DEBUG: cmd.append("-d") sys.argv = cmd self.logger.info("update") if not self.options.dryrun: try: update.main() except SystemExit: return 1 return 0
# Copyright (C) 2013 Intel Corporation # Author: Paul Eggleton <*****@*****.**> # # Licensed under the MIT license, see COPYING.MIT for details import sys import os.path sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), '..'))) import optparse import re import utils import logging logger = utils.logger_create('LayerIndexImport') def read_page(site, path): ret = {} import httplib conn = httplib.HTTPConnection(site) conn.request("GET", path) resp = conn.getresponse() if resp.status in [200, 302]: data = resp.read() in_table = False for line in data.splitlines(): if line.startswith('{|'): in_table = True continue
import glob import shutil import subprocess import string import shlex from distutils.version import LooseVersion sys.path.insert( 0, os.path.realpath(os.path.join(os.path.dirname(__file__), '..'))) sys.path.insert( 0, os.path.realpath(os.path.join(os.path.dirname(__file__), 'lib'))) import utils import recipeparse logger = utils.logger_create('LayerIndexOtherDistro') class DryRunRollbackException(Exception): pass def get_gourl(goipath): # Much more crude than the original implementation gourl = goipath if not '://' in gourl: gourl = 'https://' + gourl return gourl def update_recipe_file(path, recipe, repodir, raiseexceptions=False):
# # Licensed under the MIT license, see COPYING.MIT for details import sys import os sys.path.insert( 0, os.path.realpath(os.path.join(os.path.dirname(__file__), '..'))) import argparse import re import utils import logging import json logger = utils.logger_create('LayerIndexComparisonUpdate') class DryRunRollbackException(Exception): pass def export(args, layerbranch, skiplist): from layerindex.models import ClassicRecipe from django.db.models import F jscoverlist = [] recipequery = ClassicRecipe.objects.filter( layerbranch=layerbranch, deleted=False).order_by('pn').values( 'pn', 'cover_pn', 'cover_status',
os.path.realpath( os.path.join(os.path.dirname(__file__), '..', '..', 'layerindex'))) import argparse import re import glob import utils import logging from datetime import date, datetime class DryRunRollbackException(Exception): pass logger = utils.logger_create('RRSDump') def main(): parser = argparse.ArgumentParser(description="Dump RRS upgrade info") parser.add_argument("plan", help="Specify maintenance plan to operate on") parser.add_argument('-d', '--debug', action='store_true', help='Enable debug output') parser.add_argument('-q', '--quiet', action='store_true', help='Hide all output except error messages') args = parser.parse_args()
# Copyright (C) 2013 Intel Corporation # Author: Paul Eggleton <*****@*****.**> # # Licensed under the MIT license, see COPYING.MIT for details import sys import os.path sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), '..'))) import optparse import re import utils import logging logger = utils.logger_create('LayerIndexClassicUpdate') def main(): parser = optparse.OptionParser( usage = """ %prog [options]""") parser.add_option("-b", "--branch", help = "Specify branch to import into", action="store", dest="branch", default='oe-classic') parser.add_option("-l", "--layer", help = "Specify layer to import into", action="store", dest="layer", default='oe-classic') parser.add_option("-n", "--dry-run",
import os.path sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), '..'))) import optparse import logging from datetime import datetime import fnmatch import re import tempfile import shutil from distutils.version import LooseVersion import utils import recipeparse logger = utils.logger_create('LayerIndexUpdate') def update_recipe_file(data, path, recipe, layerdir_start, repodir): fn = str(os.path.join(path, recipe.filename)) try: logger.debug('Updating recipe %s' % fn) envdata = bb.cache.Cache.loadDataFull(fn, [], data) envdata.setVar('SRCPV', 'X') envdata.setVar('SRCDATE', 'X') envdata.setVar('SRCREV', 'X') envdata.setVar('OPIE_SRCREV', 'X') recipe.pn = envdata.getVar("PN", True) recipe.pv = envdata.getVar("PV", True) recipe.summary = envdata.getVar("SUMMARY", True) recipe.description = envdata.getVar("DESCRIPTION", True)
import sys import os import json import argparse import logging from datetime import date, datetime sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), '..'))) sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), '..', '..', 'layerindex'))) import utils class DryRunRollbackException(Exception): pass logger = utils.logger_create('RrsExport') def rrs_export(args): utils.setup_django() import settings from rrs.models import RecipeUpstreamHistory, RecipeUpstream class DatetimeEncoder(json.JSONEncoder): def default(self, obj): if isinstance(obj, (datetime, date)): return obj.isoformat() return super(DatetimeEncoder, self).default(obj) # FIXME this doesn't export the layerbranch associated with the recipe (since it # was intended to export from the original forked RRS which was OE-Core only)
import sys import os sys.path.insert( 0, os.path.realpath(os.path.join(os.path.dirname(__file__), '..'))) import optparse import utils import logging class DryRunRollbackException(Exception): pass logger = utils.logger_create('LayerIndexFixup') def main(): parser = optparse.OptionParser(usage=""" %prog [options""") parser.add_option("-n", "--dry-run", help="Don't write any data back to the database", action="store_true", dest="dryrun") parser.add_option("-d", "--debug", help="Enable debug output", action="store_const",