import logging
import subprocess
import urllib.request
import json
import datetime
from django.utils import timezone
sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), '..')))
import utils
from layerconfparse import LayerConfParse
class DryRunRollbackException(Exception):
pass
logger = utils.logger_create('LayerIndexImport')
iso8601_date_re = re.compile('^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}')
def datetime_hook(jsdict):
for key, value in jsdict.items():
if isinstance(value, str) and iso8601_date_re.match(value):
jsdict[key] = timezone.make_naive(datetime.datetime.strptime(value, "%Y-%m-%dT%H:%M:%S%z"))
return jsdict
def main():
parser = argparse.ArgumentParser(description="Layer index import utility. Imports layer information from another layer index instance using the REST API. WARNING: this will overwrite data in your database, use with caution!")
parser.add_argument('url', help='Layer index URL to fetch from')
parser.add_argument('-b', '--branch', action='store', help='Restrict to import a specific branch only (separate multiple branches with commas)')
import logging
from datetime import datetime
import re
import tempfile
import shutil
import errno
from distutils.version import LooseVersion
import itertools
import utils
import recipeparse
import layerconfparse
import warnings
warnings.filterwarnings("ignore", category=DeprecationWarning)
logger = utils.logger_create('LayerIndexUpdate')
# Ensure PythonGit is installed (buildhistory_analysis needs it)
try:
import git
except ImportError:
logger.error(
"Please install PythonGit 0.3.1 or later in order to use this script")
sys.exit(1)
class DryRunRollbackException(Exception):
pass
def check_machine_conf(path, subdir_start):
# Copyright (C) 2013 Intel Corporation
# Author: Paul Eggleton <*****@*****.**>
#
# Licensed under the MIT license, see COPYING.MIT for details
import sys
import os.path
sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), '..')))
import optparse
import re
import utils
import logging
logger = utils.logger_create('LayerIndexClassicUpdate')
class DryRunRollbackException(Exception):
pass
def main():
parser = optparse.OptionParser(
usage = """
%prog [options]""")
parser.add_option("-b", "--branch",
help = "Specify branch to import into",
action="store", dest="branch", default='oe-classic')
parser.add_option("-l", "--layer",
import os
import argparse
import subprocess
import logging
import urllib.request
import json
import shutil
import tempfile
import tarfile
sys.path.insert(
0, os.path.realpath(os.path.join(os.path.dirname(__file__), '..')))
import utils
logger = utils.logger_create('ClearImport')
def import_derivative(args, tempdir):
def extract_tar(tarballfn):
with tarfile.open(tarballfn, 'r') as tar:
if not utils.check_tar_contents(tar):
logger.error('Invalid source tarball')
return None
tar.extractall(tempdir)
for entry in os.listdir(tempdir):
pth = os.path.join(tempdir, entry)
if os.path.isdir(pth):
return pth
logger.error('No directory found after extracting source tarball')
return None
class ImportProject:
logger = utils.logger_create('ProjectIndexImport')
def find_layers(self, path):
self.logger.debug("finding layer..")
result = []
for root, _, files in os.walk(path, followlinks=True):
for _ in fnmatch.filter(files, 'layer.conf'):
if not root.endswith('conf'):
continue
self.logger.debug("Found %s" % root)
result.append(root)
return result
def main(self):
parser = optparse.OptionParser(usage="""
%prog [options] [directory]""")
parser.add_option("-d",
"--debug",
help="Enable debug output",
action="store_const",
const=logging.DEBUG,
dest="loglevel",
default=logging.INFO)
parser.add_option("-n",
"--dry-run",
help="Don't write any data back to the database",
action="store_true",
dest="dryrun")
self.options, args = parser.parse_args(sys.argv)
self.logger.setLevel(self.options.loglevel)
if len(args) == 1:
print("Please provide a directory.")
sys.exit(1)
install_dir = args[1]
lc_list = self.find_layers(install_dir)
core_layer = self.add_core(lc_list)
if core_layer:
lc_list.remove(core_layer)
for layer in lc_list:
self.add_layer(layer)
def add_layer(self, layer):
self.logger.debug("Processing layer %s" % layer)
try:
git_dir = utils.runcmd("git rev-parse --show-toplevel",
destdir=layer,
logger=self.logger)
except Exception as e:
self.logger.error(
"Cannot get root dir for layer %s: %s - Skipping." %
(layer, str(e)))
return 1
layer_name = layer.split('/')[-2]
layer_subdir = None
if os.path.basename(git_dir) != layer_name:
layer_subdir = layer_name
layer_name = self.get_layer_name(layer)
for i in [1, 2, 3]:
remote = utils.runcmd("git remote",
destdir=git_dir,
logger=self.logger)
if not remote:
self.logger.warning("Cannot find remote git for %s" %
layer_name)
return 1
try:
git_url = utils.runcmd("git config --get remote.%s.url" %
remote,
destdir=git_dir,
logger=self.logger)
except Exception as e:
self.logger.info(
"Cannot get remote.%s.url for git dir %s: %s" %
(remote, git_dir, str(e)))
if not os.path.exists(git_url):
# Assume this is remote.
self.logger.debug("Found git url = %s" % git_url)
remote_branch = utils.runcmd(
"git rev-parse --abbrev-ref --symbolic-full-name @\{u\}",
destdir=git_dir,
logger=self.logger)
if remote_branch.startswith(remote):
actual_branch = remote_branch[len(remote) + 1:]
break
self.logger.debug("Iterating to find git url into %s" % git_dir)
git_dir = git_url
if not git_url:
self.logger.warning("Cannot find layer %s git url" % layer)
return 1
cmd = ['import_layer.py']
if self.options.loglevel == logging.DEBUG:
cmd.append("-d")
if layer_subdir:
cmd.append("-s")
cmd.append(layer_subdir)
if actual_branch:
cmd.append("-a")
cmd.append(actual_branch)
cmd.append(git_url)
cmd.append(layer_name)
prefix = "Calling"
if self.options.dryrun:
prefix = "Would Call"
self.logger.info("%s import_layer.main with %s for dir %s" %
(prefix, str(cmd), layer))
sys.argv = cmd
if not self.options.dryrun:
try:
import_layer.main()
except SystemExit as see:
return see.code
return 0
def get_layer_name(self, layerconfdir):
layer_name = layerconfdir.split('/')[-2]
self.logger.debug('getting layer %s' % layerconfdir)
layer_conf = os.path.join(layerconfdir, 'layer.conf')
if os.path.isfile(layer_conf):
with open(layer_conf) as conf:
for line in conf:
if 'BBLAYERS_LAYERINDEX_NAME' in line:
layer_name = line.split('=')[1].strip(' "\n')
return layer_name
def add_core(self, layers):
utils.setup_django()
core = None
import settings
for layer in layers:
layer_name = self.get_layer_name(layer)
if layer_name == settings.CORE_LAYER_NAME:
if self.add_layer(layer):
self.logger.info('Failed to add core layer\n')
core = layer
self.update()
break
return core
def update(self):
update_py = os.path.realpath(
os.path.join(os.path.dirname(__file__), '../update.py'))
cmd = [update_py]
if self.options.loglevel == logging.DEBUG:
cmd.append("-d")
sys.argv = cmd
self.logger.info("update")
if not self.options.dryrun:
try:
update.main()
except SystemExit:
return 1
return 0
# Copyright (C) 2013 Intel Corporation
# Author: Paul Eggleton <*****@*****.**>
#
# Licensed under the MIT license, see COPYING.MIT for details
import sys
import os.path
sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), '..')))
import optparse
import re
import utils
import logging
logger = utils.logger_create('LayerIndexImport')
def read_page(site, path):
ret = {}
import httplib
conn = httplib.HTTPConnection(site)
conn.request("GET", path)
resp = conn.getresponse()
if resp.status in [200, 302]:
data = resp.read()
in_table = False
for line in data.splitlines():
if line.startswith('{|'):
in_table = True
continue
import glob
import shutil
import subprocess
import string
import shlex
from distutils.version import LooseVersion
sys.path.insert(
0, os.path.realpath(os.path.join(os.path.dirname(__file__), '..')))
sys.path.insert(
0, os.path.realpath(os.path.join(os.path.dirname(__file__), 'lib')))
import utils
import recipeparse
logger = utils.logger_create('LayerIndexOtherDistro')
class DryRunRollbackException(Exception):
pass
def get_gourl(goipath):
# Much more crude than the original implementation
gourl = goipath
if not '://' in gourl:
gourl = 'https://' + gourl
return gourl
def update_recipe_file(path, recipe, repodir, raiseexceptions=False):
#
# Licensed under the MIT license, see COPYING.MIT for details
import sys
import os
sys.path.insert(
0, os.path.realpath(os.path.join(os.path.dirname(__file__), '..')))
import argparse
import re
import utils
import logging
import json
logger = utils.logger_create('LayerIndexComparisonUpdate')
class DryRunRollbackException(Exception):
pass
def export(args, layerbranch, skiplist):
from layerindex.models import ClassicRecipe
from django.db.models import F
jscoverlist = []
recipequery = ClassicRecipe.objects.filter(
layerbranch=layerbranch, deleted=False).order_by('pn').values(
'pn',
'cover_pn',
'cover_status',
os.path.realpath(
os.path.join(os.path.dirname(__file__), '..', '..', 'layerindex')))
import argparse
import re
import glob
import utils
import logging
from datetime import date, datetime
class DryRunRollbackException(Exception):
pass
logger = utils.logger_create('RRSDump')
def main():
parser = argparse.ArgumentParser(description="Dump RRS upgrade info")
parser.add_argument("plan", help="Specify maintenance plan to operate on")
parser.add_argument('-d',
'--debug',
action='store_true',
help='Enable debug output')
parser.add_argument('-q',
'--quiet',
action='store_true',
help='Hide all output except error messages')
args = parser.parse_args()
# Copyright (C) 2013 Intel Corporation
# Author: Paul Eggleton <*****@*****.**>
#
# Licensed under the MIT license, see COPYING.MIT for details
import sys
import os.path
sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), '..')))
import optparse
import re
import utils
import logging
logger = utils.logger_create('LayerIndexClassicUpdate')
def main():
parser = optparse.OptionParser(
usage = """
%prog [options]""")
parser.add_option("-b", "--branch",
help = "Specify branch to import into",
action="store", dest="branch", default='oe-classic')
parser.add_option("-l", "--layer",
help = "Specify layer to import into",
action="store", dest="layer", default='oe-classic')
parser.add_option("-n", "--dry-run",
import os.path
sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), '..')))
import optparse
import logging
from datetime import datetime
import fnmatch
import re
import tempfile
import shutil
from distutils.version import LooseVersion
import utils
import recipeparse
logger = utils.logger_create('LayerIndexUpdate')
def update_recipe_file(data, path, recipe, layerdir_start, repodir):
fn = str(os.path.join(path, recipe.filename))
try:
logger.debug('Updating recipe %s' % fn)
envdata = bb.cache.Cache.loadDataFull(fn, [], data)
envdata.setVar('SRCPV', 'X')
envdata.setVar('SRCDATE', 'X')
envdata.setVar('SRCREV', 'X')
envdata.setVar('OPIE_SRCREV', 'X')
recipe.pn = envdata.getVar("PN", True)
recipe.pv = envdata.getVar("PV", True)
recipe.summary = envdata.getVar("SUMMARY", True)
recipe.description = envdata.getVar("DESCRIPTION", True)
import sys
import os
import json
import argparse
import logging
from datetime import date, datetime
sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), '..')))
sys.path.insert(0, os.path.realpath(os.path.join(os.path.dirname(__file__), '..', '..', 'layerindex')))
import utils
class DryRunRollbackException(Exception):
pass
logger = utils.logger_create('RrsExport')
def rrs_export(args):
utils.setup_django()
import settings
from rrs.models import RecipeUpstreamHistory, RecipeUpstream
class DatetimeEncoder(json.JSONEncoder):
def default(self, obj):
if isinstance(obj, (datetime, date)):
return obj.isoformat()
return super(DatetimeEncoder, self).default(obj)
# FIXME this doesn't export the layerbranch associated with the recipe (since it
# was intended to export from the original forked RRS which was OE-Core only)
import sys
import os
sys.path.insert(
0, os.path.realpath(os.path.join(os.path.dirname(__file__), '..')))
import optparse
import utils
import logging
class DryRunRollbackException(Exception):
pass
logger = utils.logger_create('LayerIndexFixup')
def main():
parser = optparse.OptionParser(usage="""
%prog [options""")
parser.add_option("-n",
"--dry-run",
help="Don't write any data back to the database",
action="store_true",
dest="dryrun")
parser.add_option("-d",
"--debug",
help="Enable debug output",
action="store_const",