def get_layerbranch(layername, local_path):
layerbranch = layerbranches.get(layername, None)
if layerbranch:
return layerbranch
actualname = layername
if layername == 'meta':
actualname = settings.CORE_LAYER_NAME
jslayer = jsdata['layers'][layername]
layer, created = LayerItem.objects.get_or_create(
name=actualname)
if created:
layer.status = 'X'
layer.layer_type = 'M'
layer.summary = 'N/A'
layer.description = 'N/A'
layer.vcs_url = jslayer.get('vcs_url', '')
utils.validate_vcs_url(layer.vcs_url)
layer.comparison = True
utils.validate_fields(layer)
layer.save()
layerbranch = LayerBranch()
layerbranch.layer = layer
layerbranch.branch = branch
layerbranch.vcs_subdir = jslayer.get('vcs_subdir', '')
layerbranch.actual_branch = jslayer.get(
'actual_branch', '')
layerbranch.local_path = local_path
utils.validate_fields(layerbranch)
layerbranch.save()
layerbranches[layername] = layerbranch
return layerbranch
def main():
valid_layer_name = re.compile('[-\w]+$')
parser = optparse.OptionParser(usage="""
%prog [options] <url> [name]""")
utils.setup_django()
layer_type_help, layer_type_choices = get_layer_type_choices()
parser.add_option("-s",
"--subdir",
help="Specify subdirectory",
action="store",
dest="subdir")
parser.add_option("-t",
"--type",
help=layer_type_help,
choices=layer_type_choices,
action="store",
dest="layer_type",
default='')
parser.add_option("-n",
"--dry-run",
help="Don't write any data back to the database",
action="store_true",
dest="dryrun")
parser.add_option("-d",
"--debug",
help="Enable debug output",
action="store_const",
const=logging.DEBUG,
dest="loglevel",
default=logging.INFO)
parser.add_option("",
"--github-auth",
help="Specify github username:password",
action="store",
dest="github_auth")
parser.add_option("-q",
"--quiet",
help="Hide all output except error messages",
action="store_const",
const=logging.ERROR,
dest="loglevel")
parser.add_option("-a",
"--actual-branch",
help="Set actual branch",
action="store",
dest="actual_branch")
options, args = parser.parse_args(sys.argv)
if len(args) < 2:
print("Please specify URL of repository for layer")
sys.exit(1)
layer_url = args[1]
if len(args) > 2:
layer_name = args[2]
else:
if options.subdir:
layer_name = options.subdir
else:
layer_name = [x for x in layer_url.split('/') if x][-1]
if layer_name.endswith('.git'):
layer_name = layer_name[:-4]
if not valid_layer_name.match(layer_name):
logger.error(
'Invalid layer name "%s" - Layer name can only include letters, numbers and dashes.',
layer_name)
sys.exit(1)
if options.github_auth:
if not ':' in options.github_auth:
logger.error(
'--github-auth value must be specified as username:password')
sys.exit(1)
splitval = options.github_auth.split(':')
github_login = splitval[0]
github_password = splitval[1]
else:
github_login = None
github_password = None
import settings
from layerindex.models import LayerItem, LayerBranch, LayerDependency, LayerMaintainer
from django.db import transaction
logger.setLevel(options.loglevel)
fetchdir = settings.LAYER_FETCH_DIR
if not fetchdir:
logger.error("Please set LAYER_FETCH_DIR in settings.py")
sys.exit(1)
if not os.path.exists(fetchdir):
os.makedirs(fetchdir)
master_branch = utils.get_branch('master')
core_layer = None
try:
with transaction.atomic():
# Fetch layer
logger.info('Fetching repository %s' % layer_url)
layer = LayerItem()
layer.name = layer_name
layer.status = 'P'
layer.summary = 'tempvalue'
layer.description = layer.summary
set_vcs_fields(layer, layer_url)
urldir = layer.get_fetch_dir()
repodir = os.path.join(fetchdir, urldir)
out = None
try:
if not os.path.exists(repodir):
out = utils.runcmd("git clone %s %s" %
(layer.vcs_url, urldir),
fetchdir,
logger=logger)
else:
out = utils.runcmd("git fetch", repodir, logger=logger)
except Exception as e:
logger.error("Fetch failed: %s" % str(e))
sys.exit(1)
actual_branch = 'master'
if (options.actual_branch):
actual_branch = options.actual_branch
try:
out = utils.runcmd("git checkout origin/%s" % actual_branch,
repodir,
logger=logger)
except subprocess.CalledProcessError:
actual_branch = None
branches = utils.runcmd("git branch -r",
repodir,
logger=logger)
for line in branches.splitlines():
if 'origin/HEAD ->' in line:
actual_branch = line.split('-> origin/')[-1]
break
if not actual_branch:
logger.error(
"Repository has no master branch nor origin/HEAD")
sys.exit(1)
out = utils.runcmd("git checkout origin/%s" % actual_branch,
repodir,
logger=logger)
layer_paths = []
if options.subdir:
layerdir = os.path.join(repodir, options.subdir)
if not os.path.exists(layerdir):
logger.error(
"Subdirectory %s does not exist in repository for master branch"
% options.subdir)
sys.exit(1)
if not os.path.exists(os.path.join(layerdir,
'conf/layer.conf')):
logger.error(
"conf/layer.conf not found in subdirectory %s" %
options.subdir)
sys.exit(1)
layer_paths.append(layerdir)
else:
if os.path.exists(os.path.join(repodir, 'conf/layer.conf')):
layer_paths.append(repodir)
# Find subdirs with a conf/layer.conf
for subdir in os.listdir(repodir):
subdir_path = os.path.join(repodir, subdir)
if os.path.isdir(subdir_path):
if os.path.exists(
os.path.join(subdir_path, 'conf/layer.conf')):
layer_paths.append(subdir_path)
if not layer_paths:
logger.error(
"conf/layer.conf not found in repository or first level subdirectories - is subdirectory set correctly?"
)
sys.exit(1)
if 'github.com' in layer.vcs_url:
json_data, owner_json_data = get_github_layerinfo(
layer.vcs_url, github_login, github_password)
for layerdir in layer_paths:
layer.pk = None
if layerdir != repodir:
subdir = os.path.relpath(layerdir, repodir)
if len(layer_paths) > 1:
layer.name = subdir
else:
subdir = ''
if LayerItem.objects.filter(name=layer.name).exists():
if LayerItem.objects.filter(name=layer.name).exclude(
vcs_url=layer.vcs_url).exists():
conflict_list = LayerItem.objects.filter(
name=layer.name).exclude(vcs_url=layer.vcs_url)
conflict_list_urls = []
for conflict in conflict_list:
conflict_list_urls.append(conflict.vcs_url)
cln = ', '.join(conflict_list_urls)
logger.error(
'A layer named "%s" already exists in the database. Possible name collision with %s.vcs_url = %s'
% (layer.name, layer.name, cln))
sys.exit(1)
else:
logger.info(
'The layer named "%s" already exists in the database. Skipping this layer with same vcs_url'
% layer.name)
layer_paths = [x for x in layer_paths if x != layerdir]
continue
logger.info('Creating layer %s' % layer.name)
# Guess layer type if not specified
if options.layer_type:
layer.layer_type = options.layer_type
elif layer.name in ['openembedded-core', 'meta-oe']:
layer.layer_type = 'A'
elif glob.glob(os.path.join(layerdir, 'conf/distro/*.conf')):
layer.layer_type = 'D'
elif glob.glob(os.path.join(layerdir, 'conf/machine/*.conf')):
layer.layer_type = 'B'
else:
layer.layer_type = 'M'
layer.save()
layerbranch = LayerBranch()
layerbranch.layer = layer
layerbranch.branch = master_branch
if layerdir != repodir:
layerbranch.vcs_subdir = subdir
if actual_branch:
layerbranch.actual_branch = actual_branch
layerbranch.save()
if layer.name != settings.CORE_LAYER_NAME:
if not core_layer:
core_layer = utils.get_layer(settings.CORE_LAYER_NAME)
if core_layer:
logger.debug('Adding dep %s to %s' %
(core_layer.name, layer.name))
layerdep = LayerDependency()
layerdep.layerbranch = layerbranch
layerdep.dependency = core_layer
layerdep.save()
layerconfparser = LayerConfParse(logger=logger)
try:
config_data = layerconfparser.parse_layer(layerdir)
if config_data:
utils.add_dependencies(layerbranch,
config_data,
logger=logger)
utils.add_recommends(layerbranch,
config_data,
logger=logger)
finally:
layerconfparser.shutdown()
# Get some extra meta-information
readme_files = glob.glob(os.path.join(layerdir, 'README*'))
if (not readme_files) and subdir:
readme_files = glob.glob(os.path.join(repodir, 'README*'))
maintainer_files = glob.glob(
os.path.join(layerdir, 'MAINTAINERS'))
if (not maintainer_files) and subdir:
maintainer_files = glob.glob(
os.path.join(repodir, 'MAINTAINERS'))
maintainers = []
if readme_files:
(desc, maintainers, deps) = readme_extract(readme_files[0])
if desc:
layer.summary = layer.name
layer.description = desc
if maintainer_files:
maintainers.extend(maintainers_extract(readme_files[0]))
if (not maintainers) and 'github.com' in layer.vcs_url:
if json_data:
layer.summary = json_data['description']
layer.description = layer.summary
if owner_json_data:
owner_name = owner_json_data.get('name', None)
owner_email = owner_json_data.get('email', None)
if owner_name and owner_email:
maintainers.append('%s <%s>' %
(owner_name, owner_email))
if layer.name == 'openembedded-core':
layer.summary = 'Core metadata'
elif layer.name == 'meta-oe':
layer.summary = 'Additional shared OE metadata'
layer.description = layer.summary
if maintainers:
maint_re = re.compile(
r'^"?([^"@$<>]+)"? *<([^<> ]+)>[ -]*(.+)?$')
for maintentry in maintainers:
res = maint_re.match(maintentry)
if res:
maintainer = LayerMaintainer()
maintainer.layerbranch = layerbranch
maintainer.name = res.group(1).strip()
maintainer.email = res.group(2)
if res.group(3):
maintainer.responsibility = res.group(
3).strip()
maintainer.save()
layer.save()
if not layer_paths:
logger.error('No layers added.')
sys.exit(1)
if options.dryrun:
raise DryRunRollbackException()
except DryRunRollbackException:
pass
sys.exit(0)
def main():
parser = optparse.OptionParser(
usage = """
%prog [options] <url> [name]""")
parser.add_option("-s", "--subdir",
help = "Specify subdirectory",
action="store", dest="subdir")
parser.add_option("-n", "--dry-run",
help = "Don't write any data back to the database",
action="store_true", dest="dryrun")
parser.add_option("-d", "--debug",
help = "Enable debug output",
action="store_const", const=logging.DEBUG, dest="loglevel", default=logging.INFO)
parser.add_option("", "--github-auth",
help = "Specify github username:password",
action="store", dest="github_auth")
parser.add_option("-q", "--quiet",
help = "Hide all output except error messages",
action="store_const", const=logging.ERROR, dest="loglevel")
options, args = parser.parse_args(sys.argv)
if len(args) < 2:
print("Please specify URL of repository for layer")
sys.exit(1)
layer_url = args[1]
if len(args) > 2:
layer_name = args[2]
else:
if options.subdir:
layer_name = options.subdir
else:
layer_name = filter(None, layer_url.split('/'))[-1]
if layer_name.endswith('.git'):
layer_name = layer_name[:-4]
if options.github_auth:
if not ':' in options.github_auth:
logger.error('--github-auth value must be specified as username:password')
sys.exit(1)
splitval = options.github_auth.split(':')
github_login = splitval[0]
github_password = splitval[1]
else:
github_login = None
github_password = None
utils.setup_django()
import settings
from layerindex.models import LayerItem, LayerBranch, LayerDependency, LayerMaintainer
from django.db import transaction
logger.setLevel(options.loglevel)
fetchdir = settings.LAYER_FETCH_DIR
if not fetchdir:
logger.error("Please set LAYER_FETCH_DIR in settings.py")
sys.exit(1)
if not os.path.exists(fetchdir):
os.makedirs(fetchdir)
master_branch = utils.get_branch('master')
core_layer = None
transaction.enter_transaction_management()
transaction.managed(True)
try:
# Fetch layer
logger.info('Fetching repository %s' % layer_url)
layer = LayerItem()
layer.name = layer_name
layer.status = 'P'
layer.layer_type = 'M'
layer.summary = 'tempvalue'
layer.description = layer.summary
set_vcs_fields(layer, layer_url)
urldir = layer.get_fetch_dir()
repodir = os.path.join(fetchdir, urldir)
out = None
try:
if not os.path.exists(repodir):
out = utils.runcmd("git clone %s %s" % (layer.vcs_url, urldir), fetchdir, logger=logger)
else:
out = utils.runcmd("git fetch", repodir, logger=logger)
except Exception as e:
logger.error("Fetch failed: %s" % str(e))
sys.exit(1)
actual_branch = ''
try:
out = utils.runcmd("git checkout origin/master", repodir, logger=logger)
except subprocess.CalledProcessError:
branches = utils.runcmd("git branch -r", repodir, logger=logger)
for line in branches.splitlines():
if 'origin/HEAD ->' in line:
actual_branch = line.split('-> origin/')[-1]
break
if not actual_branch:
logger.error("Repository has no master branch nor origin/HEAD")
sys.exit(1)
out = utils.runcmd("git checkout origin/%s" % actual_branch, repodir, logger=logger)
layer_paths = []
if options.subdir:
layerdir = os.path.join(repodir, options.subdir)
if not os.path.exists(layerdir):
logger.error("Subdirectory %s does not exist in repository for master branch" % options.subdir)
sys.exit(1)
if not os.path.exists(os.path.join(layerdir, 'conf/layer.conf')):
logger.error("conf/layer.conf not found in subdirectory %s" % options.subdir)
sys.exit(1)
layer_paths.append(layerdir)
else:
if os.path.exists(os.path.join(repodir, 'conf/layer.conf')):
layer_paths.append(repodir)
# Find subdirs with a conf/layer.conf
for subdir in os.listdir(repodir):
subdir_path = os.path.join(repodir, subdir)
if os.path.isdir(subdir_path):
if os.path.exists(os.path.join(subdir_path, 'conf/layer.conf')):
layer_paths.append(subdir_path)
if not layer_paths:
logger.error("conf/layer.conf not found in repository or first level subdirectories - is subdirectory set correctly?")
sys.exit(1)
if 'github.com' in layer.vcs_url:
json_data, owner_json_data = get_github_layerinfo(layer.vcs_url, github_login, github_password)
for layerdir in layer_paths:
layer.pk = None
if layerdir != repodir:
subdir = os.path.relpath(layerdir, repodir)
if len(layer_paths) > 1:
layer.name = subdir
else:
subdir = ''
if LayerItem.objects.filter(name=layer.name).exists():
logger.error('A layer named "%s" already exists in the database' % layer_name)
sys.exit(1)
logger.info('Creating layer %s' % layer.name)
# Guess layer type
if glob.glob(os.path.join(layerdir, 'conf/distro/*.conf')):
layer.layer_type = 'D'
elif glob.glob(os.path.join(layerdir, 'conf/machine/*.conf')):
layer.layer_type = 'B'
layer.save()
layerbranch = LayerBranch()
layerbranch.layer = layer
layerbranch.branch = master_branch
if layerdir != repodir:
layerbranch.vcs_subdir = subdir
if actual_branch:
layerbranch.actual_branch = actual_branch
layerbranch.save()
if layer.name != settings.CORE_LAYER_NAME:
if not core_layer:
core_layer = utils.get_layer(settings.CORE_LAYER_NAME)
if core_layer:
layerdep = LayerDependency()
layerdep.layerbranch = layerbranch
layerdep.dependency = core_layer
layerdep.save()
# Get some extra meta-information
readme_files = glob.glob(os.path.join(layerdir, 'README*'))
if (not readme_files) and subdir:
readme_files = glob.glob(os.path.join(repodir, 'README*'))
maintainer_files = glob.glob(os.path.join(layerdir, 'MAINTAINERS'))
if (not maintainer_files) and subdir:
maintainer_files = glob.glob(os.path.join(repodir, 'MAINTAINERS'))
maintainers = []
if readme_files:
(desc, maintainers, deps) = readme_extract(readme_files[0])
if desc:
layer.summary = layer.name
layer.description = desc
if maintainer_files:
maintainers.extend(maintainers_extract(readme_files[0]))
if (not maintainers) and 'github.com' in layer.vcs_url:
if json_data:
layer.summary = json_data['description']
layer.description = layer.summary
if owner_json_data:
owner_name = owner_json_data.get('name', None)
owner_email = owner_json_data.get('email', None)
if owner_name and owner_email:
maintainers.append('%s <%s>' % (owner_name, owner_email))
if layer.name == 'openembedded-core':
layer.summary = 'Core metadata'
layer.layer_type = 'A'
elif layer.name == 'meta-oe':
layer.summary = 'Additional shared OE metadata'
layer.description = layer.summary
layer.layer_type = 'A'
if maintainers:
maint_re = re.compile(r'^"?([^"@$<>]+)"? *<([^<> ]+)>[ -]*(.+)?$')
for maintentry in maintainers:
res = maint_re.match(maintentry)
if res:
maintainer = LayerMaintainer()
maintainer.layerbranch = layerbranch
maintainer.name = res.group(1).strip()
maintainer.email = res.group(2)
if res.group(3):
maintainer.responsibility = res.group(3).strip()
maintainer.save()
layer.save()
if options.dryrun:
transaction.rollback()
else:
transaction.commit()
except:
transaction.rollback()
raise
finally:
transaction.leave_transaction_management()
sys.exit(0)