def check_branch_layer(args):
from layerindex.models import Branch, LayerItem, LayerBranch
branch = utils.get_branch(args.branch)
if branch:
if not branch.comparison:
logger.error("Specified branch %s is not a comparison branch" %
args.branch)
return 1, None
else:
branch = Branch()
branch.name = args.branch
branch.bitbake_branch = '-'
branch.short_description = '' # FIXME
branch.updates_enabled = False
branch.comparison = True
branch.save()
layer = LayerItem.objects.filter(name=args.layer).first()
if layer:
if not layer.comparison:
logger.error("Specified layer %s is not a comparison layer" %
args.branch)
return 1, None
else:
layer = LayerItem()
layer.name = args.layer
layer.layer_type = 'M'
layer.summary = layer.name
layer.description = layer.name
layer.comparison = True
layer.save()
layerbranch = layer.get_layerbranch(args.branch)
if not layerbranch:
layerbranch = LayerBranch()
layerbranch.layer = layer
layerbranch.branch = branch
layerbranch.save()
return 0, layerbranch
def main():
parser = argparse.ArgumentParser(description="Layer index import utility. Imports layer information from another layer index instance using the REST API. WARNING: this will overwrite data in your database, use with caution!")
parser.add_argument('url', help='Layer index URL to fetch from')
parser.add_argument('-b', '--branch', action='store', help='Restrict to import a specific branch only (separate multiple branches with commas)')
parser.add_argument('-l', '--layer', action='store', help='Restrict to import a specific layer only (regular expressions allowed)')
parser.add_argument('-r', '--reload', action='store_true', help='Reload data even if it is up-to-date')
parser.add_argument('-n', '--dry-run', action='store_true', help="Don't write any data back to the database")
parser.add_argument('-d', '--debug', action='store_true', help='Enable debug output')
parser.add_argument('-q', '--quiet', action='store_true', help='Hide all output except error messages')
args = parser.parse_args()
if args.debug:
loglevel = logging.DEBUG
elif args.quiet:
loglevel = logging.WARNING
else:
loglevel = logging.INFO
utils.setup_django()
import settings
from layerindex.models import Branch, LayerItem, LayerBranch, LayerDependency, LayerMaintainer, LayerNote, Recipe, Source, Patch, PackageConfig, StaticBuildDep, DynamicBuildDep, RecipeFileDependency, Machine, Distro, BBClass, BBAppend, IncFile
from django.db import transaction
logger.setLevel(loglevel)
fetchdir = settings.LAYER_FETCH_DIR
if not fetchdir:
logger.error("Please set LAYER_FETCH_DIR in settings.py")
sys.exit(1)
if not os.path.exists(fetchdir):
os.makedirs(fetchdir)
layerindex_url = args.url
if not layerindex_url.endswith('/'):
layerindex_url += '/'
if not '/layerindex/api/' in layerindex_url:
layerindex_url += 'layerindex/api/'
rq = urllib.request.Request(layerindex_url)
data = urllib.request.urlopen(rq).read()
jsdata = json.loads(data.decode('utf-8'))
branches_url = jsdata['branches']
layers_url = jsdata['layerItems']
layerdeps_url = jsdata['layerDependencies']
layerbranches_url = jsdata['layerBranches']
layermaintainers_url = jsdata.get('layerMaintainers', None)
layernotes_url = jsdata.get('layerNotes', None)
recipes_url = jsdata.get('recipesExtended', None)
machines_url = jsdata.get('machines', None)
distros_url = jsdata.get('distros', None)
classes_url = jsdata.get('classes', None)
appends_url = jsdata.get('appends', None)
incfiles_url = jsdata.get('incFiles', None)
logger.debug('Getting branches')
# Get branches (we assume the ones we want are already there, so skip any that aren't)
rq = urllib.request.Request(branches_url)
data = urllib.request.urlopen(rq).read()
jsdata = json.loads(data.decode('utf-8'))
branch_idmap = {}
filter_branches = []
if args.branch:
for branch in args.branch.split(','):
if not Branch.objects.filter(name=branch).exists():
logger.error('"%s" is not a valid branch in this database (branches must be created manually first)' % branch)
sys.exit(1)
filter_branches.append(branch)
for branchjs in jsdata:
if filter_branches and branchjs['name'] not in filter_branches:
logger.debug('Skipping branch %s, not in specified branch list' % branchjs['name'])
continue
res = Branch.objects.filter(name=branchjs['name'])
if res:
branch = res.first()
branch_idmap[branchjs['id']] = branch
else:
logger.debug('Skipping branch %s, not in database' % branchjs['name'])
if args.layer:
layer_re = re.compile('^' + args.layer + '$')
else:
layer_re = None
try:
with transaction.atomic():
# Get layers
logger.info('Importing layers')
rq = urllib.request.Request(layers_url)
data = urllib.request.urlopen(rq).read()
jsdata = json.loads(data.decode('utf-8'), object_hook=datetime_hook)
layer_idmap = {}
exclude_fields = ['id', 'updated']
for layerjs in jsdata:
if layer_re and not layer_re.match(layerjs['name']):
logger.debug('Skipping layer %s, does not match layer restriction' % layerjs['name'])
continue
layeritem = LayerItem.objects.filter(name=layerjs['name']).first()
if layeritem:
# Already have this layer
if layerjs['updated'] <= layeritem.updated and not args.reload:
logger.debug('Skipping layer %s, already up-to-date' % layerjs['name'])
layer_idmap[layerjs['id']] = layeritem
continue
else:
logger.debug('Updating layer %s' % layerjs['name'])
else:
logger.debug('Adding layer %s' % layerjs['name'])
layeritem = LayerItem()
for key, value in layerjs.items():
if key in exclude_fields:
continue
setattr(layeritem, key, value)
layeritem.save()
layer_idmap[layerjs['id']] = layeritem
# Get layer branches
logger.debug('Importing layer branches')
rq = urllib.request.Request(layerbranches_url)
data = urllib.request.urlopen(rq).read()
jsdata = json.loads(data.decode('utf-8'), object_hook=datetime_hook)
layerbranch_idmap = {}
def import_child_items(parentobj, objclass, childlist=None, url=None, parent_orig_id=None, parentfield=None, exclude_fields=None, key_fields=None, custom_fields=None, custom_field_cb=None):
logger.debug('Importing %s for %s' % (objclass._meta.verbose_name_plural, parentobj))
if parentfield is None:
parentfield = parentobj.__class__.__name__.lower()
if exclude_fields is None:
exclude = ['id', parentfield]
else:
exclude = exclude_fields[:]
if custom_fields is not None:
exclude += custom_fields
if key_fields is None:
keys = None
else:
# The parent field always needs to be part of the keys
keys = key_fields + [parentfield]
def fetch_api_url(api_url):
rq = urllib.request.Request(api_url)
data = urllib.request.urlopen(rq).read()
return json.loads(data.decode('utf-8'))
if url:
if parent_orig_id is None:
raise Exception('import_child_items: if url is specified then parent_orig_id must also be specified')
childjsdata = fetch_api_url(url + '?filter=%s:%s' % (parentfield, parent_orig_id))
elif childlist is not None:
childjsdata = childlist
else:
raise Exception('import_child_items: either url or childlist must be specified')
manager = getattr(parentobj, objclass.__name__.lower() + '_set')
existing_ids = list(manager.values_list('id', flat=True))
updated_ids = []
def import_list(childjslist):
for childjs in childjslist:
vals = {}
for key, value in childjs.items():
if key in exclude:
continue
vals[key] = value
vals[parentfield] = parentobj
if keys:
keyvals = {k: vals[k] for k in keys}
else:
keyvals = vals
# In the case of multiple records with the same keys (e.g. multiple recipes with same pn),
# we need to skip ones we've already touched
obj = None
created = False
for entry in manager.filter(**keyvals):
if entry.id not in updated_ids:
obj = entry
break
else:
created = True
obj = objclass(**keyvals)
for key, value in vals.items():
setattr(obj, key, value)
# Need to have saved before calling custom_field_cb since the function might be adding child objects
obj.save()
updated_ids.append(obj.id)
if custom_field_cb is not None:
custom_field_cb(obj, childjs)
if not created:
if obj.id in existing_ids:
existing_ids.remove(obj.id)
if 'results' in childjsdata:
while True:
import_list(childjsdata['results'])
if childjsdata.get('next', None):
childjsdata = fetch_api_url(childjsdata['next'])
if not 'results' in childjsdata:
break
else:
break
else:
import_list(childjsdata)
for idv in existing_ids:
objclass.objects.filter(id=idv).delete()
def package_config_field_handler(package_config, pjsdata):
for dep in pjsdata['builddeps']:
dynamic_build_dependency, created = DynamicBuildDep.objects.get_or_create(name=dep)
if created:
dynamic_build_dependency.save()
dynamic_build_dependency.package_configs.add(package_config)
dynamic_build_dependency.recipes.add(package_config.recipe)
def recipe_field_handler(recipe, recipejs):
sources = recipejs.get('sources', [])
import_child_items(recipe, Source, childlist=sources, key_fields=['url'])
patches = recipejs.get('patches', [])
import_child_items(recipe, Patch, childlist=patches, key_fields=['path'])
existing_deps = list(recipe.staticbuilddep_set.values_list('name', flat=True))
for dep in recipejs['staticbuilddeps']:
depobj, created = StaticBuildDep.objects.get_or_create(name=dep)
if created:
depobj.save()
elif dep in existing_deps:
existing_deps.remove(dep)
depobj.recipes.add(recipe)
for existing_dep in existing_deps:
recipe.staticbuilddep_set.filter(name=existing_dep).first().recipes.remove(recipe)
package_configs = recipejs.get('package_configs', [])
import_child_items(recipe, PackageConfig, childlist=package_configs, custom_fields=['builddeps'], custom_field_cb=package_config_field_handler, key_fields=['feature'])
# RecipeFileDependency objects need to be handled specially (since they link to a separate LayerBranch)
existing_filedeps = list(recipe.recipefiledependency_set.values_list('id', flat=True))
filedeps = recipejs.get('filedeps', [])
for filedep in filedeps:
target_layerbranch = layerbranch_idmap.get(filedep['layerbranch'], None)
if target_layerbranch is None:
logger.debug('Skipping recipe file dependency on layerbranch %s, branch not imported' % filedep['layerbranch'])
continue
depobj, created = RecipeFileDependency.objects.get_or_create(recipe=recipe, layerbranch=target_layerbranch, path=filedep['path'])
if created:
depobj.save()
elif depobj.id in existing_filedeps:
existing_filedeps.remove(depobj.id)
for idv in existing_filedeps:
RecipeFileDependency.objects.filter(id=idv).delete()
# Get list of layerbranches that currently exist, so we can delete any that
# we don't find in the remote layer index (assuming they are on branches
# that *do* exist in the remote index and are in the list specified by
# -b/--branch, if any)
existing_layerbranches = list(LayerBranch.objects.filter(branch__in=branch_idmap.values()).values_list('id', flat=True))
exclude_fields = ['id', 'layer', 'branch', 'yp_compatible_version', 'updated']
layercount = len(jsdata)
for i, layerbranchjs in enumerate(jsdata):
branch = branch_idmap.get(layerbranchjs['branch'], None)
if not branch:
# We don't have this branch, skip it
logger.debug('Skipping layerbranch %s, branch not imported' % layerbranchjs['id'])
continue
layer = layer_idmap.get(layerbranchjs['layer'], None)
if not layer:
# We didn't import this layer, skip it
logger.debug('Skipping layerbranch %s, layer not imported' % layerbranchjs['id'])
continue
layerbranch = LayerBranch.objects.filter(layer=layer).filter(branch=branch).first()
if layerbranch:
# The layerbranch already exists (this will occur for layers
# that already existed, since we need to have those in layer_idmap
# to be able to import layer dependencies)
existing_layerbranches.remove(layerbranch.id)
if layerbranchjs['updated'] <= layerbranch.updated and not args.reload:
logger.debug('Skipping layerbranch %s, already up-to-date' % layerbranchjs['id'])
layerbranch_idmap[layerbranchjs['id']] = layerbranch
continue
logger.info('Updating %s (%d/%d)' % (layerbranch, i+1, layercount))
else:
layerbranch = LayerBranch()
layerbranch.branch = branch
layerbranch.layer = layer
logger.info('Importing %s (%d/%d)' % (layerbranch, i+1, layercount))
for key, value in layerbranchjs.items():
if key in exclude_fields:
continue
setattr(layerbranch, key, value)
layerbranch.save()
layerbranch_idmap[layerbranchjs['id']] = layerbranch
if recipes_url:
import_child_items(layerbranch,
Recipe,
url=recipes_url,
parent_orig_id=layerbranchjs['id'],
exclude_fields=['id', 'layerbranch', 'updated'],
custom_fields=['sources', 'patches', 'package_configs'],
custom_field_cb=recipe_field_handler,
key_fields=['pn'])
if machines_url:
import_child_items(layerbranch,
Machine,
url=machines_url,
parent_orig_id=layerbranchjs['id'],
exclude_fields=['id', 'layerbranch', 'updated'],
key_fields=['name'])
if distros_url:
import_child_items(layerbranch,
Distro,
url=distros_url,
parent_orig_id=layerbranchjs['id'],
exclude_fields=['id', 'layerbranch', 'updated'],
key_fields=['name'])
# The models below don't have an "updated" field at present, but it does
# no harm to leave it as excluded in case it does get added in the future
if classes_url:
import_child_items(layerbranch,
BBClass,
url=classes_url,
parent_orig_id=layerbranchjs['id'],
exclude_fields=['id', 'layerbranch', 'updated'],
key_fields=['name'])
if appends_url:
import_child_items(layerbranch,
BBAppend,
url=appends_url,
parent_orig_id=layerbranchjs['id'],
exclude_fields=['id', 'layerbranch', 'updated'],
key_fields=['filename'])
if incfiles_url:
import_child_items(layerbranch,
IncFile,
url=incfiles_url,
parent_orig_id=layerbranchjs['id'],
exclude_fields=['id', 'layerbranch', 'updated'],
key_fields=['path'])
for idv in existing_layerbranches:
layerbranch = LayerBranch.objects.get(id=idv)
if layer_re is None or layer_re.match(layerbranch.layer.name):
logger.debug('Deleting layerbranch %s' % layerbranch)
layerbranch.delete()
# Get layer dependencies
logger.info('Importing layer dependencies')
rq = urllib.request.Request(layerdeps_url)
data = urllib.request.urlopen(rq).read()
jsdata = json.loads(data.decode('utf-8'))
exclude_fields = ['id', 'layerbranch', 'dependency', 'updated']
existing_deps = []
for layerbranch in layerbranch_idmap.values():
existing_deps += list(LayerDependency.objects.filter(layerbranch=layerbranch).values_list('id', flat=True))
for layerdepjs in jsdata:
layerbranch = layerbranch_idmap.get(layerdepjs['layerbranch'], None)
if not layerbranch:
# We didn't import this layerbranch, skip it
continue
dependency = layer_idmap.get(layerdepjs['dependency'], None)
if not dependency:
# We didn't import the dependency, skip it
continue
layerdep, created = LayerDependency.objects.get_or_create(layerbranch=layerbranch, dependency=dependency)
if not created and layerdep.id in existing_deps:
existing_deps.remove(layerdep.id)
for key, value in layerdepjs.items():
if key in exclude_fields:
continue
setattr(layerdep, key, value)
layerdep.save()
for idv in existing_deps:
LayerDependency.objects.filter(id=idv).delete()
def import_items(desc, url, exclude_fields, objclass, idmap, parentfield):
logger.debug('Importing %s' % desc)
rq = urllib.request.Request(url)
data = urllib.request.urlopen(rq).read()
jsdata = json.loads(data.decode('utf-8'))
existing_ids = []
for parentobj in idmap.values():
existing_ids += list(objclass.objects.values_list('id', flat=True))
for itemjs in jsdata:
parentobj = idmap.get(itemjs[parentfield], None)
if not parentobj:
# We didn't import the parent, skip it
continue
vals = {}
for key, value in itemjs.items():
if key in exclude_fields:
continue
vals[key] = value
vals[parentfield] = parentobj
manager = getattr(parentobj, objclass.__name__.lower() + '_set')
obj, created = manager.get_or_create(**vals)
for key, value in vals.items():
setattr(obj, key, value)
obj.save()
for idv in existing_deps:
objclass.objects.filter(id=idv).delete()
if layermaintainers_url:
import_items('layer maintainers',
layermaintainers_url,
['id', 'layerbranch'],
LayerMaintainer,
layerbranch_idmap,
'layerbranch')
if layernotes_url:
import_items('layer notes',
layernotes_url,
['id', 'layer'],
LayerNote,
layer_idmap,
'layer')
if args.dry_run:
raise DryRunRollbackException()
except DryRunRollbackException:
pass
sys.exit(0)
def main():
valid_layer_name = re.compile('[-\w]+$')
parser = optparse.OptionParser(usage="""
%prog [options] <url>""")
parser.add_option("-n",
"--dry-run",
help="Don't write any data back to the database",
action="store_true",
dest="dryrun")
parser.add_option("-d",
"--debug",
help="Enable debug output",
action="store_const",
const=logging.DEBUG,
dest="loglevel",
default=logging.INFO)
parser.add_option("-q",
"--quiet",
help="Hide all output except error messages",
action="store_const",
const=logging.ERROR,
dest="loglevel")
options, args = parser.parse_args(sys.argv)
if len(args) < 2:
print("Please specify URL of the layer index")
sys.exit(1)
layerindex_url = args[1]
utils.setup_django()
import settings
from layerindex.models import Branch, LayerItem, LayerBranch, LayerDependency, LayerMaintainer, LayerNote
from django.db import transaction
logger.setLevel(options.loglevel)
fetchdir = settings.LAYER_FETCH_DIR
if not fetchdir:
logger.error("Please set LAYER_FETCH_DIR in settings.py")
sys.exit(1)
if not os.path.exists(fetchdir):
os.makedirs(fetchdir)
if not layerindex_url.endswith('/'):
layerindex_url += '/'
if not '/layerindex/api/' in layerindex_url:
layerindex_url += '/layerindex/api/'
rq = urllib.request.Request(layerindex_url)
data = urllib.request.urlopen(rq).read()
jsdata = json.loads(data.decode('utf-8'))
branches_url = jsdata['branches']
layers_url = jsdata['layerItems']
layerdeps_url = jsdata['layerDependencies']
layerbranches_url = jsdata['layerBranches']
layermaintainers_url = jsdata.get('layerMaintainers', None)
layernotes_url = jsdata.get('layerNotes', None)
logger.debug('Getting branches')
# Get branches (we assume the ones we want are already there, so skip any that aren't)
rq = urllib.request.Request(branches_url)
data = urllib.request.urlopen(rq).read()
jsdata = json.loads(data.decode('utf-8'))
branch_idmap = {}
for branchjs in jsdata:
res = Branch.objects.filter(name=branchjs['name'])
if res:
branch = res.first()
branch_idmap[branchjs['id']] = branch
try:
with transaction.atomic():
# Get layers
logger.debug('Importing layers')
rq = urllib.request.Request(layers_url)
data = urllib.request.urlopen(rq).read()
jsdata = json.loads(data.decode('utf-8'))
layer_idmap = {}
exclude_fields = ['id', 'updated']
for layerjs in jsdata:
res = LayerItem.objects.filter(name=layerjs['name'])
if res:
# Already have this layer
logger.debug('Skipping layer %s, already in database' %
layerjs['name'])
layer_idmap[layerjs['id']] = res[0]
continue
logger.debug('Adding layer %s' % layerjs['name'])
layeritem = LayerItem()
for key, value in layerjs.items():
if key in exclude_fields:
continue
setattr(layeritem, key, value)
layeritem.save()
layer_idmap[layerjs['id']] = layeritem
# Get layer branches
logger.debug('Importing layer branches')
rq = urllib.request.Request(layerbranches_url)
data = urllib.request.urlopen(rq).read()
jsdata = json.loads(data.decode('utf-8'))
layerbranch_idmap = {}
exclude_fields = [
'id', 'layer', 'branch', 'vcs_last_fetch', 'vcs_last_rev',
'vcs_last_commit', 'yp_compatible_version', 'updated'
]
for layerbranchjs in jsdata:
branch = branch_idmap.get(layerbranchjs['branch'], None)
if not branch:
# We don't have this branch, skip it
logger.debug(
'Skipping layerbranch %s, branch not imported' %
layerbranchjs['id'])
continue
layer = layer_idmap.get(layerbranchjs['layer'], None)
if not layer:
# We didn't import this layer, skip it
logger.debug(
'Skipping layerbranch %s, layer not imported' %
layerbranchjs['id'])
continue
res = LayerBranch.objects.filter(layer=layer).filter(
branch=branch)
if res:
# The layerbranch already exists (this will occur for layers
# that already existed, since we need to have those in layer_idmap
# to be able to import layer dependencies)
logger.debug('Skipping layerbranch %s, already exists' %
layerbranchjs['id'])
continue
layerbranch = LayerBranch()
for key, value in layerbranchjs.items():
if key in exclude_fields:
continue
setattr(layerbranch, key, value)
layerbranch.branch = branch
layerbranch.layer = layer
layerbranch.save()
layerbranch_idmap[layerbranchjs['id']] = layerbranch
# Get layer dependencies
logger.debug('Importing layer dependencies')
rq = urllib.request.Request(layerdeps_url)
data = urllib.request.urlopen(rq).read()
jsdata = json.loads(data.decode('utf-8'))
exclude_fields = ['id', 'layerbranch', 'dependency', 'updated']
for layerdepjs in jsdata:
layerbranch = layerbranch_idmap.get(layerdepjs['layerbranch'],
None)
if not layerbranch:
# We didn't import this layerbranch, skip it
continue
dependency = layer_idmap.get(layerdepjs['dependency'], None)
if not dependency:
# We didn't import the dependency, skip it
continue
layerdep = LayerDependency()
for key, value in layerdepjs.items():
if key in exclude_fields:
continue
setattr(layerdep, key, value)
layerdep.layerbranch = layerbranch
layerdep.dependency = dependency
layerdep.save()
if layermaintainers_url:
# Get layer maintainers (only available in latest code)
logger.debug('Importing layer maintainers')
rq = urllib.request.Request(layermaintainers_url)
data = urllib.request.urlopen(rq).read()
jsdata = json.loads(data.decode('utf-8'))
exclude_fields = ['id', 'layerbranch']
for layermaintainerjs in jsdata:
layerbranch = layerbranch_idmap.get(
layermaintainerjs['layerbranch'], None)
if not layerbranch:
# We didn't import this layerbranch, skip it
continue
layermaintainer = LayerMaintainer()
for key, value in layermaintainerjs.items():
if key in exclude_fields:
continue
setattr(layermaintainer, key, value)
layermaintainer.layerbranch = layerbranch
layermaintainer.save()
if layernotes_url:
# Get layer notes (only available in latest code)
logger.debug('Importing layer notes')
rq = urllib.request.Request(layernotes_url)
data = urllib.request.urlopen(rq).read()
jsdata = json.loads(data.decode('utf-8'))
exclude_fields = ['id', 'layer']
for layernotejs in jsdata:
layer = layer_idmap.get(layernotejs['layer'], None)
if not layer:
# We didn't import this layer, skip it
continue
res = LayerNote.objects.filter(layer=layer).filter(
text=layernotejs['text'])
if res:
# The note already exists (this will occur for layers
# that already existed, since we need to have those in layer_idmap
# to be able to import layer dependencies)
logger.debug('Skipping note %s, already exists' %
layernotejs['id'])
continue
layernote = LayerNote()
for key, value in layernotejs.items():
if key in exclude_fields:
continue
setattr(layernote, key, value)
layernote.layer = layer
layernote.save()
if options.dryrun:
raise DryRunRollbackException()
except DryRunRollbackException:
pass
sys.exit(0)
def main():
valid_layer_name = re.compile('[-\w]+$')
parser = optparse.OptionParser(usage="""
%prog [options] <url> [name]""")
utils.setup_django()
layer_type_help, layer_type_choices = get_layer_type_choices()
parser.add_option("-s",
"--subdir",
help="Specify subdirectory",
action="store",
dest="subdir")
parser.add_option("-t",
"--type",
help=layer_type_help,
choices=layer_type_choices,
action="store",
dest="layer_type",
default='')
parser.add_option("-n",
"--dry-run",
help="Don't write any data back to the database",
action="store_true",
dest="dryrun")
parser.add_option("-d",
"--debug",
help="Enable debug output",
action="store_const",
const=logging.DEBUG,
dest="loglevel",
default=logging.INFO)
parser.add_option("",
"--github-auth",
help="Specify github username:password",
action="store",
dest="github_auth")
parser.add_option("-q",
"--quiet",
help="Hide all output except error messages",
action="store_const",
const=logging.ERROR,
dest="loglevel")
parser.add_option("-a",
"--actual-branch",
help="Set actual branch",
action="store",
dest="actual_branch")
options, args = parser.parse_args(sys.argv)
if len(args) < 2:
print("Please specify URL of repository for layer")
sys.exit(1)
layer_url = args[1]
if len(args) > 2:
layer_name = args[2]
else:
if options.subdir:
layer_name = options.subdir
else:
layer_name = [x for x in layer_url.split('/') if x][-1]
if layer_name.endswith('.git'):
layer_name = layer_name[:-4]
if not valid_layer_name.match(layer_name):
logger.error(
'Invalid layer name "%s" - Layer name can only include letters, numbers and dashes.',
layer_name)
sys.exit(1)
if options.github_auth:
if not ':' in options.github_auth:
logger.error(
'--github-auth value must be specified as username:password')
sys.exit(1)
splitval = options.github_auth.split(':')
github_login = splitval[0]
github_password = splitval[1]
else:
github_login = None
github_password = None
import settings
from layerindex.models import LayerItem, LayerBranch, LayerDependency, LayerMaintainer
from django.db import transaction
logger.setLevel(options.loglevel)
fetchdir = settings.LAYER_FETCH_DIR
if not fetchdir:
logger.error("Please set LAYER_FETCH_DIR in settings.py")
sys.exit(1)
if not os.path.exists(fetchdir):
os.makedirs(fetchdir)
master_branch = utils.get_branch('master')
core_layer = None
try:
with transaction.atomic():
# Fetch layer
logger.info('Fetching repository %s' % layer_url)
layer = LayerItem()
layer.name = layer_name
layer.status = 'P'
layer.summary = 'tempvalue'
layer.description = layer.summary
set_vcs_fields(layer, layer_url)
urldir = layer.get_fetch_dir()
repodir = os.path.join(fetchdir, urldir)
out = None
try:
if not os.path.exists(repodir):
out = utils.runcmd("git clone %s %s" %
(layer.vcs_url, urldir),
fetchdir,
logger=logger)
else:
out = utils.runcmd("git fetch", repodir, logger=logger)
except Exception as e:
logger.error("Fetch failed: %s" % str(e))
sys.exit(1)
actual_branch = 'master'
if (options.actual_branch):
actual_branch = options.actual_branch
try:
out = utils.runcmd("git checkout origin/%s" % actual_branch,
repodir,
logger=logger)
except subprocess.CalledProcessError:
actual_branch = None
branches = utils.runcmd("git branch -r",
repodir,
logger=logger)
for line in branches.splitlines():
if 'origin/HEAD ->' in line:
actual_branch = line.split('-> origin/')[-1]
break
if not actual_branch:
logger.error(
"Repository has no master branch nor origin/HEAD")
sys.exit(1)
out = utils.runcmd("git checkout origin/%s" % actual_branch,
repodir,
logger=logger)
layer_paths = []
if options.subdir:
layerdir = os.path.join(repodir, options.subdir)
if not os.path.exists(layerdir):
logger.error(
"Subdirectory %s does not exist in repository for master branch"
% options.subdir)
sys.exit(1)
if not os.path.exists(os.path.join(layerdir,
'conf/layer.conf')):
logger.error(
"conf/layer.conf not found in subdirectory %s" %
options.subdir)
sys.exit(1)
layer_paths.append(layerdir)
else:
if os.path.exists(os.path.join(repodir, 'conf/layer.conf')):
layer_paths.append(repodir)
# Find subdirs with a conf/layer.conf
for subdir in os.listdir(repodir):
subdir_path = os.path.join(repodir, subdir)
if os.path.isdir(subdir_path):
if os.path.exists(
os.path.join(subdir_path, 'conf/layer.conf')):
layer_paths.append(subdir_path)
if not layer_paths:
logger.error(
"conf/layer.conf not found in repository or first level subdirectories - is subdirectory set correctly?"
)
sys.exit(1)
if 'github.com' in layer.vcs_url:
json_data, owner_json_data = get_github_layerinfo(
layer.vcs_url, github_login, github_password)
for layerdir in layer_paths:
layer.pk = None
if layerdir != repodir:
subdir = os.path.relpath(layerdir, repodir)
if len(layer_paths) > 1:
layer.name = subdir
else:
subdir = ''
if LayerItem.objects.filter(name=layer.name).exists():
if LayerItem.objects.filter(name=layer.name).exclude(
vcs_url=layer.vcs_url).exists():
conflict_list = LayerItem.objects.filter(
name=layer.name).exclude(vcs_url=layer.vcs_url)
conflict_list_urls = []
for conflict in conflict_list:
conflict_list_urls.append(conflict.vcs_url)
cln = ', '.join(conflict_list_urls)
logger.error(
'A layer named "%s" already exists in the database. Possible name collision with %s.vcs_url = %s'
% (layer.name, layer.name, cln))
sys.exit(1)
else:
logger.info(
'The layer named "%s" already exists in the database. Skipping this layer with same vcs_url'
% layer.name)
layer_paths = [x for x in layer_paths if x != layerdir]
continue
logger.info('Creating layer %s' % layer.name)
# Guess layer type if not specified
if options.layer_type:
layer.layer_type = options.layer_type
elif layer.name in ['openembedded-core', 'meta-oe']:
layer.layer_type = 'A'
elif glob.glob(os.path.join(layerdir, 'conf/distro/*.conf')):
layer.layer_type = 'D'
elif glob.glob(os.path.join(layerdir, 'conf/machine/*.conf')):
layer.layer_type = 'B'
else:
layer.layer_type = 'M'
layer.save()
layerbranch = LayerBranch()
layerbranch.layer = layer
layerbranch.branch = master_branch
if layerdir != repodir:
layerbranch.vcs_subdir = subdir
if actual_branch:
layerbranch.actual_branch = actual_branch
layerbranch.save()
if layer.name != settings.CORE_LAYER_NAME:
if not core_layer:
core_layer = utils.get_layer(settings.CORE_LAYER_NAME)
if core_layer:
logger.debug('Adding dep %s to %s' %
(core_layer.name, layer.name))
layerdep = LayerDependency()
layerdep.layerbranch = layerbranch
layerdep.dependency = core_layer
layerdep.save()
layerconfparser = LayerConfParse(logger=logger)
try:
config_data = layerconfparser.parse_layer(layerdir)
if config_data:
utils.add_dependencies(layerbranch,
config_data,
logger=logger)
utils.add_recommends(layerbranch,
config_data,
logger=logger)
finally:
layerconfparser.shutdown()
# Get some extra meta-information
readme_files = glob.glob(os.path.join(layerdir, 'README*'))
if (not readme_files) and subdir:
readme_files = glob.glob(os.path.join(repodir, 'README*'))
maintainer_files = glob.glob(
os.path.join(layerdir, 'MAINTAINERS'))
if (not maintainer_files) and subdir:
maintainer_files = glob.glob(
os.path.join(repodir, 'MAINTAINERS'))
maintainers = []
if readme_files:
(desc, maintainers, deps) = readme_extract(readme_files[0])
if desc:
layer.summary = layer.name
layer.description = desc
if maintainer_files:
maintainers.extend(maintainers_extract(readme_files[0]))
if (not maintainers) and 'github.com' in layer.vcs_url:
if json_data:
layer.summary = json_data['description']
layer.description = layer.summary
if owner_json_data:
owner_name = owner_json_data.get('name', None)
owner_email = owner_json_data.get('email', None)
if owner_name and owner_email:
maintainers.append('%s <%s>' %
(owner_name, owner_email))
if layer.name == 'openembedded-core':
layer.summary = 'Core metadata'
elif layer.name == 'meta-oe':
layer.summary = 'Additional shared OE metadata'
layer.description = layer.summary
if maintainers:
maint_re = re.compile(
r'^"?([^"@$<>]+)"? *<([^<> ]+)>[ -]*(.+)?$')
for maintentry in maintainers:
res = maint_re.match(maintentry)
if res:
maintainer = LayerMaintainer()
maintainer.layerbranch = layerbranch
maintainer.name = res.group(1).strip()
maintainer.email = res.group(2)
if res.group(3):
maintainer.responsibility = res.group(
3).strip()
maintainer.save()
layer.save()
if not layer_paths:
logger.error('No layers added.')
sys.exit(1)
if options.dryrun:
raise DryRunRollbackException()
except DryRunRollbackException:
pass
sys.exit(0)
def main():
parser = optparse.OptionParser(
usage = """
%prog [options]""")
options, args = parser.parse_args(sys.argv)
utils.setup_django()
from layerindex.models import LayerItem, LayerBranch, LayerDependency
from django.db import transaction
import httplib
conn = httplib.HTTPConnection("www.openembedded.org")
conn.request("GET", "/wiki/LayerIndex?action=raw")
resp = conn.getresponse()
if resp.status in [200, 302]:
data = resp.read()
in_table = False
layer_type = 'M'
nowiki_re = re.compile(r'</?nowiki>')
link_re = re.compile(r'\[(http.*) +link\]')
readme_re = re.compile(r';f=[a-zA-Z0-9/-]*README;')
master_branch = utils.get_branch('master')
core_layer = None
with transaction.atomic():
for line in data.splitlines():
if line.startswith('{|'):
in_table = True
continue
if in_table:
if line.startswith('|}'):
# We're done
break
elif line.startswith('!'):
section = line.split('|', 1)[1].strip("'")
if section.startswith('Base'):
layer_type = 'A'
elif section.startswith('Board'):
layer_type = 'B'
elif section.startswith('Software'):
layer_type = 'S'
elif section.startswith('Distribution'):
layer_type = 'D'
else:
layer_type = 'M'
elif not line.startswith('|-'):
if line.startswith("|| ''"):
continue
fields = line.split('||')
layer = LayerItem()
layer.name = fields[1].strip()
if ' ' in layer.name:
logger.warn('Skipping layer %s - name invalid' % layer.name)
continue
logger.info('Adding layer %s' % layer.name)
layer.status = 'P'
layer.layer_type = layer_type
layer.summary = fields[2].strip()
layer.description = layer.summary
if len(fields) > 6:
res = link_re.match(fields[6].strip())
if res:
link = res.groups(1)[0].strip()
if link.endswith('/README') or readme_re.search(link):
link = 'README'
layer.usage_url = link
repoval = nowiki_re.sub('', fields[4]).strip()
layer.vcs_url = repoval
if repoval.startswith('git://git.openembedded.org/'):
reponame = re.sub('^.*/', '', repoval)
layer.vcs_web_url = 'http://cgit.openembedded.org/' + reponame
layer.vcs_web_tree_base_url = 'http://cgit.openembedded.org/' + reponame + '/tree/%path%?h=%branch%'
layer.vcs_web_file_base_url = 'http://cgit.openembedded.org/' + reponame + '/tree/%path%?h=%branch%'
layer.vcs_web_commit_url = 'http://cgit.openembedded.org/' + reponame + '/commit/?id=%hash%'
elif repoval.startswith('git://git.yoctoproject.org/'):
reponame = re.sub('^.*/', '', repoval)
layer.vcs_web_url = 'http://git.yoctoproject.org/cgit/cgit.cgi/' + reponame
layer.vcs_web_tree_base_url = 'http://git.yoctoproject.org/cgit/cgit.cgi/' + reponame + '/tree/%path%?h=%branch%'
layer.vcs_web_file_base_url = 'http://git.yoctoproject.org/cgit/cgit.cgi/' + reponame + '/tree/%path%?h=%branch%'
layer.vcs_web_commit_url = 'http://git.yoctoproject.org/cgit/cgit.cgi/' + reponame + '/commit/?id=%hash%'
elif repoval.startswith('git://github.com/') or repoval.startswith('http://github.com/') or repoval.startswith('https://github.com/'):
reponame = re.sub('^.*github.com/', '', repoval)
reponame = re.sub('.git$', '', reponame)
layer.vcs_web_url = 'http://github.com/' + reponame
layer.vcs_web_tree_base_url = 'http://github.com/' + reponame + '/tree/%branch%/'
layer.vcs_web_file_base_url = 'http://github.com/' + reponame + '/blob/%branch%/'
layer.vcs_web_commit_url = 'http://github.com/' + reponame + '/commit/%hash%'
elif repoval.startswith('git://gitlab.com/') or repoval.startswith('http://gitlab.com/') or repoval.startswith('https://gitlab.com/'):
reponame = re.sub('^.*gitlab.com/', '', repoval)
reponame = re.sub('.git$', '', reponame)
layer.vcs_web_url = 'http://gitlab.com/' + reponame
layer.vcs_web_tree_base_url = 'http://gitlab.com/' + reponame + '/tree/%branch%/'
layer.vcs_web_file_base_url = 'http://gitlab.com/' + reponame + '/blob/%branch%/'
layer.vcs_web_commit_url = 'http://gitlab.com/' + reponame + '/commit/%hash%'
elif repoval.startswith('git://bitbucket.org/') or repoval.startswith('http://bitbucket.org/') or repoval.startswith('https://bitbucket.org/'):
reponame = re.sub('^.*bitbucket.org/', '', repoval)
reponame = re.sub('.git$', '', reponame)
layer.vcs_web_url = 'http://bitbucket.org/' + reponame
layer.vcs_web_tree_base_url = 'http://bitbucket.org/' + reponame + '/src/%branch%/%path%?at=%branch%'
layer.vcs_web_file_base_url = 'http://bitbucket.org/' + reponame + '/src/%branch%/%path%?at=%branch%'
layer.vcs_web_commit_url = 'http://bitbucket.org/' + reponame + '/commits/%hash%'
elif '.git' in repoval:
res = link_re.match(fields[5].strip())
layer.vcs_web_url = res.groups(1)[0]
layer.vcs_web_tree_base_url = re.sub(r'\.git.*', '.git;a=tree;f=%path%;hb=%branch%', layer.vcs_web_url)
layer.vcs_web_file_base_url = re.sub(r'\.git.*', '.git;a=blob;f=%path%;hb=%branch%', layer.vcs_web_url)
layer.vcs_web_file_base_url = re.sub(r'\.git.*', '.git;a=commit;h=%hash%', layer.vcs_web_url)
layer.save()
layerbranch = LayerBranch()
layerbranch.layer = layer
layerbranch.branch = master_branch
layerbranch.vcs_subdir = fields[3].strip()
layerbranch.save()
if layer.name != 'openembedded-core':
if not core_layer:
core_layer = utils.get_layer('openembedded-core')
if core_layer:
layerdep = LayerDependency()
layerdep.layerbranch = layerbranch
layerdep.dependency = core_layer
layerdep.save()
else:
logger.error('Fetch failed: %d: %s' % (resp.status, resp.reason))
sys.exit(0)
def main():
parser = optparse.OptionParser(usage="""
%prog [options] <bitbakepath> <oeclassicpath>""")
parser.add_option("-b",
"--branch",
help="Specify branch to import into",
action="store",
dest="branch",
default='oe-classic')
parser.add_option("-l",
"--layer",
help="Specify layer to import into",
action="store",
dest="layer",
default='oe-classic')
parser.add_option("-n",
"--dry-run",
help="Don't write any data back to the database",
action="store_true",
dest="dryrun")
parser.add_option("-d",
"--debug",
help="Enable debug output",
action="store_const",
const=logging.DEBUG,
dest="loglevel",
default=logging.INFO)
parser.add_option("-q",
"--quiet",
help="Hide all output except error messages",
action="store_const",
const=logging.ERROR,
dest="loglevel")
options, args = parser.parse_args(sys.argv)
if len(args) < 3:
logger.error('You must specify bitbakepath and oeclassicpath')
parser.print_help()
sys.exit(1)
if len(args) > 3:
logger.error('unexpected argument "%s"' % args[3])
parser.print_help()
sys.exit(1)
utils.setup_django()
import settings
from layerindex.models import LayerItem, LayerBranch, Recipe, ClassicRecipe, Machine, BBAppend, BBClass
from django.db import transaction
logger.setLevel(options.loglevel)
branch = utils.get_branch(options.branch)
if not branch:
logger.error("Specified branch %s is not valid" % options.branch)
sys.exit(1)
res = list(LayerItem.objects.filter(name=options.layer)[:1])
if res:
layer = res[0]
else:
layer = LayerItem()
layer.name = options.layer
layer.status = 'P'
layer.layer_type = 'M'
layer.summary = 'OE-Classic'
layer.description = 'OpenEmbedded-Classic'
layer.vcs_url = 'git://git.openembedded.org/openembedded'
layer.vcs_web_url = 'http://cgit.openembedded.org/openembedded'
layer.vcs_web_tree_base_url = 'http://cgit.openembedded.org/openembedded/tree/%path%'
layer.vcs_web_file_base_url = 'http://cgit.openembedded.org/openembedded/tree/%path%'
layer.comparison = True
layer.save()
layerbranch = layer.get_layerbranch(options.branch)
if not layerbranch:
# LayerBranch doesn't exist for this branch, create it
layerbranch = LayerBranch()
layerbranch.layer = layer
layerbranch.branch = branch
layerbranch.save()
fetchdir = settings.LAYER_FETCH_DIR
if not fetchdir:
logger.error("Please set LAYER_FETCH_DIR in settings.py")
sys.exit(1)
if not os.path.exists(fetchdir):
os.makedirs(fetchdir)
fetchedrepos = []
failedrepos = []
bitbakepath = args[1]
oeclassicpath = args[2]
confparentdir = os.path.abspath(
os.path.join(os.path.dirname(os.path.abspath(__file__)),
'../../oe-classic'))
os.environ['BBPATH'] = str("%s:%s" % (confparentdir, oeclassicpath))
try:
(tinfoil, tempdir) = recipeparse.init_parser(settings,
branch,
bitbakepath,
nocheckout=True,
classic=True,
logger=logger)
except recipeparse.RecipeParseError as e:
logger.error(str(e))
sys.exit(1)
# Clear the default value of SUMMARY so that we can use DESCRIPTION instead if it hasn't been set
tinfoil.config_data.setVar('SUMMARY', '')
# Clear the default value of DESCRIPTION so that we can see where it's not set
tinfoil.config_data.setVar('DESCRIPTION', '')
# Clear the default value of HOMEPAGE ('unknown')
tinfoil.config_data.setVar('HOMEPAGE', '')
try:
with transaction.atomic():
layerdir_start = os.path.normpath(oeclassicpath) + os.sep
layerrecipes = Recipe.objects.filter(layerbranch=layerbranch)
layermachines = Machine.objects.filter(layerbranch=layerbranch)
layerdistros = Distro.objects.filter(layerbranch=layerbranch)
layerappends = BBAppend.objects.filter(layerbranch=layerbranch)
layerclasses = BBClass.objects.filter(layerbranch=layerbranch)
try:
config_data_copy = recipeparse.setup_layer(
tinfoil.config_data, fetchdir, oeclassicpath, layer,
layerbranch, logger)
except recipeparse.RecipeParseError as e:
logger.error(str(e))
sys.exit(1)
layerrecipes.delete()
layermachines.delete()
layerdistros.delete()
layerappends.delete()
layerclasses.delete()
for root, dirs, files in os.walk(oeclassicpath):
if '.git' in dirs:
dirs.remove('.git')
for f in files:
fullpath = os.path.join(root, f)
(typename, filepath,
filename) = recipeparse.detect_file_type(
fullpath, layerdir_start)
if typename == 'recipe':
recipe = ClassicRecipe()
recipe.layerbranch = layerbranch
recipe.filename = filename
recipe.filepath = filepath
update_recipe_file(tinfoil, config_data_copy, root,
recipe, layerdir_start,
oeclassicpath)
recipe.save()
layerbranch.vcs_last_fetch = datetime.now()
layerbranch.save()
if options.dryrun:
raise DryRunRollbackException()
except DryRunRollbackException:
pass
except:
import traceback
traceback.print_exc()
finally:
tinfoil.shutdown()
shutil.rmtree(tempdir)
sys.exit(0)
def main():
parser = optparse.OptionParser(
usage = """
%prog [options] <bitbakepath> <oeclassicpath>""")
parser.add_option("-b", "--branch",
help = "Specify branch to import into",
action="store", dest="branch", default='oe-classic')
parser.add_option("-l", "--layer",
help = "Specify layer to import into",
action="store", dest="layer", default='oe-classic')
parser.add_option("-n", "--dry-run",
help = "Don't write any data back to the database",
action="store_true", dest="dryrun")
parser.add_option("-d", "--debug",
help = "Enable debug output",
action="store_const", const=logging.DEBUG, dest="loglevel", default=logging.INFO)
parser.add_option("-q", "--quiet",
help = "Hide all output except error messages",
action="store_const", const=logging.ERROR, dest="loglevel")
options, args = parser.parse_args(sys.argv)
if len(args) < 3:
logger.error('You must specify bitbakepath and oeclassicpath')
parser.print_help()
sys.exit(1)
if len(args) > 3:
logger.error('unexpected argument "%s"' % args[3])
parser.print_help()
sys.exit(1)
utils.setup_django()
import settings
from layerindex.models import LayerItem, LayerBranch, Recipe, ClassicRecipe, Machine, BBAppend, BBClass
from django.db import transaction
logger.setLevel(options.loglevel)
branch = utils.get_branch(options.branch)
if not branch:
logger.error("Specified branch %s is not valid" % options.branch)
sys.exit(1)
res = list(LayerItem.objects.filter(name=options.layer)[:1])
if res:
layer = res[0]
else:
layer = LayerItem()
layer.name = options.layer
layer.status = 'P'
layer.layer_type = 'M'
layer.summary = 'OE-Classic'
layer.description = 'OpenEmbedded-Classic'
layer.vcs_url = 'git://git.openembedded.org/openembedded'
layer.vcs_web_url = 'http://cgit.openembedded.org/cgit.cgi/openembedded'
layer.vcs_web_tree_base_url = 'http://cgit.openembedded.org/cgit.cgi/openembedded/tree/%path%'
layer.vcs_web_file_base_url = 'http://cgit.openembedded.org/cgit.cgi/openembedded/tree/%path%'
layer.classic = True
layer.save()
layerbranch = layer.get_layerbranch(options.branch)
if not layerbranch:
# LayerBranch doesn't exist for this branch, create it
layerbranch = LayerBranch()
layerbranch.layer = layer
layerbranch.branch = branch
layerbranch.save()
fetchdir = settings.LAYER_FETCH_DIR
if not fetchdir:
logger.error("Please set LAYER_FETCH_DIR in settings.py")
sys.exit(1)
if not os.path.exists(fetchdir):
os.makedirs(fetchdir)
fetchedrepos = []
failedrepos = []
bitbakepath = args[1]
oeclassicpath = args[2]
confparentdir = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '../../oe-classic'))
os.environ['BBPATH'] = str("%s:%s" % (confparentdir, oeclassicpath))
try:
(tinfoil, tempdir) = recipeparse.init_parser(settings, branch, bitbakepath, nocheckout=True, classic=True, logger=logger)
except recipeparse.RecipeParseError as e:
logger.error(str(e))
sys.exit(1)
# Clear the default value of SUMMARY so that we can use DESCRIPTION instead if it hasn't been set
tinfoil.config_data.setVar('SUMMARY', '')
# Clear the default value of DESCRIPTION so that we can see where it's not set
tinfoil.config_data.setVar('DESCRIPTION', '')
# Clear the default value of HOMEPAGE ('unknown')
tinfoil.config_data.setVar('HOMEPAGE', '')
transaction.enter_transaction_management()
transaction.managed(True)
try:
layerdir_start = os.path.normpath(oeclassicpath) + os.sep
layerrecipes = Recipe.objects.filter(layerbranch=layerbranch)
layermachines = Machine.objects.filter(layerbranch=layerbranch)
layerappends = BBAppend.objects.filter(layerbranch=layerbranch)
layerclasses = BBClass.objects.filter(layerbranch=layerbranch)
try:
config_data_copy = recipeparse.setup_layer(tinfoil.config_data, fetchdir, oeclassicpath, layer, layerbranch)
except recipeparse.RecipeParseError as e:
logger.error(str(e))
transaction.rollback()
sys.exit(1)
layerrecipes.delete()
layermachines.delete()
layerappends.delete()
layerclasses.delete()
for root, dirs, files in os.walk(oeclassicpath):
if '.git' in dirs:
dirs.remove('.git')
for f in files:
fullpath = os.path.join(root, f)
(typename, filepath, filename) = recipeparse.detect_file_type(fullpath, layerdir_start)
if typename == 'recipe':
recipe = ClassicRecipe()
recipe.layerbranch = layerbranch
recipe.filename = filename
recipe.filepath = filepath
update_recipe_file(config_data_copy, root, recipe, layerdir_start, oeclassicpath)
recipe.save()
layerbranch.vcs_last_fetch = datetime.now()
layerbranch.save()
if options.dryrun:
transaction.rollback()
else:
transaction.commit()
except:
import traceback
traceback.print_exc()
transaction.rollback()
finally:
transaction.leave_transaction_management()
shutil.rmtree(tempdir)
sys.exit(0)
def main():
parser = optparse.OptionParser(
usage = """
%prog [options] <url> [name]""")
parser.add_option("-s", "--subdir",
help = "Specify subdirectory",
action="store", dest="subdir")
parser.add_option("-n", "--dry-run",
help = "Don't write any data back to the database",
action="store_true", dest="dryrun")
parser.add_option("-d", "--debug",
help = "Enable debug output",
action="store_const", const=logging.DEBUG, dest="loglevel", default=logging.INFO)
parser.add_option("", "--github-auth",
help = "Specify github username:password",
action="store", dest="github_auth")
parser.add_option("-q", "--quiet",
help = "Hide all output except error messages",
action="store_const", const=logging.ERROR, dest="loglevel")
options, args = parser.parse_args(sys.argv)
if len(args) < 2:
print("Please specify URL of repository for layer")
sys.exit(1)
layer_url = args[1]
if len(args) > 2:
layer_name = args[2]
else:
if options.subdir:
layer_name = options.subdir
else:
layer_name = filter(None, layer_url.split('/'))[-1]
if layer_name.endswith('.git'):
layer_name = layer_name[:-4]
if options.github_auth:
if not ':' in options.github_auth:
logger.error('--github-auth value must be specified as username:password')
sys.exit(1)
splitval = options.github_auth.split(':')
github_login = splitval[0]
github_password = splitval[1]
else:
github_login = None
github_password = None
utils.setup_django()
import settings
from layerindex.models import LayerItem, LayerBranch, LayerDependency, LayerMaintainer
from django.db import transaction
logger.setLevel(options.loglevel)
fetchdir = settings.LAYER_FETCH_DIR
if not fetchdir:
logger.error("Please set LAYER_FETCH_DIR in settings.py")
sys.exit(1)
if not os.path.exists(fetchdir):
os.makedirs(fetchdir)
master_branch = utils.get_branch('master')
core_layer = None
transaction.enter_transaction_management()
transaction.managed(True)
try:
# Fetch layer
logger.info('Fetching repository %s' % layer_url)
layer = LayerItem()
layer.name = layer_name
layer.status = 'P'
layer.layer_type = 'M'
layer.summary = 'tempvalue'
layer.description = layer.summary
set_vcs_fields(layer, layer_url)
urldir = layer.get_fetch_dir()
repodir = os.path.join(fetchdir, urldir)
out = None
try:
if not os.path.exists(repodir):
out = utils.runcmd("git clone %s %s" % (layer.vcs_url, urldir), fetchdir, logger=logger)
else:
out = utils.runcmd("git fetch", repodir, logger=logger)
except Exception as e:
logger.error("Fetch failed: %s" % str(e))
sys.exit(1)
actual_branch = ''
try:
out = utils.runcmd("git checkout origin/master", repodir, logger=logger)
except subprocess.CalledProcessError:
branches = utils.runcmd("git branch -r", repodir, logger=logger)
for line in branches.splitlines():
if 'origin/HEAD ->' in line:
actual_branch = line.split('-> origin/')[-1]
break
if not actual_branch:
logger.error("Repository has no master branch nor origin/HEAD")
sys.exit(1)
out = utils.runcmd("git checkout origin/%s" % actual_branch, repodir, logger=logger)
layer_paths = []
if options.subdir:
layerdir = os.path.join(repodir, options.subdir)
if not os.path.exists(layerdir):
logger.error("Subdirectory %s does not exist in repository for master branch" % options.subdir)
sys.exit(1)
if not os.path.exists(os.path.join(layerdir, 'conf/layer.conf')):
logger.error("conf/layer.conf not found in subdirectory %s" % options.subdir)
sys.exit(1)
layer_paths.append(layerdir)
else:
if os.path.exists(os.path.join(repodir, 'conf/layer.conf')):
layer_paths.append(repodir)
# Find subdirs with a conf/layer.conf
for subdir in os.listdir(repodir):
subdir_path = os.path.join(repodir, subdir)
if os.path.isdir(subdir_path):
if os.path.exists(os.path.join(subdir_path, 'conf/layer.conf')):
layer_paths.append(subdir_path)
if not layer_paths:
logger.error("conf/layer.conf not found in repository or first level subdirectories - is subdirectory set correctly?")
sys.exit(1)
if 'github.com' in layer.vcs_url:
json_data, owner_json_data = get_github_layerinfo(layer.vcs_url, github_login, github_password)
for layerdir in layer_paths:
layer.pk = None
if layerdir != repodir:
subdir = os.path.relpath(layerdir, repodir)
if len(layer_paths) > 1:
layer.name = subdir
else:
subdir = ''
if LayerItem.objects.filter(name=layer.name).exists():
logger.error('A layer named "%s" already exists in the database' % layer_name)
sys.exit(1)
logger.info('Creating layer %s' % layer.name)
# Guess layer type
if glob.glob(os.path.join(layerdir, 'conf/distro/*.conf')):
layer.layer_type = 'D'
elif glob.glob(os.path.join(layerdir, 'conf/machine/*.conf')):
layer.layer_type = 'B'
layer.save()
layerbranch = LayerBranch()
layerbranch.layer = layer
layerbranch.branch = master_branch
if layerdir != repodir:
layerbranch.vcs_subdir = subdir
if actual_branch:
layerbranch.actual_branch = actual_branch
layerbranch.save()
if layer.name != settings.CORE_LAYER_NAME:
if not core_layer:
core_layer = utils.get_layer(settings.CORE_LAYER_NAME)
if core_layer:
layerdep = LayerDependency()
layerdep.layerbranch = layerbranch
layerdep.dependency = core_layer
layerdep.save()
# Get some extra meta-information
readme_files = glob.glob(os.path.join(layerdir, 'README*'))
if (not readme_files) and subdir:
readme_files = glob.glob(os.path.join(repodir, 'README*'))
maintainer_files = glob.glob(os.path.join(layerdir, 'MAINTAINERS'))
if (not maintainer_files) and subdir:
maintainer_files = glob.glob(os.path.join(repodir, 'MAINTAINERS'))
maintainers = []
if readme_files:
(desc, maintainers, deps) = readme_extract(readme_files[0])
if desc:
layer.summary = layer.name
layer.description = desc
if maintainer_files:
maintainers.extend(maintainers_extract(readme_files[0]))
if (not maintainers) and 'github.com' in layer.vcs_url:
if json_data:
layer.summary = json_data['description']
layer.description = layer.summary
if owner_json_data:
owner_name = owner_json_data.get('name', None)
owner_email = owner_json_data.get('email', None)
if owner_name and owner_email:
maintainers.append('%s <%s>' % (owner_name, owner_email))
if layer.name == 'openembedded-core':
layer.summary = 'Core metadata'
layer.layer_type = 'A'
elif layer.name == 'meta-oe':
layer.summary = 'Additional shared OE metadata'
layer.description = layer.summary
layer.layer_type = 'A'
if maintainers:
maint_re = re.compile(r'^"?([^"@$<>]+)"? *<([^<> ]+)>[ -]*(.+)?$')
for maintentry in maintainers:
res = maint_re.match(maintentry)
if res:
maintainer = LayerMaintainer()
maintainer.layerbranch = layerbranch
maintainer.name = res.group(1).strip()
maintainer.email = res.group(2)
if res.group(3):
maintainer.responsibility = res.group(3).strip()
maintainer.save()
layer.save()
if options.dryrun:
transaction.rollback()
else:
transaction.commit()
except:
transaction.rollback()
raise
finally:
transaction.leave_transaction_management()
sys.exit(0)
def main():
parser = optparse.OptionParser(
usage = """
%prog [options]""")
options, args = parser.parse_args(sys.argv)
utils.setup_django()
from layerindex.models import LayerItem, LayerBranch, LayerDependency
from django.db import transaction
import httplib
conn = httplib.HTTPConnection("www.openembedded.org")
conn.request("GET", "/wiki/LayerIndex?action=raw")
resp = conn.getresponse()
if resp.status in [200, 302]:
data = resp.read()
in_table = False
layer_type = 'M'
nowiki_re = re.compile(r'</?nowiki>')
link_re = re.compile(r'\[(http.*) +link\]')
readme_re = re.compile(r';f=[a-zA-Z0-9/-]*README;')
master_branch = utils.get_branch('master')
core_layer = None
transaction.enter_transaction_management()
transaction.managed(True)
try:
for line in data.splitlines():
if line.startswith('{|'):
in_table = True
continue
if in_table:
if line.startswith('|}'):
# We're done
break
elif line.startswith('!'):
section = line.split('|', 1)[1].strip("'")
if section.startswith('Base'):
layer_type = 'A'
elif section.startswith('Board'):
layer_type = 'B'
elif section.startswith('Software'):
layer_type = 'S'
elif section.startswith('Distribution'):
layer_type = 'D'
else:
layer_type = 'M'
elif not line.startswith('|-'):
if line.startswith("|| ''"):
continue
fields = line.split('||')
layer = LayerItem()
layer.name = fields[1].strip()
if ' ' in layer.name:
logger.warn('Skipping layer %s - name invalid' % layer.name)
continue
logger.info('Adding layer %s' % layer.name)
layer.status = 'P'
layer.layer_type = layer_type
layer.summary = fields[2].strip()
layer.description = layer.summary
if len(fields) > 6:
res = link_re.match(fields[6].strip())
if res:
link = res.groups(1)[0].strip()
if link.endswith('/README') or readme_re.search(link):
link = 'README'
layer.usage_url = link
repoval = nowiki_re.sub('', fields[4]).strip()
layer.vcs_url = repoval
if repoval.startswith('git://git.openembedded.org/'):
reponame = re.sub('^.*/', '', repoval)
layer.vcs_web_url = 'http://cgit.openembedded.org/cgit.cgi/' + reponame
layer.vcs_web_tree_base_url = 'http://cgit.openembedded.org/cgit.cgi/' + reponame + '/tree/%path%?h=%branch%'
layer.vcs_web_file_base_url = 'http://cgit.openembedded.org/cgit.cgi/' + reponame + '/tree/%path%?h=%branch%'
elif 'git.yoctoproject.org/' in repoval:
reponame = re.sub('^.*/', '', repoval)
layer.vcs_web_url = 'http://git.yoctoproject.org/cgit/cgit.cgi/' + reponame
layer.vcs_web_tree_base_url = 'http://git.yoctoproject.org/cgit/cgit.cgi/' + reponame + '/tree/%path%?h=%branch%'
layer.vcs_web_file_base_url = 'http://git.yoctoproject.org/cgit/cgit.cgi/' + reponame + '/tree/%path%?h=%branch%'
elif 'github.com/' in repoval:
reponame = re.sub('^.*github.com/', '', repoval)
reponame = re.sub('.git$', '', reponame)
layer.vcs_web_url = 'http://github.com/' + reponame
layer.vcs_web_tree_base_url = 'http://github.com/' + reponame + '/tree/%branch%/'
layer.vcs_web_file_base_url = 'http://github.com/' + reponame + '/blob/%branch%/'
elif 'gitorious.org/' in repoval:
reponame = re.sub('^.*gitorious.org/', '', repoval)
reponame = re.sub('.git$', '', reponame)
layer.vcs_web_url = 'http://gitorious.org/' + reponame
layer.vcs_web_tree_base_url = 'http://gitorious.org/' + reponame + '/trees/%branch%/'
layer.vcs_web_file_base_url = 'http://gitorious.org/' + reponame + '/blobs/%branch%/'
elif 'bitbucket.org/' in repoval:
reponame = re.sub('^.*bitbucket.org/', '', repoval)
reponame = re.sub('.git$', '', reponame)
layer.vcs_web_url = 'http://bitbucket.org/' + reponame
layer.vcs_web_tree_base_url = 'http://bitbucket.org/' + reponame + '/src/%branch%/%path%?at=%branch%'
layer.vcs_web_file_base_url = 'http://bitbucket.org/' + reponame + '/src/%branch%/%path%?at=%branch%'
elif '.git' in repoval:
res = link_re.match(fields[5].strip())
layer.vcs_web_url = res.groups(1)[0]
layer.vcs_web_tree_base_url = re.sub(r'\.git.*', '.git;a=tree;f=%path%;hb=%branch%', layer.vcs_web_url)
layer.vcs_web_file_base_url = re.sub(r'\.git.*', '.git;a=blob;f=%path%;hb=%branch%', layer.vcs_web_url)
layer.save()
layerbranch = LayerBranch()
layerbranch.layer = layer
layerbranch.branch = master_branch
layerbranch.vcs_subdir = fields[3].strip()
layerbranch.save()
if layer.name != 'openembedded-core':
if not core_layer:
core_layer = utils.get_layer('openembedded-core')
if core_layer:
layerdep = LayerDependency()
layerdep.layerbranch = layerbranch
layerdep.dependency = core_layer
layerdep.save()
transaction.commit()
except:
transaction.rollback()
raise
finally:
transaction.leave_transaction_management()
else:
logger.error('Fetch failed: %d: %s' % (resp.status, resp.reason))
sys.exit(0)
