def _add_dependency(var, name, layerbranch, config_data, logger=None, required=True): from layerindex.models import LayerBranch, LayerDependency layer_name = layerbranch.layer.name var_name = layer_name if layerbranch.collection: var_name = layerbranch.collection dep_list = config_data.getVar("%s_%s" % (var, var_name), True) if not dep_list: return try: dep_dict = bb.utils.explode_dep_versions2(dep_list) except bb.utils.VersionStringException as vse: logger.debug('Error parsing %s_%s for %s\n%s' % (var, var_name, layer_name, str(vse))) return need_remove = LayerDependency.objects.filter( layerbranch=layerbranch).filter(required=required) for dep, ver_list in list(dep_dict.items()): ver_str = None if ver_list: ver_str = ver_list[0] try: dep_layer = get_dependency_layer(dep, ver_str, logger) except bb.utils.VersionStringException as vse: if logger: logger.error('Error getting %s %s for %s\n%s' % (name, dep.layer_name, str(vse))) continue # No layer found. if not dep_layer: if logger: if required: logger.error('Cannot resolve %s %s (version %s) for %s' % (name, dep, ver_str, layer_name)) else: logger.warning('Cannot resolve %s %s (version %s) for %s' % (name, dep, ver_str, layer_name)) continue # Preparing to remove obsolete ones if need_remove: need_remove = need_remove.exclude(dependency=dep_layer) # Skip existing entries. existing = list( LayerDependency.objects.filter(layerbranch=layerbranch).filter( required=required).filter(dependency=dep_layer)) if existing: logger.debug('Skipping %s - already a dependency for %s' % (dep, layer_name)) continue if logger: logger.debug('Adding %s %s to %s' % (name, dep_layer.name, layer_name)) layerdep = LayerDependency() layerdep.layerbranch = layerbranch layerdep.dependency = dep_layer layerdep.required = required layerdep.save() if need_remove: import settings remove_layer_dependencies = getattr(settings, 'REMOVE_LAYER_DEPENDENCIES', False) if remove_layer_dependencies: logger.info('Removing obsolete dependencies "%s" for layer %s' % (need_remove, layer_name)) need_remove.delete() else: logger.warn('Dependencies "%s" are not in %s\'s conf/layer.conf' % (need_remove, layer_name)) logger.warn( 'Either set REMOVE_LAYER_DEPENDENCIES to remove them from the database, or fix conf/layer.conf' )
def main(): valid_layer_name = re.compile('[-\w]+$') parser = optparse.OptionParser(usage=""" %prog [options] <url> [name]""") utils.setup_django() layer_type_help, layer_type_choices = get_layer_type_choices() parser.add_option("-s", "--subdir", help="Specify subdirectory", action="store", dest="subdir") parser.add_option("-t", "--type", help=layer_type_help, choices=layer_type_choices, action="store", dest="layer_type", default='') parser.add_option("-n", "--dry-run", help="Don't write any data back to the database", action="store_true", dest="dryrun") parser.add_option("-d", "--debug", help="Enable debug output", action="store_const", const=logging.DEBUG, dest="loglevel", default=logging.INFO) parser.add_option("", "--github-auth", help="Specify github username:password", action="store", dest="github_auth") parser.add_option("-q", "--quiet", help="Hide all output except error messages", action="store_const", const=logging.ERROR, dest="loglevel") parser.add_option("-a", "--actual-branch", help="Set actual branch", action="store", dest="actual_branch") options, args = parser.parse_args(sys.argv) if len(args) < 2: print("Please specify URL of repository for layer") sys.exit(1) layer_url = args[1] if len(args) > 2: layer_name = args[2] else: if options.subdir: layer_name = options.subdir else: layer_name = [x for x in layer_url.split('/') if x][-1] if layer_name.endswith('.git'): layer_name = layer_name[:-4] if not valid_layer_name.match(layer_name): logger.error( 'Invalid layer name "%s" - Layer name can only include letters, numbers and dashes.', layer_name) sys.exit(1) if options.github_auth: if not ':' in options.github_auth: logger.error( '--github-auth value must be specified as username:password') sys.exit(1) splitval = options.github_auth.split(':') github_login = splitval[0] github_password = splitval[1] else: github_login = None github_password = None import settings from layerindex.models import LayerItem, LayerBranch, LayerDependency, LayerMaintainer from django.db import transaction logger.setLevel(options.loglevel) fetchdir = settings.LAYER_FETCH_DIR if not fetchdir: logger.error("Please set LAYER_FETCH_DIR in settings.py") sys.exit(1) if not os.path.exists(fetchdir): os.makedirs(fetchdir) master_branch = utils.get_branch('master') core_layer = None try: with transaction.atomic(): # Fetch layer logger.info('Fetching repository %s' % layer_url) layer = LayerItem() layer.name = layer_name layer.status = 'P' layer.summary = 'tempvalue' layer.description = layer.summary set_vcs_fields(layer, layer_url) urldir = layer.get_fetch_dir() repodir = os.path.join(fetchdir, urldir) out = None try: if not os.path.exists(repodir): out = utils.runcmd("git clone %s %s" % (layer.vcs_url, urldir), fetchdir, logger=logger) else: out = utils.runcmd("git fetch", repodir, logger=logger) except Exception as e: logger.error("Fetch failed: %s" % str(e)) sys.exit(1) actual_branch = 'master' if (options.actual_branch): actual_branch = options.actual_branch try: out = utils.runcmd("git checkout origin/%s" % actual_branch, repodir, logger=logger) except subprocess.CalledProcessError: actual_branch = None branches = utils.runcmd("git branch -r", repodir, logger=logger) for line in branches.splitlines(): if 'origin/HEAD ->' in line: actual_branch = line.split('-> origin/')[-1] break if not actual_branch: logger.error( "Repository has no master branch nor origin/HEAD") sys.exit(1) out = utils.runcmd("git checkout origin/%s" % actual_branch, repodir, logger=logger) layer_paths = [] if options.subdir: layerdir = os.path.join(repodir, options.subdir) if not os.path.exists(layerdir): logger.error( "Subdirectory %s does not exist in repository for master branch" % options.subdir) sys.exit(1) if not os.path.exists(os.path.join(layerdir, 'conf/layer.conf')): logger.error( "conf/layer.conf not found in subdirectory %s" % options.subdir) sys.exit(1) layer_paths.append(layerdir) else: if os.path.exists(os.path.join(repodir, 'conf/layer.conf')): layer_paths.append(repodir) # Find subdirs with a conf/layer.conf for subdir in os.listdir(repodir): subdir_path = os.path.join(repodir, subdir) if os.path.isdir(subdir_path): if os.path.exists( os.path.join(subdir_path, 'conf/layer.conf')): layer_paths.append(subdir_path) if not layer_paths: logger.error( "conf/layer.conf not found in repository or first level subdirectories - is subdirectory set correctly?" ) sys.exit(1) if 'github.com' in layer.vcs_url: json_data, owner_json_data = get_github_layerinfo( layer.vcs_url, github_login, github_password) for layerdir in layer_paths: layer.pk = None if layerdir != repodir: subdir = os.path.relpath(layerdir, repodir) if len(layer_paths) > 1: layer.name = subdir else: subdir = '' if LayerItem.objects.filter(name=layer.name).exists(): if LayerItem.objects.filter(name=layer.name).exclude( vcs_url=layer.vcs_url).exists(): conflict_list = LayerItem.objects.filter( name=layer.name).exclude(vcs_url=layer.vcs_url) conflict_list_urls = [] for conflict in conflict_list: conflict_list_urls.append(conflict.vcs_url) cln = ', '.join(conflict_list_urls) logger.error( 'A layer named "%s" already exists in the database. Possible name collision with %s.vcs_url = %s' % (layer.name, layer.name, cln)) sys.exit(1) else: logger.info( 'The layer named "%s" already exists in the database. Skipping this layer with same vcs_url' % layer.name) layer_paths = [x for x in layer_paths if x != layerdir] continue logger.info('Creating layer %s' % layer.name) # Guess layer type if not specified if options.layer_type: layer.layer_type = options.layer_type elif layer.name in ['openembedded-core', 'meta-oe']: layer.layer_type = 'A' elif glob.glob(os.path.join(layerdir, 'conf/distro/*.conf')): layer.layer_type = 'D' elif glob.glob(os.path.join(layerdir, 'conf/machine/*.conf')): layer.layer_type = 'B' else: layer.layer_type = 'M' layer.save() layerbranch = LayerBranch() layerbranch.layer = layer layerbranch.branch = master_branch if layerdir != repodir: layerbranch.vcs_subdir = subdir if actual_branch: layerbranch.actual_branch = actual_branch layerbranch.save() if layer.name != settings.CORE_LAYER_NAME: if not core_layer: core_layer = utils.get_layer(settings.CORE_LAYER_NAME) if core_layer: logger.debug('Adding dep %s to %s' % (core_layer.name, layer.name)) layerdep = LayerDependency() layerdep.layerbranch = layerbranch layerdep.dependency = core_layer layerdep.save() layerconfparser = LayerConfParse(logger=logger) try: config_data = layerconfparser.parse_layer(layerdir) if config_data: utils.add_dependencies(layerbranch, config_data, logger=logger) utils.add_recommends(layerbranch, config_data, logger=logger) finally: layerconfparser.shutdown() # Get some extra meta-information readme_files = glob.glob(os.path.join(layerdir, 'README*')) if (not readme_files) and subdir: readme_files = glob.glob(os.path.join(repodir, 'README*')) maintainer_files = glob.glob( os.path.join(layerdir, 'MAINTAINERS')) if (not maintainer_files) and subdir: maintainer_files = glob.glob( os.path.join(repodir, 'MAINTAINERS')) maintainers = [] if readme_files: (desc, maintainers, deps) = readme_extract(readme_files[0]) if desc: layer.summary = layer.name layer.description = desc if maintainer_files: maintainers.extend(maintainers_extract(readme_files[0])) if (not maintainers) and 'github.com' in layer.vcs_url: if json_data: layer.summary = json_data['description'] layer.description = layer.summary if owner_json_data: owner_name = owner_json_data.get('name', None) owner_email = owner_json_data.get('email', None) if owner_name and owner_email: maintainers.append('%s <%s>' % (owner_name, owner_email)) if layer.name == 'openembedded-core': layer.summary = 'Core metadata' elif layer.name == 'meta-oe': layer.summary = 'Additional shared OE metadata' layer.description = layer.summary if maintainers: maint_re = re.compile( r'^"?([^"@$<>]+)"? *<([^<> ]+)>[ -]*(.+)?$') for maintentry in maintainers: res = maint_re.match(maintentry) if res: maintainer = LayerMaintainer() maintainer.layerbranch = layerbranch maintainer.name = res.group(1).strip() maintainer.email = res.group(2) if res.group(3): maintainer.responsibility = res.group( 3).strip() maintainer.save() layer.save() if not layer_paths: logger.error('No layers added.') sys.exit(1) if options.dryrun: raise DryRunRollbackException() except DryRunRollbackException: pass sys.exit(0)
def main(): valid_layer_name = re.compile('[-\w]+$') parser = optparse.OptionParser(usage=""" %prog [options] <url>""") parser.add_option("-n", "--dry-run", help="Don't write any data back to the database", action="store_true", dest="dryrun") parser.add_option("-d", "--debug", help="Enable debug output", action="store_const", const=logging.DEBUG, dest="loglevel", default=logging.INFO) parser.add_option("-q", "--quiet", help="Hide all output except error messages", action="store_const", const=logging.ERROR, dest="loglevel") options, args = parser.parse_args(sys.argv) if len(args) < 2: print("Please specify URL of the layer index") sys.exit(1) layerindex_url = args[1] utils.setup_django() import settings from layerindex.models import Branch, LayerItem, LayerBranch, LayerDependency, LayerMaintainer, LayerNote from django.db import transaction logger.setLevel(options.loglevel) fetchdir = settings.LAYER_FETCH_DIR if not fetchdir: logger.error("Please set LAYER_FETCH_DIR in settings.py") sys.exit(1) if not os.path.exists(fetchdir): os.makedirs(fetchdir) if not layerindex_url.endswith('/'): layerindex_url += '/' if not '/layerindex/api/' in layerindex_url: layerindex_url += '/layerindex/api/' rq = urllib.request.Request(layerindex_url) data = urllib.request.urlopen(rq).read() jsdata = json.loads(data.decode('utf-8')) branches_url = jsdata['branches'] layers_url = jsdata['layerItems'] layerdeps_url = jsdata['layerDependencies'] layerbranches_url = jsdata['layerBranches'] layermaintainers_url = jsdata.get('layerMaintainers', None) layernotes_url = jsdata.get('layerNotes', None) logger.debug('Getting branches') # Get branches (we assume the ones we want are already there, so skip any that aren't) rq = urllib.request.Request(branches_url) data = urllib.request.urlopen(rq).read() jsdata = json.loads(data.decode('utf-8')) branch_idmap = {} for branchjs in jsdata: res = Branch.objects.filter(name=branchjs['name']) if res: branch = res.first() branch_idmap[branchjs['id']] = branch try: with transaction.atomic(): # Get layers logger.debug('Importing layers') rq = urllib.request.Request(layers_url) data = urllib.request.urlopen(rq).read() jsdata = json.loads(data.decode('utf-8')) layer_idmap = {} exclude_fields = ['id', 'updated'] for layerjs in jsdata: res = LayerItem.objects.filter(name=layerjs['name']) if res: # Already have this layer logger.debug('Skipping layer %s, already in database' % layerjs['name']) layer_idmap[layerjs['id']] = res[0] continue logger.debug('Adding layer %s' % layerjs['name']) layeritem = LayerItem() for key, value in layerjs.items(): if key in exclude_fields: continue setattr(layeritem, key, value) layeritem.save() layer_idmap[layerjs['id']] = layeritem # Get layer branches logger.debug('Importing layer branches') rq = urllib.request.Request(layerbranches_url) data = urllib.request.urlopen(rq).read() jsdata = json.loads(data.decode('utf-8')) layerbranch_idmap = {} exclude_fields = [ 'id', 'layer', 'branch', 'vcs_last_fetch', 'vcs_last_rev', 'vcs_last_commit', 'yp_compatible_version', 'updated' ] for layerbranchjs in jsdata: branch = branch_idmap.get(layerbranchjs['branch'], None) if not branch: # We don't have this branch, skip it logger.debug( 'Skipping layerbranch %s, branch not imported' % layerbranchjs['id']) continue layer = layer_idmap.get(layerbranchjs['layer'], None) if not layer: # We didn't import this layer, skip it logger.debug( 'Skipping layerbranch %s, layer not imported' % layerbranchjs['id']) continue res = LayerBranch.objects.filter(layer=layer).filter( branch=branch) if res: # The layerbranch already exists (this will occur for layers # that already existed, since we need to have those in layer_idmap # to be able to import layer dependencies) logger.debug('Skipping layerbranch %s, already exists' % layerbranchjs['id']) continue layerbranch = LayerBranch() for key, value in layerbranchjs.items(): if key in exclude_fields: continue setattr(layerbranch, key, value) layerbranch.branch = branch layerbranch.layer = layer layerbranch.save() layerbranch_idmap[layerbranchjs['id']] = layerbranch # Get layer dependencies logger.debug('Importing layer dependencies') rq = urllib.request.Request(layerdeps_url) data = urllib.request.urlopen(rq).read() jsdata = json.loads(data.decode('utf-8')) exclude_fields = ['id', 'layerbranch', 'dependency', 'updated'] for layerdepjs in jsdata: layerbranch = layerbranch_idmap.get(layerdepjs['layerbranch'], None) if not layerbranch: # We didn't import this layerbranch, skip it continue dependency = layer_idmap.get(layerdepjs['dependency'], None) if not dependency: # We didn't import the dependency, skip it continue layerdep = LayerDependency() for key, value in layerdepjs.items(): if key in exclude_fields: continue setattr(layerdep, key, value) layerdep.layerbranch = layerbranch layerdep.dependency = dependency layerdep.save() if layermaintainers_url: # Get layer maintainers (only available in latest code) logger.debug('Importing layer maintainers') rq = urllib.request.Request(layermaintainers_url) data = urllib.request.urlopen(rq).read() jsdata = json.loads(data.decode('utf-8')) exclude_fields = ['id', 'layerbranch'] for layermaintainerjs in jsdata: layerbranch = layerbranch_idmap.get( layermaintainerjs['layerbranch'], None) if not layerbranch: # We didn't import this layerbranch, skip it continue layermaintainer = LayerMaintainer() for key, value in layermaintainerjs.items(): if key in exclude_fields: continue setattr(layermaintainer, key, value) layermaintainer.layerbranch = layerbranch layermaintainer.save() if layernotes_url: # Get layer notes (only available in latest code) logger.debug('Importing layer notes') rq = urllib.request.Request(layernotes_url) data = urllib.request.urlopen(rq).read() jsdata = json.loads(data.decode('utf-8')) exclude_fields = ['id', 'layer'] for layernotejs in jsdata: layer = layer_idmap.get(layernotejs['layer'], None) if not layer: # We didn't import this layer, skip it continue res = LayerNote.objects.filter(layer=layer).filter( text=layernotejs['text']) if res: # The note already exists (this will occur for layers # that already existed, since we need to have those in layer_idmap # to be able to import layer dependencies) logger.debug('Skipping note %s, already exists' % layernotejs['id']) continue layernote = LayerNote() for key, value in layernotejs.items(): if key in exclude_fields: continue setattr(layernote, key, value) layernote.layer = layer layernote.save() if options.dryrun: raise DryRunRollbackException() except DryRunRollbackException: pass sys.exit(0)
def main(): parser = optparse.OptionParser( usage = """ %prog [options]""") options, args = parser.parse_args(sys.argv) utils.setup_django() from layerindex.models import LayerItem, LayerBranch, LayerDependency from django.db import transaction import httplib conn = httplib.HTTPConnection("www.openembedded.org") conn.request("GET", "/wiki/LayerIndex?action=raw") resp = conn.getresponse() if resp.status in [200, 302]: data = resp.read() in_table = False layer_type = 'M' nowiki_re = re.compile(r'</?nowiki>') link_re = re.compile(r'\[(http.*) +link\]') readme_re = re.compile(r';f=[a-zA-Z0-9/-]*README;') master_branch = utils.get_branch('master') core_layer = None with transaction.atomic(): for line in data.splitlines(): if line.startswith('{|'): in_table = True continue if in_table: if line.startswith('|}'): # We're done break elif line.startswith('!'): section = line.split('|', 1)[1].strip("'") if section.startswith('Base'): layer_type = 'A' elif section.startswith('Board'): layer_type = 'B' elif section.startswith('Software'): layer_type = 'S' elif section.startswith('Distribution'): layer_type = 'D' else: layer_type = 'M' elif not line.startswith('|-'): if line.startswith("|| ''"): continue fields = line.split('||') layer = LayerItem() layer.name = fields[1].strip() if ' ' in layer.name: logger.warn('Skipping layer %s - name invalid' % layer.name) continue logger.info('Adding layer %s' % layer.name) layer.status = 'P' layer.layer_type = layer_type layer.summary = fields[2].strip() layer.description = layer.summary if len(fields) > 6: res = link_re.match(fields[6].strip()) if res: link = res.groups(1)[0].strip() if link.endswith('/README') or readme_re.search(link): link = 'README' layer.usage_url = link repoval = nowiki_re.sub('', fields[4]).strip() layer.vcs_url = repoval if repoval.startswith('git://git.openembedded.org/'): reponame = re.sub('^.*/', '', repoval) layer.vcs_web_url = 'http://cgit.openembedded.org/' + reponame layer.vcs_web_tree_base_url = 'http://cgit.openembedded.org/' + reponame + '/tree/%path%?h=%branch%' layer.vcs_web_file_base_url = 'http://cgit.openembedded.org/' + reponame + '/tree/%path%?h=%branch%' layer.vcs_web_commit_url = 'http://cgit.openembedded.org/' + reponame + '/commit/?id=%hash%' elif repoval.startswith('git://git.yoctoproject.org/'): reponame = re.sub('^.*/', '', repoval) layer.vcs_web_url = 'http://git.yoctoproject.org/cgit/cgit.cgi/' + reponame layer.vcs_web_tree_base_url = 'http://git.yoctoproject.org/cgit/cgit.cgi/' + reponame + '/tree/%path%?h=%branch%' layer.vcs_web_file_base_url = 'http://git.yoctoproject.org/cgit/cgit.cgi/' + reponame + '/tree/%path%?h=%branch%' layer.vcs_web_commit_url = 'http://git.yoctoproject.org/cgit/cgit.cgi/' + reponame + '/commit/?id=%hash%' elif repoval.startswith('git://github.com/') or repoval.startswith('http://github.com/') or repoval.startswith('https://github.com/'): reponame = re.sub('^.*github.com/', '', repoval) reponame = re.sub('.git$', '', reponame) layer.vcs_web_url = 'http://github.com/' + reponame layer.vcs_web_tree_base_url = 'http://github.com/' + reponame + '/tree/%branch%/' layer.vcs_web_file_base_url = 'http://github.com/' + reponame + '/blob/%branch%/' layer.vcs_web_commit_url = 'http://github.com/' + reponame + '/commit/%hash%' elif repoval.startswith('git://gitlab.com/') or repoval.startswith('http://gitlab.com/') or repoval.startswith('https://gitlab.com/'): reponame = re.sub('^.*gitlab.com/', '', repoval) reponame = re.sub('.git$', '', reponame) layer.vcs_web_url = 'http://gitlab.com/' + reponame layer.vcs_web_tree_base_url = 'http://gitlab.com/' + reponame + '/tree/%branch%/' layer.vcs_web_file_base_url = 'http://gitlab.com/' + reponame + '/blob/%branch%/' layer.vcs_web_commit_url = 'http://gitlab.com/' + reponame + '/commit/%hash%' elif repoval.startswith('git://bitbucket.org/') or repoval.startswith('http://bitbucket.org/') or repoval.startswith('https://bitbucket.org/'): reponame = re.sub('^.*bitbucket.org/', '', repoval) reponame = re.sub('.git$', '', reponame) layer.vcs_web_url = 'http://bitbucket.org/' + reponame layer.vcs_web_tree_base_url = 'http://bitbucket.org/' + reponame + '/src/%branch%/%path%?at=%branch%' layer.vcs_web_file_base_url = 'http://bitbucket.org/' + reponame + '/src/%branch%/%path%?at=%branch%' layer.vcs_web_commit_url = 'http://bitbucket.org/' + reponame + '/commits/%hash%' elif '.git' in repoval: res = link_re.match(fields[5].strip()) layer.vcs_web_url = res.groups(1)[0] layer.vcs_web_tree_base_url = re.sub(r'\.git.*', '.git;a=tree;f=%path%;hb=%branch%', layer.vcs_web_url) layer.vcs_web_file_base_url = re.sub(r'\.git.*', '.git;a=blob;f=%path%;hb=%branch%', layer.vcs_web_url) layer.vcs_web_file_base_url = re.sub(r'\.git.*', '.git;a=commit;h=%hash%', layer.vcs_web_url) layer.save() layerbranch = LayerBranch() layerbranch.layer = layer layerbranch.branch = master_branch layerbranch.vcs_subdir = fields[3].strip() layerbranch.save() if layer.name != 'openembedded-core': if not core_layer: core_layer = utils.get_layer('openembedded-core') if core_layer: layerdep = LayerDependency() layerdep.layerbranch = layerbranch layerdep.dependency = core_layer layerdep.save() else: logger.error('Fetch failed: %d: %s' % (resp.status, resp.reason)) sys.exit(0)
def edit_layer_view(request, template_name, branch='master', slug=None): return_url = None branchobj = Branch.objects.filter(name=branch)[:1].get() if slug: # Edit mode layeritem = get_object_or_404(LayerItem, name=slug) if layeritem.classic: raise Http404 if not (request.user.is_authenticated() and (request.user.has_perm('layerindex.publish_layer') or layeritem.user_can_edit(request.user))): raise PermissionDenied layerbranch = get_object_or_404(LayerBranch, layer=layeritem, branch=branchobj) deplistlayers = LayerItem.objects.exclude( id=layeritem.id).order_by('name') returnto = request.GET.get('returnto', 'layer_item') if returnto: if returnto == 'layer_review': return_url = reverse_lazy(returnto, args=(layeritem.name, )) else: return_url = reverse_lazy(returnto, args=(branch, layeritem.name)) else: # Submit mode layeritem = LayerItem() layerbranch = LayerBranch(layer=layeritem, branch=branchobj) deplistlayers = LayerItem.objects.filter( classic=False).order_by('name') if request.method == 'POST': last_vcs_url = layeritem.vcs_url form = EditLayerForm(request.user, layerbranch, request.POST, instance=layeritem) maintainerformset = LayerMaintainerFormSet(request.POST, instance=layerbranch) if form.is_valid() and maintainerformset.is_valid(): with transaction.atomic(): reset_last_rev = False form.save() layerbranch.layer = layeritem new_subdir = form.cleaned_data['vcs_subdir'] if layerbranch.vcs_subdir != new_subdir: layerbranch.vcs_subdir = new_subdir reset_last_rev = True layerbranch.save() maintainerformset.save() if slug: new_deps = form.cleaned_data['deps'] existing_deps = [ deprec.dependency for deprec in layerbranch.dependencies_set.all() ] reset_last_rev = False for dep in new_deps: if dep not in existing_deps: deprec = LayerDependency() deprec.layerbranch = layerbranch deprec.dependency = dep deprec.save() reset_last_rev = True for dep in existing_deps: if dep not in new_deps: layerbranch.dependencies_set.filter( dependency=dep).delete() reset_last_rev = True if layeritem.vcs_url != last_vcs_url: reset_last_rev = True if reset_last_rev: layerbranch.vcs_last_rev = '' layerbranch.save() else: # Save dependencies for dep in form.cleaned_data['deps']: deprec = LayerDependency() deprec.layerbranch = layerbranch deprec.dependency = dep deprec.save() # Send email plaintext = get_template('layerindex/submitemail.txt') perm = Permission.objects.get(codename='publish_layer') users = User.objects.filter( Q(groups__permissions=perm) | Q(user_permissions=perm)).distinct() for user in users: if user.first_name: user_name = user.first_name else: user_name = user.username layer_url = request.build_absolute_uri( reverse('layer_review', args=(layeritem.name, ))) if getattr(settings, 'FORCE_REVIEW_HTTPS', False) and layer_url.startswith('http:'): layer_url = 'https:' + layer_url.split(':', 1)[1] d = Context({ 'user_name': user_name, 'layer_name': layeritem.name, 'layer_url': layer_url, }) subject = '%s - %s' % (settings.SUBMIT_EMAIL_SUBJECT, layeritem.name) from_email = settings.SUBMIT_EMAIL_FROM to_email = user.email text_content = plaintext.render(d) tasks.send_email.apply_async( (subject, text_content, from_email, [to_email])) return HttpResponseRedirect(reverse('submit_layer_thanks')) messages.success(request, 'Layer %s saved successfully.' % layeritem.name) if return_url: if returnto == 'layer_review': return_url = reverse_lazy(returnto, args=(layeritem.name, )) else: return_url = reverse_lazy(returnto, args=(branch, layeritem.name)) return HttpResponseRedirect(return_url) else: form = EditLayerForm(request.user, layerbranch, instance=layeritem) maintainerformset = LayerMaintainerFormSet(instance=layerbranch) return render( request, template_name, { 'form': form, 'maintainerformset': maintainerformset, 'deplistlayers': deplistlayers, 'return_url': return_url, })
def main(): parser = optparse.OptionParser( usage = """ %prog [options] <url> [name]""") parser.add_option("-s", "--subdir", help = "Specify subdirectory", action="store", dest="subdir") parser.add_option("-n", "--dry-run", help = "Don't write any data back to the database", action="store_true", dest="dryrun") parser.add_option("-d", "--debug", help = "Enable debug output", action="store_const", const=logging.DEBUG, dest="loglevel", default=logging.INFO) parser.add_option("", "--github-auth", help = "Specify github username:password", action="store", dest="github_auth") parser.add_option("-q", "--quiet", help = "Hide all output except error messages", action="store_const", const=logging.ERROR, dest="loglevel") options, args = parser.parse_args(sys.argv) if len(args) < 2: print("Please specify URL of repository for layer") sys.exit(1) layer_url = args[1] if len(args) > 2: layer_name = args[2] else: if options.subdir: layer_name = options.subdir else: layer_name = filter(None, layer_url.split('/'))[-1] if layer_name.endswith('.git'): layer_name = layer_name[:-4] if options.github_auth: if not ':' in options.github_auth: logger.error('--github-auth value must be specified as username:password') sys.exit(1) splitval = options.github_auth.split(':') github_login = splitval[0] github_password = splitval[1] else: github_login = None github_password = None utils.setup_django() import settings from layerindex.models import LayerItem, LayerBranch, LayerDependency, LayerMaintainer from django.db import transaction logger.setLevel(options.loglevel) fetchdir = settings.LAYER_FETCH_DIR if not fetchdir: logger.error("Please set LAYER_FETCH_DIR in settings.py") sys.exit(1) if not os.path.exists(fetchdir): os.makedirs(fetchdir) master_branch = utils.get_branch('master') core_layer = None transaction.enter_transaction_management() transaction.managed(True) try: # Fetch layer logger.info('Fetching repository %s' % layer_url) layer = LayerItem() layer.name = layer_name layer.status = 'P' layer.layer_type = 'M' layer.summary = 'tempvalue' layer.description = layer.summary set_vcs_fields(layer, layer_url) urldir = layer.get_fetch_dir() repodir = os.path.join(fetchdir, urldir) out = None try: if not os.path.exists(repodir): out = utils.runcmd("git clone %s %s" % (layer.vcs_url, urldir), fetchdir, logger=logger) else: out = utils.runcmd("git fetch", repodir, logger=logger) except Exception as e: logger.error("Fetch failed: %s" % str(e)) sys.exit(1) actual_branch = '' try: out = utils.runcmd("git checkout origin/master", repodir, logger=logger) except subprocess.CalledProcessError: branches = utils.runcmd("git branch -r", repodir, logger=logger) for line in branches.splitlines(): if 'origin/HEAD ->' in line: actual_branch = line.split('-> origin/')[-1] break if not actual_branch: logger.error("Repository has no master branch nor origin/HEAD") sys.exit(1) out = utils.runcmd("git checkout origin/%s" % actual_branch, repodir, logger=logger) layer_paths = [] if options.subdir: layerdir = os.path.join(repodir, options.subdir) if not os.path.exists(layerdir): logger.error("Subdirectory %s does not exist in repository for master branch" % options.subdir) sys.exit(1) if not os.path.exists(os.path.join(layerdir, 'conf/layer.conf')): logger.error("conf/layer.conf not found in subdirectory %s" % options.subdir) sys.exit(1) layer_paths.append(layerdir) else: if os.path.exists(os.path.join(repodir, 'conf/layer.conf')): layer_paths.append(repodir) # Find subdirs with a conf/layer.conf for subdir in os.listdir(repodir): subdir_path = os.path.join(repodir, subdir) if os.path.isdir(subdir_path): if os.path.exists(os.path.join(subdir_path, 'conf/layer.conf')): layer_paths.append(subdir_path) if not layer_paths: logger.error("conf/layer.conf not found in repository or first level subdirectories - is subdirectory set correctly?") sys.exit(1) if 'github.com' in layer.vcs_url: json_data, owner_json_data = get_github_layerinfo(layer.vcs_url, github_login, github_password) for layerdir in layer_paths: layer.pk = None if layerdir != repodir: subdir = os.path.relpath(layerdir, repodir) if len(layer_paths) > 1: layer.name = subdir else: subdir = '' if LayerItem.objects.filter(name=layer.name).exists(): logger.error('A layer named "%s" already exists in the database' % layer_name) sys.exit(1) logger.info('Creating layer %s' % layer.name) # Guess layer type if glob.glob(os.path.join(layerdir, 'conf/distro/*.conf')): layer.layer_type = 'D' elif glob.glob(os.path.join(layerdir, 'conf/machine/*.conf')): layer.layer_type = 'B' layer.save() layerbranch = LayerBranch() layerbranch.layer = layer layerbranch.branch = master_branch if layerdir != repodir: layerbranch.vcs_subdir = subdir if actual_branch: layerbranch.actual_branch = actual_branch layerbranch.save() if layer.name != settings.CORE_LAYER_NAME: if not core_layer: core_layer = utils.get_layer(settings.CORE_LAYER_NAME) if core_layer: layerdep = LayerDependency() layerdep.layerbranch = layerbranch layerdep.dependency = core_layer layerdep.save() # Get some extra meta-information readme_files = glob.glob(os.path.join(layerdir, 'README*')) if (not readme_files) and subdir: readme_files = glob.glob(os.path.join(repodir, 'README*')) maintainer_files = glob.glob(os.path.join(layerdir, 'MAINTAINERS')) if (not maintainer_files) and subdir: maintainer_files = glob.glob(os.path.join(repodir, 'MAINTAINERS')) maintainers = [] if readme_files: (desc, maintainers, deps) = readme_extract(readme_files[0]) if desc: layer.summary = layer.name layer.description = desc if maintainer_files: maintainers.extend(maintainers_extract(readme_files[0])) if (not maintainers) and 'github.com' in layer.vcs_url: if json_data: layer.summary = json_data['description'] layer.description = layer.summary if owner_json_data: owner_name = owner_json_data.get('name', None) owner_email = owner_json_data.get('email', None) if owner_name and owner_email: maintainers.append('%s <%s>' % (owner_name, owner_email)) if layer.name == 'openembedded-core': layer.summary = 'Core metadata' layer.layer_type = 'A' elif layer.name == 'meta-oe': layer.summary = 'Additional shared OE metadata' layer.description = layer.summary layer.layer_type = 'A' if maintainers: maint_re = re.compile(r'^"?([^"@$<>]+)"? *<([^<> ]+)>[ -]*(.+)?$') for maintentry in maintainers: res = maint_re.match(maintentry) if res: maintainer = LayerMaintainer() maintainer.layerbranch = layerbranch maintainer.name = res.group(1).strip() maintainer.email = res.group(2) if res.group(3): maintainer.responsibility = res.group(3).strip() maintainer.save() layer.save() if options.dryrun: transaction.rollback() else: transaction.commit() except: transaction.rollback() raise finally: transaction.leave_transaction_management() sys.exit(0)
def main(): parser = optparse.OptionParser( usage = """ %prog [options]""") options, args = parser.parse_args(sys.argv) utils.setup_django() from layerindex.models import LayerItem, LayerBranch, LayerDependency from django.db import transaction import httplib conn = httplib.HTTPConnection("www.openembedded.org") conn.request("GET", "/wiki/LayerIndex?action=raw") resp = conn.getresponse() if resp.status in [200, 302]: data = resp.read() in_table = False layer_type = 'M' nowiki_re = re.compile(r'</?nowiki>') link_re = re.compile(r'\[(http.*) +link\]') readme_re = re.compile(r';f=[a-zA-Z0-9/-]*README;') master_branch = utils.get_branch('master') core_layer = None transaction.enter_transaction_management() transaction.managed(True) try: for line in data.splitlines(): if line.startswith('{|'): in_table = True continue if in_table: if line.startswith('|}'): # We're done break elif line.startswith('!'): section = line.split('|', 1)[1].strip("'") if section.startswith('Base'): layer_type = 'A' elif section.startswith('Board'): layer_type = 'B' elif section.startswith('Software'): layer_type = 'S' elif section.startswith('Distribution'): layer_type = 'D' else: layer_type = 'M' elif not line.startswith('|-'): if line.startswith("|| ''"): continue fields = line.split('||') layer = LayerItem() layer.name = fields[1].strip() if ' ' in layer.name: logger.warn('Skipping layer %s - name invalid' % layer.name) continue logger.info('Adding layer %s' % layer.name) layer.status = 'P' layer.layer_type = layer_type layer.summary = fields[2].strip() layer.description = layer.summary if len(fields) > 6: res = link_re.match(fields[6].strip()) if res: link = res.groups(1)[0].strip() if link.endswith('/README') or readme_re.search(link): link = 'README' layer.usage_url = link repoval = nowiki_re.sub('', fields[4]).strip() layer.vcs_url = repoval if repoval.startswith('git://git.openembedded.org/'): reponame = re.sub('^.*/', '', repoval) layer.vcs_web_url = 'http://cgit.openembedded.org/cgit.cgi/' + reponame layer.vcs_web_tree_base_url = 'http://cgit.openembedded.org/cgit.cgi/' + reponame + '/tree/%path%?h=%branch%' layer.vcs_web_file_base_url = 'http://cgit.openembedded.org/cgit.cgi/' + reponame + '/tree/%path%?h=%branch%' elif 'git.yoctoproject.org/' in repoval: reponame = re.sub('^.*/', '', repoval) layer.vcs_web_url = 'http://git.yoctoproject.org/cgit/cgit.cgi/' + reponame layer.vcs_web_tree_base_url = 'http://git.yoctoproject.org/cgit/cgit.cgi/' + reponame + '/tree/%path%?h=%branch%' layer.vcs_web_file_base_url = 'http://git.yoctoproject.org/cgit/cgit.cgi/' + reponame + '/tree/%path%?h=%branch%' elif 'github.com/' in repoval: reponame = re.sub('^.*github.com/', '', repoval) reponame = re.sub('.git$', '', reponame) layer.vcs_web_url = 'http://github.com/' + reponame layer.vcs_web_tree_base_url = 'http://github.com/' + reponame + '/tree/%branch%/' layer.vcs_web_file_base_url = 'http://github.com/' + reponame + '/blob/%branch%/' elif 'gitorious.org/' in repoval: reponame = re.sub('^.*gitorious.org/', '', repoval) reponame = re.sub('.git$', '', reponame) layer.vcs_web_url = 'http://gitorious.org/' + reponame layer.vcs_web_tree_base_url = 'http://gitorious.org/' + reponame + '/trees/%branch%/' layer.vcs_web_file_base_url = 'http://gitorious.org/' + reponame + '/blobs/%branch%/' elif 'bitbucket.org/' in repoval: reponame = re.sub('^.*bitbucket.org/', '', repoval) reponame = re.sub('.git$', '', reponame) layer.vcs_web_url = 'http://bitbucket.org/' + reponame layer.vcs_web_tree_base_url = 'http://bitbucket.org/' + reponame + '/src/%branch%/%path%?at=%branch%' layer.vcs_web_file_base_url = 'http://bitbucket.org/' + reponame + '/src/%branch%/%path%?at=%branch%' elif '.git' in repoval: res = link_re.match(fields[5].strip()) layer.vcs_web_url = res.groups(1)[0] layer.vcs_web_tree_base_url = re.sub(r'\.git.*', '.git;a=tree;f=%path%;hb=%branch%', layer.vcs_web_url) layer.vcs_web_file_base_url = re.sub(r'\.git.*', '.git;a=blob;f=%path%;hb=%branch%', layer.vcs_web_url) layer.save() layerbranch = LayerBranch() layerbranch.layer = layer layerbranch.branch = master_branch layerbranch.vcs_subdir = fields[3].strip() layerbranch.save() if layer.name != 'openembedded-core': if not core_layer: core_layer = utils.get_layer('openembedded-core') if core_layer: layerdep = LayerDependency() layerdep.layerbranch = layerbranch layerdep.dependency = core_layer layerdep.save() transaction.commit() except: transaction.rollback() raise finally: transaction.leave_transaction_management() else: logger.error('Fetch failed: %d: %s' % (resp.status, resp.reason)) sys.exit(0)
def edit_layer_view(request, template_name, branch='master', slug=None): return_url = None branchobj = Branch.objects.filter(name=branch)[:1].get() if slug: # Edit mode layeritem = get_object_or_404(LayerItem, name=slug) if layeritem.classic: raise Http404 if not (request.user.is_authenticated() and (request.user.has_perm('layerindex.publish_layer') or layeritem.user_can_edit(request.user))): raise PermissionDenied layerbranch = get_object_or_404(LayerBranch, layer=layeritem, branch=branchobj) deplistlayers = LayerItem.objects.exclude(id=layeritem.id).order_by('name') returnto = request.GET.get('returnto', 'layer_item') if returnto: if returnto == 'layer_review': return_url = reverse_lazy(returnto, args=(layeritem.name,)) else: return_url = reverse_lazy(returnto, args=(branch, layeritem.name)) else: # Submit mode layeritem = LayerItem() layerbranch = LayerBranch(layer=layeritem, branch=branchobj) deplistlayers = LayerItem.objects.filter(classic=False).order_by('name') if request.method == 'POST': last_vcs_url = layeritem.vcs_url form = EditLayerForm(request.user, layerbranch, request.POST, instance=layeritem) maintainerformset = LayerMaintainerFormSet(request.POST, instance=layerbranch) if form.is_valid() and maintainerformset.is_valid(): with transaction.commit_on_success(): reset_last_rev = False form.save() layerbranch.layer = layeritem new_subdir = form.cleaned_data['vcs_subdir'] if layerbranch.vcs_subdir != new_subdir: layerbranch.vcs_subdir = new_subdir reset_last_rev = True layerbranch.save() maintainerformset.save() if slug: new_deps = form.cleaned_data['deps'] existing_deps = [deprec.dependency for deprec in layerbranch.dependencies_set.all()] reset_last_rev = False for dep in new_deps: if dep not in existing_deps: deprec = LayerDependency() deprec.layerbranch = layerbranch deprec.dependency = dep deprec.save() reset_last_rev = True for dep in existing_deps: if dep not in new_deps: layerbranch.dependencies_set.filter(dependency=dep).delete() reset_last_rev = True if layeritem.vcs_url != last_vcs_url: reset_last_rev = True if reset_last_rev: layerbranch.vcs_last_rev = '' layerbranch.save() else: # Save dependencies for dep in form.cleaned_data['deps']: deprec = LayerDependency() deprec.layerbranch = layerbranch deprec.dependency = dep deprec.save() # Send email plaintext = get_template('layerindex/submitemail.txt') perm = Permission.objects.get(codename='publish_layer') users = User.objects.filter(Q(groups__permissions=perm) | Q(user_permissions=perm) ).distinct() for user in users: if user.first_name: user_name = user.first_name else: user_name = user.username d = Context({ 'user_name': user_name, 'layer_name': layeritem.name, 'layer_url': request.build_absolute_uri(reverse('layer_review', args=(layeritem.name,))), }) subject = '%s - %s' % (settings.SUBMIT_EMAIL_SUBJECT, layeritem.name) from_email = settings.SUBMIT_EMAIL_FROM to_email = user.email text_content = plaintext.render(d) msg = EmailMessage(subject, text_content, from_email, [to_email]) msg.send() return HttpResponseRedirect(reverse('submit_layer_thanks')) messages.success(request, 'Layer %s saved successfully.' % layeritem.name) if return_url: return HttpResponseRedirect(return_url) else: form = EditLayerForm(request.user, layerbranch, instance=layeritem) maintainerformset = LayerMaintainerFormSet(instance=layerbranch) return render(request, template_name, { 'form': form, 'maintainerformset': maintainerformset, 'deplistlayers': deplistlayers, 'return_url': return_url, })