def rrs_remove_duplicates(args): utils.setup_django() import settings from django.db import transaction from rrs.models import RecipeUpstreamHistory from layerindex.models import Recipe core_layer = utils.get_layer(settings.CORE_LAYER_NAME) if not core_layer: logger.error('Unable to find core layer %s' % settings.CORE_LAYER_NAME) return 1 core_layerbranch = core_layer.get_layerbranch('master') if not core_layerbranch: logger.error('Unable to find branch master of layer %s' % core_layerbranch.name) return 1 try: with transaction.atomic(): for row in RecipeUpstreamHistory.objects.filter(layerbranch=core_layerbranch).order_by('-id'): if RecipeUpstreamHistory.objects.filter(layerbranch=row.layerbranch, start_date=row.start_date).count() > 1: logger.info('Deleting duplicate %d' % row.id) row.delete() if args.dry_run: raise DryRunRollbackException except DryRunRollbackException: pass return 0
def _forward(self): """ Helper to create the forward pass in the graph :return: """ x = self.x # Reshape input x = tf.reshape(x, (-1, self.input_size, self.input_size, 1)) # List of length maintained during the building because some # dimensions can't be inferred at build time, such as the Dense layer size length = [self.input_size, self.input_size, 1] # Implement dropout every three layers nb_dropout_layers = len(self.layers) // 3 # Iterate over all built State for index in range(len(self.layers)): prev_layer = None # Pass the previous shape (needed for reshaping) if index > 0: prev_layer = self.layers[index - 1] if index % 3 == 0 and index > 0: keep_prob = 1 - (index // 3) / (2 * nb_dropout_layers) x = layers.dropout(inputs=x, is_training=self.is_training, keep_prob=keep_prob) x, length = utils.get_layer(x, self.layers[index], self.nb_targets, length, prev_layer) # No activation, at the last layer if index != len(self.layers) - 1: x = self.activation(x) return x
def __init__( self, filters: int, kernel_size: int = 3, stride: int = 1, padding: int = 0, norm_layer: str = None, act_layer: str = "relu", use_bias: bool = True, l2_reg: float = 1e-5, name: str = "ConvNormAct", ): super().__init__(name=name) if padding > 0: self.pad = tf.keras.layers.ZeroPadding2D( padding=padding, name=f"Padding{padding}x{padding}", ) else: self.pad = Identity() self.conv = tf.keras.layers.Conv2D( filters=filters, kernel_size=kernel_size, strides=stride, name=f"Conv{kernel_size}x{kernel_size}", kernel_regularizer=tf.keras.regularizers.l2(l2_reg), use_bias=use_bias, ) _available_normalization = { "bn": BatchNormalization(), } self.norm = get_layer(norm_layer, _available_normalization, Identity()) _available_activation = { "relu": tf.keras.layers.ReLU(name="ReLU"), "relu6": ReLU6(), "hswish": HardSwish(), "hsigmoid": HardSigmoid(), "sigmoid": tf.keras.layers.Activation('sigmoid', name='sigmoid'), "softmax": tf.keras.layers.Softmax(name="Softmax"), } self.act = get_layer(act_layer, _available_activation, Identity())
def do_layer_selection_changed(self): """ Takes the names of currently selected stations with water measurements and display them in the combobox """ # Can't use activeLayer() cause if I have stations selected, # and select another layer it creates an error, when zooming to the stations lay = utils.get_layer("CurrentW", self.i_face) self.c_box.clear() for feat in lay.selectedFeatures(): self.c_box.addItem(feat['shortname'])
def rrs_import(args): utils.setup_django() import settings from django.db import transaction from rrs.models import RecipeUpstreamHistory, RecipeUpstream from layerindex.models import Recipe core_layer = utils.get_layer(settings.CORE_LAYER_NAME) if not core_layer: logger.error('Unable to find core layer %s' % settings.CORE_LAYER_NAME) return 1 core_layerbranch = core_layer.get_layerbranch('master') if not core_layerbranch: logger.error('Unable to find branch master of layer %s' % core_layerbranch.name) return 1 layerbranch = core_layerbranch try: with transaction.atomic(): with open(args.infile, 'r') as f: data = json.load(f) for item, itemdata in data.items(): if item == 'recipeupstreamhistory': for histdata in itemdata: ruh = RecipeUpstreamHistory() ruh.start_date = histdata['start_date'] ruh.end_date = histdata['end_date'] ruh.layerbranch = layerbranch ruh.save() for upstreamdata in histdata['upstreams']: ru = RecipeUpstream() ru.history = ruh pn = upstreamdata['recipe'] recipe = Recipe.objects.filter(layerbranch=layerbranch, pn=pn).first() if not recipe: logger.warning('Could not find recipe %s in layerbranch %s' % (pn, layerbranch)) continue ru.recipe = recipe ru.version = upstreamdata['version'] ru.type = upstreamdata['type'] ru.status = upstreamdata['status'] ru.no_update_reason = upstreamdata['no_update_reason'] ru.date = upstreamdata['date'] ru.save() if args.dry_run: raise DryRunRollbackException except DryRunRollbackException: pass return 0
def init_params(options): params = OrderedDict() # embedding params['Wemb'] = norm_weight(options['n_words_src'], options['dim_word']) params['Wemb_dec'] = norm_weight(options['n_words'], options['dim_word']) # encoder: bidirectional RNN params = get_layer(options['encoder'])[0](options, params, prefix='encoder', nin=options['dim_word'], dim=options['dim']) params = get_layer(options['encoder'])[0](options, params, prefix='encoder_r', nin=options['dim_word'], dim=options['dim']) ctxdim = 2 * options['dim'] # init_state, init_cell params = get_layer('ff')[0](options, params, prefix='ff_state', nin=ctxdim, nout=options['dim']) # decoder params = get_layer(options['decoder'])[0](options, params, prefix='decoder', nin=options['dim_word'], dim=options['dim'], dimctx=ctxdim) # readout params = get_layer('ff')[0](options, params, prefix='ff_logit_lstm', nin=options['dim'], nout=options['dim_word'], ortho=False) params = get_layer('ff')[0](options, params, prefix='ff_logit_prev', nin=options['dim_word'], nout=options['dim_word'], ortho=False) params = get_layer('ff')[0](options, params, prefix='ff_logit_ctx', nin=ctxdim, nout=options['dim_word'], ortho=False) params = get_layer('ff')[0](options, params, prefix='ff_logit', nin=options['dim_word'], nout=options['n_words']) return params
def timeit(self, args): print_action = [] get_layer = lambda x: utils.get_layer(x, args.layer) # consume argument layer # consume argument action if settings.TIME_ACTION_SEPARATELY == args.mode: cal = CategoricalTimer(settings.TIMER_CATEGORY_PATTERN, settings.TIMER_CATEGORY_KEY) elif settings.TIME_ACTION_TOGETHER == args.mode: cal = PatternTimer(settings.TIMER_ANYTEXT_PATTERN) else: raise ValueError log_path = utils.get_log_path(settings.LOG_DURATION_TYPE_NAME, args.target) fd = open(log_path, 'w') cal_out = lambda x: cal.measure(x).output_duration(fd) # consumte arguments print_option if settings.TIME_PRINT_OPTION_NOT_EMPTY in args.print_option: print_action.append(cal_out) if settings.TIME_PRINT_OPTION_VALID in args.print_option: censor = TextCensor(settings.CENSOR_RULES) print_action.append(lambda x: cal_out(censor.validate(x).qualified)) if settings.TIME_PRINT_OPTION_TOTAL in args.print_option: all_cal = OverallTimer(cal) print_action.append(lambda x: all_cal.measure(x)) for src_file, _ in utils.traverser(args.target, "", settings.LOOKUP_NAMES_PATTERN): intervals = get_layer(self.tbp.read(src_file).parse_blocks()) fd.write(">>" + src_file.decode(settings.DECODING).encode(settings.ENCODING) + ':' + os.linesep) for action in print_action: action(intervals) if "all_cal" in locals().keys(): fd.write(">>" + args.target.decode(settings.DECODING).encode(settings.ENCODING) + ' ') all_cal.output_duration(fd) fd.close()
def init_parser(settings, branch, bitbakepath, enable_tracking=False, nocheckout=False, classic=False, logger=None): if not (nocheckout or classic): # Check out the branch of BitBake appropriate for this branch and clean out any stale files (e.g. *.pyc) if re.match("[0-9a-f]{40}", branch.bitbake_branch): # SHA1 hash bitbake_ref = branch.bitbake_branch else: # Branch name bitbake_ref = "origin/%s" % branch.bitbake_branch out = utils.runcmd("git checkout %s" % bitbake_ref, bitbakepath, logger=logger) out = utils.runcmd("git clean -f -x", bitbakepath, logger=logger) # Skip sanity checks os.environ["BB_ENV_EXTRAWHITE"] = "DISABLE_SANITY_CHECKS" os.environ["DISABLE_SANITY_CHECKS"] = "1" fetchdir = settings.LAYER_FETCH_DIR if not classic: # Ensure we have OE-Core set up to get some base configuration core_layer = utils.get_layer(settings.CORE_LAYER_NAME) if not core_layer: raise RecipeParseError( "Unable to find core layer %s in database; create this layer or set the CORE_LAYER_NAME setting to point to the core layer" % settings.CORE_LAYER_NAME ) core_layerbranch = core_layer.get_layerbranch(branch.name) core_branchname = branch.name if core_layerbranch: core_subdir = core_layerbranch.vcs_subdir if core_layerbranch.actual_branch: core_branchname = core_layerbranch.actual_branch else: core_subdir = "meta" core_urldir = core_layer.get_fetch_dir() core_repodir = os.path.join(fetchdir, core_urldir) core_layerdir = os.path.join(core_repodir, core_subdir) if not nocheckout: out = utils.runcmd("git checkout origin/%s" % core_branchname, core_repodir, logger=logger) out = utils.runcmd("git clean -f -x", core_repodir, logger=logger) if not os.path.exists(os.path.join(core_layerdir, "conf/bitbake.conf")): raise RecipeParseError( "conf/bitbake.conf not found in core layer %s - is subdirectory set correctly?" % core_layer.name ) # The directory above where this script exists should contain our conf/layer.conf, # so add it to BBPATH along with the core layer directory confparentdir = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")) os.environ["BBPATH"] = str("%s:%s" % (confparentdir, core_layerdir)) # Change into a temporary directory so we don't write the cache and other files to the current dir if not os.path.exists(settings.TEMP_BASE_DIR): os.makedirs(settings.TEMP_BASE_DIR) tempdir = tempfile.mkdtemp(dir=settings.TEMP_BASE_DIR) os.chdir(tempdir) tinfoil = _setup_tinfoil(bitbakepath, enable_tracking) # Ensure TMPDIR exists (or insane.bbclass will blow up trying to write to the QA log) oe_tmpdir = tinfoil.config_data.getVar("TMPDIR", True) if not os.path.exists(oe_tmpdir): os.makedirs(oe_tmpdir) # Ensure BBFILES as an initial value so that the old mode of BBFILES := "${BBFILES} ..." works if not tinfoil.config_data.getVar("BBFILES", False): tinfoil.config_data.setVar("BBFILES", "") return (tinfoil, tempdir)
def main(): valid_layer_name = re.compile('[-\w]+$') parser = optparse.OptionParser(usage=""" %prog [options] <url> [name]""") utils.setup_django() layer_type_help, layer_type_choices = get_layer_type_choices() parser.add_option("-s", "--subdir", help="Specify subdirectory", action="store", dest="subdir") parser.add_option("-t", "--type", help=layer_type_help, choices=layer_type_choices, action="store", dest="layer_type", default='') parser.add_option("-n", "--dry-run", help="Don't write any data back to the database", action="store_true", dest="dryrun") parser.add_option("-d", "--debug", help="Enable debug output", action="store_const", const=logging.DEBUG, dest="loglevel", default=logging.INFO) parser.add_option("", "--github-auth", help="Specify github username:password", action="store", dest="github_auth") parser.add_option("-q", "--quiet", help="Hide all output except error messages", action="store_const", const=logging.ERROR, dest="loglevel") parser.add_option("-a", "--actual-branch", help="Set actual branch", action="store", dest="actual_branch") options, args = parser.parse_args(sys.argv) if len(args) < 2: print("Please specify URL of repository for layer") sys.exit(1) layer_url = args[1] if len(args) > 2: layer_name = args[2] else: if options.subdir: layer_name = options.subdir else: layer_name = [x for x in layer_url.split('/') if x][-1] if layer_name.endswith('.git'): layer_name = layer_name[:-4] if not valid_layer_name.match(layer_name): logger.error( 'Invalid layer name "%s" - Layer name can only include letters, numbers and dashes.', layer_name) sys.exit(1) if options.github_auth: if not ':' in options.github_auth: logger.error( '--github-auth value must be specified as username:password') sys.exit(1) splitval = options.github_auth.split(':') github_login = splitval[0] github_password = splitval[1] else: github_login = None github_password = None import settings from layerindex.models import LayerItem, LayerBranch, LayerDependency, LayerMaintainer from django.db import transaction logger.setLevel(options.loglevel) fetchdir = settings.LAYER_FETCH_DIR if not fetchdir: logger.error("Please set LAYER_FETCH_DIR in settings.py") sys.exit(1) if not os.path.exists(fetchdir): os.makedirs(fetchdir) master_branch = utils.get_branch('master') core_layer = None try: with transaction.atomic(): # Fetch layer logger.info('Fetching repository %s' % layer_url) layer = LayerItem() layer.name = layer_name layer.status = 'P' layer.summary = 'tempvalue' layer.description = layer.summary set_vcs_fields(layer, layer_url) urldir = layer.get_fetch_dir() repodir = os.path.join(fetchdir, urldir) out = None try: if not os.path.exists(repodir): out = utils.runcmd("git clone %s %s" % (layer.vcs_url, urldir), fetchdir, logger=logger) else: out = utils.runcmd("git fetch", repodir, logger=logger) except Exception as e: logger.error("Fetch failed: %s" % str(e)) sys.exit(1) actual_branch = 'master' if (options.actual_branch): actual_branch = options.actual_branch try: out = utils.runcmd("git checkout origin/%s" % actual_branch, repodir, logger=logger) except subprocess.CalledProcessError: actual_branch = None branches = utils.runcmd("git branch -r", repodir, logger=logger) for line in branches.splitlines(): if 'origin/HEAD ->' in line: actual_branch = line.split('-> origin/')[-1] break if not actual_branch: logger.error( "Repository has no master branch nor origin/HEAD") sys.exit(1) out = utils.runcmd("git checkout origin/%s" % actual_branch, repodir, logger=logger) layer_paths = [] if options.subdir: layerdir = os.path.join(repodir, options.subdir) if not os.path.exists(layerdir): logger.error( "Subdirectory %s does not exist in repository for master branch" % options.subdir) sys.exit(1) if not os.path.exists(os.path.join(layerdir, 'conf/layer.conf')): logger.error( "conf/layer.conf not found in subdirectory %s" % options.subdir) sys.exit(1) layer_paths.append(layerdir) else: if os.path.exists(os.path.join(repodir, 'conf/layer.conf')): layer_paths.append(repodir) # Find subdirs with a conf/layer.conf for subdir in os.listdir(repodir): subdir_path = os.path.join(repodir, subdir) if os.path.isdir(subdir_path): if os.path.exists( os.path.join(subdir_path, 'conf/layer.conf')): layer_paths.append(subdir_path) if not layer_paths: logger.error( "conf/layer.conf not found in repository or first level subdirectories - is subdirectory set correctly?" ) sys.exit(1) if 'github.com' in layer.vcs_url: json_data, owner_json_data = get_github_layerinfo( layer.vcs_url, github_login, github_password) for layerdir in layer_paths: layer.pk = None if layerdir != repodir: subdir = os.path.relpath(layerdir, repodir) if len(layer_paths) > 1: layer.name = subdir else: subdir = '' if LayerItem.objects.filter(name=layer.name).exists(): if LayerItem.objects.filter(name=layer.name).exclude( vcs_url=layer.vcs_url).exists(): conflict_list = LayerItem.objects.filter( name=layer.name).exclude(vcs_url=layer.vcs_url) conflict_list_urls = [] for conflict in conflict_list: conflict_list_urls.append(conflict.vcs_url) cln = ', '.join(conflict_list_urls) logger.error( 'A layer named "%s" already exists in the database. Possible name collision with %s.vcs_url = %s' % (layer.name, layer.name, cln)) sys.exit(1) else: logger.info( 'The layer named "%s" already exists in the database. Skipping this layer with same vcs_url' % layer.name) layer_paths = [x for x in layer_paths if x != layerdir] continue logger.info('Creating layer %s' % layer.name) # Guess layer type if not specified if options.layer_type: layer.layer_type = options.layer_type elif layer.name in ['openembedded-core', 'meta-oe']: layer.layer_type = 'A' elif glob.glob(os.path.join(layerdir, 'conf/distro/*.conf')): layer.layer_type = 'D' elif glob.glob(os.path.join(layerdir, 'conf/machine/*.conf')): layer.layer_type = 'B' else: layer.layer_type = 'M' layer.save() layerbranch = LayerBranch() layerbranch.layer = layer layerbranch.branch = master_branch if layerdir != repodir: layerbranch.vcs_subdir = subdir if actual_branch: layerbranch.actual_branch = actual_branch layerbranch.save() if layer.name != settings.CORE_LAYER_NAME: if not core_layer: core_layer = utils.get_layer(settings.CORE_LAYER_NAME) if core_layer: logger.debug('Adding dep %s to %s' % (core_layer.name, layer.name)) layerdep = LayerDependency() layerdep.layerbranch = layerbranch layerdep.dependency = core_layer layerdep.save() layerconfparser = LayerConfParse(logger=logger) try: config_data = layerconfparser.parse_layer(layerdir) if config_data: utils.add_dependencies(layerbranch, config_data, logger=logger) utils.add_recommends(layerbranch, config_data, logger=logger) finally: layerconfparser.shutdown() # Get some extra meta-information readme_files = glob.glob(os.path.join(layerdir, 'README*')) if (not readme_files) and subdir: readme_files = glob.glob(os.path.join(repodir, 'README*')) maintainer_files = glob.glob( os.path.join(layerdir, 'MAINTAINERS')) if (not maintainer_files) and subdir: maintainer_files = glob.glob( os.path.join(repodir, 'MAINTAINERS')) maintainers = [] if readme_files: (desc, maintainers, deps) = readme_extract(readme_files[0]) if desc: layer.summary = layer.name layer.description = desc if maintainer_files: maintainers.extend(maintainers_extract(readme_files[0])) if (not maintainers) and 'github.com' in layer.vcs_url: if json_data: layer.summary = json_data['description'] layer.description = layer.summary if owner_json_data: owner_name = owner_json_data.get('name', None) owner_email = owner_json_data.get('email', None) if owner_name and owner_email: maintainers.append('%s <%s>' % (owner_name, owner_email)) if layer.name == 'openembedded-core': layer.summary = 'Core metadata' elif layer.name == 'meta-oe': layer.summary = 'Additional shared OE metadata' layer.description = layer.summary if maintainers: maint_re = re.compile( r'^"?([^"@$<>]+)"? *<([^<> ]+)>[ -]*(.+)?$') for maintentry in maintainers: res = maint_re.match(maintentry) if res: maintainer = LayerMaintainer() maintainer.layerbranch = layerbranch maintainer.name = res.group(1).strip() maintainer.email = res.group(2) if res.group(3): maintainer.responsibility = res.group( 3).strip() maintainer.save() layer.save() if not layer_paths: logger.error('No layers added.') sys.exit(1) if options.dryrun: raise DryRunRollbackException() except DryRunRollbackException: pass sys.exit(0)
def main(): if LooseVersion(git.__version__) < '0.3.1': logger.error( "Version of GitPython is too old, please install GitPython (python-git) 0.3.1 or later in order to use this script" ) sys.exit(1) parser = optparse.OptionParser(usage=""" %prog [options]""") parser.add_option("-b", "--branch", help="Specify branch to update", action="store", dest="branch", default='master') parser.add_option("-l", "--layer", help="Layer to update", action="store", dest="layer") parser.add_option( "-r", "--reload", help="Reload recipe data instead of updating since last update", action="store_true", dest="reload") parser.add_option( "", "--fullreload", help="Discard existing recipe data and fetch it from scratch", action="store_true", dest="fullreload") parser.add_option("-n", "--dry-run", help="Don't write any data back to the database", action="store_true", dest="dryrun") parser.add_option("", "--nocheckout", help="Don't check out branches", action="store_true", dest="nocheckout") parser.add_option("", "--stop-on-error", help="Stop on first parsing error", action="store_true", default=False, dest="stop_on_error") parser.add_option("-i", "--initial", help="Print initial values parsed from layer.conf only", action="store_true") parser.add_option("-d", "--debug", help="Enable debug output", action="store_const", const=logging.DEBUG, dest="loglevel", default=logging.INFO) parser.add_option("-q", "--quiet", help="Hide all output except error messages", action="store_const", const=logging.ERROR, dest="loglevel") parser.add_option( "", "--keep-temp", help="Preserve temporary directory at the end instead of deleting it", action="store_true") options, args = parser.parse_args(sys.argv) if len(args) > 1: logger.error('unexpected argument "%s"' % args[1]) parser.print_help() sys.exit(1) if options.fullreload: options.reload = True utils.setup_django() import settings from layerindex.models import LayerItem, LayerBranch, Recipe, RecipeFileDependency, Machine, Distro, BBAppend, BBClass, IncFile from django.db import transaction logger.setLevel(options.loglevel) branch = utils.get_branch(options.branch) if not branch: logger.error("Specified branch %s is not valid" % options.branch) sys.exit(1) fetchdir = settings.LAYER_FETCH_DIR if not fetchdir: logger.error("Please set LAYER_FETCH_DIR in settings.py") sys.exit(1) bitbakepath = os.path.join(fetchdir, 'bitbake') layer = utils.get_layer(options.layer) urldir = layer.get_fetch_dir() repodir = os.path.join(fetchdir, urldir) layerbranch = layer.get_layerbranch(options.branch) branchname = options.branch branchdesc = options.branch if layerbranch: if layerbranch.actual_branch: branchname = layerbranch.actual_branch branchdesc = "%s (%s)" % (options.branch, branchname) # Collect repo info repo = git.Repo(repodir) if repo.bare: logger.error('Repository %s is bare, not supported' % repodir) sys.exit(1) topcommit = repo.commit('origin/%s' % branchname) if options.nocheckout: topcommit = repo.commit('HEAD') tinfoil = None tempdir = None try: with transaction.atomic(): newbranch = False if not layerbranch: # LayerBranch doesn't exist for this branch, create it newbranch = True layerbranch = LayerBranch() layerbranch.layer = layer layerbranch.branch = branch layerbranch_source = layer.get_layerbranch(branch) if not layerbranch_source: layerbranch_source = layer.get_layerbranch(None) if layerbranch_source: layerbranch.vcs_subdir = layerbranch_source.vcs_subdir layerbranch.save() if layerbranch_source: for maintainer in layerbranch_source.layermaintainer_set.all( ): maintainer.pk = None maintainer.id = None maintainer.layerbranch = layerbranch maintainer.save() if layerbranch.vcs_subdir and not options.nocheckout: # Find latest commit in subdirectory # A bit odd to do it this way but apparently there's no other way in the GitPython API topcommit = next( repo.iter_commits('origin/%s' % branchname, paths=layerbranch.vcs_subdir), None) layerdir = os.path.join(repodir, layerbranch.vcs_subdir) layerdir_start = os.path.normpath(layerdir) + os.sep layerrecipes = Recipe.objects.filter(layerbranch=layerbranch) layermachines = Machine.objects.filter(layerbranch=layerbranch) layerdistros = Distro.objects.filter(layerbranch=layerbranch) layerappends = BBAppend.objects.filter(layerbranch=layerbranch) layerclasses = BBClass.objects.filter(layerbranch=layerbranch) layerincfiles = IncFile.objects.filter(layerbranch=layerbranch) if layerbranch.vcs_last_rev != topcommit.hexsha or options.reload or options.initial: # Check out appropriate branch if not options.nocheckout: utils.checkout_layer_branch(layerbranch, repodir, logger=logger) logger.info("Collecting data for layer %s on branch %s" % (layer.name, branchdesc)) try: (tinfoil, tempdir) = recipeparse.init_parser( settings, branch, bitbakepath, nocheckout=options.nocheckout, logger=logger) except recipeparse.RecipeParseError as e: logger.error(str(e)) sys.exit(1) logger.debug('Using temp directory %s' % tempdir) # Clear the default value of SUMMARY so that we can use DESCRIPTION instead if it hasn't been set tinfoil.config_data.setVar('SUMMARY', '') # Clear the default value of DESCRIPTION so that we can see where it's not set tinfoil.config_data.setVar('DESCRIPTION', '') # Clear the default value of HOMEPAGE ('unknown') tinfoil.config_data.setVar('HOMEPAGE', '') # Set a blank value for LICENSE so that it doesn't cause the parser to die (e.g. with meta-ti - # why won't they just fix that?!) tinfoil.config_data.setVar('LICENSE', '') layerconfparser = layerconfparse.LayerConfParse( logger=logger, tinfoil=tinfoil) layer_config_data = layerconfparser.parse_layer(layerdir) if not layer_config_data: logger.info( "Skipping update of layer %s for branch %s - conf/layer.conf may have parse issues" % (layer.name, branchdesc)) layerconfparser.shutdown() sys.exit(1) utils.set_layerbranch_collection_version(layerbranch, layer_config_data, logger=logger) if options.initial: # Use print() rather than logger.info() since "-q" makes it print nothing. for i in [ "BBFILE_COLLECTIONS", "LAYERVERSION", "LAYERDEPENDS", "LAYERRECOMMENDS" ]: print( '%s = "%s"' % (i, utils.get_layer_var( layer_config_data, i, logger))) sys.exit(0) # Set up for recording patch info utils.setup_core_layer_sys_path(settings, branch.name) skip_patches = False try: import oe.recipeutils except ImportError: logger.warn( 'Failed to find lib/oe/recipeutils.py in layers - patch information will not be collected' ) skip_patches = True utils.add_dependencies(layerbranch, layer_config_data, logger=logger) utils.add_recommends(layerbranch, layer_config_data, logger=logger) layerbranch.save() try: config_data_copy = recipeparse.setup_layer( tinfoil.config_data, fetchdir, layerdir, layer, layerbranch, logger) except recipeparse.RecipeParseError as e: logger.error(str(e)) sys.exit(1) if layerbranch.vcs_last_rev and not options.reload: try: diff = repo.commit( layerbranch.vcs_last_rev).diff(topcommit) except Exception as e: logger.warn( "Unable to get diff from last commit hash for layer %s - falling back to slow update: %s" % (layer.name, str(e))) diff = None else: diff = None # We handle recipes specially to try to preserve the same id # when recipe upgrades happen (so that if a user bookmarks a # recipe page it remains valid) layerrecipes_delete = [] layerrecipes_add = [] # Check if any paths should be ignored because there are layers within this layer removedirs = [] for root, dirs, files in os.walk(layerdir): for diritem in dirs: if os.path.exists( os.path.join(root, diritem, 'conf', 'layer.conf')): removedirs.append( os.path.join(root, diritem) + os.sep) if diff: # Apply git changes to existing recipe list if layerbranch.vcs_subdir: subdir_start = os.path.normpath( layerbranch.vcs_subdir) + os.sep else: subdir_start = "" updatedrecipes = set() dirtyrecipes = set() other_deletes = [] other_adds = [] for diffitem in diff.iter_change_type('R'): oldpath = diffitem.a_blob.path newpath = diffitem.b_blob.path skip = False for removedir in removedirs: # FIXME what about files moved into removedirs? if oldpath.startswith(removedir): skip = True break if skip: continue if oldpath.startswith(subdir_start): if not newpath.startswith(subdir_start): logger.debug( "Treating rename of %s to %s as a delete since new path is outside layer" % (oldpath, newpath)) other_deletes.append(diffitem) continue (oldtypename, oldfilepath, oldfilename) = recipeparse.detect_file_type( oldpath, subdir_start) (newtypename, newfilepath, newfilename) = recipeparse.detect_file_type( newpath, subdir_start) if oldtypename != newtypename: # This is most likely to be a .inc file renamed to a .bb - and since # there may be another recipe deleted at the same time we probably want # to consider that, so just treat it as a delete and an add logger.debug( "Treating rename of %s to %s as a delete and add (since type changed)" % (oldpath, newpath)) other_deletes.append(diffitem) other_adds.append(diffitem) elif oldtypename == 'recipe': results = layerrecipes.filter( filepath=oldfilepath).filter( filename=oldfilename) if len(results): recipe = results[0] logger.debug("Rename recipe %s to %s" % (recipe, newpath)) recipe.filepath = newfilepath recipe.filename = newfilename recipe.save() update_recipe_file( tinfoil, config_data_copy, os.path.join(layerdir, newfilepath), recipe, layerdir_start, repodir, options.stop_on_error, skip_patches) updatedrecipes.add( os.path.join(oldfilepath, oldfilename)) updatedrecipes.add( os.path.join(newfilepath, newfilename)) else: logger.warn( "Renamed recipe %s could not be found" % oldpath) other_adds.append(diffitem) elif oldtypename == 'bbappend': results = layerappends.filter( filepath=oldfilepath).filter( filename=oldfilename) if len(results): logger.debug( "Rename bbappend %s to %s" % (results[0], os.path.join(newfilepath, newfilename))) results[0].filepath = newfilepath results[0].filename = newfilename results[0].save() else: logger.warn( "Renamed bbappend %s could not be found" % oldpath) other_adds.append(diffitem) elif oldtypename == 'machine': results = layermachines.filter( name=oldfilename) if len(results): logger.debug("Rename machine %s to %s" % (results[0], newfilename)) results[0].name = newfilename results[0].save() else: logger.warn( "Renamed machine %s could not be found" % oldpath) other_adds.append(diffitem) elif oldtypename == 'distro': results = layerdistros.filter(name=oldfilename) if len(results): logger.debug("Rename distro %s to %s" % (results[0], newfilename)) results[0].name = newfilename results[0].save() else: logger.warn( "Renamed distro %s could not be found" % oldpath) other_adds.append(diffitem) elif oldtypename == 'bbclass': results = layerclasses.filter(name=oldfilename) if len(results): logger.debug("Rename class %s to %s" % (results[0], newfilename)) results[0].name = newfilename results[0].save() else: logger.warn( "Renamed class %s could not be found" % oldpath) other_adds.append(diffitem) elif oldtypename == 'incfile': results = layerincfiles.filter( path=os.path.join(oldfilepath, oldfilename)) if len(results): logger.debug("Rename inc file %s to %s" % (results[0], newfilename)) results[0].name = newfilename results[0].save() else: logger.warn( "Renamed inc file %s could not be found" % oldpath) other_adds.append(diffitem) deps = RecipeFileDependency.objects.filter( layerbranch=layerbranch).filter(path=oldpath) for dep in deps: dirtyrecipes.add(dep.recipe) for diffitem in itertools.chain(diff.iter_change_type('D'), other_deletes): path = diffitem.a_blob.path if path.startswith(subdir_start): skip = False for removedir in removedirs: if path.startswith(removedir): skip = True break if skip: continue (typename, filepath, filename) = recipeparse.detect_file_type( path, subdir_start) if typename == 'recipe': values = layerrecipes.filter( filepath=filepath).filter( filename=filename).values( 'id', 'filepath', 'filename', 'pn') if len(values): layerrecipes_delete.append(values[0]) logger.debug("Mark %s for deletion" % values[0]) updatedrecipes.add( os.path.join(values[0]['filepath'], values[0]['filename'])) else: logger.warn( "Deleted recipe %s could not be found" % path) elif typename == 'bbappend': layerappends.filter(filepath=filepath).filter( filename=filename).delete() elif typename == 'machine': layermachines.filter(name=filename).delete() elif typename == 'distro': layerdistros.filter(name=filename).delete() elif typename == 'bbclass': layerclasses.filter(name=filename).delete() elif typename == 'incfile': layerincfiles.filter(path=os.path.join( filepath, filename)).delete() for diffitem in itertools.chain(diff.iter_change_type('A'), other_adds): path = diffitem.b_blob.path if path.startswith(subdir_start): skip = False for removedir in removedirs: if path.startswith(removedir): skip = True break if skip: continue (typename, filepath, filename) = recipeparse.detect_file_type( path, subdir_start) if typename == 'recipe': layerrecipes_add.append( os.path.join(repodir, path)) logger.debug("Mark %s for addition" % path) updatedrecipes.add( os.path.join(filepath, filename)) elif typename == 'bbappend': append = BBAppend() append.layerbranch = layerbranch append.filename = filename append.filepath = filepath append.save() elif typename == 'machine': machine = Machine() machine.layerbranch = layerbranch machine.name = filename update_machine_conf_file( os.path.join(repodir, path), machine) machine.save() elif typename == 'distro': distro = Distro() distro.layerbranch = layerbranch distro.name = filename update_distro_conf_file( os.path.join(repodir, path), distro, config_data_copy) distro.save() elif typename == 'bbclass': bbclass = BBClass() bbclass.layerbranch = layerbranch bbclass.name = filename bbclass.save() elif typename == 'incfile': incfile = IncFile() incfile.layerbranch = layerbranch incfile.path = os.path.join(filepath, filename) incfile.save() for diffitem in diff.iter_change_type('M'): path = diffitem.b_blob.path if path.startswith(subdir_start): skip = False for removedir in removedirs: if path.startswith(removedir): skip = True break if skip: continue (typename, filepath, filename) = recipeparse.detect_file_type( path, subdir_start) if typename == 'recipe': logger.debug("Mark %s for update" % path) results = layerrecipes.filter( filepath=filepath).filter( filename=filename)[:1] if results: recipe = results[0] update_recipe_file( tinfoil, config_data_copy, os.path.join(layerdir, filepath), recipe, layerdir_start, repodir, options.stop_on_error, skip_patches) recipe.save() updatedrecipes.add(recipe.full_path()) elif typename == 'machine': results = layermachines.filter(name=filename) if results: machine = results[0] update_machine_conf_file( os.path.join(repodir, path), machine) machine.save() elif typename == 'distro': results = layerdistros.filter(name=filename) if results: distro = results[0] update_distro_conf_file( os.path.join(repodir, path), distro, config_data_copy) distro.save() deps = RecipeFileDependency.objects.filter( layerbranch=layerbranch).filter(path=path) for dep in deps: dirtyrecipes.add(dep.recipe) for recipe in dirtyrecipes: if not recipe.full_path() in updatedrecipes: update_recipe_file( tinfoil, config_data_copy, os.path.join(layerdir, recipe.filepath), recipe, layerdir_start, repodir, options.stop_on_error, skip_patches) else: # Collect recipe data from scratch layerrecipe_fns = [] if options.fullreload: layerrecipes.delete() else: # First, check which recipes still exist layerrecipe_values = layerrecipes.values( 'id', 'filepath', 'filename', 'pn') for v in layerrecipe_values: if v['filepath'].startswith('../'): # FIXME: These recipes were present due to a bug (not handling renames # to paths outside the layer) - this can be removed at some point in the future preserve = False else: root = os.path.join(layerdir, v['filepath']) fullpath = os.path.join(root, v['filename']) if os.path.exists(fullpath): preserve = True for removedir in removedirs: if fullpath.startswith(removedir): preserve = False break else: preserve = False if preserve: # Recipe still exists, update it results = layerrecipes.filter(id=v['id'])[:1] recipe = results[0] update_recipe_file(tinfoil, config_data_copy, root, recipe, layerdir_start, repodir, options.stop_on_error, skip_patches) else: # Recipe no longer exists, mark it for later on layerrecipes_delete.append(v) layerrecipe_fns.append(fullpath) layermachines.delete() layerdistros.delete() layerappends.delete() layerclasses.delete() for root, dirs, files in os.walk(layerdir): if '.git' in dirs: dirs.remove('.git') for diritem in dirs[:]: fullpath = os.path.join(root, diritem) + os.sep if fullpath in removedirs: dirs.remove(diritem) for f in files: fullpath = os.path.join(root, f) (typename, _, filename) = recipeparse.detect_file_type( fullpath, layerdir_start) if typename == 'recipe': if fullpath not in layerrecipe_fns: layerrecipes_add.append(fullpath) elif typename == 'bbappend': append = BBAppend() append.layerbranch = layerbranch append.filename = f append.filepath = os.path.relpath( root, layerdir) append.save() elif typename == 'machine': machine = Machine() machine.layerbranch = layerbranch machine.name = filename update_machine_conf_file(fullpath, machine) machine.save() elif typename == 'distro': distro = Distro() distro.layerbranch = layerbranch distro.name = filename update_distro_conf_file( fullpath, distro, config_data_copy) distro.save() elif typename == 'bbclass': bbclass = BBClass() bbclass.layerbranch = layerbranch bbclass.name = filename bbclass.save() elif typename == 'incfile': incfile = IncFile() incfile.layerbranch = layerbranch incfile.path = os.path.relpath( fullpath, layerdir) incfile.save() for added in layerrecipes_add: # This is good enough without actually parsing the file (pn, pv) = split_recipe_fn(added) oldid = -1 for deleted in layerrecipes_delete: if deleted['pn'] == pn: oldid = deleted['id'] layerrecipes_delete.remove(deleted) break if oldid > -1: # Reclaim a record we would have deleted results = Recipe.objects.filter(id=oldid)[:1] recipe = results[0] logger.debug("Reclaim %s for %s %s" % (recipe, pn, pv)) else: # Create new record logger.debug("Add new recipe %s" % added) recipe = Recipe() recipe.layerbranch = layerbranch recipe.filename = os.path.basename(added) root = os.path.dirname(added) recipe.filepath = os.path.relpath(root, layerdir) update_recipe_file(tinfoil, config_data_copy, root, recipe, layerdir_start, repodir, options.stop_on_error, skip_patches) recipe.save() for deleted in layerrecipes_delete: logger.debug("Delete %s" % deleted) results = Recipe.objects.filter(id=deleted['id'])[:1] recipe = results[0] recipe.delete() # Save repo info layerbranch.vcs_last_rev = topcommit.hexsha layerbranch.vcs_last_commit = datetime.fromtimestamp( topcommit.committed_date) else: logger.info("Layer %s is already up-to-date for branch %s" % (layer.name, branchdesc)) layerbranch.vcs_last_fetch = datetime.now() layerbranch.save() if options.dryrun: raise DryRunRollbackException() except KeyboardInterrupt: logger.warn("Update interrupted, changes to %s rolled back" % layer.name) sys.exit(254) except SystemExit: raise except DryRunRollbackException: pass except: import traceback logger.error(traceback.format_exc().rstrip()) sys.exit(1) finally: if tinfoil and (LooseVersion(bb.__version__) > LooseVersion("1.27")): tinfoil.shutdown() if tempdir: if options.keep_temp: logger.debug('Preserving temp directory %s' % tempdir) else: logger.debug('Deleting temp directory') utils.rmtree_force(tempdir) sys.exit(0)
def train(path='model.npy'): print('Loading data !') train_csv, test_csv = pd.read_csv(dir_train), pd.read_csv(dir_test) raw_data = train_csv.append(test_csv, sort=False) max_, min_ = max(raw_data['y']), min(raw_data['y']) train_data, valid_data, test_data = make_dataset(data_process(raw_data)) REGULARIZATION = regularization / len(train_data) try: net = utils.load_model(path, True) print('Successfully load model !') except: print('Initialize the model parameters !') net = [ utils.get_layer(3, 5, activation=activation, optimizer=optimization, regularization=REGULARIZATION), utils.get_layer(5, 2, 'ReLU', optimizer=optimization, regularization=REGULARIZATION), utils.get_layer(2, 1, 'linear', optimizer=optimization, regularization=REGULARIZATION) ] valid_loss_record = [] test_loss_record = [] for q in range(epoch): batch = utils.batch(train_data, True) train_batch, valid_batch = utils.batch(train_data, True), utils.batch( valid_data, True) print('Training begins !') time_start = time.time() train_loss, valid_loss = 0, 0 for p in range(round): for i in range(len(batch)): net, _ = utils.train([batch[i][0]], [[batch[i][1]]], net, learning_rate=0.01, decay_rate=0.99) train_loss, valid_loss = cal_loss(train_batch, net), cal_loss(valid_batch, net) print('epoch : ' + str(q) + ' train_loss:' + str(train_loss / len(train_batch)) + ' valid_loss:' + str(valid_loss / len(valid_batch))) valid_loss_record.append(train_loss / len(train_batch)) test_loss_record.append(valid_loss / len(valid_batch)) time_end = time.time() print('Train finished in ' + str((time_end - time_start) * 60) + ' seconds !') validation(valid_data, net, max_, min_) plt.plot(valid_loss_record, label='train', color='r') plt.plot(test_loss_record, label='valid', color='b') plt.savefig('Loss.jpg') plt.show() return net
import utils import numpy as np layer = [utils.get_layer(3, 8, activation='ReLU', optimizer='Adamoptimizer', regularization=0.1), utils.get_layer(8, 1, activation='linear',optimizer='Adamoptimizer',regularization=0.1)] batch_x = [[0.5,0.5,0.5]] batch_y = [[0.5]] for i in range(10): layer, total_loss = utils.train(batch_x, batch_y, layer) print(total_loss) #print(layer) utils.save_model(layer, 'model_test.npy') utils.load_model('model_test.npy') utils.forward(batch_x, layer) print(utils.last_layer(layer)['out_'])
def main(): parser = optparse.OptionParser( usage = """ %prog [options] <url> [name]""") parser.add_option("-s", "--subdir", help = "Specify subdirectory", action="store", dest="subdir") parser.add_option("-n", "--dry-run", help = "Don't write any data back to the database", action="store_true", dest="dryrun") parser.add_option("-d", "--debug", help = "Enable debug output", action="store_const", const=logging.DEBUG, dest="loglevel", default=logging.INFO) parser.add_option("", "--github-auth", help = "Specify github username:password", action="store", dest="github_auth") parser.add_option("-q", "--quiet", help = "Hide all output except error messages", action="store_const", const=logging.ERROR, dest="loglevel") options, args = parser.parse_args(sys.argv) if len(args) < 2: print("Please specify URL of repository for layer") sys.exit(1) layer_url = args[1] if len(args) > 2: layer_name = args[2] else: if options.subdir: layer_name = options.subdir else: layer_name = filter(None, layer_url.split('/'))[-1] if layer_name.endswith('.git'): layer_name = layer_name[:-4] if options.github_auth: if not ':' in options.github_auth: logger.error('--github-auth value must be specified as username:password') sys.exit(1) splitval = options.github_auth.split(':') github_login = splitval[0] github_password = splitval[1] else: github_login = None github_password = None utils.setup_django() import settings from layerindex.models import LayerItem, LayerBranch, LayerDependency, LayerMaintainer from django.db import transaction logger.setLevel(options.loglevel) fetchdir = settings.LAYER_FETCH_DIR if not fetchdir: logger.error("Please set LAYER_FETCH_DIR in settings.py") sys.exit(1) if not os.path.exists(fetchdir): os.makedirs(fetchdir) master_branch = utils.get_branch('master') core_layer = None transaction.enter_transaction_management() transaction.managed(True) try: # Fetch layer logger.info('Fetching repository %s' % layer_url) layer = LayerItem() layer.name = layer_name layer.status = 'P' layer.layer_type = 'M' layer.summary = 'tempvalue' layer.description = layer.summary set_vcs_fields(layer, layer_url) urldir = layer.get_fetch_dir() repodir = os.path.join(fetchdir, urldir) out = None try: if not os.path.exists(repodir): out = utils.runcmd("git clone %s %s" % (layer.vcs_url, urldir), fetchdir, logger=logger) else: out = utils.runcmd("git fetch", repodir, logger=logger) except Exception as e: logger.error("Fetch failed: %s" % str(e)) sys.exit(1) actual_branch = '' try: out = utils.runcmd("git checkout origin/master", repodir, logger=logger) except subprocess.CalledProcessError: branches = utils.runcmd("git branch -r", repodir, logger=logger) for line in branches.splitlines(): if 'origin/HEAD ->' in line: actual_branch = line.split('-> origin/')[-1] break if not actual_branch: logger.error("Repository has no master branch nor origin/HEAD") sys.exit(1) out = utils.runcmd("git checkout origin/%s" % actual_branch, repodir, logger=logger) layer_paths = [] if options.subdir: layerdir = os.path.join(repodir, options.subdir) if not os.path.exists(layerdir): logger.error("Subdirectory %s does not exist in repository for master branch" % options.subdir) sys.exit(1) if not os.path.exists(os.path.join(layerdir, 'conf/layer.conf')): logger.error("conf/layer.conf not found in subdirectory %s" % options.subdir) sys.exit(1) layer_paths.append(layerdir) else: if os.path.exists(os.path.join(repodir, 'conf/layer.conf')): layer_paths.append(repodir) # Find subdirs with a conf/layer.conf for subdir in os.listdir(repodir): subdir_path = os.path.join(repodir, subdir) if os.path.isdir(subdir_path): if os.path.exists(os.path.join(subdir_path, 'conf/layer.conf')): layer_paths.append(subdir_path) if not layer_paths: logger.error("conf/layer.conf not found in repository or first level subdirectories - is subdirectory set correctly?") sys.exit(1) if 'github.com' in layer.vcs_url: json_data, owner_json_data = get_github_layerinfo(layer.vcs_url, github_login, github_password) for layerdir in layer_paths: layer.pk = None if layerdir != repodir: subdir = os.path.relpath(layerdir, repodir) if len(layer_paths) > 1: layer.name = subdir else: subdir = '' if LayerItem.objects.filter(name=layer.name).exists(): logger.error('A layer named "%s" already exists in the database' % layer_name) sys.exit(1) logger.info('Creating layer %s' % layer.name) # Guess layer type if glob.glob(os.path.join(layerdir, 'conf/distro/*.conf')): layer.layer_type = 'D' elif glob.glob(os.path.join(layerdir, 'conf/machine/*.conf')): layer.layer_type = 'B' layer.save() layerbranch = LayerBranch() layerbranch.layer = layer layerbranch.branch = master_branch if layerdir != repodir: layerbranch.vcs_subdir = subdir if actual_branch: layerbranch.actual_branch = actual_branch layerbranch.save() if layer.name != settings.CORE_LAYER_NAME: if not core_layer: core_layer = utils.get_layer(settings.CORE_LAYER_NAME) if core_layer: layerdep = LayerDependency() layerdep.layerbranch = layerbranch layerdep.dependency = core_layer layerdep.save() # Get some extra meta-information readme_files = glob.glob(os.path.join(layerdir, 'README*')) if (not readme_files) and subdir: readme_files = glob.glob(os.path.join(repodir, 'README*')) maintainer_files = glob.glob(os.path.join(layerdir, 'MAINTAINERS')) if (not maintainer_files) and subdir: maintainer_files = glob.glob(os.path.join(repodir, 'MAINTAINERS')) maintainers = [] if readme_files: (desc, maintainers, deps) = readme_extract(readme_files[0]) if desc: layer.summary = layer.name layer.description = desc if maintainer_files: maintainers.extend(maintainers_extract(readme_files[0])) if (not maintainers) and 'github.com' in layer.vcs_url: if json_data: layer.summary = json_data['description'] layer.description = layer.summary if owner_json_data: owner_name = owner_json_data.get('name', None) owner_email = owner_json_data.get('email', None) if owner_name and owner_email: maintainers.append('%s <%s>' % (owner_name, owner_email)) if layer.name == 'openembedded-core': layer.summary = 'Core metadata' layer.layer_type = 'A' elif layer.name == 'meta-oe': layer.summary = 'Additional shared OE metadata' layer.description = layer.summary layer.layer_type = 'A' if maintainers: maint_re = re.compile(r'^"?([^"@$<>]+)"? *<([^<> ]+)>[ -]*(.+)?$') for maintentry in maintainers: res = maint_re.match(maintentry) if res: maintainer = LayerMaintainer() maintainer.layerbranch = layerbranch maintainer.name = res.group(1).strip() maintainer.email = res.group(2) if res.group(3): maintainer.responsibility = res.group(3).strip() maintainer.save() layer.save() if options.dryrun: transaction.rollback() else: transaction.commit() except: transaction.rollback() raise finally: transaction.leave_transaction_management() sys.exit(0)
def __init__( self, out_channels: int, exp_channels: int, kernel_size: int, stride: int, use_se: bool, act_layer: str, l2_reg: float = 1e-5, let_res: bool = True, ): super().__init__(name="Bneck") self.out_channels = out_channels self.stride = stride self.use_se = use_se self.let_res = let_res # print('[out_ch] ', out_channels) # print('[exp_ch] ', exp_channels) # Expand self.expand = ConvNormAct( exp_channels, kernel_size=1, norm_layer="bn", act_layer=act_layer, use_bias=False, l2_reg=l2_reg, name="Expand", ) # Depthwise dw_padding = (kernel_size - 1) // 2 self.pad = tf.keras.layers.ZeroPadding2D( padding=dw_padding, name=f"Depthwise/Padding{dw_padding}x{dw_padding}", ) self.depthwise = tf.keras.layers.DepthwiseConv2D( kernel_size=kernel_size, strides=stride, name=f"Depthwise/DWConv{kernel_size}x{kernel_size}", depthwise_regularizer=tf.keras.regularizers.l2(l2_reg), use_bias=False, ) self.bn = BatchNormalization(name="Depthwise/BatchNormalization") if self.use_se: self.se = SEBottleneck( l2_reg=l2_reg, name="Depthwise/SEBottleneck", ) _available_activation = { "relu": tf.keras.layers.ReLU(name="Depthwise/ReLU"), "hswish": HardSwish(name="Depthwise/HardSwish"), } self.act = get_layer(act_layer, _available_activation, Identity()) # Project self.project = ConvNormAct( out_channels, kernel_size=1, norm_layer="bn", act_layer=None, use_bias=False, l2_reg=l2_reg, name="Project", )
def main(): parser = optparse.OptionParser( usage = """ %prog [options]""") options, args = parser.parse_args(sys.argv) utils.setup_django() from layerindex.models import LayerItem, LayerBranch, LayerDependency from django.db import transaction import httplib conn = httplib.HTTPConnection("www.openembedded.org") conn.request("GET", "/wiki/LayerIndex?action=raw") resp = conn.getresponse() if resp.status in [200, 302]: data = resp.read() in_table = False layer_type = 'M' nowiki_re = re.compile(r'</?nowiki>') link_re = re.compile(r'\[(http.*) +link\]') readme_re = re.compile(r';f=[a-zA-Z0-9/-]*README;') master_branch = utils.get_branch('master') core_layer = None with transaction.atomic(): for line in data.splitlines(): if line.startswith('{|'): in_table = True continue if in_table: if line.startswith('|}'): # We're done break elif line.startswith('!'): section = line.split('|', 1)[1].strip("'") if section.startswith('Base'): layer_type = 'A' elif section.startswith('Board'): layer_type = 'B' elif section.startswith('Software'): layer_type = 'S' elif section.startswith('Distribution'): layer_type = 'D' else: layer_type = 'M' elif not line.startswith('|-'): if line.startswith("|| ''"): continue fields = line.split('||') layer = LayerItem() layer.name = fields[1].strip() if ' ' in layer.name: logger.warn('Skipping layer %s - name invalid' % layer.name) continue logger.info('Adding layer %s' % layer.name) layer.status = 'P' layer.layer_type = layer_type layer.summary = fields[2].strip() layer.description = layer.summary if len(fields) > 6: res = link_re.match(fields[6].strip()) if res: link = res.groups(1)[0].strip() if link.endswith('/README') or readme_re.search(link): link = 'README' layer.usage_url = link repoval = nowiki_re.sub('', fields[4]).strip() layer.vcs_url = repoval if repoval.startswith('git://git.openembedded.org/'): reponame = re.sub('^.*/', '', repoval) layer.vcs_web_url = 'http://cgit.openembedded.org/' + reponame layer.vcs_web_tree_base_url = 'http://cgit.openembedded.org/' + reponame + '/tree/%path%?h=%branch%' layer.vcs_web_file_base_url = 'http://cgit.openembedded.org/' + reponame + '/tree/%path%?h=%branch%' layer.vcs_web_commit_url = 'http://cgit.openembedded.org/' + reponame + '/commit/?id=%hash%' elif repoval.startswith('git://git.yoctoproject.org/'): reponame = re.sub('^.*/', '', repoval) layer.vcs_web_url = 'http://git.yoctoproject.org/cgit/cgit.cgi/' + reponame layer.vcs_web_tree_base_url = 'http://git.yoctoproject.org/cgit/cgit.cgi/' + reponame + '/tree/%path%?h=%branch%' layer.vcs_web_file_base_url = 'http://git.yoctoproject.org/cgit/cgit.cgi/' + reponame + '/tree/%path%?h=%branch%' layer.vcs_web_commit_url = 'http://git.yoctoproject.org/cgit/cgit.cgi/' + reponame + '/commit/?id=%hash%' elif repoval.startswith('git://github.com/') or repoval.startswith('http://github.com/') or repoval.startswith('https://github.com/'): reponame = re.sub('^.*github.com/', '', repoval) reponame = re.sub('.git$', '', reponame) layer.vcs_web_url = 'http://github.com/' + reponame layer.vcs_web_tree_base_url = 'http://github.com/' + reponame + '/tree/%branch%/' layer.vcs_web_file_base_url = 'http://github.com/' + reponame + '/blob/%branch%/' layer.vcs_web_commit_url = 'http://github.com/' + reponame + '/commit/%hash%' elif repoval.startswith('git://gitlab.com/') or repoval.startswith('http://gitlab.com/') or repoval.startswith('https://gitlab.com/'): reponame = re.sub('^.*gitlab.com/', '', repoval) reponame = re.sub('.git$', '', reponame) layer.vcs_web_url = 'http://gitlab.com/' + reponame layer.vcs_web_tree_base_url = 'http://gitlab.com/' + reponame + '/tree/%branch%/' layer.vcs_web_file_base_url = 'http://gitlab.com/' + reponame + '/blob/%branch%/' layer.vcs_web_commit_url = 'http://gitlab.com/' + reponame + '/commit/%hash%' elif repoval.startswith('git://bitbucket.org/') or repoval.startswith('http://bitbucket.org/') or repoval.startswith('https://bitbucket.org/'): reponame = re.sub('^.*bitbucket.org/', '', repoval) reponame = re.sub('.git$', '', reponame) layer.vcs_web_url = 'http://bitbucket.org/' + reponame layer.vcs_web_tree_base_url = 'http://bitbucket.org/' + reponame + '/src/%branch%/%path%?at=%branch%' layer.vcs_web_file_base_url = 'http://bitbucket.org/' + reponame + '/src/%branch%/%path%?at=%branch%' layer.vcs_web_commit_url = 'http://bitbucket.org/' + reponame + '/commits/%hash%' elif '.git' in repoval: res = link_re.match(fields[5].strip()) layer.vcs_web_url = res.groups(1)[0] layer.vcs_web_tree_base_url = re.sub(r'\.git.*', '.git;a=tree;f=%path%;hb=%branch%', layer.vcs_web_url) layer.vcs_web_file_base_url = re.sub(r'\.git.*', '.git;a=blob;f=%path%;hb=%branch%', layer.vcs_web_url) layer.vcs_web_file_base_url = re.sub(r'\.git.*', '.git;a=commit;h=%hash%', layer.vcs_web_url) layer.save() layerbranch = LayerBranch() layerbranch.layer = layer layerbranch.branch = master_branch layerbranch.vcs_subdir = fields[3].strip() layerbranch.save() if layer.name != 'openembedded-core': if not core_layer: core_layer = utils.get_layer('openembedded-core') if core_layer: layerdep = LayerDependency() layerdep.layerbranch = layerbranch layerdep.dependency = core_layer layerdep.save() else: logger.error('Fetch failed: %d: %s' % (resp.status, resp.reason)) sys.exit(0)
def build_sampler(tparams, options, trng): x = tensor.matrix('x', dtype='int64') xr = x[::-1] n_timesteps = x.shape[0] n_samples = x.shape[1] # word embedding (source) emb = tparams['Wemb'][x.flatten()] emb = emb.reshape([n_timesteps, n_samples, options['dim_word']]) embr = tparams['Wemb'][xr.flatten()] embr = embr.reshape([n_timesteps, n_samples, options['dim_word']]) # encoder proj = get_layer(options['encoder'])[1](tparams, emb, options, prefix='encoder') projr = get_layer(options['encoder'])[1](tparams, embr, options, prefix='encoder_r') ctx = concatenate([proj[0], projr[0][::-1]], axis=proj[0].ndim-1) ctx_mean = ctx.mean(0) init_state = get_layer('ff')[1](tparams, ctx_mean, options, prefix='ff_state', activ='tanh') print 'Building f_init...', outs = [init_state, ctx] f_init = theano.function([x], outs, name='f_init', profile=profile) print 'Done' # x: 1 x 1 y = tensor.vector('y_sampler', dtype='int64') init_state = tensor.matrix('init_state', dtype='float32') # if it's the first word, emb should be all zero emb = tensor.switch(y[:, None] < 0, tensor.alloc(0., 1, tparams['Wemb_dec'].shape[1]), tparams['Wemb_dec'][y]) proj = get_layer(options['decoder'])[1](tparams, emb, options, prefix='decoder', mask=None, context=ctx, one_step=True, init_state=init_state) next_state = proj[0] ctxs = proj[1] logit_lstm = get_layer('ff')[1](tparams, next_state, options, prefix='ff_logit_lstm', activ='linear') logit_prev = get_layer('ff')[1](tparams, emb, options, prefix='ff_logit_prev', activ='linear') logit_ctx = get_layer('ff')[1](tparams, ctxs, options, prefix='ff_logit_ctx', activ='linear') logit = tensor.tanh(logit_lstm + logit_prev + logit_ctx) logit = get_layer('ff')[1](tparams, logit, options, prefix='ff_logit', activ='linear') next_probs = tensor.nnet.softmax(logit) next_sample = trng.multinomial(pvals=next_probs).argmax(1) # next word probability print 'Building f_next..', inps = [y, ctx, init_state] outs = [next_probs, next_sample, next_state] f_next = theano.function(inps, outs, name='f_next', profile=profile) print 'Done' return f_init, f_next
def build_model(tparams, options): opt_ret = dict() trng = RandomStreams(1234) use_noise = theano.shared(numpy.float32(0.)) # description string: #words x #samples x = tensor.matrix('x', dtype='int64') x_mask = tensor.matrix('x_mask', dtype='float32') y = tensor.matrix('y', dtype='int64') y_mask = tensor.matrix('y_mask', dtype='float32') xr = x[::-1] xr_mask = x_mask[::-1] n_timesteps = x.shape[0] n_timesteps_trg = y.shape[0] n_samples = x.shape[1] emb = tparams['Wemb'][x.flatten()] emb = emb.reshape([n_timesteps, n_samples, options['dim_word']]) proj = get_layer(options['encoder'])[1](tparams, emb, options, prefix='encoder', mask=x_mask) embr = tparams['Wemb'][xr.flatten()] embr = embr.reshape([n_timesteps, n_samples, options['dim_word']]) projr = get_layer(options['encoder'])[1](tparams, embr, options, prefix='encoder_r', mask=xr_mask) ctx = concatenate([proj[0], projr[0][::-1]], axis=proj[0].ndim-1) ctx_mean = (ctx * x_mask[:, :, None]).sum(0) / x_mask.sum(0)[:,None] # initial decoder state init_state = get_layer('ff')[1](tparams, ctx_mean, options, prefix='ff_state', activ='tanh') # word embedding (target) emb = tparams['Wemb_dec'][y.flatten()] emb = emb.reshape([n_timesteps_trg, n_samples, options['dim_word']]) emb_shifted = tensor.zeros_like(emb) emb_shifted = tensor.set_subtensor(emb_shifted[1:], emb[:-1]) emb = emb_shifted # decoder proj = get_layer(options['decoder'])[1](tparams, emb, options, prefix='decoder', mask=y_mask, context=ctx, context_mask=x_mask, one_step=False, init_state=init_state) proj_h = proj[0] ctxs = proj[1] opt_ret['dec_alphas'] = proj[2] # compute word probabilities logit_lstm = get_layer('ff')[1](tparams, proj_h, options, prefix='ff_logit_lstm', activ='linear') logit_prev = get_layer('ff')[1](tparams, emb, options, prefix='ff_logit_prev', activ='linear') logit_ctx = get_layer('ff')[1](tparams, ctxs, options, prefix='ff_logit_ctx', activ='linear') logit = tensor.tanh(logit_lstm+logit_prev+logit_ctx) logit = get_layer('ff')[1](tparams, logit, options, prefix='ff_logit', activ='linear') logit_shp = logit.shape probs = tensor.nnet.softmax(logit.reshape([logit_shp[0]*logit_shp[1], logit_shp[2]])) # cost y_flat = y.flatten() y_flat_idx = tensor.arange(y_flat.shape[0]) * options['n_words'] + y_flat cost = -tensor.log(probs.flatten()[y_flat_idx]) cost = cost.reshape([y.shape[0], y.shape[1]]) cost = (cost * y_mask).sum(0) return trng, use_noise, x, x_mask, y, y_mask, opt_ret, cost
def build_sampler(tparams, options, trng): x = tensor.matrix('x', dtype='int64') xr = x[::-1] n_timesteps = x.shape[0] n_samples = x.shape[1] # word embedding (source) emb = tparams['Wemb'][x.flatten()] emb = emb.reshape([n_timesteps, n_samples, options['dim_word']]) embr = tparams['Wemb'][xr.flatten()] embr = embr.reshape([n_timesteps, n_samples, options['dim_word']]) # encoder proj = get_layer(options['encoder'])[1](tparams, emb, options, prefix='encoder') projr = get_layer(options['encoder'])[1](tparams, embr, options, prefix='encoder_r') ctx = concatenate([proj[0], projr[0][::-1]], axis=proj[0].ndim - 1) ctx_mean = ctx.mean(0) init_state = get_layer('ff')[1](tparams, ctx_mean, options, prefix='ff_state', activ='tanh') print 'Building f_init...', outs = [init_state, ctx] f_init = theano.function([x], outs, name='f_init', profile=profile) print 'Done' # x: 1 x 1 y = tensor.vector('y_sampler', dtype='int64') init_state = tensor.matrix('init_state', dtype='float32') # if it's the first word, emb should be all zero emb = tensor.switch(y[:, None] < 0, tensor.alloc(0., 1, tparams['Wemb_dec'].shape[1]), tparams['Wemb_dec'][y]) proj = get_layer(options['decoder'])[1](tparams, emb, options, prefix='decoder', mask=None, context=ctx, one_step=True, init_state=init_state) next_state = proj[0] ctxs = proj[1] logit_lstm = get_layer('ff')[1](tparams, next_state, options, prefix='ff_logit_lstm', activ='linear') logit_prev = get_layer('ff')[1](tparams, emb, options, prefix='ff_logit_prev', activ='linear') logit_ctx = get_layer('ff')[1](tparams, ctxs, options, prefix='ff_logit_ctx', activ='linear') logit = tensor.tanh(logit_lstm + logit_prev + logit_ctx) logit = get_layer('ff')[1](tparams, logit, options, prefix='ff_logit', activ='linear') next_probs = tensor.nnet.softmax(logit) next_sample = trng.multinomial(pvals=next_probs).argmax(1) # next word probability print 'Building f_next..', inps = [y, ctx, init_state] outs = [next_probs, next_sample, next_state] f_next = theano.function(inps, outs, name='f_next', profile=profile) print 'Done' return f_init, f_next
m = np.mean(y) data = [] for i in range(5000): x1[i] = (x1[i] - min(x1)) / (max(x1) - min(x1)) x2[i] = (x2[i] - min(x2)) / (max(x2) - min(x2)) x3[i] = (x3[i] - min(x3)) / (max(x3) - min(x3)) y[i] = [1, 0] if y[i] > 1.5 else [0, 1] data.append([[x1[i], x2[i], x3[i]], y[i]]) for i in range(5000): print(x1[i], x2[i], x3[i], y[i]) net = [ utils.get_layer(3, 8, activation='ReLU', optimizer='Adamoptimizer', regularization=0.01), utils.get_layer(8, 2, activation='softmax', optimizer='Adamoptimizer', regularization=0.01) ] epoch = 16 round = 1 batch_size = 1000 print('traning begins !') for p in range(epoch): print('epoch ' + str(p) + ' begins !')
def build_model(tparams, options): opt_ret = dict() trng = RandomStreams(1234) use_noise = theano.shared(numpy.float32(0.)) # description string: #words x #samples x = tensor.matrix('x', dtype='int64') x_mask = tensor.matrix('x_mask', dtype='float32') y = tensor.matrix('y', dtype='int64') y_mask = tensor.matrix('y_mask', dtype='float32') xr = x[::-1] xr_mask = x_mask[::-1] n_timesteps = x.shape[0] n_timesteps_trg = y.shape[0] n_samples = x.shape[1] emb = tparams['Wemb'][x.flatten()] emb = emb.reshape([n_timesteps, n_samples, options['dim_word']]) proj = get_layer(options['encoder'])[1](tparams, emb, options, prefix='encoder', mask=x_mask) embr = tparams['Wemb'][xr.flatten()] embr = embr.reshape([n_timesteps, n_samples, options['dim_word']]) projr = get_layer(options['encoder'])[1](tparams, embr, options, prefix='encoder_r', mask=xr_mask) ctx = concatenate([proj[0], projr[0][::-1]], axis=proj[0].ndim - 1) ctx_mean = (ctx * x_mask[:, :, None]).sum(0) / x_mask.sum(0)[:, None] # initial decoder state init_state = get_layer('ff')[1](tparams, ctx_mean, options, prefix='ff_state', activ='tanh') # word embedding (target) emb = tparams['Wemb_dec'][y.flatten()] emb = emb.reshape([n_timesteps_trg, n_samples, options['dim_word']]) emb_shifted = tensor.zeros_like(emb) emb_shifted = tensor.set_subtensor(emb_shifted[1:], emb[:-1]) emb = emb_shifted # decoder proj = get_layer(options['decoder'])[1](tparams, emb, options, prefix='decoder', mask=y_mask, context=ctx, context_mask=x_mask, one_step=False, init_state=init_state) proj_h = proj[0] ctxs = proj[1] opt_ret['dec_alphas'] = proj[2] # compute word probabilities logit_lstm = get_layer('ff')[1](tparams, proj_h, options, prefix='ff_logit_lstm', activ='linear') logit_prev = get_layer('ff')[1](tparams, emb, options, prefix='ff_logit_prev', activ='linear') logit_ctx = get_layer('ff')[1](tparams, ctxs, options, prefix='ff_logit_ctx', activ='linear') logit = tensor.tanh(logit_lstm + logit_prev + logit_ctx) logit = get_layer('ff')[1](tparams, logit, options, prefix='ff_logit', activ='linear') logit_shp = logit.shape probs = tensor.nnet.softmax( logit.reshape([logit_shp[0] * logit_shp[1], logit_shp[2]])) # cost y_flat = y.flatten() y_flat_idx = tensor.arange(y_flat.shape[0]) * options['n_words'] + y_flat cost = -tensor.log(probs.flatten()[y_flat_idx]) cost = cost.reshape([y.shape[0], y.shape[1]]) cost = (cost * y_mask).sum(0) return trng, use_noise, x, x_mask, y, y_mask, opt_ret, cost
def init_parser(settings, branch, bitbakepath, enable_tracking=False, nocheckout=False, classic=False, logger=None): if not (nocheckout or classic): # Check out the branch of BitBake appropriate for this branch and clean out any stale files (e.g. *.pyc) if re.match('[0-9a-f]{40}', branch.bitbake_branch): # SHA1 hash bitbake_ref = branch.bitbake_branch else: # Branch name bitbake_ref = 'origin/%s' % branch.bitbake_branch utils.checkout_repo(bitbakepath, bitbake_ref, logger=logger) # Skip sanity checks os.environ['BB_ENV_EXTRAWHITE'] = 'DISABLE_SANITY_CHECKS' os.environ['DISABLE_SANITY_CHECKS'] = '1' fetchdir = settings.LAYER_FETCH_DIR if not classic: # Ensure we have OE-Core set up to get some base configuration core_layer = utils.get_layer(settings.CORE_LAYER_NAME) if not core_layer: raise RecipeParseError("Unable to find core layer %s in database; create this layer or set the CORE_LAYER_NAME setting to point to the core layer" % settings.CORE_LAYER_NAME) core_layerbranch = core_layer.get_layerbranch(branch.name) core_branchname = branch.name if core_layerbranch: core_subdir = core_layerbranch.vcs_subdir if core_layerbranch.actual_branch: core_branchname = core_layerbranch.actual_branch else: core_subdir = 'meta' core_urldir = core_layer.get_fetch_dir() core_repodir = os.path.join(fetchdir, core_urldir) core_layerdir = os.path.join(core_repodir, core_subdir) if not nocheckout: utils.checkout_repo(core_repodir, "origin/%s" % core_branchname, logger=logger) if not os.path.exists(os.path.join(core_layerdir, 'conf/bitbake.conf')): raise RecipeParseError("conf/bitbake.conf not found in core layer %s - is subdirectory set correctly?" % core_layer.name) # The directory above where this script exists should contain our conf/layer.conf, # so add it to BBPATH along with the core layer directory confparentdir = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..')) os.environ['BBPATH'] = str("%s:%s" % (confparentdir, core_layerdir)) # Change into a temporary directory so we don't write the cache and other files to the current dir if not os.path.exists(settings.TEMP_BASE_DIR): os.makedirs(settings.TEMP_BASE_DIR) tempdir = tempfile.mkdtemp(dir=settings.TEMP_BASE_DIR) saved_cwd = os.getcwd() os.chdir(tempdir) # We need to create a dummy bblayers.conf to avoid bitbake-cookerdaemon.log being created in <oecore>/meta/ # (see findTopdir() in bitbake/lib/bb/cookerdata.py) os.mkdir(os.path.join(tempdir, 'conf')) with open(os.path.join(tempdir, 'conf', 'bblayers.conf'), 'w') as f: pass if logger: tinfoil = utils.setup_tinfoil(bitbakepath, enable_tracking, loglevel=logger.getEffectiveLevel()) else: tinfoil = utils.setup_tinfoil(bitbakepath, enable_tracking) os.chdir(saved_cwd) # Ensure TMPDIR exists (or insane.bbclass will blow up trying to write to the QA log) oe_tmpdir = tinfoil.config_data.getVar('TMPDIR', True) if not os.path.exists(oe_tmpdir): os.makedirs(oe_tmpdir) # Ensure BBFILES as an initial value so that the old mode of BBFILES := "${BBFILES} ..." works if not tinfoil.config_data.getVar('BBFILES', False): tinfoil.config_data.setVar('BBFILES', '') return (tinfoil, tempdir)
def main(): parser = optparse.OptionParser( usage = """ %prog [options]""") options, args = parser.parse_args(sys.argv) utils.setup_django() from layerindex.models import LayerItem, LayerBranch, LayerDependency from django.db import transaction import httplib conn = httplib.HTTPConnection("www.openembedded.org") conn.request("GET", "/wiki/LayerIndex?action=raw") resp = conn.getresponse() if resp.status in [200, 302]: data = resp.read() in_table = False layer_type = 'M' nowiki_re = re.compile(r'</?nowiki>') link_re = re.compile(r'\[(http.*) +link\]') readme_re = re.compile(r';f=[a-zA-Z0-9/-]*README;') master_branch = utils.get_branch('master') core_layer = None transaction.enter_transaction_management() transaction.managed(True) try: for line in data.splitlines(): if line.startswith('{|'): in_table = True continue if in_table: if line.startswith('|}'): # We're done break elif line.startswith('!'): section = line.split('|', 1)[1].strip("'") if section.startswith('Base'): layer_type = 'A' elif section.startswith('Board'): layer_type = 'B' elif section.startswith('Software'): layer_type = 'S' elif section.startswith('Distribution'): layer_type = 'D' else: layer_type = 'M' elif not line.startswith('|-'): if line.startswith("|| ''"): continue fields = line.split('||') layer = LayerItem() layer.name = fields[1].strip() if ' ' in layer.name: logger.warn('Skipping layer %s - name invalid' % layer.name) continue logger.info('Adding layer %s' % layer.name) layer.status = 'P' layer.layer_type = layer_type layer.summary = fields[2].strip() layer.description = layer.summary if len(fields) > 6: res = link_re.match(fields[6].strip()) if res: link = res.groups(1)[0].strip() if link.endswith('/README') or readme_re.search(link): link = 'README' layer.usage_url = link repoval = nowiki_re.sub('', fields[4]).strip() layer.vcs_url = repoval if repoval.startswith('git://git.openembedded.org/'): reponame = re.sub('^.*/', '', repoval) layer.vcs_web_url = 'http://cgit.openembedded.org/cgit.cgi/' + reponame layer.vcs_web_tree_base_url = 'http://cgit.openembedded.org/cgit.cgi/' + reponame + '/tree/%path%?h=%branch%' layer.vcs_web_file_base_url = 'http://cgit.openembedded.org/cgit.cgi/' + reponame + '/tree/%path%?h=%branch%' elif 'git.yoctoproject.org/' in repoval: reponame = re.sub('^.*/', '', repoval) layer.vcs_web_url = 'http://git.yoctoproject.org/cgit/cgit.cgi/' + reponame layer.vcs_web_tree_base_url = 'http://git.yoctoproject.org/cgit/cgit.cgi/' + reponame + '/tree/%path%?h=%branch%' layer.vcs_web_file_base_url = 'http://git.yoctoproject.org/cgit/cgit.cgi/' + reponame + '/tree/%path%?h=%branch%' elif 'github.com/' in repoval: reponame = re.sub('^.*github.com/', '', repoval) reponame = re.sub('.git$', '', reponame) layer.vcs_web_url = 'http://github.com/' + reponame layer.vcs_web_tree_base_url = 'http://github.com/' + reponame + '/tree/%branch%/' layer.vcs_web_file_base_url = 'http://github.com/' + reponame + '/blob/%branch%/' elif 'gitorious.org/' in repoval: reponame = re.sub('^.*gitorious.org/', '', repoval) reponame = re.sub('.git$', '', reponame) layer.vcs_web_url = 'http://gitorious.org/' + reponame layer.vcs_web_tree_base_url = 'http://gitorious.org/' + reponame + '/trees/%branch%/' layer.vcs_web_file_base_url = 'http://gitorious.org/' + reponame + '/blobs/%branch%/' elif 'bitbucket.org/' in repoval: reponame = re.sub('^.*bitbucket.org/', '', repoval) reponame = re.sub('.git$', '', reponame) layer.vcs_web_url = 'http://bitbucket.org/' + reponame layer.vcs_web_tree_base_url = 'http://bitbucket.org/' + reponame + '/src/%branch%/%path%?at=%branch%' layer.vcs_web_file_base_url = 'http://bitbucket.org/' + reponame + '/src/%branch%/%path%?at=%branch%' elif '.git' in repoval: res = link_re.match(fields[5].strip()) layer.vcs_web_url = res.groups(1)[0] layer.vcs_web_tree_base_url = re.sub(r'\.git.*', '.git;a=tree;f=%path%;hb=%branch%', layer.vcs_web_url) layer.vcs_web_file_base_url = re.sub(r'\.git.*', '.git;a=blob;f=%path%;hb=%branch%', layer.vcs_web_url) layer.save() layerbranch = LayerBranch() layerbranch.layer = layer layerbranch.branch = master_branch layerbranch.vcs_subdir = fields[3].strip() layerbranch.save() if layer.name != 'openembedded-core': if not core_layer: core_layer = utils.get_layer('openembedded-core') if core_layer: layerdep = LayerDependency() layerdep.layerbranch = layerbranch layerdep.dependency = core_layer layerdep.save() transaction.commit() except: transaction.rollback() raise finally: transaction.leave_transaction_management() else: logger.error('Fetch failed: %d: %s' % (resp.status, resp.reason)) sys.exit(0)