def rrs_remove_duplicates(args):
utils.setup_django()
import settings
from django.db import transaction
from rrs.models import RecipeUpstreamHistory
from layerindex.models import Recipe
core_layer = utils.get_layer(settings.CORE_LAYER_NAME)
if not core_layer:
logger.error('Unable to find core layer %s' % settings.CORE_LAYER_NAME)
return 1
core_layerbranch = core_layer.get_layerbranch('master')
if not core_layerbranch:
logger.error('Unable to find branch master of layer %s' % core_layerbranch.name)
return 1
try:
with transaction.atomic():
for row in RecipeUpstreamHistory.objects.filter(layerbranch=core_layerbranch).order_by('-id'):
if RecipeUpstreamHistory.objects.filter(layerbranch=row.layerbranch, start_date=row.start_date).count() > 1:
logger.info('Deleting duplicate %d' % row.id)
row.delete()
if args.dry_run:
raise DryRunRollbackException
except DryRunRollbackException:
pass
return 0
def _forward(self):
"""
Helper to create the forward pass in the graph
:return:
"""
x = self.x
# Reshape input
x = tf.reshape(x, (-1, self.input_size, self.input_size, 1))
# List of length maintained during the building because some
# dimensions can't be inferred at build time, such as the Dense layer size
length = [self.input_size, self.input_size, 1]
# Implement dropout every three layers
nb_dropout_layers = len(self.layers) // 3
# Iterate over all built State
for index in range(len(self.layers)):
prev_layer = None
# Pass the previous shape (needed for reshaping)
if index > 0:
prev_layer = self.layers[index - 1]
if index % 3 == 0 and index > 0:
keep_prob = 1 - (index // 3) / (2 * nb_dropout_layers)
x = layers.dropout(inputs=x,
is_training=self.is_training,
keep_prob=keep_prob)
x, length = utils.get_layer(x, self.layers[index], self.nb_targets,
length, prev_layer)
# No activation, at the last layer
if index != len(self.layers) - 1:
x = self.activation(x)
return x
def __init__(
self,
filters: int,
kernel_size: int = 3,
stride: int = 1,
padding: int = 0,
norm_layer: str = None,
act_layer: str = "relu",
use_bias: bool = True,
l2_reg: float = 1e-5,
name: str = "ConvNormAct",
):
super().__init__(name=name)
if padding > 0:
self.pad = tf.keras.layers.ZeroPadding2D(
padding=padding,
name=f"Padding{padding}x{padding}",
)
else:
self.pad = Identity()
self.conv = tf.keras.layers.Conv2D(
filters=filters,
kernel_size=kernel_size,
strides=stride,
name=f"Conv{kernel_size}x{kernel_size}",
kernel_regularizer=tf.keras.regularizers.l2(l2_reg),
use_bias=use_bias,
)
_available_normalization = {
"bn": BatchNormalization(),
}
self.norm = get_layer(norm_layer, _available_normalization, Identity())
_available_activation = {
"relu": tf.keras.layers.ReLU(name="ReLU"),
"relu6": ReLU6(),
"hswish": HardSwish(),
"hsigmoid": HardSigmoid(),
"sigmoid": tf.keras.layers.Activation('sigmoid', name='sigmoid'),
"softmax": tf.keras.layers.Softmax(name="Softmax"),
}
self.act = get_layer(act_layer, _available_activation, Identity())
def do_layer_selection_changed(self):
"""
Takes the names of currently selected stations with water measurements
and display them in the combobox
"""
# Can't use activeLayer() cause if I have stations selected,
# and select another layer it creates an error, when zooming to the stations
lay = utils.get_layer("CurrentW", self.i_face)
self.c_box.clear()
for feat in lay.selectedFeatures():
self.c_box.addItem(feat['shortname'])
def rrs_import(args):
utils.setup_django()
import settings
from django.db import transaction
from rrs.models import RecipeUpstreamHistory, RecipeUpstream
from layerindex.models import Recipe
core_layer = utils.get_layer(settings.CORE_LAYER_NAME)
if not core_layer:
logger.error('Unable to find core layer %s' % settings.CORE_LAYER_NAME)
return 1
core_layerbranch = core_layer.get_layerbranch('master')
if not core_layerbranch:
logger.error('Unable to find branch master of layer %s' % core_layerbranch.name)
return 1
layerbranch = core_layerbranch
try:
with transaction.atomic():
with open(args.infile, 'r') as f:
data = json.load(f)
for item, itemdata in data.items():
if item == 'recipeupstreamhistory':
for histdata in itemdata:
ruh = RecipeUpstreamHistory()
ruh.start_date = histdata['start_date']
ruh.end_date = histdata['end_date']
ruh.layerbranch = layerbranch
ruh.save()
for upstreamdata in histdata['upstreams']:
ru = RecipeUpstream()
ru.history = ruh
pn = upstreamdata['recipe']
recipe = Recipe.objects.filter(layerbranch=layerbranch, pn=pn).first()
if not recipe:
logger.warning('Could not find recipe %s in layerbranch %s' % (pn, layerbranch))
continue
ru.recipe = recipe
ru.version = upstreamdata['version']
ru.type = upstreamdata['type']
ru.status = upstreamdata['status']
ru.no_update_reason = upstreamdata['no_update_reason']
ru.date = upstreamdata['date']
ru.save()
if args.dry_run:
raise DryRunRollbackException
except DryRunRollbackException:
pass
return 0
def init_params(options):
params = OrderedDict()
# embedding
params['Wemb'] = norm_weight(options['n_words_src'], options['dim_word'])
params['Wemb_dec'] = norm_weight(options['n_words'], options['dim_word'])
# encoder: bidirectional RNN
params = get_layer(options['encoder'])[0](options, params, prefix='encoder',
nin=options['dim_word'],
dim=options['dim'])
params = get_layer(options['encoder'])[0](options, params, prefix='encoder_r',
nin=options['dim_word'],
dim=options['dim'])
ctxdim = 2 * options['dim']
# init_state, init_cell
params = get_layer('ff')[0](options, params, prefix='ff_state',
nin=ctxdim, nout=options['dim'])
# decoder
params = get_layer(options['decoder'])[0](options, params, prefix='decoder',
nin=options['dim_word'],
dim=options['dim'],
dimctx=ctxdim)
# readout
params = get_layer('ff')[0](options, params, prefix='ff_logit_lstm',
nin=options['dim'], nout=options['dim_word'],
ortho=False)
params = get_layer('ff')[0](options, params, prefix='ff_logit_prev',
nin=options['dim_word'], nout=options['dim_word'],
ortho=False)
params = get_layer('ff')[0](options, params, prefix='ff_logit_ctx',
nin=ctxdim, nout=options['dim_word'],
ortho=False)
params = get_layer('ff')[0](options, params, prefix='ff_logit',
nin=options['dim_word'], nout=options['n_words'])
return params
def timeit(self, args):
print_action = []
get_layer = lambda x: utils.get_layer(x, args.layer) # consume argument layer
# consume argument action
if settings.TIME_ACTION_SEPARATELY == args.mode:
cal = CategoricalTimer(settings.TIMER_CATEGORY_PATTERN, settings.TIMER_CATEGORY_KEY)
elif settings.TIME_ACTION_TOGETHER == args.mode:
cal = PatternTimer(settings.TIMER_ANYTEXT_PATTERN)
else:
raise ValueError
log_path = utils.get_log_path(settings.LOG_DURATION_TYPE_NAME, args.target)
fd = open(log_path, 'w')
cal_out = lambda x: cal.measure(x).output_duration(fd)
# consumte arguments print_option
if settings.TIME_PRINT_OPTION_NOT_EMPTY in args.print_option:
print_action.append(cal_out)
if settings.TIME_PRINT_OPTION_VALID in args.print_option:
censor = TextCensor(settings.CENSOR_RULES)
print_action.append(lambda x: cal_out(censor.validate(x).qualified))
if settings.TIME_PRINT_OPTION_TOTAL in args.print_option:
all_cal = OverallTimer(cal)
print_action.append(lambda x: all_cal.measure(x))
for src_file, _ in utils.traverser(args.target, "", settings.LOOKUP_NAMES_PATTERN):
intervals = get_layer(self.tbp.read(src_file).parse_blocks())
fd.write(">>" + src_file.decode(settings.DECODING).encode(settings.ENCODING) + ':' + os.linesep)
for action in print_action:
action(intervals)
if "all_cal" in locals().keys():
fd.write(">>" + args.target.decode(settings.DECODING).encode(settings.ENCODING) + ' ')
all_cal.output_duration(fd)
fd.close()
def init_parser(settings, branch, bitbakepath, enable_tracking=False, nocheckout=False, classic=False, logger=None):
if not (nocheckout or classic):
# Check out the branch of BitBake appropriate for this branch and clean out any stale files (e.g. *.pyc)
if re.match("[0-9a-f]{40}", branch.bitbake_branch):
# SHA1 hash
bitbake_ref = branch.bitbake_branch
else:
# Branch name
bitbake_ref = "origin/%s" % branch.bitbake_branch
out = utils.runcmd("git checkout %s" % bitbake_ref, bitbakepath, logger=logger)
out = utils.runcmd("git clean -f -x", bitbakepath, logger=logger)
# Skip sanity checks
os.environ["BB_ENV_EXTRAWHITE"] = "DISABLE_SANITY_CHECKS"
os.environ["DISABLE_SANITY_CHECKS"] = "1"
fetchdir = settings.LAYER_FETCH_DIR
if not classic:
# Ensure we have OE-Core set up to get some base configuration
core_layer = utils.get_layer(settings.CORE_LAYER_NAME)
if not core_layer:
raise RecipeParseError(
"Unable to find core layer %s in database; create this layer or set the CORE_LAYER_NAME setting to point to the core layer"
% settings.CORE_LAYER_NAME
)
core_layerbranch = core_layer.get_layerbranch(branch.name)
core_branchname = branch.name
if core_layerbranch:
core_subdir = core_layerbranch.vcs_subdir
if core_layerbranch.actual_branch:
core_branchname = core_layerbranch.actual_branch
else:
core_subdir = "meta"
core_urldir = core_layer.get_fetch_dir()
core_repodir = os.path.join(fetchdir, core_urldir)
core_layerdir = os.path.join(core_repodir, core_subdir)
if not nocheckout:
out = utils.runcmd("git checkout origin/%s" % core_branchname, core_repodir, logger=logger)
out = utils.runcmd("git clean -f -x", core_repodir, logger=logger)
if not os.path.exists(os.path.join(core_layerdir, "conf/bitbake.conf")):
raise RecipeParseError(
"conf/bitbake.conf not found in core layer %s - is subdirectory set correctly?" % core_layer.name
)
# The directory above where this script exists should contain our conf/layer.conf,
# so add it to BBPATH along with the core layer directory
confparentdir = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), ".."))
os.environ["BBPATH"] = str("%s:%s" % (confparentdir, core_layerdir))
# Change into a temporary directory so we don't write the cache and other files to the current dir
if not os.path.exists(settings.TEMP_BASE_DIR):
os.makedirs(settings.TEMP_BASE_DIR)
tempdir = tempfile.mkdtemp(dir=settings.TEMP_BASE_DIR)
os.chdir(tempdir)
tinfoil = _setup_tinfoil(bitbakepath, enable_tracking)
# Ensure TMPDIR exists (or insane.bbclass will blow up trying to write to the QA log)
oe_tmpdir = tinfoil.config_data.getVar("TMPDIR", True)
if not os.path.exists(oe_tmpdir):
os.makedirs(oe_tmpdir)
# Ensure BBFILES as an initial value so that the old mode of BBFILES := "${BBFILES} ..." works
if not tinfoil.config_data.getVar("BBFILES", False):
tinfoil.config_data.setVar("BBFILES", "")
return (tinfoil, tempdir)
def main():
valid_layer_name = re.compile('[-\w]+$')
parser = optparse.OptionParser(usage="""
%prog [options] <url> [name]""")
utils.setup_django()
layer_type_help, layer_type_choices = get_layer_type_choices()
parser.add_option("-s",
"--subdir",
help="Specify subdirectory",
action="store",
dest="subdir")
parser.add_option("-t",
"--type",
help=layer_type_help,
choices=layer_type_choices,
action="store",
dest="layer_type",
default='')
parser.add_option("-n",
"--dry-run",
help="Don't write any data back to the database",
action="store_true",
dest="dryrun")
parser.add_option("-d",
"--debug",
help="Enable debug output",
action="store_const",
const=logging.DEBUG,
dest="loglevel",
default=logging.INFO)
parser.add_option("",
"--github-auth",
help="Specify github username:password",
action="store",
dest="github_auth")
parser.add_option("-q",
"--quiet",
help="Hide all output except error messages",
action="store_const",
const=logging.ERROR,
dest="loglevel")
parser.add_option("-a",
"--actual-branch",
help="Set actual branch",
action="store",
dest="actual_branch")
options, args = parser.parse_args(sys.argv)
if len(args) < 2:
print("Please specify URL of repository for layer")
sys.exit(1)
layer_url = args[1]
if len(args) > 2:
layer_name = args[2]
else:
if options.subdir:
layer_name = options.subdir
else:
layer_name = [x for x in layer_url.split('/') if x][-1]
if layer_name.endswith('.git'):
layer_name = layer_name[:-4]
if not valid_layer_name.match(layer_name):
logger.error(
'Invalid layer name "%s" - Layer name can only include letters, numbers and dashes.',
layer_name)
sys.exit(1)
if options.github_auth:
if not ':' in options.github_auth:
logger.error(
'--github-auth value must be specified as username:password')
sys.exit(1)
splitval = options.github_auth.split(':')
github_login = splitval[0]
github_password = splitval[1]
else:
github_login = None
github_password = None
import settings
from layerindex.models import LayerItem, LayerBranch, LayerDependency, LayerMaintainer
from django.db import transaction
logger.setLevel(options.loglevel)
fetchdir = settings.LAYER_FETCH_DIR
if not fetchdir:
logger.error("Please set LAYER_FETCH_DIR in settings.py")
sys.exit(1)
if not os.path.exists(fetchdir):
os.makedirs(fetchdir)
master_branch = utils.get_branch('master')
core_layer = None
try:
with transaction.atomic():
# Fetch layer
logger.info('Fetching repository %s' % layer_url)
layer = LayerItem()
layer.name = layer_name
layer.status = 'P'
layer.summary = 'tempvalue'
layer.description = layer.summary
set_vcs_fields(layer, layer_url)
urldir = layer.get_fetch_dir()
repodir = os.path.join(fetchdir, urldir)
out = None
try:
if not os.path.exists(repodir):
out = utils.runcmd("git clone %s %s" %
(layer.vcs_url, urldir),
fetchdir,
logger=logger)
else:
out = utils.runcmd("git fetch", repodir, logger=logger)
except Exception as e:
logger.error("Fetch failed: %s" % str(e))
sys.exit(1)
actual_branch = 'master'
if (options.actual_branch):
actual_branch = options.actual_branch
try:
out = utils.runcmd("git checkout origin/%s" % actual_branch,
repodir,
logger=logger)
except subprocess.CalledProcessError:
actual_branch = None
branches = utils.runcmd("git branch -r",
repodir,
logger=logger)
for line in branches.splitlines():
if 'origin/HEAD ->' in line:
actual_branch = line.split('-> origin/')[-1]
break
if not actual_branch:
logger.error(
"Repository has no master branch nor origin/HEAD")
sys.exit(1)
out = utils.runcmd("git checkout origin/%s" % actual_branch,
repodir,
logger=logger)
layer_paths = []
if options.subdir:
layerdir = os.path.join(repodir, options.subdir)
if not os.path.exists(layerdir):
logger.error(
"Subdirectory %s does not exist in repository for master branch"
% options.subdir)
sys.exit(1)
if not os.path.exists(os.path.join(layerdir,
'conf/layer.conf')):
logger.error(
"conf/layer.conf not found in subdirectory %s" %
options.subdir)
sys.exit(1)
layer_paths.append(layerdir)
else:
if os.path.exists(os.path.join(repodir, 'conf/layer.conf')):
layer_paths.append(repodir)
# Find subdirs with a conf/layer.conf
for subdir in os.listdir(repodir):
subdir_path = os.path.join(repodir, subdir)
if os.path.isdir(subdir_path):
if os.path.exists(
os.path.join(subdir_path, 'conf/layer.conf')):
layer_paths.append(subdir_path)
if not layer_paths:
logger.error(
"conf/layer.conf not found in repository or first level subdirectories - is subdirectory set correctly?"
)
sys.exit(1)
if 'github.com' in layer.vcs_url:
json_data, owner_json_data = get_github_layerinfo(
layer.vcs_url, github_login, github_password)
for layerdir in layer_paths:
layer.pk = None
if layerdir != repodir:
subdir = os.path.relpath(layerdir, repodir)
if len(layer_paths) > 1:
layer.name = subdir
else:
subdir = ''
if LayerItem.objects.filter(name=layer.name).exists():
if LayerItem.objects.filter(name=layer.name).exclude(
vcs_url=layer.vcs_url).exists():
conflict_list = LayerItem.objects.filter(
name=layer.name).exclude(vcs_url=layer.vcs_url)
conflict_list_urls = []
for conflict in conflict_list:
conflict_list_urls.append(conflict.vcs_url)
cln = ', '.join(conflict_list_urls)
logger.error(
'A layer named "%s" already exists in the database. Possible name collision with %s.vcs_url = %s'
% (layer.name, layer.name, cln))
sys.exit(1)
else:
logger.info(
'The layer named "%s" already exists in the database. Skipping this layer with same vcs_url'
% layer.name)
layer_paths = [x for x in layer_paths if x != layerdir]
continue
logger.info('Creating layer %s' % layer.name)
# Guess layer type if not specified
if options.layer_type:
layer.layer_type = options.layer_type
elif layer.name in ['openembedded-core', 'meta-oe']:
layer.layer_type = 'A'
elif glob.glob(os.path.join(layerdir, 'conf/distro/*.conf')):
layer.layer_type = 'D'
elif glob.glob(os.path.join(layerdir, 'conf/machine/*.conf')):
layer.layer_type = 'B'
else:
layer.layer_type = 'M'
layer.save()
layerbranch = LayerBranch()
layerbranch.layer = layer
layerbranch.branch = master_branch
if layerdir != repodir:
layerbranch.vcs_subdir = subdir
if actual_branch:
layerbranch.actual_branch = actual_branch
layerbranch.save()
if layer.name != settings.CORE_LAYER_NAME:
if not core_layer:
core_layer = utils.get_layer(settings.CORE_LAYER_NAME)
if core_layer:
logger.debug('Adding dep %s to %s' %
(core_layer.name, layer.name))
layerdep = LayerDependency()
layerdep.layerbranch = layerbranch
layerdep.dependency = core_layer
layerdep.save()
layerconfparser = LayerConfParse(logger=logger)
try:
config_data = layerconfparser.parse_layer(layerdir)
if config_data:
utils.add_dependencies(layerbranch,
config_data,
logger=logger)
utils.add_recommends(layerbranch,
config_data,
logger=logger)
finally:
layerconfparser.shutdown()
# Get some extra meta-information
readme_files = glob.glob(os.path.join(layerdir, 'README*'))
if (not readme_files) and subdir:
readme_files = glob.glob(os.path.join(repodir, 'README*'))
maintainer_files = glob.glob(
os.path.join(layerdir, 'MAINTAINERS'))
if (not maintainer_files) and subdir:
maintainer_files = glob.glob(
os.path.join(repodir, 'MAINTAINERS'))
maintainers = []
if readme_files:
(desc, maintainers, deps) = readme_extract(readme_files[0])
if desc:
layer.summary = layer.name
layer.description = desc
if maintainer_files:
maintainers.extend(maintainers_extract(readme_files[0]))
if (not maintainers) and 'github.com' in layer.vcs_url:
if json_data:
layer.summary = json_data['description']
layer.description = layer.summary
if owner_json_data:
owner_name = owner_json_data.get('name', None)
owner_email = owner_json_data.get('email', None)
if owner_name and owner_email:
maintainers.append('%s <%s>' %
(owner_name, owner_email))
if layer.name == 'openembedded-core':
layer.summary = 'Core metadata'
elif layer.name == 'meta-oe':
layer.summary = 'Additional shared OE metadata'
layer.description = layer.summary
if maintainers:
maint_re = re.compile(
r'^"?([^"@$<>]+)"? *<([^<> ]+)>[ -]*(.+)?$')
for maintentry in maintainers:
res = maint_re.match(maintentry)
if res:
maintainer = LayerMaintainer()
maintainer.layerbranch = layerbranch
maintainer.name = res.group(1).strip()
maintainer.email = res.group(2)
if res.group(3):
maintainer.responsibility = res.group(
3).strip()
maintainer.save()
layer.save()
if not layer_paths:
logger.error('No layers added.')
sys.exit(1)
if options.dryrun:
raise DryRunRollbackException()
except DryRunRollbackException:
pass
sys.exit(0)
def main():
if LooseVersion(git.__version__) < '0.3.1':
logger.error(
"Version of GitPython is too old, please install GitPython (python-git) 0.3.1 or later in order to use this script"
)
sys.exit(1)
parser = optparse.OptionParser(usage="""
%prog [options]""")
parser.add_option("-b",
"--branch",
help="Specify branch to update",
action="store",
dest="branch",
default='master')
parser.add_option("-l",
"--layer",
help="Layer to update",
action="store",
dest="layer")
parser.add_option(
"-r",
"--reload",
help="Reload recipe data instead of updating since last update",
action="store_true",
dest="reload")
parser.add_option(
"",
"--fullreload",
help="Discard existing recipe data and fetch it from scratch",
action="store_true",
dest="fullreload")
parser.add_option("-n",
"--dry-run",
help="Don't write any data back to the database",
action="store_true",
dest="dryrun")
parser.add_option("",
"--nocheckout",
help="Don't check out branches",
action="store_true",
dest="nocheckout")
parser.add_option("",
"--stop-on-error",
help="Stop on first parsing error",
action="store_true",
default=False,
dest="stop_on_error")
parser.add_option("-i",
"--initial",
help="Print initial values parsed from layer.conf only",
action="store_true")
parser.add_option("-d",
"--debug",
help="Enable debug output",
action="store_const",
const=logging.DEBUG,
dest="loglevel",
default=logging.INFO)
parser.add_option("-q",
"--quiet",
help="Hide all output except error messages",
action="store_const",
const=logging.ERROR,
dest="loglevel")
parser.add_option(
"",
"--keep-temp",
help="Preserve temporary directory at the end instead of deleting it",
action="store_true")
options, args = parser.parse_args(sys.argv)
if len(args) > 1:
logger.error('unexpected argument "%s"' % args[1])
parser.print_help()
sys.exit(1)
if options.fullreload:
options.reload = True
utils.setup_django()
import settings
from layerindex.models import LayerItem, LayerBranch, Recipe, RecipeFileDependency, Machine, Distro, BBAppend, BBClass, IncFile
from django.db import transaction
logger.setLevel(options.loglevel)
branch = utils.get_branch(options.branch)
if not branch:
logger.error("Specified branch %s is not valid" % options.branch)
sys.exit(1)
fetchdir = settings.LAYER_FETCH_DIR
if not fetchdir:
logger.error("Please set LAYER_FETCH_DIR in settings.py")
sys.exit(1)
bitbakepath = os.path.join(fetchdir, 'bitbake')
layer = utils.get_layer(options.layer)
urldir = layer.get_fetch_dir()
repodir = os.path.join(fetchdir, urldir)
layerbranch = layer.get_layerbranch(options.branch)
branchname = options.branch
branchdesc = options.branch
if layerbranch:
if layerbranch.actual_branch:
branchname = layerbranch.actual_branch
branchdesc = "%s (%s)" % (options.branch, branchname)
# Collect repo info
repo = git.Repo(repodir)
if repo.bare:
logger.error('Repository %s is bare, not supported' % repodir)
sys.exit(1)
topcommit = repo.commit('origin/%s' % branchname)
if options.nocheckout:
topcommit = repo.commit('HEAD')
tinfoil = None
tempdir = None
try:
with transaction.atomic():
newbranch = False
if not layerbranch:
# LayerBranch doesn't exist for this branch, create it
newbranch = True
layerbranch = LayerBranch()
layerbranch.layer = layer
layerbranch.branch = branch
layerbranch_source = layer.get_layerbranch(branch)
if not layerbranch_source:
layerbranch_source = layer.get_layerbranch(None)
if layerbranch_source:
layerbranch.vcs_subdir = layerbranch_source.vcs_subdir
layerbranch.save()
if layerbranch_source:
for maintainer in layerbranch_source.layermaintainer_set.all(
):
maintainer.pk = None
maintainer.id = None
maintainer.layerbranch = layerbranch
maintainer.save()
if layerbranch.vcs_subdir and not options.nocheckout:
# Find latest commit in subdirectory
# A bit odd to do it this way but apparently there's no other way in the GitPython API
topcommit = next(
repo.iter_commits('origin/%s' % branchname,
paths=layerbranch.vcs_subdir), None)
layerdir = os.path.join(repodir, layerbranch.vcs_subdir)
layerdir_start = os.path.normpath(layerdir) + os.sep
layerrecipes = Recipe.objects.filter(layerbranch=layerbranch)
layermachines = Machine.objects.filter(layerbranch=layerbranch)
layerdistros = Distro.objects.filter(layerbranch=layerbranch)
layerappends = BBAppend.objects.filter(layerbranch=layerbranch)
layerclasses = BBClass.objects.filter(layerbranch=layerbranch)
layerincfiles = IncFile.objects.filter(layerbranch=layerbranch)
if layerbranch.vcs_last_rev != topcommit.hexsha or options.reload or options.initial:
# Check out appropriate branch
if not options.nocheckout:
utils.checkout_layer_branch(layerbranch,
repodir,
logger=logger)
logger.info("Collecting data for layer %s on branch %s" %
(layer.name, branchdesc))
try:
(tinfoil, tempdir) = recipeparse.init_parser(
settings,
branch,
bitbakepath,
nocheckout=options.nocheckout,
logger=logger)
except recipeparse.RecipeParseError as e:
logger.error(str(e))
sys.exit(1)
logger.debug('Using temp directory %s' % tempdir)
# Clear the default value of SUMMARY so that we can use DESCRIPTION instead if it hasn't been set
tinfoil.config_data.setVar('SUMMARY', '')
# Clear the default value of DESCRIPTION so that we can see where it's not set
tinfoil.config_data.setVar('DESCRIPTION', '')
# Clear the default value of HOMEPAGE ('unknown')
tinfoil.config_data.setVar('HOMEPAGE', '')
# Set a blank value for LICENSE so that it doesn't cause the parser to die (e.g. with meta-ti -
# why won't they just fix that?!)
tinfoil.config_data.setVar('LICENSE', '')
layerconfparser = layerconfparse.LayerConfParse(
logger=logger, tinfoil=tinfoil)
layer_config_data = layerconfparser.parse_layer(layerdir)
if not layer_config_data:
logger.info(
"Skipping update of layer %s for branch %s - conf/layer.conf may have parse issues"
% (layer.name, branchdesc))
layerconfparser.shutdown()
sys.exit(1)
utils.set_layerbranch_collection_version(layerbranch,
layer_config_data,
logger=logger)
if options.initial:
# Use print() rather than logger.info() since "-q" makes it print nothing.
for i in [
"BBFILE_COLLECTIONS", "LAYERVERSION",
"LAYERDEPENDS", "LAYERRECOMMENDS"
]:
print(
'%s = "%s"' % (i,
utils.get_layer_var(
layer_config_data, i, logger)))
sys.exit(0)
# Set up for recording patch info
utils.setup_core_layer_sys_path(settings, branch.name)
skip_patches = False
try:
import oe.recipeutils
except ImportError:
logger.warn(
'Failed to find lib/oe/recipeutils.py in layers - patch information will not be collected'
)
skip_patches = True
utils.add_dependencies(layerbranch,
layer_config_data,
logger=logger)
utils.add_recommends(layerbranch,
layer_config_data,
logger=logger)
layerbranch.save()
try:
config_data_copy = recipeparse.setup_layer(
tinfoil.config_data, fetchdir, layerdir, layer,
layerbranch, logger)
except recipeparse.RecipeParseError as e:
logger.error(str(e))
sys.exit(1)
if layerbranch.vcs_last_rev and not options.reload:
try:
diff = repo.commit(
layerbranch.vcs_last_rev).diff(topcommit)
except Exception as e:
logger.warn(
"Unable to get diff from last commit hash for layer %s - falling back to slow update: %s"
% (layer.name, str(e)))
diff = None
else:
diff = None
# We handle recipes specially to try to preserve the same id
# when recipe upgrades happen (so that if a user bookmarks a
# recipe page it remains valid)
layerrecipes_delete = []
layerrecipes_add = []
# Check if any paths should be ignored because there are layers within this layer
removedirs = []
for root, dirs, files in os.walk(layerdir):
for diritem in dirs:
if os.path.exists(
os.path.join(root, diritem, 'conf',
'layer.conf')):
removedirs.append(
os.path.join(root, diritem) + os.sep)
if diff:
# Apply git changes to existing recipe list
if layerbranch.vcs_subdir:
subdir_start = os.path.normpath(
layerbranch.vcs_subdir) + os.sep
else:
subdir_start = ""
updatedrecipes = set()
dirtyrecipes = set()
other_deletes = []
other_adds = []
for diffitem in diff.iter_change_type('R'):
oldpath = diffitem.a_blob.path
newpath = diffitem.b_blob.path
skip = False
for removedir in removedirs:
# FIXME what about files moved into removedirs?
if oldpath.startswith(removedir):
skip = True
break
if skip:
continue
if oldpath.startswith(subdir_start):
if not newpath.startswith(subdir_start):
logger.debug(
"Treating rename of %s to %s as a delete since new path is outside layer"
% (oldpath, newpath))
other_deletes.append(diffitem)
continue
(oldtypename, oldfilepath,
oldfilename) = recipeparse.detect_file_type(
oldpath, subdir_start)
(newtypename, newfilepath,
newfilename) = recipeparse.detect_file_type(
newpath, subdir_start)
if oldtypename != newtypename:
# This is most likely to be a .inc file renamed to a .bb - and since
# there may be another recipe deleted at the same time we probably want
# to consider that, so just treat it as a delete and an add
logger.debug(
"Treating rename of %s to %s as a delete and add (since type changed)"
% (oldpath, newpath))
other_deletes.append(diffitem)
other_adds.append(diffitem)
elif oldtypename == 'recipe':
results = layerrecipes.filter(
filepath=oldfilepath).filter(
filename=oldfilename)
if len(results):
recipe = results[0]
logger.debug("Rename recipe %s to %s" %
(recipe, newpath))
recipe.filepath = newfilepath
recipe.filename = newfilename
recipe.save()
update_recipe_file(
tinfoil, config_data_copy,
os.path.join(layerdir, newfilepath),
recipe, layerdir_start, repodir,
options.stop_on_error, skip_patches)
updatedrecipes.add(
os.path.join(oldfilepath, oldfilename))
updatedrecipes.add(
os.path.join(newfilepath, newfilename))
else:
logger.warn(
"Renamed recipe %s could not be found"
% oldpath)
other_adds.append(diffitem)
elif oldtypename == 'bbappend':
results = layerappends.filter(
filepath=oldfilepath).filter(
filename=oldfilename)
if len(results):
logger.debug(
"Rename bbappend %s to %s" %
(results[0],
os.path.join(newfilepath,
newfilename)))
results[0].filepath = newfilepath
results[0].filename = newfilename
results[0].save()
else:
logger.warn(
"Renamed bbappend %s could not be found"
% oldpath)
other_adds.append(diffitem)
elif oldtypename == 'machine':
results = layermachines.filter(
name=oldfilename)
if len(results):
logger.debug("Rename machine %s to %s" %
(results[0], newfilename))
results[0].name = newfilename
results[0].save()
else:
logger.warn(
"Renamed machine %s could not be found"
% oldpath)
other_adds.append(diffitem)
elif oldtypename == 'distro':
results = layerdistros.filter(name=oldfilename)
if len(results):
logger.debug("Rename distro %s to %s" %
(results[0], newfilename))
results[0].name = newfilename
results[0].save()
else:
logger.warn(
"Renamed distro %s could not be found"
% oldpath)
other_adds.append(diffitem)
elif oldtypename == 'bbclass':
results = layerclasses.filter(name=oldfilename)
if len(results):
logger.debug("Rename class %s to %s" %
(results[0], newfilename))
results[0].name = newfilename
results[0].save()
else:
logger.warn(
"Renamed class %s could not be found" %
oldpath)
other_adds.append(diffitem)
elif oldtypename == 'incfile':
results = layerincfiles.filter(
path=os.path.join(oldfilepath,
oldfilename))
if len(results):
logger.debug("Rename inc file %s to %s" %
(results[0], newfilename))
results[0].name = newfilename
results[0].save()
else:
logger.warn(
"Renamed inc file %s could not be found"
% oldpath)
other_adds.append(diffitem)
deps = RecipeFileDependency.objects.filter(
layerbranch=layerbranch).filter(path=oldpath)
for dep in deps:
dirtyrecipes.add(dep.recipe)
for diffitem in itertools.chain(diff.iter_change_type('D'),
other_deletes):
path = diffitem.a_blob.path
if path.startswith(subdir_start):
skip = False
for removedir in removedirs:
if path.startswith(removedir):
skip = True
break
if skip:
continue
(typename, filepath,
filename) = recipeparse.detect_file_type(
path, subdir_start)
if typename == 'recipe':
values = layerrecipes.filter(
filepath=filepath).filter(
filename=filename).values(
'id', 'filepath', 'filename', 'pn')
if len(values):
layerrecipes_delete.append(values[0])
logger.debug("Mark %s for deletion" %
values[0])
updatedrecipes.add(
os.path.join(values[0]['filepath'],
values[0]['filename']))
else:
logger.warn(
"Deleted recipe %s could not be found"
% path)
elif typename == 'bbappend':
layerappends.filter(filepath=filepath).filter(
filename=filename).delete()
elif typename == 'machine':
layermachines.filter(name=filename).delete()
elif typename == 'distro':
layerdistros.filter(name=filename).delete()
elif typename == 'bbclass':
layerclasses.filter(name=filename).delete()
elif typename == 'incfile':
layerincfiles.filter(path=os.path.join(
filepath, filename)).delete()
for diffitem in itertools.chain(diff.iter_change_type('A'),
other_adds):
path = diffitem.b_blob.path
if path.startswith(subdir_start):
skip = False
for removedir in removedirs:
if path.startswith(removedir):
skip = True
break
if skip:
continue
(typename, filepath,
filename) = recipeparse.detect_file_type(
path, subdir_start)
if typename == 'recipe':
layerrecipes_add.append(
os.path.join(repodir, path))
logger.debug("Mark %s for addition" % path)
updatedrecipes.add(
os.path.join(filepath, filename))
elif typename == 'bbappend':
append = BBAppend()
append.layerbranch = layerbranch
append.filename = filename
append.filepath = filepath
append.save()
elif typename == 'machine':
machine = Machine()
machine.layerbranch = layerbranch
machine.name = filename
update_machine_conf_file(
os.path.join(repodir, path), machine)
machine.save()
elif typename == 'distro':
distro = Distro()
distro.layerbranch = layerbranch
distro.name = filename
update_distro_conf_file(
os.path.join(repodir, path), distro,
config_data_copy)
distro.save()
elif typename == 'bbclass':
bbclass = BBClass()
bbclass.layerbranch = layerbranch
bbclass.name = filename
bbclass.save()
elif typename == 'incfile':
incfile = IncFile()
incfile.layerbranch = layerbranch
incfile.path = os.path.join(filepath, filename)
incfile.save()
for diffitem in diff.iter_change_type('M'):
path = diffitem.b_blob.path
if path.startswith(subdir_start):
skip = False
for removedir in removedirs:
if path.startswith(removedir):
skip = True
break
if skip:
continue
(typename, filepath,
filename) = recipeparse.detect_file_type(
path, subdir_start)
if typename == 'recipe':
logger.debug("Mark %s for update" % path)
results = layerrecipes.filter(
filepath=filepath).filter(
filename=filename)[:1]
if results:
recipe = results[0]
update_recipe_file(
tinfoil, config_data_copy,
os.path.join(layerdir, filepath),
recipe, layerdir_start, repodir,
options.stop_on_error, skip_patches)
recipe.save()
updatedrecipes.add(recipe.full_path())
elif typename == 'machine':
results = layermachines.filter(name=filename)
if results:
machine = results[0]
update_machine_conf_file(
os.path.join(repodir, path), machine)
machine.save()
elif typename == 'distro':
results = layerdistros.filter(name=filename)
if results:
distro = results[0]
update_distro_conf_file(
os.path.join(repodir, path), distro,
config_data_copy)
distro.save()
deps = RecipeFileDependency.objects.filter(
layerbranch=layerbranch).filter(path=path)
for dep in deps:
dirtyrecipes.add(dep.recipe)
for recipe in dirtyrecipes:
if not recipe.full_path() in updatedrecipes:
update_recipe_file(
tinfoil, config_data_copy,
os.path.join(layerdir, recipe.filepath),
recipe, layerdir_start, repodir,
options.stop_on_error, skip_patches)
else:
# Collect recipe data from scratch
layerrecipe_fns = []
if options.fullreload:
layerrecipes.delete()
else:
# First, check which recipes still exist
layerrecipe_values = layerrecipes.values(
'id', 'filepath', 'filename', 'pn')
for v in layerrecipe_values:
if v['filepath'].startswith('../'):
# FIXME: These recipes were present due to a bug (not handling renames
# to paths outside the layer) - this can be removed at some point in the future
preserve = False
else:
root = os.path.join(layerdir, v['filepath'])
fullpath = os.path.join(root, v['filename'])
if os.path.exists(fullpath):
preserve = True
for removedir in removedirs:
if fullpath.startswith(removedir):
preserve = False
break
else:
preserve = False
if preserve:
# Recipe still exists, update it
results = layerrecipes.filter(id=v['id'])[:1]
recipe = results[0]
update_recipe_file(tinfoil, config_data_copy,
root, recipe,
layerdir_start, repodir,
options.stop_on_error,
skip_patches)
else:
# Recipe no longer exists, mark it for later on
layerrecipes_delete.append(v)
layerrecipe_fns.append(fullpath)
layermachines.delete()
layerdistros.delete()
layerappends.delete()
layerclasses.delete()
for root, dirs, files in os.walk(layerdir):
if '.git' in dirs:
dirs.remove('.git')
for diritem in dirs[:]:
fullpath = os.path.join(root, diritem) + os.sep
if fullpath in removedirs:
dirs.remove(diritem)
for f in files:
fullpath = os.path.join(root, f)
(typename, _,
filename) = recipeparse.detect_file_type(
fullpath, layerdir_start)
if typename == 'recipe':
if fullpath not in layerrecipe_fns:
layerrecipes_add.append(fullpath)
elif typename == 'bbappend':
append = BBAppend()
append.layerbranch = layerbranch
append.filename = f
append.filepath = os.path.relpath(
root, layerdir)
append.save()
elif typename == 'machine':
machine = Machine()
machine.layerbranch = layerbranch
machine.name = filename
update_machine_conf_file(fullpath, machine)
machine.save()
elif typename == 'distro':
distro = Distro()
distro.layerbranch = layerbranch
distro.name = filename
update_distro_conf_file(
fullpath, distro, config_data_copy)
distro.save()
elif typename == 'bbclass':
bbclass = BBClass()
bbclass.layerbranch = layerbranch
bbclass.name = filename
bbclass.save()
elif typename == 'incfile':
incfile = IncFile()
incfile.layerbranch = layerbranch
incfile.path = os.path.relpath(
fullpath, layerdir)
incfile.save()
for added in layerrecipes_add:
# This is good enough without actually parsing the file
(pn, pv) = split_recipe_fn(added)
oldid = -1
for deleted in layerrecipes_delete:
if deleted['pn'] == pn:
oldid = deleted['id']
layerrecipes_delete.remove(deleted)
break
if oldid > -1:
# Reclaim a record we would have deleted
results = Recipe.objects.filter(id=oldid)[:1]
recipe = results[0]
logger.debug("Reclaim %s for %s %s" % (recipe, pn, pv))
else:
# Create new record
logger.debug("Add new recipe %s" % added)
recipe = Recipe()
recipe.layerbranch = layerbranch
recipe.filename = os.path.basename(added)
root = os.path.dirname(added)
recipe.filepath = os.path.relpath(root, layerdir)
update_recipe_file(tinfoil, config_data_copy, root, recipe,
layerdir_start, repodir,
options.stop_on_error, skip_patches)
recipe.save()
for deleted in layerrecipes_delete:
logger.debug("Delete %s" % deleted)
results = Recipe.objects.filter(id=deleted['id'])[:1]
recipe = results[0]
recipe.delete()
# Save repo info
layerbranch.vcs_last_rev = topcommit.hexsha
layerbranch.vcs_last_commit = datetime.fromtimestamp(
topcommit.committed_date)
else:
logger.info("Layer %s is already up-to-date for branch %s" %
(layer.name, branchdesc))
layerbranch.vcs_last_fetch = datetime.now()
layerbranch.save()
if options.dryrun:
raise DryRunRollbackException()
except KeyboardInterrupt:
logger.warn("Update interrupted, changes to %s rolled back" %
layer.name)
sys.exit(254)
except SystemExit:
raise
except DryRunRollbackException:
pass
except:
import traceback
logger.error(traceback.format_exc().rstrip())
sys.exit(1)
finally:
if tinfoil and (LooseVersion(bb.__version__) > LooseVersion("1.27")):
tinfoil.shutdown()
if tempdir:
if options.keep_temp:
logger.debug('Preserving temp directory %s' % tempdir)
else:
logger.debug('Deleting temp directory')
utils.rmtree_force(tempdir)
sys.exit(0)
def train(path='model.npy'):
print('Loading data !')
train_csv, test_csv = pd.read_csv(dir_train), pd.read_csv(dir_test)
raw_data = train_csv.append(test_csv, sort=False)
max_, min_ = max(raw_data['y']), min(raw_data['y'])
train_data, valid_data, test_data = make_dataset(data_process(raw_data))
REGULARIZATION = regularization / len(train_data)
try:
net = utils.load_model(path, True)
print('Successfully load model !')
except:
print('Initialize the model parameters !')
net = [
utils.get_layer(3,
5,
activation=activation,
optimizer=optimization,
regularization=REGULARIZATION),
utils.get_layer(5,
2,
'ReLU',
optimizer=optimization,
regularization=REGULARIZATION),
utils.get_layer(2,
1,
'linear',
optimizer=optimization,
regularization=REGULARIZATION)
]
valid_loss_record = []
test_loss_record = []
for q in range(epoch):
batch = utils.batch(train_data, True)
train_batch, valid_batch = utils.batch(train_data, True), utils.batch(
valid_data, True)
print('Training begins !')
time_start = time.time()
train_loss, valid_loss = 0, 0
for p in range(round):
for i in range(len(batch)):
net, _ = utils.train([batch[i][0]], [[batch[i][1]]],
net,
learning_rate=0.01,
decay_rate=0.99)
train_loss, valid_loss = cal_loss(train_batch,
net), cal_loss(valid_batch, net)
print('epoch : ' + str(q) + ' train_loss:' +
str(train_loss / len(train_batch)) + ' valid_loss:' +
str(valid_loss / len(valid_batch)))
valid_loss_record.append(train_loss / len(train_batch))
test_loss_record.append(valid_loss / len(valid_batch))
time_end = time.time()
print('Train finished in ' + str((time_end - time_start) * 60) +
' seconds !')
validation(valid_data, net, max_, min_)
plt.plot(valid_loss_record, label='train', color='r')
plt.plot(test_loss_record, label='valid', color='b')
plt.savefig('Loss.jpg')
plt.show()
return net
import utils
import numpy as np
layer = [utils.get_layer(3, 8, activation='ReLU', optimizer='Adamoptimizer', regularization=0.1), utils.get_layer(8, 1, activation='linear',optimizer='Adamoptimizer',regularization=0.1)]
batch_x = [[0.5,0.5,0.5]]
batch_y = [[0.5]]
for i in range(10):
layer, total_loss = utils.train(batch_x, batch_y, layer)
print(total_loss)
#print(layer)
utils.save_model(layer, 'model_test.npy')
utils.load_model('model_test.npy')
utils.forward(batch_x, layer)
print(utils.last_layer(layer)['out_'])
def main():
parser = optparse.OptionParser(
usage = """
%prog [options] <url> [name]""")
parser.add_option("-s", "--subdir",
help = "Specify subdirectory",
action="store", dest="subdir")
parser.add_option("-n", "--dry-run",
help = "Don't write any data back to the database",
action="store_true", dest="dryrun")
parser.add_option("-d", "--debug",
help = "Enable debug output",
action="store_const", const=logging.DEBUG, dest="loglevel", default=logging.INFO)
parser.add_option("", "--github-auth",
help = "Specify github username:password",
action="store", dest="github_auth")
parser.add_option("-q", "--quiet",
help = "Hide all output except error messages",
action="store_const", const=logging.ERROR, dest="loglevel")
options, args = parser.parse_args(sys.argv)
if len(args) < 2:
print("Please specify URL of repository for layer")
sys.exit(1)
layer_url = args[1]
if len(args) > 2:
layer_name = args[2]
else:
if options.subdir:
layer_name = options.subdir
else:
layer_name = filter(None, layer_url.split('/'))[-1]
if layer_name.endswith('.git'):
layer_name = layer_name[:-4]
if options.github_auth:
if not ':' in options.github_auth:
logger.error('--github-auth value must be specified as username:password')
sys.exit(1)
splitval = options.github_auth.split(':')
github_login = splitval[0]
github_password = splitval[1]
else:
github_login = None
github_password = None
utils.setup_django()
import settings
from layerindex.models import LayerItem, LayerBranch, LayerDependency, LayerMaintainer
from django.db import transaction
logger.setLevel(options.loglevel)
fetchdir = settings.LAYER_FETCH_DIR
if not fetchdir:
logger.error("Please set LAYER_FETCH_DIR in settings.py")
sys.exit(1)
if not os.path.exists(fetchdir):
os.makedirs(fetchdir)
master_branch = utils.get_branch('master')
core_layer = None
transaction.enter_transaction_management()
transaction.managed(True)
try:
# Fetch layer
logger.info('Fetching repository %s' % layer_url)
layer = LayerItem()
layer.name = layer_name
layer.status = 'P'
layer.layer_type = 'M'
layer.summary = 'tempvalue'
layer.description = layer.summary
set_vcs_fields(layer, layer_url)
urldir = layer.get_fetch_dir()
repodir = os.path.join(fetchdir, urldir)
out = None
try:
if not os.path.exists(repodir):
out = utils.runcmd("git clone %s %s" % (layer.vcs_url, urldir), fetchdir, logger=logger)
else:
out = utils.runcmd("git fetch", repodir, logger=logger)
except Exception as e:
logger.error("Fetch failed: %s" % str(e))
sys.exit(1)
actual_branch = ''
try:
out = utils.runcmd("git checkout origin/master", repodir, logger=logger)
except subprocess.CalledProcessError:
branches = utils.runcmd("git branch -r", repodir, logger=logger)
for line in branches.splitlines():
if 'origin/HEAD ->' in line:
actual_branch = line.split('-> origin/')[-1]
break
if not actual_branch:
logger.error("Repository has no master branch nor origin/HEAD")
sys.exit(1)
out = utils.runcmd("git checkout origin/%s" % actual_branch, repodir, logger=logger)
layer_paths = []
if options.subdir:
layerdir = os.path.join(repodir, options.subdir)
if not os.path.exists(layerdir):
logger.error("Subdirectory %s does not exist in repository for master branch" % options.subdir)
sys.exit(1)
if not os.path.exists(os.path.join(layerdir, 'conf/layer.conf')):
logger.error("conf/layer.conf not found in subdirectory %s" % options.subdir)
sys.exit(1)
layer_paths.append(layerdir)
else:
if os.path.exists(os.path.join(repodir, 'conf/layer.conf')):
layer_paths.append(repodir)
# Find subdirs with a conf/layer.conf
for subdir in os.listdir(repodir):
subdir_path = os.path.join(repodir, subdir)
if os.path.isdir(subdir_path):
if os.path.exists(os.path.join(subdir_path, 'conf/layer.conf')):
layer_paths.append(subdir_path)
if not layer_paths:
logger.error("conf/layer.conf not found in repository or first level subdirectories - is subdirectory set correctly?")
sys.exit(1)
if 'github.com' in layer.vcs_url:
json_data, owner_json_data = get_github_layerinfo(layer.vcs_url, github_login, github_password)
for layerdir in layer_paths:
layer.pk = None
if layerdir != repodir:
subdir = os.path.relpath(layerdir, repodir)
if len(layer_paths) > 1:
layer.name = subdir
else:
subdir = ''
if LayerItem.objects.filter(name=layer.name).exists():
logger.error('A layer named "%s" already exists in the database' % layer_name)
sys.exit(1)
logger.info('Creating layer %s' % layer.name)
# Guess layer type
if glob.glob(os.path.join(layerdir, 'conf/distro/*.conf')):
layer.layer_type = 'D'
elif glob.glob(os.path.join(layerdir, 'conf/machine/*.conf')):
layer.layer_type = 'B'
layer.save()
layerbranch = LayerBranch()
layerbranch.layer = layer
layerbranch.branch = master_branch
if layerdir != repodir:
layerbranch.vcs_subdir = subdir
if actual_branch:
layerbranch.actual_branch = actual_branch
layerbranch.save()
if layer.name != settings.CORE_LAYER_NAME:
if not core_layer:
core_layer = utils.get_layer(settings.CORE_LAYER_NAME)
if core_layer:
layerdep = LayerDependency()
layerdep.layerbranch = layerbranch
layerdep.dependency = core_layer
layerdep.save()
# Get some extra meta-information
readme_files = glob.glob(os.path.join(layerdir, 'README*'))
if (not readme_files) and subdir:
readme_files = glob.glob(os.path.join(repodir, 'README*'))
maintainer_files = glob.glob(os.path.join(layerdir, 'MAINTAINERS'))
if (not maintainer_files) and subdir:
maintainer_files = glob.glob(os.path.join(repodir, 'MAINTAINERS'))
maintainers = []
if readme_files:
(desc, maintainers, deps) = readme_extract(readme_files[0])
if desc:
layer.summary = layer.name
layer.description = desc
if maintainer_files:
maintainers.extend(maintainers_extract(readme_files[0]))
if (not maintainers) and 'github.com' in layer.vcs_url:
if json_data:
layer.summary = json_data['description']
layer.description = layer.summary
if owner_json_data:
owner_name = owner_json_data.get('name', None)
owner_email = owner_json_data.get('email', None)
if owner_name and owner_email:
maintainers.append('%s <%s>' % (owner_name, owner_email))
if layer.name == 'openembedded-core':
layer.summary = 'Core metadata'
layer.layer_type = 'A'
elif layer.name == 'meta-oe':
layer.summary = 'Additional shared OE metadata'
layer.description = layer.summary
layer.layer_type = 'A'
if maintainers:
maint_re = re.compile(r'^"?([^"@$<>]+)"? *<([^<> ]+)>[ -]*(.+)?$')
for maintentry in maintainers:
res = maint_re.match(maintentry)
if res:
maintainer = LayerMaintainer()
maintainer.layerbranch = layerbranch
maintainer.name = res.group(1).strip()
maintainer.email = res.group(2)
if res.group(3):
maintainer.responsibility = res.group(3).strip()
maintainer.save()
layer.save()
if options.dryrun:
transaction.rollback()
else:
transaction.commit()
except:
transaction.rollback()
raise
finally:
transaction.leave_transaction_management()
sys.exit(0)
def __init__(
self,
out_channels: int,
exp_channels: int,
kernel_size: int,
stride: int,
use_se: bool,
act_layer: str,
l2_reg: float = 1e-5,
let_res: bool = True,
):
super().__init__(name="Bneck")
self.out_channels = out_channels
self.stride = stride
self.use_se = use_se
self.let_res = let_res
# print('[out_ch] ', out_channels)
# print('[exp_ch] ', exp_channels)
# Expand
self.expand = ConvNormAct(
exp_channels,
kernel_size=1,
norm_layer="bn",
act_layer=act_layer,
use_bias=False,
l2_reg=l2_reg,
name="Expand",
)
# Depthwise
dw_padding = (kernel_size - 1) // 2
self.pad = tf.keras.layers.ZeroPadding2D(
padding=dw_padding,
name=f"Depthwise/Padding{dw_padding}x{dw_padding}",
)
self.depthwise = tf.keras.layers.DepthwiseConv2D(
kernel_size=kernel_size,
strides=stride,
name=f"Depthwise/DWConv{kernel_size}x{kernel_size}",
depthwise_regularizer=tf.keras.regularizers.l2(l2_reg),
use_bias=False,
)
self.bn = BatchNormalization(name="Depthwise/BatchNormalization")
if self.use_se:
self.se = SEBottleneck(
l2_reg=l2_reg,
name="Depthwise/SEBottleneck",
)
_available_activation = {
"relu": tf.keras.layers.ReLU(name="Depthwise/ReLU"),
"hswish": HardSwish(name="Depthwise/HardSwish"),
}
self.act = get_layer(act_layer, _available_activation, Identity())
# Project
self.project = ConvNormAct(
out_channels,
kernel_size=1,
norm_layer="bn",
act_layer=None,
use_bias=False,
l2_reg=l2_reg,
name="Project",
)
def main():
parser = optparse.OptionParser(
usage = """
%prog [options]""")
options, args = parser.parse_args(sys.argv)
utils.setup_django()
from layerindex.models import LayerItem, LayerBranch, LayerDependency
from django.db import transaction
import httplib
conn = httplib.HTTPConnection("www.openembedded.org")
conn.request("GET", "/wiki/LayerIndex?action=raw")
resp = conn.getresponse()
if resp.status in [200, 302]:
data = resp.read()
in_table = False
layer_type = 'M'
nowiki_re = re.compile(r'</?nowiki>')
link_re = re.compile(r'\[(http.*) +link\]')
readme_re = re.compile(r';f=[a-zA-Z0-9/-]*README;')
master_branch = utils.get_branch('master')
core_layer = None
with transaction.atomic():
for line in data.splitlines():
if line.startswith('{|'):
in_table = True
continue
if in_table:
if line.startswith('|}'):
# We're done
break
elif line.startswith('!'):
section = line.split('|', 1)[1].strip("'")
if section.startswith('Base'):
layer_type = 'A'
elif section.startswith('Board'):
layer_type = 'B'
elif section.startswith('Software'):
layer_type = 'S'
elif section.startswith('Distribution'):
layer_type = 'D'
else:
layer_type = 'M'
elif not line.startswith('|-'):
if line.startswith("|| ''"):
continue
fields = line.split('||')
layer = LayerItem()
layer.name = fields[1].strip()
if ' ' in layer.name:
logger.warn('Skipping layer %s - name invalid' % layer.name)
continue
logger.info('Adding layer %s' % layer.name)
layer.status = 'P'
layer.layer_type = layer_type
layer.summary = fields[2].strip()
layer.description = layer.summary
if len(fields) > 6:
res = link_re.match(fields[6].strip())
if res:
link = res.groups(1)[0].strip()
if link.endswith('/README') or readme_re.search(link):
link = 'README'
layer.usage_url = link
repoval = nowiki_re.sub('', fields[4]).strip()
layer.vcs_url = repoval
if repoval.startswith('git://git.openembedded.org/'):
reponame = re.sub('^.*/', '', repoval)
layer.vcs_web_url = 'http://cgit.openembedded.org/' + reponame
layer.vcs_web_tree_base_url = 'http://cgit.openembedded.org/' + reponame + '/tree/%path%?h=%branch%'
layer.vcs_web_file_base_url = 'http://cgit.openembedded.org/' + reponame + '/tree/%path%?h=%branch%'
layer.vcs_web_commit_url = 'http://cgit.openembedded.org/' + reponame + '/commit/?id=%hash%'
elif repoval.startswith('git://git.yoctoproject.org/'):
reponame = re.sub('^.*/', '', repoval)
layer.vcs_web_url = 'http://git.yoctoproject.org/cgit/cgit.cgi/' + reponame
layer.vcs_web_tree_base_url = 'http://git.yoctoproject.org/cgit/cgit.cgi/' + reponame + '/tree/%path%?h=%branch%'
layer.vcs_web_file_base_url = 'http://git.yoctoproject.org/cgit/cgit.cgi/' + reponame + '/tree/%path%?h=%branch%'
layer.vcs_web_commit_url = 'http://git.yoctoproject.org/cgit/cgit.cgi/' + reponame + '/commit/?id=%hash%'
elif repoval.startswith('git://github.com/') or repoval.startswith('http://github.com/') or repoval.startswith('https://github.com/'):
reponame = re.sub('^.*github.com/', '', repoval)
reponame = re.sub('.git$', '', reponame)
layer.vcs_web_url = 'http://github.com/' + reponame
layer.vcs_web_tree_base_url = 'http://github.com/' + reponame + '/tree/%branch%/'
layer.vcs_web_file_base_url = 'http://github.com/' + reponame + '/blob/%branch%/'
layer.vcs_web_commit_url = 'http://github.com/' + reponame + '/commit/%hash%'
elif repoval.startswith('git://gitlab.com/') or repoval.startswith('http://gitlab.com/') or repoval.startswith('https://gitlab.com/'):
reponame = re.sub('^.*gitlab.com/', '', repoval)
reponame = re.sub('.git$', '', reponame)
layer.vcs_web_url = 'http://gitlab.com/' + reponame
layer.vcs_web_tree_base_url = 'http://gitlab.com/' + reponame + '/tree/%branch%/'
layer.vcs_web_file_base_url = 'http://gitlab.com/' + reponame + '/blob/%branch%/'
layer.vcs_web_commit_url = 'http://gitlab.com/' + reponame + '/commit/%hash%'
elif repoval.startswith('git://bitbucket.org/') or repoval.startswith('http://bitbucket.org/') or repoval.startswith('https://bitbucket.org/'):
reponame = re.sub('^.*bitbucket.org/', '', repoval)
reponame = re.sub('.git$', '', reponame)
layer.vcs_web_url = 'http://bitbucket.org/' + reponame
layer.vcs_web_tree_base_url = 'http://bitbucket.org/' + reponame + '/src/%branch%/%path%?at=%branch%'
layer.vcs_web_file_base_url = 'http://bitbucket.org/' + reponame + '/src/%branch%/%path%?at=%branch%'
layer.vcs_web_commit_url = 'http://bitbucket.org/' + reponame + '/commits/%hash%'
elif '.git' in repoval:
res = link_re.match(fields[5].strip())
layer.vcs_web_url = res.groups(1)[0]
layer.vcs_web_tree_base_url = re.sub(r'\.git.*', '.git;a=tree;f=%path%;hb=%branch%', layer.vcs_web_url)
layer.vcs_web_file_base_url = re.sub(r'\.git.*', '.git;a=blob;f=%path%;hb=%branch%', layer.vcs_web_url)
layer.vcs_web_file_base_url = re.sub(r'\.git.*', '.git;a=commit;h=%hash%', layer.vcs_web_url)
layer.save()
layerbranch = LayerBranch()
layerbranch.layer = layer
layerbranch.branch = master_branch
layerbranch.vcs_subdir = fields[3].strip()
layerbranch.save()
if layer.name != 'openembedded-core':
if not core_layer:
core_layer = utils.get_layer('openembedded-core')
if core_layer:
layerdep = LayerDependency()
layerdep.layerbranch = layerbranch
layerdep.dependency = core_layer
layerdep.save()
else:
logger.error('Fetch failed: %d: %s' % (resp.status, resp.reason))
sys.exit(0)
def build_sampler(tparams, options, trng):
x = tensor.matrix('x', dtype='int64')
xr = x[::-1]
n_timesteps = x.shape[0]
n_samples = x.shape[1]
# word embedding (source)
emb = tparams['Wemb'][x.flatten()]
emb = emb.reshape([n_timesteps, n_samples, options['dim_word']])
embr = tparams['Wemb'][xr.flatten()]
embr = embr.reshape([n_timesteps, n_samples, options['dim_word']])
# encoder
proj = get_layer(options['encoder'])[1](tparams, emb, options, prefix='encoder')
projr = get_layer(options['encoder'])[1](tparams, embr, options, prefix='encoder_r')
ctx = concatenate([proj[0], projr[0][::-1]], axis=proj[0].ndim-1)
ctx_mean = ctx.mean(0)
init_state = get_layer('ff')[1](tparams, ctx_mean, options,
prefix='ff_state', activ='tanh')
print 'Building f_init...',
outs = [init_state, ctx]
f_init = theano.function([x], outs, name='f_init', profile=profile)
print 'Done'
# x: 1 x 1
y = tensor.vector('y_sampler', dtype='int64')
init_state = tensor.matrix('init_state', dtype='float32')
# if it's the first word, emb should be all zero
emb = tensor.switch(y[:, None] < 0,
tensor.alloc(0., 1, tparams['Wemb_dec'].shape[1]),
tparams['Wemb_dec'][y])
proj = get_layer(options['decoder'])[1](tparams, emb, options,
prefix='decoder',
mask=None, context=ctx,
one_step=True,
init_state=init_state)
next_state = proj[0]
ctxs = proj[1]
logit_lstm = get_layer('ff')[1](tparams, next_state, options,
prefix='ff_logit_lstm', activ='linear')
logit_prev = get_layer('ff')[1](tparams, emb, options,
prefix='ff_logit_prev', activ='linear')
logit_ctx = get_layer('ff')[1](tparams, ctxs, options,
prefix='ff_logit_ctx', activ='linear')
logit = tensor.tanh(logit_lstm + logit_prev + logit_ctx)
logit = get_layer('ff')[1](tparams, logit, options,
prefix='ff_logit', activ='linear')
next_probs = tensor.nnet.softmax(logit)
next_sample = trng.multinomial(pvals=next_probs).argmax(1)
# next word probability
print 'Building f_next..',
inps = [y, ctx, init_state]
outs = [next_probs, next_sample, next_state]
f_next = theano.function(inps, outs, name='f_next', profile=profile)
print 'Done'
return f_init, f_next
def init_params(options):
params = OrderedDict()
# embedding
params['Wemb'] = norm_weight(options['n_words_src'], options['dim_word'])
params['Wemb_dec'] = norm_weight(options['n_words'], options['dim_word'])
# encoder: bidirectional RNN
params = get_layer(options['encoder'])[0](options,
params,
prefix='encoder',
nin=options['dim_word'],
dim=options['dim'])
params = get_layer(options['encoder'])[0](options,
params,
prefix='encoder_r',
nin=options['dim_word'],
dim=options['dim'])
ctxdim = 2 * options['dim']
# init_state, init_cell
params = get_layer('ff')[0](options,
params,
prefix='ff_state',
nin=ctxdim,
nout=options['dim'])
# decoder
params = get_layer(options['decoder'])[0](options,
params,
prefix='decoder',
nin=options['dim_word'],
dim=options['dim'],
dimctx=ctxdim)
# readout
params = get_layer('ff')[0](options,
params,
prefix='ff_logit_lstm',
nin=options['dim'],
nout=options['dim_word'],
ortho=False)
params = get_layer('ff')[0](options,
params,
prefix='ff_logit_prev',
nin=options['dim_word'],
nout=options['dim_word'],
ortho=False)
params = get_layer('ff')[0](options,
params,
prefix='ff_logit_ctx',
nin=ctxdim,
nout=options['dim_word'],
ortho=False)
params = get_layer('ff')[0](options,
params,
prefix='ff_logit',
nin=options['dim_word'],
nout=options['n_words'])
return params
def build_model(tparams, options):
opt_ret = dict()
trng = RandomStreams(1234)
use_noise = theano.shared(numpy.float32(0.))
# description string: #words x #samples
x = tensor.matrix('x', dtype='int64')
x_mask = tensor.matrix('x_mask', dtype='float32')
y = tensor.matrix('y', dtype='int64')
y_mask = tensor.matrix('y_mask', dtype='float32')
xr = x[::-1]
xr_mask = x_mask[::-1]
n_timesteps = x.shape[0]
n_timesteps_trg = y.shape[0]
n_samples = x.shape[1]
emb = tparams['Wemb'][x.flatten()]
emb = emb.reshape([n_timesteps, n_samples, options['dim_word']])
proj = get_layer(options['encoder'])[1](tparams, emb, options,
prefix='encoder',
mask=x_mask)
embr = tparams['Wemb'][xr.flatten()]
embr = embr.reshape([n_timesteps, n_samples, options['dim_word']])
projr = get_layer(options['encoder'])[1](tparams, embr, options,
prefix='encoder_r',
mask=xr_mask)
ctx = concatenate([proj[0], projr[0][::-1]], axis=proj[0].ndim-1)
ctx_mean = (ctx * x_mask[:, :, None]).sum(0) / x_mask.sum(0)[:,None]
# initial decoder state
init_state = get_layer('ff')[1](tparams, ctx_mean, options,
prefix='ff_state', activ='tanh')
# word embedding (target)
emb = tparams['Wemb_dec'][y.flatten()]
emb = emb.reshape([n_timesteps_trg, n_samples, options['dim_word']])
emb_shifted = tensor.zeros_like(emb)
emb_shifted = tensor.set_subtensor(emb_shifted[1:], emb[:-1])
emb = emb_shifted
# decoder
proj = get_layer(options['decoder'])[1](tparams, emb, options,
prefix='decoder',
mask=y_mask, context=ctx,
context_mask=x_mask,
one_step=False,
init_state=init_state)
proj_h = proj[0]
ctxs = proj[1]
opt_ret['dec_alphas'] = proj[2]
# compute word probabilities
logit_lstm = get_layer('ff')[1](tparams, proj_h, options,
prefix='ff_logit_lstm', activ='linear')
logit_prev = get_layer('ff')[1](tparams, emb, options,
prefix='ff_logit_prev', activ='linear')
logit_ctx = get_layer('ff')[1](tparams, ctxs, options,
prefix='ff_logit_ctx', activ='linear')
logit = tensor.tanh(logit_lstm+logit_prev+logit_ctx)
logit = get_layer('ff')[1](tparams, logit, options,
prefix='ff_logit', activ='linear')
logit_shp = logit.shape
probs = tensor.nnet.softmax(logit.reshape([logit_shp[0]*logit_shp[1],
logit_shp[2]]))
# cost
y_flat = y.flatten()
y_flat_idx = tensor.arange(y_flat.shape[0]) * options['n_words'] + y_flat
cost = -tensor.log(probs.flatten()[y_flat_idx])
cost = cost.reshape([y.shape[0], y.shape[1]])
cost = (cost * y_mask).sum(0)
return trng, use_noise, x, x_mask, y, y_mask, opt_ret, cost
def build_sampler(tparams, options, trng):
x = tensor.matrix('x', dtype='int64')
xr = x[::-1]
n_timesteps = x.shape[0]
n_samples = x.shape[1]
# word embedding (source)
emb = tparams['Wemb'][x.flatten()]
emb = emb.reshape([n_timesteps, n_samples, options['dim_word']])
embr = tparams['Wemb'][xr.flatten()]
embr = embr.reshape([n_timesteps, n_samples, options['dim_word']])
# encoder
proj = get_layer(options['encoder'])[1](tparams,
emb,
options,
prefix='encoder')
projr = get_layer(options['encoder'])[1](tparams,
embr,
options,
prefix='encoder_r')
ctx = concatenate([proj[0], projr[0][::-1]], axis=proj[0].ndim - 1)
ctx_mean = ctx.mean(0)
init_state = get_layer('ff')[1](tparams,
ctx_mean,
options,
prefix='ff_state',
activ='tanh')
print 'Building f_init...',
outs = [init_state, ctx]
f_init = theano.function([x], outs, name='f_init', profile=profile)
print 'Done'
# x: 1 x 1
y = tensor.vector('y_sampler', dtype='int64')
init_state = tensor.matrix('init_state', dtype='float32')
# if it's the first word, emb should be all zero
emb = tensor.switch(y[:, None] < 0,
tensor.alloc(0., 1, tparams['Wemb_dec'].shape[1]),
tparams['Wemb_dec'][y])
proj = get_layer(options['decoder'])[1](tparams,
emb,
options,
prefix='decoder',
mask=None,
context=ctx,
one_step=True,
init_state=init_state)
next_state = proj[0]
ctxs = proj[1]
logit_lstm = get_layer('ff')[1](tparams,
next_state,
options,
prefix='ff_logit_lstm',
activ='linear')
logit_prev = get_layer('ff')[1](tparams,
emb,
options,
prefix='ff_logit_prev',
activ='linear')
logit_ctx = get_layer('ff')[1](tparams,
ctxs,
options,
prefix='ff_logit_ctx',
activ='linear')
logit = tensor.tanh(logit_lstm + logit_prev + logit_ctx)
logit = get_layer('ff')[1](tparams,
logit,
options,
prefix='ff_logit',
activ='linear')
next_probs = tensor.nnet.softmax(logit)
next_sample = trng.multinomial(pvals=next_probs).argmax(1)
# next word probability
print 'Building f_next..',
inps = [y, ctx, init_state]
outs = [next_probs, next_sample, next_state]
f_next = theano.function(inps, outs, name='f_next', profile=profile)
print 'Done'
return f_init, f_next
m = np.mean(y)
data = []
for i in range(5000):
x1[i] = (x1[i] - min(x1)) / (max(x1) - min(x1))
x2[i] = (x2[i] - min(x2)) / (max(x2) - min(x2))
x3[i] = (x3[i] - min(x3)) / (max(x3) - min(x3))
y[i] = [1, 0] if y[i] > 1.5 else [0, 1]
data.append([[x1[i], x2[i], x3[i]], y[i]])
for i in range(5000):
print(x1[i], x2[i], x3[i], y[i])
net = [
utils.get_layer(3,
8,
activation='ReLU',
optimizer='Adamoptimizer',
regularization=0.01),
utils.get_layer(8,
2,
activation='softmax',
optimizer='Adamoptimizer',
regularization=0.01)
]
epoch = 16
round = 1
batch_size = 1000
print('traning begins !')
for p in range(epoch):
print('epoch ' + str(p) + ' begins !')
def build_model(tparams, options):
opt_ret = dict()
trng = RandomStreams(1234)
use_noise = theano.shared(numpy.float32(0.))
# description string: #words x #samples
x = tensor.matrix('x', dtype='int64')
x_mask = tensor.matrix('x_mask', dtype='float32')
y = tensor.matrix('y', dtype='int64')
y_mask = tensor.matrix('y_mask', dtype='float32')
xr = x[::-1]
xr_mask = x_mask[::-1]
n_timesteps = x.shape[0]
n_timesteps_trg = y.shape[0]
n_samples = x.shape[1]
emb = tparams['Wemb'][x.flatten()]
emb = emb.reshape([n_timesteps, n_samples, options['dim_word']])
proj = get_layer(options['encoder'])[1](tparams,
emb,
options,
prefix='encoder',
mask=x_mask)
embr = tparams['Wemb'][xr.flatten()]
embr = embr.reshape([n_timesteps, n_samples, options['dim_word']])
projr = get_layer(options['encoder'])[1](tparams,
embr,
options,
prefix='encoder_r',
mask=xr_mask)
ctx = concatenate([proj[0], projr[0][::-1]], axis=proj[0].ndim - 1)
ctx_mean = (ctx * x_mask[:, :, None]).sum(0) / x_mask.sum(0)[:, None]
# initial decoder state
init_state = get_layer('ff')[1](tparams,
ctx_mean,
options,
prefix='ff_state',
activ='tanh')
# word embedding (target)
emb = tparams['Wemb_dec'][y.flatten()]
emb = emb.reshape([n_timesteps_trg, n_samples, options['dim_word']])
emb_shifted = tensor.zeros_like(emb)
emb_shifted = tensor.set_subtensor(emb_shifted[1:], emb[:-1])
emb = emb_shifted
# decoder
proj = get_layer(options['decoder'])[1](tparams,
emb,
options,
prefix='decoder',
mask=y_mask,
context=ctx,
context_mask=x_mask,
one_step=False,
init_state=init_state)
proj_h = proj[0]
ctxs = proj[1]
opt_ret['dec_alphas'] = proj[2]
# compute word probabilities
logit_lstm = get_layer('ff')[1](tparams,
proj_h,
options,
prefix='ff_logit_lstm',
activ='linear')
logit_prev = get_layer('ff')[1](tparams,
emb,
options,
prefix='ff_logit_prev',
activ='linear')
logit_ctx = get_layer('ff')[1](tparams,
ctxs,
options,
prefix='ff_logit_ctx',
activ='linear')
logit = tensor.tanh(logit_lstm + logit_prev + logit_ctx)
logit = get_layer('ff')[1](tparams,
logit,
options,
prefix='ff_logit',
activ='linear')
logit_shp = logit.shape
probs = tensor.nnet.softmax(
logit.reshape([logit_shp[0] * logit_shp[1], logit_shp[2]]))
# cost
y_flat = y.flatten()
y_flat_idx = tensor.arange(y_flat.shape[0]) * options['n_words'] + y_flat
cost = -tensor.log(probs.flatten()[y_flat_idx])
cost = cost.reshape([y.shape[0], y.shape[1]])
cost = (cost * y_mask).sum(0)
return trng, use_noise, x, x_mask, y, y_mask, opt_ret, cost
def init_parser(settings, branch, bitbakepath, enable_tracking=False, nocheckout=False, classic=False, logger=None):
if not (nocheckout or classic):
# Check out the branch of BitBake appropriate for this branch and clean out any stale files (e.g. *.pyc)
if re.match('[0-9a-f]{40}', branch.bitbake_branch):
# SHA1 hash
bitbake_ref = branch.bitbake_branch
else:
# Branch name
bitbake_ref = 'origin/%s' % branch.bitbake_branch
utils.checkout_repo(bitbakepath, bitbake_ref, logger=logger)
# Skip sanity checks
os.environ['BB_ENV_EXTRAWHITE'] = 'DISABLE_SANITY_CHECKS'
os.environ['DISABLE_SANITY_CHECKS'] = '1'
fetchdir = settings.LAYER_FETCH_DIR
if not classic:
# Ensure we have OE-Core set up to get some base configuration
core_layer = utils.get_layer(settings.CORE_LAYER_NAME)
if not core_layer:
raise RecipeParseError("Unable to find core layer %s in database; create this layer or set the CORE_LAYER_NAME setting to point to the core layer" % settings.CORE_LAYER_NAME)
core_layerbranch = core_layer.get_layerbranch(branch.name)
core_branchname = branch.name
if core_layerbranch:
core_subdir = core_layerbranch.vcs_subdir
if core_layerbranch.actual_branch:
core_branchname = core_layerbranch.actual_branch
else:
core_subdir = 'meta'
core_urldir = core_layer.get_fetch_dir()
core_repodir = os.path.join(fetchdir, core_urldir)
core_layerdir = os.path.join(core_repodir, core_subdir)
if not nocheckout:
utils.checkout_repo(core_repodir, "origin/%s" % core_branchname, logger=logger)
if not os.path.exists(os.path.join(core_layerdir, 'conf/bitbake.conf')):
raise RecipeParseError("conf/bitbake.conf not found in core layer %s - is subdirectory set correctly?" % core_layer.name)
# The directory above where this script exists should contain our conf/layer.conf,
# so add it to BBPATH along with the core layer directory
confparentdir = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..'))
os.environ['BBPATH'] = str("%s:%s" % (confparentdir, core_layerdir))
# Change into a temporary directory so we don't write the cache and other files to the current dir
if not os.path.exists(settings.TEMP_BASE_DIR):
os.makedirs(settings.TEMP_BASE_DIR)
tempdir = tempfile.mkdtemp(dir=settings.TEMP_BASE_DIR)
saved_cwd = os.getcwd()
os.chdir(tempdir)
# We need to create a dummy bblayers.conf to avoid bitbake-cookerdaemon.log being created in <oecore>/meta/
# (see findTopdir() in bitbake/lib/bb/cookerdata.py)
os.mkdir(os.path.join(tempdir, 'conf'))
with open(os.path.join(tempdir, 'conf', 'bblayers.conf'), 'w') as f:
pass
if logger:
tinfoil = utils.setup_tinfoil(bitbakepath, enable_tracking, loglevel=logger.getEffectiveLevel())
else:
tinfoil = utils.setup_tinfoil(bitbakepath, enable_tracking)
os.chdir(saved_cwd)
# Ensure TMPDIR exists (or insane.bbclass will blow up trying to write to the QA log)
oe_tmpdir = tinfoil.config_data.getVar('TMPDIR', True)
if not os.path.exists(oe_tmpdir):
os.makedirs(oe_tmpdir)
# Ensure BBFILES as an initial value so that the old mode of BBFILES := "${BBFILES} ..." works
if not tinfoil.config_data.getVar('BBFILES', False):
tinfoil.config_data.setVar('BBFILES', '')
return (tinfoil, tempdir)
def main():
parser = optparse.OptionParser(
usage = """
%prog [options]""")
options, args = parser.parse_args(sys.argv)
utils.setup_django()
from layerindex.models import LayerItem, LayerBranch, LayerDependency
from django.db import transaction
import httplib
conn = httplib.HTTPConnection("www.openembedded.org")
conn.request("GET", "/wiki/LayerIndex?action=raw")
resp = conn.getresponse()
if resp.status in [200, 302]:
data = resp.read()
in_table = False
layer_type = 'M'
nowiki_re = re.compile(r'</?nowiki>')
link_re = re.compile(r'\[(http.*) +link\]')
readme_re = re.compile(r';f=[a-zA-Z0-9/-]*README;')
master_branch = utils.get_branch('master')
core_layer = None
transaction.enter_transaction_management()
transaction.managed(True)
try:
for line in data.splitlines():
if line.startswith('{|'):
in_table = True
continue
if in_table:
if line.startswith('|}'):
# We're done
break
elif line.startswith('!'):
section = line.split('|', 1)[1].strip("'")
if section.startswith('Base'):
layer_type = 'A'
elif section.startswith('Board'):
layer_type = 'B'
elif section.startswith('Software'):
layer_type = 'S'
elif section.startswith('Distribution'):
layer_type = 'D'
else:
layer_type = 'M'
elif not line.startswith('|-'):
if line.startswith("|| ''"):
continue
fields = line.split('||')
layer = LayerItem()
layer.name = fields[1].strip()
if ' ' in layer.name:
logger.warn('Skipping layer %s - name invalid' % layer.name)
continue
logger.info('Adding layer %s' % layer.name)
layer.status = 'P'
layer.layer_type = layer_type
layer.summary = fields[2].strip()
layer.description = layer.summary
if len(fields) > 6:
res = link_re.match(fields[6].strip())
if res:
link = res.groups(1)[0].strip()
if link.endswith('/README') or readme_re.search(link):
link = 'README'
layer.usage_url = link
repoval = nowiki_re.sub('', fields[4]).strip()
layer.vcs_url = repoval
if repoval.startswith('git://git.openembedded.org/'):
reponame = re.sub('^.*/', '', repoval)
layer.vcs_web_url = 'http://cgit.openembedded.org/cgit.cgi/' + reponame
layer.vcs_web_tree_base_url = 'http://cgit.openembedded.org/cgit.cgi/' + reponame + '/tree/%path%?h=%branch%'
layer.vcs_web_file_base_url = 'http://cgit.openembedded.org/cgit.cgi/' + reponame + '/tree/%path%?h=%branch%'
elif 'git.yoctoproject.org/' in repoval:
reponame = re.sub('^.*/', '', repoval)
layer.vcs_web_url = 'http://git.yoctoproject.org/cgit/cgit.cgi/' + reponame
layer.vcs_web_tree_base_url = 'http://git.yoctoproject.org/cgit/cgit.cgi/' + reponame + '/tree/%path%?h=%branch%'
layer.vcs_web_file_base_url = 'http://git.yoctoproject.org/cgit/cgit.cgi/' + reponame + '/tree/%path%?h=%branch%'
elif 'github.com/' in repoval:
reponame = re.sub('^.*github.com/', '', repoval)
reponame = re.sub('.git$', '', reponame)
layer.vcs_web_url = 'http://github.com/' + reponame
layer.vcs_web_tree_base_url = 'http://github.com/' + reponame + '/tree/%branch%/'
layer.vcs_web_file_base_url = 'http://github.com/' + reponame + '/blob/%branch%/'
elif 'gitorious.org/' in repoval:
reponame = re.sub('^.*gitorious.org/', '', repoval)
reponame = re.sub('.git$', '', reponame)
layer.vcs_web_url = 'http://gitorious.org/' + reponame
layer.vcs_web_tree_base_url = 'http://gitorious.org/' + reponame + '/trees/%branch%/'
layer.vcs_web_file_base_url = 'http://gitorious.org/' + reponame + '/blobs/%branch%/'
elif 'bitbucket.org/' in repoval:
reponame = re.sub('^.*bitbucket.org/', '', repoval)
reponame = re.sub('.git$', '', reponame)
layer.vcs_web_url = 'http://bitbucket.org/' + reponame
layer.vcs_web_tree_base_url = 'http://bitbucket.org/' + reponame + '/src/%branch%/%path%?at=%branch%'
layer.vcs_web_file_base_url = 'http://bitbucket.org/' + reponame + '/src/%branch%/%path%?at=%branch%'
elif '.git' in repoval:
res = link_re.match(fields[5].strip())
layer.vcs_web_url = res.groups(1)[0]
layer.vcs_web_tree_base_url = re.sub(r'\.git.*', '.git;a=tree;f=%path%;hb=%branch%', layer.vcs_web_url)
layer.vcs_web_file_base_url = re.sub(r'\.git.*', '.git;a=blob;f=%path%;hb=%branch%', layer.vcs_web_url)
layer.save()
layerbranch = LayerBranch()
layerbranch.layer = layer
layerbranch.branch = master_branch
layerbranch.vcs_subdir = fields[3].strip()
layerbranch.save()
if layer.name != 'openembedded-core':
if not core_layer:
core_layer = utils.get_layer('openembedded-core')
if core_layer:
layerdep = LayerDependency()
layerdep.layerbranch = layerbranch
layerdep.dependency = core_layer
layerdep.save()
transaction.commit()
except:
transaction.rollback()
raise
finally:
transaction.leave_transaction_management()
else:
logger.error('Fetch failed: %d: %s' % (resp.status, resp.reason))
sys.exit(0)
