def install(defs, this, component):
if os.path.exists(os.path.join(this['sandbox'], 'baserock',
component['name'] + '.meta')):
return
if app.config.get('log-verbose'):
app.log(this, 'Installing %s' % component['cache'])
unpackdir = cache.get_cache(defs, component) + '.unpacked'
if this.get('kind') is 'system':
utils.copy_all_files(unpackdir, this['sandbox'])
else:
utils.hardlink_all_files(unpackdir, this['sandbox'])
def install(defs, this, component):
# populate this['sandbox'] with the artifact files from component
if os.path.exists(os.path.join(this["sandbox"], "baserock", component["name"] + ".meta")):
return
if app.config.get("log-verbose"):
app.log(this, "Installing %s" % component["cache"])
if cache.get_cache(defs, component) is False:
app.exit(this, "ERROR: unable to get cache for", component["name"])
unpackdir = cache.get_cache(defs, component) + ".unpacked"
if this.get("kind") is "system":
utils.copy_all_files(unpackdir, this["sandbox"])
else:
utils.hardlink_all_files(unpackdir, this["sandbox"])
def install(dn, component):
# populate dn['sandbox'] with the artifact files from component
if os.path.exists(os.path.join(dn['sandbox'], 'baserock',
component['name'] + '.meta')):
return
app.log(dn, 'Sandbox: installing %s' % component['cache'], verbose=True)
if cache.get_cache(component) is False:
app.log(dn, 'Unable to get cache for', component['name'], exit=True)
unpackdir = cache.get_cache(component) + '.unpacked'
if dn.get('kind') is 'system':
utils.copy_all_files(unpackdir, dn['sandbox'])
else:
utils.hardlink_all_files(unpackdir, dn['sandbox'])
def install(defs, this, component):
# populate this['sandbox'] with the artifact files from component
if os.path.exists(os.path.join(this['sandbox'], 'baserock',
component['name'] + '.meta')):
return
if app.config.get('log-verbose'):
app.log(this, 'Sandbox: installing %s' % component['cache'])
if cache.get_cache(defs, component) is False:
app.exit(this, 'ERROR: unable to get cache for', component['name'])
unpackdir = cache.get_cache(defs, component) + '.unpacked'
if this.get('kind') is 'system':
utils.copy_all_files(unpackdir, this['sandbox'])
else:
utils.hardlink_all_files(unpackdir, this['sandbox'])
def install(dn, component):
# populate dn['sandbox'] with the artifact files from component
if os.path.exists(
os.path.join(dn['sandbox'], 'baserock',
component['name'] + '.meta')):
return
app.log(dn, 'Sandbox: installing %s' % component['cache'], verbose=True)
if cache.get_cache(component) is False:
app.log(dn, 'Unable to get cache for', component['name'], exit=True)
unpackdir = cache.get_cache(component) + '.unpacked'
if dn.get('kind') is 'system':
utils.copy_all_files(unpackdir, dn['sandbox'])
else:
utils.hardlink_all_files(unpackdir, dn['sandbox'])
def install(defs, this, component):
# populate this['sandbox'] with the artifact files from component
if os.path.exists(
os.path.join(this['sandbox'], 'baserock',
component['name'] + '.meta')):
return
if app.config.get('log-verbose'):
app.log(this, 'Installing %s' % component['cache'])
if cache.get_cache(defs, component) is False:
app.exit(this, 'ERROR: unable to get cache for', component['name'])
unpackdir = cache.get_cache(defs, component) + '.unpacked'
if this.get('kind') is 'system':
utils.copy_all_files(unpackdir, this['sandbox'])
else:
utils.hardlink_all_files(unpackdir, this['sandbox'])
def _install(defs, this, component):
if os.path.exists(os.path.join(this['sandbox'], 'baserock',
component['name'] + '.meta')):
return
for it in component.get('build-depends', []):
dependency = defs.get(it)
if (dependency.get('build-mode', 'staging') ==
component.get('build-mode', 'staging')):
_install(defs, this, dependency)
for it in component.get('contents', []):
subcomponent = defs.get(it)
if subcomponent.get('build-mode', 'staging') != 'bootstrap':
_install(defs, this, subcomponent)
unpackdir = cache.unpack(defs, component)
if this.get('kind') is 'system':
utils.copy_all_files(unpackdir, this['sandbox'])
else:
utils.hardlink_all_files(unpackdir, this['sandbox'])
def brightness_contrast_augment_data(num_of_imgs_to_have=103,
overwrite_folder=False):
if os.path.exists(augmented_data_path) and overwrite_folder:
shutil.rmtree(augmented_data_path)
random.seed(time.time())
utils.makedirs_ok(augmented_data_path)
utils.copy_all_files(my_model_data, augmented_data_path, "*.jpg")
original_images = glob.glob(augmented_data_path + "/*.jpg")
healthy_imgs = glob.glob(augmented_data_path + "/healthy_*.jpg")
kc_imgs = glob.glob(augmented_data_path + "/kc_*.jpg")
cly_imgs = glob.glob(augmented_data_path + "/cly_*.jpg")
sus_imgs = glob.glob(augmented_data_path + "/sus_*.jpg")
functions = [contrast, gaussian_blur]
for target in [healthy_imgs, kc_imgs, cly_imgs, sus_imgs]:
img_i = 0
for i in range(num_of_imgs_to_have - len(target)):
img_basename = path.basename((target[img_i]))
new_basename = img_basename.partition(".jpg")[0] + "_" + str(
i) + ".jpg"
new_path = path.dirname(target[img_i]) + "/" + new_basename
shutil.copy(target[img_i], new_path)
new_img = misc.imread(new_path)
#
num_of_functions_to_use = random.randint(1, len(functions))
# new_img = random.choice(functions)(new_img)
for function in random.sample(functions, num_of_functions_to_use):
new_img = function(new_img)
# new_img = contrast(new_img)
misc.imsave(new_path, new_img, "JPEG")
img_i = (img_i + 1) % len(target)
def augment_every_image_twice(overwrite_folder=False):
if os.path.exists(augment_every_image_twice_path) and overwrite_folder:
shutil.rmtree(augment_every_image_twice_path)
random.seed(time.time())
utils.makedirs_ok(augment_every_image_twice_path)
utils.copy_all_files(my_model_data, augment_every_image_twice_path,
"*.jpg")
healthy_imgs = glob.glob(augment_every_image_twice_path + "/healthy_*.jpg")
kc_imgs = glob.glob(augment_every_image_twice_path + "/kc_*.jpg")
cly_imgs = glob.glob(augment_every_image_twice_path + "/cly_*.jpg")
sus_imgs = glob.glob(augment_every_image_twice_path + "/sus_*.jpg")
functions = [contrast, gaussian_blur]
for target in [healthy_imgs, kc_imgs, cly_imgs, sus_imgs]:
for img in target:
img_basename = path.basename(img)
contrast_new_basename = img_basename.partition(
".jpg")[0] + "_" + str(1) + ".jpg"
new_path = path.dirname(img) + "/" + contrast_new_basename
shutil.copy(img, new_path)
new_img = misc.imread(new_path)
new_img = contrast(new_img)
misc.imsave(new_path, new_img, "JPEG")
blur_new_basename = img_basename.partition(".jpg")[0] + "_" + str(
2) + ".jpg"
new_path = path.dirname(img) + "/" + blur_new_basename
shutil.copy(img, new_path)
new_img = misc.imread(new_path)
new_img = gaussian_blur(new_img)
misc.imsave(new_path, new_img, "JPEG")
def install_split_artifacts(defs, component, stratum, artifacts):
'''Create the .meta files for a split stratum
Given a stratum and a list of artifacts to split, writes new .meta files to
the baserock dir in the 'sandbox' dir of the component and copies the files
from the .unpacked directory of each individual chunk to the sandbox
'''
if os.path.exists(os.path.join(component['sandbox'], 'baserock',
stratum['name'] + '.meta')):
return
if artifacts is None:
artifacts = []
default_artifacts = defs.defaults.get_split_rules('stratum')
for split in config.get('default-splits', []):
artifacts += [stratum['name'] + split]
log(component, 'Installing %s splits' % stratum['name'], artifacts)
stratum_metadata = get_metadata(defs, stratum)
split_stratum_metadata = {}
split_stratum_metadata['products'] = []
components = []
for product in stratum_metadata['products']:
for artifact in artifacts:
if artifact == product['artifact']:
components += product['components']
split_stratum_metadata['products'].append(product)
log(component, 'Splitting artifacts:', artifacts, verbose=True)
log(component, 'Splitting components:', components, verbose=True)
baserockpath = os.path.join(component['sandbox'], 'baserock')
if not os.path.isdir(baserockpath):
os.mkdir(baserockpath)
split_stratum_metafile = os.path.join(baserockpath,
stratum['name'] + '.meta')
with open(split_stratum_metafile, "w") as f:
yaml.safe_dump(split_stratum_metadata, f, default_flow_style=False)
for path in stratum['contents']:
chunk = defs.get(path)
if chunk.get('build-mode', 'staging') == 'bootstrap':
continue
if not get_cache(defs, chunk):
exit(stratum, 'ERROR: artifact not found', chunk.get('name'))
try:
metafile = path_to_metafile(defs, chunk)
with open(metafile, "r") as f:
filelist = []
metadata = yaml.safe_load(f)
split_metadata = {'ref': metadata.get('ref'),
'repo': metadata.get('repo'),
'products': []}
if config.get('artifact-version', 0) not in [0, 1]:
metadata['cache'] = component.get('cache')
for product in metadata['products']:
if product['artifact'] in components:
filelist += product.get('components', [])
# handle old artifacts still containing 'files'
filelist += product.get('files', [])
split_metadata['products'].append(product)
if split_metadata['products'] != []:
split_metafile = os.path.join(baserockpath,
os.path.basename(metafile))
with open(split_metafile, "w") as f:
yaml.safe_dump(split_metadata, f,
default_flow_style=False)
cachepath, cachedir = os.path.split(get_cache(defs, chunk))
path = os.path.join(cachepath, cachedir + '.unpacked')
utils.copy_file_list(path, component['sandbox'], filelist)
except:
# if we got here, something has gone badly wrong parsing metadata
# or copying files into the sandbox...
log(stratum, 'WARNING: failed copying files from', metafile)
log(stratum, 'WARNING: copying *all* files')
utils.copy_all_files(path, component['sandbox'])