def extract(self, where):
for filename in self._FILES:
content = pkgutil.get_data('bes', filename)
dst_path = path.join(where, filename)
file_util.save(dst_path, content = content, mode = 0o755)
if file_util.read(dst_path) != content:
raise RuntimeError('Failed to save %s to %s.' % (filename, dst_path))
def test_update_script(self):
config = '''
[common]
artifacts_dir: /somewhere/not/there
[test1]
description: test1
packages: orange_juice
'''
test = self._setup_test(config)
args = [
'packages', 'update', '--artifacts',
test.artifact_manager.root_dir, '--root-dir', test.tmp_dir,
'--system', 'linux', '--distro', 'ubuntu', '--distro-version',
'18', '--level', 'release', 'test1'
]
rv = self.run_script(args)
self.assertEqual(0, rv.exit_code)
args = [
'packages', 'print', '--root-dir', test.tmp_dir, '--system',
'linux', '--distro', 'ubuntu', '--distro-version', '18', '--level',
'release', 'test1'
]
rv = self.run_script(args)
self.assertEqual([
'citrus', 'fiber', 'fructose', 'fruit', 'orange', 'orange_juice',
'water'
], rv.stdout.split('\n'))
config = '''
[common]
artifacts_dir: /somewhere/not/there
[test1]
description: test1
packages: orange_juice pear_juice
'''
file_util.save(path.join(test.tmp_dir, 'config'), content=config)
cmd = [
path.join(test.tmp_dir,
'update.sh'), '--system', 'linux', '--distro', 'ubuntu',
'--distro-version', '18', '--level', 'release', 'test1'
]
env = os_env.make_clean_env(keep_keys=['PYTHONPATH'],
update={'PATH': path.dirname(self.script)})
rv = execute.execute(cmd,
raise_error=False,
env=env,
stderr_to_stdout=True)
if rv.exit_code != 0 or self.DEBUG:
self.spew(rv.stdout)
self.assertEqual(0, rv.exit_code)
args = [
'packages', 'print', '--root-dir', test.tmp_dir, '--level',
'release', '--system', 'linux', '--distro', 'ubuntu',
'--distro-version', '18', '--level', 'release', 'test1'
]
rv = self.run_script(args)
self.assertEqual([
'citrus', 'fiber', 'fructose', 'fruit', 'orange', 'orange_juice',
'pear', 'pear_juice', 'water'
], rv.stdout.split('\n'))
def save(self, where):
for inst in iter(self):
filename = '%s.rci' % (inst.name)
filepath = path.join(where, filename)
if path.exists(filepath):
return False, 'Instruction already exists: %s' % (filepath)
file_util.save(filepath, content=str(inst), mode=0o644)
return True, None
def __make_tmp_files(self): tmp_dir = temp_file.make_temp_dir() file_util.save(path.join(tmp_dir, 'foo.txt'), content = 'foo.txt\n') file_util.save(path.join(tmp_dir, 'bar.txt'), content = 'bar.txt\n') file_util.save(path.join(tmp_dir, 'kiwi.jpg'), content = 'kiwi.jpg\n') file_util.save(path.join(tmp_dir, 'kiwi.png'), content = 'kiwi.png\n') file_util.save(path.join(tmp_dir, 'orange.png'), content = 'orange.png\n') return tmp_dir
def execute(self, script, env, values, inputs):
import os.path as path
from rebuild.tools import install
from bes.fs import file_util
src_filename = inputs.get('foo_filename')
install.install(src_filename, script.python_lib_dir, mode = 0o644)
file_util.save(path.join(script.python_lib_dir, '__init__.py'), content = '', mode = 0o644)
return self.result(True)
def save(self, root_dir, variables):
filename = path.join(root_dir, self.basename)
variables = copy.deepcopy(variables)
content = string_util.replace(self.template, variables)
content = content.replace(path.expanduser('~'), '${HOME}')
if self._content_changed(filename, content):
file_util.save(filename, content=content, mode=self.mode)
return True
return False
def _command_sync(self, local_folder, remote_folder, dry_run):
print('sync %s to %s' % (local_folder, remote_folder))
print('reading local db: %s' % (local_folder))
db = source_finder_db(local_folder)
#source_tool.update_sources_index(local_folder)
#local_checksums = file_checksum_list.load_checksums_file(path.join(local_folder, 'sources_index.json'))
local_dict = db.checksum_dict()
# for k, v in local_dict.items():
# print(' LOCAL: %s %s' % (k, v))
remote_db_path = path.join(remote_folder, source_finder_db.DB_FILENAME)
print('fetching remote db: %s' % (remote_folder))
remote_db_content = self._pcloud.download_to_bytes(
file_path=remote_db_path)
remote_db = source_finder_db.make_temp_db(remote_db_content)
print('done fetching remote db: %s' % (remote_folder))
file_util.save('/tmp/caca.json', content=remote_db_content)
print('temp: %s' % (remote_db.db_filename))
#print(remote_db_json)
assert False
print('listing remote files and checksums: %s' % (remote_folder))
remote_items = self._pcloud.list_folder(folder_path=remote_folder,
recursive=True,
checksums=True)
print('done fetching remote files: %s' % (remote_folder))
remote_dict = self._items_to_dict(remote_folder, remote_items)
# for k, v in remote_dict.items():
# print('REMOTE: %s %s' % (k, v))
local_set = set(local_dict.items())
remote_set = set(remote_dict.items())
not_in_remote = local_set - remote_set
not_in_local = remote_set - local_set
for k, v in sorted(dict(not_in_remote).items()):
local_path = path.join(local_folder, k)
remote_path = path.join(remote_folder, k)
#print('NOT IN REMOTE: %s %s' % (k, v))
if dry_run:
print('DRY-RUN: would upload %s -> %s' %
(local_path, remote_path))
else:
print('Uploading: %s' % (remote_path))
self._pcloud.upload_file(local_path,
path.basename(local_path),
folder_path=path.dirname(remote_path))
if dry_run:
print('DRY-RUN: would upload db file: %s -> %s' %
(db.db_filename, remote_folder))
else:
print('Uploading db file: %s' % (db.db_filename))
self._pcloud.upload_file(db.db_filename,
path.basename(db.db_filename),
folder_path=remote_folder)
return 0
def write_file(self, filename, backup=True):
new_content = str(self)
if path.exists(filename):
old_content = file_util.read(filename)
if new_content == old_content:
return False
if backup:
file_util.backup(filename)
file_util.save(filename, new_content)
return True
def _make_temp_archive_dmg(clazz, items, filename, mode):
tmp_dir = temp_file.make_temp_dir()
for item in items:
assert item
assert item.arcname
file_util.save(path.join(tmp_dir, item.arcname), content = item.content)
tmp_dmg = temp_file.make_temp_file()
cmd = 'hdiutil create -srcfolder %s -ov -format UDZO %s' % (tmp_dir, filename)
execute.execute(cmd)
file_util.remove(tmp_dir)
def save(self):
'Save the db from its source.'
db_json = self._dict_db.to_json()
tmp_dir = temp_file.make_temp_dir()
tmp_db_filename = path.join(tmp_dir, self.DB_FILENAME)
file_util.save(tmp_db_filename, content=db_json)
self._pcloud.upload_file(tmp_db_filename,
path.basename(self._remote_db_path),
folder_path=path.dirname(
self._remote_db_path))
def create_package(clazz,
tarball_path,
pkg_desc,
build_target,
stage_dir,
timer=None):
timer = timer or debug_timer('package', disabled=True)
properties = dict_util.filter_without_keys(
pkg_desc.properties, ['export_compilation_flags_requirements'])
# Hack the export_compilation_flags_requirements property to be a plain
# string list instead of the masked config it is
key = 'export_compilation_flags_requirements'
if key in pkg_desc.properties:
properties[key] = [str(x) for x in pkg_desc.properties[key]]
files_dir = path.join(stage_dir, 'files')
timer.start('create_package - find files')
files = file_find.find(files_dir,
relative=True,
file_type=file_find.FILE | file_find.LINK)
timer.stop()
timer.start('create_package - files checksums')
files_checksum_list = file_checksum_list.from_files(files,
root_dir=files_dir)
timer.stop()
env_files_dir = path.join(stage_dir, 'env')
timer.start('create_package - find env_files')
if path.isdir(env_files_dir):
env_files = file_find.find(env_files_dir,
relative=True,
file_type=file_find.FILE
| file_find.LINK)
else:
env_files = []
timer.stop()
timer.start('create_package - env_files checksums')
env_files_checksum_list = file_checksum_list.from_files(
env_files, root_dir=env_files_dir)
timer.stop()
pkg_files = package_files(files_checksum_list, env_files_checksum_list)
metadata = package_metadata(
'', pkg_desc.name, pkg_desc.version.upstream_version,
pkg_desc.version.revision, pkg_desc.version.epoch,
build_target.system, build_target.level, build_target.arch,
build_target.distro, build_target.distro_version,
pkg_desc.requirements, properties, pkg_files)
metadata_filename = path.join(stage_dir, clazz.METADATA_FILENAME)
file_util.save(metadata_filename, content=metadata.to_json())
clazz._create_package(tarball_path, stage_dir, timer)
return clazz._create_package_result(tarball_path, metadata)
def test_install(self):
tmp_basename = 'foo.sh'
tmp_src_dir = temp_file.make_temp_dir()
tmp_filename = path.join(tmp_src_dir, tmp_basename)
file_util.save(tmp_filename, content='this is foo.')
tmp_dest_dir = temp_file.make_temp_dir()
install.install(tmp_filename, tmp_dest_dir)
dest_path = path.join(tmp_dest_dir, tmp_basename)
self.assertTrue(path.exists(dest_path))
def _make_temp_archive_xz(clazz, items, filename, mode):
tmp_dir = temp_file.make_temp_dir()
for item in items:
assert item
assert item.arcname
file_util.save(path.join(tmp_dir, item.arcname), content = item.content)
tmp_xz = temp_file.make_temp_file()
manifest_content = '\n'.join([ item.arcname for item in items ])
manifest = temp_file.make_temp_file(content = manifest_content)
cmd = 'tar Jcf %s -C %s -T %s' % (filename, tmp_dir, manifest)
execute.execute(cmd)
file_util.remove(tmp_dir)
def test_update_two_times(self):
config = '''
[common]
artifacts_dir: /somewhere/not/there
[test1]
description: test1
packages: orange_juice
'''
test = self._setup_test(config)
args = [
'packages', 'update', '--artifacts',
test.artifact_manager.root_dir, '--root-dir', test.tmp_dir,
'--system', 'linux', '--distro', 'ubuntu', '--distro-version',
'18', '--level', 'release', 'test1'
]
rv = self.run_script(args)
self.assertEqual(0, rv.exit_code)
args = [
'packages', 'print', '--root-dir', test.tmp_dir, '--system',
'linux', '--distro', 'ubuntu', '--distro-version', '18', '--level',
'release', 'test1'
]
rv = self.run_script(args)
self.assertEqual([
'citrus', 'fiber', 'fructose', 'fruit', 'orange', 'orange_juice',
'water'
], rv.stdout.split('\n'))
config = '''
[common]
artifacts_dir: /somewhere/not/there
[test1]
description: test1
packages: orange_juice pear_juice
'''
file_util.save(path.join(test.tmp_dir, 'config'), content=config)
args = [
'packages', 'update', '--artifacts',
test.artifact_manager.root_dir, '--root-dir', test.tmp_dir,
'--system', 'linux', '--distro', 'ubuntu', '--distro-version',
'18', '--level', 'release', 'test1'
]
rv = self.run_script(args)
self.assertEqual(0, rv.exit_code)
args = [
'packages', 'print', '--root-dir', test.tmp_dir, '--system',
'linux', '--distro', 'ubuntu', '--distro-version', '18', '--level',
'release', 'test1'
]
rv = self.run_script(args)
self.assertEqual([
'citrus', 'fiber', 'fructose', 'fruit', 'orange', 'orange_juice',
'pear', 'pear_juice', 'water'
], rv.stdout.split('\n'))
def update_egg_directory(clazz, d):
if not path.exists(d):
return
if not path.isdir(d):
raise RuntimeError('Not a directory: %s' % (d))
eggs = clazz.list_eggs(d)
eggs = [ './%s' % (egg) for egg in eggs ]
eggs_content = '\n'.join(eggs)
easy_install_dot_pth = path.join(d, clazz.EASY_INSTALL_DOT_PTH_FILENAME)
easy_install_dot_pth_content = clazz.EASY_INSTALL_DOT_PTH_TEMPLATE % (eggs_content)
file_util.save(easy_install_dot_pth, content = easy_install_dot_pth_content, mode = 0o644)
clazz.update_site_dot_py(d)
def _load_db_remote(self):
try:
self.blurb('pcloud: using remote db: %s' %
(self._remote_db_file_path))
content = self.pcloud.download_to_bytes(
file_path=self._remote_db_file_path)
sf = source_finder_db_dict.from_json(content)
file_util.save(path.join(self.local_root_dir,
source_finder_db_dict.DB_FILENAME),
content=content)
return sf
except pcloud_error as ex:
return source_finder_db_dict()
def update_site_dot_py(clazz, d):
if not path.exists(d):
return
if not path.isdir(d):
raise RuntimeError('Not a directory: %s' % (d))
site_py_path = path.join(d, clazz.SITE_DOT_PY_FILENAME)
old_content = None
if path.exists(site_py_path):
if not path.isfile(site_py_path):
raise RuntimeError('Not a regular file: %s' % (site_py_path))
old_content = file_util.read(site_py_path)
if old_content == clazz.SITE_DOT_PY_CONTENT:
return
file_util.save(site_py_path, content = clazz.SITE_DOT_PY_CONTENT, mode = 0o644)
def test_update_egg_directory(self):
tmp_dir = temp_file.make_temp_dir()
eggs = [
'foo-1.2.3-py2.7.egg',
'bar-6.6.6-py2.7.egg',
'baz-10.11.12-py2.7.egg',
]
for egg in eggs:
file_util.save(path.join(tmp_dir, egg), content = '%s\n' % (egg))
setup_tools.update_egg_directory(tmp_dir)
easy_install_dot_pth_path = path.join(tmp_dir, setup_tools.EASY_INSTALL_DOT_PTH_FILENAME)
actual_eggs = setup_tools.read_easy_install_pth(easy_install_dot_pth_path)
self.assertEqual( sorted(eggs), sorted(actual_eggs) )
def test_trash_one(self): ctx = self._make_context(timeout = 0.100) ctx.trash.start() tmp = file_util.save(path.join(ctx.stuff_dir, 'foo.txt'), content = 'foo\n') self.assertEqual( [ 'foo.txt' ], dir_util.list(ctx.stuff_dir, relative = True) ) ctx.trash.trash(tmp) time.sleep(0.250) self.assertEqual( [], dir_util.list(ctx.stuff_dir, relative = True) ) ctx.trash.stop()
def main():
ap = argparse.ArgumentParser()
ap.add_argument('filename', type = str, action = 'store', help = 'The platforms.txt file.')
ap.add_argument('output', type = str, action = 'store', help = 'The platforms.txt file.')
args = ap.parse_args()
content = file_util.read(args.filename)
parser = TLP(content)
lines = TLP.parse_lines(content, strip_comments = True, strip_text = True, remove_empties = True)
infos = parse_host_infos(lines)
s = make_host_info_python_list(infos)
file_util.save(args.output, content = s)
# j = json_util.to_json(infos, indent = 2)
# for info in infos:
# print(host_info_to_python_str(info))
# print(j)
return 0
def _write_retry_script(clazz, command, env, script):
from bes.compat import StringIO
s = StringIO()
s.write('#!/bin/bash\n')
s.write('mkdir -p %s\n' % (script.staged_files_dir))
items = sorted(env.items())
last_item = items.pop(-1)
def _item_str(key, value, slash):
return '%s=\"%s\"%s\n' % (key, value, slash)
for key, value in items:
s.write(_item_str(key, value, '\\'))
s.write(_item_str(last_item[0], last_item[1], ''))
if string_util.is_string(command):
s.write(command)
else:
s.write(' '.join(command))
content = s.getvalue()
file_path = path.join(script.build_dir, clazz.RETRY_SCRIPT_FILENAME)
file_util.save(file_path, content = content, mode = 0o755)
return file_path
def __write_example_modules(self):
tmp_dir = temp_file.make_temp_dir()
file_util.save(path.join(tmp_dir, 'foo.pc'), self.FOO_PC)
file_util.save(path.join(tmp_dir, 'bar.pc'), self.BAR_PC)
file_util.save(path.join(tmp_dir, 'foo_dup.pc'), self.FOO_DUP_PC)
return tmp_dir
def _make_junk(name, content):
obj = _resolve_filename(_object_from_source(name, 'txt', None))
file_util.save(obj, content=content, mode=0644)
def execute(self, script, env, values, inputs):
import os.path as path
from bes.fs import file_util
foo_filename = path.join(script.build_dir, 'foo.py')
file_util.save(foo_filename, content = self.CONTENT, mode = 0o644)
return self.result(True, outputs = { 'foo_filename': foo_filename })
def save(self, info): assert archiver.is_valid(self._archive) archiver.extract_member_to_file(self._archive, self._member, info.cached_filename) # The checksum here is for the archive not the member file_util.save(info.checksum_filename, self._checksum + '\n')
def _create_tmp_files(self, tmp_repo): foo = path.join(tmp_repo, 'foo.txt') bar = path.join(tmp_repo, 'bar.txt') file_util.save(foo, content = 'foo.txt\n') file_util.save(bar, content = 'bar.txt\n') return [ 'bar.txt', 'foo.txt' ]
def write_items(clazz, root_dir, items):
'Write the content to disk.'
for item in items:
assert item
assert item.arcname
file_util.save(path.join(root_dir, item.arcname), item.content)
def write(self, where): return file_util.save(path.join(where, self.filename), content = self.source_code)
def _make_temp_source(tmp_dir, filename, content):
return file_util.save(path.join(tmp_dir, filename), content=content)
def make_temp_db(clazz, db_content, delete=True):
root = temp_file.make_temp_dir(delete=delete)
db_filename = path.join(root, clazz.DB_FILENAME)
file_util.save(db_filename, content=db_content)
return clazz(root)
def _setup_test(clazz, config):
tmp_dir = clazz._make_temp_dir()
am = clazz._make_test_artifact_manager()
file_util.save(path.join(tmp_dir, 'config'), content=config)
return clazz._setup(tmp_dir, am)