def extract(self, where): for filename in self._FILES: content = pkgutil.get_data('bes', filename) dst_path = path.join(where, filename) file_util.save(dst_path, content = content, mode = 0o755) if file_util.read(dst_path) != content: raise RuntimeError('Failed to save %s to %s.' % (filename, dst_path))
def test_update_script(self): config = ''' [common] artifacts_dir: /somewhere/not/there [test1] description: test1 packages: orange_juice ''' test = self._setup_test(config) args = [ 'packages', 'update', '--artifacts', test.artifact_manager.root_dir, '--root-dir', test.tmp_dir, '--system', 'linux', '--distro', 'ubuntu', '--distro-version', '18', '--level', 'release', 'test1' ] rv = self.run_script(args) self.assertEqual(0, rv.exit_code) args = [ 'packages', 'print', '--root-dir', test.tmp_dir, '--system', 'linux', '--distro', 'ubuntu', '--distro-version', '18', '--level', 'release', 'test1' ] rv = self.run_script(args) self.assertEqual([ 'citrus', 'fiber', 'fructose', 'fruit', 'orange', 'orange_juice', 'water' ], rv.stdout.split('\n')) config = ''' [common] artifacts_dir: /somewhere/not/there [test1] description: test1 packages: orange_juice pear_juice ''' file_util.save(path.join(test.tmp_dir, 'config'), content=config) cmd = [ path.join(test.tmp_dir, 'update.sh'), '--system', 'linux', '--distro', 'ubuntu', '--distro-version', '18', '--level', 'release', 'test1' ] env = os_env.make_clean_env(keep_keys=['PYTHONPATH'], update={'PATH': path.dirname(self.script)}) rv = execute.execute(cmd, raise_error=False, env=env, stderr_to_stdout=True) if rv.exit_code != 0 or self.DEBUG: self.spew(rv.stdout) self.assertEqual(0, rv.exit_code) args = [ 'packages', 'print', '--root-dir', test.tmp_dir, '--level', 'release', '--system', 'linux', '--distro', 'ubuntu', '--distro-version', '18', '--level', 'release', 'test1' ] rv = self.run_script(args) self.assertEqual([ 'citrus', 'fiber', 'fructose', 'fruit', 'orange', 'orange_juice', 'pear', 'pear_juice', 'water' ], rv.stdout.split('\n'))
def save(self, where): for inst in iter(self): filename = '%s.rci' % (inst.name) filepath = path.join(where, filename) if path.exists(filepath): return False, 'Instruction already exists: %s' % (filepath) file_util.save(filepath, content=str(inst), mode=0o644) return True, None
def __make_tmp_files(self): tmp_dir = temp_file.make_temp_dir() file_util.save(path.join(tmp_dir, 'foo.txt'), content = 'foo.txt\n') file_util.save(path.join(tmp_dir, 'bar.txt'), content = 'bar.txt\n') file_util.save(path.join(tmp_dir, 'kiwi.jpg'), content = 'kiwi.jpg\n') file_util.save(path.join(tmp_dir, 'kiwi.png'), content = 'kiwi.png\n') file_util.save(path.join(tmp_dir, 'orange.png'), content = 'orange.png\n') return tmp_dir
def execute(self, script, env, values, inputs): import os.path as path from rebuild.tools import install from bes.fs import file_util src_filename = inputs.get('foo_filename') install.install(src_filename, script.python_lib_dir, mode = 0o644) file_util.save(path.join(script.python_lib_dir, '__init__.py'), content = '', mode = 0o644) return self.result(True)
def save(self, root_dir, variables): filename = path.join(root_dir, self.basename) variables = copy.deepcopy(variables) content = string_util.replace(self.template, variables) content = content.replace(path.expanduser('~'), '${HOME}') if self._content_changed(filename, content): file_util.save(filename, content=content, mode=self.mode) return True return False
def _command_sync(self, local_folder, remote_folder, dry_run): print('sync %s to %s' % (local_folder, remote_folder)) print('reading local db: %s' % (local_folder)) db = source_finder_db(local_folder) #source_tool.update_sources_index(local_folder) #local_checksums = file_checksum_list.load_checksums_file(path.join(local_folder, 'sources_index.json')) local_dict = db.checksum_dict() # for k, v in local_dict.items(): # print(' LOCAL: %s %s' % (k, v)) remote_db_path = path.join(remote_folder, source_finder_db.DB_FILENAME) print('fetching remote db: %s' % (remote_folder)) remote_db_content = self._pcloud.download_to_bytes( file_path=remote_db_path) remote_db = source_finder_db.make_temp_db(remote_db_content) print('done fetching remote db: %s' % (remote_folder)) file_util.save('/tmp/caca.json', content=remote_db_content) print('temp: %s' % (remote_db.db_filename)) #print(remote_db_json) assert False print('listing remote files and checksums: %s' % (remote_folder)) remote_items = self._pcloud.list_folder(folder_path=remote_folder, recursive=True, checksums=True) print('done fetching remote files: %s' % (remote_folder)) remote_dict = self._items_to_dict(remote_folder, remote_items) # for k, v in remote_dict.items(): # print('REMOTE: %s %s' % (k, v)) local_set = set(local_dict.items()) remote_set = set(remote_dict.items()) not_in_remote = local_set - remote_set not_in_local = remote_set - local_set for k, v in sorted(dict(not_in_remote).items()): local_path = path.join(local_folder, k) remote_path = path.join(remote_folder, k) #print('NOT IN REMOTE: %s %s' % (k, v)) if dry_run: print('DRY-RUN: would upload %s -> %s' % (local_path, remote_path)) else: print('Uploading: %s' % (remote_path)) self._pcloud.upload_file(local_path, path.basename(local_path), folder_path=path.dirname(remote_path)) if dry_run: print('DRY-RUN: would upload db file: %s -> %s' % (db.db_filename, remote_folder)) else: print('Uploading db file: %s' % (db.db_filename)) self._pcloud.upload_file(db.db_filename, path.basename(db.db_filename), folder_path=remote_folder) return 0
def write_file(self, filename, backup=True): new_content = str(self) if path.exists(filename): old_content = file_util.read(filename) if new_content == old_content: return False if backup: file_util.backup(filename) file_util.save(filename, new_content) return True
def _make_temp_archive_dmg(clazz, items, filename, mode): tmp_dir = temp_file.make_temp_dir() for item in items: assert item assert item.arcname file_util.save(path.join(tmp_dir, item.arcname), content = item.content) tmp_dmg = temp_file.make_temp_file() cmd = 'hdiutil create -srcfolder %s -ov -format UDZO %s' % (tmp_dir, filename) execute.execute(cmd) file_util.remove(tmp_dir)
def save(self): 'Save the db from its source.' db_json = self._dict_db.to_json() tmp_dir = temp_file.make_temp_dir() tmp_db_filename = path.join(tmp_dir, self.DB_FILENAME) file_util.save(tmp_db_filename, content=db_json) self._pcloud.upload_file(tmp_db_filename, path.basename(self._remote_db_path), folder_path=path.dirname( self._remote_db_path))
def create_package(clazz, tarball_path, pkg_desc, build_target, stage_dir, timer=None): timer = timer or debug_timer('package', disabled=True) properties = dict_util.filter_without_keys( pkg_desc.properties, ['export_compilation_flags_requirements']) # Hack the export_compilation_flags_requirements property to be a plain # string list instead of the masked config it is key = 'export_compilation_flags_requirements' if key in pkg_desc.properties: properties[key] = [str(x) for x in pkg_desc.properties[key]] files_dir = path.join(stage_dir, 'files') timer.start('create_package - find files') files = file_find.find(files_dir, relative=True, file_type=file_find.FILE | file_find.LINK) timer.stop() timer.start('create_package - files checksums') files_checksum_list = file_checksum_list.from_files(files, root_dir=files_dir) timer.stop() env_files_dir = path.join(stage_dir, 'env') timer.start('create_package - find env_files') if path.isdir(env_files_dir): env_files = file_find.find(env_files_dir, relative=True, file_type=file_find.FILE | file_find.LINK) else: env_files = [] timer.stop() timer.start('create_package - env_files checksums') env_files_checksum_list = file_checksum_list.from_files( env_files, root_dir=env_files_dir) timer.stop() pkg_files = package_files(files_checksum_list, env_files_checksum_list) metadata = package_metadata( '', pkg_desc.name, pkg_desc.version.upstream_version, pkg_desc.version.revision, pkg_desc.version.epoch, build_target.system, build_target.level, build_target.arch, build_target.distro, build_target.distro_version, pkg_desc.requirements, properties, pkg_files) metadata_filename = path.join(stage_dir, clazz.METADATA_FILENAME) file_util.save(metadata_filename, content=metadata.to_json()) clazz._create_package(tarball_path, stage_dir, timer) return clazz._create_package_result(tarball_path, metadata)
def test_install(self): tmp_basename = 'foo.sh' tmp_src_dir = temp_file.make_temp_dir() tmp_filename = path.join(tmp_src_dir, tmp_basename) file_util.save(tmp_filename, content='this is foo.') tmp_dest_dir = temp_file.make_temp_dir() install.install(tmp_filename, tmp_dest_dir) dest_path = path.join(tmp_dest_dir, tmp_basename) self.assertTrue(path.exists(dest_path))
def _make_temp_archive_xz(clazz, items, filename, mode): tmp_dir = temp_file.make_temp_dir() for item in items: assert item assert item.arcname file_util.save(path.join(tmp_dir, item.arcname), content = item.content) tmp_xz = temp_file.make_temp_file() manifest_content = '\n'.join([ item.arcname for item in items ]) manifest = temp_file.make_temp_file(content = manifest_content) cmd = 'tar Jcf %s -C %s -T %s' % (filename, tmp_dir, manifest) execute.execute(cmd) file_util.remove(tmp_dir)
def test_update_two_times(self): config = ''' [common] artifacts_dir: /somewhere/not/there [test1] description: test1 packages: orange_juice ''' test = self._setup_test(config) args = [ 'packages', 'update', '--artifacts', test.artifact_manager.root_dir, '--root-dir', test.tmp_dir, '--system', 'linux', '--distro', 'ubuntu', '--distro-version', '18', '--level', 'release', 'test1' ] rv = self.run_script(args) self.assertEqual(0, rv.exit_code) args = [ 'packages', 'print', '--root-dir', test.tmp_dir, '--system', 'linux', '--distro', 'ubuntu', '--distro-version', '18', '--level', 'release', 'test1' ] rv = self.run_script(args) self.assertEqual([ 'citrus', 'fiber', 'fructose', 'fruit', 'orange', 'orange_juice', 'water' ], rv.stdout.split('\n')) config = ''' [common] artifacts_dir: /somewhere/not/there [test1] description: test1 packages: orange_juice pear_juice ''' file_util.save(path.join(test.tmp_dir, 'config'), content=config) args = [ 'packages', 'update', '--artifacts', test.artifact_manager.root_dir, '--root-dir', test.tmp_dir, '--system', 'linux', '--distro', 'ubuntu', '--distro-version', '18', '--level', 'release', 'test1' ] rv = self.run_script(args) self.assertEqual(0, rv.exit_code) args = [ 'packages', 'print', '--root-dir', test.tmp_dir, '--system', 'linux', '--distro', 'ubuntu', '--distro-version', '18', '--level', 'release', 'test1' ] rv = self.run_script(args) self.assertEqual([ 'citrus', 'fiber', 'fructose', 'fruit', 'orange', 'orange_juice', 'pear', 'pear_juice', 'water' ], rv.stdout.split('\n'))
def update_egg_directory(clazz, d): if not path.exists(d): return if not path.isdir(d): raise RuntimeError('Not a directory: %s' % (d)) eggs = clazz.list_eggs(d) eggs = [ './%s' % (egg) for egg in eggs ] eggs_content = '\n'.join(eggs) easy_install_dot_pth = path.join(d, clazz.EASY_INSTALL_DOT_PTH_FILENAME) easy_install_dot_pth_content = clazz.EASY_INSTALL_DOT_PTH_TEMPLATE % (eggs_content) file_util.save(easy_install_dot_pth, content = easy_install_dot_pth_content, mode = 0o644) clazz.update_site_dot_py(d)
def _load_db_remote(self): try: self.blurb('pcloud: using remote db: %s' % (self._remote_db_file_path)) content = self.pcloud.download_to_bytes( file_path=self._remote_db_file_path) sf = source_finder_db_dict.from_json(content) file_util.save(path.join(self.local_root_dir, source_finder_db_dict.DB_FILENAME), content=content) return sf except pcloud_error as ex: return source_finder_db_dict()
def update_site_dot_py(clazz, d): if not path.exists(d): return if not path.isdir(d): raise RuntimeError('Not a directory: %s' % (d)) site_py_path = path.join(d, clazz.SITE_DOT_PY_FILENAME) old_content = None if path.exists(site_py_path): if not path.isfile(site_py_path): raise RuntimeError('Not a regular file: %s' % (site_py_path)) old_content = file_util.read(site_py_path) if old_content == clazz.SITE_DOT_PY_CONTENT: return file_util.save(site_py_path, content = clazz.SITE_DOT_PY_CONTENT, mode = 0o644)
def test_update_egg_directory(self): tmp_dir = temp_file.make_temp_dir() eggs = [ 'foo-1.2.3-py2.7.egg', 'bar-6.6.6-py2.7.egg', 'baz-10.11.12-py2.7.egg', ] for egg in eggs: file_util.save(path.join(tmp_dir, egg), content = '%s\n' % (egg)) setup_tools.update_egg_directory(tmp_dir) easy_install_dot_pth_path = path.join(tmp_dir, setup_tools.EASY_INSTALL_DOT_PTH_FILENAME) actual_eggs = setup_tools.read_easy_install_pth(easy_install_dot_pth_path) self.assertEqual( sorted(eggs), sorted(actual_eggs) )
def test_trash_one(self): ctx = self._make_context(timeout = 0.100) ctx.trash.start() tmp = file_util.save(path.join(ctx.stuff_dir, 'foo.txt'), content = 'foo\n') self.assertEqual( [ 'foo.txt' ], dir_util.list(ctx.stuff_dir, relative = True) ) ctx.trash.trash(tmp) time.sleep(0.250) self.assertEqual( [], dir_util.list(ctx.stuff_dir, relative = True) ) ctx.trash.stop()
def main(): ap = argparse.ArgumentParser() ap.add_argument('filename', type = str, action = 'store', help = 'The platforms.txt file.') ap.add_argument('output', type = str, action = 'store', help = 'The platforms.txt file.') args = ap.parse_args() content = file_util.read(args.filename) parser = TLP(content) lines = TLP.parse_lines(content, strip_comments = True, strip_text = True, remove_empties = True) infos = parse_host_infos(lines) s = make_host_info_python_list(infos) file_util.save(args.output, content = s) # j = json_util.to_json(infos, indent = 2) # for info in infos: # print(host_info_to_python_str(info)) # print(j) return 0
def _write_retry_script(clazz, command, env, script): from bes.compat import StringIO s = StringIO() s.write('#!/bin/bash\n') s.write('mkdir -p %s\n' % (script.staged_files_dir)) items = sorted(env.items()) last_item = items.pop(-1) def _item_str(key, value, slash): return '%s=\"%s\"%s\n' % (key, value, slash) for key, value in items: s.write(_item_str(key, value, '\\')) s.write(_item_str(last_item[0], last_item[1], '')) if string_util.is_string(command): s.write(command) else: s.write(' '.join(command)) content = s.getvalue() file_path = path.join(script.build_dir, clazz.RETRY_SCRIPT_FILENAME) file_util.save(file_path, content = content, mode = 0o755) return file_path
def __write_example_modules(self): tmp_dir = temp_file.make_temp_dir() file_util.save(path.join(tmp_dir, 'foo.pc'), self.FOO_PC) file_util.save(path.join(tmp_dir, 'bar.pc'), self.BAR_PC) file_util.save(path.join(tmp_dir, 'foo_dup.pc'), self.FOO_DUP_PC) return tmp_dir
def _make_junk(name, content): obj = _resolve_filename(_object_from_source(name, 'txt', None)) file_util.save(obj, content=content, mode=0644)
def execute(self, script, env, values, inputs): import os.path as path from bes.fs import file_util foo_filename = path.join(script.build_dir, 'foo.py') file_util.save(foo_filename, content = self.CONTENT, mode = 0o644) return self.result(True, outputs = { 'foo_filename': foo_filename })
def save(self, info): assert archiver.is_valid(self._archive) archiver.extract_member_to_file(self._archive, self._member, info.cached_filename) # The checksum here is for the archive not the member file_util.save(info.checksum_filename, self._checksum + '\n')
def _create_tmp_files(self, tmp_repo): foo = path.join(tmp_repo, 'foo.txt') bar = path.join(tmp_repo, 'bar.txt') file_util.save(foo, content = 'foo.txt\n') file_util.save(bar, content = 'bar.txt\n') return [ 'bar.txt', 'foo.txt' ]
def write_items(clazz, root_dir, items): 'Write the content to disk.' for item in items: assert item assert item.arcname file_util.save(path.join(root_dir, item.arcname), item.content)
def write(self, where): return file_util.save(path.join(where, self.filename), content = self.source_code)
def _make_temp_source(tmp_dir, filename, content): return file_util.save(path.join(tmp_dir, filename), content=content)
def make_temp_db(clazz, db_content, delete=True): root = temp_file.make_temp_dir(delete=delete) db_filename = path.join(root, clazz.DB_FILENAME) file_util.save(db_filename, content=db_content) return clazz(root)
def _setup_test(clazz, config): tmp_dir = clazz._make_temp_dir() am = clazz._make_test_artifact_manager() file_util.save(path.join(tmp_dir, 'config'), content=config) return clazz._setup(tmp_dir, am)