def copy_osg_post_scripts(stage_dir_abs, post_scripts_dir, dver, basearch):
"""Copy osg scripts from post_scripts_dir to the stage2 directory"""
if not os.path.isdir(post_scripts_dir):
raise Error("script directory (%r) not found" % post_scripts_dir)
post_scripts_dir_abs = os.path.abspath(post_scripts_dir)
dest_dir = os.path.join(stage_dir_abs, "osg")
safe_makedirs(dest_dir)
for script_name in 'osg-post-install', 'osgrun.in':
script_path = os.path.join(post_scripts_dir_abs, script_name)
dest_path = os.path.join(dest_dir, script_name)
try:
shutil.copyfile(script_path, dest_path)
os.chmod(dest_path, 0755)
except EnvironmentError as err:
raise Error("unable to copy script (%r) to (%r): %s" %
(script_path, dest_dir, str(err)))
try:
envsetup.write_setup_in_files(dest_dir, dver, basearch)
except EnvironmentError as err:
raise Error(
"unable to create environment script templates (setup.csh.in, setup.sh.in): %s"
% str(err))
def extract(*, infile, outfile, platform, quiet=False):
"""Extract keycodes from a header file.
Arguments:
infile: Input file path, or None to automatically locate it
outfile: Output file path, or None to automatically place it
platform: Name of platform
"""
read_table = PLATFORMS[platform]
try:
table = list(read_table(infile))
except FileNotFoundError as ex:
raise Error("Input file not found: {!r}".format(infile))
if not table:
raise Error(
"Could not find keycode definitions in input file {!r}".format(
infile))
die("Could not find keycode definitions in input file.")
if outfile is None:
repo = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
outfile = os.path.join(repo, "data",
"{}_scancodes.csv".format(platform))
if not quiet:
print("Writing {}".format(outfile))
with open(outfile, "w") as fp:
write_table(table, fp)
def apply_regex(self, match, name):
"""Apply a regulare expression rule."""
if len(match) <= 2 or not match.endswith("/"):
raise Error("Invalid regular expression {!r}".format(match))
try:
regex = re.compile(match[1:-1])
except ValueError as ex:
raise Error("Invalid regular expression {!r}: {}".format(
match, ex))
used_size = len(self.used)
for sname in set(self.scancode_map).difference(self.used):
m = regex.fullmatch(sname)
if m is None:
continue
if not name:
self.used.add(sname)
continue
hname = m.expand(name)
hid_key = self.hid_names.get(hname)
if hid_key is None:
continue
code = self.scancode_map[sname]
self.keymap[code] = hid_key
self.used.add(sname)
if len(self.used) == used_size:
raise Error("No scancodes match {!r}".format(match))
def __init__(self, dirname, filename):
try:
fp = open(os.path.join(dirname, filename))
except FileNotFoundError:
raise Error("Data file not found", filename=filename)
except OSError as ex:
raise Error("Could not open data file: {}".format(ex),
filename=filename)
self.filename = filename
self.fp = fp
def apply_single(self, match, name):
"""Apply a direct mapping rule."""
code = self.scancode_map.get(match)
if code is None:
raise Error("No scancode has name {!r}".format(match))
if match in self.used:
raise Error("Scancode {!r} already has mapping".format(match))
self.used.add(match)
if not name:
return
hid_key = self.hid_names.get(name)
if hid_key is None:
raise Error("No HID key is named {!r}".format(name))
self.keymap[code] = hid_key
def patch_installed_packages(stage_dir_abs, patch_dirs, dver):
"""Apply all patches in patch_dir to the files in stage_dir_abs
Assumptions:
- stage_dir_abs exists and has packages installed into it
- patch files are to be applied in sorted order (by filename; directory
name does not matter)
- patch files are -p1
- patch files end with .patch
Return success or failure as a bool
"""
patch_dirs_abs = [os.path.abspath(x) for x in patch_dirs]
oldwd = os.getcwd()
try:
os.chdir(stage_dir_abs)
patch_files = []
for patch_dir_abs in patch_dirs_abs:
patch_files += glob.glob(os.path.join(patch_dir_abs, "*.patch"))
patch_files.sort(cmp=_cmp_basename)
for patch_file in patch_files:
statusmsg("Applying patch %r" % patch_file)
#statusmsg("Applying patch %r" % os.path.basename(patch_file))
err = subprocess.call(['patch', '-p1', '--force', '--input', patch_file])
if err:
raise Error("patch file %r failed to apply" % patch_file)
finally:
os.chdir(oldwd)
def make_namemap(table, fname):
"""Create a function that maps integers to strings.
Arguments:
table: List of (input, output) pairs, where inputs are integers and
outputs are strings
fname: The output function name
Returns:
A string, contaning C source code which defines a function converting
integers to strings according to the table.
"""
kmap = {}
for code, kname in table:
kname2 = kmap.get(code)
if kname2 is not None and kname != kname2:
raise Error("Name conflict: code {} is {!r} and {!r}".format(
code, kname, kname2))
kmap[code] = kname
data, strmap = make_string_table([kname for (code, kname) in table])
count = max(code for code, kname in table) + 1
stridx = [0] * count
for code, kname in table:
stridx[code] = strmap[kname]
return NAMEMAP_TEMPLATE.format(
lname=fname.lower(),
uname=fname.upper(),
ddata=format_data(data, " "),
odata=format_numbers(stridx, " "),
otype=ctype(max(stridx)),
count=count,
)
def read_keytable(datadir, name, size, hid_names):
"""Read keycode tables.
Arguments:
datadir: Directory containing input data
name: Platform name
Returns:
A Keytable object for the platform
"""
with ReadFile(datadir, "{}_scancodes.csv".format(name)) as fp:
scancodes = read_scancodes(fp)
builder = KeymapBuilder([key for key in scancodes if key.code < size],
hid_names)
with ReadFile(datadir, "{}_map.csv".format(name)) as fp:
builder.apply_keymap(fp)
with ReadFile(datadir, "{}_names.csv".format(name)) as fp:
name_table = read_names(fp)
displaynames = []
for key in builder.keymap.values():
displayname = name_table.pop(key.name, key.displayname)
displaynames.append((key.code, displayname))
if name_table:
raise Error("Unused name mapping for {}".format(", ".join(
sorted(name_table))))
to_hid_table = [0] * size
from_hid_table = [255] * 256
for code, key in sorted(builder.keymap.items()):
to_hid_table[code] = key.code
if from_hid_table[key.code] == 255:
from_hid_table[key.code] = code
return Keytable(name, scancodes, displaynames, to_hid_table,
from_hid_table)
def init_stage1_rpmdb(stage1_root):
"""Create an rpmdb"""
err = subprocess.call(["rpm", "--initdb", "--root", stage1_root])
if err:
raise Error(
"Could not initialize rpmdb into %r (rpm process returned %d)" %
(stage1_root, err))
def apply_row(self, row):
"""Apply the rule in a single row of a keymap file."""
try:
match, name = row
except ValueError:
raise Error("Got {} columns, expected 2".format(len(row)))
if match.startswith("/"):
self.apply_regex(match, name)
else:
self.apply_single(match, name)
def _get_yum_major_version(self):
proc = subprocess.Popen("yum --version | head -n1",
shell=True,
stdout=subprocess.PIPE)
output = to_str(proc.communicate()[0]).strip()
version = output.split(".")
try:
return int(version[0])
except (IndexError, ValueError):
raise Error("unexpected output from yum --version: %s" % output)
def make_stage1_root_dir(stage1_root):
"""Make or empty a directory to be used for building the stage1.
Prompt the user before removing anything (if the dir already exists).
"""
if stage1_root == "/":
raise Error("You may not use '/' as the output directory")
try:
if os.path.isdir(stage1_root):
print "Stage 1 directory (%r) already exists. Reuse it? Note that the contents will be emptied! " % stage1_root
user_choice = raw_input("[y/n] ? ").strip().lower()
if not user_choice.startswith('y'):
raise Error(
"Not overwriting %r. Remove it or pass a different directory"
% stage1_root)
shutil.rmtree(stage1_root)
os.makedirs(stage1_root)
except OSError as err:
raise Error("Could not create stage 1 root dir %s: %s" %
(stage1_root, str(err)))
def read_hid(fp):
"""Read the HID keycode table.
Arguments:
fp: Input file
Returns:
Alist of Keycode objects
"""
codes = set()
names = set()
displaynames = set()
result = []
reader = csv.reader(fp)
def error(msg):
return Error(msg, lineno=lineno)
row = next(reader)
headers = ["Keycode", "Name", "Display Name"]
if row != headers:
raise Error("Got headers {!r}, expected {!r}".format(row, headers),
lineno=1)
for lineno, row in enumerate(reader, 2):
if not row:
continue
try:
codestr, name, displayname = row
except ValueError:
raise error("Got {} columns, expected 3".format(len(row)))
try:
code = int(codestr, 0)
except ValueError:
raise error("Invalid keycode {}".format(rowstr))
if code in codes:
raise error("Duplicate code {}".format(code))
codes.add(code)
if not name:
if displayname:
raise error("Name is empty but display name is present")
continue
if not VALID_NAME.fullmatch(name):
raise error("Invalid name {!r}".format(name))
if name in names:
raise error("Duplicate name {!r}".format(name))
names.add(name)
if not displayname:
displayname = name
if not VALID_DISPLAYNAME.fullmatch(displayname):
raise error("Invalid display name {!r}".format(displayname))
if displayname in displaynames:
raise error("Duplicate display name {!r}".format(displayname))
result.append(Keycode(code, name, displayname))
return result
def init_stage1_devices(stage1_root):
"""Make /dev file system in the chroot"""
devdir = os.path.join(stage1_root, 'dev')
os.makedirs(devdir)
for name, major, minor, group, perms in DEVICES:
path = opj(devdir, name)
try:
os.mknod(path, perms | stat.S_IFCHR, os.makedev(major, minor))
os.chown(path, -1, grp.getgrnam(group).gr_gid)
except EnvironmentError as err:
raise Error("Could not create /dev/%s in the chroot: %s" %
(name, str(err)))
def fix_osg_version(stage_dir_abs, relnum=""):
osg_version_path = os.path.join(stage_dir_abs, 'etc/osg-version')
version_str = new_version_str = ""
_relnum = ""
if relnum:
_relnum = "-" + str(relnum)
with open(osg_version_path) as osg_version_fh:
version_str = osg_version_fh.readline()
if not version_str:
raise Error("Could not read version string from %r" % osg_version_path)
if not re.match(r'[0-9.]+', version_str):
raise Error("%r does not contain version" % osg_version_path)
if 'tarball' in version_str:
new_version_str = version_str
else:
new_version_str = re.sub(r'^([0-9.]+)(?!-tarball)', r'\1-tarball%s' % (_relnum), version_str)
with open(osg_version_path, 'w') as osg_version_write_fh:
osg_version_write_fh.write(new_version_str)
def ReadUsage(file, length):
HEX = r'[\da-fA-F]'
LOC = r'(%s{2}):(%s{4})' % (HEX, HEX)
RE = r'^%s\.\.%s:\s+(.*)$' % (LOC, LOC)
KINDS = {'Unknown': 0, 'Data': 2, 'Code': 3, 'Addr': 4}
usage = bytearray(length)
for i, line in enumerate(file.readlines()):
m = re.match(RE, line)
if m:
bank0, addr0, bank1, addr1, kind = m.groups()
loc0 = LocFromBankAddr(int(bank0, 16), int(addr0, 16))
loc1 = LocFromBankAddr(int(bank1, 16), int(addr1, 16))
if loc0 > loc1:
raise Error('Invalid range %s:%s..%s:%s' %
(bank0, addr0, bank1, addr1))
if kind not in KINDS:
raise Error('Invalid kind: %s' % kind)
kind = KINDS[kind]
for l in range(loc0, loc1 + 1):
usage[l] = kind
else:
raise Error('Invalid line: %s' % line)
return usage
def install_packages(stage_dir_abs, packages, repofile, dver, basearch, extra_repos=None):
"""Install packages into a stage1 dir"""
if isinstance(packages, str):
packages = [packages]
with common.MountProcFS(stage_dir_abs):
with yumconf.YumInstaller(repofile, dver, basearch, extra_repos) as yum:
yum.install(installroot=stage_dir_abs, packages=packages)
# Check that the packages got installed
for pkg in packages:
if pkg.startswith('@'):
continue # can't check on groups
if not package_installed(stage_dir_abs, pkg):
raise Error("%r not installed after yum install" % pkg)
def fix_gsissh_config_dir(stage_dir_abs):
"""A hack to fix gsissh, which looks for $GLOBUS_LOCATION/etc/ssh.
The actual files are in $OSG_LOCATION/etc/gsissh, so make a symlink.
Make it a relative symlink so we don't have to fix it in post-install.
"""
if not os.path.isdir(os.path.join(stage_dir_abs, 'etc/gsissh')):
return
try:
usr_etc = os.path.join(stage_dir_abs, 'usr/etc')
safe_makedirs(usr_etc)
os.symlink('../../etc/gsissh', os.path.join(usr_etc, 'ssh'))
except EnvironmentError as err:
raise Error("unable to fix gsissh config dir: %s" % str(err))
def apply_keymap(self, fp):
"""Apply the rules in a keymap file."""
reader = csv.reader(fp)
row = next(reader)
headers = ["Platform Name", "HID Name"]
if row != headers:
raise Error("Got headers {!r}, expected {!r}".format(row, headers),
lineno=1)
for lineno, row in enumerate(reader, 2):
if not row:
continue
try:
self.apply_row(row)
except Error as ex:
ex.lineno = lineno
raise ex
def tar_stage_dir(stage_dir_abs, tarball):
"""tar up the stage_dir
Assume: valid stage2 dir
"""
tarball_abs = os.path.abspath(tarball)
stage_dir_parent = os.path.dirname(stage_dir_abs)
stage_dir_base = os.path.basename(stage_dir_abs)
excludes = [
"var/log/yum.log",
"tmp/*",
"var/cache/yum/*",
"var/lib/rpm/*",
"var/lib/yum/*",
"var/tmp/*",
"dev/*",
"proc/*",
"etc/rc.d/rc?.d",
"etc/alternatives",
"var/lib/alternatives",
"usr/bin/[[]",
"usr/share/man/man1/[[].1.gz",
"bin/dbus*",
"lib/libcap*",
"lib/dbus*",
"lib/security/pam*.so",
"lib64/libcap*",
"lib64/dbus*",
"lib64/security/pam*.so",
"usr/bin/gnome*",
"*~",
]
cmd = ["tar", "-C", stage_dir_parent, "-czf", tarball_abs, stage_dir_base]
stage1_filelist = os.path.join(stage_dir_abs, 'stage1_filelist')
if os.path.isfile(stage1_filelist):
exclude_list = os.path.join(stage_dir_parent, 'exclude_list')
_write_exclude_list(stage1_filelist, exclude_list, stage_dir_base,
excludes)
cmd.append('--exclude-from=%s' % exclude_list)
err = subprocess.call(cmd)
if err:
raise Error("unable to create tarball (%r) from stage 2 dir (%r)" %
(tarball_abs, stage_dir_abs))
def __init__(self, dirname, filename, quiet=False, guard=None):
if not quiet:
print("Writing", filename, file=sys.stderr)
try:
fp = open(os.path.join(dirname, filename), "w")
except Exception as ex:
raise Error("Could not create output file: {}".format(ex),
filename=filename)
try:
fp.write("/* This file is automatically generated. */\n")
if guard is not None:
fp.write("#ifndef {0}\n#define {0}\n".format(guard))
except:
fp.close()
raise
self.fp = fp
self.guard = guard
def read_scancodes(fp):
"""Read a scancodes table mapping platform-specific scancodes to names.
If a scancode appears twice in the file, only the first entry is included.
Entries with empty names are filtered out.
Arguments:
fp: Input file
Returns:
A list of Scancode objects
"""
result = []
codes = set()
names = set()
reader = csv.reader(fp)
def error(msg):
return Error(msg, lineno=lineno)
row = next(reader)
headers = ["Keycode", "Name"]
if row != headers:
raise Error("Got headers {!r}, expected {!r}".format(row, headers),
lineno=1)
for lineno, row in enumerate(reader, 2):
if not row:
continue
if len(row) != 2:
raise error("Got {} columns, expected 2".format(len(row)))
codestr, name = row
try:
code = int(codestr)
except ValueError:
raise error("Invalid scancode value {!r}".format(codestr))
if not VALID_NAME.fullmatch(name):
raise error("Invalid scancode name {!r}".format(name))
if name and name in names:
raise error("Duplicate scancode name {!r}".format(name))
names.add(name)
if code not in codes:
codes.add(code)
if name:
result.append(Scancode(code, name))
return result
def read_names(fp):
"""Read a display name table mapping HID names to platform-specific names.
Arguments:
fp: Input file
Returns:
A dictionary mapping HID names to display names
"""
result = {}
reader = csv.reader(fp)
def error(msg):
return Error(msg, lineno=lineno)
row = next(reader)
headers = ["Name", "Display Name"]
if row != headers:
raise Error("Got headers {!r}, expected {!r}".format(row, headers),
lineno=1)
for lineno, row in enumerate(reader, 2):
if not row:
continue
try:
name, displayname = row
except ValueError:
raise error("Got {} columns, expected 2".format(len(row)))
if not name:
continue
if not VALID_NAME.fullmatch(name):
raise error("Invalid name {!r}".format(name))
if name in result:
raise error("Duplicate name {!r}".format(name))
if not displayname:
continue
if not VALID_DISPLAYNAME:
raise error("Invalid display name {!r}".format(displayname))
result[name] = displayname
return result
def error(msg):
return Error(msg, lineno=lineno)