def insert_bottom_labels(sig, lines, v):
"""
Insert bottom labels for each function.
"""
# Look for the structure and insert:
# > .globl <bottom-label>
# foo:
# ...
# > <bottom-label>
# .cc_bottom foo.function
vmsg(v, ' Inserting function labels')
# For each function, for each line...
# (Create a new list and modify it each time...)
b = False
for x in sig.mobile_proc_names:
new = []
for (i, y) in enumerate(lines):
new.append(y)
if y == x+':\n' and not b:
new.insert(len(new)-1,
'.globl '+function_label_bottom(x)+'\n')
b = True
elif y[0] == '.' and b:
if y.split()[0] == '.cc_bottom':
new.insert(len(new)-1,
function_label_bottom(x)+':\n')
b = False
lines = new
return lines
def assemble_builtins(show_calls, v):
vmsg(v, 'Compiling builtins:')
for x in BUILTIN_FILES:
objfile = x+'.o'
vmsg(v, ' '+x+' -> '+objfile)
util.call([CC, config.MPI_SYSTEM_PATH+'/'+x,
'-o', objfile] + ASSEMBLE_FLAGS, show_calls)
def replace_images(show_calls, v): vmsg(v, 'Creating new executable') util.call([XCC, target_2core(), config.XS1_RUNTIME_PATH+'/container.xc', '-o', FINAL_XE]) util.call([XOBJDUMP, '--split', MASTER_XE], v=show_calls) util.call([XOBJDUMP, FINAL_XE, '-r', '0,0,image_n0c0.elf'], v=show_calls) util.call([XOBJDUMP, '--split', SLAVE_XE], v=show_calls) util.call([XOBJDUMP, FINAL_XE, '-r', '0,1,image_n0c0.elf'], v=show_calls)
def link_master(device, show_calls, v):
"""
The jump table must be located at _cp and the common elements of the
constant and data pools must be in the same positions relative to _cp and
_dp in the master and slave images.
"""
vmsg(v, 'Linking master -> '+MASTER_XE)
s = util.call([XCC, target_1core(),
'system.S.o',
'system.xc.o',
'control.xc.o',
'worker.xc.o',
'source.xc.o',
'host.xc.o',
'host.S.o',
'connect.xc.o',
'master.xc.o',
'master.S.o',
'program.o',
'memory.c.o',
'pointer.c.o',
'util.xc.o',
MASTER_TABLES+'.o',
CONST_POOL+'.o',
'globals.S.o',
'-o', MASTER_XE] + LINK_FLAGS,
v=show_calls)
print(s, end='')
def assemble_runtime(show_calls, v):
vmsg(v, 'Compiling runtime:')
for x in RUNTIME_FILES:
objfile = x+'.o'
vmsg(v, ' '+x+' -> '+objfile)
util.call([MPICC, config.MPI_RUNTIME_PATH+'/'+x,
'-o', objfile] + ASSEMBLE_FLAGS, show_calls)
def translate(ast, sig, child, device, outfile, translate_only, v):
"""
Translate the AST to target system.
"""
vmsg(v, 'Translating AST...')
buf = io.StringIO()
ext = None
# Create a tranlator AST walker
if device.system == SYSTEM_TYPE_XS1:
walker = TranslateXS1(sig, child, buf)
elif device.system == SYSTEM_TYPE_MPI:
walker = TranslateMPI(sig, child, buf)
walker.walk_program(ast)
if translate_only:
outfile = (outfile if outfile!=defs.DEFAULT_OUT_FILE else
outfile+'.'+device.source_file_ext())
util.write_file(outfile, buf.getvalue())
vmsg(v, 'Produced file: '+outfile)
raise SystemExit()
return buf
def assemble_runtime(device, show_calls, v):
vmsg(v, 'Compiling runtime:')
for x in RUNTIME_FILES:
objfile = x+'.o'
vmsg(v, ' '+x+' -> '+objfile)
s = util.call([XCC, config.XS1_RUNTIME_PATH+'/'+x,
'-o', objfile] + ASSEMBLE_FLAGS, v=show_calls)
print(s, end='')
def child_analysis(sig, ast): """ Determine children. """ vmsg(v, "Performing child analysis") child = Children(sig) ast.accept(child) child.build() #child.display() return child
def link(show_calls, v):
"""
Link the complete executable.
"""
vmsg(v, 'Linking executable -> '+BINARY)
util.call([MPICC, 'program.c.o']
+ [x+'.o' for x in RUNTIME_FILES]
+ [x+'.o' for x in BUILTIN_FILES]
+ ['-o', BINARY]
+ LINK_FLAGS,
show_calls)
def append_header(device, outfile, show_calls, v):
vmsg(v, 'Appending binary header')
xe = open(FINAL_XE, "rb")
se = open(outfile, "wb")
try:
se.write(bytes('SIRE', 'UTF-8'))
se.write(struct.pack('I', device.num_cores()))
se.write(xe.read())
finally:
xe.close()
se.close()
def assemble_str(name, string, show_calls, v, cleanup=True):
"""
Assemble a buffer containing a c program.
"""
srcfile = name + '.c'
outfile = name + '.o'
vmsg(v, 'Assembling '+srcfile+' -> '+outfile)
util.write_file(srcfile, string)
util.call([MPICC, srcfile, '-o', outfile] + ASSEMBLE_FLAGS, show_calls)
if cleanup:
os.remove(srcfile)
def compile_str(name, string, show_calls, v, save_temps=True):
"""
Compile a buffer containing an XC program.
"""
srcfile = name + '.xc'
outfile = name + '.S'
vmsg(v, 'Compiling '+srcfile+' -> '+outfile)
util.write_file(srcfile, string)
util.call([XCC, srcfile, '-o', outfile] + COMPILE_FLAGS, v=show_calls)
if not save_temps:
os.remove(srcfile)
def cleanup(v):
"""
Renanme the output file and delete any temporary files.
"""
vmsg(v, 'Cleaning up')
# Remove specific files
util.remove_file(DEVICE_HDR)
# Remove runtime objects
for x in glob.glob('*.o'):
util.remove_file(x)
def rewrite_calls(sig, lines, v):
"""
Rewrite calls to program functions to branch through the jump table.
"""
vmsg(v, ' Rewriting calls')
for (i, x) in enumerate(lines):
frags = x.strip().split()
names = builtin.runtime_functions + sig.mobile_proc_names
if frags and frags[0] == 'bl' and frags[1] in names:
lines[i] = '\tbla cp[{}+{}]\n'.format(defs.LABEL_JUMP_TABLE,
names.index(frags[1])*defs.BYTES_PER_WORD)
return lines
def build(sig, buf, device, outfile,
compile_only, display_memory, show_calls, save_temps, v):
"""
Compile the translated AST for the target system.
"""
vmsg(v, 'Creating executable...')
# Create a Build object
if device.system == SYSTEM_TYPE_XS1:
build_xs1(sig, device, buf, outfile,
compile_only, display_memory, show_calls, save_temps, v)
elif device.system == SYSTEM_TYPE_MPI:
build_mpi(device, buf, outfile, compile_only, show_calls, v)
def modify_assembly(sig, lines, v):
"""
Perform modifications on assembly output.
"""
vmsg(v, 'Modifying assembly output')
#print(''.join(lines))
(lines, cp) = extract_constants(lines, v)
lines = insert_bottom_labels(sig, lines, v)
#lines = insert_frame_sizes(sig, lines, v)
lines = rewrite_calls(sig, lines, v)
lines.insert(0, '###### MODIFIED ######\n')
#print(''.join(lines))
return (lines, cp)
def assemble_str(name, ext, string, show_calls, v, save_temps=False):
"""
Assemble a buffer containing an XC or assembly program.
"""
srcfile = name + '.' + ext
outfile = name + '.o'
vmsg(v, 'Assembling '+srcfile+' -> '+outfile)
util.write_file(srcfile, string)
if ext == 'xc':
s = util.call([XCC, srcfile, '-o', outfile] + ASSEMBLE_FLAGS, v=show_calls)
print(s, end='')
elif ext == 'S':
s = util.call([XAS, srcfile, '-o', outfile], v=show_calls)
print(s, end='')
if not save_temps:
os.remove(srcfile)
def walk_program(self, node, v):
# All processes have been flattened into 'main'
self.defs = node.defs
debug(self.debug, 'd before program = {}'.format(0))
d = self.stmt(node.defs[-1].stmt, node.defs[-1].name, 0)
debug(self.debug, 'd after program = {}'.format(d))
# Check the available number of processors has not been exceeded
if d > self.device.num_cores():
self.errorlog.report_error(
'insufficient processors: {} required, {} available'.format(
d, self.device.num_cores()))
# Report processor usage
vmsg(v, ' {}/{} processors used'.format(d, self.device.num_cores()))
def walk_program(self, node, v):
# All processes have been flattened into 'main'
self.defs = node.defs
debug(self.debug, 'd before program = {}'.format(0))
d = self.stmt(node.defs[-1].stmt, node.defs[-1].name, 0)
debug(self.debug, 'd after program = {}'.format(d))
# Check the available number of processors has not been exceeded
if d > self.device.num_cores():
self.errorlog.report_error(
'insufficient processors: {} required, {} available'
.format(d, self.device.num_cores()))
# Report processor usage
vmsg(v, ' {}/{} processors used'.format(d, self.device.num_cores()))
def semantic_analysis(sym, sig, ast, device, errorlog):
"""
Perform semantic analysis on an AST.
"""
vmsg(v, "Performing semantic analysis")
sem = Semantics(sym, sig, device, errorlog)
sem.walk_program(ast)
# Check for any errors
if errorlog.any():
raise QuietError()
# Quit if we're only performing semantic analysis
if sem_only:
raise SystemExit()
return sem
def create_headers(device, v):
vmsg(v, 'Creating device header '+DEVICE_HDR)
s = '#define NUM_CORES {}\n'.format(device.num_cores())
s += '#define NUM_CORES_LOG {}\n'.format(
int(math.log(device.num_cores())/math.log(2)))
s += '#define NUM_CORES_SQRT {}\n'.format(
int(math.sqrt(device.num_cores())))
s += '#define NUM_NODES 0\n'
s += '#define NUM_CORES_PER_NODE NUM_CORES\n'
s += '#define XS1_L\n'
#s += '#define NUM_NODES {}\n'.format(device.num_nodes)
#s += '#define NUM_CORES_PER_NODE {}\n'.format(device.num_cores_per_node)
#if device.type == XS1_DEVICE_TYPE_G:
# s += '#define XS1_G\n'
#elif device.type == XS1_DEVICE_TYPE_L:
# s += '#define XS1_L\n'
util.write_file(DEVICE_HDR, s)
def produce_ast(input_file, errorlog, log=True):
"""
Parse an input string to produce an AST.
"""
vmsg(v, "Parsing file '{}'".format(infile if infile else 'stdin'))
# Setup logging if we need to
if log:
logging.basicConfig(
level = logging.DEBUG,
filename = defs.PARSE_LOG_FILE,
filemode = "w",
format = "%(filename)10s:%(lineno)4d:%(message)s")
logger = logging.getLogger()
else:
logger = 0
# Create the parser and produce the AST
parser = Parser(errorlog, lex_optimise=True,
yacc_debug=False, yacc_optimise=False)
ast = parser.parse(input_file, infile, debug=logger)
if errorlog.any():
raise QuietError()
# Perform parsing only
if parse_only:
raise SystemExit()
# Display (dump) the AST
if print_ast:
ast.accept(Dump())
raise SystemExit()
# Display (pretty-print) the AST
if pprint_raw_ast:
Printer().walk_program(ast)
raise SystemExit()
return ast
def build_mpi(device, buf, outfile,
compile_only, show_calls=False, v=False):
"""
Run the build process to create either the assembly output or the complete
binary.
"""
# Add the include paths once they have been set
include_dirs = ['-I', '.']
include_dirs += ['-I', config.INSTALL_PATH]
global COMPILE_FLAGS
global ASSEMBLE_FLAGS
COMPILE_FLAGS += include_dirs
ASSEMBLE_FLAGS += include_dirs
try:
# Create headers
create_headers(device, v)
if compile_only:
raise SystemExit()
# Compile the program and runtime
assemble_str(PROGRAM, buf.getvalue(), show_calls, v)
buf.close()
assemble_runtime(show_calls, v)
assemble_builtins(show_calls, v)
link(show_calls, v)
# Rename the output file
outfile = (outfile if outfile!=defs.DEFAULT_OUT_FILE else
outfile+'.'+device.binary_file_ext())
os.rename(BINARY, outfile)
vmsg(v, 'Produced file: '+outfile)
finally:
cleanup(v)
def cleanup(v):
"""
Renanme the output file and delete any temporary files.
"""
vmsg(v, 'Cleaning up')
# Remove specific files
util.remove_file(MASTER_XE)
util.remove_file(SLAVE_XE)
util.remove_file('image_n0c0.elf')
util.remove_file('config.xml')
util.remove_file('platform_def.xn')
util.remove_file('program_info.txt')
util.remove_file(DEVICE_HDR)
# Remove unused master images
for x in glob.glob('image_n*c*elf'):
util.remove_file(x)
# Remove runtime objects
for x in glob.glob('*.o'):
util.remove_file(x)
def build_master_tab_init(sig, buf, v):
assert (len(sig.mobile_proc_names) + defs.JUMP_INDEX_OFFSET) <= defs.JUMP_TABLE_SIZE
vmsg(v, 'Building master table initialisation ({}/{})'.format(
len(sig.mobile_proc_names), defs.JUMP_TABLE_SIZE))
buf.write('#include "xs1/definitions.h"\n')
# Data section
buf.write('.section .dp.data, "awd", @progbits\n')
buf.write('.align {}\n'.format(defs.BYTES_PER_WORD))
buf.write('_numProgEntries:\n')
buf.write('.globl _numProgEntries\n')
buf.write('.set _numProgEntries.globound, {}\n'.format(defs.BYTES_PER_WORD))
buf.write('\t.word {}\n'.format(len(sig.mobile_proc_names)))
buf.write('_jumpLocations:\n')
buf.write('.globl _jumpLocations\n')
buf.write('.set _jumpLocations.globound, {}\n'.format(
len(sig.mobile_proc_names)*defs.BYTES_PER_WORD))
for x in sig.mobile_proc_names:
buf.write('\t.word {}\n'.format(x))
buf.write('.align {}\n'.format(defs.BYTES_PER_WORD))
buf.write('.globl '+defs.LABEL_SIZE_TABLE+', "a(:ui)"\n')
buf.write('.set {}.globound, {}\n'.format(
defs.LABEL_SIZE_TABLE, defs.BYTES_PER_WORD*defs.SIZE_TABLE_SIZE))
buf.write(defs.LABEL_SIZE_TABLE+':\n')
# Pad runtime entries
for x in range(defs.JUMP_INDEX_OFFSET):
buf.write('\t.word 0\n')
# Program procedure entries
for x in sig.mobile_proc_names:
buf.write('\t.word {}-{}+{}\n'.format(
function_label_bottom(x), x, defs.BYTES_PER_WORD))
# Pad any unused space
remaining = defs.SIZE_TABLE_SIZE - (defs.JUMP_INDEX_OFFSET +
len(sig.mobile_proc_names))
buf.write('\t.space {}\n'.format(remaining*defs.BYTES_PER_WORD))
def link_slave(device, show_calls, v):
"""
As above.
"""
vmsg(v, 'Linking slave -> '+SLAVE_XE)
s = util.call([XCC, target_1core(),
'system.S.o',
'system.xc.o',
'control.xc.o',
'worker.xc.o',
'source.xc.o',
'host.xc.o',
'host.S.o',
'connect.xc.o',
'slave.S.o',
'memory.c.o',
'pointer.c.o',
'util.xc.o',
CONST_POOL+'.o',
'globals.S.o',
'-o', SLAVE_XE] + LINK_FLAGS,
v=show_calls)
print(s, end='')
def create_headers(device, v):
vmsg(v, 'Creating device header '+DEVICE_HDR)
s = ''
s += '#define NUM_CORES {}\n'.format(device.num_cores())
util.write_file(DEVICE_HDR, s)
def extract_constants(lines, v):
"""
Extract constant sections only within the elimination block of a function.
This covers all constants local to a function. This is to differentiate
constants associated with and declared global.
- Don't include elimination blocks for strings.
- Make extracted labels global.
NOTE: this assumes a constant section will be terminated with a .text,
(which may not always be true?).
"""
vmsg(v, ' Extracting constants')
cp = []
new = []
sect = False
func = False
for x in lines:
# If we have entered a function elimination block
if re.match(r'\.cc_top [_A-Za-z][A-Za-z0-9_]*\.function', x):
func = True
# If we have left
if re.match(r'\.cc_bottom [_A-Za-z][A-Za-z0-9_]*\.function', x):
func = False
if func:
# If this is a cp section
if x.find('.section .cp') >= 0:
sect = True
# If we have left the cp section
if x.find('.text') >= 0:
sect = False
if sect:
# If we are in the seciton: replace labels with externs,
# declare them as global in the new cp.
if x.find(':\n') > 0:
cp.append('\t.globl '+x[:-2]+'\n')
new.insert(0, '\t.extern '+x[:-2]+'\n')
# Rename .const4 and const8 to rodata
if x.find('.section .cp.const') >= 0:
x = '\t.section .cp.rodata, "ac", @progbits\n'
# Leave .call and .globreads where they are
if (x.find('.call')!=-1 or x.find('.globread')!=-1):
new.append(x)
# Omit elimination directives (for strings),
elif (x.find('.cc_top')==-1 and x.find('.cc_bottom')==-1):
cp.append(x)
# If we're outside a cp section, add to a new list
else:
if x.find('.text')==-1:
new.append(x)
# If we're outside a function block, add to a new list
else:
new.append(x)
return (new, cp)
def transform_ast(sem, sym, sig, ast, errorlog, device, v):
"""
Perform transformations on the AST.
"""
# 1. Distribute processes
vmsg(v, "Expanding processes")
ExpandProcs(sig, ast).walk_program(ast)
if errorlog.any(): raise Error()
# 2. Flatten nested parallel composition
#vmsg(v, "Flattening nested parallel composition")
#FlattenPar().walk_program(ast)
# 3. Distribute processes
vmsg(v, "Distributing processes")
InsertOns(device, errorlog).walk_program(ast, v)
if errorlog.any(): raise Error()
# 4. Label process locations
vmsg(v, "Labelling processes")
LabelProcs(sym, device).walk_program(ast)
# 5. Label channels
vmsg(v, "Labelling channels")
LabelChans(device, errorlog).walk_program(ast)
if errorlog.any(): raise Error()
# 6. Label connections
vmsg(v, "Labelling connections")
LabelConns().walk_program(ast)
#DisplayConns(device).walk_program(ast)
# 7. Insert channel ends
vmsg(v, "Inserting connections")
InsertConns(sym).walk_program(ast)
# 8. Rename channel uses
vmsg(v, "Renaming channel uses")
RenameChans().walk_program(ast)
# 9. Build the control-flow graph and initialise sets for liveness analysis
vmsg(v, "Building the control flow graph")
BuildCFG().run(ast)
# 10. Perform liveness analysis
vmsg(v, "Performing liveness analysis")
Liveness().run(ast)
# 13. Transform server processes
vmsg(v, "Transforming server processes")
TransformServer().walk_program(ast)
# 11. Transform parallel composition
vmsg(v, "Transforming parallel composition")
TransformPar(sem, sig).walk_program(ast)
# 12. Transform parallel replication
vmsg(v, "Transforming parallel replication")
TransformRep(sym, sem, sig, device).walk_program(ast)
# 14. Flatten nested calls
vmsg(v, "Flattening nested calls")
FlattenCalls(sig).walk_program(ast)
# 15. Remove unused declarations
vmsg(v, "Removing unused declarations")
RemoveDecls().walk_program(ast)
# Display (pretty-print) the transformed AST
if pprint_trans_ast:
Printer().walk_program(ast)
raise SystemExit()
def build_xs1(sig, device, program_buf, outfile,
compile_only, display_memory,
show_calls=False, save_temps=False, v=False):
"""
Run the build process to create either the assembly output or the complete
binary.
"""
# Add the include paths once they have been set
include_dirs = ['-I', '.']
include_dirs += ['-I', config.INSTALL_PATH]
global COMPILE_FLAGS
global ASSEMBLE_FLAGS
COMPILE_FLAGS += include_dirs
ASSEMBLE_FLAGS += include_dirs
# Try to run the build, pass any Error or SystemExit exceptions along to
# main driver after having cleaned up and temporary files.
try:
# Create headers
create_headers(device, v)
# Generate the assembly
(lines, cp) = generate_assembly(sig, program_buf, show_calls, v, save_temps)
if compile_only:
# Write the program back out and assemble
util.write_file(PROGRAM_ASM, ''.join(lines))
# Rename the output file
outfile = (outfile if outfile!=defs.DEFAULT_OUT_FILE else
outfile+'.'+device.assembly_file_ext())
os.rename(PROGRAM_ASM, outfile)
raise SystemExit()
# Write the program back out and assemble
assemble_str(PROGRAM, 'S', ''.join(lines), show_calls, v)
# Write the cp out and assemble
assemble_str(CONST_POOL, 'S', ''.join(cp), show_calls, v)
# Output and assemble the master jump table
buf = io.StringIO()
build_master_tab_init(sig, buf, v)
#print(buf.getvalue())
assemble_str(MASTER_TABLES, 'S', buf.getvalue(), show_calls, v, save_temps)
# Assemble and link the rest
assemble_runtime(device, show_calls, v)
link_master(device, show_calls, v)
link_slave(device, show_calls, v)
replace_images(show_calls, v)
# Dump memory usage information
if display_memory:
dump_memory_use()
# Append XE to header in output file
outfile = (outfile if outfile!=defs.DEFAULT_OUT_FILE else
outfile+'.se')
append_header(device, outfile, show_calls, v)
vmsg(v, 'Produced file: '+outfile)
except Error as e:
raise Error(e.args)
except SystemExit:
raise SystemExit()
except:
raise
finally:
if not save_temps:
cleanup(v)