def parse_and_filter(indir, namefilter=None, recursive=False, printlog=False):
''' read and parse headers and definitions of component and instrument files '''
instr_files, comp_files = utils.get_instr_comp_files(indir, recursive)
print("parsing root folder:", indir)
comp_info_lst = []
for f in comp_files:
try:
if printlog:
print("parsing comp... %s" % f)
info = CompParser(f).parse()
info.filepath = os.path.abspath(f)
comp_info_lst.append(info)
except:
print("failed parsing file: %s" % f)
comp_info_lst.append(CompParser(f).stub())
print("comp files: %s" % str(len(comp_files)))
instr_info_lst = []
for f in instr_files:
try:
if printlog:
print("parsing instr... %s" % f)
info = InstrParser(f).parse()
info.filepath = os.path.abspath(f)
instr_info_lst.append(info)
except:
print("failed parsing file: %s" % f)
instr_info_lst.append(InstrParser(f).stub())
print("instr files: %s" % str(len(instr_files)))
if namefilter != None:
comp_info_lst = [
c for c in comp_info_lst
if re.search(namefilter.lower(), c.name.lower())
]
instr_info_lst = [
c for c in instr_info_lst
if re.search(namefilter.lower(), c.name.lower())
]
comp_files = [
f for f in comp_files
if re.search(namefilter.lower(),
os.path.splitext(os.path.basename(f))[0].lower())
]
instr_files = [
f for f in instr_files
if re.search(namefilter.lower(),
os.path.splitext(os.path.basename(f))[0].lower())
]
return comp_info_lst, instr_info_lst, comp_files, instr_files
def initDynamicView(self):
# load installed mcstas instruments:
# construct args = [site, instr_fullpath[], instr_path_lst[]]
args = []
files_instr, files_comp = get_instr_comp_files(mccode_config.configuration["MCCODE_LIB_DIR"])
# temporary list consisting of instrument files with site names:
files_instr_and_site = []
for f in files_instr:
files_instr_and_site.append([f, get_instr_site(f)])
# order instrument files by site:
sites = {s for s in list(map(lambda f: f[1], files_instr_and_site))}
for s in sites:
# extract instruments file paths of this site
instr_path_lst = list(map(lambda f: f[0], filter(lambda f: f[1] in [s], files_instr_and_site)))
# sort instrument of this site by file name
instr_path_lst.sort(key=lambda instrpath: os.path.splitext(os.path.basename(instrpath))[0])
# extract file names
instr_name_lst = list(map(lambda instrpath: os.path.splitext(os.path.basename(instrpath))[0], instr_path_lst))
arg = []
arg.append(s)
arg.append(instr_name_lst)
arg.append(instr_path_lst)
args.append(arg)
# sort sites
args.sort(key=lambda arg: arg[0])
# hand on for menu generation
self.view.initMainWindowDynamicElements(args, self.handleNewFromTemplate)
# load installed mcstas components:
# args - [category, comp_names[], comp_parsers[]]
args = []
categories = {0 : 'Source', 1 : 'Optics', 2 : 'Sample', 3 : 'Monitor', 4 : 'Misc', 5 : 'Contrib', 6: 'Union', 7 : 'Obsolete'}
dirnames = {0 : 'sources', 1 : 'optics', 2 : 'samples', 3 : 'monitors', 4 : 'misc', 5 : 'contrib', 6: 'contrib/union', 7 : 'obsolete'}
i = 0
while i < 8:
arg = [] # arg - category, comp_names[], comp_parsers[]
compnames = []
parsers = []
for f in files_comp:
if i==6:
if re.search(dirnames[i], os.path.dirname(f)):
compnames.append(os.path.splitext(os.path.basename(f))[0]) # get filename without extension - this is the component name
parsers.append(ComponentParser(f)) # append a parser, for ease of parsing on-the-fly
else:
if re.search(dirnames[i], os.path.basename(os.path.dirname(f))):
compnames.append(os.path.splitext(os.path.basename(f))[0]) # get filename without extension - this is the component name
parsers.append(ComponentParser(f)) # append a parser, for ease of parsing on-the-fly
arg.append(categories[i])
arg.append(compnames)
arg.append(parsers)
args.append(arg)
i += 1
# sort components in each category (using Python default string sort on filename)
for arg in args:
arg[1].sort()
arg[2].sort(key=lambda parser: os.path.splitext(os.path.basename(parser.file))[0])
# hand on for menu generation
self.view.initCodeEditorComponentMenu(args)
def repair_instr(localdir):
'''
Dev function used to alter instr file headers.
'''
local_instr_files, local_comp_files = utils.get_instr_comp_files(localdir)
files = []
rows = []
for filename in local_instr_files:
try:
print("parsing... %s" % filename)
info = InstrParser(filename).parse()
files.append(filename)
rows.append(info)
except:
print("failed parsing instr file: %s" % filename)
quit()
cnt = 0
for filename in local_instr_files:
f = open(filename, 'r')
# read the first two instr file sections
header = utils.read_header(f)
define = utils.read_define_instr(f)
# doc lines
print('*****')
print(filename)
print()
seen_P = False
par_docs = []
idxs = []
idxs_remove = []
lines = header.splitlines()
for i in range(len(lines)):
l = lines[i]
# remove unwanted lines
m = re.match('\* Release:', l)
if m:
idxs_remove.append(i)
continue
m = re.match('\* Version:', l)
if m:
idxs_remove.append(i)
continue
m = re.match('\* INPUT PARAMETERS:', l)
if m:
idxs_remove.append(i)
continue
# fast-forward to %P / %Parameters tag
if not seen_P and re.match('\* \%Parameters', l):
seen_P = True
elif not seen_P:
continue
# exit if we reach %L / %Link tag
if re.match('\* \%L', l):
break
l = l.lstrip('*').strip()
m = re.match(
'(\w+):[ \t]*\[([ \w\/\(\)\\\~\-.,\":\%\^\|\{\};\*]+)\][ \t]*(.*)',
l)
if m:
healthy_par_doc = (m.group(1), m.group(2), m.group(3).strip())
par_docs.append(healthy_par_doc)
idxs.append(i)
continue
elif re.match('(\w+):', l):
# empty docstrings
m3 = re.match('(\w+):[ \t]*$', l)
if m3:
empty_par_doc = (m3.group(1), '', '')
par_docs.append(empty_par_doc)
idxs.append(i)
continue
# no-unit docstrings
m4 = re.match(
'(\w+):[ \t]*([ \t\w,.\-\(\)\=\^\/:\"\'\%\<\>\|\{\};\*]+)$',
l)
if m4:
limp_par_doc = (m4.group(1), '', m4.group(2).strip())
par_docs.append(limp_par_doc)
idxs.append(i)
continue
# "inversed" docstrings
m2 = re.match(
'(\w+):[ \t]*(.*)[ \t]*\[([ \w\/\(\)\\\~\-.,\":\%\|\{\}]+)\]',
l)
if m2:
par_doc = (m2.group(1), m2.group(3).strip(), m2.group(2))
par_docs.append(par_doc)
idxs.append(i)
continue
# edit par doc lines, remove superfluous
if True:
if len(par_docs) == 0:
continue
l01 = max([len(p[0]) + len(p[1]) for p in par_docs])
name, real_pars = utils.parse_define_instr(define)
real_parnames = [par[1] for par in real_pars]
# rewrite par doc lines, remove "bonus" doc lines without a parameter to document
for i in range(len(par_docs)):
p = par_docs[i]
idx = idxs[i]
# reorganize the docstring line
format_str = '* %s: %-' + str(l01 - len(p[0]) + 3) + 's %s'
l = format_str % (p[0], '[' + p[1] + ']', p[2])
print(l)
# replace l in lines:
lines[idx] = l
# flag superfluous doc lines for removal
if p[0] not in real_parnames:
# (!!!!)
# TODO: take care of the ordering of idxs_remove. Today, we know that all previuosly
# removed lines are above, but this may change
# (!!!!)
idxs_remove.append(idx)
# add a stub par doc line for each par that isn't represented (WARNING: do not use while removing lines!
if False:
print()
extra_pardoc_lines = []
par_doc_names = [q[0] for q in par_docs]
for i in range(len(real_pars)):
par_name = real_pars[i][1]
if par_name not in par_doc_names:
l = '* %s:' % par_name
print(l)
extra_pardoc_lines.append(l)
# insert those extra lines ...
good_idx = idxs[-1]
for i in range(len(extra_pardoc_lines)):
l = extra_pardoc_lines[i]
lines.insert(good_idx + i + 1, l)
# append/read-append remaining lines
for l in define.splitlines():
lines.append(l)
for l in f:
lines.append(l.rstrip('\n'))
# remove unwanted lines:
for idx in reversed(idxs_remove):
del lines[idx]
for l in lines:
print(l)
continue
f.close()
f = open(filename, 'w')
f.write('\n'.join(lines) + '\n')
f.close()
cnt += 1
print(cnt)
quit()
def main(args):
logging.basicConfig(level=logging.INFO)
usedir = mccode_config.configuration["MCCODE_LIB_DIR"]
if args.dir == None and args.install == False and args.searchterm == None and args.manual == False and args.comps == False and args.web == False:
''' browse system docs and exit '''
subprocess.Popen(
'%s %s' %
(mccode_config.configuration['BROWSER'],
os.path.join(usedir,
mccode_config.get_mccode_prefix() + 'doc.html')),
shell=True)
quit()
elif args.manual == True:
''' open manual and exit '''
subprocess.Popen(
'%s %s' %
(mccode_config.configuration['BROWSER'],
os.path.join(
usedir, 'doc', 'manuals',
mccode_config.configuration['MCCODE'] + '-manual.pdf')),
shell=True)
quit()
elif args.comps == True:
''' open component manual and exit '''
subprocess.Popen(
'%s %s' %
(mccode_config.configuration['BROWSER'],
os.path.join(
usedir, 'doc', 'manuals',
mccode_config.configuration['MCCODE'] + '-components.pdf')),
shell=True)
quit()
elif args.web == True:
''' open website and exit '''
subprocess.Popen(
'%s %s' % (mccode_config.configuration['BROWSER'], 'http://www.' +
mccode_config.configuration['MCCODE'] + '.org'),
shell=True)
quit()
elif args.install == True:
''' write system doc files '''
if args.searchterm:
print("will not write filtered system docs, exiting...")
quit()
print("writing mccode distribution docs...")
if args.dir:
usedir = args.dir
print("using custom dir: %s" % usedir)
comp_infos, instr_infos, comp_files, instr_files = parse_and_filter(
usedir, recursive=True, printlog=args.verbose)
write_doc_files_or_continue(comp_infos, instr_infos, comp_files,
instr_files, args.verbose)
masterdoc = OverviewDocWriter(
comp_infos, instr_infos, [], [],
mccode_config.configuration['MCCODE_LIB_DIR'])
text = masterdoc.create()
mcdoc_html_filepath = os.path.join(
usedir,
mccode_config.get_mccode_prefix() + 'doc.html')
try:
write_file(mcdoc_html_filepath, text)
print("master doc file: %s" % mcdoc_html_filepath)
except Exception as e:
print('ERROR writing master doc file: %s', e)
elif args.dir != None or args.searchterm != None:
''' filtered and/or local results '''
flter = '.*'
usedir = mccode_config.configuration['MCCODE_LIB_DIR']
rec = True
if args.searchterm:
flter = args.searchterm
# single, specific file
if args.searchterm is not None and re.search('\.', args.searchterm):
usedir2 = '.'
if args.dir is not None:
usedir2 = args.dir
f = os.path.join(usedir2, args.searchterm)
# find matcing filenames
instr_files, comp_files = utils.get_instr_comp_files(
mccode_config.configuration['MCCODE_LIB_DIR'], True)
comp_files = [
f for f in comp_files if os.path.basename(f) == args.searchterm
]
instr_files = [
f for f in instr_files
if os.path.basename(f) == args.searchterm
]
# accumulate results
results = []
if os.path.isfile(f):
results.append(f)
results.extend(instr_files)
results.extend(comp_files)
if len(results) == 1:
f = results[0]
instr = re.search('[\w0-9]+\.instr', args.searchterm)
comp = re.search('[\w0-9]+\.comp', args.searchterm)
if instr:
f_html = os.path.splitext(f)[0] + ".html"
info = InstrParser(f).parse()
info.filepath = os.path.abspath(f)
write_doc_files_or_continue([], [info], [], [f])
subprocess.Popen(
'%s %s' %
(mccode_config.configuration['BROWSER'], f_html),
shell=True)
elif comp:
f_html = os.path.splitext(f)[0] + ".html"
info = CompParser(f).parse()
info.filepath = os.path.abspath(f)
write_doc_files_or_continue([info], [], [f], [])
subprocess.Popen(
'%s %s' %
(mccode_config.configuration['BROWSER'], f_html),
shell=True)
quit()
# there were multiple matches - fall back to general search term mode
else:
flter = os.path.splitext(os.path.basename(args.searchterm))[0]
# system
comp_infos, instr_infos, comp_files, instr_files = parse_and_filter(
usedir, flter, recursive=True)
write_doc_files_or_continue(comp_infos, instr_infos, comp_files,
instr_files)
# local
comp_infos_local = []
instr_infos_local = []
if args.dir != None:
usedir = args.dir
comp_infos_local, instr_infos_local, comp_files, instr_files = parse_and_filter(
args.dir, flter, recursive=False)
write_doc_files_or_continue(comp_infos_local, instr_infos_local,
comp_files, instr_files)
if len(comp_infos_local) + len(instr_infos_local) + len(
comp_infos) + len(instr_infos) == 0:
print("no matches found")
quit()
masterdoc = OverviewDocWriter(comp_infos, instr_infos,
comp_infos_local, instr_infos_local,
usedir)
text = masterdoc.create()
mcdoc_html_filepath = os.path.join(
'.',
mccode_config.get_mccode_prefix() + 'doc.html')
if args.verbose:
print('writing local overview doc file... %s' %
mcdoc_html_filepath)
write_file(mcdoc_html_filepath, text)
subprocess.Popen(
'%s %s' %
(mccode_config.configuration['BROWSER'],
os.path.join('.',
mccode_config.get_mccode_prefix() + 'doc.html')),
shell=True)
def main(args):
'''
Behavior: See file header.
'''
logging.basicConfig(level=logging.INFO)
usedir = args.crawl or mccode_config.configuration["MCCODE_LIB_DIR"]
print("using source directory: " + usedir)
'''
# repair mode - do not run mcdoc, just the "repair" function
if args.repair:
#repair_instr(localdir)
repair_comp(usedir)
quit()
'''
# local files
instr_files, comp_files = utils.get_instr_comp_files(usedir)
# parse comp files
comp_info_lst = []
for f in comp_files:
try:
print("parsing... %s" % f)
info = CompParser(f).parse()
info.filepath = f
comp_info_lst.append(info)
except:
print("failed parsing file: %s" % f)
comp_info_lst.append(CompParser(f).stub())
print("parsed comp files: %s" % str(len(comp_files)))
# parse instr files
instr_info_lst = []
for f in instr_files:
try:
print("parsing... %s" % f)
info = InstrParser(f).parse()
info.filepath = f
instr_info_lst.append(info)
except:
print("failed parsing file: %s" % f)
instr_info_lst.append(InstrParser(f).stub())
print("parsed instr files: %s" % str(len(instr_files)))
# apply a name-filter (matches instr / comp name, not filename)
if args.namefilter:
comp_info_lst = [
c for c in comp_info_lst
if re.search(args.namefilter.lower(), c.name.lower())
]
instr_info_lst = [
c for c in comp_info_lst
if re.search(args.namefilter.lower(), c.name.lower())
]
'''
# debug mode - write files with a header property each, then quit
if args.debug:
text = '\n'.join(['%4d: \n%s' % (i, files[i]) for i in range(len(files))])
write_file('files', text)
for i in range(utils.InstrCompHeaderInfo.__len__()-3):
text = '\n'.join(['%4d: %s' % (j, instr_info_lst[j][i]) for j in range(len(instr_info_lst))])
write_file(utils.InstrCompHeaderInfo.colname(i), text)
for i in range(8, 10):
text = '\n'.join(['%4d: \n%s' % (j, '\n'.join(['%-20s, %-10s, %s' % (str(k[0]), str(k[1]), str(k[2])) for k in instr_info_lst[j][i]])) for j in range(len(instr_info_lst))])
write_file(utils.InstrCompHeaderInfo.colname(i), text)
text = '\n'.join(['%4d: \n%s' % (j, '\n'.join(instr_info_lst[j][10])) for j in range(len(instr_info_lst))])
# '\n'.join(info.links)
write_file(utils.InstrCompHeaderInfo.colname(10), text)
quit()
'''
mcdoc_html = os.path.join(usedir, 'mcdoc.html')
# try-catch section for file write
try:
# generate and save comp html doc pages
for i in range(len(comp_info_lst)):
p = comp_info_lst[i]
f = comp_files[i]
doc = CompDocWriter(p)
text = doc.create()
h = os.path.splitext(f)[0] + '.html'
print("writing doc file... %s" % h)
write_file(h, text)
# generate and save instr html doc pages
for i in range(len(instr_info_lst)):
p = instr_info_lst[i]
f = instr_files[i]
doc = InstrDocWriter(p)
text = doc.create()
h = os.path.splitext(f)[0] + '.html'
print("writing doc file... %s" % h)
write_file(h, text)
# write overview files, properly assembling links to instr- and html-files
masterdoc = OverviewDocWriter(
comp_info_lst, instr_info_lst,
mccode_config.configuration['MCCODE_LIB_DIR'])
text = masterdoc.create()
print('writing master doc file... %s' % os.path.abspath('mcdoc.html'))
write_file(mcdoc_html, text)
except Exception as e:
print("Could not write to disk: %s" % e.__str__())
# open a web-browser in a cross-platform way, unless --nobrowse has been enabled
if not args.nobrowse:
subprocess.Popen('%s %s' %
(mccode_config.configuration['BROWSER'], mcdoc_html),
shell=True)
def mccode_test(branchdir, testdir, limitinstrs=None, instrfilter=None):
''' this main test function tests the given mccode branch/version '''
# copy instr files and record info
logging.info("Finding instruments in: %s" % branchdir)
instrs, _ = utils.get_instr_comp_files(join(branchdir, "examples"),
recursive=True,
instrfilter=instrfilter)
instrs.sort()
# limt runs if required
if limitinstrs:
instrs = instrs[:limitinstrs]
# max instr name length for pretty-output
maxnamelen = 0
for f in instrs:
l = len(basename(f)) - 5
if l > maxnamelen:
maxnamelen = l
# create test objects and copy instrument files
logging.info("Copying instruments to: %s" % testdir)
tests = []
for f in instrs:
# create the test foldr for this instrument
instrname = splitext(basename(f))[0]
instrdir = join(testdir, instrname)
mkdir(instrdir)
# create a new file with the instr text in it - e.g. a local copy of the instrument file
text = open(f, encoding='utf-8').read()
f_new = join(instrdir, basename(f))
open(f_new, 'w', encoding='utf-8').write(text)
# create a test object for every test defined in the instrument header
instrtests = create_instr_test_objs(sourcefile=f,
localfile=f_new,
header=text)
tests = tests + instrtests
# extract and record %Example info from text
numtests = len([t for t in instrtests if t.testnb > 0])
if numtests == 0:
formatstr = "%-" + "%ds: NO TEST" % maxnamelen
logging.debug(formatstr % instrname)
elif numtests == 1:
formatstr = "%-" + "%ds: TEST" % maxnamelen
logging.debug(formatstr % instrname)
else:
formatstr = "%-" + "%ds: TESTS (%d)" % (maxnamelen, numtests)
logging.debug(formatstr % instrname)
# compile, record time
logging.info("")
logging.info("Compiling instruments [seconds]...")
for test in tests:
# if binary exists, set compile time = 0 and continue
binfile = os.path.splitext(test.localfile)[0] + ".out"
if os.path.exists(binfile):
test.compiled = True
test.compiletime = 0
else:
if test.testnb > 0 or (not args.skipnontest):
log = LineLogger()
t1 = time.time()
cmd = "mcrun --info %s &> compile_stdout.txt" % test.localfile
utils.run_subtool_noread(cmd,
cwd=join(testdir, test.instrname))
t2 = time.time()
test.compiled = os.path.exists(binfile)
test.compiletime = t2 - t1
# log to terminal
if test.compiled:
formatstr = "%-" + "%ds: " % maxnamelen + \
"{:3d}.".format(math.floor(test.compiletime)) + str(test.compiletime-int(test.compiletime)).split('.')[1][:2]
logging.info(formatstr % test.get_display_name())
else:
formatstr = "%-" + "%ds: COMPILE ERROR using:\n" % maxnamelen
logging.info(formatstr % test.instrname + cmd)
else:
logging.info("Skipping compile of " + test.instrname)
# save (incomplete) test results to disk
test.save(infolder=join(testdir, test.instrname))
# run, record time
logging.info("")
logging.info("Running tests...")
for test in tests:
if not test.compiled:
formatstr = "%-" + "%ds: NO COMPILE" % (maxnamelen + 1)
logging.info(formatstr % test.instrname)
continue
# runable tests have testnb > 0
if test.testnb <= 0:
formatstr = "%-" + "%ds: NO TEST" % (maxnamelen + 1)
logging.info(formatstr % test.get_display_name())
continue
# run the test, record time and runtime success/fail
t1 = time.time()
global ncount, mpi
if mpi is not None:
cmd = "mcrun -s 1000 %s %s -n%s --mpi=%s -d%d &> run_stdout.txt" % (
test.localfile, test.parvals, ncount, mpi, test.testnb)
else:
cmd = "mcrun -s 1000 %s %s -n%s -d%d &> run_stdout.txt" % (
test.localfile, test.parvals, ncount, test.testnb)
retcode = utils.run_subtool_noread(cmd,
cwd=join(testdir, test.instrname))
t2 = time.time()
didwrite = os.path.exists(
join(testdir, test.instrname, str(test.testnb), "mccode.sim"))
test.didrun = retcode != 0 or didwrite
test.runtime = t2 - t1
# log to terminal
if test.didrun:
formatstr = "%-" + "%ds: " % (maxnamelen+1) + \
"{:3d}.".format(math.floor(test.runtime)) + str(test.runtime-int(test.runtime)).split('.')[1][:2]
logging.info(formatstr % test.get_display_name())
else:
formatstr = "%-" + "%ds: RUNTIME ERROR" % (maxnamelen + 1)
logging.info(formatstr % instrname + ", " + cmd)
continue
# test value extraction
extraction = extract_testvals(
join(testdir, test.instrname, str(test.testnb)), test.detector)
if type(extraction) is tuple:
test.testval = extraction[0]
else:
test.testval = -1
# save test result to disk
test.testcomplete = True
test.save(infolder=join(testdir, test.instrname))
# cpu type: cat /proc/cpuinfo |grep name |uniq | cut -f2- -d:
# gpu type: nvidia-smi -L | head -1 |cut -f2- -d: |cut -f1 -d\(
metalog = LineLogger()
utils.run_subtool_to_completion(
"cat /proc/cpuinfo |grep name |uniq | cut -f2- -d: | xargs echo",
stdout_cb=metalog.logline)
cpu_type = ",".join(metalog.lst)
metalog = LineLogger()
utils.run_subtool_to_completion(
"nvidia-smi -L | head -1 |cut -f2- -d: |cut -f1 -d\(",
stdout_cb=metalog.logline)
gpu_type = ",".join(metalog.lst)
if "failed because" in gpu_type:
gpu_type = "none"
metalog = LineLogger()
utils.run_subtool_to_completion("hostname", stdout_cb=metalog.logline)
hostnamestr = ",".join(metalog.lst)
metalog = LineLogger()
utils.run_subtool_to_completion('echo "$USER"', stdout_cb=metalog.logline)
username = "******".join(metalog.lst)
metainfo = OrderedDict()
metainfo["ncount"] = ncount
metainfo["mpi"] = mpi
metainfo["date"] = utils.get_datetimestr()
metainfo["hostname"] = hostnamestr
metainfo["user"] = username
metainfo["cpu_type"] = cpu_type
metainfo["gpu_type"] = gpu_type
# displayname must be unique, we can return a dict, which eases comparison between tests
obj = {}
for t in tests:
obj[t.get_display_name()] = t.get_json_repr()
obj["_meta"] = metainfo
return obj