def run(context):
meta101.os.environ = context.get_env()
meta101.derive(suffix =[".metrics.json", ".tokens.json"],
dump =os.environ["metrics101dump"],
oninit =initdump,
getvalue =getgeshi,
callback =derive,
ondump =preparedump,
entirerepo=changed)
def run(context):
meta101.os.environ = context.get_env()
meta101.derive(suffix=[".metrics.json", ".tokens.json"],
dump=os.environ["metrics101dump"],
oninit=initdump,
getvalue=getgeshi,
callback=derive,
ondump=preparedump,
entirerepo=changed)
deriver.dump["extractors"] = set(deriver.dump["extractors"])
else:
deriver.dump["extractors"] = set()
def derive(deriver, language, filename, **kwargs):
extractorPath = os.path.join(os.environ["extractor101dir"],language, "extractor")
if os.path.isfile(extractorPath):
deriver.dump["extractors"].add(extractorPath)
# extractors take their input via stdin, so we gotta open the file
with open(filename) as f:
return json.loads(subprocess.check_output([extractorPath], stdin=f))
def preparedump(deriver):
deriver.dump["extractors"] = sorted(list(deriver.dump["extractors"]))
edir = os.environ["extractor101dir"]
extractor = [os.path.join(edir, d, "extractor") for d in os.listdir(edir)]
changed = meta101.havechanged(__file__, "module.json", *extractor )
meta101.derive(suffix =".extractor.json",
dump =os.environ["extractor101dump"],
oninit =initdump,
getvalue ="language",
callback =derive,
ondump =preparedump,
entirerepo=changed)
def derive(deriver, value, resources, filename, **kwargs):
extractor = value["extractor"]
geshicode = value["geshi"]
extractorpath = resources[1]
deriver.dump["geshicodes"].add(geshicode)
deriver.dump["extractors"].add(extractor)
command = ["php", "helper.php", filename, geshicode, extractorpath]
result = json.loads(subprocess.check_output(command))
return (result["metrics"], result["tokens"])
def preparedump(deriver):
for key in ["geshicodes", "extractors"]:
deriver.dump[key] = sorted(list(deriver.dump[key]))
changed = meta101.havechanged(__file__, "helper.php", "megalib_leftover.php")
meta101.derive(suffix=[".fragments.metrics.json", ".fragments.tokens.json"],
resources=[Json(".matches.json"),
File(".extractor.json")],
dump=os.environ["fragmentMetrics101dump"],
oninit=initdump,
getvalue=getextractor,
callback=derive,
ondump=preparedump,
entirerepo=changed)
import meta101
geshi = os.environ["gatheredGeshi101dir"] + "/run.php"
def initdump(deriver):
if "geshicodes" in deriver.dump:
deriver.dump["geshicodes"] = set(deriver.dump["geshicodes"])
else:
deriver.dump["geshicodes"] = set()
def derive(deriver, geshicode, filename, **kwargs):
deriver.dump["geshicodes"].add(geshicode)
command = ["php", geshi, filename, "php://stdout", geshicode]
return subprocess.check_output(command)
def preparedump(deriver):
deriver.dump["geshicodes"] = sorted(list(deriver.dump["geshicodes"]))
meta101.derive(suffix =".geshi.html",
dump =os.environ["geshi101dump"],
oninit =initdump,
getvalue ="geshi",
callback =derive,
ondump =preparedump,
entirerepo=meta101.havechanged(__file__))
def derive(deriver, value, resources, filename, **kwargs):
extractor = value["extractor"]
geshicode = value["geshi"]
extractorpath = resources[1]
deriver.dump["geshicodes"].add(geshicode)
deriver.dump["extractors"].add(extractor)
command = ["php", "helper.php", filename, geshicode, extractorpath]
result = json.loads(subprocess.check_output(command))
return (result["metrics"], result["tokens"])
def preparedump(deriver):
for key in ["geshicodes", "extractors"]:
deriver.dump[key] = sorted(list(deriver.dump[key]))
changed = meta101.havechanged(__file__, "helper.php", "megalib_leftover.php")
meta101.derive(suffix =[".fragments.metrics.json", ".fragments.tokens.json"],
resources =[Json(".matches.json"), File(".extractor.json")],
dump =os.environ["fragmentMetrics101dump"],
oninit =initdump,
getvalue =getextractor,
callback =derive,
ondump =preparedump,
entirerepo=changed)
eq_ok(f.read(), want, name)
else:
eq_ok(json.load(f), want, name)
except Exception as e:
ok(False, "{} ({}: {})".format(name, e.__class__.__name__, e))
def derivelanguage(deriver, language, **kwargs):
if language == "Java":
raise ValueError("Blech")
return language
reset()
lives_ok(lambda: derive(suffix =".lang",
dump =os.path.join(tmp, "lang.json"),
callback=derivelanguage,
getvalue="key"),
"empty input runs ok")
file_ok(tmp, "lang.json", {"problems" : {}}, "...new, empty dump is created")
reset()
lives_ok(lambda: derive(suffix =".lang",
dump =os.path.join(tmp, "lang.json"),
callback =derivelanguage,
getvalue ="language",
entirerepo=True),
"simple language derivation over entire repo")
file_ok(tmp, "lang.json", {"problems" : {"java" : "Blech"}},
import subprocess
import meta101
geshi = os.environ["gatheredGeshi101dir"] + "/run.php"
def initdump(deriver):
if "geshicodes" in deriver.dump:
deriver.dump["geshicodes"] = set(deriver.dump["geshicodes"])
else:
deriver.dump["geshicodes"] = set()
def derive(deriver, geshicode, filename, **kwargs):
deriver.dump["geshicodes"].add(geshicode)
command = ["php", geshi, filename, "php://stdout", geshicode]
return subprocess.check_output(command)
def preparedump(deriver):
deriver.dump["geshicodes"] = sorted(list(deriver.dump["geshicodes"]))
meta101.derive(suffix=".geshi.html",
dump=os.environ["geshi101dump"],
oninit=initdump,
getvalue="geshi",
callback=derive,
ondump=preparedump,
entirerepo=meta101.havechanged(__file__))
("refinedTokens", []),
])
def readif(filename, default):
try:
with open(filename) as f:
return json.load(f)
except (IOError, ValueError):
return default
def derive(deriver, value, resources, **kwargs):
zipped = zip(resources, defaults.keys(), defaults.values())
got = {key: readif(path, default) for path, key, default in zipped}
return {
"units": got["matches"] + got["predicates"] + got["fragments"],
"metrics": got["metrics"],
"refinedTokens": got["refinedTokens"],
}
resrc = [meta101.resource.Pass(".{}.json".format(k)) for k in defaults.keys()]
meta101.derive(suffix=".summary.json",
dump=os.environ["summary101dump"],
getvalue=lambda *args, **kwargs: True,
callback=derive,
resources=resrc,
entirerepo=meta101.havechanged(__file__))
# subprocess has no safe getstatusoutput, so this will have to do
try:
output, status = (subprocess.check_output(command), 0)
except subprocess.CalledProcessError as e:
output, status = (e.output, e.returncode)
return {
"validator": path,
"command": command,
"status": status,
"output": output,
}
def preparedump(deriver):
deriver.dump["validators"] = sorted(list(deriver.dump["validators"]))
vdir = os.environ["validator101dir"]
validator = [os.path.join(vdir, d, "validator") for d in os.listdir(vdir)]
changed = meta101.havechanged(__file__, "module.json", *validator)
meta101.derive(suffix=".validator.json",
dump=os.environ["validator101dump"],
oninit=initdump,
getvalue="language",
callback=derive,
ondump=preparedump,
entirerepo=changed)
