def derive(deriver, value, resources, filename, **kwargs):
extractor = value["extractor"]
geshicode = value["geshi"]
extractorpath = resources[1]
deriver.dump["geshicodes"].add(geshicode)
deriver.dump["extractors"].add(extractor)
command = ["php", "helper.php", filename, geshicode, extractorpath]
result = json.loads(subprocess.check_output(command))
return (result["metrics"], result["tokens"])
def preparedump(deriver):
for key in ["geshicodes", "extractors"]:
deriver.dump[key] = sorted(list(deriver.dump[key]))
changed = meta101.havechanged(__file__, "helper.php", "megalib_leftover.php")
meta101.derive(suffix=[".fragments.metrics.json", ".fragments.tokens.json"],
resources=[Json(".matches.json"),
File(".extractor.json")],
dump=os.environ["fragmentMetrics101dump"],
oninit=initdump,
getvalue=getextractor,
callback=derive,
ondump=preparedump,
entirerepo=changed)
deriver.dump["extractors"] = set(deriver.dump["extractors"])
else:
deriver.dump["extractors"] = set()
def derive(deriver, language, filename, **kwargs):
extractorPath = os.path.join(os.environ["extractor101dir"],language, "extractor")
if os.path.isfile(extractorPath):
deriver.dump["extractors"].add(extractorPath)
# extractors take their input via stdin, so we gotta open the file
with open(filename) as f:
return json.loads(subprocess.check_output([extractorPath], stdin=f))
def preparedump(deriver):
deriver.dump["extractors"] = sorted(list(deriver.dump["extractors"]))
edir = os.environ["extractor101dir"]
extractor = [os.path.join(edir, d, "extractor") for d in os.listdir(edir)]
changed = meta101.havechanged(__file__, "module.json", *extractor )
meta101.derive(suffix =".extractor.json",
dump =os.environ["extractor101dump"],
oninit =initdump,
getvalue ="language",
callback =derive,
ondump =preparedump,
entirerepo=changed)
def derive(deriver, value, resources, filename, **kwargs):
extractor = value["extractor"]
geshicode = value["geshi"]
extractorpath = resources[1]
deriver.dump["geshicodes"].add(geshicode)
deriver.dump["extractors"].add(extractor)
command = ["php", "helper.php", filename, geshicode, extractorpath]
result = json.loads(subprocess.check_output(command))
return (result["metrics"], result["tokens"])
def preparedump(deriver):
for key in ["geshicodes", "extractors"]:
deriver.dump[key] = sorted(list(deriver.dump[key]))
changed = meta101.havechanged(__file__, "helper.php", "megalib_leftover.php")
meta101.derive(suffix =[".fragments.metrics.json", ".fragments.tokens.json"],
resources =[Json(".matches.json"), File(".extractor.json")],
dump =os.environ["fragmentMetrics101dump"],
oninit =initdump,
getvalue =getextractor,
callback =derive,
ondump =preparedump,
entirerepo=changed)
import meta101
geshi = os.environ["gatheredGeshi101dir"] + "/run.php"
def initdump(deriver):
if "geshicodes" in deriver.dump:
deriver.dump["geshicodes"] = set(deriver.dump["geshicodes"])
else:
deriver.dump["geshicodes"] = set()
def derive(deriver, geshicode, filename, **kwargs):
deriver.dump["geshicodes"].add(geshicode)
command = ["php", geshi, filename, "php://stdout", geshicode]
return subprocess.check_output(command)
def preparedump(deriver):
deriver.dump["geshicodes"] = sorted(list(deriver.dump["geshicodes"]))
meta101.derive(suffix =".geshi.html",
dump =os.environ["geshi101dump"],
oninit =initdump,
getvalue ="geshi",
callback =derive,
ondump =preparedump,
entirerepo=meta101.havechanged(__file__))
try:
output, status = (subprocess.check_output(command), 0)
except subprocess.CalledProcessError as e:
output, status = (e.output, e.returncode)
return {
"validator" : path,
"command" : command,
"status" : status,
"output" : output,
}
def preparedump(deriver):
deriver.dump["validators"] = sorted(list(deriver.dump["validators"]))
vdir = os.environ["validator101dir"]
validator = [os.path.join(vdir, d, "validator") for d in os.listdir(vdir)]
changed = meta101.havechanged(__file__, "module.json", *validator )
meta101.derive(suffix =".validator.json",
dump =os.environ["validator101dump"],
oninit =initdump,
getvalue ="language",
callback =derive,
ondump =preparedump,
entirerepo=changed)
#!/usr/bin/env python
import os
import meta101
pdir = os.environ["predicates101dir"]
predicates = [os.path.join(pdir, d, "predicate.py") for d in os.listdir(pdir)]
changed = meta101.havechanged(__file__, "module.json", *predicates )
meta101.matchall("predicates", entirerepo=changed)
import subprocess
import meta101
geshi = os.environ["gatheredGeshi101dir"] + "/run.php"
def initdump(deriver):
if "geshicodes" in deriver.dump:
deriver.dump["geshicodes"] = set(deriver.dump["geshicodes"])
else:
deriver.dump["geshicodes"] = set()
def derive(deriver, geshicode, filename, **kwargs):
deriver.dump["geshicodes"].add(geshicode)
command = ["php", geshi, filename, "php://stdout", geshicode]
return subprocess.check_output(command)
def preparedump(deriver):
deriver.dump["geshicodes"] = sorted(list(deriver.dump["geshicodes"]))
meta101.derive(suffix=".geshi.html",
dump=os.environ["geshi101dump"],
oninit=initdump,
getvalue="geshi",
callback=derive,
ondump=preparedump,
entirerepo=meta101.havechanged(__file__))
#!/usr/bin/env python
import meta101
# FIXME also check changes on fragments when they get moved into 101worker
meta101.matchall("fragments", entirerepo=meta101.havechanged(__file__))
#!/usr/bin/env python
import os
import meta101
pdir = os.environ["predicates101dir"]
predicates = [os.path.join(pdir, d, "predicate.py") for d in os.listdir(pdir)]
changed = meta101.havechanged(__file__, "module.json", *predicates)
meta101.matchall("predicates", entirerepo=changed)
#!/usr/bin/env python
import meta101
meta101.matchall("matches", meta101.havechanged(__file__))
# subprocess has no safe getstatusoutput, so this will have to do
try:
output, status = (subprocess.check_output(command), 0)
except subprocess.CalledProcessError as e:
output, status = (e.output, e.returncode)
return {
"validator": path,
"command": command,
"status": status,
"output": output,
}
def preparedump(deriver):
deriver.dump["validators"] = sorted(list(deriver.dump["validators"]))
vdir = os.environ["validator101dir"]
validator = [os.path.join(vdir, d, "validator") for d in os.listdir(vdir)]
changed = meta101.havechanged(__file__, "module.json", *validator)
meta101.derive(suffix=".validator.json",
dump=os.environ["validator101dump"],
oninit=initdump,
getvalue="language",
callback=derive,
ondump=preparedump,
entirerepo=changed)