def clone_repos(config, target):
if not target:
Out.error(
"Unable to determine the target path for the cloned repos, make sure the config defines a target value."
)
exit(-2)
if not os.path.exists(target):
Out.info(f"Dir [{target}] is missing, creating... ")
os.mkdir(target)
cloned_repos = []
git_uris = config["repositories"]
if not git_uris:
Out.error(
"The repositories to clone are missing, make sure the config file defines a repos section."
)
exit(-2)
for g_uri in git_uris:
try:
cloned_repo = git.Git(target).clone(g_uri)
if cloned_repo:
cloned_repos.append(cloned_repo)
Out.info(f"Repository [{g_uri}] cloned to [{target}].")
except git.exc.GitCommandError as git_error:
Out.warn(f"Unable to clone [{g_uri}].", git_error)
def _report(self):
template = u"\n{title}\n{issues}\n\n{statistics}"
title = u"Scaning <{directory}> at {time}".format(
directory=Out.R(conf.target),
time=Out.R(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())))
issues_content = \
Out.Y("-"*80 + "\nFound security issues as follows:\n\n")
for issue in issuemgr:
issues_content = issues_content + self._format_issue(issue)
statistics = Out.Y("-" * 80 + "\nStatistics information:\n")
sinfo = issuemgr.statistics()
for s in sinfo:
statistics = statistics + \
"{key}: {value}".format(key=severity_map[s][0].capitalize(),
value=sinfo[s]) + "\n"
content = template.format(title=title,
issues=issues_content,
statistics=statistics)
return content
def infer_version():
regex = re.compile(
r"Inferred project: graphql-dgs-codegen, version: ([0-9A-Z\-.]+)")
out = subprocess.check_output([gradlew, "-p", gradle_wd,
"project"]).decode("utf-8")
Out.debug(f"Process output:\n{out}", State.verbose)
match = re.search(regex, out)
return match.group(1) if match else ""
def to_asm(section, base_name):
c_file = os.path.join(sys_compiler.TEMPDIR, '%s.c' % base_name)
out = Out(c_file)
out << '#include <inttypes.h>\n'
to_c(section, out)
out.close()
code = sys_compiler.c_to_asm(base_name)
if code:
raise GVMTException("GVMT Internal Error -- %s failed with exit code %d" % (CC_PATH, code))
def infer_build_file(project_dir):
if os.path.isfile(f"{project_dir}/build.gradle.kts"):
build_file = f"{project_dir}/build.gradle.kts"
elif os.path.isfile(f"{project_dir}/build.gradle"):
Out.error(f"We only support Gradle Kotlin (build.gradle.kts) files.")
sys.exit(2)
else:
Out.error(
f"Unable to infer the build file for project [{project_dir}]~")
sys.exit(2)
return build_file
def _format_issue(self, issue):
template = (u"[{id}:{name}]\n"
u"<Match:{pattern}> <Severity:{severity}> "
u"<Confidence:{confidence}>\n"
u"@{filename}\n"
u"{context}\n")
return template.format(
id=Out.R(issue['ID']),
name=Out.G(issue['name']),
pattern=Out.Y(issue['pattern']),
severity=severity_map[issue['severity']][0].capitalize(),
confidence=confidence_map[issue['confidence']][0].capitalize(),
filename=Out.B(issue['filename']),
context=self._format_issue_context(issue))
def analyze(self):
filemgr.init()
featuremgr.init()
for file in filemgr.walk():
if conf.verbose:
Out.info("scanning file {0}".format(file.filename))
try:
for feature in featuremgr[file.scope]:
matchctxes = file.match(feature.patterns, conf.ectx)
for matchctx in matchctxes:
feature.evaluate(matchctx, conf.sctx)
except KeyError:
continue
def analyze(self):
filemgr.init()
featuremgr.init()
for file in filemgr.walk():
if conf.verbose:
Out.info("scanning file {0}".format(file.filename))
try:
for feature in featuremgr[file.scope]:
matchctxes = file.match(feature.patterns, conf.ectx)
for matchctx in matchctxes:
feature.evaluate(matchctx, conf.sctx)
except KeyError:
continue
def to_object(src_file, base_name, library, optimise, sys_headers, gc_name):
code = src_file.code
bytecodes = src_file.bytecodes
info = src_file.info
c_file = os.path.join(sys_compiler.TEMPDIR, '%s.c' % base_name)
out = Out(c_file)
out << '#include <alloca.h>\n'
out << '#include "gvmt/internal/core.h"\n'
out << 'extern void* malloc(uintptr_t size);\n'
out << 'extern int strcmp(void* s1, void* s2);\n'
out << 'extern GVMT_Object gvmt_%s_malloc(GVMT_StackItem* sp, GVMT_Frame fp, uintptr_t size);\n' % gc_name
out << 'extern void __gvmt_fatal(const char* fmt, ...);\n'
out << '#define GVMT_COMPILED\n'
out << '\n'
out << '\n'
write_info(info, out)
if bytecodes:
write_interpreter(bytecodes, out, gc_name)
write_code(code, out, gc_name)
out.close()
code = sys_compiler.c_to_object(base_name, optimise, library, True, sys_headers)
if code:
raise GVMTException("GVMT Internal Error -- %s failed with exit code %d" % (CC_PATH, code))
def _format_issue_context(self, issue):
result = ""
if not issue['context']:
return result
largest_lineno = issue['context'][-1][0]
no_fmt = "{0:>" + str(len(str(largest_lineno))) + "}"
for line in issue['context']:
if line[0] == issue['lineno']:
result = result + Out.Y(no_fmt.format(str(line[0])) + ": " +\
line[1].rstrip() + "\n")
else:
result = result + no_fmt.format(str(line[0])) + "- " +\
line[1].rstrip() + "\n"
return result
def run_example_build(settings_file, project_dir):
command = [
gradlew, "-p", project_dir, "-c", settings_file, "clean", "check"
]
str_cmd = " ".join(command)
try:
Out.info(f"Running {str_cmd}")
p = subprocess.check_output(command)
Out.info(p.decode("utf-8"))
except subprocess.SubprocessError as error:
Out.error(f"Unable to test {project_dir}, command '{str_cmd}' failed!",
error)
sys.exit(2)
def run_example_build(project_dir, build_file="", settings_file=""):
command = [
gradlew, "-p", project_dir, "-s", "-w", "--info", "--stacktrace"
]
if settings_file:
command.extend(["-c", settings_file])
else:
Out.error(
f"The project {project_dir} is missing a Gradle settings file!")
sys.exit(2)
command.extend(["clean", "check"])
str_cmd = " ".join(command)
try:
Out.info(f"Running {str_cmd}")
p = subprocess.check_output(command)
Out.info(p.decode("utf-8"))
except subprocess.SubprocessError as error:
Out.error(f"Unable to test {project_dir}, command '{str_cmd}' failed!",
error)
sys.exit(2)
def load_config():
with open(config_file, "r") as yml_file:
config = yaml.load(yml_file, Loader=yaml.SafeLoader)
Out.debug(f"Configuration loaded from [{config_file}]:\n{config}\n",
State.verbose)
return config
def main(argv):
script_name = os.path.basename(__file__)
help_message = f"""
{BColors.HEADER}Options{BColors.ENDC}:
-c | codegen= : Version we need to apply, if left empty it will calculate the version based on Gradle's Project task.
-p | path= : Path where the examples should be found. Defaults to [{examples_path}].
-k : By default the directory defined in the path will be removed on success. If this flag is set the directory will be kept.
-v : Verbose
{BColors.HEADER}Example{BColors.ENDC}: {script_name} -c <codegen version>
"""
projects_dir = examples_path
codegen_version = ""
keep_project_dir = False
try:
opts, args = getopt.getopt(argv, "hvkc:p:", ["codegen=", "path="])
except getopt.GetoptError:
Out.usage(script_name, help_message)
sys.exit(2)
for opt, arg in opts:
if opt == '-h':
Out.usage(script_name, help_message)
sys.exit()
elif opt in ("-c", "--codegen"):
codegen_version = arg
elif opt in ("-p", "--path"):
projects_dir = os.path.abspath(arg)
elif opt in ("-k"):
keep_project_dir = True
elif opt in ("-v"):
State.verbose = True
if not codegen_version:
Out.info("Codgen version not supplied, inferring...")
codegen_version = infer_version()
if codegen_version:
Out.info(
f"Codegen Version resolved to {BColors.OKGREEN}{codegen_version}{BColors.ENDC}"
)
else:
Out.error("Unable to resolved a Codegen Version!")
exit(2)
clone_repos(load_config(), projects_dir)
projects = next(os.walk(projects_dir))[1]
if not projects:
Out.error(f"No projects available at [{projects_dir}]!")
exit(2)
for project in projects:
project_root = f"{projects_dir}/{project}"
Out.info(f"Processing project [{project_root}]...")
infer_build_file(project_root)
gradle_settings_file_path = f"{project_root}/settings.gradle.kts"
generate_gradle_settings(settings_gradle_kts_template, codegen_version,
gradle_settings_file_path)
run_example_build(gradle_settings_file_path, project_root)
if not keep_project_dir:
Out.info(f"Removing {projects_dir}...")
try:
shutil.rmtree(projects_dir)
except Exception as error:
Out.error(f"Failed deleting {projects_dir}.", error)
Out.ok(f"Build successful.")
def main(argv):
script_name = os.path.basename(__file__)
help_message = f"""
{BColors.HEADER}Options{BColors.ENDC}:
-c | codegen= : Version we need to apply, if left empty it will calculate the version based
on Gradle's Project task.
-p | path= : Path where the examples should be found. Defaults to [{examples_path}].
-g : Clone the examples before attempting to execute them, this is done via git-clone.
The repositories are specified via the "repositories:" section in {config_file}.
-k : By default the directory defined in the path will be removed on success.
If this flag is set the directory will be kept.
-v : Verbose
{BColors.HEADER}Example{BColors.ENDC}: {script_name} -c <codegen version>
"""
projects_dir = examples_path
codegen_version = ""
git_clone_projects = False
keep_project_dir = False
try:
opts, args = getopt.getopt(argv, "hvkgc:p:", ["codegen=", "path="])
except getopt.GetoptError:
Out.usage(script_name, help_message)
sys.exit(2)
for opt, arg in opts:
if opt == '-h':
Out.usage(script_name, help_message)
sys.exit()
elif opt in ("-c", "--codegen"):
codegen_version = arg
elif opt in ("-p", "--path"):
projects_dir = os.path.abspath(arg)
elif opt in "-g":
git_clone_projects = True
elif opt in "-k":
keep_project_dir = True
elif opt in "-v":
State.verbose = True
if not codegen_version:
Out.info("Version not supplied, inferring...")
codegen_version = infer_version()
if codegen_version:
Out.info(
f"Version resolved to {BColors.OKGREEN}{codegen_version}{BColors.ENDC}"
)
else:
Out.error("Unable to resolved a version!")
exit(2)
if git_clone_projects:
Out.info(f"Cloning example repositories to {projects_dir}...")
clone_repos(load_config(), projects_dir)
if not os.path.exists(projects_dir):
Out.error(
f"Can not find projects to build, the path {projects_dir} doesn't exist!"
)
exit(2)
projects = next(os.walk(projects_dir))[1]
if not projects:
Out.error(f"No projects available at [{projects_dir}]!")
exit(2)
for project in projects:
project_root = f"{projects_dir}/{project}"
Out.info(f"Processing project [{project_root}]...")
build_file = infer_build_file(project_root)
settings_file = infer_gradle_settings_file(project_root)
update_build(build_file, codegen_version)
run_example_build(project_root,
build_file=build_file,
settings_file=settings_file)
if not keep_project_dir:
Out.info(f"Removing {projects_dir}...")
try:
shutil.rmtree(projects_dir)
except Exception as error:
Out.error(f"Failed deleting {projects_dir}.", error)
Out.ok(f"Build successful.")