def test_parse_valid_and_save(self):
mp = ManifestParser(self.valid_xml_file)
mp.parse()
self.assertEqual(len(mp.media_file_objs), 2)
mp.save()
self.assertEqual(MediaFile.objects.count(), 2)
manifest = Manifest.objects.get(id=1)
self.assertEqual(manifest.filename, valid_manifest_file)
self.assertEqual(manifest.media_files.count(), 2)
def clean_file(self):
self._parser = ManifestParser(self.cleaned_data['file'])
try:
self._parser.parse()
except ParseError:
raise forms.ValidationError('Invalid manifest file uploaded')
class ManifestForm(forms.Form):
"""
Manifest file upload form
"""
file = forms.FileField(label='Manifest File')
def __init__(self, *args, **kwargs):
self._parser = None # XML parser
super(ManifestForm, self).__init__(*args, **kwargs)
def clean_file(self):
self._parser = ManifestParser(self.cleaned_data['file'])
try:
self._parser.parse()
except ParseError:
raise forms.ValidationError('Invalid manifest file uploaded')
def save(self):
return self._parser.save()
def parse_manifest(self):
if self.manifest_dict:
return
if self.isparsed is True or self.isfetched is False:
return
if self.manifest is None:
self.manifest = self._search_for_manifest()
if self.path is None:
raise RuntimeError()
manifest_parser = ManifestParser()
manifest_parser.add_arbitrary_code(global_mod.options.arbitrary_code)
if self.manifest is None:
logging.debug("No manifest found in module "+str(self))
else:
logging.debug("Parse manifest in: %s" % self.path)
manifest_parser.add_manifest(self.manifest)
if self.parent is None:
allow_unknown = True
extra_context = {}
else:
if global_mod.options.allow_unknown is True:
allow_unknown = True
else:
allow_unknown = False
extra_context = dict(global_mod.top_module.manifest_dict) # copy the dictionary
del extra_context["modules"]
del extra_context["files"]
del extra_context["include_dirs"]
del extra_context["sim_only_files"]
del extra_context["incl_makefiles"]
del extra_context["bit_file_targets"]
del extra_context["library"]
extra_context["__manifest"] = self.path
opt_map = None
try:
opt_map = manifest_parser.parse(allow_unknown=allow_unknown,
extra_context=extra_context)
except NameError as ne:
logging.error("Error while parsing {0}:\n{1}: {2}.".format(self.manifest, type(ne), ne))
quit()
self.manifest_dict = opt_map
def setUp(self):
mp = ManifestParser(open(test_upload_manifest, 'r'))
mp.parse()
self.manifest = mp.save()
self.upload_url = '/upload/%d' % self.manifest.id
def parse_manifest(self):
if self.isparsed == True or self.isfetched == False:
return
if self.manifest == None:
self.manifest = self.__search_for_manifest()
if self.path == None:
raise RuntimeError()
manifest_parser = ManifestParser()
# For non-top modules
if(self.parent != None):
manifest_parser.add_arbitrary_code("target=\""+str(global_mod.top_module.target)+"\"")
manifest_parser.add_arbitrary_code("action=\""+str(global_mod.top_module.action)+"\"")
# syn_device and sim_tool will be set for non-top modules
manifest_parser.add_arbitrary_code("syn_device=\""+str(global_mod.top_module.syn_device)+"\"")
manifest_parser.add_arbitrary_code("__manifest=\""+self.path+"\"")
manifest_parser.add_arbitrary_code(global_mod.options.arbitrary_code)
if self.manifest == None:
p.vprint("No manifest found in module "+str(self))
else:
manifest_parser.add_manifest(self.manifest)
p.vprint("Parsing manifest file: " + str(self.manifest))
opt_map = None
try:
opt_map = manifest_parser.parse()
except NameError as ne:
p.echo("Error while parsing {0}:\n{1}: {2}.".format(self.manifest, type(ne), ne))
quit()
if(opt_map["fetchto"] != None):
fetchto = path_mod.rel2abs(opt_map["fetchto"], self.path)
self.fetchto = fetchto
else:
fetchto = self.fetchto
if self.ise == None:
self.ise = "13.1"
if "local" in opt_map["modules"]:
local_paths = self.__make_list(opt_map["modules"]["local"])
local_mods = []
for path in local_paths:
if path_mod.is_abs_path(path):
p.error("Found an absolute path (" + path + ") in a manifest")
p.rawprint("(" + self.path + ")")
quit()
path = path_mod.rel2abs(path, self.path)
local_mods.append(self.pool.new_module(parent=self, url=path, source="local", fetchto=fetchto))
self.local = local_mods
else:
self.local = []
self.vmap_opt = opt_map["vmap_opt"]
self.vcom_opt = opt_map["vcom_opt"]
self.vsim_opt = opt_map["vsim_opt"]
self.vlog_opt = opt_map["vlog_opt"]
#if self.vlog_opt == "":
# self.vlog_opt = global_mod.top_module.vlog_opt
#if self.vcom_opt == "":
# self.vcom_opt = global_mod.top_module.vcom_opt
#if self.vsim_opt == "":
# self.vsim_opt = global_mod.top_module.vsim_opt
# if self.vmap_opt == "":
# self.vmap_opt = global_mod.top_module.vmap_opt
self.library = opt_map["library"]
self.include_dirs = []
if opt_map["include_dirs"] != None:
if isinstance(opt_map["include_dirs"], basestring):
self.include_dirs.append(opt_map["include_dirs"])
else:
self.include_dirs.extend(opt_map["include_dirs"])
for dir in self.include_dirs:
if path_mod.is_abs_path(dir):
p.warning(self.path + " contains absolute path to an include directory: " +
dir)
if not os.path.exists(dir):
p.warning(self.path + " has an unexisting include directory: " + dir)
if opt_map["files"] == []:
self.files = SourceFileSet()
else:
opt_map["files"] = self.__make_list(opt_map["files"])
paths = []
for path in opt_map["files"]:
if not path_mod.is_abs_path(path):
path = path_mod.rel2abs(path, self.path)
paths.append(path)
else:
p.warning(path + " is an absolute path. Omitting.")
if not os.path.exists(path):
p.error("File listed in " + self.manifest.path + " doesn't exist: "
+ path +".\nExiting.")
quit()
from srcfile import VerilogFile, VHDLFile
self.files = self.__create_file_list_from_paths(paths=paths);
for f in self.files:
if isinstance(f, VerilogFile):
f.vsim_opt = self.vsim_opt
elif isinstance(f, VHDLFile):
f.vcom_opt = self.vcom_opt
if "svn" in opt_map["modules"]:
opt_map["modules"]["svn"] = self.__make_list(opt_map["modules"]["svn"])
svn_mods = []
for url in opt_map["modules"]["svn"]:
svn_mods.append(self.pool.new_module(parent=self, url=url, source="svn", fetchto=fetchto))
self.svn = svn_mods
else:
self.svn = []
if "git" in opt_map["modules"]:
opt_map["modules"]["git"] = self.__make_list(opt_map["modules"]["git"])
git_mods = []
for url in opt_map["modules"]["git"]:
git_mods.append(self.pool.new_module(parent=self, url=url, source="git", fetchto=fetchto))
self.git = git_mods
else:
self.git = []
self.target = opt_map["target"]
self.action = opt_map["action"]
if opt_map["syn_name"] == None and opt_map["syn_project"] != None:
self.syn_name = opt_map["syn_project"][:-5] #cut out .xise from the end
else:
self.syn_name = opt_map["syn_name"]
self.syn_device = opt_map["syn_device"];
self.syn_grade = opt_map["syn_grade"];
self.syn_package= opt_map["syn_package"];
self.syn_project = opt_map["syn_project"];
self.syn_top = opt_map["syn_top"];
self.isparsed = True
for m in self.submodules():
m.parse_manifest()
def main():
usage = "usage: %prog [options]\n"
usage += "type %prog --help to get help message"
parser = optparse.OptionParser(usage=usage)
parser.add_option("--manifest-help",
action="store_true",
dest="manifest_help",
help="print manifest file variables description")
parser.add_option("--make-vsim",
dest="make_vsim",
action="store_true",
default=None,
help="generate a ModelSim simulation Makefile")
parser.add_option(
"--make-isim",
dest="make_isim",
action="store_true",
default=None,
help="generate a ISE Simulation (ISim) simulation Makefile")
parser.add_option("--make-fetch",
dest="make_fetch",
action="store_true",
default=None,
help="generate a makefile for modules' fetching")
parser.add_option("--make-ise",
dest="make_ise",
action="store_true",
default=None,
help="generate a makefile for local ISE synthesis")
parser.add_option("--make-remote",
dest="make_remote",
action="store_true",
default=None,
help="generate a makefile for remote synthesis")
parser.add_option(
"-f",
"--fetch",
action="store_true",
dest="fetch",
default=None,
help="fetch and/or update remote modules listed in Manifest")
parser.add_option("--clean",
action="store_true",
dest="clean",
default=None,
help="remove all modules fetched for this one")
parser.add_option("--list",
action="store_true",
dest="list",
default=None,
help="List all modules together with their files")
parser.add_option(
"--list-files",
action="store_true",
dest="list_files",
default=None,
help="List all files in a from of a space-separated string")
parser.add_option(
"--merge-cores=name",
default=None,
dest="merge_cores",
help=
"Merges entire synthesizable content of an project into a pair of VHDL/Verilog files"
)
parser.add_option(
"--ise-proj",
action="store_true",
dest="ise_proj",
default=None,
help="create/update an ise project including list of project"
"files")
parser.add_option(
"--quartus-proj",
action="store_true",
dest="quartus_proj",
default=None,
help="create/update a quartus project including list of project"
"files")
parser.add_option("-l",
"--synthesize-locally",
dest="local",
default=None,
action="store_true",
help="perform a local synthesis")
parser.add_option("-r",
"--synthesize-remotelly",
dest="remote",
default=None,
action="store_true",
help="perform a remote synthesis")
parser.add_option("--synth-server",
dest="synth_server",
default=None,
help="use given SERVER for remote synthesis",
metavar="SERVER")
parser.add_option("--synth-user",
dest="synth_user",
default=None,
help="use given USER for remote synthesis",
metavar="USER")
parser.add_option("--force-ise",
dest="force_ise",
default=None,
type=float,
help="Force given ISE version to be used in"
" synthesis,use 0 for current version",
metavar="ISE")
parser.add_option("--py",
dest="arbitrary_code",
default="",
help="add arbitrary code to all manifests' evaluation")
parser.add_option("-v",
"--verbose",
dest="verbose",
action="store_true",
default="false",
help="verbose mode")
parser.add_option("--version",
dest="print_version",
action="store_true",
default="false",
help="print version id of this Hdlmake build")
(options, _) = parser.parse_args()
global_mod.options = options
#HANDLE PROJECT INDEPENDENT OPTIONS
if options.manifest_help == True:
from manifest_parser import ManifestParser
ManifestParser().help()
quit()
if options.print_version == True:
p.print_version()
quit()
# Check later if a simulation tool should have been specified
if options.make_isim == True:
global_mod.sim_tool = "isim"
elif options.make_vsim == True:
global_mod.sim_tool = "vsim"
p.info("Simulation tool: " + str(global_mod.sim_tool))
p.vprint("LoadTopManifest")
pool = ModulePool()
pool.new_module(parent=None, url=os.getcwd(), source="local", fetchto=".")
if pool.get_top_module().manifest == None:
p.rawprint("No manifest found. At least an empty one is needed")
p.rawprint("To see some help, type hdlmake --help")
quit()
global_mod.top_module = pool.get_top_module()
global_mod.global_target = global_mod.top_module.target
ssh = Connection(ssh_user=options.synth_user,
ssh_server=options.synth_server)
from hdlmake_kernel import HdlmakeKernel
kernel = HdlmakeKernel(modules_pool=pool, connection=ssh, options=options)
options_kernel_mapping = {
"fetch": "fetch",
"make_vsim": "generate_vsim_makefile",
"make_isim": "generate_isim_makefile",
"ise_proj": "generate_ise_project",
"quartus_proj": "generate_quartus_project",
"local": "run_local_synthesis",
"remote": "run_remote_synthesis",
"make_fetch": "generate_fetch_makefile",
"make_ise": "generate_ise_makefile",
"make_remote": "generate_remote_synthesis_makefile",
"list": "list_modules",
"clean": "clean_modules",
"merge_cores": "merge_cores"
}
sth_chosen = False
import traceback
for option, function in options_kernel_mapping.items():
try:
is_set = getattr(options, option)
if is_set:
sth_chosen = True
getattr(kernel, function)()
except Exception, unknown_error:
p.echo(
"Oooops! We've got an error. Here is the appropriate info:\n")
p.print_version()
print(unknown_error)
traceback.print_exc()
def main():
"""This is the main funcion, where HDLMake starts.
Here, we make the next processes:
-- parse command
-- check and set the environment
-- prepare the global module containing the heavy common stuff
"""
#
# SET & GET PARSER
#
parser = _get_parser()
#
# PARSE & GET OPTIONS
#
options = _get_options(sys, parser)
global_mod.options = options
# global_mod_assigned!!!
env = Env(options)
global_mod.env = env
numeric_level = getattr(logging, options.log.upper(), None)
if not isinstance(numeric_level, int):
sys.exit('Invalid log level: %s' % options.log)
logging.basicConfig(
format=colored("%(levelname)s", "yellow") +
colored("\t%(filename)s:%(lineno)d: %(funcName)s()\t", "blue") +
"%(message)s",
level=numeric_level)
logging.debug(str(options))
modules_pool = ModulePool()
modules_pool.new_module(parent=None,
url=os.getcwd(),
source=fetch_mod.LOCAL,
fetchto=".",
process_manifest=False)
# Setting top_module as top module of design (ModulePool class)
if modules_pool.get_top_module().manifest is None:
logging.info("No manifest found. At least an empty one is needed")
logging.info("To see some help, type hdlmake --help")
sys.exit("Exiting")
# Setting global variable (global_mod.py)
top_mod = modules_pool.get_top_module()
global_mod.top_module = top_mod
#global_mod.global_target = global_mod.top_module.target
global_mod.mod_pool = modules_pool
modules_pool.process_top_module_manifest()
#
# Load global tool object (global_mod.py)
#
if not top_mod.action:
logging.error(
"`action' manifest variable has to be specified. "
"Otherwise hdlmake doesn't know how to handle the project")
quit()
if top_mod.action == "synthesis":
if not top_mod.syn_tool:
logging.error(
"`syn_tool' manifest variable has to be specified. "
"Otherwise hdlmake doesn't know how to synthesize the project")
quit()
tool_name = top_mod.syn_tool
elif top_mod.action == "simulation":
if not top_mod.sim_tool:
logging.error(
"`sim_tool' manifest variable has to be specified. "
"Otherwise hdlmake doesn't know how to simulate the project")
quit()
tool_name = top_mod.sim_tool
logging.info('import tool module: ' + tool_name)
try:
tool_module = importlib.import_module("tools.%s.%s" %
(tool_name, tool_name))
except Exception as e:
logging.error(e)
quit()
global_mod.tool_module = tool_module
#env.top_module = modules_pool.get_top_module()
env.check_env(verbose=False)
#env.check_env_wrt_manifest(verbose=False)
# #
# EXECUTE THE COMMANDS/ACTIONS HERE #
# #
if options.command == "check-env":
env.check_env(verbose=True)
quit()
if options.command == "check-manifest":
env.check_manifest(modules_pool.get_top_module().manifest,
verbose=True)
quit()
if options.command == "manifest-help":
ManifestParser().print_help()
quit()
if options.command == "auto":
logging.info("Running automatic flow.")
if not top_mod.action:
logging.error(
"`action' manifest variable has to be specified. "
"Otherwise hdlmake doesn't know how to handle the project")
quit()
if top_mod.action == "simulation":
if not top_mod.sim_tool:
logging.error(
"`sim_tool' manifest variable has to be specified. "
"Otherwise hdlmake doesn't know how to simulate the project"
)
quit()
action = [GenerateSimulationMakefile]
elif top_mod.action == "synthesis":
if not top_mod.syn_tool:
logging.error(
"`syn_tool' manifest variable has to be specified. "
"Otherwise hdlmake doesn't know how to synthesize the project"
)
quit()
action = [
GenerateSynthesisProject, GenerateSynthesisMakefile,
GenerateRemoteSynthesisMakefile
]
# TODO: I advocate for removing the remote options -- too much pain, too little gain!!
# Why shouldn't we directly use ssh, screen, scp and rsync on a remote HDLMake deployment??
#elif options.command == "make-simulation":
# action = [ GenerateSimulationMakefile ]
#elif options.command == "make-fetch":
# action = [ GenerateFetchMakefile ]
#elif options.command == "make-ise":
# action = [ GenerateSynthesisMakefile ]
#elif options.command == "make-remote":
# action = [ GenerateRemoteSynthesisMakefile ]
elif options.command == "fetch":
action = [FetchModules]
elif options.command == "clean":
action = [CleanModules]
elif options.command == "list-mods":
action = [ListModules]
elif options.command == "list-files":
action = [ListFiles]
elif options.command == "merge-cores":
action = [MergeCores]
elif options.command == "ise-project":
action = [GenerateSynthesisProject]
elif options.command == "quartus-project":
action = [GenerateSynthesisProject]
elif options.command == "project":
action = [GenerateSynthesisProject]
try:
for command in action:
action_instance = command(modules_pool=modules_pool,
options=options,
env=env)
action_instance.run()
except Exception as e:
import traceback
logging.error(e)
print("Trace:")
traceback.print_exc()
def parse_manifest(self):
if self.isparsed == True or self.isfetched == False:
return
if self.manifest == None:
self.manifest = self.__search_for_manifest()
if self.path == None:
raise RuntimeError()
manifest_parser = ManifestParser()
# For non-top modules
if (self.parent != None):
manifest_parser.add_arbitrary_code(
"target=\"" + str(global_mod.top_module.target) + "\"")
manifest_parser.add_arbitrary_code(
"action=\"" + str(global_mod.top_module.action) + "\"")
# syn_device and sim_tool will be set for non-top modules
manifest_parser.add_arbitrary_code(
"syn_device=\"" + str(global_mod.top_module.syn_device) + "\"")
manifest_parser.add_arbitrary_code("__manifest=\"" + self.path + "\"")
manifest_parser.add_arbitrary_code(global_mod.options.arbitrary_code)
if self.manifest == None:
p.vprint("No manifest found in module " + str(self))
else:
manifest_parser.add_manifest(self.manifest)
p.vprint("Parsing manifest file: " + str(self.manifest))
opt_map = None
try:
opt_map = manifest_parser.parse()
except NameError as ne:
p.echo("Error while parsing {0}:\n{1}: {2}.".format(
self.manifest, type(ne), ne))
quit()
if (opt_map["fetchto"] != None):
fetchto = path_mod.rel2abs(opt_map["fetchto"], self.path)
self.fetchto = fetchto
else:
fetchto = self.fetchto
if self.ise == None:
self.ise = "13.1"
if "local" in opt_map["modules"]:
local_paths = self.__make_list(opt_map["modules"]["local"])
local_mods = []
for path in local_paths:
if path_mod.is_abs_path(path):
p.error("Found an absolute path (" + path +
") in a manifest")
p.rawprint("(" + self.path + ")")
quit()
path = path_mod.rel2abs(path, self.path)
local_mods.append(
self.pool.new_module(parent=self,
url=path,
source="local",
fetchto=fetchto))
self.local = local_mods
else:
self.local = []
self.vmap_opt = opt_map["vmap_opt"]
self.vcom_opt = opt_map["vcom_opt"]
self.vsim_opt = opt_map["vsim_opt"]
self.vlog_opt = opt_map["vlog_opt"]
#if self.vlog_opt == "":
# self.vlog_opt = global_mod.top_module.vlog_opt
#if self.vcom_opt == "":
# self.vcom_opt = global_mod.top_module.vcom_opt
#if self.vsim_opt == "":
# self.vsim_opt = global_mod.top_module.vsim_opt
# if self.vmap_opt == "":
# self.vmap_opt = global_mod.top_module.vmap_opt
self.library = opt_map["library"]
self.include_dirs = []
if opt_map["include_dirs"] != None:
if isinstance(opt_map["include_dirs"], basestring):
self.include_dirs.append(opt_map["include_dirs"])
else:
self.include_dirs.extend(opt_map["include_dirs"])
for dir in self.include_dirs:
if path_mod.is_abs_path(dir):
p.warning(self.path +
" contains absolute path to an include directory: " +
dir)
if not os.path.exists(dir):
p.warning(self.path +
" has an unexisting include directory: " + dir)
if opt_map["files"] == []:
self.files = SourceFileSet()
else:
opt_map["files"] = self.__make_list(opt_map["files"])
paths = []
for path in opt_map["files"]:
if not path_mod.is_abs_path(path):
path = path_mod.rel2abs(path, self.path)
paths.append(path)
else:
p.warning(path + " is an absolute path. Omitting.")
if not os.path.exists(path):
p.error("File listed in " + self.manifest.path +
" doesn't exist: " + path + ".\nExiting.")
quit()
from srcfile import VerilogFile, VHDLFile
self.files = self.__create_file_list_from_paths(paths=paths)
for f in self.files:
if isinstance(f, VerilogFile):
f.vsim_opt = self.vsim_opt
elif isinstance(f, VHDLFile):
f.vcom_opt = self.vcom_opt
if "svn" in opt_map["modules"]:
opt_map["modules"]["svn"] = self.__make_list(
opt_map["modules"]["svn"])
svn_mods = []
for url in opt_map["modules"]["svn"]:
svn_mods.append(
self.pool.new_module(parent=self,
url=url,
source="svn",
fetchto=fetchto))
self.svn = svn_mods
else:
self.svn = []
if "git" in opt_map["modules"]:
opt_map["modules"]["git"] = self.__make_list(
opt_map["modules"]["git"])
git_mods = []
for url in opt_map["modules"]["git"]:
git_mods.append(
self.pool.new_module(parent=self,
url=url,
source="git",
fetchto=fetchto))
self.git = git_mods
else:
self.git = []
self.target = opt_map["target"]
self.action = opt_map["action"]
if opt_map["syn_name"] == None and opt_map["syn_project"] != None:
self.syn_name = opt_map[
"syn_project"][:-5] #cut out .xise from the end
else:
self.syn_name = opt_map["syn_name"]
self.syn_device = opt_map["syn_device"]
self.syn_grade = opt_map["syn_grade"]
self.syn_package = opt_map["syn_package"]
self.syn_project = opt_map["syn_project"]
self.syn_top = opt_map["syn_top"]
self.isparsed = True
for m in self.submodules():
m.parse_manifest()
