def type(self, name):
result = None
for pattern, biitype in self:
if pattern == 'NOEXT':
if os.path.splitext(name)[1] == '':
result = biitype
elif fnmatch.fnmatch(name, pattern):
result = biitype
if result is None:
if "CMakeLists.txt" in name:
result = BiiType(CMAKE)
else:
extension = os.path.splitext(name)[1]
result = BiiType.from_extension(extension)
return result
def _process_virtual_config(self, block_name, virtual, virtuals, realizations, biiout):
'''Params:
block_name : BlockName
virtual: dictionary
'''
for virtual_parse_result in virtual.itervalues():
for cell_name in virtual_parse_result.apply:
block_cell_name = block_name + cell_name
try:
old_cell = self.block_holder[cell_name].cell
except KeyError:
old_cell = None
#Build a new virtual cell with the new data, keeping old if existing
new_cell = VirtualCell(block_cell_name, virtual_parse_result.code,
virtual_parse_result.leaves)
if old_cell is None:
new_cell.type = BiiType.from_extension(block_cell_name.extension)
else:
new_cell.type = old_cell.type
virtual_resource = Resource(new_cell, None)
self.block_holder.add_resource(virtual_resource)
self._process_leaves(new_cell, realizations, biiout)
virtuals.add(block_cell_name)
def make_content(brl, lang=BiiType(UNKNOWN), read_file=True):
'''Reads a test file as binary or systext depending on lang
Parameters:
brl: BlockCellName or ID
'''
if isinstance(lang, int):
lang = BiiType(lang)
binary = lang.is_binary()
if isinstance(brl, basestring):
name = '/'.join(brl.split('/')[1:])
parser = parser_factory(lang, brl.split('/')[-1])
if read_file:
blob = Blob(path=testfileutils.file_path(name), is_binary=binary)
else:
blob = Blob("Blob example content", is_binary=binary)
return Content(brl, blob, parser)
def make_published_resource(block, block_cell_name):
if isinstance(block_cell_name, basestring):
moduleCellName = BlockCellName(block_cell_name)
sr = SimpleCell(block_cell_name)
sr.type = BiiType.from_extension(moduleCellName.extension)
sr.ID = UserID(1) + 1 + 2
return sr
def _cells_setup(self, block_name, cells_names, biitype):
resources = []
block_name = BlockName(block_name)
for x in cells_names:
cell = SimpleCell(block_name + x, BiiType(biitype))
resources.append(Resource(cell, Mock()))
return BlockHolder(block_name, resources)
def _process_from_files(self, names):
'''param names: iterable of BlockCellNames. The user will be stripped, and the remaining
will be loaded from the test folder.
Will create a HiveHolder, fill with the data and process it'''
block_holder = mother.get_block_holder(names, BiiType(CPP))
self._process(block_holder)
return block_holder
def make_content(brl, lang=BiiType(UNKNOWN), read_file=True):
'''Reads a test file as binary or systext depending on lang
Parameters:
brl: BlockCellName or ID
'''
if isinstance(lang, int):
lang = BiiType(lang)
binary = lang.is_binary()
if isinstance(brl, basestring):
name = '/'.join(brl.split('/')[1:])
parser = parser_factory(lang, brl.split('/')[-1])
if read_file:
blob = Blob(path=testfileutils.file_path(name), is_binary=binary)
else:
blob = Blob("Blob example content", is_binary=binary)
return Content(brl, blob, parser)
def make_published_resource(block, block_cell_name):
if isinstance(block_cell_name, basestring):
moduleCellName = BlockCellName(block_cell_name)
sr = SimpleCell(block_cell_name)
sr.type = BiiType.from_extension(moduleCellName.extension)
sr.ID = UserID(1) + 1 + 2
return sr
def __init__(self, main):
CPPTarget.__init__(self)
assert isinstance(main, BlockCellName)
assert not BiiType.isCppHeader(main.extension)
self.main = main
self.files.add(main.cell_name)
self.name = _binary_name(main)
self.block_interface = _binary_name(main.block_name) + "_interface"
self.simple_name = _binary_name(main.cell_name)
def __init__(self, main):
CPPTarget.__init__(self)
assert isinstance(main, BlockCellName)
assert not BiiType.isCppHeader(main.extension)
self.main = main
self.files.add(main.cell_name)
self.name = _binary_name(main)
self.block_interface = _binary_name(main.block_name) + "_interface"
self.simple_name = _binary_name(main.cell_name)
def make_simple_cell(block_cell_name, hive=None):
if isinstance(block_cell_name, basestring):
block_cell_name = BlockCellName(block_cell_name)
cell = SimpleCell(block_cell_name)
cell.type = BiiType.from_extension(block_cell_name.extension)
if hive:
changes = ProcessorChanges()
changes.upsert(cell.name, cell, None)
hive.update(changes)
return cell
def deserialize(data):
return SystemID(
data[SystemID.SERIAL_NAME_ID_KEY],
version_id=Version.deserialize(data[SystemID.SERIAL_VERSION_ID_KEY]),
biiType=BiiType.deserialize(data[SystemID.SERIAL_BIITYPE_KEY]),
language_version=Version.deserialize(data[SystemID.SERIAL_LANGUAGE_VERSION_KEY]),
os_info=OSInfo.deserialize(data[SystemID.SERIAL_OS_INFO_KEY]),
tool_info=ToolInfo.deserialize(data[SystemID.SERIAL_TOOL_INFO_KEY]),
path=data[SystemID.SERIAL_PATH_KEY],
)
def make_simple_cell(block_cell_name, hive=None):
if isinstance(block_cell_name, basestring):
block_cell_name = BlockCellName(block_cell_name)
cell = SimpleCell(block_cell_name)
cell.type = BiiType.from_extension(block_cell_name.extension)
if hive:
changes = ProcessorChanges()
changes.upsert(cell.name, cell, None)
hive.update(changes)
return cell
def __init__(self, name=None):
if name and not isinstance(name, BlockCellName):
name = BlockCellName(name)
self.name = name
self.type = BiiType(UNKNOWN)
'''the root property is intended to be always pointing to a CellID'''
self.root = None
'''polimorphic ID, it could be ID published or BlockCellName for Edition'''
self.ID = name # By defaults, the ID (edition is the name)
self.hasMain = False # FIXME: Probably only in SimpleCell
def loads(text):
result = TypeFilter()
for (line_no, line) in enumerate(text.splitlines()):
line = line.strip()
if not line or line.startswith('#'):
continue
try:
tokens = line.split()
biitype = BiiType.from_text(tokens[1]) if len(tokens) > 1 else None
result.append((tokens[0], biitype)) # Pattern, extension
except Exception:
raise BiiException('Wrong type format in line %d: %s' % (line_no, line))
return result
def migrate_block_files(block_name, block_files, biiout):
""" actual migration, takes params
param block_name: BlockName
param block_files: {CellName: Blob load}
param biiout: standar bii output
"""
old_names = ['requirements', 'parents', 'paths', 'dependencies', 'mains']
old_names = OrderedDict([(k, k) for k in old_names])
old_names['parents'] = 'parent'
old_files = [
name for name in old_names if 'bii/%s.bii' % name in block_files
]
if not old_files:
return
deleted = []
if BIICODE_FILE in block_files:
biiout.warn(
"The following old configuration files exist in your %s block\n"
"%s\nMigrating them to your existing %s file, please check it" %
(block_name, ', '.join(old_files), BIICODE_FILE))
current_config = [
"# Biicode configuration file migrated from old config files\n"
]
for file_name, config_name in old_names.iteritems():
current_config.append("[%s]" % config_name)
if file_name in old_files:
config_file = 'bii/%s.bii' % file_name
biiout.warn("Migrating old %s file to new %s configuration file" %
(config_file, BIICODE_FILE))
content = block_files.pop(config_file)
deleted.append(config_file)
for line in content.splitlines():
current_config.append("\t%s" % line)
current_config.append("")
current_config.append('[hooks]\n\n[includes]\n\n[data]')
# This migrate the old data defined in the bii:// way
for name, content in block_files.iteritems():
biitype = BiiType.from_extension(CellName(name).extension)
if biitype == CPP:
_, data, _, _, _ = FileParser().parse(content)
if data:
data_str = ' '.join(d.name for d in data)
current_config.append('\t%s + %s' % (name, data_str))
config_text = '\n'.join(current_config)
return config_text, deleted
def get_block_holder(block_cell_names, biitype=BiiType(UNKNOWN)):
'''Given a working set, a list of resources and a biitype constant
- Adds every resource to the working set and assign given type
- Read test files from disk as binary or systext depending on given type and creates contents
- Fills a wsHolder with given resources and created contents
'''
resources = []
for name in block_cell_names:
cell = SimpleCell(name, biitype)
content = make_content(name, biitype)
resources.append(Resource(cell, content))
block_name = BlockCellName(iter(block_cell_names).next()).block_name
return BlockHolder(block_name, resources)
def compute_mains(settings, resources, output):
"""Return a dictionary {main_name: MainInfo} with all the information
with all necessary resources to build these mains.
"""
cells_with_main = [r.cell for r in resources.itervalues() if r.cell.hasMain]
cells = {r.name: r.cell for r in resources.itervalues()}
mains = {}
for main_cell in cells_with_main:
# filtering out headers
if BiiType.isCppHeader(main_cell.name.extension):
output.info("skipping main function in header file %s" % main_cell.name)
mains[main_cell.name] = _compute_main_info(main_cell, cells, settings)
return mains
def migrate_block_files(block_name, block_files, biiout):
""" actual migration, takes params
param block_name: BlockName
param block_files: {CellName: Blob load}
param biiout: standar bii output
"""
old_names = ['requirements', 'parents', 'paths', 'dependencies', 'mains']
old_names = OrderedDict([(k, k) for k in old_names])
old_names['parents'] = 'parent'
old_files = [name for name in old_names if 'bii/%s.bii' % name in block_files]
if not old_files:
return
deleted = []
if BIICODE_FILE in block_files:
biiout.warn("The following old configuration files exist in your %s block\n"
"%s\nMigrating them to your existing %s file, please check it"
% (block_name, ', '.join(old_files), BIICODE_FILE))
current_config = ["# Biicode configuration file migrated from old config files\n"]
for file_name, config_name in old_names.iteritems():
current_config.append("[%s]" % config_name)
if file_name in old_files:
config_file = 'bii/%s.bii' % file_name
biiout.warn("Migrating old %s file to new %s configuration file"
% (config_file, BIICODE_FILE))
content = block_files.pop(config_file)
deleted.append(config_file)
for line in content.splitlines():
current_config.append("\t%s" % line)
current_config.append("")
current_config.append('[hooks]\n\n[includes]\n\n[data]')
# This migrate the old data defined in the bii:// way
for name, content in block_files.iteritems():
biitype = BiiType.from_extension(CellName(name).extension)
if biitype == CPP:
_, data, _, _, _ = FileParser().parse(content)
if data:
data_str = ' '.join(d.name for d in data)
current_config.append('\t%s + %s' % (name, data_str))
config_text = '\n'.join(current_config)
return config_text, deleted
def generate_dynlib_adapter(self):
db = self.workspace.hivedb
hive = db.read_hive()
mains = hive.mains
graph = hive.block_graph
dep_blocks = set([x.block_name for x in graph.dep.nodes])
dynlibs_files = map(get_dynlib_name, dep_blocks)
for main_name, main in mains.iteritems():
main_deps = main.dep_code
headers = [block_cell_name for block_cell_name in main_deps if BiiType.isCppHeader(block_cell_name.extension)]
headers_contents = self._read_header_contents(headers)
if headers_contents:
py_adapter = self.template.render(headers=''.join(headers_contents),
dynlib_file=dynlibs_files[0])
self._create_py_file(py_adapter, main_name)
def deserialize(self, data):
if data is None:
return None
try:
kls = Deserializer.get_polymorphic_class(data)
r = kls.deserialize(data)
r.name = BlockCellName(data[Cell.SERIAL_NAME_KEY])
r.root = ID.deserialize(data[Cell.SERIAL_ROOT_KEY])
r.ID = self.id_type.deserialize(data[Cell.SERIAL_ID_KEY])
r.type = BiiType(data[Cell.SERIAL_TYPE_KEY])
r.hasMain = data[Cell.SERIAL_MAIN_KEY]
try:
r.dependencies.cell_name = r.name
except AttributeError:
pass
return r
except Exception as e:
tb = traceback.format_exc()
logger.warning(tb)
raise BiiSerializationException(e)
def generate_dynlib_adapter(self):
db = self.workspace.hivedb
hive = db.read_hive()
mains = hive.mains
graph = hive.block_graph
dep_blocks = set([x.block_name for x in graph.dep.nodes])
dynlibs_files = map(get_dynlib_name, dep_blocks)
for main_name, main in mains.iteritems():
main_deps = main.dep_code
headers = [
block_cell_name for block_cell_name in main_deps
if BiiType.isCppHeader(block_cell_name.extension)
]
headers_contents = self._read_header_contents(headers)
if headers_contents:
py_adapter = self.template.render(
headers=''.join(headers_contents),
dynlib_file=dynlibs_files[0])
self._create_py_file(py_adapter, main_name)
def test_set_of_types(self):
self.assertFalse(BiiType.from_extension(".cpp").is_binary())
self.assertTrue(BiiType.from_extension(".wav").is_binary())
self.assertTrue(BiiType.from_extension(".cpp") == CPP)
def make_simple_cell(block_cell_name):
if isinstance(block_cell_name, basestring):
block_cell_name = BlockCellName(block_cell_name)
cell = SimpleCell(block_cell_name)
cell.type = BiiType.from_extension(block_cell_name.extension)
return cell
def test_bii_type(self):
b1 = BiiType(SOUND)
s1 = serialize(b1)
b2 = BiiType.deserialize(s1)
self.assertEqual(b1, b2)
def make_variable_content(brl, lang=BiiType(UNKNOWN), up_limit=2048):
'''Generates a content with variable size, from 0 byte to 2kb. Returns the content'''
size = random.choice(range(up_limit))
blob = Blob(generate_text(size), False)
parser = parser_factory(lang, brl.cell_name)
return Content(brl, blob, parser)
def make_too_big_content(brl, lang=BiiType(UNKNOWN)):
blob_load = load(testfileutils.file_path("limits/largefile.txt"))
blob = Blob(blob_load, is_binary=True)
parser = parser_factory(lang, brl.cell_name)
return Content(brl, blob, parser)
def test_bii_type(self):
b1 = BiiType(SOUND)
s1 = serialize(b1)
b2 = BiiType.deserialize(s1)
self.assertEqual(b1, b2)
def test_bii_types_from_name(self):
self.assertEqual(CPP, BiiType.from_extension(".cpp"))
self.assertEqual(CPP, BiiType.from_extension(".c"))
self.assertEqual(CPP, BiiType.from_extension(".ino"))
self.assertEqual(CPP, BiiType.from_extension(".h"))
self.assertEqual(CPP, BiiType.from_extension(".hh"))
self.assertEqual(CPP, BiiType.from_extension(".cc"))
self.assertEqual(CPP, BiiType.from_extension(".inl"))
self.assertEqual(CPP, BiiType.from_extension(".ipp"))
self.assertEqual(TEXT, BiiType.from_extension(".txt"))
self.assertEqual(XML, BiiType.from_extension(".xml"))
self.assertEqual(HTML, BiiType.from_extension(".html"))
self.assertEqual(HTML, BiiType.from_extension(".htm"))
self.assertEqual(SOUND, BiiType.from_extension(".wav"))
self.assertEqual(IMAGE, BiiType.from_extension(".jpg"))
self.assertEqual(IMAGE, BiiType.from_extension(".gif"))
self.assertEqual(IMAGE, BiiType.from_extension(".png"))
self.assertEqual(IMAGE, BiiType.from_extension(".bmp"))
self.assertEqual(PYTHON, BiiType.from_extension(".py"))
self.assertEqual(TEXT, BiiType.from_extension(".bii"))
self.assertEqual(UNKNOWN, BiiType.from_extension(".unknow"))
def test_set_of_types(self):
self.assertFalse(BiiType.from_extension(".cpp").is_binary())
self.assertTrue(BiiType.from_extension(".wav").is_binary())
self.assertTrue(BiiType.from_extension(".cpp") == CPP)
def generate_python_sys_libs(self):
cell_mock = Mock()
cell_mock.type = BiiType(PYTHON)
validator = getSystemNameValidatorFor(cell_mock).names()
return validator
def __init__(self, name, biitype=UNKNOWN):
super(SimpleCell, self).__init__(name)
self.dependencies = DependencySet()
self.dependencies.cell_name = self.name
self.container = None # CellID or BlockCellName (of the virtual resource)
self.type = BiiType(biitype)
def clean_metadata(self):
self.hasMain = False
self.root = None
self.type = BiiType(UNKNOWN)
def test_bii_types_from_name(self):
self.assertEqual(CPP, BiiType.from_extension(".cpp"))
self.assertEqual(CPP, BiiType.from_extension(".c"))
self.assertEqual(CPP, BiiType.from_extension(".ino"))
self.assertEqual(CPP, BiiType.from_extension(".h"))
self.assertEqual(CPP, BiiType.from_extension(".hh"))
self.assertEqual(CPP, BiiType.from_extension(".cc"))
self.assertEqual(CPP, BiiType.from_extension(".inl"))
self.assertEqual(CPP, BiiType.from_extension(".ipp"))
self.assertEqual(TEXT, BiiType.from_extension(".txt"))
self.assertEqual(XML, BiiType.from_extension(".xml"))
self.assertEqual(HTML, BiiType.from_extension(".html"))
self.assertEqual(HTML, BiiType.from_extension(".htm"))
self.assertEqual(SOUND, BiiType.from_extension(".wav"))
self.assertEqual(IMAGE, BiiType.from_extension(".jpg"))
self.assertEqual(IMAGE, BiiType.from_extension(".gif"))
self.assertEqual(IMAGE, BiiType.from_extension(".png"))
self.assertEqual(IMAGE, BiiType.from_extension(".bmp"))
self.assertEqual(PYTHON, BiiType.from_extension(".py"))
self.assertEqual(TEXT, BiiType.from_extension(".bii"))
self.assertEqual(UNKNOWN, BiiType.from_extension(".unknow"))
def make_simple_cell(block_cell_name):
if isinstance(block_cell_name, basestring):
block_cell_name = BlockCellName(block_cell_name)
cell = SimpleCell(block_cell_name)
cell.type = BiiType.from_extension(block_cell_name.extension)
return cell