def test_rmdir_nonempty(self):
# Remove any prexisting blockstore.
CONFIG.unmap_bs("rootbs")
CONFIG.remove_bs(self.bspath)
# Create the filesystem
bsargs = CONFIG.BSARGS(self.bspath)
self.bs = utp.BlockStore.create(CONFIG.BSTYPE,
"rootbs",
CONFIG.BSSIZE,
bsargs)
self.fs = utp.FileSystem.mkfs(CONFIG.FSTYPE, self.bs, "", "",
CONFIG.UNAME, CONFIG.GNAME, CONFIG.FSARGS)
# Create a directory.
self.fs.fs_mkdir("/top", 0555, CONFIG.UNAME, CONFIG.GNAME)
# Create a file.
self.fs.fs_mknod("/top/inside", 0666, 0, CONFIG.UNAME, CONFIG.GNAME)
# Shouldn't be able to rmdir the directory.
try:
self.fs.fs_rmdir("/top");
assert False
except OSError, ex:
assert ex.errno == ENOTEMPTY
def setup_class(self):
self.bspath = "fs_getattr_01.bs"
# Remove any prexisting blockstore.
CONFIG.unmap_bs("rootbs")
CONFIG.remove_bs(self.bspath)
# Create the filesystem
bsargs = CONFIG.BSARGS(self.bspath)
self.bs = utp.BlockStore.create(CONFIG.BSTYPE,
"rootbs",
CONFIG.BSSIZE,
bsargs)
self.fs = utp.FileSystem.mkfs(CONFIG.FSTYPE, self.bs, "", "",
CONFIG.UNAME, CONFIG.GNAME, CONFIG.FSARGS)
# Make a directory.
self.fs.fs_mkdir("/foo", 0755, CONFIG.UNAME, CONFIG.GNAME)
# Make a file.
self.fs.fs_mknod("/foo/bar", 0666, 0, CONFIG.UNAME, CONFIG.GNAME)
# Now we unmount the filesystem.
self.fs.fs_umount()
self.bs.bs_close()
# Now mount it again.
bsargs = CONFIG.BSARGS(self.bspath)
self.bs = utp.BlockStore.open(CONFIG.BSTYPE, "rootbs", bsargs)
self.fs = utp.FileSystem.mount(CONFIG.FSTYPE, self.bs,
"", "", CONFIG.FSARGS)
def test_rmdir(self):
# Remove any prexisting blockstore.
CONFIG.unmap_bs("rootbs")
CONFIG.remove_bs(self.bspath)
# Create the filesystem
bsargs = CONFIG.BSARGS(self.bspath)
self.bs = utp.BlockStore.create(CONFIG.BSTYPE,
"rootbs",
CONFIG.BSSIZE,
bsargs)
self.fs = utp.FileSystem.mkfs(CONFIG.FSTYPE, self.bs, "", "",
CONFIG.UNAME, CONFIG.GNAME, CONFIG.FSARGS)
# Create a file.
self.fs.fs_mknod("/bar", 0666, 0, CONFIG.UNAME, CONFIG.GNAME)
# Create a directory.
self.fs.fs_mkdir("/foo", 0555, CONFIG.UNAME, CONFIG.GNAME)
# Now we should be able to stat the file.
st = self.fs.fs_getattr("/bar");
# Now we should be able to stat the dir.
st = self.fs.fs_getattr("/foo");
# Shouldn't be able to rmdir the file.
try:
self.fs.fs_rmdir("/bar");
assert False
except OSError, ex:
assert ex.errno == ENOTDIR
def test_unlink(self):
print "Remove any prexisting blockstore."
CONFIG.unmap_bs("rootbs")
CONFIG.remove_bs(self.bspath)
print "Create the filesystem"
bsargs = CONFIG.BSARGS(self.bspath)
self.bs = utp.BlockStore.create(CONFIG.BSTYPE,
"rootbs",
CONFIG.BSSIZE,
bsargs)
self.fs = utp.FileSystem.mkfs(CONFIG.FSTYPE, self.bs, "", "",
CONFIG.UNAME, CONFIG.GNAME, CONFIG.FSARGS)
print "Create a file"
self.fs.fs_mknod("/bar", 0666, 0, CONFIG.UNAME, CONFIG.GNAME)
print "Create a directory"
self.fs.fs_mkdir("/foo", 0555, CONFIG.UNAME, CONFIG.GNAME)
print "Now we should be able to stat the file."
st = self.fs.fs_getattr("/bar");
print "Now we should be able to stat the dir."
st = self.fs.fs_getattr("/foo");
print "Shouldn't be able to unlink the directory."
try:
self.fs.fs_unlink("/foo");
assert False
except OSError, ex:
assert ex.errno == EISDIR
print "Should be able to unlink the file."
def test_symlink(self):
# Remove any prexisting blockstore.
CONFIG.unmap_bs("rootbs")
CONFIG.remove_bs(self.bspath)
# Create the filesystem
bsargs = CONFIG.BSARGS(self.bspath)
self.bs = utp.BlockStore.create(CONFIG.BSTYPE,
"rootbs",
CONFIG.BSSIZE,
bsargs)
self.fs = utp.FileSystem.mkfs(CONFIG.FSTYPE, self.bs, "", "",
CONFIG.UNAME, CONFIG.GNAME, CONFIG.FSARGS)
# Create a directory.
self.fs.fs_mkdir("/foo", 0555, CONFIG.UNAME, CONFIG.GNAME)
# Create a file.
self.fs.fs_mknod("/foo/bar", 0666, 0, CONFIG.UNAME, CONFIG.GNAME)
# Open the file.
self.fs.fs_open("/foo/bar", O_RDWR)
# Write some bytes into the file.
self.fs.fs_write("/foo/bar", buffer("testdata"))
# Check clash w/ existing file.
try:
self.fs.fs_symlink("/blahblah", "/foo/bar");
assert False
except OSError, ex:
assert ex.errno == EEXIST
def setup_class(self):
self.bspath = "bs_refresh_01"
CONFIG.unmap_bs("rootbs")
CONFIG.remove_bs(self.bspath)
self.bs = utp.BlockStore.create(CONFIG.BSTYPE,
"rootbs",
CONFIG.BSSIZE,
CONFIG.BSARGS(self.bspath))
def test_read(self):
# Remove any prexisting blockstore.
CONFIG.unmap_bs("rootbs")
CONFIG.remove_bs(self.bspath)
# Create the filesystem
bsargs = CONFIG.BSARGS(self.bspath)
self.bs = utp.BlockStore.create(CONFIG.BSTYPE, "rootbs", CONFIG.BSSIZE, bsargs)
self.fs = utp.FileSystem.mkfs(CONFIG.FSTYPE, self.bs, "", "", CONFIG.UNAME, CONFIG.GNAME, CONFIG.FSARGS)
# Create a file.
self.fs.fs_mknod("/foo", 0666, 0, CONFIG.UNAME, CONFIG.GNAME)
# Write some data into the file.
self.fs.fs_write("/foo", buffer("testdata"))
# Now we should be able to stat the file.
st = self.fs.fs_getattr("/foo")
assert S_ISREG(st[ST_MODE])
assert st[ST_SIZE] == 8
# Now try and read 4096 bytes.
buf = self.fs.fs_read("/foo", 4096)
assert str(buf) == "testdata"
# Now try and read 4097 bytes.
buf = self.fs.fs_read("/foo", 4097)
assert str(buf) == "testdata"
# Now we unmount the filesystem.
self.fs.fs_umount()
self.bs.bs_close()
# Now mount it again.
bsargs = CONFIG.BSARGS(self.bspath)
self.bs = utp.BlockStore.open(CONFIG.BSTYPE, "rootbs", bsargs)
self.fs = utp.FileSystem.mount(CONFIG.FSTYPE, self.bs, "", "", CONFIG.FSARGS)
# We should be able to stat the same file.
st = self.fs.fs_getattr("/foo")
assert S_ISREG(st[ST_MODE])
assert st[ST_SIZE] == 8
# We should be able to read the data.
self.fs.fs_open("/foo", O_RDONLY)
buf = self.fs.fs_read("/foo", 4096)
assert str(buf) == "testdata"
# We should be able to read the data.
self.fs.fs_open("/foo", O_RDONLY)
buf = self.fs.fs_read("/foo", 4097)
assert str(buf) == "testdata"
# WORKAROUND - py.test doesn't correctly capture the DTOR logging.
self.bs.bs_close()
self.bs = None
self.fs = None
def teardown_class(self):
# WORKAROUND - py.test doesn't correctly capture the DTOR logging.
olvl = utp.FileSystem.loglevel(-1)
self.bs.bs_close()
self.bs = None
self.fs = None
utp.FileSystem.loglevel(olvl)
CONFIG.remove_bs(self.bspath)
def setup_class(self):
self.bspath = "fs_rename_01.bs"
# Remove any prexisting blockstore.
CONFIG.unmap_bs("rootbs")
CONFIG.remove_bs(self.bspath)
# Create the filesystem
bsargs = CONFIG.BSARGS(self.bspath)
self.bs = utp.BlockStore.create(CONFIG.BSTYPE, "rootbs", CONFIG.BSSIZE, bsargs)
self.fs = utp.FileSystem.mkfs(CONFIG.FSTYPE, self.bs, "", "", CONFIG.UNAME, CONFIG.GNAME, CONFIG.FSARGS)
def test_persistence(self):
# Remove any prexisting blockstore.
CONFIG.unmap_bs("rootbs")
CONFIG.remove_bs(self.bspath)
# Create the filesystem
bsargs = CONFIG.BSARGS(self.bspath)
self.bs = utp.BlockStore.create(CONFIG.BSTYPE,
"rootbs",
CONFIG.BSSIZE,
bsargs)
self.fs = utp.FileSystem.mkfs(CONFIG.FSTYPE, self.bs, "", "",
CONFIG.UNAME, CONFIG.GNAME, CONFIG.FSARGS)
# Create a diretory.
self.fs.fs_mkdir("/foo", 0755, CONFIG.UNAME, CONFIG.GNAME)
# Create a file in the directory.
self.fs.fs_mknod("/foo/bar", 0666, 0, CONFIG.UNAME, CONFIG.GNAME)
# Now we should be able to stat the directory.
st = self.fs.fs_getattr("/foo");
assert S_ISDIR(st[ST_MODE])
# Now we should be able to stat the file.
st = self.fs.fs_getattr("/foo/bar");
assert S_ISREG(st[ST_MODE])
# Now we unmount the filesystem.
self.fs.fs_umount()
self.bs.bs_close()
# Now mount it again.
bsargs = CONFIG.BSARGS(self.bspath)
self.bs = utp.BlockStore.open(CONFIG.BSTYPE,
"rootbs",
bsargs)
self.fs = utp.FileSystem.mount(CONFIG.FSTYPE, self.bs,
"", "", CONFIG.FSARGS)
# Now we should be able to stat the directory.
st = self.fs.fs_getattr("/foo");
assert S_ISDIR(st[ST_MODE])
# We should be able to stat the same file.
st = self.fs.fs_getattr("/foo/bar");
assert S_ISREG(st[ST_MODE])
# WORKAROUND - py.test doesn't correctly capture the DTOR logging.
self.bs.bs_close()
self.bs = None
self.fs = None
def detect_token(self, token):
"""
Identifies (classifies, validates) tokens.
:param token: Token
:return: None
:raises: ScannerException
"""
if token in CONFIG.RESERVED_WORDS:
logging.debug('reserved word:{0}'.format(token))
self.add_to_pif(code=CONFIG.CODIFICATION_MAP[token], index=-1)
return
elif CONFIG.match_int_constant(token):
logging.debug('integer constant:{0}'.format(token))
st_index = self.add_to_st(token=token, constant=True)
self.add_to_pif(code=CONFIG.CODIFICATION_MAP['constant'], index=st_index)
return
elif CONFIG.match_char_constant(token):
logging.debug('character constant:{0}'.format(token))
st_index = self.add_to_st(token=token.split('\'')[1], constant=True)
self.add_to_pif(code=CONFIG.CODIFICATION_MAP['constant'], index=st_index)
return
elif token in CONFIG.SEPARATORS:
logging.debug('separator:{0}'.format(token))
self.add_to_pif(code=CONFIG.CODIFICATION_MAP[token], index=-1)
return
elif token in CONFIG.ARITHMETIC_OPERATORS:
logging.debug('arithmetic operator:{0}'.format(token))
self.add_to_pif(code=CONFIG.CODIFICATION_MAP[token], index=-1)
return
elif token in CONFIG.COMPARATORS:
logging.debug('comparator:{0}'.format(token))
self.add_to_pif(code=CONFIG.CODIFICATION_MAP[token], index=-1)
return
elif CONFIG.match_identifier(token):
logging.debug('identifier:{0}'.format(token))
st_index = self.add_to_st(token=token, constant=False)
self.add_to_pif(code=CONFIG.CODIFICATION_MAP['identifier'], index=st_index)
return
# Token couldn't be identified, raise exception
raise ScannerException('Syntax error: {0}'.format(token))
def test_statfs(self):
# Remove any prexisting blockstore.
CONFIG.unmap_bs("rootbs")
CONFIG.remove_bs(self.bspath)
# Create the filesystem
bsargs = CONFIG.BSARGS(self.bspath)
self.bs = utp.BlockStore.create(CONFIG.BSTYPE,
"rootbs",
CONFIG.BSSIZE,
bsargs)
self.fs = utp.FileSystem.mkfs(CONFIG.FSTYPE, self.bs, "", "",
CONFIG.UNAME, CONFIG.GNAME, CONFIG.FSARGS)
BLKSZ = 4 * 1024
BLKSUSED = 2
stvfs = self.fs.fs_statfs()
assert stvfs.f_bsize == BLKSZ
assert stvfs.f_blocks == CONFIG.BSSIZE / BLKSZ
assert stvfs.f_bfree == (CONFIG.BSSIZE / BLKSZ) - BLKSUSED
assert stvfs.f_bavail == (CONFIG.BSSIZE / BLKSZ) - BLKSUSED
# Now we unmount the filesystem.
self.fs.fs_umount()
self.bs.bs_close()
# Now mount it again.
bsargs = CONFIG.BSARGS(self.bspath)
self.bs = utp.BlockStore.open(CONFIG.BSTYPE, "rootbs", bsargs)
self.fs = utp.FileSystem.mount(CONFIG.FSTYPE, self.bs,
"", "", CONFIG.FSARGS)
stvfs = self.fs.fs_statfs()
assert stvfs.f_bsize == BLKSZ
assert stvfs.f_blocks == CONFIG.BSSIZE / BLKSZ
assert stvfs.f_bfree == (CONFIG.BSSIZE / BLKSZ) - BLKSUSED
assert stvfs.f_bavail == (CONFIG.BSSIZE / BLKSZ) - BLKSUSED
# WORKAROUND - py.test doesn't correctly capture the DTOR logging.
self.bs.bs_close()
self.bs = None
self.fs = None
def test_data_single_child(self):
print "test_data_single_child"
bspath1 = "vbs_data_01_c1"
CONFIG.unmap_bs("child1")
CONFIG.remove_bs(bspath1)
self.bs1 = utp.BlockStore.create(CONFIG.BSTYPE,
"child1",
CONFIG.BSSIZE,
CONFIG.BSARGS(bspath1))
CONFIG.unmap_bs("rootbs")
self.vbs = utp.BlockStore.open("VBS",
"rootbs",
("child1",))
print "Put a block of data."
key1 = buffer("key1")
val1 = buffer("val1")
self.vbs.bs_block_put(key1, val1)
print "Retrieve the block."
blk1 = self.vbs.bs_block_get(key1)
assert blk1 == val1
print "Test block that doesn't exist."
key2 = buffer("key2")
py.test.raises(utp.NotFoundError, self.vbs.bs_block_get, key2)
print "Close and reopen everything."
self.vbs.bs_close()
self.bs1.bs_close()
self.bs1 = utp.BlockStore.open(CONFIG.BSTYPE,
"child1",
CONFIG.BSARGS(bspath1))
self.vbs = utp.BlockStore.open("VBS",
"rootbs",
("child1",))
print "Retrieve the block."
blk1 = self.vbs.bs_block_get(key1)
assert blk1 == val1
print "Test block that doesn't exist."
key2 = buffer("key2")
py.test.raises(utp.NotFoundError, self.vbs.bs_block_get, key2)
print "Close for good."
self.vbs.bs_close()
self.vbs = None
self.bs1.bs_close()
CONFIG.remove_bs(bspath1)
def test_follow_on_single_w_miss(self):
CONFIG.unmap_bs("rootbs")
CONFIG.remove_bs(self.bspath)
self.bs = utp.BlockStore.create(CONFIG.BSTYPE,
"rootbs",
CONFIG.BSSIZE,
CONFIG.BSARGS(self.bspath))
# Insert a single SHE.
node = utp.SignedHeadEdge(("fsid", "rootref", 0, time.time() * 1e6, 0, 0))
self.bs.bs_head_insert(node)
# Follow should miss w/ bad key.
miss = (buffer("fsid"), buffer("notref"))
py.test.raises(utp.NotFoundError, self.bs.bs_head_follow, miss)
# Reopen the blockstore.
self.bs.bs_close()
self.bs = utp.BlockStore.open(CONFIG.BSTYPE,
"rootbs",
CONFIG.BSARGS(self.bspath))
# Follow should miss w/ bad key.
miss = (buffer("fsid"), buffer("notref"))
py.test.raises(utp.NotFoundError, self.bs.bs_head_follow, miss)
# Close the blockstore.
self.bs.bs_close()
CONFIG.remove_bs(self.bspath)
def test_follow_on_empty(self):
CONFIG.unmap_bs("rootbs")
CONFIG.remove_bs(self.bspath)
self.bs = utp.BlockStore.create(CONFIG.BSTYPE,
"rootbs",
CONFIG.BSSIZE,
CONFIG.BSARGS(self.bspath))
# Follow should generate NotFound on empty BS.
seed = (buffer(""), buffer(""))
py.test.raises(utp.NotFoundError, self.bs.bs_head_follow, seed)
# Follow should generate NotFound on empty BS, even w/ fsid
seed = (buffer(""), buffer(""))
py.test.raises(utp.NotFoundError, self.bs.bs_head_follow, seed)
# Reopen the blockstore.
self.bs.bs_close()
self.bs = utp.BlockStore.open(CONFIG.BSTYPE,
"rootbs",
CONFIG.BSARGS(self.bspath))
# Follow should generate NotFound on empty BS.
seed = (buffer(""), buffer(""))
py.test.raises(utp.NotFoundError, self.bs.bs_head_follow, seed)
# Follow should generate NotFound on empty BS, even w/ fsid
seed = (buffer("fsid"), buffer(""))
py.test.raises(utp.NotFoundError, self.bs.bs_head_follow, seed)
# Close the blockstore.
self.bs.bs_close()
CONFIG.remove_bs(self.bspath)
def test_furthest_on_single_w_miss(self):
CONFIG.unmap_bs("rootbs")
CONFIG.remove_bs(self.bspath)
self.bs = utp.BlockStore.create(CONFIG.BSTYPE,
"rootbs",
CONFIG.BSSIZE,
CONFIG.BSARGS(self.bspath))
# Insert a single SHE.
node = utp.SignedHeadEdge(("fsid", "rootref", 0, time.time() * 1e6, 0, 0))
self.bs.bs_head_insert(node)
# Furthest should miss w/ bad key.
miss = (buffer("fsid"), buffer("notref"))
shes = self.bs.bs_head_furthest(miss)
assert lenhack(shes) == 0
# Reopen the blockstore.
self.bs.bs_close()
self.bs = utp.BlockStore.open(CONFIG.BSTYPE,
"rootbs",
CONFIG.BSARGS(self.bspath))
# Furthest should miss w/ bad key.
miss = (buffer("fsid"), buffer("notref"))
shes = self.bs.bs_head_furthest(miss)
assert lenhack(shes) == 0
# Close the blockstore.
self.bs.bs_close()
CONFIG.remove_bs(self.bspath)
def test_furthest_on_single(self):
CONFIG.unmap_bs("rootbs")
CONFIG.remove_bs(self.bspath)
self.bs = utp.BlockStore.create(CONFIG.BSTYPE,
"rootbs",
CONFIG.BSSIZE,
CONFIG.BSARGS(self.bspath))
# Insert a single SHE.
node = utp.SignedHeadEdge(("fsid", "rootref", 0, time.time() * 1e6, 0, 0))
self.bs.bs_head_insert(node)
# Furthest should return the node we inserted.
seed = (buffer("fsid"), buffer(""))
shes = self.bs.bs_head_furthest(seed)
assert lenhack(shes) == 1
assert shes[0] == (buffer("fsid"), buffer("rootref"))
# Reopen the blockstore.
self.bs.bs_close()
self.bs = utp.BlockStore.open(CONFIG.BSTYPE,
"rootbs",
CONFIG.BSARGS(self.bspath))
# Furthest should return the node we inserted.
seed = (buffer("fsid"), buffer(""))
shes = self.bs.bs_head_furthest(seed)
assert lenhack(shes) == 1
assert shes[0] == (buffer("fsid"), buffer("rootref"))
# Close the blockstore.
self.bs.bs_close()
CONFIG.remove_bs(self.bspath)
def test_doubleslash(self):
# Remove any prexisting blockstore.
CONFIG.unmap_bs("rootbs")
CONFIG.remove_bs(self.bspath)
# Create the filesystem
bsargs = CONFIG.BSARGS(self.bspath)
self.bs = utp.BlockStore.create(CONFIG.BSTYPE,
"rootbs",
CONFIG.BSSIZE,
bsargs)
self.fs = utp.FileSystem.mkfs(CONFIG.FSTYPE, self.bs, "", "",
CONFIG.UNAME, CONFIG.GNAME, CONFIG.FSARGS)
# Create a directory.
self.fs.fs_mkdir("/mydir", 0555, CONFIG.UNAME, CONFIG.GNAME)
# Create a file in the directory.
self.fs.fs_mknod("/mydir/myfile", 0666, 0, CONFIG.UNAME, CONFIG.GNAME)
# Make sure we can stat the file normally.
st = self.fs.fs_getattr("/mydir/myfile")
assert S_ISREG(st[ST_MODE])
# Make sure we can stat the file with an extra slash.
st = self.fs.fs_getattr("/mydir//myfile")
assert S_ISREG(st[ST_MODE])
# Make sure we can stat the file with a lot of extra slashes.
st = self.fs.fs_getattr("/mydir//////myfile")
assert S_ISREG(st[ST_MODE])
st = self.fs.fs_getattr("/mydir///////myfile")
assert S_ISREG(st[ST_MODE])
# WORKAROUND - py.test doesn't correctly capture the DTOR logging.
self.bs.bs_close()
self.bs = None
self.fs = None
def test_create(self):
CONFIG.unmap_bs("rootbs")
CONFIG.remove_bs(self.bspath)
bs = utp.BlockStore.create(CONFIG.BSTYPE,
"rootbs",
CONFIG.BSSIZE,
CONFIG.BSARGS(self.bspath))
bs.bs_close()
CONFIG.remove_bs(self.bspath)
def test_open_single_child(self):
bspath1 = "vbs_open_01_c1"
CONFIG.unmap_bs("child1")
CONFIG.remove_bs(bspath1)
bs1 = utp.BlockStore.create(CONFIG.BSTYPE,
"child1",
CONFIG.BSSIZE,
CONFIG.BSARGS(bspath1))
CONFIG.unmap_bs("rootbs")
vbs = utp.BlockStore.open("VBS",
"rootbs",
("child1",))
vbs.bs_close()
bs1.bs_close()
CONFIG.remove_bs(bspath1)
def test_bs_stat(self):
# In case it already exists ...
CONFIG.unmap_bs("rootbs")
CONFIG.remove_bs(self.bspath)
# Create the blockstore.
self.bs = utp.BlockStore.create(CONFIG.BSTYPE,
"rootbs",
CONFIG.BSSIZE,
CONFIG.BSARGS(self.bspath))
# Check the blockstore stats.
bss = self.bs.bs_stat();
assert bss.bss_size == CONFIG.BSSIZE
assert bss.bss_free == CONFIG.BSSIZE
# Check that it looks like a tuple too.
assert bss == (CONFIG.BSSIZE, CONFIG.BSSIZE)
# Use some bytes.
k = buffer("samplekey")
v = buffer("sampledata")
self.bs.bs_block_put(k, v)
# Check the blockstore stats.
bss = self.bs.bs_stat();
assert bss.bss_size == CONFIG.BSSIZE
assert bss.bss_free == CONFIG.BSSIZE - 10
# Use some more bytes.
k = buffer("samplekey2")
v = buffer("sampledata2")
self.bs.bs_block_put(k, v)
# Check the blockstore stats.
bss = self.bs.bs_stat();
assert bss.bss_size == CONFIG.BSSIZE
assert bss.bss_free == CONFIG.BSSIZE - 21
# Close and reopen the blockstore.
self.bs.bs_close()
self.bs = utp.BlockStore.open(CONFIG.BSTYPE,
"rootbs",
CONFIG.BSARGS(self.bspath))
# Check the blockstore stats.
bss = self.bs.bs_stat();
assert bss.bss_size == CONFIG.BSSIZE
assert bss.bss_free == CONFIG.BSSIZE - 21
# Close the blockstore.
self.bs.bs_close()
CONFIG.remove_bs(self.bspath)
def test_stat_single_child(self):
bspath1 = "vbs_stat_01_c1"
CONFIG.unmap_bs("child1")
CONFIG.remove_bs(bspath1)
self.bs1 = utp.BlockStore.create(CONFIG.BSTYPE,
"child1",
CONFIG.BSSIZE,
CONFIG.BSARGS(bspath1))
CONFIG.unmap_bs("rootbs")
self.vbs = utp.BlockStore.open("VBS",
"rootbs",
("child1",))
# Check the blockstore stats.
bss = self.vbs.bs_stat();
assert bss.bss_size == CONFIG.BSSIZE
assert bss.bss_free == CONFIG.BSSIZE
# Close and reopen everything.
self.vbs.bs_close()
self.bs1.bs_close()
self.bs1 = utp.BlockStore.open(CONFIG.BSTYPE,
"child1",
CONFIG.BSARGS(bspath1))
self.vbs = utp.BlockStore.open("VBS",
"rootbs",
("child1",))
# Check the blockstore stats again.
bss = self.vbs.bs_stat();
assert bss.bss_size == CONFIG.BSSIZE
assert bss.bss_free == CONFIG.BSSIZE
# Close for good.
self.vbs.bs_close()
self.vbs = None
self.bs1.bs_close()
CONFIG.remove_bs(bspath1)
def test_create_on_prexisting_should_throw_error(self):
CONFIG.unmap_bs("rootbs")
CONFIG.remove_bs(self.bspath)
bs = utp.BlockStore.create(CONFIG.BSTYPE,
"rootbs",
CONFIG.BSSIZE,
CONFIG.BSARGS(self.bspath))
py.test.raises(utp.NotUniqueError,
utp.BlockStore.create,
CONFIG.BSTYPE,
"rootbs",
CONFIG.BSSIZE,
CONFIG.BSARGS(self.bspath))
bs.bs_close()
CONFIG.remove_bs(self.bspath)
def test_should_be_able_to_create_close_and_open_a_block_store(self):
CONFIG.unmap_bs("rootbs")
CONFIG.remove_bs(self.bspath)
bs = utp.BlockStore.create(CONFIG.BSTYPE,
"rootbs",
CONFIG.BSSIZE,
CONFIG.BSARGS(self.bspath))
k = buffer("persistentkey%(random.randrange(999999999))")
v = buffer("persistentvalue")
bs.bs_block_put(k, v)
bs.bs_close()
bs1 = utp.BlockStore.open(CONFIG.BSTYPE,
"rootbs",
CONFIG.BSARGS(self.bspath))
b = bs1.bs_block_get(k)
bs1.bs_close()
CONFIG.remove_bs(self.bspath)
assert b == v
def teardown_class(self):
CONFIG.unmap_bs("rootbs")
CONFIG.remove_bs(self.bspath)
def test_two_nodes(self):
self.bs = utp.BlockStore.create(CONFIG.BSTYPE, "rootbs", CONFIG.BSSIZE, CONFIG.BSARGS(self.bspath))
# Insert the first SHN.
node1 = utp.SignedHeadEdge(("fsid", "node1", 0, time.time() * 1e6, 0, 0))
self.bs.bs_head_insert(node1)
# Insert the second SHN.
node2 = utp.SignedHeadEdge(("fsid", "node2", "node1", time.time() * 1e6, 0, 0))
self.bs.bs_head_insert(node2)
# Furthest w/ empty should return the second node we inserted.
seed0 = (buffer("fsid"), buffer(""))
shes = self.bs.bs_head_furthest(seed0)
assert lenhack(shes) == 1
assert shes[0] == (buffer("fsid"), buffer("node2"))
# Furthest w/ first should return the second node we inserted.
seed1 = (buffer("fsid"), buffer("node1"))
shes = self.bs.bs_head_furthest(seed1)
assert lenhack(shes) == 1
assert shes[0] == (buffer("fsid"), buffer("node2"))
# Furthest w/ last should return the second node we inserted.
seed2 = (buffer("fsid"), buffer("node2"))
shes = self.bs.bs_head_furthest(seed2)
assert lenhack(shes) == 1
assert shes[0] == (buffer("fsid"), buffer("node2"))
# Follow w/ empty should return both nodes.
shes = self.bs.bs_head_follow(seed0)
assert lenhack(shes) == 2
assert str(shes[0].rootref) == "node1"
assert str(shes[1].rootref) == "node2"
# Follow w/ first should return the second node.
shes = self.bs.bs_head_follow(seed1)
assert lenhack(shes) == 1
assert str(shes[0].rootref) == "node2"
# Follow w/ last should return nothing.
shes = self.bs.bs_head_follow(seed2)
assert lenhack(shes) == 0
# Reopen the blockstore.
self.bs.bs_close()
self.bs = utp.BlockStore.open(CONFIG.BSTYPE, "rootbs", CONFIG.BSARGS(self.bspath))
# Furthest w/ empty should return the second node we inserted.
seed0 = (buffer("fsid"), buffer(""))
shes = self.bs.bs_head_furthest(seed0)
assert lenhack(shes) == 1
assert shes[0] == (buffer("fsid"), buffer("node2"))
# Furthest w/ first should return the second node we inserted.
seed1 = (buffer("fsid"), buffer("node1"))
shes = self.bs.bs_head_furthest(seed1)
assert lenhack(shes) == 1
assert shes[0] == (buffer("fsid"), buffer("node2"))
# Furthest w/ last should return the second node we inserted.
seed2 = (buffer("fsid"), buffer("node2"))
shes = self.bs.bs_head_furthest(seed2)
assert lenhack(shes) == 1
assert shes[0] == (buffer("fsid"), buffer("node2"))
# Follow w/ empty should return both nodes.
shes = self.bs.bs_head_follow(seed0)
assert lenhack(shes) == 2
assert str(shes[0].rootref) == "node1"
assert str(shes[1].rootref) == "node2"
# Follow w/ first should return the second node.
shes = self.bs.bs_head_follow(seed1)
assert lenhack(shes) == 1
assert str(shes[0].rootref) == "node2"
# Follow w/ last should return nothing.
shes = self.bs.bs_head_follow(seed2)
assert lenhack(shes) == 0
# Close the blockstore.
self.bs.bs_close()
CONFIG.remove_bs(self.bspath)
def setup_class(self):
self.bspath = "bs_head_02"
CONFIG.unmap_bs("rootbs")
CONFIG.remove_bs(self.bspath)
def test_on_empty(self):
print "test_on_empty starting"
# First child.
bspath1 = "vbs_head_01_c1"
CONFIG.unmap_bs("child1")
CONFIG.remove_bs(bspath1)
self.bs1 = utp.BlockStore.create(CONFIG.BSTYPE,
"child1",
CONFIG.BSSIZE,
CONFIG.BSARGS(bspath1))
# Second child.
bspath2 = "vbs_head_01_c2"
CONFIG.unmap_bs("child2")
CONFIG.remove_bs(bspath2)
self.bs2 = utp.BlockStore.create(CONFIG.BSTYPE,
"child2",
CONFIG.BSSIZE,
CONFIG.BSARGS(bspath2))
# Third child.
bspath3 = "vbs_head_01_c3"
CONFIG.unmap_bs("child3")
CONFIG.remove_bs(bspath3)
self.bs3 = utp.BlockStore.create(CONFIG.BSTYPE,
"child3",
CONFIG.BSSIZE,
CONFIG.BSARGS(bspath3))
# Open the virtual block store.
CONFIG.unmap_bs("rootbs")
self.vbs = utp.BlockStore.open("VBS",
"rootbs",
("child1", "child2", "child3"))
# NOTE - We used to generate NotFound on empty BS, but this
# was changed to return an empty set instead. See comment
# in LameHeadNodeGraph::head_furthest_async for more info.
#
# print "Furthest should generate NotFound on empty BS."
# seed = (buffer(""), buffer(""))
# py.test.raises(utp.NotFoundError, self.vbs.bs_head_furthest, seed)
#
# print "Furthest should generate NotFound on empty BS, even w/ fsid"
# seed = (buffer("fsid"), buffer(""))
# py.test.raises(utp.NotFoundError, self.vbs.bs_head_furthest, seed)
print "Furthest should generate empty on empty BS."
seed = (buffer(""), buffer(""))
shes = self.vbs.bs_head_furthest(seed)
assert lenhack(shes) == 0
print "Furthest should generate empty on empty BS, even w/ fsid"
seed = (buffer("fsid"), buffer(""))
shes = self.vbs.bs_head_furthest(seed)
assert lenhack(shes) == 0
print "Follow should generate NotFound on empty BS."
seed = (buffer(""), buffer(""))
py.test.raises(utp.NotFoundError, self.vbs.bs_head_follow, seed)
print "Follow should generate NotFound on empty BS, even w/ fsid"
seed = (buffer("fsid"), buffer(""))
py.test.raises(utp.NotFoundError, self.vbs.bs_head_follow, seed)
self.vbs.bs_sync()
print "Close for good."
self.vbs.bs_close()
self.vbs = None
self.bs3.bs_close()
self.bs3 = None
self.bs2.bs_close()
self.bs2 = None
self.bs1.bs_close()
self.bs1 = None
CONFIG.remove_bs(bspath3)
CONFIG.remove_bs(bspath2)
CONFIG.remove_bs(bspath1)
print "test_on_empty finished"
def test_on_independent(self):
print "test_on_independent starting"
# First child.
bspath1 = "vbs_head_01_c1"
CONFIG.unmap_bs("child1")
CONFIG.remove_bs(bspath1)
self.bs1 = utp.BlockStore.create(CONFIG.BSTYPE,
"child1",
CONFIG.BSSIZE,
CONFIG.BSARGS(bspath1))
# Insert a node into this child only
node1 = utp.SignedHeadEdge(("fsid", "node1", 0, time.time() * 1e6, 0, 0))
self.bs1.bs_head_insert(node1)
# Second child.
bspath2 = "vbs_head_01_c2"
CONFIG.unmap_bs("child2")
CONFIG.remove_bs(bspath2)
self.bs2 = utp.BlockStore.create(CONFIG.BSTYPE,
"child2",
CONFIG.BSSIZE,
CONFIG.BSARGS(bspath2))
# Insert a node into this child only
node2 = utp.SignedHeadEdge(("fsid", "node2", 0, time.time() * 1e6, 0, 0))
self.bs2.bs_head_insert(node2)
# Third child.
bspath3 = "vbs_head_01_c3"
CONFIG.unmap_bs("child3")
CONFIG.remove_bs(bspath3)
self.bs3 = utp.BlockStore.create(CONFIG.BSTYPE,
"child3",
CONFIG.BSSIZE,
CONFIG.BSARGS(bspath3))
# Insert a node into this child only
node3 = utp.SignedHeadEdge(("fsid", "node3", 0, time.time() * 1e6, 0, 0))
self.bs3.bs_head_insert(node3)
# Open the virtual block store.
CONFIG.unmap_bs("rootbs")
self.vbs = utp.BlockStore.open("VBS",
"rootbs",
("child1", "child2", "child3"))
seed0 = (buffer("fsid"), buffer(""))
# Follow w/ empty should return all edges.
shes = self.vbs.bs_head_follow(seed0)
assert lenhack(shes) == 3
assert sorted((str(shes[0].rootref),
str(shes[1].rootref),
str(shes[2].rootref))) == ["node1", "node2", "node3"]
# Further w/ empty should return all nodes.
shes = self.vbs.bs_head_furthest(seed0)
assert lenhack(shes) == 3
assert sorted(shes) == [ (buffer("fsid"), buffer("node1")),
(buffer("fsid"), buffer("node2")),
(buffer("fsid"), buffer("node3")), ]
self.vbs.bs_sync()
# Close for good.
self.vbs.bs_close()
self.vbs = None
self.bs3.bs_close()
self.bs3 = None
self.bs2.bs_close()
self.bs2 = None
self.bs1.bs_close()
self.bs1 = None
CONFIG.remove_bs(bspath3)
CONFIG.remove_bs(bspath2)
CONFIG.remove_bs(bspath1)
print "test_on_independent finished"
def test_on_single(self):
print "test_on_single starting"
# First child.
bspath1 = "vbs_head_01_c1"
CONFIG.unmap_bs("child1")
CONFIG.remove_bs(bspath1)
self.bs1 = utp.BlockStore.create(CONFIG.BSTYPE,
"child1",
CONFIG.BSSIZE,
CONFIG.BSARGS(bspath1))
# Second child.
bspath2 = "vbs_head_01_c2"
CONFIG.unmap_bs("child2")
CONFIG.remove_bs(bspath2)
self.bs2 = utp.BlockStore.create(CONFIG.BSTYPE,
"child2",
CONFIG.BSSIZE,
CONFIG.BSARGS(bspath2))
# Third child.
bspath3 = "vbs_head_01_c3"
CONFIG.unmap_bs("child3")
CONFIG.remove_bs(bspath3)
self.bs3 = utp.BlockStore.create(CONFIG.BSTYPE,
"child3",
CONFIG.BSSIZE,
CONFIG.BSARGS(bspath3))
# Open the virtual block store.
CONFIG.unmap_bs("rootbs")
self.vbs = utp.BlockStore.open("VBS",
"rootbs",
("child1", "child2", "child3"))
# Insert a single SHN.
node = utp.SignedHeadEdge(("fsid", "rootref", 0, time.time() * 1e6, 0, 0))
self.vbs.bs_head_insert(node)
# Follow should return the node we inserted.
seed = (buffer("fsid"), buffer(""))
shes = self.vbs.bs_head_follow(seed)
assert lenhack(shes) == 1
assert str(shes[0].rootref) == "rootref"
# Furthest should return the node we inserted.
seed = (buffer("fsid"), buffer(""))
shes = self.vbs.bs_head_furthest(seed)
assert lenhack(shes) == 1
assert shes[0] == (buffer("fsid"), buffer("rootref"))
self.vbs.bs_sync()
# Close for good.
self.vbs.bs_close()
self.vbs = None
self.bs3.bs_close()
self.bs3 = None
self.bs2.bs_close()
self.bs2 = None
self.bs1.bs_close()
self.bs1 = None
CONFIG.remove_bs(bspath3)
CONFIG.remove_bs(bspath2)
CONFIG.remove_bs(bspath1)
print "test_on_single finished"
import cv2
import numpy as np
import ip_preprocessing as pre
import line
import CONFIG as cfg
import line_to_block as l2b
import code_generation as code
C = cfg.CONFIG()
org, gray = pre.read_img('input/4.png', (0, 3000)) # cut out partial img
binary = pre.preprocess(gray, 1)
# detect lines
lines_h, lines_v = line.detect_line(binary, C.LINE_MIN_LENGTH_HORIZONTAL,
C.LINE_MIN_LENGTH_VERTICAL,
C.LINE_MAX_THICKNESS,
C.LINE_MAX_CROSS_POINT)
# divide blocks
blocks_h = l2b.divide_blocks_by_lines(lines_h, org.shape[0],
C.BLOCK_MIN_HEIGHT)
# calculate the hierarchy among blocks
hierarchies_h = l2b.hierarchical_blocks(blocks_h)
# generate code according to hierarchy
html = code.gen_html(blocks_h, hierarchies_h)
# save results
broad_line = np.zeros(org.shape, dtype=np.uint8)
broad_block = np.zeros(org.shape, dtype=np.uint8)
line.draw_line(broad_line, lines_h, (0, 255, 0))
#gatypeArray = ['nsga2','weightedga']
seriesList = ['total','spread','resources','makespan']
ylabel={}
ylabel['total']='weighted sum'
ylabel['spread']='CV'
ylabel['resources']='ratio'
ylabel['makespan']='t'
file_path = "./20180410202755"
if not os.path.exists(file_path+'/plots'):
os.makedirs(file_path+'/plots')
cnf = config.CONFIG()
for networkTopology in networkTopologyArray:
for numApps in numAppsArray:
for series in seriesList:
generationSolution = {}
generationPareto = {}
mydataserie={}
for gatype in gatypeArray:
generationSolution[gatype] = list()
generationPareto[gatype] = list()