def rename(fs: FileSystem, *args):
"""
修改名字 rename name1 name2
:param fs:
:param args:
:return:
"""
if args[0] == '-h':
print("""
修改名字
rename src_name tgt_name
不可跨目录层级使用
可以修改文件/目录的名字
""")
return
name1 = args[0]
name2 = args[1]
pwd_cat = fs.load_pwd_obj() # 当前目录
flag = pwd_cat.is_exist_son_files(name1)
if flag == -1:
print("{} 文件不存在".format(name1))
else:
if name2 in pwd_cat.son_files or name2 in pwd_cat.son_dirs:
print("{} 文件重名".format(name2))
else:
if flag == FILE_TYPE:
pwd_cat.add_new_file(name2, pwd_cat.son_files[name1])
if flag == DIR_TYPE:
pwd_cat.add_new_cat(name2, pwd_cat.son_dirs[name1])
pwd_cat.remove(name1, flag)
fs.write_back(fs.pwd_inode, bytes(pwd_cat))
def app(destination: str, file_size: int, chunk_size: int, debug=False):
"""
Simple program that creates fake a file, splits the file into chunks
and makes lock file.
"""
if debug:
logging.basicConfig(level=logging.DEBUG)
file_name = 'data'
# Generate a fake file
FileGenerator.gen_fake_file(
file_path=destination,
file_name=file_name,
size=file_size,
)
# Generate a lock file
FileGenerator.gen_lock_file(
file_path=destination,
file_name=file_name,
)
# Split the fake file into chunks
FileSystem.split_equal(
file_name=file_name,
source_path=destination,
destination_path=destination,
chunk_size=chunk_size,
)
def tree_x(fs: FileSystem, depth: int, level=0):
"""
生成文件目录结构，具体样式百度
.
└── test
├── css
├── img
│ └── head
└── js
:param fs:
:param depth: 打印的深度
:param level: 已经到达的文件深度
:return:
"""
if depth == 0:
return
pwd_cat = fs.load_pwd_obj() # 获取当前目录
file_list = pwd_cat.file_name_and_types()
for name, flag in file_list:
if flag == DIR_TYPE: # 文件夹
print("│ " * level, end="")
print("├──", color(name, DIR_COLOR_F, DIR_COLOR_B))
flag_x = fs.ch_sig_dir(name, info=False)
if flag_x:
tree_x(fs, depth - 1, level + 1)
fs.ch_sig_dir("..")
if flag == FILE_TYPE: # 文件
print("│ " * level, end="")
print("├──", color(name, FILE_COLOR_F, FILE_COLOR_B))
def clear(fs: FileSystem, *args):
"""
清空终端输出
:param fs:
:return:
"""
fs.clear()
def setUp(self):
self.__testDir = "tempTestDir"
self.fs = FileSystem(self.__testDir)
self.removeDirectory(self.__testDir)
assert(os.path.exists(self.__testDir)==False)
try: os.mkdir(self.__testDir)
except: pass
def touch(fs: FileSystem, name: str):
"""
新建文件
:param fs:
:param name:
:return:
"""
if name == '-h':
print("""
新建文件
touch file_name
不可跨目录层级调用
""")
return
pwd_cat = fs.load_pwd_obj() # 当前目录
flag, info = pwd_cat.check_name(name)
if not flag:
print(info)
return
new_inode = fs.get_new_inode(user_id=fs.current_user_id)
new_inode.target_type = 0 # 文件
pwd_cat.add_new_file(name, new_inode.i_no)
fs.write_back(fs.pwd_inode, bytes(pwd_cat))
new_inode.write_back(fs.fp)
def more(fs: FileSystem, name: str):
"""
展示文件
:param fs:
:param name:
:return:
"""
if name and name == '-h':
print("""
显示文本内容
more file_name
不可跨目录层级调用
完全显示文本内容
""")
return
pwd_cat = fs.load_pwd_obj() # 当前目录
flag = pwd_cat.is_exist_son_files(name)
if flag == -1:
print("{} 文件不存在".format(name))
if flag == DIR_TYPE:
print("{} 是文件夹".format(name))
if flag == FILE_TYPE:
inode_io = pwd_cat.get_file(name)
inode = fs.get_inode(inode_id=inode_io)
flag, text = fs.load_files_block(inode)
if flag:
print(text)
else:
print("cannot open file .: Permission denied")
def setup_db_and_file_system(self, app): # SETUP THE DATABASE AND FILE SYSTEM ENVIRONMENTS
self.database = Database()
self.mysql_database = MySQLDatabase()
self.file_system = FileSystem()
app = self.database.setup_db(app)
app = self.mysql_database.setup_mysql_db(app)
app = self.file_system.setup_file_system(app)
return app
def __init__(self, conf_file):
self.pidfile = "/tmp/hermes.pid"
self.conf_file = conf_file
self.conf_data = self.__read_conf()
# Initiate objects
self.file_system = FileSystem(self.conf_data['archive_path'])
self.mail = SendEmail(self.conf_data['mail'])
def detail(fs: FileSystem, *args):
if args and args[0] == '-h':
print("""
显示系统信息：
系统名
索引块使用情况
数据块使用情况
""")
return
fs.show()
def setUp(self):
self.file_system = FileSystem()
drive = self.file_system.get_root_node()
self.test_folder = self.file_system.create(2, 'test_folder', drive.path)
self.test_folder2 = self.file_system.create(2, 'test_folder2', drive.path)
self.text_file1 = self.file_system.create(3, 'text_file1', self.test_folder.path)
self.text_file2 = self.file_system.create(3, 'text_file2', self.test_folder.path)
self.text_file3 = self.file_system.create(3, 'text_file3', self.test_folder2.path)
self.zip_file1 = self.file_system.create(4, 'zip_file1', self.test_folder2.path)
self.text_file4 = self.file_system.create(3, 'text_file4', self.zip_file1.path)
def test_savetax(self):
form_number = "form_number"
base_directory = constants.BASE_DIR + "/tests_files"
year = 2000
url = "https://www.irs.gov/pub/irs-prior/p1--2017.pdf"
irs_tax = IrsTax(form_number, "", year, url)
file_system = FileSystem(base_directory)
file_system.save_tax(irs_tax)
directory = "{}/{}".format(file_system.base_directory,
irs_tax.form_number)
self.assertTrue(
os.path.isfile(irs_tax.format_and_return_filename(directory)))
def rm(fs: FileSystem, *args):
"""
删除文件
:param fs:
:param args: 参数
:return:
"""
if args[0] == '-h':
print("""
删除文件或目录
rm filename 删除当前文件
rm -r dirname 递归删除目录及其下所有内容
不可跨目录层级调用
""")
return
pwd_cat = fs.load_pwd_obj()
power = False # 删除力度
if args[0] == '-r':
power = True
name = args[1]
else:
name = args[0]
flag = pwd_cat.is_exist_son_files(name)
if flag == -1:
print("文件不存在")
else:
inode_id = -1
if flag == DIR_TYPE:
if not power:
print("无法直接删除目录请使用 rm -r dir_name")
return
inode_id = pwd_cat.get_dir(name)
elif flag == FILE_TYPE:
if not power:
dose = input(f"rm:是否删除一般文件“{name}”[Y/N]:")
if dose.lower() != 'y':
return
inode_id = pwd_cat.get_file(name)
if inode_id != -1:
flag_x = fs.free_up_inode(inode_id)
if flag_x:
# 从当前目录中删除并将当前目录写回
pwd_cat.remove(name, flag)
fs.write_back(fs.pwd_inode, bytes(pwd_cat))
else:
print("cannot delete directory/file .: Permission denied")
def stat(fs: FileSystem, name):
if name == '-h':
print("""
显示文件详情
stat filename/dirname
不可跨目录层级调用
""")
return
pwd_cat = fs.load_pwd_obj()
type_x = pwd_cat.is_exist_son_files(name)
if type_x == -1:
return
inode_id = pwd_cat.get_inode_id(name, type_x)
inode = fs.get_inode(inode_id)
inode.show_detail_info(fs.fp)
def __init__(self):
self.open_files_table = {}
self.process_files_table = {}
self.users = set()
self.system_user = User()
self.users.add(self.system_user)
self.root_file_system = FileSystem()
async def task_builder(self):
"""
Client tasks runner.
"""
tasks = []
# Select files for sending
files = []
for file_ in FileSystem.get_files_with_size(source_path=self.source):
files.append({
'name': file_['name'],
'size': file_['size'],
})
meta = {'chunks': files}
for file in files:
task = asyncio.ensure_future(
self.send_file(file_path=file['name'], **meta))
tasks.append(task)
contents = await asyncio.gather(*tasks)
return contents
def testInit(self):
from file_system import FileSystem
from system import System, system
fileSystem = FileSystem()
print IceCommon.openOfficeName
IceCommon.setup(system, fileSystem)
def pwd(fs: FileSystem, *args):
"""
打印当前目录的完整路径
:param fs:
:return:
"""
print(fs.pwd())
def __init__(self, addr, port, network=None):
Base.__init__(self, DFS(addr, port))
self.fileSystem_ = FileSystem(self.dfs_)
if not network:
network = Network(self.dfs_, self.fileSystem_)
self.network_ = network
def mv(fs: FileSystem, *args):
"""
剪切文件或目录 mv home/ywq/demo home/caohang
:param fs:
:param args:
:return:
"""
if args[0] == '-h':
print("""
剪切文件或目录：
mv xx/xx/src xx/xx/tgt
可以跨目录层级剪切目录或文件到其他目录
""")
return
path_src = args[0] # home/ywq/demo
path_tgt = args[1] # home/caohang
name = path_src.split('/')[-1] # 取出文件名
cnt1 = len(path_src.split('/')) - 1 # 第一个目录的深度
cnt2 = len(path_tgt.split('/')) # 第二个目录的深度
inode_io = 0
# 删掉原来的目录
cd(fs, '/'.join(path_src.split('/')[:-1]))
pwd_cat = fs.load_pwd_obj() # 当前目录块
flag = pwd_cat.is_exist_son_files(name)
if flag == -1:
print("{} 文件不存在".format(name))
cd(fs, '/'.join(['..'] * cnt1))
return
else:
if flag == FILE_TYPE:
inode_io = pwd_cat.son_files[name]
if flag == DIR_TYPE:
inode_io = pwd_cat.son_dirs[name]
pwd_cat.remove(name, flag)
fs.write_back(fs.pwd_inode, bytes(pwd_cat))
cd(fs, '/'.join(['..'] * cnt1))
# 增加到现在的目录下
cd(fs, '/'.join(path_tgt.split('/')))
pwd_cat_new = fs.load_pwd_obj() # 要增加的目录块
if flag == FILE_TYPE:
pwd_cat_new.add_new_file(name, inode_io)
if flag == DIR_TYPE:
inode = fs.get_inode(inode_io)
new_cat = inode.get_target_obj(fs.fp)
new_cat.parent_inode_id = fs.pwd_inode.i_no
pwd_cat_new.add_new_cat(name, inode_io)
fs.write_back(inode, bytes(new_cat))
fs.write_back(fs.pwd_inode, bytes(pwd_cat_new))
cd(fs, '/'.join(['..'] * cnt2))
def mkdir(fs: FileSystem, name: str):
"""
新建目录
新建索引对象-->新建目录对象-->将前两者作为键值对加入当前索引对应的目录-->申请空间存放新建的目录
:param fs:
:param name: 文件夹名称
:return:
"""
if name == '-h':
print("""
新建文件夹
mkdir dirname
只可以在当前目录的新建不可跳级新建
获得当前目录对象pwd_obj
检查命名冲突，pwd_obj.check_name(name)
获取新的inode对象
将新建文件夹的名字和inode号作为键值对写回pwd_obj
写回新建的目录对象new_obj，并且将其开辟的新的地址块号添加到对应的inode对象中
写回新的inode对象
""")
return
pwd_cat = fs.load_pwd_obj() # 当前目录
flag, info = pwd_cat.check_name(name)
if not flag:
print(info)
return
new_inode = fs.get_new_inode(user_id=fs.current_user_id)
pwd_cat.add_new_cat(name=name, inode_id=new_inode.i_no)
new_cat = fs.get_new_cat(name=name, parent_inode_id=fs.pwd_inode.i_no)
fs.write_back(new_inode, bytes(new_cat)) # 写回新建目录
fs.write_back(fs.pwd_inode, bytes(pwd_cat)) # 写回当前目录，因为新建的目录挂载当前目录也被修改了
new_inode.write_back(fs.fp)
def vim(fs: FileSystem, name: str):
"""
向文件中输入内容，或者是修改文件
:param fs:
:param name:
:return:
"""
if name == '-h':
print("""
编辑文件中的内容
vim file_name
不可跨层级调用
命令使用后会打开一个线程单独运行文本编辑器
""")
return
pwd_cat = fs.load_pwd_obj() # 当前目录
flag = pwd_cat.is_exist_son_files(name)
if flag == -1:
touch(fs, name)
vim(fs, name)
return
if flag == DIR_TYPE:
print("{} 是文件夹".format(name))
if flag == FILE_TYPE:
inode_io = pwd_cat.get_file(name)
inode = fs.get_inode(inode_id=inode_io)
if check_auth(inode.user_id, fs.current_user_id):
def func():
flag_x, s = fs.load_files_block(inode)
if not s:
s = ''
te = TextEdit(s)
te.run()
s = te.s
if len(pickle.dumps(s)) <= (13 * BLOCK_SIZE - 100):
fs.write_back(inode, pickle.dumps(s))
inode.write_back(fs.fp)
else:
print("out of size")
vim_thread = Thread(target=func)
vim_thread.start()
else:
print("cannot edit file .: Permission denied")
def testHasTransparency2(self):
testFormula = "testData/1_a.gif"
import Image
from cStringIO import StringIO
#use image directly
im = Image.open(testFormula)
#Make it transparent first before convert to L (grayscale)
s = StringIO()
newSize = (212, 45)
im.save(s, "GIF")
data = s.getvalue()
iceImage = IceImage(data, ".gif")
data = iceImage.bwResizeImage(newSize)
fs = FileSystem()
fs.writeFile("testData/iceImage1.gif", data)
def cd(fs: FileSystem, *args: str):
"""
切换目录,可以多级目录切换
:param fs:
:param args: 切换到的目录名
:return: True/False 表示切换是否完全成功
"""
if args[0] == '-h':
print("""
切换目录
dirName:要切换到的目标目录名
cd hello 切换一级目录
cd hello\hello 切换多级目录
cd ..\.. 切换上层目录
cd ~ 切换到根目录
""")
else:
fs.chdir(args[0])
def testHasTransparency2(self):
testFormula = "testData/1_a.gif"
import Image
from cStringIO import StringIO
#use image directly
im = Image.open(testFormula)
#Make it transparent first before convert to L (grayscale)
s = StringIO()
newSize = (212, 45)
im.save(s, "GIF")
data = s.getvalue()
iceImage = IceImage(data, ".gif")
data = iceImage.bwResizeImage(newSize)
fs = FileSystem()
fs.writeFile("testData/iceImage1.gif", data)
def test_find(self):
with FileSystem(":memory:", self.matryoshka) as fs:
with File.create(fs, Path("folder1", "file"), self.example_file):
pass
with File.create(fs, Path("folder2", "file"), self.example_file):
pass
files = File.find(fs, Path("folder*", "file"))
self.assertEqual(len(files), 2)
async def _handle_echo(self, reader, writer):
"""
Callback.
:param reader: a StreamReader object
:param writer: a StreamWriter object
"""
peer_name = writer.get_extra_info('peername')
logging.info(f'Accepted connection from {peer_name}')
received_data = await reader.read()
# parse information from the received data
data_frame = bson.loads(received_data)
FileSystem.chunk_merging(data_frame=data_frame,
destination_path=self.destination)
writer.close()
def running_pfs(fs: FileSystem):
while True:
bar(fs.current_user_name, fs.get_current_path_name())
cmd = input().split()
if cmd[0] == 'exit':
break
try:
func = getattr(commands, cmd[0])
func(fs, *cmd[1:])
except AttributeError:
print("\n命令不支持\n")
def cp(fs: FileSystem, *args):
"""
复制文件/目录参数-r
:param fs:
:param args:
:return:
"""
if args[0] == '-h':
print("""
复制文件
cp xx/xx/src_filename xx/xx/tgt_dir
复制文件到其他目录
支持跨目录层级调用
仅支持复制文件
""")
return
if args[0] == '-r':
path_src = args[1]
path_tgt = args[2]
else:
path_src = args[0]
path_tgt = args[1]
name = path_src.split('/')[-1] # 取出文件名
cnt1 = len(path_src.split('/')) - 1 # 第一个目录的深度
cnt2 = len(path_tgt.split('/')) # 第二个目录的深度
text_copy = "" # 文件内容
cd(fs, '/'.join(path_src.split('/')[:-1]))
pwd_cat = fs.load_pwd_obj()
flag = pwd_cat.is_exist_son_files(name)
if flag == -1:
print("{} 文件不存在".format(name))
cd(fs, '/'.join(['..'] * cnt1))
return
else:
if flag == FILE_TYPE:
inode_io = pwd_cat.son_files[name]
inode = fs.get_inode(inode_id=inode_io)
flag2, text = fs.load_files_block(inode)
if flag2:
text_copy = text # 传递内容
if flag == DIR_TYPE:
print("不能复制文件夹")
cd(fs, '/'.join(['..'] * cnt1))
return
cd(fs, '/'.join(['..'] * cnt1))
# 增加到现在的目录下
cd(fs, '/'.join(path_tgt.split('/')))
touch(fs, name)
pwd_cat_new = fs.load_pwd_obj()
new_inode_io = pwd_cat_new.son_files[name]
new_inode = fs.get_inode(inode_id=new_inode_io)
fs.write_back(new_inode, pickle.dumps(text_copy))
new_inode.write_back(fs.fp)
cd(fs, '/'.join(['..'] * cnt2))
def ll(fs: FileSystem, *args):
"""
打印当前目录下的具体信息
:param fs:
:return:
"""
if args and args[0] == '-h':
print("""
打印当前目录下的全部文件及详细信息
ll
绿色表示目录
白色表示普通文件
打印内容分别是：
1.文件数（普通文件是1，目录是其下一级所包含的文件数目）
2.上次修改日期（2020-06-19 00:22:42）
3.文件大小 （512B 整数倍）
4.拥有者的ID
5.文件/目录 名
""")
return
fs.show_lls_info()
def test_right_format_and_filename(self):
form_number = "form_number"
directory = "/directory"
year = 2000
irs_tax = IrsTax(form_number, "", year, "#")
pattern = f"{directory}/{form_number} - {year}.pdf"
file_system = FileSystem(directory)
file_path = irs_tax.format_and_return_filename(
file_system.base_directory)
self.assertEqual(file_path, pattern)
class ooo2xhtmlTests(TestCase):
def setUp(self):
self.__fs = FileSystem(".")
def tearDown(self):
pass
def testName(self):
o2xhtml = Ooo2xhtml(iceContext)
if self.__fs.isFile(testOdtData)==False:
msg = "testData '%s' not found!" % testOdtData
print msg
print self.__fs.absolutePath(testOdtData)
self.assertFail(msg)
return
contentXmlStr = self.__fs.readFromZipFile(testOdtData, "content.xml")
o2xhtml.process(contentXmlStr)
html = str(o2xhtml.shtml)
print len(html)
for k, i in o2xhtml.meta.iteritems():
print "%s - '%s'" % (k, str(i))
print
def testInit(self):
fs = FileSystem("tempTest")
path = "."
sClient = MockSimpleClient(path=None, fs=fs, create=True)
print "rev#", sClient.rootRevisionNumber
fs.writeFile("one.txt", "testing one")
sClient.addFile("one.txt")
print sClient.list()
sClient.commit(paths=["one.txt"], logMessage="Commit Test")
print sClient.list()
fs.writeFile("one.txt", "testing one modified")
print sClient.list()
print "---"
sClient = MockSimpleClient(path=None, fs=fs)
print sClient.list()
sClient.commit(paths=["one.txt"], logMessage="2nd commit")
print sClient.list()
print sClient.logFile("one.txt")
fs.writeFile("one.txt", "testing one modified2")
print sClient.list()
sClient.revertFile("one.txt")
print sClient.list()
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Soutoftware
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
#
import os
from file_system import FileSystem
from system import System
fs = FileSystem()
system = System()
def tts_available(engine = "native"):
nativeAvailable = False
if engine == "native":
nativeAvailable = macosx_say_available() or windows_say_available()
# fallback to other if possible
if nativeAvailable:
return True
else:
if other_available():
return True
else:
return False
class FileSystemTests(IceCommon.TestCase):
def removeDirectory(self, dir):
if os.path.exists(dir):
files = os.listdir(dir)
for file in files:
file = dir + "/" + file
if os.path.isdir(file):
self.removeDirectory(file)
else:
os.remove(file)
os.rmdir(dir)
def setUp(self):
self.__testDir = "tempTestDir"
self.fs = FileSystem(self.__testDir)
self.removeDirectory(self.__testDir)
assert(os.path.exists(self.__testDir)==False)
try: os.mkdir(self.__testDir)
except: pass
def tearDown(self):
self.removeDirectory(self.__testDir)
pass
def testCurrentWorkingDirectory(self):
testDir = os.path.abspath(self.__testDir).replace("\\", "/")
self.assertEqual(self.fs.currentWorkingDirectory, testDir)
# absolutePath(path)
def testAbsolutePath(self):
# on windows root will equal "c:/"
root = os.path.abspath("/").replace("\\", "/")
#print "root='%s'" % root
self.assertEqual(self.fs.absolutePath("."), self.fs.currentWorkingDirectory)
self.assertEqual(self.fs.absolutePath(".."), os.path.abspath(".").replace("\\", "/"))
self.assertEqual(self.fs.absolutePath("/"), root)
self.assertEqual(self.fs.absolutePath("/test/one"), root + "test/one")
self.assertEqual(self.fs.absolutePath("/test\\one"), root + "test/one")
# join(*args)
def testJoin(self):
self.assertEqual(self.fs.join("one", "two", "3"), "one/two/3")
self.assertEqual(self.fs.join("one/", "two/"), "one/two/")
self.assertEqual(self.fs.join("one", "/two"), "/two")
self.assertEqual(self.fs.join(".", ".."), "./..")
# split(path)
def testSplit(self):
self.assertEqual(self.fs.split("one/two/three"), ("one/two", "three"))
self.assertEqual(self.fs.split("one"), ("", "one"))
self.assertEqual(self.fs.split("../."), ("..", "."))
self.assertEqual(self.fs.split(""), ("", ""))
# splitExt(path)
def testSplitExt(self):
self.assertEqual(self.fs.splitExt("one/two"), ("one/two", ""))
self.assertEqual(self.fs.splitExt("one.two.ext"), ("one.two", ".ext"))
self.assertEqual(self.fs.splitExt("one/two.ext/three"), ("one/two.ext/three", ""))
self.assertEqual(self.fs.splitExt(".ext"), ("", ".ext"))
self.assertEqual(self.fs.splitExt(""), ("", ""))
# splitPathFileExt(path)\
def testSplitPathFileExt(self):
self.assertEqual(self.fs.splitPathFileExt("/one/two"), ("/one", "two", ""))
self.assertEqual(self.fs.splitPathFileExt("one.two.ext"), ("", "one.two", ".ext"))
self.assertEqual(self.fs.splitPathFileExt("one/two.ext/three"), ("one/two.ext", "three", ""))
self.assertEqual(self.fs.splitPathFileExt(".ext"), ("", "", ".ext"))
self.assertEqual(self.fs.splitPathFileExt(""), ("", "", ""))
# exists(path)
def testExists(self):
testFile = "testExistsFile"
testDir = "testExistsDir"
self.assertFalse(self.fs.exists(testFile))
self.assertFalse(self.fs.exists(testDir))
f = open(os.path.join(self.__testDir, testFile), "wb")
f.write("Testing")
f.close()
os.mkdir(os.path.join(self.__testDir, testDir))
self.assertTrue(self.fs.exists(testFile))
self.assertTrue(self.fs.exists(testDir))
# isFile(path)
def testIsFile(self):
testFile = "testIsFile"
testDir = "testIsFileDir"
self.assertFalse(self.fs.isFile(testFile))
self.assertFalse(self.fs.isFile(testDir))
f = open(os.path.join(self.__testDir, testFile), "wb")
f.write("Testing")
f.close()
os.mkdir(os.path.join(self.__testDir, testDir))
self.assertTrue(self.fs.isFile(testFile))
self.assertFalse(self.fs.isFile(testDir))
# isDirectory(path)
def testIsDirectory(self):
testFile = "testIsDirFile"
testDir = "testIsDir"
self.assertFalse(self.fs.isDirectory(testFile))
self.assertFalse(self.fs.isDirectory(testDir))
f = open(os.path.join(self.__testDir, testFile), "wb")
f.write("Testing")
f.close()
os.mkdir(os.path.join(self.__testDir, testDir))
self.assertFalse(self.fs.isDirectory(testFile))
self.assertTrue(self.fs.isDirectory(testDir))
# makeDirectory(path)
def testMakeDirectory(self):
dir1 = "makeDir"
dir2 = "makeDir2/subDir/subSubDir"
self.assertFalse(self.fs.isDirectory(dir1))
self.assertFalse(self.fs.isDirectory(dir2))
self.fs.makeDirectory(dir1)
self.fs.makeDirectory(dir2)
self.assertTrue(self.fs.isDirectory(dir1))
self.assertTrue(self.fs.isDirectory(dir2))
# delete(path)
def testDelete(self):
file = "deleteFile"
dir = "dir/subDir/subSubDir"
self.fs.writeFile(file, "testing")
self.fs.writeFile(self.fs.join(dir, "file"), "test")
self.assertTrue(self.fs.isDirectory(dir))
self.assertTrue(self.fs.isFile(file))
self.fs.delete(file)
self.fs.delete(dir)
self.assertFalse(self.fs.isDirectory(dir))
self.assertFalse(self.fs.isFile(file))
# copy(fromPath, toPath)
def testCopy(self):
# Simple file copy test
fromFile = "copyFile"
toFile = "copyToFile"
toDirFile = "dir/subDir/copyToFile"
self.fs.writeFile(fromFile, "test data")
self.assertFalse(self.fs.exists(toFile))
self.fs.copy(fromFile, toFile)
self.fs.copy(fromFile, toDirFile)
self.assertTrue(self.fs.exists(toFile))
self.assertEqual(self.fs.readFile(toFile), self.fs.readFile(fromFile))
self.assertEqual(self.fs.readFile(toDirFile), self.fs.readFile(fromFile))
# Directory, subDirectory and files copy test
dir = "copyDir"
toDir = "copyToDir"
self.fs.writeFile("copyDir/subDir/file", "testing")
self.assertFalse(self.fs.exists(toDir))
self.fs.copy(dir, toDir)
self.assertEqual(self.fs.readFile("copyDir/subDir/file"), self.fs.readFile("copyToDir/subDir/file"))
# move(fromPath, toPath)
def testMove(self):
# move file test
file = "file"
toFile = "toFile"
toDirFile = "dir/subDir/toFile"
self.fs.writeFile(file, "testing")
self.fs.move(file, toFile)
self.assertFalse(self.fs.isFile(file))
self.assertTrue(self.fs.isFile(toFile))
self.fs.move(toFile, toDirFile)
self.assertFalse(self.fs.isFile(toFile))
self.assertTrue(self.fs.isFile(toDirFile))
# move dir test
dir = "dirToMove"
toDir = "movedDir"
self.fs.writeFile(self.fs.join(dir, "file"), "testing")
self.fs.move(dir, toDir)
self.assertFalse(self.fs.isDirectory(dir))
self.assertTrue(self.fs.isDirectory(toDir))
self.assertEqual(self.fs.readFile(self.fs.join(toDir, "file")), "testing")
# walk(path)
def testWalk(self):
tree = ["./one.txt", "./test/two.txt", "./test/subDir/three.txt"]
tree.sort()
for f in tree:
self.fs.writeFile(f, "testData")
rFiles = []
for path, dirs, files in self.fs.walk("."):
self.assertTrue(path.endswith("/"))
for file in files:
rFiles.append(path + file)
rFiles.sort()
self.assertEqual(tree, rFiles)
# walker(path, func)
def testWalker(self):
tree = ["./one.txt", "./test/two.txt", "./test/subDir/three.txt"]
tree.sort()
for f in tree:
self.fs.writeFile(f, "testData")
def callback(path, dirs, files):
self.assertTrue(path.endswith("/"))
for file in files:
rFiles.append(path + file)
rFiles = []
self.fs.walker(".", callback)
rFiles.sort()
self.assertEqual(tree, rFiles)
# createTempDirectory
def testCreateTempDirectory(self):
tempDir = self.fs.createTempDirectory()
self.assertNotEqual(tempDir, None)
self.assertTrue(len(str(tempDir))>1)
self.assertTrue(tempDir.isTempDirectory)
self.assertTrue(os.path.isdir(str(tempDir)))
tempDir.delete()
#print str(tempDir)
# writeFile(path, data)
def testWriteFile(self):
file = "writeFile"
data = "Testing\nThe End.\n"
self.assertFalse(self.fs.isFile(file))
self.fs.writeFile(file, data)
self.assertTrue(self.fs.isFile(file))
self.assertEqual(self.fs.readFile(file), data)
# readFile(path)
def testReadFile(self):
file = "writeFile"
data = "Testing\nThe End.\n"
self.assertFalse(self.fs.isFile(file))
self.fs.writeFile(file, data)
self.assertTrue(self.fs.isFile(file))
self.assertEqual(self.fs.readFile(file), data)
# zip(toZipFile, path="./")
def testZip(self, data="TestData\n", files=None):
pathToZip = "zipDir"
if files is None:
files = ["one.txt", "two.txt", "dir/three.txt", "dir2/subDir/four.txt"]
zipFile = "test.zip"
for file in files:
path = self.fs.join(pathToZip, file)
self.fs.writeFile(path, data)
self.fs.zip(zipFile, pathToZip)
self.assertTrue(self.fs.isFile(zipFile))
# unzipToTempDirectory(zipFile)
def testUnzipToTempDirectory(self):
self.testZip()
tempDir = self.fs.unzipToTempDirectory("test.zip")
self.assertTrue(tempDir.isFile("one.txt"))
tempDir.delete()
# unzipToDirectory(zipFile, path)
def testUnzipToDirectory(self):
data="Testing\n123\n"
self.testZip(data=data)
self.fs.unzipToDirectory("test.zip", "temp")
self.assertTrue(self.fs.isDirectory("temp"))
self.assertEqual(self.fs.readFile("temp/dir2/subDir/four.txt"), data)
def testZipList(self):
files = ["one.txt", "two.txt", "dir/three.txt", "dir2/subDir/four.txt"]
self.testZip(files=files)
list = self.fs.zipList("test.zip")
files.sort()
list.sort()
self.assertEqual(list, files)
# readFromZipFile(zipFile, filename)
def testReadFromZipFile(self):
data="Testing\n123\n"
self.testZip(data=data)
result = self.fs.readFromZipFile("test.zip", "one.txt")
self.assertEqual(data, result)
result = self.fs.readFromZipFile("test.zip", "dir2/subDir/four.txt")
self.assertEqual(data, result)
# addToZipFile(zipFile, filename, data)
def testAddToZipFile(self):
files = ["one.txt", "two.txt", "dir/three.txt", "dir2/subDir/four.txt"]
self.testZip(files=files)
# Test adding a new File
self.fs.addToZipFile("test.zip", "newFile.txt", "testing")
files.append("newFile.txt")
files.sort()
rList = self.fs.zipList("test.zip")
rList.sort()
self.assertEqual(rList, files)
# Test replacing a file
self.fs.addToZipFile("test.zip", "one.txt", "replaceData")
rList = self.fs.zipList("test.zip")
rList.sort()
self.assertEqual(rList, files)
self.assertEqual(self.fs.readFromZipFile("test.zip", "one.txt"), "replaceData")
# list(path=".")
def testList(self):
dir = "listDir"
files = ["one", "two", "three"]
dirs = ["four", "five"]
for f in files:
self.fs.writeFile(self.fs.join(dir, f), "testing")
for d in dirs:
self.fs.makeDirectory(self.fs.join(dir, d))
list = self.fs.list(dir)
list.sort()
all = files + dirs
all.sort()
self.assertEqual(list, all)
# listFiles(path=".")
def testListFiles(self):
dir = "listDir"
files = ["one", "two", "three"]
dirs = ["four", "five"]
files.sort()
dirs.sort()
for f in files:
self.fs.writeFile(self.fs.join(dir, f), "testing")
for d in dirs:
self.fs.makeDirectory(self.fs.join(dir, d))
list = self.fs.listFiles(dir)
list.sort()
self.assertEqual(list, files)
# listDirectories(path=".")
def testDirectories(self):
dir = "listDir"
files = ["one", "two", "three"]
dirs = ["four", "five"]
for f in files:
self.fs.writeFile(self.fs.join(dir, f), "testing")
for d in dirs:
self.fs.makeDirectory(self.fs.join(dir, d))
list = self.fs.listDirectories(dir)
list.sort()
dirs.sort()
self.assertEqual(list, dirs)
# listDirsFiles(path=".")
def testListDirsFiles(self):
dir = "listDir"
files = ["one", "two", "three"]
dirs = ["four", "five"]
files.sort()
dirs.sort()
for f in files:
self.fs.writeFile(self.fs.join(dir, f), "testing")
for d in dirs:
self.fs.makeDirectory(self.fs.join(dir, d))
rdirs, rfiles = self.fs.listDirsFiles(dir)
rdirs.sort()
rfiles.sort()
self.assertEqual(rdirs, dirs)
self.assertEqual(rfiles, files)
class Peer(Base):
def __init__(self, addr, port, network=None):
Base.__init__(self, DFS(addr, port))
self.fileSystem_ = FileSystem(self.dfs_)
if not network:
network = Network(self.dfs_, self.fileSystem_)
self.network_ = network
# TODO not loading state for now
# self.loadState()
##
# Public API
##
def open(self, fileName, op):
exists = self.fileSystem_.exists(fileName)
if exists and self.fileSystem_.isDeleted(fileName):
self.log_.i('WARNING: Trying to open non-existant file ' + fileName + ' in "r". Aborting open...')
return err.CannotOpenFile
status = self.updateFile(fileName)
self.log_.i('Opening ' + fileName + ' in "' + op + '"...')
if op is "r":
if exists and not self.fileSystem_.physical_.exists(fileName):
self.log_.i('WARNING: Unable to reteived ' + fileName + '. Aborting open...')
return err.CannotOpenFile
if exists:
if self.fileSystem_.canRead(fileName):
self.fileSystem_.logical_.fileList_[fileName].readCounter += 1
self.fileSystem_.logical_.fileList_[fileName].state = "r"
else:
self.log_.i('WARNING: Trying to open ' + fileName + ' in "r", but it is already open in "w". Aborting open...')
return err.CannotOpenFile
else:
self.log_.i('WARNING: Trying to open non-existant file ' + fileName + ' in "r". Aborting open...')
self.log_.w('tried to open ' + fileName + ' in read mode, but it doesnt exist')
return err.FileNotFound
elif op is "w":
if not exists:
buf = [' ']
self.fileSystem_.physical_.fillEmptyFile(fileName, 1)
self.fileSystem_.add(fileName)
self.log_.v('creating ' + fileName + ' b/c opened it in write mode for the first time')
if self.fileSystem_.canWrite(fileName):
if not self.fileSystem_.physical_.exists(fileName):
self.fileSystem_.physical_.fillEmptyFile(fileName, 1)
self.fileSystem_.logical_.fileList_[fileName].state = "w"
else:
self.log_.i('WARNING: Trying to open ' + fileName + ' in "w", but it is already open. Aborting open...')
return err.CannotOpenFile
else:
self.log_.i('WARNING: Trying to open ' + fileName + ' in unknown mode "' + op + '". Aborting open...')
return err.InvalidOp
self.log_.v('-- opened ' + fileName + ' in ' + op)
return status
def close(self, fileName, noLogging=False):
if not self.fileSystem_.exists(fileName) or self.fileSystem_.isDeleted(fileName):
self.log_.i('WARNING: Trying to close non-existant file ' + fileName + ' in "r". Aborting close...')
self.log_.w('tried to close ' + fileName + ', which doesnt exist: ' + str(self.fileSystem_.exists(fileName)))
return err.FileNotFound
file = self.fileSystem_.logical_.fileList_[fileName]
if file.state is "":
self.log_.i('WARNING: Trying to close ' + fileName + ', but it is already closed. Aborting close...')
return err.FileNotOpen
if not noLogging:
self.log_.i('Closing ' + fileName + '...')
if file.state is "r":
if file.readCounter > 0:
file.readCounter -= 1
if file.readCounter == 0:
file.state = ''
else:
file.state = ""
else: # state is 'w'
if file.hasLocalChanges and self.dfs_.online:
file.latestVersion = file.localVersion.copy()
self.network_.fileEdited()
self.network_.waitForPropagation()
file.state = ""
self.log_.v('-- closed ' + fileName + ', state is now: ' + file.state)
return err.OK
def read(self, fileName, buf=None, offset=0, bufsize=-1):
if not self.fileSystem_.exists(fileName):
self.log_.i('WARNING: Unable to retreive ' + fileName + '. Aborting read...')
self.log_.w('tried to read from ' + fileName + ', which doesnt exist')
return err.FileNotFound
if self.fileSystem_.logical_.fileList_[fileName].state is "r":
if not self.fileSystem_.physical_.exists(fileName):
self.log_.i('WARNING: Unable to retrieve ' + fileName + '. Aborting read...')
self.log_.e('tried to read from ' + fileName + ', which doesnt exist physically')
return err.FileNotFound
if self.fileSystem_.isMissingChunks(fileName):
self.log_.i('WARNING: ' + fileName + ' was not fully retreived. You are reading a partially downloaded file.')
elif self.fileSystem_.logical_.getFile(fileName).isOutOfDate():
self.log_.i('WARNING: ' + fileName + ' could not be retreived from the system. You are reading an out of date file.')
if not buf:
buf = [0] * self.fileSystem_.physical_.getFileSize(fileName)
if bufsize < 0:
bufsize = len(buf)
status = self.fileSystem_.readIntoBuffer(fileName, buf, offset, bufsize)
else:
self.log_.i('WARNING: Cannot read from ' + fileName + ' because it is not in read mode. Aborting read....')
self.log_.w('tried to read from ' + fileName + ' while not in read mode')
return err.FileNotOpen
try:
text = ''.join(buf)
except Exception:
text = str(buf)
textToPrint = text
if len(textToPrint) > 500:
textToPrint = textToPrint[:500]
self.log_.i('Read ' + fileName + ':\n"""\n' + textToPrint + '\n"""')
return text
def getSize(self, fileName):
if self.fileSystem_.physical_.exists(fileName):
return self.fileSystem_.physical_.getFileSize(fileName)
return 0
def hasLocalCopy(self, fileName):
if not self.fileSystem_.exists(fileName):
return False
return self.fileSystem_.isMissingChunks(fileName) or not self.fileSystem_.isUpToDate(fileName)
def write(self, fileName, buf, offset=0, bufsize=-1, deleteOriginalContent=False):
if bufsize < 0:
bufsize = len(buf)
self.log_.i('Writing buffer of size ' + str(bufsize) + ' to ' + fileName + '...')
if not self.fileSystem_.exists(fileName):
self.log_.i('WARNING: Cannot write to ' + fileName + ' because it is not in write mode. Aborting write....')
return err.FileNotFound
if self.fileSystem_.logical_.fileList_[fileName].state is "w":
if deleteOriginalContent:
self.fileSystem_.physical_.trim(fileName, 1)
self.fileSystem_.write(fileName, buf, offset, bufsize)
else:
self.log_.i('WARNING: Cannot write to ' + fileName + ' because it is not in write mode. Aborting write....')
self.log_.w('tried to write to ' + fileName + ' while not in write mode')
return err.FileNotOpenForWrite
return err.OK
def delete(self, fileName):
if not self.fileSystem_.exists(fileName) or self.fileSystem_.isDeleted(fileName):
self.log_.w(fileName + ' doest exist or is already deleted')
return 1
file = self.fileSystem_.logical_.getFile(fileName)
if file.state != '':
self.close(fileName, noLogging=True)
self.log_.i('Deleting ' + fileName + '...')
self.fileSystem_.delete(fileName)
self.network_.fileEdited()
self.network_.waitForPropagation()
return err.OK
def listFiles(self, files=[]):
while len(files) > 0:
files.pop()
for f in self.fileSystem_.list():
files.append(f)
self.printInfo(files)
return err.OK
# mark the file as stable
def markStable(self, fileName):
if not self.fileSystem_.exists(fileName):
self.log_.w('tried to mark ' + fileName + ' as stable, but doesnt exist')
return 1
self.log_.i('Marking ' + fileName + ' as stable...')
newFileName = fileName + ".stable";
while self.fileSystem_.exists(newFileName):
newFileName = newFileName + ".stable"
self.fileSystem_.copyFile(fileName, newFileName)
if self.dfs_.online:
self.network_.fileEdited()
self.network_.waitForPropagation()
# save the most recent version of the file locally
def pin(self, fileName):
if not self.fileSystem_.exists(fileName):
return err.FileNotFound
# TODO check write / read access
self.log_.i('Pinning ' + fileName + '...')
status = self.updateFile(fileName, 3)
return status
# delete the local copy of the file
def unpin(self, fileName):
if not self.fileSystem_.exists(fileName):
return err.FileNotFound
# TODO check write / read access
self.log_.i('Unpinning ' + fileName + '...')
self.fileSystem_.deleteLocalCopy(fileName)
return err.OK
# join DFS, connecting to the peer at the given addr and port if given
def join(self, addr, port=None):
if not port:
peer = addr
addr = peer.dfs_.addr
port = peer.dfs_.port
self.log_.i('Connecting to DFS from starting point ' + addr + ':' + str(port) + '...')
status = self.network_.connectTo(DFS(addr, port))
self.network_.waitForPropagation()
return status
# retire from the system
def retire(self):
self.log_.i('Retiring...')
self.goOffline()
return err.OK
# connect to the internet
def goOnline(self):
if not self.dfs_.online:
self.log_.i('Going online...')
self.network_.connect()
self.dfs_.online = True
self.network_.waitForPropagation()
return err.OK
else:
return err.AlreadyOnline
# disconnect from the internet
def goOffline(self):
if self.dfs_.online:
self.log_.i('Going offline...')
self.network_.disconnect()
self.dfs_.online = False
self.network_.waitForPropagation()
return err.OK
else:
return err.AlreadyOffline
# exits the program
def exit(self):
self.log_.i('exiting...')
self.goOffline()
fs = self.fileSystem_.getState()
ns = self.network_.getState()
self.log_.v('writing ' + str(len(fs)) + ' files and ' + str(len(ns)) + ' peers to disk')
state = (fs, ns)
s = serializer.serialize(state)
self.fileSystem_.writeState(s)
##
# Private functions
##
def loadState(self):
state = self.fileSystem_.readState()
if state:
try:
fs, ns = serializer.deserialize(state)
except Exception, ex:
self.log_.e('found state, but failed to deserialize: ' + str(ex))
return
if len(fs) + len(ns) > 0:
self.log_.i('Loaded ' + str(len(fs)) + ' files and ' + str(len(ns)) + ' device addresses from state')
self.fileSystem_.loadFromState(fs)
self.network_.loadFromState(ns)
