def __build_boot_filesystems(self, device_map):
if 'efi' in device_map:
log.info(
'Creating EFI(fat16) filesystem on %s',
device_map['efi'].get_device()
)
filesystem = FileSystem(
'fat16', device_map['efi'], self.root_dir + '/boot/efi/'
)
filesystem.create_on_device(
label=self.disk_setup.get_efi_label()
)
self.system_efi = filesystem
if 'boot' in device_map:
boot_filesystem = self.requested_boot_filesystem
if not boot_filesystem:
boot_filesystem = self.requested_filesystem
boot_directory = self.root_dir + '/boot/'
if self.bootloader == 'grub2_s390x_emu':
boot_directory = self.root_dir + '/boot/zipl/'
boot_filesystem = 'ext2'
log.info(
'Creating boot(%s) filesystem on %s',
boot_filesystem, device_map['boot'].get_device()
)
filesystem = FileSystem(
boot_filesystem, device_map['boot'], boot_directory
)
filesystem.create_on_device(
label=self.disk_setup.get_boot_label()
)
self.system_boot = filesystem
def test_read_and_write_file(self):
fs = FileSystem()
text_path = None
binary_path = None
unicode_text_string = u'\u016An\u012Dc\u014Dde\u033D'
hex_equivalent = '\xC5\xAA\x6E\xC4\xAD\x63\xC5\x8D\x64\x65\xCC\xBD'
malformed_text_hex = '\x4D\x69\x63\x72\x6F\x73\x6F\x66\x74\xAE\x20\x56\x69\x73\x75\x61\x6C\x20\x53\x74\x75\x64\x69\x6F\xAE\x20\x32\x30\x31\x30\x0D\x0A'
malformed_ignored_text_hex = '\x4D\x69\x63\x72\x6F\x73\x6F\x66\x74\x20\x56\x69\x73\x75\x61\x6C\x20\x53\x74\x75\x64\x69\x6F\x20\x32\x30\x31\x30\x0D\x0A'
try:
text_path = tempfile.mktemp(prefix='tree_unittest_')
binary_path = tempfile.mktemp(prefix='tree_unittest_')
fs.write_text_file(text_path, unicode_text_string)
contents = fs.read_binary_file(text_path)
self.assertEqual(contents, hex_equivalent)
fs.write_binary_file(binary_path, hex_equivalent)
text_contents = fs.read_text_file(binary_path)
self.assertEqual(text_contents, unicode_text_string)
self.assertRaises(ValueError, fs.write_text_file, binary_path,
malformed_text_hex)
fs.write_binary_file(binary_path, malformed_text_hex)
self.assertRaises(ValueError, fs.read_text_file, binary_path)
text_contents = fs.read_binary_file(binary_path).decode(
'utf8', 'ignore')
self.assertEquals(text_contents, malformed_ignored_text_hex)
fs.open_text_file_for_reading(binary_path, 'replace').readline()
finally:
if text_path and fs.isfile(text_path):
os.remove(text_path)
if binary_path and fs.isfile(binary_path):
os.remove(binary_path)
def export(self, export_filename):
log = logger()
log.info("About to export %s to %s" % (self.ident, export_filename))
zfs = FileSystem(base_path="")
zipComment = {"Ver": 1, "Id": self.ident, "Type": self.save_type.name}
# TODO: error checking.
with zipfile.ZipFile(export_filename, 'w') as zf:
zf.comment = json.dumps(zipComment).encode('utf-8')
log.debug("Writing {} (base {}) to {}".format(
self.filename, os.path.basename(self.filename),
zfs.get_path_by_type(os.path.basename(self.filename),
self.save_type)))
zf.write(
self.filename,
zfs.get_path_by_type(os.path.basename(self.filename),
self.save_type))
for url in self.models:
log.debug("Writing {} to {}".format(
url.location,
zfs.get_model_path(os.path.basename(url.location))))
zf.write(url.location,
zfs.get_model_path(os.path.basename(url.location)))
for url in self.images:
log.debug("Writing {} to {}".format(
url.location,
zfs.get_model_path(os.path.basename(url.location))))
zf.write(url.location,
zfs.get_image_path(os.path.basename(url.location)))
log.info("File exported.")
def save(self, path, filename):
#with open(os.path.join(path, filename), 'w') as stream:
#stream.write(self.text_input.text)
filename = os.path.join(path, filename)
organisation = cache['organisation']
password = cache['password']
# only save an encrypted version of this file
if (cache['organisation'] != None):
fs = FileSystem()
fs.createFile(filename, organisation, password, TextEditor.text)
TextEditor.currentFile = filename + '.enc'
else:
# the user needs to be prompted to login first
popup_msg = "Please login to an organisation first!"
layout = BoxLayout(orientation='vertical')
popup_title = "Error"
label1 = Label(text=popup_title)
label = Label(text=popup_msg)
button = Button(text='Dismiss')
layout.add_widget(label1)
layout.add_widget(label)
layout.add_widget(button)
popup = Popup(title=popup_title,
content=layout,
size_hint=(0.6, 0.6))
popup.open()
button.bind(on_press=popup.dismiss)
self.dismiss_popup()
def test_append_to_text_file(self):
fs = FileSystem()
text_path = None
unicode_text_string1 = u'\u016An\u012Dc\u014Dde\u033D'
unicode_text_string2 = 'Hello'
try:
text_path = tempfile.mktemp(prefix='tree_unittest_')
file = fs.open_text_file_for_writing(text_path)
file.write(unicode_text_string1)
file.close()
file = fs.open_text_file_for_writing(text_path, should_append=True)
file.write(unicode_text_string2)
file.close()
file = fs.open_text_file_for_reading(text_path)
read_text = file.read()
file.close()
self.assertEqual(read_text,
unicode_text_string1 + unicode_text_string2)
finally:
if text_path and fs.isfile(text_path):
os.remove(text_path)
def confirm_clicked(self, x):
RunArgsPopup.popup.dismiss()
organisation = cache['organisation']
password = cache['organisation']
filename = TextEditor.currentFile
args = RunArgsPopup.org.split(' ')
if (organisation != None and password != None and filename != None):
fs = FileSystem()
print("File: " + TextEditor.currentFile)
fs.run(filename, organisation, password, args)
else:
# the user needs to be prompted to login first/load a file
popup_msg = "Please login to an organisation and load a file!"
layout = BoxLayout(orientation='vertical')
popup_title = "Error"
label1 = Label(text=popup_title)
label = Label(text=popup_msg)
button = Button(text='Dismiss')
layout.add_widget(label1)
layout.add_widget(label)
layout.add_widget(button)
popup = Popup(title=popup_title,
content=layout,
size_hint=(0.6, 0.6))
popup.open()
button.bind(on_press=popup.dismiss)
def load(self, path, filename):
string = ""
with open(os.path.join(path, filename[0])) as stream:
string = stream.read()
print(path, filename)
ext = filename[0].split('.')[-1]
fs = FileSystem()
# ask the user for organisation name and password
layout = BoxLayout(orientation='vertical')
labelOrg = Label(text='Organisation')
inputOrg = TextInput(multiline=False)
labelPas = Label(text='Password')
inputPas = TextInput(multiline=False)
button = Button(text='Confirm')
layout.add_widget(labelOrg)
layout.add_widget(inputOrg)
layout.add_widget(labelPas)
layout.add_widget(inputPas)
layout.add_widget(button)
inputOrg.bind(text=PopupInput.set_org)
inputPas.bind(text=PopupInput.set_pas)
popup = Popup(title='Enter credentials',
content=layout,
size_hint=(None, None),
size=(400, 250))
popup.open()
if ext != 'enc':
# the file needs to be encrypted
pass
self.dismiss_popup()
def getLocations(self, files, locations, forceBundle=False):
"""find local copies of a file.
files: list of file names
locations: list with locations (output)
forceBundle: treat all files as bundle files
"""
if not self.isConnected():
return False
retval = 0
debug("getLocations: " + str(files) + " " + str(locations))
try:
fileSystem = FileSystem(self.config)
fetcher = CacheFetcher(self.config, fileSystem, self.connection)
nFiles = len(files)
for i in range(nFiles):
filename = os.path.realpath(files[i])
debug("locate " + filename)
sendExit = (i == (nFiles - 1))
if forceBundle or self._isBundleFile(filename):
loc, retval = self._findBundleLocations(filename, fileSystem, fetcher, sendExit)
else:
loc, retval = self._findLocations(filename, fileSystem, fetcher, sendExit)
locations += loc
except Exception, e:
error("unknown error: %s" % str(e))
retval = 2
def setUp(self):
self.fs = FileSystem()
self.setup_generic_test_dir()
self._this_dir = os.path.dirname(os.path.abspath(__file__))
self._missing_file = os.path.join(self._this_dir, 'missing_file.py')
self._this_file = os.path.join(self._this_dir, 'filesystem_unittest.py')
def __init__(self):
from codec import Codec
from window import Window
from system import System
from datajar import DataJar
from filesystem import FileSystem
try:
manifest = json.load(codecs.open('manifest.json', 'r', 'utf-8'))
except:
manifest = {}
for key in assets.manifest:
if key in manifest:
assets.manifest[key] = manifest[key]
self.app = QApplication(sys.argv)
self.app.setApplicationName(assets.manifest['name'])
self.app.setApplicationVersion(assets.manifest['version'])
assets.sys = System()
assets.codec = Codec()
assets.fs = FileSystem()
assets.dataJar = DataJar()
translator = QTranslator()
if translator.load("zh_CN.qm"):
self.app.installTranslator(translator)
self.window = Window(None, assets.manifest['path'] + 'index.html')
sys.exit(self.app.exec_())
def __init__(self, args, parent=None):
QObject.__init__(self, parent)
# variable declarations
self.m_defaultPageSettings = {}
self.m_pages = []
self.m_verbose = args.verbose
self.m_page = WebPage(self)
self.m_returnValue = 0
self.m_terminated = False
# setup the values from args
self.m_scriptFile = args.script
self.m_args = args.script_args
self.m_filesystem = FileSystem(self)
self.m_pages.append(self.m_page)
do_action('PhantomInitPre')
if not args.proxy:
QNetworkProxyFactory.setUseSystemConfiguration(True)
else:
proxy = QNetworkProxy(QNetworkProxy.HttpProxy, args.proxy[0],
int(args.proxy[1]))
QNetworkProxy.setApplicationProxy(proxy)
# Provide WebPage with a non-standard Network Access Manager
self.m_netAccessMan = NetworkAccessManager(args.disk_cache,
args.ignore_ssl_errors,
self)
self.m_page.setNetworkAccessManager(self.m_netAccessMan)
self.m_page.javaScriptConsoleMessageSent.connect(
self.printConsoleMessage)
self.m_defaultPageSettings['loadImages'] = args.load_images
self.m_defaultPageSettings['loadPlugins'] = args.load_plugins
self.m_defaultPageSettings['userAgent'] = self.m_page.userAgent()
self.m_defaultPageSettings[
'localAccessRemote'] = args.local_access_remote
self.m_page.applySettings(self.m_defaultPageSettings)
self.libraryPath = os.path.dirname(os.path.abspath(self.m_scriptFile))
# inject our properties and slots into javascript
self.m_page.mainFrame().addToJavaScriptWindowObject('phantom', self)
self.m_page.mainFrame().addToJavaScriptWindowObject(
'fs', self.m_filesystem)
bootstrap = QFile(':/bootstrap.js')
if not bootstrap.open(QFile.ReadOnly):
sys.exit('Can not bootstrap!')
bootstrapper = str(bootstrap.readAll())
bootstrap.close()
if not bootstrapper:
sys.exit('Can not bootstrap!')
self.m_page.mainFrame().evaluateJavaScript(bootstrapper)
do_action('PhantomInitPost')
def save():
if (TextEditor.currentFile != None):
fs = FileSystem()
fs.updateFile(TextEditor.currentFile, TextEditor.text,
cache['organisation'], cache['password'])
else:
lsd = LoadSaveDialog()
lsd.show_save()
def test_listdir(self):
fs = FileSystem()
with fs.mkdtemp(prefix='filesystem_unittest_') as d:
self.assertEqual(fs.listdir(d), [])
new_file = os.path.join(d, 'foo')
fs.write_text_file(new_file, u'foo')
self.assertEqual(fs.listdir(d), ['foo'])
os.remove(new_file)
def __init__(self):
self.log = get_logger('honey_pot')
self.log.info("Server Initiated")
self.soc = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.soc.bind(('', 23))
self.soc.listen(5)
self.log.info('Server Active')
self.thread_lock = threading.Lock()
self.file_system = FileSystem()
def __create_filesystem(self, volume_name, filesystem_name):
device_node = self.volume_map[volume_name]
label = None
if volume_name == 'LVRoot':
label = self.custom_args['root_label']
filesystem = FileSystem(
filesystem_name,
MappedDevice(device=device_node, device_provider=self))
filesystem.create_on_device(label=label)
def test_chdir(self):
fs = FileSystem()
cwd = fs.getcwd()
newdir = '/'
if sys.platform == 'win32':
newdir = 'c:\\'
fs.chdir(newdir)
self.assertEqual(fs.getcwd(), newdir)
fs.chdir(cwd)
def choose_lexer(filename):
fs = FileSystem()
filetype = fs.getFileType(filename)
if filetype == 'py':
TextEditor.lex = PythonLexer()
elif filetype == 'java':
TextEditor.lex = JavaLexer()
elif filetype == 'cpp':
TextEditor.lex = CppLexer()
TextEditor.codeinput.lexer = TextEditor.lex
def __enter__(self):
self._filesystem = FileSystem(self._disk)
try:
self._filesystem.mount()
except:
# TODO: exit the program if the disk cannot be scanned
logging.warning("The disk cannot be mounted. Skip the scanning.")
sys.exit(-1)
return self
def main(argc, argv):
options = Options(argv)
if options.version:
sys.stderr.write("%s\n" % __version__)
return 1
if options.help or len(options.arg) < 1:
usage()
return 1
if options.debug:
LogLevel.enableDebug()
if options.copy:
if len(options.arg) < 2:
usage()
return 1
filename = options.arg[0]
config = ClientConfiguration(options.config)
if options.config:
if not config.read(options.config):
error("cannot read config file %s. using build-in defaults" %
options.config)
else:
if not config.loadDefault():
error("cannot read default config file. using build-in defaults")
if options.nobundle:
config.IGNORE_BUNDLE = True
if options.printDestination:
print CmClient.getDestination(FileSystem(config),
os.path.realpath(filename))
return 0
client = CmClient(config)
if not client.isConnected():
error("cannot connect to server")
else:
log("connected to %s" % str(client.connection.getPeerName()))
r = False
if options.copy:
r = client.copy(filename, options.arg[1], options.register,
options.bundle)
elif options.locate:
locations = []
r = client.getLocations(options.arg, locations, options.bundle)
log("%d locations found" % len(locations))
for l in locations:
print "%s:%s:%s" % l
else:
f, r = client.fetch(filename, options.bundle, options.conjunct)
print f
return not r
def _fetchFile(self, filename, fileSystem=None, fetcher=None, closeOnError=True):
if not fileSystem:
fileSystem = FileSystem(self.config)
if not fetcher:
fetcher = CacheFetcher(self.config, fileSystem, self.connection)
if not os.path.isfile(filename):
error("file not found '%s'" % filename)
if closeOnError and not self.single: fetcher.sendExit()
return (filename, 1)
fileinfo = fileSystem.getFileInfo(filename)
destination = self.getDestination(fileSystem, filename)
if destination is None:
return (filename, 1)
log("destination: " + str(destination))
if os.path.isfile(destination):
wait = fetcher.isActive(destination)
while wait > 0:
log("file transfer in progress. wait %ds" % wait)
time.sleep(wait)
wait = fetcher.isActive(destination)
fileExists, canCopy, removed = fileSystem.destinationExists(fileinfo, destination)
if fileExists:
if removed:
log("removed " + destination)
fetcher.sendFileRemoved(fileinfo, destination)
if not canCopy:
error("cannot copy file to %s" % destination)
return (filename, 1)
else:
log("using existing file")
fileSystem.setATime(destination)
fetcher.sendFileLocation(fileinfo, destination)
return (destination, 0)
freeSpace, removed = fileSystem.checkFreeSpace(int(fileinfo[1]), destination)
if not freeSpace:
log("not enough free space in %s" % fileSystem.cacheDir)
if closeOnError and self.single: fetcher.sendExit()
return (filename, 1)
log("request: " + filename)
fetcher.requestFile(fileinfo, destination)
msg = self.connection.receiveMessage()
resultFile, ok, term = fetcher.handleMessage(fileinfo, destination, msg)
while not ok:
msg = self.connection.receiveMessage()
resultFile, ok, term = fetcher.handleMessage(fileinfo, destination, msg)
return (resultFile, 0)
def _copyFile(self, source, destination, register, isBundle):
debug("copyFile: %s, %s" % (source, destination))
if not os.path.isfile(source):
error("file not found '%s'" % source)
return 1
debug("isBundle: %d" % isBundle)
if not isBundle:
if os.path.isdir(destination):
destination = os.path.normpath(destination + "/" + os.path.basename(source))
debug("destination: " + destination)
if os.path.isfile(destination):
log("warning: will overwrite %s" % destination)
else:
if not os.path.isdir(destination):
error("destination is not a directory")
return 1
fs = FileSystem(self.config)
fileinfo = fs.getFileInfo(source)
debug("fileinfo: " + str(fileinfo))
fileserver = fs.getFileServer(destination)
debug("fileserver: " + fileserver)
request = Message(Message.REGISTER_COPY, fileinfo + [fileserver])
if not self.connection.sendMessage(request):
error("cannot send message")
return 1
retval = 0
while True:
msg = self.connection.receiveMessage()
if msg == None:
error("cannot receive message")
retval = 1
break
elif msg.type == Message.WAIT:
time.sleep(int(msg.content[0]))
if not self.connection.sendMessage(request):
error("cannot send message")
retval = 1
break
elif msg.type == Message.FILE_OK:
if not isBundle:
retval, reply = self._copySingleFile(source, destination, register)
else:
retval, reply = self._copyBundleFile(source, destination)
if not self.connection.sendMessage(reply):
error("cannot send message")
retval = 1
break
else:
error("unexpected message: %s" % str(msg))
retval = 1
break
return retval
def __operate_on_loop(self):
filesystem = None
loop_provider = LoopDevice(self.filename,
self.filesystem_setup.get_size_mbytes(),
self.blocksize)
loop_provider.create()
filesystem = FileSystem(self.requested_filesystem, loop_provider,
self.root_dir, self.custom_args)
filesystem.create_on_device(self.label)
log.info('--> Syncing data to filesystem on %s',
loop_provider.get_device())
exclude_list = ['image', '.profile', '.kconfig', 'var/cache/kiwi']
filesystem.sync_data(exclude_list)
def test_dirs_under(self):
fs = FileSystem()
parentDir = fs.normpath(fs.join(self._this_dir, ".."))
self.assertTrue(self._this_dir in fs.dirs_under(parentDir))
def filter_no_dir(fs, dirpath):
return False
self.assertEqual(len(fs.dirs_under(parentDir, filter_no_dir)), 0)
def filter_this_dir(fs, dirpath):
return dirpath != self._this_dir
self.assertFalse(self._this_dir in fs.dirs_under(parentDir, filter_this_dir))
def choose_lexer(self):
fs = FileSystem()
if TextEditor.filepath != None:
ext = fs.getFileType(TextEditor.filepath)
if ext == 'py':
TextEditor.lex = PythonLexer()
elif ext == 'java':
TextEditor.lex = JavaLexer()
elif ext == 'cpp':
TextEditor.lex = CppLexer()
else:
# No extension
TextEditor.lex = PythonLexer()
def test_remove_file_with_retry(self):
RealFileSystemTest._remove_failures = 2
def remove_with_exception(filename):
RealFileSystemTest._remove_failures -= 1
if RealFileSystemTest._remove_failures >= 0:
try:
raise WindowsError
except NameError:
raise FileSystem._WindowsError
fs = FileSystem()
self.assertTrue(fs.remove('filename', remove_with_exception))
self.assertEqual(-1, RealFileSystemTest._remove_failures)
def test_read_text_file_unicode_decode_error(self):
fs = FileSystem()
text_path = None
try:
text_path = tempfile.mktemp(prefix='write_text_unittest_')
fs.write_binary_file(text_path, bytearray(b'\x73\x74\x72\x8b'))
self.assertRaises(UnicodeDecodeError, fs.read_text_file, text_path)
self.assertEqual(u'str\ufffd',
fs.read_text_file(text_path, errors='replace'))
self.assertEqual('str',
fs.read_text_file(text_path, errors='ignore'))
finally:
if text_path and fs.isfile(text_path):
os.remove(text_path)
def parse_dependencies(self, kw):
http_request = kw.get('http_request', None)
model_base = kw.get('model_base', None)
manager = kw.get('manager', None)
model = kw.get('model', None)
queryset = kw.get('queryset', None)
settings = kw.get('settings', None)
site = kw.get('site', None)
fs = kw.get('fs', None)
self.http_request = http_request
if not http_request:
from django.http import HttpRequest
self.http_request = HttpRequest
self.model_base = model_base
if not model_base:
from django.db.models.base import ModelBase
self.model_base = ModelBase
self.manager = manager
if not manager:
from django.db.models.manager import Manager
self.manager = Manager
self.model = model
if not model:
from django.db.models import Model
self.model = Model
self.queryset = queryset
if not queryset:
from django.db.models.query import QuerySet
self.queryset = QuerySet
self.settings = settings
if not settings:
from django.conf import settings
self.settings = settings
self.fs = fs
if not fs:
self.fs = FileSystem()
self.site = site
def setup(self, name=None):
filesystem = FileSystem(
'btrfs', MappedDevice(device=self.device, device_provider=self))
filesystem.create_on_device(label=self.custom_args['root_label'])
self.setup_mountpoint()
Command.run(['mount', self.device, self.mountpoint])
root_volume = self.mountpoint + '/@'
Command.run(['btrfs', 'subvolume', 'create', root_volume])
if self.custom_args['root_is_snapshot']:
snapshot_volume = self.mountpoint + '/@/.snapshots'
Command.run(['btrfs', 'subvolume', 'create', snapshot_volume])
Path.create(snapshot_volume + '/1')
snapshot = self.mountpoint + '/@/.snapshots/1/snapshot'
Command.run(
['btrfs', 'subvolume', 'snapshot', root_volume, snapshot])
self.__set_default_volume('@/.snapshots/1/snapshot')
else:
self.__set_default_volume('@')
def test_maybe_make_directory__failure(self):
# FIXME: os.chmod() doesn't work on Windows to set directories
# as readonly, so we skip this test for now.
if sys.platform in ('win32', 'cygwin'):
return
fs = FileSystem()
with fs.mkdtemp(prefix='filesystem_unittest_') as d:
# Remove write permissions on the parent directory.
os.chmod(d, stat.S_IRUSR)
# Now try to create a sub directory - should fail.
sub_dir = fs.join(d, 'subdir')
self.assertRaises(OSError, fs.maybe_make_directory, sub_dir)
# Clean up in case the test failed and we did create the
# directory.
if os.path.exists(sub_dir):
os.rmdir(sub_dir)
def test_read_and_write_text_file(self):
fs = FileSystem()
text_path = None
unicode_text_string = u'\u016An\u012Dc\u014Dde\u033D'
hex_equivalent = '\xC5\xAA\x6E\xC4\xAD\x63\xC5\x8D\x64\x65\xCC\xBD'
try:
text_path = tempfile.mktemp(prefix='tree_unittest_')
file = fs.open_text_file_for_writing(text_path)
file.write(unicode_text_string)
file.close()
file = fs.open_text_file_for_reading(text_path)
read_text = file.read()
file.close()
self.assertEqual(read_text, unicode_text_string)
finally:
if text_path and fs.isfile(text_path):
os.remove(text_path)
