def setup(self, name=None):
filesystem = FileSystem(
'btrfs', MappedDevice(device=self.device, device_provider=self)
)
filesystem.create_on_device(
label=self.custom_args['root_label']
)
self.setup_mountpoint()
Command.run(
['mount', self.device, self.mountpoint]
)
root_volume = self.mountpoint + '/@'
Command.run(
['btrfs', 'subvolume', 'create', root_volume]
)
if self.custom_args['root_is_snapshot']:
snapshot_volume = self.mountpoint + '/@/.snapshots'
Command.run(
['btrfs', 'subvolume', 'create', snapshot_volume]
)
Path.create(snapshot_volume + '/1')
snapshot = self.mountpoint + '/@/.snapshots/1/snapshot'
Command.run(
['btrfs', 'subvolume', 'snapshot', root_volume, snapshot]
)
self.__set_default_volume('@/.snapshots/1/snapshot')
else:
self.__set_default_volume('@')
def test1(self):
fs = FileSystem(TEST_DIR)
dirs = fs.listdirs()
self.assertIn('lvl_1_a', dirs)
self.assertIn('lvl_1_b', dirs)
self.assertEqual(len(dirs), 2)
def __operate_on_file(self):
default_provider = DeviceProvider()
filesystem = FileSystem(
self.requested_filesystem, default_provider,
self.root_dir, self.custom_args
)
filesystem.create_on_file(
self.filename, self.label
)
class FileSystemTest(TestCase):
def setUp(self):
self.fs = FileSystem(".")
def tearDown(self):
pass
def testOne(self):
print self.fs.absPath("../watchDir")
pass
def _copyFile(self, source, destination, register, isBundle):
debug("copyFile: %s, %s" % (source, destination))
if not os.path.isfile(source):
error("file not found '%s'" % source)
return 1
debug("isBundle: %d" % isBundle)
if not isBundle:
if os.path.isdir(destination):
destination = os.path.normpath(destination + "/" +
os.path.basename(source))
debug("destination: " + destination)
if os.path.isfile(destination):
log("warning: will overwrite %s" % destination)
else:
if not os.path.isdir(destination):
error("destination is not a directory")
return 1
fs = FileSystem(self.config)
fileinfo = fs.getFileInfo(source)
debug("fileinfo: " + str(fileinfo))
fileserver = fs.getFileServer(destination)
debug("fileserver: " + fileserver)
request = Message(Message.REGISTER_COPY, fileinfo + [fileserver])
if not self.connection.sendMessage(request):
error("cannot send message")
return 1
retval = 0
while True:
msg = self.connection.receiveMessage()
if msg == None:
error("cannot receive message")
retval = 1
break
elif msg.type == Message.WAIT:
time.sleep(int(msg.content[0]))
if not self.connection.sendMessage(request):
error("cannot send message")
retval = 1
break
elif msg.type == Message.FILE_OK:
if not isBundle:
retval, reply = self._copySingleFile(
source, destination, register)
else:
retval, reply = self._copyBundleFile(source, destination)
if not self.connection.sendMessage(reply):
error("cannot send message")
retval = 1
break
else:
error("unexpected message: %s" % str(msg))
retval = 1
break
return retval
def ui_loop():
fs = FileSystem()
print "****Welcome to the terminal emulator****"
user_command = ""
while user_command != "quit" or user_command != "exit":
user_command = raw_input("$ ")
# Separate first word from rest of command
user_command = user_command.split(" ", 1)
fs.run_command(user_command)
class Collection(object):
def __init__(self, dir_path, merge_subdirectories = 0, entry_cls = BasicEntry):
self.dir_path = dir_path
self.merge_subdirectories = merge_subdirectories
if not issubclass(entry_cls, BaseEntry):
raise ValueError("entry_generator must be subclass of BaseEntryGenerator")
self.entry_cls = entry_cls
self.fs = FileSystem(self.dir_path)
def keys(self):
dirs = self.fs.listdirs(self.merge_subdirectories)
return dirs
def list(self):
keys = self.keys()
return EntryList(self, keys)
def has(self, key):
return key in self.keys()
def get(self, key = None):
if key is None:
raise NotImplementedError("Creation of entries is not supported yet")
entry = self.entry_cls(self, key)
return entry
def clear(self, key):
raise NotImplementedError("Modification of entries is not supported yet")
def fill_entry(self, entry):
entry_path = self.fs.get_full_path(entry.key())
fs = FileSystem(entry_path)
entry.decode(fs)
def save_entry(self, entry):
entry_path = self.fs.get_full_path(entry.key())
fs = FileSystem(entry_path)
entry.encode(fs)
def test2(self):
fs = FileSystem(TEST_DIR)
dirs = fs.listdirs(1)
self.assertIn('lvl_1_a/lvl_2_aa', dirs)
self.assertIn('lvl_1_a/lvl_2_ab', dirs)
self.assertIn('lvl_1_b/lvl_2_ba', dirs)
self.assertIn('lvl_1_b/lvl_2_bb', dirs)
self.assertNotIn( 'lvl_1_a', dirs)
self.assertNotIn( 'lvl_1_b', dirs)
def __create_filesystem(self, volume_name, filesystem_name):
device_node = self.volume_map[volume_name]
label = None
if volume_name == 'LVRoot':
label = self.custom_args['root_label']
filesystem = FileSystem(
filesystem_name,
MappedDevice(device=device_node, device_provider=self)
)
filesystem.create_on_device(
label=label
)
def test_remove_file_with_retry(self):
FileSystemTest._remove_failures = 2
def remove_with_exception(filename):
FileSystemTest._remove_failures -= 1
if FileSystemTest._remove_failures >= 0:
try:
raise WindowsError
except NameError:
raise FileSystem._WindowsError
fs = FileSystem()
self.assertTrue(fs.remove('filename', remove_with_exception))
self.assertEquals(-1, FileSystemTest._remove_failures)
def __init__(self, options, option_collector):
Task.__init__(self)
option_collector.option_decls.add(self.__class__)
src_path = options.get('src-dir', None)
if src_path is None: src_path = os.getcwd()
bld_path = options.get('bld-dir', None)
if bld_path is None: bld_path = os.path.join(src_path, '++wonderbuild')
if src_path == bld_path: raise Exception, 'build dir and source dir are the same'
self.list_aliases = 'list-tasks' in options
self.task_aliases = {} # {name: [tasks]}
aliases = options.get('tasks', None)
if aliases is not None:
if len(aliases) != 0: self.requested_task_aliases = aliases.split(',')
else: self.requested_task_aliases = (None,)
else: self.requested_task_aliases = None
# TODO See note in PurgeablePersistentDict
if False: self.global_purge = options.get('purge-persistent', 'no') != 'no'
else: self.global_purge = False
self.processsing = False
gc_enabled = gc.isenabled()
if gc_enabled:
try: gc.disable()
except NotImplementedError: pass # jython uses the gc of the jvm
try:
try: f = open(os.path.join(bld_path, 'persistent.pickle'), 'rb')
except IOError: raise
else:
try:
if __debug__ and is_debug: t0 = time.time()
pickle_abi_sig = cPickle.load(f)
if pickle_abi_sig != abi_sig:
print >> sys.stderr, colored('33', 'wonderbuild: abi sig changed: discarding persistent pickle file => full rebuild will be performed')
persistent = PurgeablePersistentDict()
else:
persistent = cPickle.load(f)
if __debug__ and is_debug: debug('project: pickle: load time: ' + str(time.time() - t0) + ' s')
except:
print >> sys.stderr, colored('33', 'wonderbuild: could not load persistent pickle file => full rebuild will be performed')
import traceback; traceback.print_exc()
raise
finally: f.close()
except:
persistent = PurgeablePersistentDict()
finally:
if gc_enabled: gc.enable()
self.fs = FileSystem(persistent)
self.top_src_dir = self.fs.cur / src_path
bld_dir = self.fs.cur / bld_path
SharedTaskHolder.__init__(self, persistent, options, option_collector, bld_dir)
def test_maybe_make_directory__failure(self):
# FIXME: os.chmod() doesn't work on Windows to set directories
# as readonly, so we skip this test for now.
if sys.platform in ('win32', 'cygwin'):
return
fs = FileSystem()
with fs.mkdtemp(prefix='filesystem_unittest_') as d:
# Remove write permissions on the parent directory.
os.chmod(d, stat.S_IRUSR)
# Now try to create a sub directory - should fail.
sub_dir = fs.join(d, 'subdir')
self.assertRaises(OSError, fs.maybe_make_directory, sub_dir)
# Clean up in case the test failed and we did create the
# directory.
if os.path.exists(sub_dir):
os.rmdir(sub_dir)
def test_chdir(self):
fs = FileSystem()
cwd = fs.getcwd()
newdir = '/'
if sys.platform == 'win32':
newdir = 'c:\\'
fs.chdir(newdir)
self.assertEqual(fs.getcwd(), newdir)
fs.chdir(cwd)
def test_read_and_write_text_file(self):
fs = FileSystem()
text_path = None
unicode_text_string = u'\u016An\u012Dc\u014Dde\u033D'
hex_equivalent = '\xC5\xAA\x6E\xC4\xAD\x63\xC5\x8D\x64\x65\xCC\xBD'
try:
text_path = tempfile.mktemp(prefix='tree_unittest_')
file = fs.open_text_file_for_writing(text_path)
file.write(unicode_text_string)
file.close()
file = fs.open_text_file_for_reading(text_path)
read_text = file.read()
file.close()
self.assertEqual(read_text, unicode_text_string)
finally:
if text_path and fs.isfile(text_path):
os.remove(text_path)
def __operate_on_loop(self):
filesystem = None
loop_provider = LoopDevice(
self.filename,
self.filesystem_setup.get_size_mbytes(),
self.blocksize
)
loop_provider.create()
filesystem = FileSystem(
self.requested_filesystem, loop_provider,
self.root_dir, self.custom_args
)
filesystem.create_on_device(self.label)
log.info(
'--> Syncing data to filesystem on %s', loop_provider.get_device()
)
exclude_list = [
'image', '.profile', '.kconfig', 'var/cache/kiwi'
]
filesystem.sync_data(exclude_list)
def __init__(self, dir_path, merge_subdirectories = 0, entry_cls = BasicEntry):
self.dir_path = dir_path
self.merge_subdirectories = merge_subdirectories
if not issubclass(entry_cls, BaseEntry):
raise ValueError("entry_generator must be subclass of BaseEntryGenerator")
self.entry_cls = entry_cls
self.fs = FileSystem(self.dir_path)
def __build_boot_filesystems(self, device_map):
if 'efi' in device_map:
log.info(
'Creating EFI(fat16) filesystem on %s',
device_map['efi'].get_device()
)
filesystem = FileSystem(
'fat16', device_map['efi'], self.root_dir + '/boot/efi/'
)
filesystem.create_on_device(
label=self.disk_setup.get_efi_label()
)
self.system_efi = filesystem
if 'boot' in device_map:
boot_filesystem = self.requested_boot_filesystem
if not boot_filesystem:
boot_filesystem = self.requested_filesystem
boot_directory = self.root_dir + '/boot/'
if self.bootloader == 'grub2_s390x_emu':
boot_directory = self.root_dir + '/boot/zipl/'
boot_filesystem = 'ext2'
log.info(
'Creating boot(%s) filesystem on %s',
boot_filesystem, device_map['boot'].get_device()
)
filesystem = FileSystem(
boot_filesystem, device_map['boot'], boot_directory
)
filesystem.create_on_device(
label=self.disk_setup.get_boot_label()
)
self.system_boot = filesystem
def test_maybe_make_directory__success(self):
fs = FileSystem()
with fs.mkdtemp(prefix='filesystem_unittest_') as base_path:
sub_path = os.path.join(base_path, "newdir")
self.assertFalse(os.path.exists(sub_path))
self.assertFalse(fs.isdir(sub_path))
fs.maybe_make_directory(sub_path)
self.assertTrue(os.path.exists(sub_path))
self.assertTrue(fs.isdir(sub_path))
# Make sure we can re-create it.
fs.maybe_make_directory(sub_path)
self.assertTrue(os.path.exists(sub_path))
self.assertTrue(fs.isdir(sub_path))
# Clean up.
os.rmdir(sub_path)
self.assertFalse(os.path.exists(base_path))
self.assertFalse(fs.isdir(base_path))
def test_listdir(self):
fs = FileSystem()
with fs.mkdtemp(prefix='filesystem_unittest_') as d:
self.assertEqual(fs.listdir(d), [])
new_file = os.path.join(d, 'foo')
fs.write_text_file(new_file, u'foo')
self.assertEqual(fs.listdir(d), ['foo'])
os.remove(new_file)
def test_read_and_write_file(self):
fs = FileSystem()
text_path = None
binary_path = None
unicode_text_string = u'\u016An\u012Dc\u014Dde\u033D'
hex_equivalent = '\xC5\xAA\x6E\xC4\xAD\x63\xC5\x8D\x64\x65\xCC\xBD'
try:
text_path = tempfile.mktemp(prefix='tree_unittest_')
binary_path = tempfile.mktemp(prefix='tree_unittest_')
fs.write_text_file(text_path, unicode_text_string)
contents = fs.read_binary_file(text_path)
self.assertEqual(contents, hex_equivalent)
fs.write_binary_file(binary_path, hex_equivalent)
text_contents = fs.read_text_file(binary_path)
self.assertEqual(text_contents, unicode_text_string)
finally:
if text_path and fs.isfile(text_path):
os.remove(text_path)
if binary_path and fs.isfile(binary_path):
os.remove(binary_path)
def _fetchFile(self, filename, fileSystem=None, fetcher=None, closeOnError=True):
if not fileSystem:
fileSystem = FileSystem(self.config)
if not fetcher:
fetcher = CacheFetcher(self.config, fileSystem, self.connection)
if not os.path.isfile(filename):
error("file not found '%s'" % filename)
if closeOnError and not self.single: fetcher.sendExit()
return (filename, 1)
fileinfo = fileSystem.getFileInfo(filename)
destination = self.getDestination(fileSystem, filename)
if destination is None:
return (filename, 1)
log("destination: " + str(destination))
if os.path.isfile(destination):
wait = fetcher.isActive(destination)
while wait > 0:
log("file transfer in progress. wait %ds" % wait)
time.sleep(wait)
wait = fetcher.isActive(destination)
fileExists, canCopy, removed = fileSystem.destinationExists(fileinfo, destination)
if fileExists:
if removed:
log("removed " + destination)
fetcher.sendFileRemoved(fileinfo, destination)
if not canCopy:
error("cannot copy file to %s" % destination)
return (filename, 1)
else:
log("using existing file")
fileSystem.setATime(destination)
fetcher.sendFileLocation(fileinfo, destination)
return (destination, 0)
freeSpace, removed = fileSystem.checkFreeSpace(int(fileinfo[1]), destination)
if not freeSpace:
log("not enough free space in %s" % fileSystem.cacheDir)
if closeOnError and self.single: fetcher.sendExit()
return (filename, 1)
log("request: " + filename)
fetcher.requestFile(fileinfo, destination)
msg = self.connection.receiveMessage()
resultFile, ok, term = fetcher.handleMessage(fileinfo, destination, msg)
while not ok:
msg = self.connection.receiveMessage()
resultFile, ok, term = fetcher.handleMessage(fileinfo, destination, msg)
return (resultFile, 0)
def test_dirs_under(self):
fs = FileSystem()
parentDir = fs.normpath(fs.join(self._this_dir, ".."))
self.assertTrue(self._this_dir in fs.dirs_under(parentDir))
def filter_no_dir(fs, dirpath):
return False
self.assertEqual(len(fs.dirs_under(parentDir, filter_no_dir)), 0)
def filter_this_dir(fs, dirpath):
return dirpath != self._this_dir
self.assertFalse(self._this_dir in fs.dirs_under(parentDir, filter_this_dir))
def test_read_and_write_file(self):
fs = FileSystem()
text_path = None
binary_path = None
unicode_text_string = u'Ūnĭcōde̽'
hex_equivalent = '\xC5\xAA\x6E\xC4\xAD\x63\xC5\x8D\x64\x65\xCC\xBD'
try:
text_path = tempfile.mktemp(prefix='tree_unittest_')
binary_path = tempfile.mktemp(prefix='tree_unittest_')
fs.write_text_file(text_path, unicode_text_string)
contents = fs.read_binary_file(text_path)
self.assertEqual(contents, hex_equivalent)
fs.write_text_file(binary_path, hex_equivalent)
text_contents = fs.read_text_file(binary_path)
self.assertEqual(text_contents, unicode_text_string)
except:
if text_path:
os.remove(text_path)
if binary_path:
os.remove(binary_path)
import uos
import computer
from filesystem import FileSystem
uos.umount('/')
uos.mount(FileSystem(__path__, root=True), '/')
uos.mount(FileSystem(computer.tmp_address()), '/tmp')
def test_exists__false(self):
fs = FileSystem()
self.assertFalse(fs.exists(self._missing_file))
def test_isdir__false(self):
fs = FileSystem()
self.assertFalse(fs.isdir(self._this_file))
def __operate_on_file(self):
default_provider = DeviceProvider()
filesystem = FileSystem(self.requested_filesystem, default_provider,
self.root_dir, self.custom_args)
filesystem.create_on_file(self.filename, self.label)
def create(self):
if self.install_media and self.build_type_name != 'oem':
raise KiwiInstallMediaError(
'Install media requires oem type setup, got %s' %
self.build_type_name
)
# setup recovery archive, cleanup and create archive if requested
self.system_setup.create_recovery_archive()
# prepare boot(initrd) root system
log.info('Preparing boot system')
self.boot_image_task.prepare()
# precalculate needed disk size
disksize_mbytes = self.disk_setup.get_disksize_mbytes()
# create the disk
log.info('Creating raw disk image %s', self.diskname)
loop_provider = LoopDevice(
self.diskname, disksize_mbytes, self.blocksize
)
loop_provider.create()
self.disk = Disk(
self.firmware.get_partition_table_type(), loop_provider
)
# create the bootloader instance
self.bootloader_config = BootLoaderConfig(
self.bootloader, self.xml_state, self.root_dir,
{'targetbase': loop_provider.get_device()}
)
# create disk partitions and instance device map
device_map = self.__build_and_map_disk_partitions()
# create raid on current root device if requested
if self.mdraid:
self.raid_root = RaidDevice(device_map['root'])
self.raid_root.create_degraded_raid(raid_level=self.mdraid)
device_map['root'] = self.raid_root.get_device()
# create luks on current root device if requested
if self.luks:
self.luks_root = LuksDevice(device_map['root'])
self.luks_root.create_crypto_luks(
passphrase=self.luks, os=self.luks_os
)
device_map['root'] = self.luks_root.get_device()
# create filesystems on boot partition(s) if any
self.__build_boot_filesystems(device_map)
# create volumes and filesystems for root system
if self.volume_manager_name:
volume_manager_custom_parameters = {
'root_filesystem_args':
self.custom_filesystem_args,
'root_label':
self.disk_setup.get_root_label(),
'root_is_snapshot':
self.xml_state.build_type.get_btrfs_root_is_snapshot()
}
volume_manager = VolumeManager(
self.volume_manager_name, device_map['root'],
self.root_dir + '/',
self.volumes, volume_manager_custom_parameters
)
volume_manager.setup(
self.volume_group_name
)
volume_manager.create_volumes(
self.requested_filesystem
)
volume_manager.mount_volumes()
self.system = volume_manager
device_map['root'] = volume_manager.get_device()['root']
else:
log.info(
'Creating root(%s) filesystem on %s',
self.requested_filesystem, device_map['root'].get_device()
)
filesystem = FileSystem(
self.requested_filesystem, device_map['root'],
self.root_dir + '/',
self.custom_filesystem_args
)
filesystem.create_on_device(
label=self.disk_setup.get_root_label()
)
self.system = filesystem
# create a random image identifier
self.mbrid = ImageIdentifier()
self.mbrid.calculate_id()
# create first stage metadata to boot image
self.__write_partition_id_config_to_boot_image()
self.__write_recovery_metadata_to_boot_image()
self.__write_raid_config_to_boot_image()
self.system_setup.export_modprobe_setup(
self.boot_image_task.boot_root_directory
)
# create first stage metadata to system image
self.__write_image_identifier_to_system_image()
self.__write_crypttab_to_system_image()
# create initrd cpio archive
self.boot_image_task.create_initrd(self.mbrid)
# create second stage metadata to boot
self.__copy_first_boot_files_to_system_image()
self.__write_bootloader_config_to_system_image(device_map)
self.mbrid.write_to_disk(
self.disk.storage_provider
)
# syncing system data to disk image
log.info('Syncing system to image')
if self.system_efi:
log.info('--> Syncing EFI boot data to EFI partition')
self.system_efi.sync_data()
if self.system_boot:
log.info('--> Syncing boot data at extra partition')
self.system_boot.sync_data(
self.__get_exclude_list_for_boot_data_sync()
)
log.info('--> Syncing root filesystem data')
self.system.sync_data(
self.__get_exclude_list_for_root_data_sync(device_map)
)
# install boot loader
self.__install_bootloader(device_map)
self.result.add(
'disk_image', self.diskname
)
# create install media if requested
if self.install_media:
if self.image_format:
log.warning('Install image requested, ignoring disk format')
if self.install_iso or self.install_stick:
log.info('Creating hybrid ISO installation image')
self.install_image.create_install_iso()
self.result.add(
'installation_image', self.install_image.isoname
)
if self.install_pxe:
log.info('Creating PXE installation archive')
self.install_image.create_install_pxe_archive()
self.result.add(
'installation_pxe_archive', self.install_image.pxename
)
# create disk image format if requested
elif self.image_format:
log.info('Creating %s Disk Format', self.image_format)
disk_format = DiskFormat(
self.image_format, self.xml_state,
self.root_dir, self.target_dir
)
disk_format.create_image_format()
self.result.add(
'disk_format_image',
self.target_dir + '/' + disk_format.get_target_name_for_format(
self.image_format
)
)
return self.result
def test_getcwd(self):
fs = FileSystem()
self.assertTrue(fs.exists(fs.getcwd()))
def runBasic_btn_press(instance):
print("Basic Run")
if (TextEditor.currentFile != None):
fs = FileSystem()
fs.run(TextEditor.currentFile, cache['organisation'],
cache['password'])
def test_sep__is_readonly(self):
def assign_sep():
fs.sep = ' '
fs = FileSystem()
self.assertRaises(AttributeError, assign_sep)
def save_btn_press(instance):
print('Save')
fs = FileSystem()
fs.updateFile('finnsFile.enc', TextEditor.text, 'Student Hack',
'test1234')
print(TextEditor.text)
class VulnScanner:
"""Vulnerability scanner.
Allows to scan the given disk content and query
a CVE DB for vulnerabilities (the database is saved
in a redis server)
disk must contain the path of a valid disk image.
"""
def __init__(self, disk, redis_addr):
self._disk = disk
self._filesystem = None
if ":" in redis_addr:
redis_IP, redis_port = redis_addr.split(":")
else:
redis_IP, redis_port = redis_addr, 6379
self._cve_redis = redis.StrictRedis(host=redis_IP,
port=redis_port,
db=0,
socket_timeout=2)
self._app_redis = redis.StrictRedis(host=redis_IP,
port=redis_port,
db=1,
socket_timeout=2)
self.logger = logging.getLogger(
"%s.%s" % (self.__module__, self.__class__.__name__))
self.logger.setLevel(50)
def __enter__(self):
self._filesystem = FileSystem(self._disk)
try:
self._filesystem.mount()
except:
# TODO: exit the program if the disk cannot be scanned
logging.warning("The disk cannot be mounted. Skip the scanning.")
sys.exit(-1)
return self
def __exit__(self, *_):
self._filesystem.umount()
def __getattr__(self, attr):
return getattr(self._filesystem, attr)
def scan(self, concurrency=1):
"""Iterates over the applications installed within the disk
and queries the CVE DB to determine whether they are vulnerable.
Concurrency controls the amount of concurrent queries
against the CVE DB.
For each vulnerable application the method yields a namedtuple:
VulnApp(name -> application name
version -> application version
vulnerabilities) -> list of Vulnerabilities
Vulnerability(id -> CVE Id
summary) -> brief description of the vulnerability
"""
self.logger.debug("Scanning FS content.")
#applications = self.applications()
#print("#####application versions: ######")
#for application in applications:
# print(application.name + " : " + application.version + " : " + application.publisher)
with ThreadPoolExecutor(max_workers=concurrency) as executor:
results = executor.map(self.query_vulnerabilities,
self.applications())
for report in results:
application, vulnerabilities = report
if vulnerabilities:
yield VulnApp(application.name, application.version,
vulnerabilities)
def query_vulnerabilities(self, application):
self.logger.debug("Quering %s vulnerabilities.", application.name)
name = application.name.lower()
version = application.version
results = []
cve_set = self._app_redis.smembers(name)
for cve_id in cve_set:
cve = json.loads(self._cve_redis.get(cve_id).decode('utf-8'))
vendor_list = cve['cve']['affects']['vendor']['vendor_data']
for vendor in vendor_list:
for product in vendor['product']['product_data']:
if product['product_name'].lower() == name:
product_versions_list = product['version'][
'version_data']
if {'version_value': version} in product_versions_list:
results.append(cve)
return application, results
def query_cve_info(self, cve_id):
# query local cve database
result = [
item['cve'] for item in self._cvefeed
if item['cve']['CVE_data_meta']['ID'] == cve_id
]
return result
def applications(self):
return (Application(a['app2_name'], a['app2_version'],
a['app2_publisher'])
for a in self._filesystem.inspect_list_applications2(
self._filesystem._root))
def test_read_and_write_file(self):
fs = FileSystem()
text_path = None
binary_path = None
unicode_text_string = u'\u016An\u012Dc\u014Dde\u033D'
hex_equivalent = '\xC5\xAA\x6E\xC4\xAD\x63\xC5\x8D\x64\x65\xCC\xBD'
malformed_text_hex = '\x4D\x69\x63\x72\x6F\x73\x6F\x66\x74\xAE\x20\x56\x69\x73\x75\x61\x6C\x20\x53\x74\x75\x64\x69\x6F\xAE\x20\x32\x30\x31\x30\x0D\x0A'
malformed_ignored_text_hex = '\x4D\x69\x63\x72\x6F\x73\x6F\x66\x74\x20\x56\x69\x73\x75\x61\x6C\x20\x53\x74\x75\x64\x69\x6F\x20\x32\x30\x31\x30\x0D\x0A'
try:
text_path = tempfile.mktemp(prefix='tree_unittest_')
binary_path = tempfile.mktemp(prefix='tree_unittest_')
fs.write_text_file(text_path, unicode_text_string)
contents = fs.read_binary_file(text_path)
self.assertEqual(contents, hex_equivalent)
fs.write_binary_file(binary_path, hex_equivalent)
text_contents = fs.read_text_file(binary_path)
self.assertEqual(text_contents, unicode_text_string)
self.assertRaises(ValueError, fs.write_text_file, binary_path, malformed_text_hex)
fs.write_binary_file(binary_path, malformed_text_hex)
self.assertRaises(ValueError, fs.read_text_file, binary_path)
text_contents = fs.read_binary_file(binary_path).decode('utf8', 'ignore')
self.assertEquals(text_contents, malformed_ignored_text_hex)
fs.open_text_file_for_reading(binary_path, 'replace').readline()
finally:
if text_path and fs.isfile(text_path):
os.remove(text_path)
if binary_path and fs.isfile(binary_path):
os.remove(binary_path)
def get_instance(args):
return FileSystem(args)
def createFilesystem(self):
return FileSystem(self)
def save(self):
print('Save')
fs = FileSystem()
fs.updateFile('finnsFile.enc', TextEditor.text, 'Student Hack',
'test1234')
print(TextEditor.text)
class TTS_CLI:
def __init__(self):
self.preferences = Preferences()
parser = argparse.ArgumentParser(
description="Manipulate Tabletop Simulator files")
parser.add_argument("-d",
"--directory",
help="Override TTS cache directory")
parser.add_argument("-l",
"--loglevel",
help="Set logging level",
choices=['debug', 'info', 'warn', 'error'])
subparsers = parser.add_subparsers(dest='parser',
title='command',
description='Valid commands.')
subparsers.required = True
# add list command
parser_list = subparsers.add_parser('list',
help="List installed mods.",
description='''
List installed mods.
If no id is provided, then this will return a list of all installed modules.
If an id is provided, then this will list the contents of that modules.
''')
group_list = parser_list.add_mutually_exclusive_group()
group_list.add_argument("-w",
"--workshop",
action="store_const",
metavar='save_type',
dest='save_type',
const=SaveType.workshop,
help="List workshop files (the default).")
group_list.add_argument("-s",
"--save",
action="store_const",
metavar='save_type',
dest='save_type',
const=SaveType.save,
help="List saves.")
group_list.add_argument("-c",
"--chest",
action="store_const",
metavar='save_type',
dest='save_type',
const=SaveType.chest,
help="List chest files.")
parser_list.add_argument("id",
nargs='?',
help="ID of specific mod to list details of.")
parser_list.set_defaults(func=self.do_list)
# export command
parser_export = subparsers.add_parser(
'export',
help="Export a mod.",
description='Export a mod in a format suitible for later import.')
group_export = parser_export.add_mutually_exclusive_group()
group_export.add_argument("-w",
"--workshop",
action="store_const",
dest='save_type',
metavar='save_type',
const=SaveType.workshop,
help="ID is of workshop file (the default).")
group_export.add_argument("-s",
"--save",
action="store_const",
dest='save_type',
metavar='save_type',
const=SaveType.save,
help="ID is of savegame file.")
group_export.add_argument("-c",
"--chest",
action="store_const",
dest='save_type',
metavar='save_type',
const=SaveType.chest,
help="ID is of chest file.")
parser_export.add_argument(
"id", help="ID of mod/name of savegame to export.")
parser_export.add_argument("-o",
"--output",
help="Location/file to export to.")
parser_export.add_argument("-f",
"--force",
action="store_true",
help="Force creation of export file.")
parser_export.add_argument(
"-d",
"--download",
action="store_true",
help="Attempt to download missing cache files. (EXPERIMENTAL)")
parser_export.set_defaults(func=self.do_export)
# import command
parser_import = subparsers.add_parser(
'import',
help="Import a mod.",
description="Import an previously exported mod.")
parser_import.add_argument("file", help="Mod pak file to import.")
parser_import.set_defaults(func=self.do_import)
# download command
parser_download = subparsers.add_parser(
'download',
help='Download mod files.',
description=
'Attempt to download any missing files for an installed mod.')
group_download = parser_download.add_mutually_exclusive_group()
group_download.add_argument("-w",
"--workshop",
action="store_const",
dest='save_type',
metavar='save_type',
const=SaveType.workshop,
help="ID is of workshop file.")
group_download.add_argument("-s",
"--save",
action="store_const",
dest='save_type',
metavar='save_type',
const=SaveType.save,
help="ID is of savegame file.")
group_download.add_argument("-c",
"--chest",
action="store_const",
dest='save_type',
metavar='save_type',
const=SaveType.chest,
help="ID is of chest file.")
group_download_target = parser_download.add_mutually_exclusive_group(
required=True)
group_download_target.add_argument("-a",
"--all",
action="store_true",
help="Download all.")
group_download_target.add_argument(
"id", nargs='?', help="ID of mod/name of savegame to download.")
parser_download.set_defaults(func=self.do_download)
# cache command
parser_cache = subparsers.add_parser('cache',
help='Work with the cache.')
subparsers_cache = parser_cache.add_subparsers(
dest='parser_cache',
title='cache_command',
description='Valid sub-commands.')
subparsers_cache.required = True
parser_cache_create = subparsers_cache.add_parser(
'create', help='(re)create cache directory')
parser_cache_create.set_defaults(func=self.do_cache_create)
# config command
parser_config = subparsers.add_parser('config',
help='Configure tts manager.')
subparsers_config = parser_config.add_subparsers(
dest='parser_config',
title='config_command',
description='Valid sub-commands.')
subparsers_config.required = True
parser_config_list = subparsers_config.add_parser(
'list', help='List configuration.')
parser_config_list.set_defaults(func=self.do_config_list)
parser_config_validate = subparsers_config.add_parser(
'validate', help='Validate configuration.')
parser_config_validate.set_defaults(func=self.do_config_validate)
parser_config_reset = subparsers_config.add_parser(
'reset', help='Reset configuration.')
parser_config_reset.set_defaults(func=self.do_config_reset)
parser_config_set = subparsers_config.add_parser(
'set', help='Set configuration parameters.')
parser_config_set.set_defaults(func=self.do_config_set)
parser_config_set.add_argument("-m",
"--mod_location",
choices=['documents', 'gamedata'],
help="Where mods are stored.")
parser_config_set.add_argument("-t",
"--tts_location",
help="TTS Install directory")
args = parser.parse_args()
# set logging
if args.loglevel:
logmap = {
'debug': logging.DEBUG,
'info': logging.INFO,
'warn': logging.WARN,
'error': logging.ERROR
}
logger().setLevel(logmap[args.loglevel])
else:
logger().setLevel(logging.WARN)
# load filesystem values
if args.directory:
self.filesystem = FileSystem(os.path.abspath(args.directory))
else:
self.filesystem = self.preferences.get_filesystem()
if (args.parser == 'list'
or args.parser == 'export') and not args.save_type:
# set default
args.save_type = SaveType.workshop
if (args.parser == 'config' and args.parser_config == 'set'
and not args.mod_location and not args.tts_location):
parser_config_set.error("At least one of -m or -t is required.")
rc, message = args.func(args)
if message:
print(message)
sys.exit(rc)
def do_config_set(self, args):
if args.mod_location:
self.preferences.locationIsUser = args.mod_location == 'documents'
if args.tts_location:
self.preferences.TTSLocation = args.mod_location
self.preferences.save()
return 0, "Preferences set"
def do_config_reset(self, args):
self.preferences.reset()
return 0, "Preferences Reset."
def do_config_list(self, args):
return 0, self.preferences
def do_config_validate(self, args):
if self.preferences.validate():
return 0, "Configuration validated OK."
else:
return 1, "Configuration failed to validate."
def do_cache_create(self, args):
try:
self.filesystem.create_dirs()
except OSError as exception:
return 1, "OS error: {0}".format(exception)
return 0, "All directories created OK."
def list_by_type(self, save_type):
result = ""
for (name, id) in describe_files_by_type(self.filesystem, save_type):
result += "\n%s (%s)" % (name, id)
return 0, result
def list_item(self, data, filename, ident):
if not data:
# self.list_installed()
return
save = Save(savedata=data,
ident=ident,
filename=filename,
filesystem=self.filesystem)
return 0, save
def do_download(self, args):
successful = True
if not args.all:
if not args.save_type:
args.save_type = self.filesystem.get_json_filename_type(
args.id)
if not args.save_type:
return 1, "Unable to determine type of id %s" % args.id
successful = download_file(self.filesystem, args.id,
args.save_type)
else:
if args.save_type:
for ident in self.filesystem.get_filenames_by_type(
args.save_type):
if not download_file(self.filesystem, ident,
args.save_type):
successful = False
break
else:
for save_type in SaveType:
for ident in self.filesystem.get_filenames_by_type(
save_type):
if not download_file(self.filesystem, ident,
save_type):
successful = False
break
if successful:
return 0, "All files downloaded."
else:
return 1, "Some files failed to download."
def do_list(self, args):
rc = 0
result = None
if not args.id:
rc, result = self.list_by_type(args.save_type)
else:
if not args.save_type:
args.save_type = self.filesystem.get_json_filename_type(
args.id)
if not args.save_type:
return 1, "Unable to determine type of id %s" % args.id
filename = self.filesystem.get_json_filename_for_type(
args.id, args.save_type)
data = load_json_file(filename)
rc, result = self.list_item(data, filename, args.id)
return rc, result
def do_export(self, args):
filename = None
if args.output:
if os.path.isdir(args.output):
filename = os.path.join(args.output, args.id + ".pak")
else:
filename = args.output
else:
filename = args.id + ".pak"
data = None
json_filename = None
if not args.save_type:
args.save_type = self.filesystem.get_json_filename_type(args.id)
if not args.save_type:
return 1, "Unable to determine type of id %s" % args.id
json_filename = self.filesystem.get_json_filename_for_type(
args.id, args.save_type)
if not json_filename:
return 1, "Unable to find filename for id %s (wrong -s/-w/-c specified?)" % args.id
data = load_json_file(json_filename)
if not data:
return 1, "Unable to load data for file %s" % json_filename
save = Save(savedata=data,
filename=json_filename,
ident=args.id,
save_type=args.save_type,
filesystem=self.filesystem)
if not save.isInstalled:
if not args.download:
return 1, "Unable to find all urls required by %s. Rerun with -d to try and download them or open it within TTS.\n%s" % (
args.id, save)
else:
logger().info("Downloading missing files...")
successful = save.download()
if successful:
logger().info("Files downloaded successfully.")
else:
return 1, "Some files failed to download"
if os.path.isfile(filename) and not args.force:
return 1, "%s already exists. Please specify another file or use '-f'" % filename
logger().info("Exporting json file %s to %s" % (args.id, filename))
save.export(filename)
# TODO: exception handling
return 0, "Exported %s to %s" % (args.id, filename)
def do_import(self, args):
return 0, save_helper.importPak(self.filesystem, args.file)
def __init__(self):
self.preferences = Preferences()
parser = argparse.ArgumentParser(
description="Manipulate Tabletop Simulator files")
parser.add_argument("-d",
"--directory",
help="Override TTS cache directory")
parser.add_argument("-l",
"--loglevel",
help="Set logging level",
choices=['debug', 'info', 'warn', 'error'])
subparsers = parser.add_subparsers(dest='parser',
title='command',
description='Valid commands.')
subparsers.required = True
# add list command
parser_list = subparsers.add_parser('list',
help="List installed mods.",
description='''
List installed mods.
If no id is provided, then this will return a list of all installed modules.
If an id is provided, then this will list the contents of that modules.
''')
group_list = parser_list.add_mutually_exclusive_group()
group_list.add_argument("-w",
"--workshop",
action="store_const",
metavar='save_type',
dest='save_type',
const=SaveType.workshop,
help="List workshop files (the default).")
group_list.add_argument("-s",
"--save",
action="store_const",
metavar='save_type',
dest='save_type',
const=SaveType.save,
help="List saves.")
group_list.add_argument("-c",
"--chest",
action="store_const",
metavar='save_type',
dest='save_type',
const=SaveType.chest,
help="List chest files.")
parser_list.add_argument("id",
nargs='?',
help="ID of specific mod to list details of.")
parser_list.set_defaults(func=self.do_list)
# export command
parser_export = subparsers.add_parser(
'export',
help="Export a mod.",
description='Export a mod in a format suitible for later import.')
group_export = parser_export.add_mutually_exclusive_group()
group_export.add_argument("-w",
"--workshop",
action="store_const",
dest='save_type',
metavar='save_type',
const=SaveType.workshop,
help="ID is of workshop file (the default).")
group_export.add_argument("-s",
"--save",
action="store_const",
dest='save_type',
metavar='save_type',
const=SaveType.save,
help="ID is of savegame file.")
group_export.add_argument("-c",
"--chest",
action="store_const",
dest='save_type',
metavar='save_type',
const=SaveType.chest,
help="ID is of chest file.")
parser_export.add_argument(
"id", help="ID of mod/name of savegame to export.")
parser_export.add_argument("-o",
"--output",
help="Location/file to export to.")
parser_export.add_argument("-f",
"--force",
action="store_true",
help="Force creation of export file.")
parser_export.add_argument(
"-d",
"--download",
action="store_true",
help="Attempt to download missing cache files. (EXPERIMENTAL)")
parser_export.set_defaults(func=self.do_export)
# import command
parser_import = subparsers.add_parser(
'import',
help="Import a mod.",
description="Import an previously exported mod.")
parser_import.add_argument("file", help="Mod pak file to import.")
parser_import.set_defaults(func=self.do_import)
# download command
parser_download = subparsers.add_parser(
'download',
help='Download mod files.',
description=
'Attempt to download any missing files for an installed mod.')
group_download = parser_download.add_mutually_exclusive_group()
group_download.add_argument("-w",
"--workshop",
action="store_const",
dest='save_type',
metavar='save_type',
const=SaveType.workshop,
help="ID is of workshop file.")
group_download.add_argument("-s",
"--save",
action="store_const",
dest='save_type',
metavar='save_type',
const=SaveType.save,
help="ID is of savegame file.")
group_download.add_argument("-c",
"--chest",
action="store_const",
dest='save_type',
metavar='save_type',
const=SaveType.chest,
help="ID is of chest file.")
group_download_target = parser_download.add_mutually_exclusive_group(
required=True)
group_download_target.add_argument("-a",
"--all",
action="store_true",
help="Download all.")
group_download_target.add_argument(
"id", nargs='?', help="ID of mod/name of savegame to download.")
parser_download.set_defaults(func=self.do_download)
# cache command
parser_cache = subparsers.add_parser('cache',
help='Work with the cache.')
subparsers_cache = parser_cache.add_subparsers(
dest='parser_cache',
title='cache_command',
description='Valid sub-commands.')
subparsers_cache.required = True
parser_cache_create = subparsers_cache.add_parser(
'create', help='(re)create cache directory')
parser_cache_create.set_defaults(func=self.do_cache_create)
# config command
parser_config = subparsers.add_parser('config',
help='Configure tts manager.')
subparsers_config = parser_config.add_subparsers(
dest='parser_config',
title='config_command',
description='Valid sub-commands.')
subparsers_config.required = True
parser_config_list = subparsers_config.add_parser(
'list', help='List configuration.')
parser_config_list.set_defaults(func=self.do_config_list)
parser_config_validate = subparsers_config.add_parser(
'validate', help='Validate configuration.')
parser_config_validate.set_defaults(func=self.do_config_validate)
parser_config_reset = subparsers_config.add_parser(
'reset', help='Reset configuration.')
parser_config_reset.set_defaults(func=self.do_config_reset)
parser_config_set = subparsers_config.add_parser(
'set', help='Set configuration parameters.')
parser_config_set.set_defaults(func=self.do_config_set)
parser_config_set.add_argument("-m",
"--mod_location",
choices=['documents', 'gamedata'],
help="Where mods are stored.")
parser_config_set.add_argument("-t",
"--tts_location",
help="TTS Install directory")
args = parser.parse_args()
# set logging
if args.loglevel:
logmap = {
'debug': logging.DEBUG,
'info': logging.INFO,
'warn': logging.WARN,
'error': logging.ERROR
}
logger().setLevel(logmap[args.loglevel])
else:
logger().setLevel(logging.WARN)
# load filesystem values
if args.directory:
self.filesystem = FileSystem(os.path.abspath(args.directory))
else:
self.filesystem = self.preferences.get_filesystem()
if (args.parser == 'list'
or args.parser == 'export') and not args.save_type:
# set default
args.save_type = SaveType.workshop
if (args.parser == 'config' and args.parser_config == 'set'
and not args.mod_location and not args.tts_location):
parser_config_set.error("At least one of -m or -t is required.")
rc, message = args.func(args)
if message:
print(message)
sys.exit(rc)
def get_filesystem(self):
if self.locationIsUser:
return get_default_fs()
return FileSystem(tts_install_path=self.TTSLocation)
def test_exists__true(self):
fs = FileSystem()
self.assertTrue(fs.exists(self._this_file))
def signals(self):
return map(
lambda file: FileSignal(self.directory + file, FileSystem.instance(
)).value(), os.listdir(self.directory))
def test_exists__false(self):
fs = FileSystem()
self.assertFalse(fs.exists(self._missing_file))
def __init__(self, disk):
self.filesystem = FileSystem(disk)
def test_getcwd(self):
fs = FileSystem()
self.assertTrue(fs.exists(fs.getcwd()))
def test_sep(self):
fs = FileSystem()
self.assertEqual(fs.sep, os.sep)
self.assertEqual(fs.join("foo", "bar"),
os.path.join("foo", "bar"))
def test_isdir__true(self):
fs = FileSystem()
self.assertTrue(fs.isdir(self._this_dir))
def test_isdir__false(self):
fs = FileSystem()
self.assertFalse(fs.isdir(self._this_file))
def test_join(self):
fs = FileSystem()
self.assertEqual(fs.join('foo', 'bar'),
os.path.join('foo', 'bar'))
def test_exists__true(self):
fs = FileSystem()
self.assertTrue(fs.exists(self._this_file))
def test_maybe_make_directory__success(self):
fs = FileSystem()
with fs.mkdtemp(prefix='filesystem_unittest_') as base_path:
sub_path = os.path.join(base_path, "newdir")
self.assertFalse(os.path.exists(sub_path))
self.assertFalse(fs.isdir(sub_path))
fs.maybe_make_directory(sub_path)
self.assertTrue(os.path.exists(sub_path))
self.assertTrue(fs.isdir(sub_path))
# Make sure we can re-create it.
fs.maybe_make_directory(sub_path)
self.assertTrue(os.path.exists(sub_path))
self.assertTrue(fs.isdir(sub_path))
# Clean up.
os.rmdir(sub_path)
self.assertFalse(os.path.exists(base_path))
self.assertFalse(fs.isdir(base_path))
def test_isdir__true(self):
fs = FileSystem()
self.assertTrue(fs.isdir(self._this_dir))
def test_read_text_file__missing(self):
fs = FileSystem()
self.assertRaises(IOError, fs.read_text_file, self._missing_file)
def test_join(self):
fs = FileSystem()
self.assertEqual(fs.join('foo', 'bar'),
os.path.join('foo', 'bar'))
def test_sep(self):
fs = FileSystem()
self.assertEquals(fs.sep, os.sep)
self.assertEquals(fs.join("foo", "bar"),
os.path.join("foo", "bar"))
def test_chdir__notexists(self):
fs = FileSystem()
newdir = '/dirdoesnotexist'
if sys.platform == 'win32':
newdir = 'c:\\dirdoesnotexist'
self.assertRaises(OSError, fs.chdir, newdir)
def load(self, path, filename):
LoadSaveDialog.file_to_load = filename
#with open(os.path.join(path, filename[0])) as stream:
#LoadSaveDialog.string= stream.read()
ext = filename[0].split('.')[-1]
opt = 'rb' if ext == 'enc' else 'r'
file = open(filename[0], opt)
LoadSaveDialog.string = file.read()
file.close()
TextEditor.choose_lexer(LoadSaveDialog.file_to_load[0])
print(path, filename)
# ask the user for organisation name and password
if (cache['organisation'] != None):
# the user is logged in to an org
organisation = cache['organisation']
password = cache['password']
print(filename)
fs = FileSystem()
ext = filename[0].split('.')[-1]
if ext != 'enc':
# the file needs to be encrypted first
print('here3')
fs.updateFile(filename[0] + '.enc', LoadSaveDialog.string,
organisation, password)
TextEditor.updateCodeInput(LoadSaveDialog.string)
TextEditor.currentFile = filename[0] + '.enc'
else:
try:
print(fs.readFile(filename[0], organisation, password))
TextEditor.codeinput.text = fs.readFile(
filename[0], organisation, password)
TextEditor.currentFile = filename[0]
except ValueError:
# decryption failed
popup_msg = "Credentials invalid for the selected file!"
layout = BoxLayout(orientation='vertical')
popup_title = "Error"
label1 = Label(text=popup_title)
label = Label(text=popup_msg)
button = Button(text='Dismiss')
layout.add_widget(label1)
layout.add_widget(label)
layout.add_widget(button)
popup = Popup(title=popup_title,
content=layout,
size_hint=(0.6, 0.6))
popup.open()
button.bind(on_press=popup.dismiss)
else:
# the user needs to be prompted to login first
popup_msg = "Please login to an organisation first!"
layout = BoxLayout(orientation='vertical')
popup_title = "Error"
label1 = Label(text=popup_title)
label = Label(text=popup_msg)
button = Button(text='Dismiss')
layout.add_widget(label1)
layout.add_widget(label)
layout.add_widget(button)
popup = Popup(title=popup_title,
content=layout,
size_hint=(0.6, 0.6))
popup.open()
button.bind(on_press=popup.dismiss)
self.dismiss_popup()