def new_service(self, directory, name, desc):
try:
xml_data_object = XMLServiceDefinition.get_service_files(directory)
except:
print(
'[WARN] The service you tried to initilize has errors (check iii.xml)!'
)
raise Exception('New service files incorretcly configured')
self.service_version = Cryptographer.generate_hash(
et.tostring(xml_data_object).decode('utf8'))
self.service_xml_object = XMLServiceDefinition.create_new_service(
xml_data_object, name, desc)
def create_new_service(files, name, desc):
version_hash = Cryptographer.generate_hash(
et.tostring(files).decode('utf8'))
service = et.Element('service', version=version_hash, counter='0')
description = et.SubElement(service, 'desc', name=name)
description.text = desc
data = et.SubElement(service, 'data')
data.append(files)
tags = et.SubElement(service, 'tags')
return service
def __init__(self,
directory,
uaddress=None,
new_service=False,
name=None,
desc=None):
self.buffer = []
self.status = False
self.display = False
self.directory = directory
self.iii_dir = directory + '/.iii'
self.iii_xml_dir = directory + '/.iii/iii.xml'
self.iii_schema_dir = directory + '/.iii/iii.xsd'
self.iii_uaddress_dir = directory + '/.iii/uaddress.txt'
self.iii_start_dir_windows = directory + '/iiistart.bat'
self.iii_start_dir_posix = directory + '/iiistart.sh'
#check if required files and diretcories exist
iii_exists = os.path.isdir(self.iii_dir)
iii_xml_exists = os.path.isfile(self.iii_xml_dir)
iii_schema_exists = os.path.isfile(self.iii_schema_dir)
iii_uaddress_exists = os.path.isfile(self.iii_uaddress_dir)
iii_start_windows_exists = os.path.isfile(self.iii_start_dir_windows)
iii_start_posix_exists = os.path.isfile(self.iii_start_dir_posix)
if (iii_exists and iii_xml_exists and iii_schema_exists):
if (iii_start_windows_exists and iii_start_posix_exists):
self.os_type = 'all'
elif (iii_start_posix_exists or iii_start_windows_exists):
if iii_start_posix_exists:
if os_name == 'nt':
print(
'[ERROR] No POSIX start file found (you are running on a GNU/Linux or Unix machine)'
)
raise Exception('No POSIX start file found')
self.os_type = 'posix'
elif iii_start_windows_exists:
if os_name != 'nt':
print(
'[ERROR] No Windows start file found (you are running on a Windows machine)'
)
raise Exception('No Windows start file found')
self.os_type = 'nt'
else:
print(
'[ERROR] Service directory does not contain any start files!'
)
raise Exception(
'Service directory does not contain any start files')
else:
print(
'[ERROR] Service directory does not contain proper defining files (check .iii folder)'
)
raise Exception(
'Service directory does not conatin proper defining files')
#check data if new service
if new_service == True:
self.new_service(directory, name, desc)
else:
try:
temp_xml_data = XMLServiceDefinition.get_service_files(
directory)
self.service_version = Cryptographer.generate_hash(
et.tostring(temp_xml_data).decode('utf8'))
except:
print('[WARN] The service III tried to initilize has errors!')
raise Exception('Service files incorretcly configured')
#parse iii.xml to ensure it passes schema and that all db files and directories defined exist
with open(self.iii_xml_dir, 'r') as xml_file:
xml_string = xml_file.read()
self.variable_files = XMLServiceDefinition.parse_xml_string(
xml_string, self.directory)
if self.variable_files == False:
print(
'[ERROR] Service definitions do not correlate to files (check iii.xml)'
)
raise Exception('Service definitions do not correlate to files')
#check if uaddress file needs to be created
if iii_uaddress_exists:
pass
else:
with open(self.iii_uaddress_dir, 'w') as uaddress_file:
uaddress_file.write(uaddress)
uaddress_file.close()
#initialize address and name variables from uaddress
with open(self.iii_uaddress_dir, 'r') as uaddress_file:
uaddress_text = uaddress_file.read()
uaddress_split = uaddress_text.split('.', 1)
#add a thing to verify address follows correct format
if len(uaddress_split) != 2:
print('[ERROR] Service uaddress file is malformed!')
raise Exception('Uaddress file is malformed')
else:
self.address = uaddress_split[0]
self.name = uaddress_split[1]
self.uaddress = uaddress_text
#set start file based on current os
if os_name == 'nt':
self.iii_service_start_path = os.path.abspath(
self.iii_start_dir_windows)
else:
self.iii_service_start_path = os.path.abspath(
self.iii_start_dir_posix)
def start():
global process_handler
process_handler = multiprocessing.Process(target=ProcessHandler.main)
global processes
processes = []
print('Starting the Interweb Information Index Server (III)!')
print('-----------------------------------------------------')
print('Created by Ismaeel Mian (1creeperbomb)')
print('-----------------------------------------------------')
print(
'Check out/contribute the the project on github at https://github.com/1creeperbomb/InterwebInformationIndex'
)
print(
'----------------------------------------------------------------------------------------------------------'
)
#check if private key file exists
if os.path.isfile('keystore/private.key'):
print('Private key found! Starting normal operation!')
else:
print(
'No private key files was found. If you are importing a key, make sure the file is named \"private.key\" and is located directly in the \"keystore\" folder'
)
start_setup = filter_input(
input('Would you like to setup a new keypair? (Y/N): '), 'y/n')
while filter_input(start_setup, 'y/n') == False:
print('Please enter a valid choice')
start_setup == input(
'Would you like to setup a new keypair? (Y/N): ')
if start_setup == 'y':
first_time_setup()
elif start_setup == 'n':
print('You chose no')
shutdown(None)
password = getpass('Please enter your password to start: ')
try:
print('Attempting to decrypt...')
private_key = Cryptographer.read_key(password)
print('Succesfully decrypted!')
except:
print('Failed to decrypt!')
shutdown(None)
global crypto_main
crypto_main = Cryptographer(private_key, False)
password = None
private_key = None
print('-----------------------------------------------------')
#start services
print('[INFO] Attempting to start iii sub processes...')
process_handler.start()
#processes.append(process_handler)
time.sleep(4)
def verify_definition(dir, version, counter, name, address):
version_xpath = '/root/master[address[text()=\"' + address + '\"]]/services/service[desc[@name = \"' + name + '\"]]/@version'
counter_xpath = '/root/master[address[text()=\"' + address + '\"]]/services/service[desc[@name = \"' + name + '\"]]/@counter'
data_xpath = '/root/master[address[text()=\"' + address + '\"]]/services/service[desc[@name = \"' + name + '\"]]/data/files'
files = XMLIndex.get_data(data_xpath)
index_version = XMLIndex.get_data(version_xpath)
index_counter = XMLIndex.get_data(counter_xpath)
if index_version[0] != version or index_counter != counter:
return False
static_files = []
var_files = []
for file in files:
if file.get('type') == 'static':
s_file = [file.get('rdir'), file.text]
static_files.append(s_file)
elif file.get('type') == 'variable':
v_file = file.get('rdir')
var_files.append(v_file)
#verify file hashes and matches
root_directory = glob.glob(dir + '\\**\\*', recursive=True)
file_list = []
for file in root_directory:
new_path = os.path.relpath(os.path.realpath(file))
file_list.append(new_path)
for file in static_files:
if file[0] in file_list:
if os.path.isfile(file):
if file[1] == Cryptographer.generate_hash(message=None,
filepath=file):
file_list.remove(file)
if os.path.isdir(file):
file_list.remove(file)
var_dirs = []
for file in var_files:
if file in file_list:
if os.path.isfile(file):
file_list.remove(file)
if os.path.isdir(file):
var_dirs.append(file)
file_list.remove(file)
#check/ignore any files in variable directory
for file in file_list:
for var_dir in var_dirs:
var_dir_path = Path(var_dir)
dir_path = Path(file)
if var_dir_path in dir_path.parents:
file_list.remove(file)
if len(file_list) != 0:
print('[ERROR] Files in service do not match service definition')
return False
def load_definition(dir):
iii_dir = dir + '/.iii'
iii_xml_dir = dir + '/.iii/iii.xml'
iii_schema_dir = dir + 'debug/iii2.xsd'
try:
#try opening definition and verify against schema
with open(iii_schema_dir, 'r') as schema_file:
schema_raw = et.XML(schema_file.read())
schema_f = et.XMLSchema(schema_raw)
parser = et.XMLParser(schema=schema_f,
remove_comments=True,
remove_blank_text=True)
with open(iii_dir, 'r') as service_def:
iii_root = et.XML(service_def.read(), parser)
#get vars
service = iii_root[0]
static_files = []
var_files = []
for child in service:
if child.tag == 'desc':
description = child.text
name = child.get('name')
elif child.tag == 'address':
address = child.text
elif child.tag == 'data':
version_hash = Cryptographer.generate_hash(
et.tostring(child).decode('utf8'))
for schild in child:
if schild.tag == 'files':
xml_files = schild
elif schild.tag == 'dependencies':
dependencies = []
for source in schild:
depend = [
source.get('type'),
source.get('name'), source.text
]
dependencies.append(depend)
elif schild.tag == 'tags':
tags = []
for tag in schild:
if tag.tag == 'application':
service_type = [tag.tag, tag.get('os')]
tags.append(service_type)
elif tag.tag == 'resource':
service_type = [tag.tag]
tags.append(service_type)
elif tag.tag == 'DELETE':
delete = True
#add files to lists
for file in xml_files:
if file.get('type') == 'static':
s_file = [file.get('rdir'), file.text]
static_files.append(s_file)
elif file.get('type') == 'variable':
v_file = file.get('rdir')
var_files.append(v_file)
#verify file hashes and matches
root_directory = glob.glob(iii_dir + '\\**\\*', recursive=True)
file_list = []
for file in root_directory:
new_path = os.path.relpath(os.path.realpath(file))
file_list.append(new_path)
for file in static_files:
if file[0] in file_list:
if os.path.isfile(file):
if file[1] == Cryptographer.generate_hash(
message=None, filepath=file):
file_list.remove(file)
if os.path.isdir(file):
file_list.remove(file)
var_dirs = []
for file in var_files:
if file in file_list:
if os.path.isfile(file):
file_list.remove(file)
if os.path.isdir(file):
var_dirs.append(file)
file_list.remove(file)
#check/ignore any files in variable directory
for file in file_list:
for var_dir in var_dirs:
var_dir_path = Path(var_dir)
dir_path = Path(file)
if var_dir_path in dir_path.parents:
file_list.remove(file)
if len(file_list) != 0:
print(
'[ERROR] Files in service do not match service definition')
return False
#verify version hash
if version_hash != service.get('version'):
print('[ERROR] Definition version does not match data hash')
return False
counter = service.get('counter')
#return service data in a list
#dir, version, count, name, description, address, dependencies, tags [OS, type, etc], delete command
return [
dir, version_hash, counter, name, description, address,
dependencies, tags, delete
]
except:
print(
'[ERROR] Service defnition failed to parse (are .iii files ok?)'
)
return False
def generate_new_definition(name, desc, dir, type, addr):
#create xml
root = et.Element('def')
#xml data
service = et.SubElement(root, 'service')
description = et.SubElement(service, 'desc')
description.text = desc
description.set('name', name)
address = et.SubElement(service, 'address')
address.text = addr
data = et.SubElement(service, 'data')
files = et.SubElement(data, 'files')
#setup default files list
root_directory = glob.glob(dir + '\\**\\*', recursive=True)
file_list = []
for file in root_directory:
new_path = os.path.relpath(os.path.realpath(file))
file_list.append(new_path)
for file in file_list:
if os.path.isfile(file):
file_el = et.SubElement(files, 'file')
file_el.set('rdir', os.path.relpath(file, start=dir))
file_el.set('type', 'static')
file_hash = Cryptographer.generate_hash(message=None,
filepath=file)
file_el.text = file_hash
else:
file_el = et.SubElement(files, 'file')
file_el.set('rdir', os.path.relpath(file, start=dir))
file_el.set('type', 'static')
file_el.text = '0'
depend = et.SubElement(data, 'dependencies')
tags = et.SubElement(data, 'tags')
if type == 'resource':
resource = et.SubElement(tags, 'resource')
else:
application = et.SubElement(tags, 'application')
application.set('os', type)
version_hash = Cryptographer.generate_hash(
et.tostring(data).decode('utf8'))
service.set('version', version_hash)
service.set('counter', '1')
#create files
folder = os.path.join(dir, '.iii')
os.mkdir(folder)
copyfile('debug/iii2.xsd', os.path.join(folder, 'iii.xsd'))
#change and replace iii.xsd with iii2.xsd later
with open(os.path.join(dir, '.iii', 'iii.xsd'), 'r') as schema_file:
schema_raw = et.XML(schema_file.read())
schema = et.XMLSchema(schema_raw)
parser = et.XMLParser(schema=schema)
#print(et.tostring(root))
try:
root_check = et.fromstring(
et.tostring(root).decode('utf8'), parser)
except:
print('Schema validation failed!')
tree = et.ElementTree(root)
tree.write(os.path.join(dir, '.iii', 'iii.xml'), pretty_print=True)
def get_service_files(directory):
directory = directory.replace('services/', '')
glob_path = 'services\\' + directory + '\\**\\*'
start_path = 'services/' + directory
root_directory_raw = glob.glob(glob_path, recursive=True)
#convert glob directories into OS friendly directories
root_directory = []
for path in root_directory_raw:
new_path = os.path.relpath(os.path.realpath(path))
root_directory.append(new_path)
files = et.Element('files')
#get all variable files
iii_xml_dir = start_path + '/.iii/iii.xml'
with open(iii_xml_dir, 'r') as xml_file:
db_data = xml_file.read(
) #maybe improve to read larger files in the future?
variable_files_raw = XMLServiceDefinition.parse_xml_string(
db_data, start_path)
if variable_files_raw == False:
print(
'[WARN] The service you have created has files that do not match with the iii.xml record'
)
raise Exception('Invalid service folder')
#normailzie paths in variable files list
variable_files = []
for file in variable_files_raw:
file = os.path.normpath(file)
variable_files.append(file)
for p_file in root_directory:
p_file = os.path.normpath(p_file)
print(p_file)
if p_file in variable_files:
new_iii_path = os.path.relpath(p_file, start=start_path)
file = et.SubElement(files,
'file',
rdir=new_iii_path,
type='variable')
file.text = '0'
elif os.path.isfile(p_file):
p_hash = Cryptographer.generate_hash(message=None,
filepath=p_file)
new_iii_path = os.path.relpath(p_file, start=start_path)
file = et.SubElement(files,
'file',
rdir=new_iii_path,
type='static')
file.text = p_hash
elif os.path.isdir(p_file):
new_iii_path = os.path.relpath(p_file, start=start_path)
file = et.SubElement(files,
'file',
rdir=new_iii_path,
type='static')
file.text = '0'
return files
def verify_service(service_address, service_name, service_dir):
#/root/master[address[text()="WzJmdiSCSxk5dnT6P65UhDyNdjBnBy5E3fDxigWOHCs="]]/services/service[desc[@name = "woot"]]/data/files
xpath = '/root/master[address[text()=\"' + service_address + '\"]]/services/service[desc[@name = \"' + service_name + '\"]]/data/files'
#print(xpath)
files = XMLIndex.get_data(xpath)
service_dir = service_dir.replace('services/', '')
glob_path = 'services\\' + service_dir + '\\**\\*'
root_directory_raw = glob.glob(glob_path, recursive=True)
root_directory = []
#convert glob directories into OS friendly directories
for path in root_directory_raw:
new_path = os.path.relpath(os.path.realpath(path))
root_directory.append(new_path)
#verify files
remaining_p_files = root_directory
remaining_i_files = files
for file in files:
if file.attrib['type'] == 'static':
file_hash = file.text
file_name = file.attrib['rdir']
for p_file in root_directory:
if os.path.isfile(p_file):
p_hash = Cryptographer.generate_hash(message=None,
filepath=p_file)
if file_hash == p_hash and file_name == p_file:
remaining_p_files.remove(p_file)
remaining_i_files.remove(file)
else:
if file_name == p_file:
remaining_p_files.remove(p_file)
remaining_i_files.remove(file)
elif file.attrib['type'] == 'variable':
file_name = file.attrib['rdir']
for p_file in root_directory:
if file_name == p_file:
remaining_p_files.remove(p_file)
remaining_i_files.remove(file)
if len(remaining_i_files) != 0 or len(remainingremaining_p_files) != 0:
return False
#(Double check) Verify service version hash
version_hash = Cryptographer.generate_hash(
et.tostring(files).decode('utf8'))
iii_version = xpath = '/root/master[address[text()=\"' + service_address + '\"]]/services/service[desc[@name = \"' + service_name + '\"]]/@version'
if version_hash != iii_version[0]:
return False
return True
def parse_xml_string(xml_data_raw):
#all elements must have xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance in the root for the signature verification to work
#read the xml schema and create a schema object
with open('index-schema.xsd', 'r') as schema_file:
schema_raw = et.XML(schema_file.read())
schema = et.XMLSchema(schema_raw)
#add data to a root element
root = et.Element("root")
root.append(et.XML(xml_data_raw))
xml_data = et.tostring(root)
parser = et.XMLParser(schema=schema)
try:
root = et.fromstring(xml_data, parser)
except:
print('Schema validation failed!')
return False
#verify address
root = et.fromstring(xml_data_raw)
#check iii data version
if root.attrib['iiiVersion'] != '1':
print('Node is not from this version and not supported')
return False
node_type = root.tag
services = []
for child in root:
if child.tag == 'address':
address = child.text
elif child.tag == 'sign':
signature = child.text
salt = child.attrib['salt']
elif child.tag == 'services':
data = et.tostring(child).decode('utf8')
for service in child:
services.append(service)
try:
crypto = Cryptographer(address, True)
if crypto.verify_data(data, signature, salt):
print('Signature verified successfully!')
else:
print('Signature failed to verify')
return False
except:
print('Invalid key / key is corrupt')
return False
services_copy = services.copy()
#verify counter is latest
check = XMLIndex.get_data(node_type, address)[0]
if check != None:
for child in check:
if child.tag == 'services':
for check_service in child:
check_version = check_service.attrib['version']
check_counter = int(check_service.attrib['counter'])
for service in services_copy:
version = check_service.attrib['version']
counter = int(service.attrib['counter'])
if check_version == version:
if counter > check_counter:
services_copy.remove(service)
if len(services_copy) != 0:
print('Service counters do not match!')
return False
#parse any special tags
for service in services:
for child in service:
if child.tag == 'tags':
for tag in child:
if tag.tag == 'DELETE':
service.getparent().remove(service)
#send data to write method and socket sever
XMLIndex.__write_xml(
et.tostring(root).decode('utf8'), address, node_type)
return True