def read_yaml_text(self):
'''
Gets the data from YAML files or datastore
'''
if self.read_from_yaml_file:
print " Object interface generator: Reading object definitions from files"
data_yaml_files = list_files_recursive(
'obj/data', '*.yml', ['ion.yml', 'resource.yml', 'shared.yml'])
self.data_yaml_text = '\n\n'.join(
(file.read() for file in (open(path, 'r')
for path in data_yaml_files
if os.path.exists(path))))
service_yaml_files = list_files_recursive('obj/services', '*.yml')
service_yaml_text = '\n\n'.join(
(file.read() for file in (open(path, 'r')
for path in service_yaml_files
if os.path.exists(path))))
data = self.data_yaml_text + "\n" + service_yaml_text
else:
print " Object interface generator: Reading object definitions from datastore"
self.data_yaml_text = get_object_definition_from_datastore(
self.system_name)
if not self.data_yaml_text:
return ''
data = self.data_yaml_text + '\n' + get_service_definition_from_datastore(
self.system_name)
return data
def read_yaml_text(self):
data_yaml_files = list_files_recursive('obj/data', '*.yml', ['ion.yml', 'resource.yml','shared.yml'])
self.data_yaml_text = '\n\n'.join((file.read() for file in (open(path, 'r') for path in data_yaml_files if os.path.exists(path))))
service_yaml_files = list_files_recursive('obj/services', '*.yml')
service_yaml_text = '\n\n'.join((file.read() for file in (open(path, 'r') for path in service_yaml_files if os.path.exists(path))))
combined_yaml_text = self.data_yaml_text + "\n" + service_yaml_text
return combined_yaml_text
def get_object_definition(self):
if self.read_from_yaml_file:
data_yaml_files = list_files_recursive('obj/data', '*.yml', ['ion.yml', 'resource.yml', 'shared.yml'])
data = '\n\n'.join((file.read() for file in(open(path, 'r') for path in data_yaml_files if os.path.exists(path))))
else:
data = get_object_definition_from_datastore(self.system_name)
return data
def get_object_definition(self):
if self.read_from_yaml_file:
data_yaml_files = list_files_recursive('obj/data', '*.yml', ['ion.yml', 'resource.yml', 'shared.yml'])
data = '\n\n'.join((file.read() for file in(open(path, 'r') for path in data_yaml_files if os.path.exists(path))))
else:
data = get_object_definition_from_datastore(self.system_name)
return data
def read_yaml_text(self):
data_yaml_files = list_files_recursive(
'obj/data', '*.yml', ['ion.yml', 'resource.yml', 'shared.yml'])
self.data_yaml_text = '\n\n'.join(
(file.read() for file in (open(path, 'r')
for path in data_yaml_files
if os.path.exists(path))))
service_yaml_files = list_files_recursive('obj/services', '*.yml')
service_yaml_text = '\n\n'.join(
(file.read() for file in (open(path, 'r')
for path in service_yaml_files
if os.path.exists(path))))
combined_yaml_text = self.data_yaml_text + "\n" + service_yaml_text
return combined_yaml_text
def get_service_definition(self):
if self.read_from_yaml_file:
print " Service interface generator: reading service definitions from files"
service_yaml_files = list_files_recursive('obj/services', '*.yml')
data = '\n\n'.join((file.read() for file in(open(path, 'r') for path in service_yaml_files if os.path.exists(path))))
else:
print " Service interface generator: reading service definitions from datastore"
data = get_service_definition_from_datastore(self.system_name)
return data
def get_service_definition(self):
if self.read_from_yaml_file:
print " Service interface generator: reading service definitions from files"
service_yaml_files = list_files_recursive('obj/services', '*.yml')
data = '\n\n'.join((file.read() for file in(open(path, 'r') for path in service_yaml_files if os.path.exists(path))))
else:
print " Service interface generator: reading service definitions from datastore"
data = get_service_definition_from_datastore(self.system_name)
return data
def read_yaml_text(self):
'''
Gets the data from YAML files or datastore
'''
if self.read_from_yaml_file:
print " Object interface generator: Reading object definitions from files"
data_yaml_files = list_files_recursive('obj/data', '*.yml', ['ion.yml', 'resource.yml', 'shared.yml'])
self.data_yaml_text = '\n\n'.join((file.read() for file in(open(path, 'r') for path in data_yaml_files if os.path.exists(path))))
service_yaml_files = list_files_recursive('obj/services', '*.yml')
service_yaml_text = '\n\n'.join((file.read() for file in(open(path, 'r') for path in service_yaml_files if os.path.exists(path))))
data = self.data_yaml_text + "\n" + service_yaml_text
else:
print " Object interface generator: Reading object definitions from datastore"
self.data_yaml_text = get_object_definition_from_datastore(self.system_name)
if not self.data_yaml_text:
return ''
data = self.data_yaml_text + '\n' + get_service_definition_from_datastore(self.system_name)
return data
def store_object_interfaces(self, file=None):
#print "\nStoring object interfaces in datastore..."
if file and os.path.exists(file):
self._load_object_files([file])
elif file:
print "store_interfaces: Error couldn't find the file path\n"
else:
data_yaml_filenames = list_files_recursive('obj/data', '*.yml', ['ion.yml', 'resource.yml', 'shared.yml'])
self._load_object_files(data_yaml_filenames)
def store_service_interfaces(self, file=None):
#print "\nStoring service interfaces in datastore..."
if file and os.path.exists(file):
self._load_service_files([file])
elif file:
print "store_interfaces: Error couldn't find the file path\n"
else:
service_yaml_filenames = list_files_recursive('obj/services', '*.yml')
self._load_service_files(service_yaml_filenames)
def store_service_interfaces(self, file=None):
#print "\nStoring service interfaces in datastore..."
if file and os.path.exists(file):
self._load_service_files([file])
elif file:
print "store_interfaces: Error couldn't find the file path\n"
else:
service_yaml_filenames = list_files_recursive(
'obj/services', '*.yml')
self._load_service_files(service_yaml_filenames)
def store_object_interfaces(self, file=None):
#print "\nStoring object interfaces in datastore..."
if file and os.path.exists(file):
self._load_object_files([file])
elif file:
print "store_interfaces: Error couldn't find the file path\n"
else:
data_yaml_filenames = list_files_recursive(
'obj/data', '*.yml', ['ion.yml', 'resource.yml', 'shared.yml'])
self._load_object_files(data_yaml_filenames)
def read_yaml_text(self):
"""
Gets the data from YAML files or datastore
"""
if self.read_from_yaml_file:
print " Object interface generator: Reading object definitions from files"
data_yaml_files = list_files_recursive("obj/data", "*.yml", ["ion.yml", "resource.yml", "shared.yml"])
self.data_yaml_text = "\n\n".join(
(file.read() for file in (open(path, "r") for path in data_yaml_files if os.path.exists(path)))
)
service_yaml_files = list_files_recursive("obj/services", "*.yml")
service_yaml_text = "\n\n".join(
(file.read() for file in (open(path, "r") for path in service_yaml_files if os.path.exists(path)))
)
data = self.data_yaml_text + "\n" + service_yaml_text
else:
print " Object interface generator: Reading object definitions from datastore"
self.data_yaml_text = get_object_definition_from_datastore(self.system_name)
if not self.data_yaml_text:
return ""
data = self.data_yaml_text + "\n" + get_service_definition_from_datastore(self.system_name)
return data
def get_yaml_data(self):
data = []
if self.read_from_yaml_file:
print " Message interface generator: reading service definitions from files"
service_yaml_files = list_files_recursive('obj/services', '*.yml')
for path in service_yaml_files:
if os.path.exists(path):
file = open(path, 'r')
data.append(file.read())
file.close()
else:
print " Message interface generator: reading service definitions from datastore"
data = get_service_definition_from_datastore(self.system_name)
if not data:
data = []
return data
def get_yaml_data(self):
data = []
if self.read_from_yaml_file:
print " Message interface generator: reading service definitions from files"
service_yaml_files = list_files_recursive('obj/services', '*.yml')
for path in service_yaml_files:
if os.path.exists(path):
file = open(path, 'r')
data.append(file.read())
file.close()
else:
print " Message interface generator: reading service definitions from datastore"
data = get_service_definition_from_datastore(self.system_name)
if not data:
data = []
return data
def load_service_interface(self):
service_yaml_filenames = list_files_recursive('obj/services', '*.yml')
print "\nLoading service interfaces..."
self.load_from_files(self.directory_path, service_yaml_filenames,
self.get_service_file_content)
def store_config_files(self):
#print "\nStoring system res files in datastore..."
resource_filenames = list_files_recursive('res/config', '*.yml')
self._load_config_files(resource_filenames)
def load_resource_configuration(self):
print "\nLoading... resource config"
resource_filenames = list_files_recursive('res/config', '*.yml')
self.load_from_files(self.directory_path, resource_filenames,
self.get_resource_file_content)
def generate (self, opts):
service_yaml_files = list_files_recursive('obj/services', '*.yml')
### messageobject_output_text = "# Message Objects\n\nimport interface.objects\nfrom pyon.core.object import IonObjectBase\n"
messageobject_output_text = "# Message Objects\n\nimport interface.objects\nfrom pyon.core.object import IonMessageObjectBase\n"
current_class_schema = ""
# Now process the service definition yaml files to
# generate message classes for input and return messages.
# Do this on a per file basis to simplify figuring out
# when we've reached the end of a service's ops.
args = []
init_lines = []
for yaml_file in (open(path, 'r') for path in service_yaml_files if os.path.exists(path)):
index = 0
yaml_text = yaml_file.read()
lines = yaml_text.split('\n')
# Find service name
while index < len(lines):
if lines[index].startswith('name:'):
break
index += 1
if index >= len(lines):
continue
current_service_name = lines[index].split(':')[1].strip()
index += 1
# Find op definitions
while index < len(lines):
if lines[index].startswith('methods:'):
break
index += 1
index += 1
if index >= len(lines):
continue
# Find op name
while index < len(lines):
if lines[index].startswith(' ') and lines[index][2].isalpha():
break
index += 1
if index >= len(lines):
continue
while index < len(lines):
if len(lines[index]) == 0 or lines[index].isspace():
index += 1
continue
if not (lines[index].startswith(' ') and lines[index][2].isalpha()):
index += 1
continue
args = []
init_lines = []
current_op_name = lines[index].strip(' :')
###messageobject_output_text += '\nclass ' + current_service_name + "_" + current_op_name + "_in(IonObjectBase):\n"
messageobject_output_text += '\nclass ' + current_service_name + "_" + current_op_name + "_in(IonMessageObjectBase):\n"
messageobject_output_text += " _svc_name = '" + current_service_name + "'\n"
messageobject_output_text += " _op_name = '" + current_op_name + "'\n"
index += 1
# Find in
while index < len(lines):
if lines[index].startswith(' resource_type:'):
messageobject_output_text += " _resource_type = '" + lines[index].split(' resource_type:')[1].strip() + "'\n"
if lines[index].startswith(' resource_id:'):
messageobject_output_text += " _resource_id = '" + lines[index].split(' resource_id:')[1].strip() + "'\n"
if lines[index].startswith(' operation_type:'):
messageobject_output_text += " _operation_type = '" + lines[index].split(' operation_type:')[1].strip() + "'\n"
if lines[index].startswith(' in:'):
break
index += 1
index += 1
messageobject_output_text += '\n def __init__(self'
current_class_schema = "\n _schema = {"
while index < len(lines) and not lines[index].startswith(' out:'):
if lines[index].isspace():
index += 1
continue
line = lines[index].replace(' ', '', 1)
if line.startswith(' #'):
init_lines.append(' ' + line + '\n')
index += 1
continue
elif line.startswith(' '):
is_required = False
field = line.split(":", 1)[0].strip()
try:
value = line.split(":", 1)[1].strip()
if '#' in value:
if '_required' in value:
is_required = True
value = value.split('#')[0].strip()
except KeyError:
# Ignore key error because value is nested
index += 1
continue
if len(value) == 0:
value = "None"
value_type = "str"
default = "None"
elif value.startswith('!'):
value = value.strip("!")
if value in enums_by_name:
value_type = 'int'
# Get default enum value
enum_def = enums_by_name[value]
value = default = "interface.objects." + value + "." + enum_def["default"]
else:
value_type = value
value = default = "None"
# Hacks, find a better way in the future
elif "'" in value or '"' in value:
value_type = "str"
default = value
# Hack
else:
try:
eval_value = ast.literal_eval(value)
value_type = type(eval_value).__name__
except ValueError:
value_type = "str"
value = "'" + value + "'"
except SyntaxError:
value_type = "str"
value = "'" + value + "'"
if value_type in ['dict', 'list', 'tuple']:
default = value = "None"
else:
default = value
args.append(", ")
args.append(field + "=" + value)
# if is_required:
# init_lines.append(" if " + field + " is None:\n")
# init_lines.append(" raise BadRequest('Required parameter " + field + " was not provided')\n")
init_lines.append(' self.' + field + " = " + field + "\n")
current_class_schema += "\n '" + field + "': {'type': '" + value_type + "', 'default': " + default + ", 'required': " + str(is_required) + "},"
index += 1
if len(args) > 0:
for arg in args:
messageobject_output_text += arg
messageobject_output_text += "):\n"
for init_line in init_lines:
messageobject_output_text += init_line
else:
messageobject_output_text += "):\n"
messageobject_output_text += ' pass\n'
messageobject_output_text += current_class_schema + "\n }\n"
if index < len(lines) and lines[index].startswith(' out:'):
args = []
init_lines = []
###messageobject_output_text += '\nclass ' + current_service_name + "_" + current_op_name + "_out(IonObjectBase):\n"
messageobject_output_text += '\nclass ' + current_service_name + "_" + current_op_name + "_out(IonMessageObjectBase):\n"
messageobject_output_text += " _svc_name = '" + current_service_name + "'\n"
messageobject_output_text += " _op_name = '" + current_op_name + "'\n\n"
messageobject_output_text += ' def __init__(self'
current_class_schema = "\n _schema = {"
index += 1
while index < len(lines):
line = lines[index]
if line.isspace() or len(line) == 0:
index += 1
continue
# Ignore
if not line.startswith(' '):
index += 1
continue
# Found next op
if line.startswith(' ') and line[2].isalpha():
break
if line.startswith(' throws:'):
index += 1
while index < len(lines):
if not lines[index].startswith(' '):
break
index += 1
break
line = line.replace(' ', '', 1)
if line.startswith(' #'):
index += 1
continue
field = line.split(":", 1)[0].strip()
try:
value = line.split(":", 1)[1].strip()
if '#' in value:
value = value.split('#')[0].strip()
except KeyError:
# Ignore key error because value is nested
index += 1
continue
if len(value) == 0:
value = "None"
value_type = "str"
default = "None"
elif value.startswith('!'):
value = value.strip("!")
if value in enums_by_name:
value_type = 'int'
# Get default enum value
enum_def = enums_by_name[value]
value = default = "interface.objects." + value + "." + enum_def["default"]
else:
value_type = value
value = default = "None"
# Hacks, find a better way in the future
elif "'" in value or '"' in value:
value_type = "str"
default = value
# Hack
else:
try:
eval_value = ast.literal_eval(value)
value_type = type(eval_value).__name__
except ValueError:
value_type = "str"
value = "'" + value + "'"
except SyntaxError:
value_type = "str"
value = "'" + value + "'"
if value_type in ['dict', 'list', 'tuple']:
default = value = "None"
else:
default = value
args.append(", ")
args.append(field + "=" + value)
# messageobject_output_text += ' self.' + field + " = kwargs.get('" + field + "', " + value + ")\n"
init_lines.append(' self.' + field + " = " + field + "\n")
current_class_schema += "\n '" + field + "': {'type': '" + value_type + "', 'default': " + default + "},"
index += 1
if len(args) > 0:
for arg in args:
messageobject_output_text += arg
messageobject_output_text += "):\n"
for init_line in init_lines:
messageobject_output_text += init_line
else:
messageobject_output_text += "):\n"
messageobject_output_text += ' pass\n'
messageobject_output_text += current_class_schema + "\n }\n"
datadir = 'interface'
messagemodelfile = os.path.join(datadir, 'messages.py')
try:
os.unlink(messagemodelfile)
except:
pass
print "Writing message model to '" + messagemodelfile + "'"
with open(messagemodelfile, 'w') as f:
f.write(messageobject_output_text)
def generate(self, opts):
service_yaml_files = list_files_recursive('obj/services', '*.yml')
### messageobject_output_text = "# Message Objects\n\nimport interface.objects\nfrom pyon.core.object import IonObjectBase\n"
messageobject_output_text = "# Message Objects\n\nimport interface.objects\nfrom pyon.core.object import IonMessageObjectBase\n"
current_class_schema = ""
# Now process the service definition yaml files to
# generate message classes for input and return messages.
# Do this on a per file basis to simplify figuring out
# when we've reached the end of a service's ops.
args = []
init_lines = []
for yaml_file in (open(path, 'r') for path in service_yaml_files
if os.path.exists(path)):
index = 0
yaml_text = yaml_file.read()
lines = yaml_text.split('\n')
# Find service name
while index < len(lines):
if lines[index].startswith('name:'):
break
index += 1
if index >= len(lines):
continue
current_service_name = lines[index].split(':')[1].strip()
index += 1
# Find op definitions
while index < len(lines):
if lines[index].startswith('methods:'):
break
index += 1
index += 1
if index >= len(lines):
continue
# Find op name
while index < len(lines):
if lines[index].startswith(' ') and lines[index][2].isalpha():
break
index += 1
if index >= len(lines):
continue
while index < len(lines):
if len(lines[index]) == 0 or lines[index].isspace():
index += 1
continue
if not (lines[index].startswith(' ')
and lines[index][2].isalpha()):
index += 1
continue
args = []
init_lines = []
current_op_name = lines[index].strip(' :')
###messageobject_output_text += '\nclass ' + current_service_name + "_" + current_op_name + "_in(IonObjectBase):\n"
messageobject_output_text += '\nclass ' + current_service_name + "_" + current_op_name + "_in(IonMessageObjectBase):\n"
messageobject_output_text += " _svc_name = '" + current_service_name + "'\n"
messageobject_output_text += " _op_name = '" + current_op_name + "'\n"
index += 1
# Find in
while index < len(lines):
if lines[index].startswith(' resource_type:'):
messageobject_output_text += " _resource_type = '" + lines[
index].split(
' resource_type:')[1].strip() + "'\n"
if lines[index].startswith(' resource_id:'):
messageobject_output_text += " _resource_id = '" + lines[
index].split(' resource_id:')[1].strip() + "'\n"
if lines[index].startswith(' operation_type:'):
messageobject_output_text += " _operation_type = '" + lines[
index].split(
' operation_type:')[1].strip() + "'\n"
if lines[index].startswith(' in:'):
break
index += 1
index += 1
messageobject_output_text += '\n def __init__(self'
current_class_schema = "\n _schema = {"
while index < len(lines) and not lines[index].startswith(
' out:'):
if lines[index].isspace():
index += 1
continue
line = lines[index].replace(' ', '', 1)
if line.startswith(' #'):
init_lines.append(' ' + line + '\n')
index += 1
continue
elif line.startswith(' '):
is_required = False
field = line.split(":", 1)[0].strip()
try:
value = line.split(":", 1)[1].strip()
if '#' in value:
if '_required' in value:
is_required = True
value = value.split('#')[0].strip()
except KeyError:
# Ignore key error because value is nested
index += 1
continue
if len(value) == 0:
value = "None"
value_type = "str"
default = "None"
elif value.startswith('!'):
value = value.strip("!")
if value in enums_by_name:
value_type = 'int'
# Get default enum value
enum_def = enums_by_name[value]
value = default = "interface.objects." + value + "." + enum_def[
"default"]
else:
value_type = value
value = default = "None"
# Hacks, find a better way in the future
elif "'" in value or '"' in value:
value_type = "str"
default = value
# Hack
else:
try:
eval_value = ast.literal_eval(value)
value_type = type(eval_value).__name__
except ValueError:
value_type = "str"
value = "'" + value + "'"
except SyntaxError:
value_type = "str"
value = "'" + value + "'"
if value_type in ['dict', 'list', 'tuple']:
default = value = "None"
else:
default = value
args.append(", ")
args.append(field + "=" + value)
# if is_required:
# init_lines.append(" if " + field + " is None:\n")
# init_lines.append(" raise BadRequest('Required parameter " + field + " was not provided')\n")
init_lines.append(' self.' + field + " = " +
field + "\n")
current_class_schema += "\n '" + field + "': {'type': '" + value_type + "', 'default': " + default + ", 'required': " + str(
is_required) + "},"
index += 1
if len(args) > 0:
for arg in args:
messageobject_output_text += arg
messageobject_output_text += "):\n"
for init_line in init_lines:
messageobject_output_text += init_line
else:
messageobject_output_text += "):\n"
messageobject_output_text += ' pass\n'
messageobject_output_text += current_class_schema + "\n }\n"
if index < len(lines) and lines[index].startswith(' out:'):
args = []
init_lines = []
###messageobject_output_text += '\nclass ' + current_service_name + "_" + current_op_name + "_out(IonObjectBase):\n"
messageobject_output_text += '\nclass ' + current_service_name + "_" + current_op_name + "_out(IonMessageObjectBase):\n"
messageobject_output_text += " _svc_name = '" + current_service_name + "'\n"
messageobject_output_text += " _op_name = '" + current_op_name + "'\n\n"
messageobject_output_text += ' def __init__(self'
current_class_schema = "\n _schema = {"
index += 1
while index < len(lines):
line = lines[index]
if line.isspace() or len(line) == 0:
index += 1
continue
# Ignore
if not line.startswith(' '):
index += 1
continue
# Found next op
if line.startswith(' ') and line[2].isalpha():
break
if line.startswith(' throws:'):
index += 1
while index < len(lines):
if not lines[index].startswith(' '):
break
index += 1
break
line = line.replace(' ', '', 1)
if line.startswith(' #'):
index += 1
continue
field = line.split(":", 1)[0].strip()
try:
value = line.split(":", 1)[1].strip()
if '#' in value:
value = value.split('#')[0].strip()
except KeyError:
# Ignore key error because value is nested
index += 1
continue
if len(value) == 0:
value = "None"
value_type = "str"
default = "None"
elif value.startswith('!'):
value = value.strip("!")
if value in enums_by_name:
value_type = 'int'
# Get default enum value
enum_def = enums_by_name[value]
value = default = "interface.objects." + value + "." + enum_def[
"default"]
else:
value_type = value
value = default = "None"
# Hacks, find a better way in the future
elif "'" in value or '"' in value:
value_type = "str"
default = value
# Hack
else:
try:
eval_value = ast.literal_eval(value)
value_type = type(eval_value).__name__
except ValueError:
value_type = "str"
value = "'" + value + "'"
except SyntaxError:
value_type = "str"
value = "'" + value + "'"
if value_type in ['dict', 'list', 'tuple']:
default = value = "None"
else:
default = value
args.append(", ")
args.append(field + "=" + value)
# messageobject_output_text += ' self.' + field + " = kwargs.get('" + field + "', " + value + ")\n"
init_lines.append(' self.' + field + " = " +
field + "\n")
current_class_schema += "\n '" + field + "': {'type': '" + value_type + "', 'default': " + default + "},"
index += 1
if len(args) > 0:
for arg in args:
messageobject_output_text += arg
messageobject_output_text += "):\n"
for init_line in init_lines:
messageobject_output_text += init_line
else:
messageobject_output_text += "):\n"
messageobject_output_text += ' pass\n'
messageobject_output_text += current_class_schema + "\n }\n"
datadir = 'interface'
messagemodelfile = os.path.join(datadir, 'messages.py')
try:
os.unlink(messagemodelfile)
except:
pass
print "Writing message model to '" + messagemodelfile + "'"
with open(messagemodelfile, 'w') as f:
f.write(messageobject_output_text)
def generate (self, opts):
service_dir, interface_dir = 'obj/services', 'interface'
data_yaml_files = list_files_recursive('obj/data', '*.yml', ['ion.yml', 'resource.yml', 'shared.yml'])
data_yaml_text = '\n\n'.join((file.read() for file in (open(path, 'r') for path in data_yaml_files if os.path.exists(path))))
service_yaml_files = list_files_recursive('obj/services', '*.yml')
service_yaml_text = '\n\n'.join((file.read() for file in (open(path, 'r') for path in service_yaml_files if os.path.exists(path))))
enum_tag = u'!enum'
def enum_constructor(loader, node):
val_str = str(node.value)
val_str = val_str[1:-1].strip()
if 'name' in val_str:
name_str = val_str.split(',', 1)[0].split('=')[1].strip()
return "!" + str(name_str)
else:
return "Enum Name Not Provided"
yaml.add_constructor(enum_tag, enum_constructor, Loader=IonYamlLoader)
yaml_files = list_files_recursive('obj/data', '*.yml', ['ion.yml', 'resource.yml'])
yaml_text = '\n\n'.join((file.read() for file in (open(path, 'r') for path in yaml_files if os.path.exists(path))))
# Now walk the data model definition yaml files adding the
# necessary yaml constructors.
defs = yaml.load_all(data_yaml_text, Loader=IonYamlLoader)
def_dict = {}
for def_set in defs:
for name,_def in def_set.iteritems():
if isinstance(_def, OrderedDict):
def_dict[name] = _def
tag = u'!%s' % (name)
def constructor(loader, node):
value = node.tag.strip('!')
# See if this is an enum ref
if value in self.enums_by_name:
return {"__IsEnum": True, "value": value + "." + self.enums_by_name[value]["default"], "type": value}
else:
return str(value) + "()"
yaml.add_constructor(tag, constructor, Loader=IonYamlLoader)
xtag = u'!Extends_%s' % (name)
def extends_constructor(loader, node):
if isinstance(node, yaml.MappingNode):
value = loader.construct_mapping(node)
else:
value = {}
return value
yaml.add_constructor(xtag, extends_constructor, Loader=IonYamlLoader)
# Do the same for any data model objects in the service
# definition files.
defs = yaml.load_all(service_yaml_text, Loader=IonYamlLoader)
for def_set in defs:
for name,_def in def_set.get('obj', {}).iteritems():
if isinstance(_def, OrderedDict):
def_dict[name] = _def
tag = u'!%s' % (name)
def constructor(loader, node):
value = node.tag.strip('!')
# See if this is an enum ref
if value in self.enums_by_name:
return {"__IsEnum": True, "value": value + "." + self.enums_by_name[value]["default"], "type": value}
else:
return str(value) + "()"
yaml.add_constructor(tag, constructor, Loader=IonYamlLoader)
xtag = u'!Extends_%s' % (name)
def extends_constructor(loader, node):
if isinstance(node, yaml.MappingNode):
value = loader.construct_mapping(node)
else:
value = {}
return value
yaml.add_constructor(xtag, extends_constructor, Loader=IonYamlLoader)
yaml_files = list_files_recursive('obj/data', '*.yml', ['ion.yml', 'resource.yml'])
yaml_text = '\n\n'.join((file.read() for file in (open(path, 'r') for path in yaml_files if os.path.exists(path))))
# Load data yaml files in case services define interfaces
# in terms of common data objects
defs = yaml.load_all(yaml_text, Loader=IonYamlLoader)
for def_set in defs:
for name,_def in def_set.iteritems():
tag = u'!%s' % (name)
yaml.add_constructor(tag, self.doc_tag_constructor)
xtag = u'!Extends_%s' % (name)
yaml.add_constructor(xtag, lambda loader, node: {})
svc_signatures = {}
sigfile = os.path.join('interface', '.svc_signatures.yml')
if os.path.exists(sigfile):
with open(sigfile, 'r') as f:
cnts = f.read()
svc_signatures = yaml.load(cnts)
count = 0
# mapping of service name -> { name, docstring, deps, methods }
raw_services = {}
# dependency graph, maps svcs -> deps by service name as a list
service_dep_graph = {}
# completed service client definitions, maps service name -> full module path to find module
client_defs = {}
yaml_file_re = re.compile('(obj)/(.*)[.](yml)')
# Generate the new definitions, for now giving each
# yaml file its own python service
for root, dirs, files in os.walk(service_dir):
for filename in fnmatch.filter(files, '*.yml'):
yaml_file = os.path.join(root, filename)
file_match = yaml_file_re.match(yaml_file)
if '.svc_signatures' in filename: continue
if file_match is None: continue
file_path = file_match.group(2)
interface_base, interface_name = os.path.dirname(file_path), os.path.basename(file_path)
interface_file = os.path.join('interface', interface_base, 'i%s.py' % interface_name)
parent_dir = os.path.dirname(interface_file)
if not os.path.exists(parent_dir):
os.makedirs(parent_dir)
parent = parent_dir
while True:
# Add __init__.py files to parent dirs as necessary
curdir = os.path.split(os.path.abspath(parent))[1]
if curdir == 'services':
break
else:
parent = os.path.split(os.path.abspath(parent))[0]
pkg_file = os.path.join(parent, '__init__.py')
if not os.path.exists(pkg_file):
open(pkg_file, 'w').close()
pkg_file = os.path.join(parent_dir, '__init__.py')
if not os.path.exists(pkg_file):
open(pkg_file, 'w').close()
skip_file = False
with open(yaml_file, 'r') as f:
yaml_text = f.read()
m = hashlib.md5()
m.update(yaml_text)
cur_md5 = m.hexdigest()
if yaml_file in svc_signatures and not opts.force:
if cur_md5 == svc_signatures[yaml_file]:
print "Skipping %40s (md5 signature match)" % interface_name
skip_file = True
# do not continue here, we want to read the service deps below
if opts.dryrun:
count += 1
print "Changed %40s (needs update)" % interface_name
skip_file = True
# do not continue here, we want to read the service deps below
# update signature set
if not skip_file:
svc_signatures[yaml_file] = cur_md5
defs = yaml.load_all(yaml_text)
for def_set in defs:
# Handle object definitions first; make dummy constructors so tags will parse
if 'obj' in def_set:
for obj_name in def_set['obj']:
tag = u'!%s' % (obj_name)
yaml.add_constructor(tag, self.doc_tag_constructor)
continue
service_name = def_set.get('name', None)
class_docstring = def_set.get('docstring', "class docstring")
spec = def_set.get('spec', None)
dependencies = def_set.get('dependencies', None)
meth_list = def_set.get('methods', {}) or {}
client_path = ('.'.join(['interface', interface_base.replace('/', '.'), 'i%s' % interface_name]), '%sProcessClient' % self.service_name_from_file_name(interface_name))
# format multiline docstring
class_docstring_lines = class_docstring.split('\n')
# Annoyingly, we have to hand format the doc strings to introduce
# the correct indentation on multi-line strings
first_time = True
class_docstring_formatted = ""
for i in range(len(class_docstring_lines)):
class_docstring_line = class_docstring_lines[i]
# Potentially remove excess blank line
if class_docstring_line == "" and i == len(class_docstring_lines) - 1:
break
if first_time:
first_time = False
else:
class_docstring_formatted += "\n "
class_docstring_formatted += class_docstring_line
# load into raw_services (if we're not skipping)
if not skip_file:
if service_name in raw_services:
raise StandardError("Duplicate service name found: %s" % service_name)
raw_services[service_name] = { 'name' : service_name,
'docstring' : class_docstring_formatted,
'spec' : spec,
'dependencies' : dependencies,
'methods' : meth_list,
'interface_file' : interface_file,
'interface_name' : interface_name,
'client_path' : client_path }
# dep capturing (we check cycles when we topologically sort later)
if not service_name in service_dep_graph:
service_dep_graph[service_name] = set()
for dep in dependencies:
service_dep_graph[service_name].add(dep)
# update list of client paths for the client to this service
client_defs[service_name] = client_path
print "About to generate", len(raw_services), "services"
# topological sort of services to make sure we do things in order
# http://en.wikipedia.org/wiki/Topological_sorting
sorted_services = []
service_set = set([k for k,v in service_dep_graph.iteritems() if len(v) == 0])
while len(service_set) > 0:
n = service_set.pop()
# topo sort is over the whole dep tree, but raw_services only contains the stuff we're really generating
# so if it doesn't exist in raw_services, don't bother!
if n in raw_services:
sorted_services.append((n, raw_services[n]))
# get list of all services that depend on the current service
depending_services = [k for k,v in service_dep_graph.iteritems() if n in v]
for depending_service in depending_services:
# remove this dep
service_dep_graph[depending_service].remove(n)
# if it has no more deps, add it to the service_set list
if len(service_dep_graph[depending_service]) == 0:
service_set.add(depending_service)
# ok, check for any remaining deps that we never found - indicates a cycle
remaining_deps = set([k for k,v in service_dep_graph.iteritems() if len(v) > 0])
if len(remaining_deps):
print >> sys.stderr, "**********************************************************************"
print >> sys.stderr, "Error in dependency resolution: either a cycle or a missing dependency"
print >> sys.stderr, "Service -> Conflicting Dependency table:"
for k, v in service_dep_graph.iteritems():
if len(v) == 0:
continue
print >> sys.stderr, "\t", k, "->", ",".join(v)
print >> sys.stderr, "**********************************************************************"
raise StandardError("Cycle found in dependencies: could not resolve %s" % str(remaining_deps))
for svc in sorted_services:
svc_name, raw_def = svc
self.generate_service(raw_def['interface_file'], raw_def, client_defs, opts)
count+=1
if count > 0 and not opts.dryrun:
# write current svc_signatures
print "Writing signature file to", sigfile
with open(sigfile, 'w') as f:
f.write(yaml.dump(svc_signatures))
# Load interface base classes
self.load_mods("interface/services", True)
base_subtypes = self.find_subtypes(BaseService)
# Load impl classes
self.load_mods("ion", False)
# write client public file
# @TODO: reenable when 'as' directory goes away
'''
clientfile = os.path.join('interface', 'clients.py')
print "Writing public client file to", clientfile
with open(clientfile, 'w') as f:
f.write(templates['client_file'].substitute(when_generated=self.currtime,
client_imports="\n".join([templates['dep_client_imports'].substitute(clientmodule=x[0], clientclass=x[1]) for x in client_defs.itervalues()])))
'''
self.generate_validation_report()
exitcode = 0
# only exit with 1 if we notice changes, and we specified dryrun
if count > 0 and opts.dryrun:
exitcode = 1
sys.exit(exitcode)
def load_service_interface(self):
service_yaml_filenames = list_files_recursive('obj/services', '*.yml')
print "\nLoading service interfaces..."
self.load_from_files(self.directory_path, service_yaml_filenames, self.get_service_file_content)
def load_object_model_interface(self):
print "\nLoading object model..."
data_yaml_filenames = list_files_recursive('obj/data', '*.yml', ['ion.yml', 'resource.yml'])
self.load_from_files(self.directory_path, data_yaml_filenames, self.get_data_file_content)
def load_resource_configuration(self):
print "\nLoading... resource config"
resource_filenames = list_files_recursive('res/config', '*.yml')
self.load_from_files(self.directory_path, resource_filenames, self.get_resource_file_content)
def load_object_model_interface(self):
print "\nLoading object model..."
data_yaml_filenames = list_files_recursive('obj/data', '*.yml',
['ion.yml', 'resource.yml'])
self.load_from_files(self.directory_path, data_yaml_filenames,
self.get_data_file_content)
def store_config_files(self):
#print "\nStoring system res files in datastore..."
resource_filenames = list_files_recursive('res/config', '*.yml')
self._load_config_files(resource_filenames)
