def generateAbortNode(root, parent, settings):
global VERBOSE
# node
abortNode = LET.Element('node')
generateNodeName(abortNode, 'ABORT_')
if VERBOSE:
eprintf('Generate abort node for parent: %s named: %s\n',
parent.find('name').text if parent is not None else 'root',
abortNode.find('name').text)
# condition
abortNodeCondition = LET.Element('condition')
abortNodeCondition.text = DEFAULT_CONDITION_ABORT + (
' and intent.confidence >' +
settings.get('confidence') if 'confidence' in settings.attrib else '')
abortNode.append(abortNodeCondition)
# output
abortNodeOutput = LET.Element('output')
if parent is not None:
abortNodeOutput.text = settings.find('message').text if settings.find(
'message') is not None else DEFAULT_ABORT_MESSAGE.text
else:
abortNodeOutput.text = settings.find(
'message_cannot').text if settings.find(
'message_cannot'
) is not None else DEFAULT_ABORT_MESSAGE_CANNOT.text
abortNode.append(abortNodeOutput)
# goto
if settings.find('goto') is not None:
abortNode.append(copy.deepcopy(settings.find('goto')))
return abortNode
def generateAgainNode(root, parent, settings):
global VERBOSE
# node
againNode = LET.Element('node')
generateNodeName(againNode, 'AGAIN_')
if VERBOSE:
eprintf('Generate again node for parent: %s named: %s\n',
parent.find('name').text if parent is not None else 'root',
againNode.find('name').text)
# condition
againNodeCondition = LET.Element('condition')
againNodeCondition.text = DEFAULT_CONDITION_AGAIN + (
' and intent.confidence >' +
settings.get('confidence') if 'confidence' in settings.attrib else '')
againNode.append(againNodeCondition)
# output
againNodeOutput = LET.Element('output')
againNodeOutput.text = '$againMessage'
againNode.append(againNodeOutput)
# goto
againNodeGoto = LET.Element('goto')
againNodeGotoTarget = LET.Element('target')
againNodeGotoTarget.text = root.find('node').find('name').text
againNodeGoto.append(againNodeGotoTarget)
againNode.append(againNodeGoto)
return againNode
def generateBackNode(root, parent, settings):
global VERBOSE
# node
backNode = LET.Element('node')
generateNodeName(backNode, 'BACK_')
if VERBOSE: eprintf('Generate back node for parent: %s named: %s\n', parent.find('name').text if parent is not None else 'root', backNode.find('name').text)
# condition
backNodeCondition = LET.Element('condition')
backNodeCondition.text = DEFAULT_CONDITION_BACK + (' and intent.confidence >' + settings.get('confidence') if 'confidence' in settings.attrib else '')
backNode.append(backNodeCondition)
if parent is not None and parent in parent_map and parent_map[parent] in parent_map:
# output
backNodeOutput = LET.Element('output')
backNodeOutput.text = settings.find('message').text if settings.find('message') is not None else DEFAULT_BACK_MESSAGE.text
backNode.append(backNodeOutput)
# goto
backNodeGoto = LET.Element('goto', {'selector':'body'})
backNodeTarget = LET.Element('target')
backNodeTarget.text = parent_map[parent_map[parent]].find('name').text
backNodeGoto.append(backNodeTarget)
backNode.append(backNodeGoto)
else:
# output
backNodeOutput = LET.Element('output')
if parent is not None:
backNodeOutput.text = settings.find('message_to_main').text if settings.find('message_to_main') is not None else DEFAULT_BACK_MESSAGE_TO_MAIN.text
else:
backNodeOutput.text = settings.find('message_cannot').text if settings.find('message_cannot') is not None else DEFAULT_BACK_MESSAGE_CANNOT.text
backNode.append(backNodeOutput)
return backNode
def generateRepeatNode(parent, root, output, attempts, varName, varValue, goto):
global VERBOSE
# node
repeatNode = LET.Element('node')
generateNodeName(repeatNode, 'REPEAT_')
if VERBOSE: eprintf('Generate repeat node for parent: %s named: %s START\n', parent.find('name').text if parent is not None else 'root', repeatNode.find('name').text)
# condition
repeatNodeCondition = LET.Element('condition')
repeatNodeCondition.text = ('$' + varName + ' == null or ' if attempts == 0 else '') + '$' + varName + ' >= ' + str(attempts)
repeatNode.append(repeatNodeCondition)
# context
repeatNodeContext = LET.Element('context')
repeatVariable = LET.Element(varName)
repeatVariable.text = str(varValue)
if isinstance(varValue, int):
repeatVariable.set('type', 'number')
repeatNodeContext.append(repeatVariable)
repeatNode.append(repeatNodeContext)
# output
repeatNode.append(copy.deepcopy(output))
# goto
if goto is not None:
repeatNode.append(copy.deepcopy(goto))
root.append(repeatNode)
if VERBOSE: eprintf('Generate repeat node for parent: %s named: %s END\n', parent.find('name').text if parent is not None else 'root', repeatNode.find('name').text)
def saveConfiguration(self, configFileName):
outputConfig = configparser.ConfigParser()
for optionUniqueName in self.__dict__:
namesList = optionUniqueName.split(Cfg.sectionDelimiter)
# find section and option names
if len(namesList) > 1:
section = namesList[0]
option = Cfg.sectionDelimiter.join(namesList[1:])
else:
eprintf(
"WARNING: Missing section name in parameter name '%s', skipping.\n",
optionUniqueName)
continue
# create non existing sections
if not outputConfig.has_section(section):
outputConfig.add_section(section)
# convert types to string
if isinstance(getattr(self, optionUniqueName), list):
outputConfig.set(section, option,
','.join(getattr(self, optionUniqueName)))
elif isinstance(getattr(self, optionUniqueName), bool):
outputConfig.set(section, option,
str(getattr(self, optionUniqueName)))
else:
outputConfig.set(section, option,
getattr(self, optionUniqueName))
try:
with open(configFileName, 'w') as configFile:
outputConfig.write(configFile)
except IOError:
eprintf('ERROR: Cannot save config file %s\n', configFileName)
def validate(xml):
global schema, VERBOSE
try:
schema.assertValid(xml)
if VERBOSE: eprintf("XML is valid\n")
except LET.XMLSchemaError:
eprintf("Invalid XML %s!\n")
def validateNodeName(node):
global names
name = node.find('name').text
# check characters (Node names can only contain letters, numbers, hyphens and underscores)
pattern = re.compile("[\w-]+", re.UNICODE)
if not pattern.match(name):
eprintf("Illegal name of the node: '%s'\nNode names can only contain letters, numbers, hyphens and underscores.\n", name)
exit(1)
def isFalse(autogenerate, attributeName):
attributeValue = autogenerate.get(attributeName)
if attributeValue == None or attributeValue == 'true':
return False;
elif attributeValue == 'false':
return True
else:
eprintf('Unknown value of \'%s\' tag: %s.\n', attributeName, attributeValue)
return False
def parseXLSXIntoDataBlocks(self, filename):
""" Read Excel spreadsheet in T2C format. Store the data as tuples (domain, prefix, intent, rawBlock) into private field. """
printf('Processing xlsx file: %s\n', filename)
if not os.path.exists(filename):
eprintf('Error: File does not exist: %s\n', filename)
return {}
try:
domainName = unicode(
toIntentName(NAME_POLICY, None,
os.path.splitext(os.path.split(filename)[1])[0]),
'utf-8')
workbook = load_workbook(filename=filename, read_only=True)
except (IOError, BadZipfile):
eprintf(
'Error: File does not seem to be a valid Excel spreadsheet: %s\n',
filename)
return {}
# Process all the tabs of the file
for sheet in workbook.worksheets:
printf(' Sheet: %s\n', sheet.title)
prefix = unicode(sheet.title, 'utf-8')
currentBlock = []
# Separate all data blocks in the sheet, if the currentBlock starts with header, it is considered to be part of currentBlock
for row in sheet.iter_rows(max_col=4):
validRow = False
# Check if the row is valid. Row is valid if it contains at least one column not empty and different from comment
for columnIndex in range(0, 4):
if row[columnIndex] and row[columnIndex].value and not (
row[columnIndex].value.startswith('//')):
validRow = True
# Three slashes in the first cell cause whole rest of the line to be treated as comment
if row[0].value and row[0].value.startswith('///'):
validRow = False
# If empty line or header, we store the previous currentBlock-if any
if not validRow:
if currentBlock:
self.__createBlock(domainName, prefix, currentBlock)
currentBlock = []
else:
# if valid row - we add it to block
currentBlock.append(
(row[0].value.strip() if row[0].value
and not row[0].value.startswith('//') else None,
row[1].value.strip() if row[1].value
and not row[1].value.startswith('//') else None,
row[2].value.strip() if row[2].value
and not row[2].value.startswith('//') else None,
row[3].value.strip() if row[3].value
and not row[3].value.startswith('//') else None))
if currentBlock:
self.__createBlock(domainName, prefix, currentBlock)
def importText(importTree, config):
imports = importTree.xpath('//importText')
for imp in imports:
if VERBOSE: eprintf('Importing %s\n', os.path.join(os.path.dirname(getattr(config, 'common_dialog_main')),*filename))
filename = imp.text.split('/')
fp = io.open(os.path.join(os.path.dirname(getattr(config, 'common_dialog_main')),*filename) ,'r', encoding='utf-8')
importTxt = fp.read()
fp.close()
imp.getparent().text = ("" if imp.getparent().text is None else imp.getparent().text) + importTxt + ("" if imp.tail is None else imp.tail)
imp.getparent().remove(imp)
def findAllNodeNames(tree):
names = []
nodesWithNames = tree.xpath('//node[@name]')
for nodeWithName in nodesWithNames:
if nodeWithName.get('name') in names:
eprintf('ERROR: Duplicit node name found: %s\n', nodeWithName.get('name'))
exit(1)
else:
names.append(nodeWithName.get('name'))
return names
def importNodes(root, config):
global rootGlobal, VERBOSE, names
# IMPORT AND APPEND NODES
defaultNode = None
if len(root) > 0 and (root[len(root)-1].find('condition') is None or (root[len(root)-1].find('condition') is not None and root[len(root)-1].find('condition').text == 'anything_else')):
# IF LAST NODE DOES NOT HAVE CONDITION OR HAS CONDITION SET TO 'anything_else'
defaultNode = root[len(root)-1]
for node in root.findall('import'):
if VERBOSE: eprintf('Importing %s\n', os.path.join(os.path.dirname(getattr(config, 'common_dialog_main')),node.text))
importPath = node.text.split('/')
importTree = LET.parse(os.path.join(os.path.dirname(getattr(config, 'common_dialog_main')),*importPath))
importText(importTree, config)
replace_config_variables(importTree)
if schema is not None:
validate(importTree)
importRoot = importTree.getroot()
for importChild in importRoot.findall('node'):
#eprintf(' Importing node: %s\n', importChild)
nodeWithTheSameCondition = getNodeWithTheSameCondition(root, importChild)
"""
if nodeWithTheSameCondition is not None:
# SKIP NODES WITH SAME CONDITIONS
#eprintf(' Skipping node (same condition): %s\n', nodeWithTheSameCondition)
if importChild.find('context') is not None:
#eprintf(' Context found for node: %s\n', importChild)
if nodeWithTheSameCondition.find('context') is None:
#eprintf(' Creating context for node: %s\n', nodeWithTheSameCondition)
nodeWithTheSameConditionContext = LET.Element('context')
nodeWithTheSameCondition.append(nodeWithTheSameConditionContext)
# COPY ALL CONTEXT TO NODE WITH SAME CONDITION
for context in importChild.find('context'):
#eprintf(' Appending context: %s\n', context)
nodeWithTheSameCondition.find('context').append(context)
else:
# INSERT NODE
#eprintf(' Appending node: %s\n', importChild)
"""
root.append(importChild)
if defaultNode is not None:
# MOVE DEFAULT_NODE TO THE END
root.remove(defaultNode)
root.append(defaultNode)
# PROCESS CHILD NODES
for node in root.findall('node'):
children = node.find('nodes')
if children is not None:
importNodes(children, config)
def _createOutputElement(self, channels, buttons):
""" Convert output channels into XML structure. """
outputXml = XML.Element('output')
if channels:
for channelName, channelValues in channels.iteritems():
if channelName == '1':
textValuesXml = XML.Element('textValues')
for item in channelValues:
textValuesXml.append(
self._createXmlElement('values', item))
outputXml.append(textValuesXml)
continue
output = self._concatenateOutputs(channelValues)
if channelName == '2':
outputXml.append(self._createXmlElement('timeout', output))
elif channelName == '3':
outputXml.append(self._createXmlElement('sound', output))
elif channelName == '4':
outputXml.append(self._createXmlElement('tts', output))
elif channelName == '5':
outputXml.append(
self._createXmlElement('talking_head', output))
elif channelName == '6':
outputXml.append(
self._createXmlElement('paper_head', output))
elif channelName == '7':
outputXml.append(self._createXmlElement(
'graphics', output))
elif channelName == '8':
outputXml.append(self._createXmlElement('url', output))
else:
eprintf('Warning: Unrecognized channel: %s, value: %s\n',
channelName, output)
if buttons:
genericXml = XML.Element('generic', structure='listItem')
for buttonLabel, buttonValue in buttons.iteritems():
optionsXml = XML.Element('options')
optionsXml.append(self._createXmlElement('label', buttonLabel))
optionsXml.append(self._createXmlElement('value', buttonValue))
genericXml.append(optionsXml)
outputXml.append(genericXml)
return outputXml
def __handleIntentBlock(self, intent, block, domain):
""" Read intent definition from current block and save it into Dialog data structure. """
# blockLength = len(block)
startsWithHash = block[0][0].startswith(u'#')
if not startsWithHash and not block[0][1]:
eprintf('Warning: Wrong intent definition format for line starting with: %s\n', intent)
return
intentData = self._dialogData.getIntentData(intent, domain)
for row in block:
if row[0] and not row[0].startswith(u'#'):
intentData.addIntentAlternative(row[0]) # Collect intent definition
if row[1]:
intentData.addRawOutput(row[1:], self._labelsMap) # Collect text output
def __handleJumpToDefinition(self, jumpto, labelsMap):
selector = 'user_input'
label = jumpto
if len(jumpto) > 2 and jumpto[1] == '_':
label = jumpto[2:]
if jumpto[0] == 'b':
selector = 'body'
elif jumpto[0] == 'c':
selector = 'condition'
if label not in labelsMap:
self.setJumpTo(
label,
selector) # label can point at a node name defined externally
eprintf(
'Warning: using jumpto label that was not defined before: %s, expecting that it is external reference to a node name.\n',
label)
else:
self.setJumpTo(labelsMap[label], selector)
def addRawOutput(self, rawOutputs, labelsMap):
""" Read the raw output and store all data from it -
channel outputs, context variables and jumpto definitions. """
self._rawOutputs.append(rawOutputs)
if not isinstance(rawOutputs, tuple) or len(rawOutputs) < 1:
eprintf('Warning: rawOutput does not contain any data: %s\n',
rawOutputs)
for item in re.split('%%', rawOutputs[0]):
if not item: continue
if item.startswith(u'$'):
self.__handleVariableDefinition(item[1:])
elif item.startswith(u'B'):
self.__handleButtonDefinition(item[1:])
elif item.startswith(u':'):
self.__handleJumpToDefinition(item[1:], labelsMap)
else:
self.__handleChannelDefinition(item)
if len(rawOutputs) >= 3:
if rawOutputs[1]:
self.__handleButtonDefinition(rawOutputs[1])
if rawOutputs[2]:
self.__handleJumpToDefinition(rawOutputs[2], labelsMap)
def mergeSettings(childSettings, parentSettings):
if childSettings is None:
if VERBOSE: eprintf('Returning parent settings\n')
return parentSettings
if parentSettings is None:
if VERBOSE: eprintf('Returning child settings\n')
return childSettings
# for all child elements
for element in parentSettings:
if childSettings.find(element.tag) is None:
childSettings.append(element)
# for all attributes
for attributeName in parentSettings.attrib:
if childSettings.get(attributeName) is None:
childSettings.set(attributeName, parentSettings.get(attributeName))
if VERBOSE: eprintf('Returning merged settings\n')
return childSettings
def __init__(self, args):
logging.basicConfig(filename='log.log', level=logging.INFO)
logging.info('cfg.__init__')
self.config = {}
# Sections (names can not contain '_')
commonSection = 'common'
conversationSection = 'conversation'
cloudfunctionsSection = 'cloudfunctions'
workspaceSection = 'workspace'
weatherSection = 'weatherservice'
replaceSection = 'replace'
versionSection = 'version'
contextSection = 'context'
commonSection = 'common'
# List of attributes of framework section to be appended rather then ovewrriden (if the same parameter is defined in more config files)
frameworkAppend = [
'xls', 'intents', 'entities', 'dialogs', 'functions',
'generated_intents', 'generated_entities', 'generated_dialog'
]
if args.common_configFilePaths:
try:
for common_configFilePath in args.common_configFilePaths: # go over all the config files and collect all parameters
logging.info("Processing config file:" +
common_configFilePath)
print("Processing config file:" + common_configFilePath)
configPart = configparser.ConfigParser()
configPart.read(common_configFilePath)
# Collect all attributes from all sections
for section in configPart.sections():
options = configPart.options(section)
for option in options:
optionUniqueName = self.toOptionName(
section, option)
# value can be list
newValueList = configPart.get(section,
option).split(',')
if len(newValueList) > 1: # create list
newValue = newValueList
else: # only single value
newValue = newValueList[0]
if hasattr(self, optionUniqueName):
warning = "WARNING: '" + optionUniqueName + " already exists. "
if (section == commonSection) and (
option in frameworkAppend): # appended
logging.debug(
warning + "Appending '[" +
' '.join(newValue) + "]' to [" +
' '.join(
getattr(self, optionUniqueName)) +
"]")
setattr(self, optionUniqueName, newValue)
else: # replace
oldValue = getattr(self, optionUniqueName)
logging.debug(warning + "Replacing '" +
oldValue + "' by '[" +
' '.join(newValue) + "]'")
setattr(self, optionUniqueName, newValue)
else:
if (section == commonSection) and (
option
in frameworkAppend): # create list
setattr(self, optionUniqueName, [newValue])
else: # set value
setattr(self, optionUniqueName, newValue)
except IOError:
eprintf('ERROR: Cannot load config file %s\n',
args.configFileName)
sys.exit(1)
# Set command line parameters
# command line parameters are having precedence, therefore they are set the last
for arg in vars(args):
if hasattr(args, arg) and getattr(
args, arg): # attribute is present and not empty
if hasattr(self, arg):
eprintf(
"WARNING: Overwriting config file parameter '%s' with value '%s' from comman line argumets.\n",
arg, getattr(args, arg))
setattr(self, arg, getattr(args, arg))
if hasattr(self, 'common_output_config'):
self.saveConfiguration(getattr(self, 'common_output_config'))
parser.add_argument('-oc',
'--common_output_config',
help='output configuration file')
parser.add_argument('-v',
'--common_verbose',
required=False,
help='verbosity',
action='store_true')
args = parser.parse_args(sys.argv[1:])
config = Cfg(args)
VERBOSE = hasattr(config, 'common_verbose')
if hasattr(config, 'common_verbose') and getattr(config, 'common_verbose'):
name_policy = 'soft_verbose'
if not hasattr(config, 'common_xls'):
eprintf('ERROR: xls is not defined')
exit(1)
if not hasattr(config, 'common_generated_dialogs'):
if VERBOSE:
printf('INFO: generated_dialogs parameter is not defined\n')
if not hasattr(config, 'common_generated_intents'):
if VERBOSE:
printf('INFO: generated_intents parameter is not defined\n')
if not hasattr(config, 'common_generated_entities'):
if VERBOSE:
printf('INFO: generated_entities parameter is not defined\n')
xlsxHandler = XLSXHandler()
allDataBlocks = {
} # map of datablocks, key: Excel sheet name, value: list of all block in the sheet
# LOAD RESOURCE FILE (TRANSLATIONS)
with open(args.resource, 'r') as resourceFile:
translations = json.load(resourceFile)
# REPLACE ALL CODES WITH TEXTS
for tagToReplace in tagsToReplace:
if tagToReplace.text is None: continue
if VERBOSE: printf("%s: code '%s'\n", tagToReplace.tag, tagToReplace.text)
textParts = tagToReplace.text.split()
for textPart in textParts:
if not textPart.startswith('%%'): continue # it is not a code
code = toCode(NAME_POLICY, textPart[2:])
# if this tag code is not in translations dictionary -> error
if not code in translations:
eprintf("ERROR: code '%s' not in resource file!\n", code)
else:
# replace code (introduced with double %% and followed by white character or by the end) with its translation
newText = re.sub(r"%%"+code+"(?=\s|$)", translations[code], tagToReplace.text)
tagToReplace.text = newText
if VERBOSE: printf("-> translated as %s\n", tagToReplace.text)
# OUTPUT NEW DIALOG
if args.output is not None:
with open (args.output, 'w') as outputFile:
outputFile.write(LET.tostring(dialogXML, pretty_print=True, encoding='utf8'))
elif args.inplace:
with open (args.dialog, 'w') as outputFile:
outputFile.write(LET.tostring(dialogXML, pretty_print=True, encoding='utf8'))
else:
sys.stdout.write(LET.tostring(dialogXML, pretty_print=True, encoding='utf8'))
def printNodes(root, parent, dialogJSON):
"""Converts parsed XML to JSON structure
Args:
root (_Element): root of the parsed XML tree - (typically there is element "nodes" )
parent (_Element): initially None, then parent
dialogJSON (string): generated JSON
"""
global firstNode, VERBOSE
# PROCESS SIBLINGS
previousSibling = None
for nodeXML in root: # for each node in nodes
if not (nodeXML.tag == 'node' or nodeXML.tag == 'slot' or nodeXML.tag == 'handler' or nodeXML.tag == 'response'):
continue
# fix name
if nodeXML.find('name') is None:
generateNodeName(nodeXML, '')
else:
validateNodeName(nodeXML)
nodeJSON = {'dialog_node':nodeXML.find('name').text}
dialogJSON.append(nodeJSON)
children = []
# TYPE
if nodeXML.find('type') is not None:
nodeJSON['type'] = nodeXML.find('type').text
elif nodeXML.find('slots') is not None:
nodeJSON['type'] = "frame"
# EVENTNAME
if nodeXML.get('eventName') is not None:
nodeJSON['event_name'] = nodeXML.get('eventName')
nodeJSON['type'] = 'event_handler'
# VARIABLE
if nodeXML.get('variable') is not None:
nodeJSON['variable'] = nodeXML.get('variable')
nodeJSON['type'] = 'slot'
if nodeXML.tag == 'response':
nodeJSON['type'] = 'response_condition'
# CONDITION
if nodeXML.find('condition') is not None:
nodeJSON['conditions'] = nodeXML.find('condition').text
elif 'type' in nodeJSON:
if nodeJSON['type'] == 'default':
nodeJSON['conditions'] = DEFAULT_CONDITION_ELSE
elif nodeJSON['type'] == 'yes':
nodeJSON['conditions'] = DEFAULT_CONDITION_YES
elif nodeJSON['type'] == 'no':
nodeJSON['conditions'] = DEFAULT_CONDITION_NO
elif nodeJSON['type'] == 'slot' or nodeJSON['type'] == 'response_condition' or nodeJSON['type'] == 'event_handler':
None
else:
nodeJSON['conditions'] = DEFAULT_CONDITION_ELSE
else:
nodeJSON['conditions'] = DEFAULT_CONDITION_ELSE
# OUTPUT
if nodeXML.find('output') is not None:
outputNodeXML = nodeXML.find('output')
for responseNodeXML in outputNodeXML.findall('response'): #responses are translated to seperate nodes
children.append(responseNodeXML)
outputNodeXML.remove(responseNodeXML)
# this should be somewhere in generate
if outputNodeXML.text: # if any free text - create an element <text> txt </text> out of it and delete it
if outputNodeXML.text.strip():
outputNodeTextXML = LET.Element('text')
outputNodeTextXML.text = outputNodeXML.text
outputNodeXML.append(outputNodeTextXML)
# TODO save againMessage
outputNodeXML.text = None
if outputNodeXML.find('textValues') is not None: #rename textValues element to text
outputNodeTextXML = outputNodeXML.find('textValues')
outputNodeTextXML.tag = 'text'
convertAll(nodeJSON, outputNodeXML)
# CONTEXT
if nodeXML.find('context') is not None:
convertAll(nodeJSON, nodeXML.find('context'))
# ACTIONS
if nodeXML.find('actions') is not None:
actionsXML = nodeXML.find('actions')
nodeJSON['actions'] = []
for actionXML in actionsXML.findall('action'):
actionJSON = {}
convertAll(actionJSON, actionXML)
nodeJSON['actions'].append(actionJSON['action'])
# GO TO
if nodeXML.find('goto') is not None:
if nodeXML.find('goto').find('target') is None:
eprintf('WARNING: missing goto target in node: %s\n', nodeXML.find('name').text)
elif nodeXML.find('goto').find('target').text == '::FIRST_SIBLING':
nodeXML.find('goto').find('target').text = next(x for x in root if x.tag == 'node').find('name').text
gotoJson = {'dialog_node':nodeXML.find('goto').find('target').text}
gotoJson['selector'] = nodeXML.find('goto').find('selector').text if nodeXML.find('goto').find('selector') is not None else DEFAULT_SELECTOR
nodeJSON['go_to'] = gotoJson
# PARENT
if parent is not None:
nodeJSON['parent'] = parent.find('name').text
# PREVIOUS SIBLING
if previousSibling is not None:
nodeJSON['previous_sibling'] = previousSibling.find('name').text
# CLOSE NODE
previousSibling = nodeXML
# ADD ALL CHILDREN NODES
nodes = nodeXML.find('nodes')
if nodes is not None:
children.extend(nodes)
# ADD ALL SLOTS (FRAME FUNCTIONALITY)
slots = nodeXML.find('slots')
if slots is not None:
children.extend(slots)
# ADD ALL HANDLERS (FRAME FUNCTIONALITY)
handlers = nodeXML.find('handlers')
if handlers is not None:
children.extend(handlers)
# PROCESS ALL CHILDREN
if children:
printNodes(children, nodeXML, dialogJSON)
def generateRepeatNodes(root, parent, settings):
global VERBOSE
if parent is None: return
if VERBOSE: eprintf('Generate repeat nodes for parent: %s START\n', parent.find('name').text if parent is not None else 'root')
# ADD VARIABLE 'attempts_*' TO PARENT'S CONTEXT AND SET IT TO ZERO (FOR SURE)
repeatVarName = 'attempts_' + parent.find('name').text.replace('-', '') # remove hyphens (they cause problems in mathematical expressions where they act as minus signs)
# context
context = parent.find('context')
if context is None:
context = LET.Element('context')
parent.append(context)
contextRepeat = LET.Element(repeatVarName, {'type':'number'})
contextRepeat.text = '0'
context.append(contextRepeat)
# goto for repetation
if root.find('node') is None:
eprintf('Repeat node without options to input something!!!\n')
repeatNodeGoto = LET.Element('goto')
repeatNodeGotoTarget = LET.Element('target')
repeatNodeGotoTarget.text = root.find('node').find('name').text
repeatNodeGoto.append(repeatNodeGotoTarget)
# max attempts
maxAttempts = int(settings.find('attempts').text) if settings is not None and settings.find('attempts') is not None else DEFAULT_REPEAT_ATTEMPTS
if VERBOSE: eprintf('maxAttempts: %s\n', maxAttempts)
# output sentences
outputs = settings.find('outputs').findall('output') if settings.find('outputs') is not None and len(settings.find('outputs').findall('output')) > 0 else DEFAULT_REPEAT_MESS_TEMPLATES['default']
if VERBOSE: eprintf('nOutputs: %s\n', len(outputs))
# LAST NODE (RETURNING TO THE MAIN MENU)
generateRepeatNode(parent, root, outputs[-1], maxAttempts-1, repeatVarName, 0, settings.find('goto'))
if VERBOSE: eprintf('LAST NODE\n')
# MIDDLE NODE
for i in range(min(maxAttempts-1, len(outputs)-1) -1, 0, -1):
generateRepeatNode(parent, root, outputs[i], i, repeatVarName, '<?$' + repeatVarName +' + 1?>', repeatNodeGoto)
if VERBOSE: eprintf('MIDDLE NODE number: %d\n', i)
# FIRST (DEFAULT) NODE
generateRepeatNode(parent, root, outputs[0], 0, repeatVarName, '<? $' + repeatVarName + ' == null ? 0 : $' + repeatVarName + ' + 1 ?>', repeatNodeGoto)
if VERBOSE: eprintf('FIRST NODE\n')
if VERBOSE: eprintf('Generate repeat nodes for parent: %s \n', parent.find('name').text if parent is not None else 'root')
def generateNodes(root, parent, parentAbortSettings, parentAgainSettings, parentBackSettings, parentRepeatSettings, parentGenericSettings):
global parent_map, VERBOSE
# GENERATE NAMES
for node in root.findall('node'):
generateNodeName(node, '')
if VERBOSE: eprintf('Found node: %s in: %s\n', node.find('name').text, parent.find('name').text if parent is not None else 'root')
# READ NODES PROPERTIES
abortSettings = None
againSettings = None
backSettings = None
repeatSettings = None
genericSettings = None
for autogenerate in root.findall('autogenerate'):
if autogenerate.get('type') == 'abort':
if VERBOSE: eprintf('Abort settings found in parent: %s\n', parent.find('name').text if parent is not None else 'root')
abortSettings = autogenerate
if autogenerate.get('type') == 'again':
if VERBOSE: eprintf('Again settings found in parent: %s\n', parent.find('name').text if parent is not None else 'root')
againSettings = autogenerate
if autogenerate.get('type') == 'back':
if VERBOSE: eprintf('Back settings found in parent: %s\n', parent.find('name').text if parent is not None else 'root')
backSettings = autogenerate
if autogenerate.get('type') == 'repeat':
if VERBOSE: eprintf('Repeat settings found in parent: %s\n', parent.find('name').text if parent is not None else 'root')
repeatSettings = autogenerate
if autogenerate.get('type') == 'generic':
if VERBOSE: eprintf('Generic settings found in parent: %s\n', parent.find('name').text if parent is not None else 'root')
genericSettings = autogenerate
abortSettings = mergeSettings(abortSettings, parentAbortSettings)
# TODO discuss how those funcitonality should work and if it is possible to implement it just in conversation
#againSettings = mergeSettings(againSettings, parentAgainSettings)
#backSettings = mergeSettings(backSettings, parentBackSettings)
repeatSettings = mergeSettings(repeatSettings, parentRepeatSettings)
genericSettings = mergeSettings(genericSettings, parentGenericSettings)
# generate if settings exist and are not switched off
abort = True if (abortSettings is not None and not isFalse(abortSettings, 'on')) else False
again = True if (againSettings is not None and not isFalse(againSettings, 'on')) else False
back = True if (backSettings is not None and not isFalse(backSettings, 'on')) else False
repeat = True if (repeatSettings is not None and not isFalse(repeatSettings, 'on')) else False
generic = True if (genericSettings is not None and not isFalse(genericSettings, 'on')) else False
indexOfInsertion = len(root)
for index in range(0, len(root)):
node = root[index]
if node.tag == 'node':
condition = node.find('condition')
# TODO check if we generate condition 'anything_else' for nodes without condition
# we want to generate CONTROL nodes before repeat section
if condition is None or condition.text == 'anything_else' or condition.text.startswith(('anything_else', '$tries')):
indexOfInsertion = index
break;
# GENERATE NEW NODES
if abort:
# ABORT NODE RETURNING TO THE MAIN MENU
root.insert(indexOfInsertion, generateAbortNode(root, parent, abortSettings))
indexOfInsertion = indexOfInsertion + 1
if again:
# AGAIN NODE REPEAT CURRENT STEP
root.insert(indexOfInsertion, generateAgainNode(root, parent, againSettings))
indexOfInsertion = indexOfInsertion + 1
if back:
# BACK NODE RETURNING TO PREVIOUS NODE
root.insert(indexOfInsertion, generateBackNode(root, parent, backSettings))
indexOfInsertion = indexOfInsertion + 1
if generic:
# GENERIC NODE
for genericChild in genericSettings:
genericChildCopy = copy.deepcopy(genericChild)
generateNodeName(genericChildCopy, 'GENERIC_')
root.insert(indexOfInsertion, genericChildCopy)
indexOfInsertion = indexOfInsertion + 1
if repeat:
generateRepeatNodes(root, parent, repeatSettings)
for node in root.findall('node'):
# PROCESS CHILD NODES
children = node.find('nodes')
if children is not None:
# propagate settings only if propagation not switched off
generateNodes(
children,
node,
abortSettings if abortSettings is not None and not isFalse(abortSettings, 'propagate') else None,
againSettings if againSettings is not None and not isFalse(againSettings, 'propagate') else None,
backSettings if backSettings is not None and not isFalse(backSettings, 'propagate') else None,
repeatSettings if repeatSettings is not None and not isFalse(repeatSettings, 'propagate') else None,
genericSettings if genericSettings is not None and not isFalse(genericSettings, 'propagate') else None
)
# OUTPUT NEW DIALOG
if args.output is not None:
with open(args.output, 'w') as outputFile:
outputFile.write(
LET.tostring(dialogsXML, pretty_print=True, encoding='utf8'))
elif args.inplace:
with open(args.dialog, 'w') as outputFile:
outputFile.write(
LET.tostring(dialogsXML, pretty_print=True, encoding='utf8'))
else:
sys.stdout.write(
LET.tostring(dialogsXML, pretty_print=True, encoding='utf8'))
# EXTEND RESOURCE FILE
if args.append:
with open(args.resource, 'r') as resourceFile:
resourceJSON = json.load(resourceFile)
resourceJSON.update(
translations
) # add new translations to existing ones (Duplicate codes will be overwritten by new ones.)
translations = resourceJSON
# CREATE RESOURCE FILE
with open(args.resource, 'w') as resourceFile:
resourceFile.write(
json.dumps(translations, indent=4,
ensure_ascii=False).encode('utf8'))
if VERBOSE: eprintf('Texts were successfully replaced with codes.\n')
def _createOutputElement(self, channels, buttons):
""" Convert output channels into XML structure. """
outputXml = XML.Element('output')
if channels:
for channelName, channelValues in channels.iteritems():
if channelName == '1':
textValuesXml = XML.Element('textValues')
for item in channelValues:
textValuesXml.append(
self._createXmlElement('values', item))
outputXml.append(textValuesXml)
continue
output = self._concatenateOutputs(channelValues)
if channelName == '2':
outputXml.append(self._createXmlElement('timeout', output))
elif channelName == '3':
outputXml.append(self._createXmlElement('sound', output))
elif channelName == '4':
outputXml.append(self._createXmlElement('tts', output))
elif channelName == '5':
outputXml.append(
self._createXmlElement('talking_head', output))
elif channelName == '6':
outputXml.append(
self._createXmlElement('paper_head', output))
elif channelName == '7':
outputXml.append(self._createXmlElement(
'graphics', output))
elif channelName == '8':
outputXml.append(self._createXmlElement('url', output))
else:
eprintf('WARNING: Unrecognized channel: %s, value: %s\n',
channelName, output)
if buttons:
genericXml = XML.Element('generic', structure='listItem')
genericXml.append(self._createXmlElement('response_type',
"option"))
genericXml.append(self._createXmlElement('preference', "button"))
genericXml.append(
self._createXmlElement('title', "Fast selection buttons"))
buttonIndex = 0
for buttonLabel, buttonValue in buttons.iteritems():
if buttonIndex < MAX_OPTIONS:
optionsXml = XML.Element('options')
optionsXml.append(
self._createXmlElement('label', buttonLabel))
optionsXml.append(
self._createXmlOption('value', buttonValue))
genericXml.append(optionsXml)
else:
eprintf(
'Warning: Number of buttons is larger then %s, ignoring: %s, %s\n',
MAX_OPTIONS, buttonLabel, buttonLabel)
buttonIndex += 1
outputXml.append(genericXml)
return outputXml
config = configparser.ConfigParser()
config.read(args.config)
workspacesUrl = config.get(conversationSection, 'url')
version = config.get(conversationSection, 'version')
username = config.get(conversationSection, 'username')
printf('WCS USERNAME: %s\n', username)
password = config.get(conversationSection, 'password')
printf('WCS PASSWORD: %s\n', password)
workspaceId = config.get(conversationSection,
'workspace_id',
fallback=None)
if workspaceId:
printf('WCS WORKSPACE_ID: %s\n', workspaceId)
workspacesUrl += '/' + workspaceId
except IOError:
eprintf('ERROR: Cannot load config file %s\n', args.config)
sys.exit(1)
# wait until workspace is done with training
checkWorkspaceTime = 0
url = workspacesUrl + '?version=' + version
while True:
response = requests.get(url, auth=(username, password))
responseJson = response.json()
if 'error' in responseJson:
eprintf('ERROR: %s\n', responseJson['error'])
sys.exit(1)
status = responseJson['status']
printf('WCS WORKSPACE STATUS: %s\n', status)
if status == 'Available':
break
printf('-- TEST: ' + testName + '\n')
printf('--------------------------------------------------------------------------------\n')
# XML (new dialouge)
dialogXml = LET.Element('testsuite')
expectedJsonLine = expectedJsonFile.readline()
receivedJsonLine = receivedJsonFile.readline()
line = 0
dialogId = 0
# for every line
while expectedJsonLine:
line += 1;
if not receivedJsonLine: # no more received line
eprintf('ERROR: Missing output JSON in file %s, line %d', args.receivedFileName, line)
sys.exit(1)
expectedData = json.loads(expectedJsonLine)
expectedJson = expectedData['output_message']
receivedJson = json.loads(receivedJsonLine)
if (dialogId == 0 or dialogId != expectedData['dialog_id']):
if nDialogs > 0:
# end previous dialog
printf('--------------------------------------------------------------------------------\n')
if nFailuresInDialog: # at least one failure in this dialog
printf('-- TEST RESULT: FAILED, TOTAL FAILURES: %d, LINE OF THE FIRST FAILURE: %d\n', nFailuresInDialog, firstFailedLine)
nDialogsFailed += 1
else:
printf('-- TEST RESULT: OK\n')
def areSame(expectedOutputJson, receivedOutputJson, failureData, parentPath):
printf("ARE SAME: %s and %s\n", expectedOutputJson, receivedOutputJson)
if isinstance(expectedOutputJson, basestring):
if not isinstance(receivedOutputJson, basestring):
failureData['message'] = 'Received output differs in type from expected output.' + " (" + parentPath + ")"
failureData['expectedElement'] = "Element of the type string (" + expectedOutputJson + ")"
failureData['receivedElement'] = "Element of the type " + receivedOutputJson.__class__.__name__
printf("Different type: %s and %s\n", expectedOutputJson, receivedOutputJson)
return False
if expectedOutputJson != receivedOutputJson:
failureData['message'] = 'Received output differs from expected output.' + " (" + parentPath + ")"
failureData['expectedElement'] = expectedOutputJson
failureData['receivedElement'] = receivedOutputJson
printf("NOT SAME: %s and %s\n", expectedOutputJson, receivedOutputJson)
return False
else:
printf('SAME: basestring %s and %s are same\n', expectedOutputJson, receivedOutputJson)
return True
if isinstance(expectedOutputJson, int):
if not isinstance(receivedOutputJson, int):
failureData['message'] = 'Received output differs in type from expected output.' + " (" + parentPath + ")"
failureData['expectedElement'] = "Element of the type int (" + str(expectedOutputJson) + ")"
failureData['receivedElement'] = "Element of the type " + receivedOutputJson.__class__.__name__
printf("Different type: %s and %s\n", expectedOutputJson, receivedOutputJson)
return False
if expectedOutputJson != receivedOutputJson:
failureData['message'] = 'Received output differs from expected output.' + " (" + parentPath + ")"
failureData['expectedElement'] = str(expectedOutputJson)
failureData['receivedElement'] = str(receivedOutputJson)
printf("NOT SAME: %s and %s\n", expectedOutputJson, receivedOutputJson)
return False
else:
printf('SAME: int %s and %s are same\n', expectedOutputJson, receivedOutputJson)
return True
elif isinstance(expectedOutputJson, list):
if not isinstance(receivedOutputJson, list):
failureData['message'] = 'Received output differs in type from expected output.' + " (" + parentPath + ")"
failureData['expectedElement'] = "Element of the type list"
failureData['receivedElement'] = "Element of the type " + receivedOutputJson.__class__.__name__
printf("Different type: %s and %s\n", expectedOutputJson, receivedOutputJson)
return False
if len(expectedOutputJson) != len(receivedOutputJson):
failureData['message'] = 'List in received output differs in length from list in expected output.' + " (" + parentPath + ")"
failureData['expectedElement'] = "List of the length " + str(len(expectedOutputJson))
failureData['receivedElement'] = "List of the length " + str(len(receivedOutputJson))
printf('ERROR: Different list length!\n')
printf('expected %s\n', expectedOutputJson)
printf('received %s\n', receivedOutputJson)
return False
else:
for i in range(len(expectedOutputJson)):
printf("STEP: Item %d\n", i)
if not areSame(expectedOutputJson[i], receivedOutputJson[i], failureData, parentPath + " - " + str(i) + "th item in list"):
printf('ERROR: Different list items in positon %d!\n', i)
return False
return True
elif isinstance(expectedOutputJson, dict):
if not isinstance(receivedOutputJson, dict):
failureData['message'] = 'Received output differs in type from expected output.' + " (" + parentPath + ")"
failureData['expectedElement'] = "Element of the type dict"
failureData['receivedElement'] = "Element of the type " + receivedOutputJson.__class__.__name__
printf("Different type: %s and %s\n", expectedOutputJson, receivedOutputJson)
return False
for elementKey in expectedOutputJson:
printf("STEP: Element key %s\n", elementKey)
if expectedOutputJson[elementKey] is None:
printf("NONE: Element with key %s is none\n", elementKey)
continue
if elementKey not in receivedOutputJson or receivedOutputJson[elementKey] is None:
failureData['message'] = 'Received output has no key ' + elementKey + '.' + " (" + parentPath + ")"
failureData['expectedElement'] = "Dict with key " + elementKey
failureData['receivedElement'] = "None"
printf('ERROR: Missing key in received json!\n')
return False
else:
if not areSame(expectedOutputJson[elementKey], receivedOutputJson[elementKey], failureData, parentPath + " - " + elementKey):
printf('ERROR: Different dict items for key %s!\n', elementKey)
return False
return True
else:
eprintf('ERROR: Unsupported type of element %s, type %s!\n', str(expectedOutputJson), expectedOutputJson.__class__.__name__)
return False
if not hasattr(config, 'common_outputs_directory') or not getattr(
config, 'common_outputs_directory'):
print('ERROR: common_outputs_directory parameter not defined.')
exit(1)
if not hasattr(config, 'common_outputs_workspace') or not getattr(
config, 'common_outputs_workspace'):
print('ERROR: common_outputs_workspace parameter not defined.')
exit(1)
try:
workspaceFilePath = os.path.join(
getattr(config, 'common_outputs_directory'),
getattr(config, 'common_outputs_workspace'))
with open(workspaceFilePath, 'r') as workspaceFile:
workspace = json.load(workspaceFile)
except IOError:
eprintf('ERROR: Cannot load workspace file %s\n', workspaceFilePath)
sys.exit(1)
if hasattr(config, 'conversation_workspace_name'):
workspace['name'] = getattr(config, 'conversation_workspace_name')
else:
print('WARNING: conversation_workspace_name parameter not defined')
# credentials
if not hasattr(config, 'conversation_username') or not getattr(
config, 'conversation_username'):
print('ERROR: con_username parameter not defined.')
exit(1)
username = getattr(config, 'conversation_username')
if not hasattr(config, 'conversation_password') or not getattr(
config, 'conversation_password'):
print('ERROR: con_password parameter not defined.')
setattr(config, 'cloudfunctions_path_to_actions', '/' + '/'.join([getattr(config, 'cloudfunctions_namespace').strip("/"), getattr(config, 'cloudfunctions_package').strip("/")]).strip("/") + '/')
# load dialogue from XML
if hasattr(config, 'common_dialog_main'):
dialogTree = LET.parse(getattr(config, 'common_dialog_main'))
else:
dialogTree = LET.parse(sys.stdin)
# load schema
schemaDirname, this_filename = os.path.split(os.path.abspath(__file__))
if not hasattr(config, 'common_schema') or getattr(config, 'common_schema') is None:
setattr(config, 'common_schema', schemaDirname+'/../data_spec/dialog_schema.xml')
printf('WARNING: Schema not found, using default path /../data_spec/dialog_schema.xml\n;')
schemaFile = os.path.join(schemaDirname, getattr(config, 'common_schema'))
if not os.path.exists(schemaFile):
eprintf('ERROR: Schema file %s not found.\n', schemaFile)
exit(1)
schemaTree = LET.parse(schemaFile)
schema = LET.XMLSchema(schemaTree)
validate(dialogTree)
# process dialog tree
root = dialogTree.getroot()
rootGlobal = root
importNodes(root, config)
# remove all comments
removeAllComments(dialogTree)
# find all node names
names = findAllNodeNames(dialogTree)
