def test_ifFilePathIsFound(self):
"""Tests if getFilesAtPath returns file when it is directly supplied in input list."""
filePath = os.path.join(self.dataBasePath, 'directory1', 'file1.test')
pathList = wawCommons.getFilesAtPath([filePath])
absPath = os.path.abspath(filePath)
assert len(pathList) == 1
assert absPath in pathList
def test_ifFileInDirectoryIsNotFound(self):
"""Tests if particular file is not found by getFilesAtPath when it's not contained in supplied directory."""
directory = os.path.join(self.dataBasePath, 'directory1')
anotherDirectory = os.path.join(self.dataBasePath, 'directory2')
pathList = wawCommons.getFilesAtPath([directory])
testFile = os.path.abspath(os.path.join(anotherDirectory,
"file2.test"))
assert testFile not in pathList
def test_rangeInPattern(self):
"""Tests if getFilesAtPath can correctly handle question mark in pattern."""
directory = os.path.join(self.dataBasePath, 'directory1')
pathList = wawCommons.getFilesAtPath([directory], ["file[0-9].*"])
testFile1 = os.path.abspath(os.path.join(directory, "file1.test"))
assert testFile1 in pathList
testFile2 = os.path.abspath(os.path.join(directory, "fileA.ext"))
assert testFile2 not in pathList
def test_multiplePattterns(self):
"""Tests if mutiple patterns in getFilesAtPath behave like there is operator OR between them."""
directory = os.path.join(self.dataBasePath, 'directory1')
pathList = wawCommons.getFilesAtPath([directory], ["*.test", "*"])
testFile1 = os.path.abspath(os.path.join(directory, "file1.test"))
assert testFile1 in pathList
testFile2 = os.path.abspath(os.path.join(directory, "fileA.ext"))
assert testFile2 in pathList
def test_filePatternsInDirectory(self):
"""Tests if files returned from getFilesAtPath can be filtered by patterns parameter."""
directory = os.path.join(self.dataBasePath, 'directory1')
pathList = wawCommons.getFilesAtPath([directory], ["*.test"])
testFile1 = os.path.abspath(os.path.join(directory, "file1.test"))
assert testFile1 in pathList
testFile2 = os.path.abspath(os.path.join(directory, "fileA.ext"))
assert testFile2 not in pathList
def test_ifFileInDirectoryIsFound(self):
"""Tests if particular files are found by getFilesAtPath when it's contained in supplied directory."""
directory = os.path.join(self.dataBasePath, 'directory1')
pathList = wawCommons.getFilesAtPath([directory])
testFile1 = os.path.abspath(os.path.join(directory, "file1.test"))
assert testFile1 in pathList
testFile2 = os.path.abspath(os.path.join(directory, "fileA.ext"))
assert testFile2 in pathList
def test_pattternMatchMultipleFiles(self):
"""Tests if pattern in getFilesAtPath can match multiple files."""
directory = os.path.join(self.dataBasePath, 'directory1')
pathList = wawCommons.getFilesAtPath([directory], ["*"])
testFile1 = os.path.abspath(os.path.join(directory, "file1.test"))
assert testFile1 in pathList
testFile2 = os.path.abspath(os.path.join(directory, "fileA.ext"))
assert testFile2 in pathList
def test_filePathAndDirectoryCombination(self):
"""Tests if getFilesAtPath when there is a combination of file and directory in input parameter."""
filePath = os.path.join(self.dataBasePath, 'directory1', 'file1.test')
directory = os.path.join(self.dataBasePath, 'directory2')
pathList = wawCommons.getFilesAtPath([filePath, directory])
testFile1 = os.path.abspath(filePath)
assert testFile1 in pathList
testFile2 = os.path.abspath(os.path.join(directory, "file2.test"))
assert testFile2 in pathList
def test_ifFileInOneDirectoryIsFound(self):
"""Tests if particular file is found by getFilesAtPath when it's contained in one of supplied directories."""
directory1 = os.path.join(self.dataBasePath, 'directory1')
directory2 = os.path.join(self.dataBasePath, 'directory2')
pathList = wawCommons.getFilesAtPath([directory1, directory2])
testFile1 = os.path.abspath(os.path.join(directory1, "file1.test"))
assert testFile1 in pathList
testFile2 = os.path.abspath(os.path.join(directory2, "file2.test"))
assert testFile2 in pathList
def test_nonExistentAndExistentFile(self):
"""Tests if getFilesAtPath ignores non-existent files in list."""
nonExistentFile = os.path.join(self.dataBasePath,
'non_existent_file_123456789.test')
existentFile = os.path.join(self.dataBasePath, 'directory1',
'file1.test')
pathList = wawCommons.getFilesAtPath([nonExistentFile, existentFile])
absPath = os.path.abspath(existentFile)
assert len(pathList) == 1
assert absPath in pathList
def test_ifFilePathsIsFound(self):
"""Tests if getFilesAtPath returns files when they are directly supplied in input list."""
filePath1 = os.path.join(self.dataBasePath, 'directory1', 'file1.test')
filePath2 = os.path.join(self.dataBasePath, 'directory2', 'file2.test')
pathList = wawCommons.getFilesAtPath([filePath1, filePath2])
assert len(pathList) == 2
absPath1 = os.path.abspath(filePath1)
assert absPath1 in pathList
absPath2 = os.path.abspath(filePath2)
assert absPath2 in pathList
def test_nonExistentAndExistentDirectory(self):
"""Tests if getFilesAtPath ignores non-existent directory in list."""
nonExistentDir = os.path.join(self.dataBasePath,
'non_existent_dir_123456789/')
existentDir = os.path.join(self.dataBasePath, 'directory1')
pathList = wawCommons.getFilesAtPath([nonExistentDir, existentDir])
testFile1 = os.path.abspath(os.path.join(existentDir, "file1.test"))
assert testFile1 in pathList
testFile2 = os.path.abspath(os.path.join(existentDir, "fileA.ext"))
assert testFile2 in pathList
def test_filePatternsOnFilePath(self):
"""Tests if file returned from getFilesAtPath can be filtered by patterns parameter."""
filePath1 = os.path.join(self.dataBasePath, 'directory1', 'file1.test')
filePath2 = os.path.join(self.dataBasePath, 'directory1', 'fileA.ext')
pathList = wawCommons.getFilesAtPath([filePath1, filePath2],
["*.test"])
assert len(pathList) == 1
absPath1 = os.path.abspath(filePath1)
assert absPath1 in pathList
absPath2 = os.path.abspath(filePath2)
assert absPath2 not in pathList
def test_multiplePattternsOnMultipleDirectories(self):
"""Tests if getFilesAtPath can handle multiple patterns combined with mutiple input directories."""
directory1 = os.path.join(self.dataBasePath, 'directory1')
directory2 = os.path.join(self.dataBasePath, 'directory2')
pathList = wawCommons.getFilesAtPath([directory1, directory2],
["*.test", "*"])
testFile1 = os.path.abspath(os.path.join(directory1, "file1.test"))
assert testFile1 in pathList
testFile2 = os.path.abspath(os.path.join(directory1, "fileA.ext"))
assert testFile2 in pathList
testFile3 = os.path.abspath(os.path.join(directory2, "file2.test"))
assert testFile3 in pathList
def test_multiplePattternsOnFileDirectoryCombination(self):
"""Tests if getFilesAtPath can handle multiple patterns combined with input directory and separated file."""
filePath = os.path.join(self.dataBasePath, 'directory1', 'file1.test')
directory = os.path.join(self.dataBasePath, 'directory2')
pathList = wawCommons.getFilesAtPath([filePath, directory],
["*.test", "*"])
testFile1 = os.path.abspath(filePath)
assert testFile1 in pathList
testFile2 = os.path.abspath(
os.path.join(self.dataBasePath, 'directory1', "fileA.ext"))
assert testFile2 not in pathList
testFile3 = os.path.abspath(os.path.join(directory, "file2.test"))
assert testFile3 in pathList
) + '/packages/' + config.cloudfunctions_package + '?overwrite=true'
response = requests.put(package_url,
auth=(config.cloudfunctions_username,
config.cloudfunctions_password),
headers={'Content-Type': 'application/json'},
data='{}')
responseJson = response.json()
if 'error' in responseJson:
eprintf('\nCannot create cloud functions package\nERROR: %s\n',
responseJson['error'])
if VERBOSE: printf("%s", responseJson)
sys.exit(1)
else:
printf('\nCloud functions package successfully uploaded\n')
filesAtPath = getFilesAtPath(config.common_functions)
for functionFileName in filesAtPath:
fname = os.path.basename(functionFileName)
function_url = 'https://openwhisk.ng.bluemix.net/api/v1/namespaces/' + config.cloudfunctions_namespace + '/actions/' + config.cloudfunctions_package + '/' + fname + '?overwrite=true'
code = open(os.path.join(config.common_functions, functionFileName),
'r').read()
payload = {"exec": {"kind": "nodejs:default", "code": code}}
response = requests.put(function_url,
auth=(config.cloudfunctions_username,
config.cloudfunctions_password),
headers={'Content-Type': 'application/json'},
data=json.dumps(payload),
verify=False)
responseJson = response.json()
def test_nonExistentFile(self):
"""Tests if getFilesAtPath ignores non-existent file."""
nonExistentFile = os.path.join(self.dataBasePath,
'non_existent_file_123456789.test')
pathList = wawCommons.getFilesAtPath([nonExistentFile])
assert len(pathList) == 0
if not hasattr(config, 'common_intents'):
print('intents parameter is not defined.')
if not hasattr(config, 'common_generated_intents'):
print('generated_intents parameter is not defined, ignoring')
if not hasattr(config, 'common_outputs_intents'):
print(
'Outputs_intents parameter is not defined, output will be generated to console.'
)
intents = []
pathList = getattr(config, 'common_intents')
if hasattr(config, 'common_generated_intents'):
pathList = pathList + getattr(config, 'common_generated_intents')
filesAtPath = getFilesAtPath(pathList)
for intentFileName in filesAtPath:
intentName = toIntentName(
NAME_POLICY, args.common_intents_nameCheck,
os.path.splitext(os.path.basename(intentFileName))[0])
with codecs.open(intentFileName, encoding='utf8') as intentFile:
intent = {}
intent['intent'] = intentName
examples = []
for line in intentFile:
# remove comments
line = line.split('#')[0]
line = line.rstrip().lower()
if line and not line in examples:
examples.append(line)
elif line in examples:
def test_nonExistentDirectory(self):
"""Tests if getFilesAtPath can handle non-existent directory."""
nonExistentDir = os.path.join(self.dataBasePath,
'non_existent_dir_123456789/')
pathList = wawCommons.getFilesAtPath([nonExistentDir])
assert len(pathList) == 0
def main(argv):
# parse sequence names - because we need to get the name first and
# then create corresponding arguments for the main parser
sequenceSubparser = argparse.ArgumentParser()
sequenceSubparser.add_argument('--cloudfunctions_sequences', nargs='+')
argvWithoutHelp = list(argv)
if "--help" in argv: argvWithoutHelp.remove("--help")
if "-h" in argv: argvWithoutHelp.remove("-h")
sequenceNames = sequenceSubparser.parse_known_args(
argvWithoutHelp)[0].cloudfunctions_sequences or []
parser = argparse.ArgumentParser(
description="Deploys the cloud functions",
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('-v',
'--verbose',
required=False,
help='verbosity',
action='store_true')
parser.add_argument('-c',
'--common_configFilePaths',
help="configuaration file",
action='append')
parser.add_argument('--common_functions',
required=False,
help="directory where the cloud functions are located")
parser.add_argument('--cloudfunctions_namespace',
required=False,
help="cloud functions namespace")
parser.add_argument('--cloudfunctions_apikey',
required=False,
help="cloud functions apikey")
parser.add_argument('--cloudfunctions_username',
required=False,
help="cloud functions user name")
parser.add_argument('--cloudfunctions_password',
required=False,
help="cloud functions password")
parser.add_argument('--cloudfunctions_package',
required=False,
help="cloud functions package name")
parser.add_argument('--cloudfunctions_url',
required=False,
help="url of cloud functions API")
parser.add_argument('--log',
type=str.upper,
default=None,
choices=list(logging._levelToName.values()))
parser.add_argument('--cloudfunctions_sequences',
nargs='+',
required=False,
help="cloud functions sequence names")
for runtime in list(interpretedRuntimes.values()) + list(
compiledRuntimes.values()):
parser.add_argument('--cloudfunctions_' + runtime + '_version',
required=False,
help="cloud functions " + runtime + " version")
# Add arguments for each sequence to be able to define the functions in the sequence
for sequenceName in sequenceNames:
try:
parser.add_argument("--cloudfunctions_sequence_" + sequenceName,
required=True,
help="functions in sequence '" + sequenceName +
"'")
except argparse.ArgumentError as e:
if "conflicting option" in str(e):
# from None is needed in order to show only the custom exception and not the whole traceback
# (It would read as 'During handling of the above exception, another exception has occurred', but we DID handle it)
raise argparse.ArgumentError(
None, "Duplicate sequence name: " + sequenceName) from None
else:
raise e
args = parser.parse_args(argv)
if __name__ == '__main__':
setLoggerConfig(args.log, args.verbose)
logger.info('STARTING: ' + os.path.basename(__file__))
def handleResponse(response):
"""Get response code and show an error if it's not OK"""
code = response.status_code
if code != requests.codes.ok:
if code == 401:
logger.error(
"Authorization error. Check your credentials. (Error code "
+ str(code) + ")")
elif code == 403:
logger.error(
"Access is forbidden. Check your credentials and permissions. (Error code "
+ str(code) + ")")
elif code == 404:
logger.error(
"The resource could not be found. Check your cloudfunctions url and namespace. (Error code "
+ str(code) + ")")
elif code == 408:
logger.error("Request Timeout. (Error code " + str(code) + ")")
elif code >= 500:
logger.error("Internal server error. (Error code " +
str(code) + ")")
else:
logger.error("Unexpected error code: " + str(code))
errorsInResponse(response.json())
return False
return True
config = Cfg(args)
namespace = getRequiredParameter(config, 'cloudfunctions_namespace')
urlNamespace = quote(namespace)
auth = getParametersCombination(
config, 'cloudfunctions_apikey',
['cloudfunctions_password', 'cloudfunctions_username'])
package = getRequiredParameter(config, 'cloudfunctions_package')
cloudFunctionsUrl = getRequiredParameter(config, 'cloudfunctions_url')
functionDir = getRequiredParameter(config, 'common_functions')
# If sequence names are already defined (from console), do nothing. Else look for them in the configuration.
if not sequenceNames:
sequenceNames = getOptionalParameter(config,
'cloudfunctions_sequences') or []
# SequenceNames has to be a list
if type(sequenceNames) is str:
sequenceNames = [sequenceNames]
# Create a dict of {<seqName>: [<functions 1>, <function2> ,...]}
sequences = {
seqName: getRequiredParameter(config,
"cloudfunctions_sequence_" + seqName)
for seqName in sequenceNames
}
if 'cloudfunctions_apikey' in auth:
username, password = convertApikeyToUsernameAndPassword(
auth['cloudfunctions_apikey'])
else:
username = auth['cloudfunctions_username']
password = auth['cloudfunctions_password']
runtimeVersions = {}
for ext, runtime in list(interpretedRuntimes.items()) + list(
compiledRuntimes.items()):
runtimeVersions[runtime] = runtime + ':' + getattr(
config, 'cloudfunctions_' + runtime + '_version', 'default')
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
packageUrl = cloudFunctionsUrl + '/' + urlNamespace + '/packages/' + package + '?overwrite=true'
logger.info("Will create cloudfunctions package %s.", package)
response = requests.put(packageUrl,
auth=(username, password),
headers={'Content-Type': 'application/json'},
data='{}')
if not handleResponse(response):
logger.critical("Cannot create cloud functions package %s.", package)
sys.exit(1)
else:
logger.info('Cloud functions package successfully uploaded')
filesAtPath = getFilesAtPath(functionDir, [
'*' + ext for ext in (list(interpretedRuntimes) +
list(compiledRuntimes) + compressedFiles)
])
logger.info("Will deploy functions at paths %s.", functionDir)
for functionFilePath in filesAtPath:
fileName = os.path.basename(functionFilePath)
(funcName, ext) = os.path.splitext(fileName)
runtime = None
binary = False
# if the file is zip, it's necessary to look inside
if ext == '.zip':
runtime = _getZipPackageType(functionFilePath)
if not runtime:
logger.warning(
"Cannot determine function type from zip file '%s'. Skipping!",
functionFilePath)
continue
binary = True
else:
if ext in interpretedRuntimes:
runtime = interpretedRuntimes[ext]
binary = False
elif ext in compiledRuntimes:
runtime = compiledRuntimes[ext]
binary = True
else:
logger.warning(
"Cannot determine function type of '%s'. Skipping!",
functionFilePath)
continue
functionUrl = cloudFunctionsUrl + '/' + urlNamespace + '/actions/' + package + '/' + funcName + '?overwrite=true'
if binary:
content = base64.b64encode(open(functionFilePath,
'rb').read()).decode('utf-8')
else:
content = open(functionFilePath, 'r').read()
payload = {
'exec': {
'kind': runtimeVersions[runtime],
'binary': binary,
'code': content
}
}
logger.verbose("Deploying function %s", funcName)
response = requests.put(functionUrl,
auth=(username, password),
headers={'Content-Type': 'application/json'},
data=json.dumps(payload),
verify=False)
if not handleResponse(response):
logger.critical("Cannot deploy cloud function %s.", funcName)
sys.exit(1)
else:
logger.verbose('Cloud function %s successfully deployed.',
funcName)
logger.info("Cloudfunctions successfully deployed.")
if sequences:
logger.info("Will deploy cloudfunction sequences.")
for seqName in sequences:
sequenceUrl = cloudFunctionsUrl + '/' + urlNamespace + '/actions/' + package + '/' + seqName + '?overwrite=true'
functionNames = sequences[seqName]
fullFunctionNames = [
namespace + '/' + package + '/' + functionName
for functionName in functionNames
]
payload = {
'exec': {
'kind': 'sequence',
'binary': False,
'components': fullFunctionNames
}
}
logger.verbose("Deploying cloudfunctions sequence '%s': %s", seqName,
functionNames)
response = requests.put(sequenceUrl,
auth=(username, password),
headers={'Content-Type': 'application/json'},
data=json.dumps(payload),
verify=False)
if not handleResponse(response):
logger.critical("Cannot deploy cloudfunctions sequence %s",
seqName)
sys.exit(1)
else:
logger.verbose("Sequence '%s' deployed.", seqName)
if sequences:
logger.info("Cloudfunction sequences successfully deployed.")
logger.info('FINISHING: ' + os.path.basename(__file__))
def test_emptyPatterns(self):
"""Tests if getFilesAtPath returns no files when no patterns supplied."""
directory = self.dataBasePath
pathList = wawCommons.getFilesAtPath([directory], [])
assert len(pathList) == 0
def test_directoryPattern(self):
"""Tests if pattern containing directory separator does not match any file (this is intended behavior!)."""
directory = self.dataBasePath
pathList = wawCommons.getFilesAtPath(
[directory], [os.path.join("directory1", "*.test")])
assert len(pathList) == 0
def test_emptyInputList(self):
"""Tests if getFilesAtPath function works when pathList parameter is empty."""
pathList = wawCommons.getFilesAtPath([])
assert len(pathList) == 0
def main(argv):
parser = argparse.ArgumentParser(
description=
'Converts intent csv files to .json format of Watson Conversation Service',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('-c',
'--common_configFilePaths',
help='configuaration file',
action='append')
parser.add_argument('-oc',
'--common_output_config',
help='output configuration file')
parser.add_argument(
'-ii',
'--common_intents',
help=
'directory with intent csv files to be processed (all of them will be included in output json)',
action='append') #-gi is functionsally equivalent to -ii
parser.add_argument(
'-gi',
'--common_generated_intents',
help=
'directory with generated intent csv files to be processed (all of them will be included in output json)',
action='append')
parser.add_argument(
'-od',
'--common_outputs_directory',
required=False,
help='directory where the otputs will be stored (outputs is default)')
parser.add_argument('-oi',
'--common_outputs_intents',
help='file with output json with all the intents')
parser.add_argument(
'-ni',
'--common_intents_nameCheck',
action='append',
nargs=2,
help=
"regex and replacement for intent name check, e.g. '-' '_' for to replace hyphens for underscores or '$special' '\\L' for lowercase"
)
parser.add_argument(
'-s',
'--soft',
required=False,
help=
'soft name policy - change intents and entities names without error.',
action='store_true',
default="")
parser.add_argument('-v',
'--verbose',
required=False,
help='verbosity',
action='store_true')
parser.add_argument('--log',
type=str.upper,
default=None,
choices=list(logging._levelToName.values()))
args = parser.parse_args(argv)
if __name__ == '__main__':
setLoggerConfig(args.log, args.verbose)
config = Cfg(args)
NAME_POLICY = 'soft' if args.soft else 'hard'
logger.info('STARTING: ' + os.path.basename(__file__))
if not hasattr(config, 'common_intents'):
logger.info('intents parameter is not defined.')
if not hasattr(config, 'common_generated_intents'):
logger.info('generated_intents parameter is not defined, ignoring')
if not hasattr(config, 'common_outputs_intents'):
logger.info(
'Outputs_intents parameter is not defined, output will be generated to console.'
)
intents = []
pathList = getattr(config, 'common_intents')
if hasattr(config, 'common_generated_intents'):
pathList = pathList + getattr(config, 'common_generated_intents')
filesAtPath = getFilesAtPath(pathList)
for intentFileName in sorted(filesAtPath):
intentName = toIntentName(
NAME_POLICY, args.common_intents_nameCheck,
os.path.splitext(os.path.basename(intentFileName))[0])
with openFile(intentFileName, 'r', encoding='utf8') as intentFile:
intent = {}
intent['intent'] = intentName
examples = []
for line in intentFile:
# remove comments
line = line.split('#')[0]
line = line.rstrip().lower()
#non-ascii characters fix
#line = line.encode('utf-8')
if line:
example = processExample(line, intentName, examples)
#adding to the list
if example:
examples.append(example)
intent['examples'] = examples
intents.append(intent)
if hasattr(config, 'common_outputs_directory') and hasattr(
config, 'common_outputs_intents'):
if not os.path.exists(getattr(config, 'common_outputs_directory')):
os.makedirs(getattr(config, 'common_outputs_directory'))
logger.info('Created new output directory ' +
getattr(config, 'common_outputs_directory'))
with codecs.open(os.path.join(
getattr(config, 'common_outputs_directory'),
getattr(config, 'common_outputs_intents')),
'w',
encoding='utf8') as outputFile:
outputFile.write(json.dumps(intents, indent=4, ensure_ascii=False))
else:
print(json.dumps(intents, indent=4, ensure_ascii=False))
logger.info('FINISHING: ' + os.path.basename(__file__))
