def test_ifFilePathIsFound(self): """Tests if getFilesAtPath returns file when it is directly supplied in input list.""" filePath = os.path.join(self.dataBasePath, 'directory1', 'file1.test') pathList = wawCommons.getFilesAtPath([filePath]) absPath = os.path.abspath(filePath) assert len(pathList) == 1 assert absPath in pathList
def test_ifFileInDirectoryIsNotFound(self): """Tests if particular file is not found by getFilesAtPath when it's not contained in supplied directory.""" directory = os.path.join(self.dataBasePath, 'directory1') anotherDirectory = os.path.join(self.dataBasePath, 'directory2') pathList = wawCommons.getFilesAtPath([directory]) testFile = os.path.abspath(os.path.join(anotherDirectory, "file2.test")) assert testFile not in pathList
def test_rangeInPattern(self): """Tests if getFilesAtPath can correctly handle question mark in pattern.""" directory = os.path.join(self.dataBasePath, 'directory1') pathList = wawCommons.getFilesAtPath([directory], ["file[0-9].*"]) testFile1 = os.path.abspath(os.path.join(directory, "file1.test")) assert testFile1 in pathList testFile2 = os.path.abspath(os.path.join(directory, "fileA.ext")) assert testFile2 not in pathList
def test_multiplePattterns(self): """Tests if mutiple patterns in getFilesAtPath behave like there is operator OR between them.""" directory = os.path.join(self.dataBasePath, 'directory1') pathList = wawCommons.getFilesAtPath([directory], ["*.test", "*"]) testFile1 = os.path.abspath(os.path.join(directory, "file1.test")) assert testFile1 in pathList testFile2 = os.path.abspath(os.path.join(directory, "fileA.ext")) assert testFile2 in pathList
def test_filePatternsInDirectory(self): """Tests if files returned from getFilesAtPath can be filtered by patterns parameter.""" directory = os.path.join(self.dataBasePath, 'directory1') pathList = wawCommons.getFilesAtPath([directory], ["*.test"]) testFile1 = os.path.abspath(os.path.join(directory, "file1.test")) assert testFile1 in pathList testFile2 = os.path.abspath(os.path.join(directory, "fileA.ext")) assert testFile2 not in pathList
def test_ifFileInDirectoryIsFound(self): """Tests if particular files are found by getFilesAtPath when it's contained in supplied directory.""" directory = os.path.join(self.dataBasePath, 'directory1') pathList = wawCommons.getFilesAtPath([directory]) testFile1 = os.path.abspath(os.path.join(directory, "file1.test")) assert testFile1 in pathList testFile2 = os.path.abspath(os.path.join(directory, "fileA.ext")) assert testFile2 in pathList
def test_pattternMatchMultipleFiles(self): """Tests if pattern in getFilesAtPath can match multiple files.""" directory = os.path.join(self.dataBasePath, 'directory1') pathList = wawCommons.getFilesAtPath([directory], ["*"]) testFile1 = os.path.abspath(os.path.join(directory, "file1.test")) assert testFile1 in pathList testFile2 = os.path.abspath(os.path.join(directory, "fileA.ext")) assert testFile2 in pathList
def test_filePathAndDirectoryCombination(self): """Tests if getFilesAtPath when there is a combination of file and directory in input parameter.""" filePath = os.path.join(self.dataBasePath, 'directory1', 'file1.test') directory = os.path.join(self.dataBasePath, 'directory2') pathList = wawCommons.getFilesAtPath([filePath, directory]) testFile1 = os.path.abspath(filePath) assert testFile1 in pathList testFile2 = os.path.abspath(os.path.join(directory, "file2.test")) assert testFile2 in pathList
def test_ifFileInOneDirectoryIsFound(self): """Tests if particular file is found by getFilesAtPath when it's contained in one of supplied directories.""" directory1 = os.path.join(self.dataBasePath, 'directory1') directory2 = os.path.join(self.dataBasePath, 'directory2') pathList = wawCommons.getFilesAtPath([directory1, directory2]) testFile1 = os.path.abspath(os.path.join(directory1, "file1.test")) assert testFile1 in pathList testFile2 = os.path.abspath(os.path.join(directory2, "file2.test")) assert testFile2 in pathList
def test_nonExistentAndExistentFile(self): """Tests if getFilesAtPath ignores non-existent files in list.""" nonExistentFile = os.path.join(self.dataBasePath, 'non_existent_file_123456789.test') existentFile = os.path.join(self.dataBasePath, 'directory1', 'file1.test') pathList = wawCommons.getFilesAtPath([nonExistentFile, existentFile]) absPath = os.path.abspath(existentFile) assert len(pathList) == 1 assert absPath in pathList
def test_ifFilePathsIsFound(self): """Tests if getFilesAtPath returns files when they are directly supplied in input list.""" filePath1 = os.path.join(self.dataBasePath, 'directory1', 'file1.test') filePath2 = os.path.join(self.dataBasePath, 'directory2', 'file2.test') pathList = wawCommons.getFilesAtPath([filePath1, filePath2]) assert len(pathList) == 2 absPath1 = os.path.abspath(filePath1) assert absPath1 in pathList absPath2 = os.path.abspath(filePath2) assert absPath2 in pathList
def test_nonExistentAndExistentDirectory(self): """Tests if getFilesAtPath ignores non-existent directory in list.""" nonExistentDir = os.path.join(self.dataBasePath, 'non_existent_dir_123456789/') existentDir = os.path.join(self.dataBasePath, 'directory1') pathList = wawCommons.getFilesAtPath([nonExistentDir, existentDir]) testFile1 = os.path.abspath(os.path.join(existentDir, "file1.test")) assert testFile1 in pathList testFile2 = os.path.abspath(os.path.join(existentDir, "fileA.ext")) assert testFile2 in pathList
def test_filePatternsOnFilePath(self): """Tests if file returned from getFilesAtPath can be filtered by patterns parameter.""" filePath1 = os.path.join(self.dataBasePath, 'directory1', 'file1.test') filePath2 = os.path.join(self.dataBasePath, 'directory1', 'fileA.ext') pathList = wawCommons.getFilesAtPath([filePath1, filePath2], ["*.test"]) assert len(pathList) == 1 absPath1 = os.path.abspath(filePath1) assert absPath1 in pathList absPath2 = os.path.abspath(filePath2) assert absPath2 not in pathList
def test_multiplePattternsOnMultipleDirectories(self): """Tests if getFilesAtPath can handle multiple patterns combined with mutiple input directories.""" directory1 = os.path.join(self.dataBasePath, 'directory1') directory2 = os.path.join(self.dataBasePath, 'directory2') pathList = wawCommons.getFilesAtPath([directory1, directory2], ["*.test", "*"]) testFile1 = os.path.abspath(os.path.join(directory1, "file1.test")) assert testFile1 in pathList testFile2 = os.path.abspath(os.path.join(directory1, "fileA.ext")) assert testFile2 in pathList testFile3 = os.path.abspath(os.path.join(directory2, "file2.test")) assert testFile3 in pathList
def test_multiplePattternsOnFileDirectoryCombination(self): """Tests if getFilesAtPath can handle multiple patterns combined with input directory and separated file.""" filePath = os.path.join(self.dataBasePath, 'directory1', 'file1.test') directory = os.path.join(self.dataBasePath, 'directory2') pathList = wawCommons.getFilesAtPath([filePath, directory], ["*.test", "*"]) testFile1 = os.path.abspath(filePath) assert testFile1 in pathList testFile2 = os.path.abspath( os.path.join(self.dataBasePath, 'directory1', "fileA.ext")) assert testFile2 not in pathList testFile3 = os.path.abspath(os.path.join(directory, "file2.test")) assert testFile3 in pathList
) + '/packages/' + config.cloudfunctions_package + '?overwrite=true' response = requests.put(package_url, auth=(config.cloudfunctions_username, config.cloudfunctions_password), headers={'Content-Type': 'application/json'}, data='{}') responseJson = response.json() if 'error' in responseJson: eprintf('\nCannot create cloud functions package\nERROR: %s\n', responseJson['error']) if VERBOSE: printf("%s", responseJson) sys.exit(1) else: printf('\nCloud functions package successfully uploaded\n') filesAtPath = getFilesAtPath(config.common_functions) for functionFileName in filesAtPath: fname = os.path.basename(functionFileName) function_url = 'https://openwhisk.ng.bluemix.net/api/v1/namespaces/' + config.cloudfunctions_namespace + '/actions/' + config.cloudfunctions_package + '/' + fname + '?overwrite=true' code = open(os.path.join(config.common_functions, functionFileName), 'r').read() payload = {"exec": {"kind": "nodejs:default", "code": code}} response = requests.put(function_url, auth=(config.cloudfunctions_username, config.cloudfunctions_password), headers={'Content-Type': 'application/json'}, data=json.dumps(payload), verify=False) responseJson = response.json()
def test_nonExistentFile(self): """Tests if getFilesAtPath ignores non-existent file.""" nonExistentFile = os.path.join(self.dataBasePath, 'non_existent_file_123456789.test') pathList = wawCommons.getFilesAtPath([nonExistentFile]) assert len(pathList) == 0
if not hasattr(config, 'common_intents'): print('intents parameter is not defined.') if not hasattr(config, 'common_generated_intents'): print('generated_intents parameter is not defined, ignoring') if not hasattr(config, 'common_outputs_intents'): print( 'Outputs_intents parameter is not defined, output will be generated to console.' ) intents = [] pathList = getattr(config, 'common_intents') if hasattr(config, 'common_generated_intents'): pathList = pathList + getattr(config, 'common_generated_intents') filesAtPath = getFilesAtPath(pathList) for intentFileName in filesAtPath: intentName = toIntentName( NAME_POLICY, args.common_intents_nameCheck, os.path.splitext(os.path.basename(intentFileName))[0]) with codecs.open(intentFileName, encoding='utf8') as intentFile: intent = {} intent['intent'] = intentName examples = [] for line in intentFile: # remove comments line = line.split('#')[0] line = line.rstrip().lower() if line and not line in examples: examples.append(line) elif line in examples:
def test_nonExistentDirectory(self): """Tests if getFilesAtPath can handle non-existent directory.""" nonExistentDir = os.path.join(self.dataBasePath, 'non_existent_dir_123456789/') pathList = wawCommons.getFilesAtPath([nonExistentDir]) assert len(pathList) == 0
def main(argv): # parse sequence names - because we need to get the name first and # then create corresponding arguments for the main parser sequenceSubparser = argparse.ArgumentParser() sequenceSubparser.add_argument('--cloudfunctions_sequences', nargs='+') argvWithoutHelp = list(argv) if "--help" in argv: argvWithoutHelp.remove("--help") if "-h" in argv: argvWithoutHelp.remove("-h") sequenceNames = sequenceSubparser.parse_known_args( argvWithoutHelp)[0].cloudfunctions_sequences or [] parser = argparse.ArgumentParser( description="Deploys the cloud functions", formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('-v', '--verbose', required=False, help='verbosity', action='store_true') parser.add_argument('-c', '--common_configFilePaths', help="configuaration file", action='append') parser.add_argument('--common_functions', required=False, help="directory where the cloud functions are located") parser.add_argument('--cloudfunctions_namespace', required=False, help="cloud functions namespace") parser.add_argument('--cloudfunctions_apikey', required=False, help="cloud functions apikey") parser.add_argument('--cloudfunctions_username', required=False, help="cloud functions user name") parser.add_argument('--cloudfunctions_password', required=False, help="cloud functions password") parser.add_argument('--cloudfunctions_package', required=False, help="cloud functions package name") parser.add_argument('--cloudfunctions_url', required=False, help="url of cloud functions API") parser.add_argument('--log', type=str.upper, default=None, choices=list(logging._levelToName.values())) parser.add_argument('--cloudfunctions_sequences', nargs='+', required=False, help="cloud functions sequence names") for runtime in list(interpretedRuntimes.values()) + list( compiledRuntimes.values()): parser.add_argument('--cloudfunctions_' + runtime + '_version', required=False, help="cloud functions " + runtime + " version") # Add arguments for each sequence to be able to define the functions in the sequence for sequenceName in sequenceNames: try: parser.add_argument("--cloudfunctions_sequence_" + sequenceName, required=True, help="functions in sequence '" + sequenceName + "'") except argparse.ArgumentError as e: if "conflicting option" in str(e): # from None is needed in order to show only the custom exception and not the whole traceback # (It would read as 'During handling of the above exception, another exception has occurred', but we DID handle it) raise argparse.ArgumentError( None, "Duplicate sequence name: " + sequenceName) from None else: raise e args = parser.parse_args(argv) if __name__ == '__main__': setLoggerConfig(args.log, args.verbose) logger.info('STARTING: ' + os.path.basename(__file__)) def handleResponse(response): """Get response code and show an error if it's not OK""" code = response.status_code if code != requests.codes.ok: if code == 401: logger.error( "Authorization error. Check your credentials. (Error code " + str(code) + ")") elif code == 403: logger.error( "Access is forbidden. Check your credentials and permissions. (Error code " + str(code) + ")") elif code == 404: logger.error( "The resource could not be found. Check your cloudfunctions url and namespace. (Error code " + str(code) + ")") elif code == 408: logger.error("Request Timeout. (Error code " + str(code) + ")") elif code >= 500: logger.error("Internal server error. (Error code " + str(code) + ")") else: logger.error("Unexpected error code: " + str(code)) errorsInResponse(response.json()) return False return True config = Cfg(args) namespace = getRequiredParameter(config, 'cloudfunctions_namespace') urlNamespace = quote(namespace) auth = getParametersCombination( config, 'cloudfunctions_apikey', ['cloudfunctions_password', 'cloudfunctions_username']) package = getRequiredParameter(config, 'cloudfunctions_package') cloudFunctionsUrl = getRequiredParameter(config, 'cloudfunctions_url') functionDir = getRequiredParameter(config, 'common_functions') # If sequence names are already defined (from console), do nothing. Else look for them in the configuration. if not sequenceNames: sequenceNames = getOptionalParameter(config, 'cloudfunctions_sequences') or [] # SequenceNames has to be a list if type(sequenceNames) is str: sequenceNames = [sequenceNames] # Create a dict of {<seqName>: [<functions 1>, <function2> ,...]} sequences = { seqName: getRequiredParameter(config, "cloudfunctions_sequence_" + seqName) for seqName in sequenceNames } if 'cloudfunctions_apikey' in auth: username, password = convertApikeyToUsernameAndPassword( auth['cloudfunctions_apikey']) else: username = auth['cloudfunctions_username'] password = auth['cloudfunctions_password'] runtimeVersions = {} for ext, runtime in list(interpretedRuntimes.items()) + list( compiledRuntimes.items()): runtimeVersions[runtime] = runtime + ':' + getattr( config, 'cloudfunctions_' + runtime + '_version', 'default') requests.packages.urllib3.disable_warnings(InsecureRequestWarning) packageUrl = cloudFunctionsUrl + '/' + urlNamespace + '/packages/' + package + '?overwrite=true' logger.info("Will create cloudfunctions package %s.", package) response = requests.put(packageUrl, auth=(username, password), headers={'Content-Type': 'application/json'}, data='{}') if not handleResponse(response): logger.critical("Cannot create cloud functions package %s.", package) sys.exit(1) else: logger.info('Cloud functions package successfully uploaded') filesAtPath = getFilesAtPath(functionDir, [ '*' + ext for ext in (list(interpretedRuntimes) + list(compiledRuntimes) + compressedFiles) ]) logger.info("Will deploy functions at paths %s.", functionDir) for functionFilePath in filesAtPath: fileName = os.path.basename(functionFilePath) (funcName, ext) = os.path.splitext(fileName) runtime = None binary = False # if the file is zip, it's necessary to look inside if ext == '.zip': runtime = _getZipPackageType(functionFilePath) if not runtime: logger.warning( "Cannot determine function type from zip file '%s'. Skipping!", functionFilePath) continue binary = True else: if ext in interpretedRuntimes: runtime = interpretedRuntimes[ext] binary = False elif ext in compiledRuntimes: runtime = compiledRuntimes[ext] binary = True else: logger.warning( "Cannot determine function type of '%s'. Skipping!", functionFilePath) continue functionUrl = cloudFunctionsUrl + '/' + urlNamespace + '/actions/' + package + '/' + funcName + '?overwrite=true' if binary: content = base64.b64encode(open(functionFilePath, 'rb').read()).decode('utf-8') else: content = open(functionFilePath, 'r').read() payload = { 'exec': { 'kind': runtimeVersions[runtime], 'binary': binary, 'code': content } } logger.verbose("Deploying function %s", funcName) response = requests.put(functionUrl, auth=(username, password), headers={'Content-Type': 'application/json'}, data=json.dumps(payload), verify=False) if not handleResponse(response): logger.critical("Cannot deploy cloud function %s.", funcName) sys.exit(1) else: logger.verbose('Cloud function %s successfully deployed.', funcName) logger.info("Cloudfunctions successfully deployed.") if sequences: logger.info("Will deploy cloudfunction sequences.") for seqName in sequences: sequenceUrl = cloudFunctionsUrl + '/' + urlNamespace + '/actions/' + package + '/' + seqName + '?overwrite=true' functionNames = sequences[seqName] fullFunctionNames = [ namespace + '/' + package + '/' + functionName for functionName in functionNames ] payload = { 'exec': { 'kind': 'sequence', 'binary': False, 'components': fullFunctionNames } } logger.verbose("Deploying cloudfunctions sequence '%s': %s", seqName, functionNames) response = requests.put(sequenceUrl, auth=(username, password), headers={'Content-Type': 'application/json'}, data=json.dumps(payload), verify=False) if not handleResponse(response): logger.critical("Cannot deploy cloudfunctions sequence %s", seqName) sys.exit(1) else: logger.verbose("Sequence '%s' deployed.", seqName) if sequences: logger.info("Cloudfunction sequences successfully deployed.") logger.info('FINISHING: ' + os.path.basename(__file__))
def test_emptyPatterns(self): """Tests if getFilesAtPath returns no files when no patterns supplied.""" directory = self.dataBasePath pathList = wawCommons.getFilesAtPath([directory], []) assert len(pathList) == 0
def test_directoryPattern(self): """Tests if pattern containing directory separator does not match any file (this is intended behavior!).""" directory = self.dataBasePath pathList = wawCommons.getFilesAtPath( [directory], [os.path.join("directory1", "*.test")]) assert len(pathList) == 0
def test_emptyInputList(self): """Tests if getFilesAtPath function works when pathList parameter is empty.""" pathList = wawCommons.getFilesAtPath([]) assert len(pathList) == 0
def main(argv): parser = argparse.ArgumentParser( description= 'Converts intent csv files to .json format of Watson Conversation Service', formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('-c', '--common_configFilePaths', help='configuaration file', action='append') parser.add_argument('-oc', '--common_output_config', help='output configuration file') parser.add_argument( '-ii', '--common_intents', help= 'directory with intent csv files to be processed (all of them will be included in output json)', action='append') #-gi is functionsally equivalent to -ii parser.add_argument( '-gi', '--common_generated_intents', help= 'directory with generated intent csv files to be processed (all of them will be included in output json)', action='append') parser.add_argument( '-od', '--common_outputs_directory', required=False, help='directory where the otputs will be stored (outputs is default)') parser.add_argument('-oi', '--common_outputs_intents', help='file with output json with all the intents') parser.add_argument( '-ni', '--common_intents_nameCheck', action='append', nargs=2, help= "regex and replacement for intent name check, e.g. '-' '_' for to replace hyphens for underscores or '$special' '\\L' for lowercase" ) parser.add_argument( '-s', '--soft', required=False, help= 'soft name policy - change intents and entities names without error.', action='store_true', default="") parser.add_argument('-v', '--verbose', required=False, help='verbosity', action='store_true') parser.add_argument('--log', type=str.upper, default=None, choices=list(logging._levelToName.values())) args = parser.parse_args(argv) if __name__ == '__main__': setLoggerConfig(args.log, args.verbose) config = Cfg(args) NAME_POLICY = 'soft' if args.soft else 'hard' logger.info('STARTING: ' + os.path.basename(__file__)) if not hasattr(config, 'common_intents'): logger.info('intents parameter is not defined.') if not hasattr(config, 'common_generated_intents'): logger.info('generated_intents parameter is not defined, ignoring') if not hasattr(config, 'common_outputs_intents'): logger.info( 'Outputs_intents parameter is not defined, output will be generated to console.' ) intents = [] pathList = getattr(config, 'common_intents') if hasattr(config, 'common_generated_intents'): pathList = pathList + getattr(config, 'common_generated_intents') filesAtPath = getFilesAtPath(pathList) for intentFileName in sorted(filesAtPath): intentName = toIntentName( NAME_POLICY, args.common_intents_nameCheck, os.path.splitext(os.path.basename(intentFileName))[0]) with openFile(intentFileName, 'r', encoding='utf8') as intentFile: intent = {} intent['intent'] = intentName examples = [] for line in intentFile: # remove comments line = line.split('#')[0] line = line.rstrip().lower() #non-ascii characters fix #line = line.encode('utf-8') if line: example = processExample(line, intentName, examples) #adding to the list if example: examples.append(example) intent['examples'] = examples intents.append(intent) if hasattr(config, 'common_outputs_directory') and hasattr( config, 'common_outputs_intents'): if not os.path.exists(getattr(config, 'common_outputs_directory')): os.makedirs(getattr(config, 'common_outputs_directory')) logger.info('Created new output directory ' + getattr(config, 'common_outputs_directory')) with codecs.open(os.path.join( getattr(config, 'common_outputs_directory'), getattr(config, 'common_outputs_intents')), 'w', encoding='utf8') as outputFile: outputFile.write(json.dumps(intents, indent=4, ensure_ascii=False)) else: print(json.dumps(intents, indent=4, ensure_ascii=False)) logger.info('FINISHING: ' + os.path.basename(__file__))