def extract_sentences(self, files, inputpath): dataset_Reader = FileReader() sentence_extractor = SentenceExtractor() sentence_list = [] try: for doc in files: with open(inputpath + doc): inputdataset = dataset_Reader.read(inputpath + doc) sentence_list.extend(sentence_extractor.extract_sentences(inputdataset)) except IOError: print "IOError" return sentence_list
def preprocess(self, files, input_path): dataset_Reader = FileReader() preprocessed_list = [] try: for doc in files: with open(input_path + doc): inputdataset = (dataset_Reader.read(input_path + doc)) preprocessed_data = [] for word in inputdataset.split(): word = word.lower() if word not in self.stop_words(): filter1 = self.regularise_expression(str(word)) filter2 = self.stem_word(filter1) preprocessed_data.append(str(filter2)) preprocessed_list.append(preprocessed_data) except IOError: print "IOError" return preprocessed_list
def get_data_and_label(self): self.get_files() nlp = NLP(None) nlp.set_stop_words() file_reader = FileReader(None) contents = [] labels = [] #init titles titles = {} for title in self.files: titles[title] = con.get_setting('labels', str(title)) for title in self.files: for file in self.files[title]: file_reader.file_path = file nlp.text = file_reader.read() contents.append(nlp.get_words_feature()) labels.append(titles[title]) return (contents, labels)
def __init__(self, kubeadm_config_template_path: str, token_path: str): self.__kubeadm_config_template_path = kubeadm_config_template_path self.__token = FileReader().execute(token_path)
from aws_cdk import core from vpc.vpc_stack_constructor import VpcStackConstructor from cluster_security_group.cluster_security_group_stack_constructor import ClusterSecurityGroupStackConstructor from deployment_asset.deployment_asset_stack_constructor import DeploymentAssetStackConstructor from manager.manager_stack_constructor import ManagerStackConstructor from master.master_stack_constructor import MasterStackConstructor from worker.worker_stack_constructor import WorkerStackConstructor from utils.config_loader import ConfigLoader import os from utils.file_reader import FileReader config = ConfigLoader(config_path=os.path.join( "resource", "config", "config.json")).fetch_config_from_json_file() config["TOKEN"] = FileReader.execute( os.path.join("resource", "key", "token.txt")) app = core.App() env = core.Environment(account=config["ACCOUNT"], region=config["REGION"]) vpc_stack = VpcStackConstructor(scope=app, env=env, config=config).execute() cluster_security_group_stack = ClusterSecurityGroupStackConstructor( scope=app, env=env, config=config, vpc_stack=vpc_stack).execute() deployment_asset_stack = DeploymentAssetStackConstructor( scope=app, env=env, config=config).execute() master_stack = MasterStackConstructor( scope=app, env=env, config=config, vpc_stack=vpc_stack,