Example #1
1
 def extract_sentences(self, files, inputpath):
     dataset_Reader = FileReader()
     sentence_extractor = SentenceExtractor()
     sentence_list = []
     try:
         for doc in files:
             with open(inputpath + doc):
                 inputdataset = dataset_Reader.read(inputpath + doc)
                 sentence_list.extend(sentence_extractor.extract_sentences(inputdataset))
     except IOError:
         print "IOError"
         
     return sentence_list
Example #2
0
 def preprocess(self, files, input_path):
     dataset_Reader = FileReader()
     preprocessed_list = []
     try:
         for doc in files:
             with open(input_path + doc):
                 inputdataset = (dataset_Reader.read(input_path + doc))
                 preprocessed_data = []
                 for word in inputdataset.split():
                     word = word.lower()
                     if word not in self.stop_words():
                         filter1 = self.regularise_expression(str(word))
                         filter2 = self.stem_word(filter1)
                         preprocessed_data.append(str(filter2))
                         
                 preprocessed_list.append(preprocessed_data)
     except IOError:
         print "IOError"
     
     return preprocessed_list
Example #3
0
    def get_data_and_label(self):
        self.get_files()
        nlp = NLP(None)
        nlp.set_stop_words()
        file_reader = FileReader(None)
        contents = []
        labels = []

        #init titles
        titles = {}
        for title in self.files:
            titles[title] = con.get_setting('labels', str(title))

        for title in self.files:
            for file in self.files[title]:
                file_reader.file_path = file
                nlp.text = file_reader.read()

                contents.append(nlp.get_words_feature())
                labels.append(titles[title])

        return (contents, labels)
Example #4
0
 def __init__(self, kubeadm_config_template_path: str, token_path: str):
     self.__kubeadm_config_template_path = kubeadm_config_template_path
     self.__token = FileReader().execute(token_path)
Example #5
0
from aws_cdk import core

from vpc.vpc_stack_constructor import VpcStackConstructor
from cluster_security_group.cluster_security_group_stack_constructor import ClusterSecurityGroupStackConstructor
from deployment_asset.deployment_asset_stack_constructor import DeploymentAssetStackConstructor
from manager.manager_stack_constructor import ManagerStackConstructor
from master.master_stack_constructor import MasterStackConstructor
from worker.worker_stack_constructor import WorkerStackConstructor

from utils.config_loader import ConfigLoader
import os
from utils.file_reader import FileReader

config = ConfigLoader(config_path=os.path.join(
    "resource", "config", "config.json")).fetch_config_from_json_file()
config["TOKEN"] = FileReader.execute(
    os.path.join("resource", "key", "token.txt"))

app = core.App()
env = core.Environment(account=config["ACCOUNT"], region=config["REGION"])

vpc_stack = VpcStackConstructor(scope=app, env=env, config=config).execute()
cluster_security_group_stack = ClusterSecurityGroupStackConstructor(
    scope=app, env=env, config=config, vpc_stack=vpc_stack).execute()
deployment_asset_stack = DeploymentAssetStackConstructor(
    scope=app, env=env, config=config).execute()

master_stack = MasterStackConstructor(
    scope=app,
    env=env,
    config=config,
    vpc_stack=vpc_stack,