def load_model(self, name: Optional[str] = None) -> CNNClassificationModel: """ loads a model for us by name, to be utilized :param name: :return: """ model_config = config.config(filename=self.config_file, section="model") encoder_config = config.config(filename=self.config_file, section="encoder") if name is None: name = model_config['default'] model: CNNClassificationModel = self.factory(name)( width=int(model_config.get("width")), height=int(model_config.get("height")), plots=True if model_config.get("plot") == 'true' else False, batch_size=int(model_config['batch_size']), channels=3, tensor_board=True if model_config.get("tensor_board") == 'true' else False, early_stop=True if model_config.get("early_stop") == 'true' else False, ) return model
def train(self, input: str, model: Optional[CNNClassificationModel] = None, generator: Optional = None, gpus: int = None): """ trains the model using the internal configuration :param model: :return: """ train_config = config.config(filename=self.config_file, section="training") if model is None: model = self.load_model() if generator is None: generator = self.load_generator() model.train( input=input, generator=generator, test_size=float(train_config['test_size']), epochs=int(train_config['epoch']), gpus=int(train_config['gpus']) if gpus is None else int(gpus), verbose=int(train_config['verbose']), )
def load_generator(self) -> LabelGenerator: """ loads the generator to use :return: """ generator_config = config.config(filename=self.config_file, section="training") generator: LabelGenerator = self.factory( generator_config['generator'])() return generator
def connect_to_db(): """ connect to our internal database :return: """ c = config.config(filename="database.ini", section="machine") return PostgresqlDatabase(c.get('database'), user=c.get('user'), host=c.get('host'), password=c.get('password'))
def __init__(self, config_file="database.ini"): """ connect to the database :param server: :param port: :param database: :param user: :param password: """ db = config.config(filename=config_file, section="binbase") self.connection = psycopg2.connect(**db)
def __init__(self, config_file: str = "shares.ini", read_only: Optional[bool] = False): """ :param config_file: """ bucket_config = config.config(config_file, "s3-bucket") self.bucket_name = bucket_config["name"] self.constraint = bucket_config["constraint"] if read_only is True: self.client = boto3.client('s3', config=Config(signature_version=UNSIGNED)) self.s3 = boto3.resource('s3', config=Config(signature_version=UNSIGNED)) else: self.client = boto3.client('s3') self.s3 = boto3.resource('s3') try: self.client.create_bucket(Bucket=self.bucket_name, CreateBucketConfiguration={ 'LocationConstraint': self.constraint}) except Exception as e: pass
def load_encoder(self, name: str = None) -> Encoder: """ loads the encoder for you :return: """ encoder_config = config.config(filename=self.config_file, section="encoder") if name is None: name = encoder_config['default'] encoder: Encoder = self.factory(name)( width=int(encoder_config.get("width")), height=int(encoder_config.get("height")), min_mz=float(encoder_config.get("min_mz")), max_mz=float(encoder_config.get("max_mz")), intensity_max=float(encoder_config.get("intensity_max")), dpi=int(encoder_config.get("dpi")), plot_axis=True if encoder_config.get("axis") == 'true' else False, ) return encoder
def __init__(self, config_file: str = "machine.ini"): self.config_file = config_file self.training_config = config.config(filename=config_file, section="training")
def convert(self, pattern: str = "%%"): """ converts the given input and stores it at the defined postgres database location :param input: :return: """ # 1. query all visble sample information db = config.config(filename="database.ini", section="binbase") connection = psycopg2.connect(**db) cursor = connection.cursor() sample_count = connection.cursor() spectra = connection.cursor() bin = connection.cursor() sample_count.execute( "select count(*) from samples where sample_name like '{}' and visible = 'TRUE'" .format(pattern)) count = sample_count.fetchone()[0] pbar = tqdm(total=count + 1, desc="importing samples for pattern {}".format(pattern)) cursor.execute( "select sample_id, sample_name from samples where sample_name like '{}' and visible = 'TRUE'" .format(pattern)) row = cursor.fetchone() while row is not None: try: try: record = MZMLSampleRecord.get( MZMLSampleRecord.file_name == row[1]) record.delete_instance() except Exception: # object doesn't exist pass # 2. create sample object MZMLSampleRecord.create(file_name=row[1], instrument="gctof", name=row[1]) record = MZMLSampleRecord.get( MZMLSampleRecord.file_name == row[1]) spectra.execute( "select bin_id, spectra_id,spectra, retention_time from spectra where sample_id = {}" .format(row[0])) s = spectra.fetchone() while s is not None: splash = Splash().splash(Spectrum(s[2], SpectrumType.MS)) spectrum = [ list(map(float, x.split(':'))) for x in s[2].strip().split() ] spectrum_max = max(spectrum, key=itemgetter(1)) MZMLMSMSSpectraRecord.create( sample=record, msms=s[2], rt=s[3], splash=splash, level=1, base_peak=spectrum_max[0], base_peak_intensity=spectrum_max[1], precursor=0, precursor_intensity=0, precursor_charge=0, ion_count=len(spectrum), scan_number=int(s[1])) if s[0] is not None: DatesetToPostgresConverter.classify( "bin_id", splash, s[0]) bin.execute( "select name from bin where bin_id = {} and bin_id::text != name" .format(s[0])) result = bin.fetchone() if result is not None: DatesetToPostgresConverter.classify( "bin", splash, result[0]) s = spectra.fetchone() row = cursor.fetchone() finally: pbar.update(1)