def load_model(self, name: Optional[str] = None) -> CNNClassificationModel:
"""
loads a model for us by name, to be utilized
:param name:
:return:
"""
model_config = config.config(filename=self.config_file,
section="model")
encoder_config = config.config(filename=self.config_file,
section="encoder")
if name is None:
name = model_config['default']
model: CNNClassificationModel = self.factory(name)(
width=int(model_config.get("width")),
height=int(model_config.get("height")),
plots=True if model_config.get("plot") == 'true' else False,
batch_size=int(model_config['batch_size']),
channels=3,
tensor_board=True
if model_config.get("tensor_board") == 'true' else False,
early_stop=True
if model_config.get("early_stop") == 'true' else False,
)
return model
def train(self,
input: str,
model: Optional[CNNClassificationModel] = None,
generator: Optional = None,
gpus: int = None):
"""
trains the model using the internal configuration
:param model:
:return:
"""
train_config = config.config(filename=self.config_file,
section="training")
if model is None:
model = self.load_model()
if generator is None:
generator = self.load_generator()
model.train(
input=input,
generator=generator,
test_size=float(train_config['test_size']),
epochs=int(train_config['epoch']),
gpus=int(train_config['gpus']) if gpus is None else int(gpus),
verbose=int(train_config['verbose']),
)
def load_generator(self) -> LabelGenerator:
"""
loads the generator to use
:return:
"""
generator_config = config.config(filename=self.config_file,
section="training")
generator: LabelGenerator = self.factory(
generator_config['generator'])()
return generator
def connect_to_db():
"""
connect to our internal database
:return:
"""
c = config.config(filename="database.ini", section="machine")
return PostgresqlDatabase(c.get('database'),
user=c.get('user'),
host=c.get('host'),
password=c.get('password'))
def __init__(self, config_file="database.ini"):
"""
connect to the database
:param server:
:param port:
:param database:
:param user:
:param password:
"""
db = config.config(filename=config_file, section="binbase")
self.connection = psycopg2.connect(**db)
def __init__(self, config_file: str = "shares.ini", read_only: Optional[bool] = False):
"""
:param config_file:
"""
bucket_config = config.config(config_file, "s3-bucket")
self.bucket_name = bucket_config["name"]
self.constraint = bucket_config["constraint"]
if read_only is True:
self.client = boto3.client('s3', config=Config(signature_version=UNSIGNED))
self.s3 = boto3.resource('s3', config=Config(signature_version=UNSIGNED))
else:
self.client = boto3.client('s3')
self.s3 = boto3.resource('s3')
try:
self.client.create_bucket(Bucket=self.bucket_name, CreateBucketConfiguration={
'LocationConstraint': self.constraint})
except Exception as e:
pass
def load_encoder(self, name: str = None) -> Encoder:
"""
loads the encoder for you
:return:
"""
encoder_config = config.config(filename=self.config_file,
section="encoder")
if name is None:
name = encoder_config['default']
encoder: Encoder = self.factory(name)(
width=int(encoder_config.get("width")),
height=int(encoder_config.get("height")),
min_mz=float(encoder_config.get("min_mz")),
max_mz=float(encoder_config.get("max_mz")),
intensity_max=float(encoder_config.get("intensity_max")),
dpi=int(encoder_config.get("dpi")),
plot_axis=True if encoder_config.get("axis") == 'true' else False,
)
return encoder
def __init__(self, config_file: str = "machine.ini"):
self.config_file = config_file
self.training_config = config.config(filename=config_file,
section="training")
def convert(self, pattern: str = "%%"):
"""
converts the given input and stores it at the defined postgres database location
:param input:
:return:
"""
# 1. query all visble sample information
db = config.config(filename="database.ini", section="binbase")
connection = psycopg2.connect(**db)
cursor = connection.cursor()
sample_count = connection.cursor()
spectra = connection.cursor()
bin = connection.cursor()
sample_count.execute(
"select count(*) from samples where sample_name like '{}' and visible = 'TRUE'"
.format(pattern))
count = sample_count.fetchone()[0]
pbar = tqdm(total=count + 1,
desc="importing samples for pattern {}".format(pattern))
cursor.execute(
"select sample_id, sample_name from samples where sample_name like '{}' and visible = 'TRUE'"
.format(pattern))
row = cursor.fetchone()
while row is not None:
try:
try:
record = MZMLSampleRecord.get(
MZMLSampleRecord.file_name == row[1])
record.delete_instance()
except Exception:
# object doesn't exist
pass
# 2. create sample object
MZMLSampleRecord.create(file_name=row[1],
instrument="gctof",
name=row[1])
record = MZMLSampleRecord.get(
MZMLSampleRecord.file_name == row[1])
spectra.execute(
"select bin_id, spectra_id,spectra, retention_time from spectra where sample_id = {}"
.format(row[0]))
s = spectra.fetchone()
while s is not None:
splash = Splash().splash(Spectrum(s[2], SpectrumType.MS))
spectrum = [
list(map(float, x.split(':')))
for x in s[2].strip().split()
]
spectrum_max = max(spectrum, key=itemgetter(1))
MZMLMSMSSpectraRecord.create(
sample=record,
msms=s[2],
rt=s[3],
splash=splash,
level=1,
base_peak=spectrum_max[0],
base_peak_intensity=spectrum_max[1],
precursor=0,
precursor_intensity=0,
precursor_charge=0,
ion_count=len(spectrum),
scan_number=int(s[1]))
if s[0] is not None:
DatesetToPostgresConverter.classify(
"bin_id", splash, s[0])
bin.execute(
"select name from bin where bin_id = {} and bin_id::text != name"
.format(s[0]))
result = bin.fetchone()
if result is not None:
DatesetToPostgresConverter.classify(
"bin", splash, result[0])
s = spectra.fetchone()
row = cursor.fetchone()
finally:
pbar.update(1)
