def main():
logi('Executing main')
cwd = os.getcwd()
# raw = DataParser(os.path.join(cwd, 'data', 'pg.csv'))
background = DataParser(
os.path.join(cwd, 'datasets',
'tritrig-wab-beam_100MeV_L1L1_tight.csv'))
signal = DataParser(
os.path.join(cwd, 'datasets', 'ap_100MeV_L1L1_tight_08mm.csv'))
print(background.getDF())
background.to_numpy()
signal.to_numpy()
# print(raw.to_numpy())
# fig, ax = plt.figure(figsize=(8,6))
# ax.plot(raw.to_numpy())
print(background.CSV_HEADER)
_ = plt.hist(background.RAW_DF['vz'], bins=50)
_ = plt.hist(signal.RAW_DF['vz'], bins=50)
plt.show()
def start(self):
try:
while True:
logi("Agent run")
self.iter()
time.sleep(UPDATE_FREQUENCY)
except KeyboardInterrupt:
loge("Quitting.")
exit(0)
def filter(self, df):
# list of columns/features in the output dataframe
#column_select = ['vz','vzPull','vx','vy','vxPull','vyPull', 'uncM', 'eleZ0', 'posZ0', 'eleTrkD0', 'posTrkD0', 'projX', 'projY', 'projXPull', 'projYPull', 'uncP', 'eleP', 'posP', 'eleTrkTanLambda','eleTrkD0Err','eleTrkTanLambdaErr','eleTrkZ0Err','posTrkTanLambda','posTrkD0Err','posTrkTanLambdaErr','posTrkZ0Err']
#column_select = ['vx','vy','vxPull','vyPull', 'uncM', 'eleZ0', 'posZ0', 'eleTrkD0', 'posTrkD0', 'projX', 'projY', 'projXPull', 'projYPull', 'uncP', 'eleP', 'posP', 'eleTrkTanLambda','eleTrkD0Err','eleTrkTanLambdaErr','eleTrkZ0Err','posTrkTanLambda','posTrkD0Err','posTrkTanLambdaErr','posTrkZ0Err']
#column_select = ['vz','vzPull','vy','vyPull', 'uncM', 'eleZ0', 'posZ0', 'projY', 'projYPull', 'uncP', 'eleTrkTanLambda','eleTrkTanLambdaErr','eleTrkZ0Err','posTrkTanLambda','posTrkTanLambdaErr','posTrkZ0Err']
#column_select = ['vy','vyPull', 'uncM', 'eleZ0', 'posZ0', 'projY', 'projYPull', 'uncP', 'eleTrkTanLambda','eleTrkTanLambdaErr','eleTrkZ0Err','posTrkTanLambda','posTrkTanLambdaErr','posTrkZ0Err']
#column_select = ['vy','vyPull', 'uncM', 'eleZ0', 'posZ0', 'projY', 'eleTrkTanLambda','posTrkTanLambda']
column_select = ['vz','vzPull','vy','vyPull', 'uncM', 'eleZ0', 'posZ0', 'projY', 'eleTrkTanLambda','posTrkTanLambda']
self.DATA_HEADER = column_select
logi('Filtering data removing these columns %s' % column_select)
df = df[column_select]
return df
def build_image(release):
image_name = _image_name(release)
release_dir = _release_dir(release)
if _image_exists(image_name) == True:
loge("Image {} already exists".format(image_name))
else:
logi("Building docker image {}".format(image_name))
for line in docker('image',
'build',
'-t',
image_name,
release_dir,
_iter=True):
logi(line)
def clean(self, df):
'''
Clean RAW_DF based on a hard coded condition 'condition1'
:return: clean DataFrame
'''
condition1 = df['vz'] > 0
# condition2 ....
condition = condition1 & 1 # & all conditions
self.FILTERED_DF = df[~condition]
df = df[condition]
self.DATA_ROWS, self.DATA_COLUMNS = df.shape
self.DATA_M, self.DATA_NX = df.shape
logi('Cleaned %d samples ' % (self.RAW_M - self.DATA_M))
return df
def setup(release):
install_dir = _install_dir(release)
database_name = _database_name(release)
if not os.path.isdir(install_dir):
remove_database(MYSQL_USER, MYSQL_PASSWORD, database_name)
copy_src(installDir=install_dir, release=release)
install(installDir=install_dir,
domain=DOMAIN_NAME,
db_server=MYSQL_HOST,
db_name=database_name,
db_user=MYSQL_USER,
db_password=MYSQL_PASSWORD)
dump_database(release, db_user=MYSQL_USER, db_password=MYSQL_PASSWORD)
else:
logi("The instance '{}' ".format(release) + 'already exist ...')
copy_templates(release)
def __init__(self, csvFile, clean=True, filter=True, name='', descr='') -> None:
'''
:param csvFile: CSV File location
'''
df = pd.read_csv(csvFile)
rows = df.shape[0]
columns = df.shape[1]
logi('Reading CSV file %s found %d rows and %d columns' % (csvFile, rows, columns))
self.RAW_NX = columns
self.RAW_M = rows
self.RAW_ROWS = rows
self.RAW_COLUMNS = columns
logi('Found %d features and %d samples' % (self.RAW_NX, self.RAW_M))
self.CSV_FILE = csvFile
self.CSV_HEADER = self.getHeader(df)
self.NAME = name
self.DESCRIPTION = descr
self.updateRaw(df)
if filter:
df = self.filter(df)
if clean:
logi('Cleaning data')
df = self.clean(df)
self.updateData(df)
self.assertShapes()
def copy_db(db, user_config):
domain = user_config['HOST_DOMAIN']
release = user_config['PRESTASHOP_RELEASE']
if db_exists(db):
loge("Database '{}' already exists".format(db['MYSQL_DATABASE']))
else:
logi("Creating db '{}' ...".format(db['MYSQL_DATABASE']))
sql(db, request="CREATE DATABASE {}".format(db['MYSQL_DATABASE']))
copied_sqldump_file = sql_filepath(domain, release)
logi("Importing dump file '{}' to database {}...".format(
copied_sqldump_file, db['MYSQL_DATABASE']))
#mysql("-h", db['MYSQL_HOST'], '-P', '3306', "-u", db['MYSQL_USER'], "-p" + db['MYSQL_PASSWORD'] , db['MYSQL_DATABASE'], '<', copied_sqldump_file )
mysql("-h",
db['MYSQL_HOST'],
'-P',
'3306',
"-u",
db['MYSQL_USER'],
"-p" + db['MYSQL_PASSWORD'],
db['MYSQL_DATABASE'],
_in=sh.cat(copied_sqldump_file))
def init_domain(db, user_config):
domain = user_config['HOST_DOMAIN']
release = user_config['PRESTASHOP_RELEASE']
if not os.path.isdir(PRESTASHOPD_USERS_DIR):
logi("Creating main directory {} ".format(PRESTASHOPD_USERS_DIR))
mkdir("-p", PRESTASHOPD_USERS_DIR)
DOMAIN_DIR = PRESTASHOPD_USERS_DIR + domain
if not os.path.isdir(DOMAIN_DIR):
logi("Creating domain directory {} ".format(DOMAIN_DIR))
mkdir("-p", DOMAIN_DIR)
sqldump_file = dump_path(release)
domain_dir = domain_path(domain, release)
logi("Copying sql dump file '{}' to '{}'".format(sqldump_file, domain_dir))
cp("-arf", sqldump_file, domain_dir)
copied_sqldump_file = sql_filepath(domain, release)
logi("Configuring mysql dump file {}".format(copied_sqldump_file))
configure_mysqldump(copied_sqldump_file, domain)
logi("Configuring config files ... ")
configure_configfiles(db, user_config)
def remove_database(db_user, db_password, db_name):
logi("Removing db '{}' ...".format(db_name))
mysql("-u", db_user, "-p" + db_password, "-e",
"DROP DATABASE IF EXISTS {}".format(db_name))
def remove_db(db, domain, release):
logi("Removing db '{}' ...".format(db['MYSQL_DATABASE']))
sql(db, request="DROP DATABASE IF EXISTS {}".format(db['MYSQL_DATABASE']))
