def main(): logi('Executing main') cwd = os.getcwd() # raw = DataParser(os.path.join(cwd, 'data', 'pg.csv')) background = DataParser( os.path.join(cwd, 'datasets', 'tritrig-wab-beam_100MeV_L1L1_tight.csv')) signal = DataParser( os.path.join(cwd, 'datasets', 'ap_100MeV_L1L1_tight_08mm.csv')) print(background.getDF()) background.to_numpy() signal.to_numpy() # print(raw.to_numpy()) # fig, ax = plt.figure(figsize=(8,6)) # ax.plot(raw.to_numpy()) print(background.CSV_HEADER) _ = plt.hist(background.RAW_DF['vz'], bins=50) _ = plt.hist(signal.RAW_DF['vz'], bins=50) plt.show()
def start(self): try: while True: logi("Agent run") self.iter() time.sleep(UPDATE_FREQUENCY) except KeyboardInterrupt: loge("Quitting.") exit(0)
def filter(self, df): # list of columns/features in the output dataframe #column_select = ['vz','vzPull','vx','vy','vxPull','vyPull', 'uncM', 'eleZ0', 'posZ0', 'eleTrkD0', 'posTrkD0', 'projX', 'projY', 'projXPull', 'projYPull', 'uncP', 'eleP', 'posP', 'eleTrkTanLambda','eleTrkD0Err','eleTrkTanLambdaErr','eleTrkZ0Err','posTrkTanLambda','posTrkD0Err','posTrkTanLambdaErr','posTrkZ0Err'] #column_select = ['vx','vy','vxPull','vyPull', 'uncM', 'eleZ0', 'posZ0', 'eleTrkD0', 'posTrkD0', 'projX', 'projY', 'projXPull', 'projYPull', 'uncP', 'eleP', 'posP', 'eleTrkTanLambda','eleTrkD0Err','eleTrkTanLambdaErr','eleTrkZ0Err','posTrkTanLambda','posTrkD0Err','posTrkTanLambdaErr','posTrkZ0Err'] #column_select = ['vz','vzPull','vy','vyPull', 'uncM', 'eleZ0', 'posZ0', 'projY', 'projYPull', 'uncP', 'eleTrkTanLambda','eleTrkTanLambdaErr','eleTrkZ0Err','posTrkTanLambda','posTrkTanLambdaErr','posTrkZ0Err'] #column_select = ['vy','vyPull', 'uncM', 'eleZ0', 'posZ0', 'projY', 'projYPull', 'uncP', 'eleTrkTanLambda','eleTrkTanLambdaErr','eleTrkZ0Err','posTrkTanLambda','posTrkTanLambdaErr','posTrkZ0Err'] #column_select = ['vy','vyPull', 'uncM', 'eleZ0', 'posZ0', 'projY', 'eleTrkTanLambda','posTrkTanLambda'] column_select = ['vz','vzPull','vy','vyPull', 'uncM', 'eleZ0', 'posZ0', 'projY', 'eleTrkTanLambda','posTrkTanLambda'] self.DATA_HEADER = column_select logi('Filtering data removing these columns %s' % column_select) df = df[column_select] return df
def build_image(release): image_name = _image_name(release) release_dir = _release_dir(release) if _image_exists(image_name) == True: loge("Image {} already exists".format(image_name)) else: logi("Building docker image {}".format(image_name)) for line in docker('image', 'build', '-t', image_name, release_dir, _iter=True): logi(line)
def clean(self, df): ''' Clean RAW_DF based on a hard coded condition 'condition1' :return: clean DataFrame ''' condition1 = df['vz'] > 0 # condition2 .... condition = condition1 & 1 # & all conditions self.FILTERED_DF = df[~condition] df = df[condition] self.DATA_ROWS, self.DATA_COLUMNS = df.shape self.DATA_M, self.DATA_NX = df.shape logi('Cleaned %d samples ' % (self.RAW_M - self.DATA_M)) return df
def setup(release): install_dir = _install_dir(release) database_name = _database_name(release) if not os.path.isdir(install_dir): remove_database(MYSQL_USER, MYSQL_PASSWORD, database_name) copy_src(installDir=install_dir, release=release) install(installDir=install_dir, domain=DOMAIN_NAME, db_server=MYSQL_HOST, db_name=database_name, db_user=MYSQL_USER, db_password=MYSQL_PASSWORD) dump_database(release, db_user=MYSQL_USER, db_password=MYSQL_PASSWORD) else: logi("The instance '{}' ".format(release) + 'already exist ...') copy_templates(release)
def __init__(self, csvFile, clean=True, filter=True, name='', descr='') -> None: ''' :param csvFile: CSV File location ''' df = pd.read_csv(csvFile) rows = df.shape[0] columns = df.shape[1] logi('Reading CSV file %s found %d rows and %d columns' % (csvFile, rows, columns)) self.RAW_NX = columns self.RAW_M = rows self.RAW_ROWS = rows self.RAW_COLUMNS = columns logi('Found %d features and %d samples' % (self.RAW_NX, self.RAW_M)) self.CSV_FILE = csvFile self.CSV_HEADER = self.getHeader(df) self.NAME = name self.DESCRIPTION = descr self.updateRaw(df) if filter: df = self.filter(df) if clean: logi('Cleaning data') df = self.clean(df) self.updateData(df) self.assertShapes()
def copy_db(db, user_config): domain = user_config['HOST_DOMAIN'] release = user_config['PRESTASHOP_RELEASE'] if db_exists(db): loge("Database '{}' already exists".format(db['MYSQL_DATABASE'])) else: logi("Creating db '{}' ...".format(db['MYSQL_DATABASE'])) sql(db, request="CREATE DATABASE {}".format(db['MYSQL_DATABASE'])) copied_sqldump_file = sql_filepath(domain, release) logi("Importing dump file '{}' to database {}...".format( copied_sqldump_file, db['MYSQL_DATABASE'])) #mysql("-h", db['MYSQL_HOST'], '-P', '3306', "-u", db['MYSQL_USER'], "-p" + db['MYSQL_PASSWORD'] , db['MYSQL_DATABASE'], '<', copied_sqldump_file ) mysql("-h", db['MYSQL_HOST'], '-P', '3306', "-u", db['MYSQL_USER'], "-p" + db['MYSQL_PASSWORD'], db['MYSQL_DATABASE'], _in=sh.cat(copied_sqldump_file))
def init_domain(db, user_config): domain = user_config['HOST_DOMAIN'] release = user_config['PRESTASHOP_RELEASE'] if not os.path.isdir(PRESTASHOPD_USERS_DIR): logi("Creating main directory {} ".format(PRESTASHOPD_USERS_DIR)) mkdir("-p", PRESTASHOPD_USERS_DIR) DOMAIN_DIR = PRESTASHOPD_USERS_DIR + domain if not os.path.isdir(DOMAIN_DIR): logi("Creating domain directory {} ".format(DOMAIN_DIR)) mkdir("-p", DOMAIN_DIR) sqldump_file = dump_path(release) domain_dir = domain_path(domain, release) logi("Copying sql dump file '{}' to '{}'".format(sqldump_file, domain_dir)) cp("-arf", sqldump_file, domain_dir) copied_sqldump_file = sql_filepath(domain, release) logi("Configuring mysql dump file {}".format(copied_sqldump_file)) configure_mysqldump(copied_sqldump_file, domain) logi("Configuring config files ... ") configure_configfiles(db, user_config)
def remove_database(db_user, db_password, db_name): logi("Removing db '{}' ...".format(db_name)) mysql("-u", db_user, "-p" + db_password, "-e", "DROP DATABASE IF EXISTS {}".format(db_name))
def remove_db(db, domain, release): logi("Removing db '{}' ...".format(db['MYSQL_DATABASE'])) sql(db, request="DROP DATABASE IF EXISTS {}".format(db['MYSQL_DATABASE']))