示例#1
0
def updateJobs():
    if ctip.ON_HPCC:
        # Get all job ids
        manager = DatabaseManager()
        colnames, jobs = manager.getRecords("jobs")
        job_ids = []
        for job in jobs:
            job_ids.append(str(job["job_id"]))

        # Get output of qstat
        proc = Popen(["qstat"], stdout=PIPE)
        qstat = proc.stdout.read()
        qstat = qstat.split("\n")

        # Parse output to get the status of the active jobs
        job_stats = []
        for line in qstat:
            if line:
                line = line.split()
                id = line[0].split(".")[0]
                if id in job_ids:
                    job_stats.append((id, line[-2]))

        # Report the status' to the database
        for stat in job_stats:
            if stat[1] == "Q":
                manager.updateJobStatus(stat[0], "queued")
            elif stat[1] == "R":
                manager.updateJobStatus(stat[0], "running")
            elif stat[1] == "H":
                manager.updateJobStatus(stat[0], "held")
            elif stat[1] == "S":
                manager.updateJobStatus(stat[0], "suspended")
示例#2
0
def createConfigTable(csv_file):
    """Creates new config table from a properly formatted csv file."""
    reader = csv.reader(csv_file)

    configTableName = parseTableName(reader)

    # Get the column names
    cols = next(reader)

    configs = []
    for config in reader:
        missing_cols = len(cols) - len(config)
        for i in range(missing_cols):
            config.append("")
        configs.append(config)

    db = DatabaseManager()
    db.addConfigTable(configTableName, cols, configs)

    return configTableName
示例#3
0
def createConfigTable(csv_file):
    """Creates new config table from a properly formatted csv file."""
    reader = csv.reader(csv_file)

    configTableName = parseTableName(reader)

    # Get the column names
    cols = next(reader)

    configs = []
    for config in reader:
        missing_cols = len(cols) - len(config)
        for i in range(missing_cols):
            config.append("")
        configs.append(config)

    db = DatabaseManager()
    db.addConfigTable(configTableName, cols, configs)

    return configTableName
示例#4
0
def generateConfigTable(gen_file):
    """Generates config table from csv file of valid config parameters."""
    reader = csv.reader(gen_file)
    configTableName = parseTableName(reader)

    cols = []
    colDict = {}
    for line in gen_file:
        if not line.startswith('#'):
            line = line.strip()
            key, tokens = line.split('|')
            key = key.strip()
            tokens = tokens.strip()
            cols.append(key)
            values = []
            for token in tokens.split(','):
                parts = token.split('-')
                if len(parts) == 2:
                    start, stop = parts
                    step = '1'
                    if ':' in stop:
                        stop, step = stop.split(':')
                    if '.' in start + stop + step:
                        vals = frange(float(start), float(stop), float(step))
                    else:
                        vals = range(int(start), int(stop) + 1, int(step))
                    values.extend(vals)
                else:
                    values.append(token)
            colDict[key] = values

    configs = []
    for col in cols:
        configs = generateCombos(configs, colDict[col])

    db = DatabaseManager()
    db.addConfigTable(configTableName, cols, configs)

    return configTableName
示例#5
0
def generateConfigTable(gen_file):
    """Generates config table from csv file of valid config parameters."""
    reader = csv.reader(gen_file)
    configTableName = parseTableName(reader)

    cols = []
    colDict = {}
    for line in gen_file:
        if not line.startswith("#"):
            line = line.strip()
            key, tokens = line.split("|")
            key = key.strip()
            tokens = tokens.strip()
            cols.append(key)
            values = []
            for token in tokens.split(","):
                parts = token.split("-")
                if len(parts) == 2:
                    start, stop = parts
                    step = "1"
                    if ":" in stop:
                        stop, step = stop.split(":")
                    if "." in start + stop + step:
                        vals = frange(float(start), float(stop), float(step))
                    else:
                        vals = range(int(start), int(stop) + 1, int(step))
                    values.extend(vals)
                else:
                    values.append(token)
            colDict[key] = values

    configs = []
    for col in cols:
        configs = generateCombos(configs, colDict[col])

    db = DatabaseManager()
    db.addConfigTable(configTableName, cols, configs)

    return configTableName
示例#6
0
def log_start(args):
    db = DatabaseManager()
    db.startJob(args.job_id)
示例#7
0
def storeSnapshot(table, outfile):
    """Store the contents of a config table as a csv file."""
    db = DatabaseManager()
    cols, configs = db.getRecords(table)
    writeConfigCsv(outfile, table, cols, configs)
示例#8
0
def log_pause(args):
    db = DatabaseManager()
    db.pauseJob(args.job_id)
示例#9
0
def checkSession(session_id=None):
    updateJobs()
    manager = DatabaseManager()
    return manager.getSessionSummary(session_id)
示例#10
0
def log_end(args):
    db = DatabaseManager()
    db.endJob(args.job_id)
示例#11
0
def log_pause(args):
    db = DatabaseManager()
    db.pauseJob(args.job_id)
示例#12
0
def update_id(args):
    db = DatabaseManager()
    db.updateJobId(args.job_id, args.new_id)
示例#13
0
def updateJobs():
    if ctip.ON_HPCC:
        # Get all job ids
        manager = DatabaseManager()
        colnames, jobs = manager.getRecords("jobs")
        job_ids = []
        for job in jobs:
            job_ids.append(str(job['job_id']))

        # Get output of qstat
        proc = Popen(['qstat'], stdout=PIPE)
        qstat = proc.stdout.read()
        qstat = qstat.split('\n')

        # Parse output to get the status of the active jobs
        job_stats = []
        for line in qstat:
            if line:
                line = line.split()
                id = line[0].split('.')[0]
                if id in job_ids:
                    job_stats.append((id, line[-2]))

        # Report the status' to the database
        for stat in job_stats:
            if stat[1] == 'Q':
                manager.updateJobStatus(stat[0], 'queued')
            elif stat[1] == 'R':
                manager.updateJobStatus(stat[0], 'running')
            elif stat[1] == 'H':
                manager.updateJobStatus(stat[0], 'held')
            elif stat[1] == 'S':
                manager.updateJobStatus(stat[0], 'suspended')
示例#14
0
def storeSnapshot(table, outfile):
    """Store the contents of a config table as a csv file."""
    db = DatabaseManager()
    cols, configs = db.getRecords(table)
    writeConfigCsv(outfile, table, cols, configs)
示例#15
0
def checkSession(session_id=None):
    updateJobs()
    manager = DatabaseManager()
    return manager.getSessionSummary(session_id)
示例#16
0
def initTestSession(test_func,
                    table,
                    whereClause="",
                    outdir="",
                    qsub=None,
                    name=None):
    """
    Initialize a test session of all configs in 'table' that satisfy
    the 'whereClause'.
    """

    manager = DatabaseManager()

    # Get configs from the database
    colnames, configs = manager.getRecords(table, whereClause)

    #
    # Create a folder for this test session based on the date and time
    #   -> The directory being created is:
    #           <outDir>/<configTable>/<date_time>/
    #   -> If a session_name is specified, the directory is:
    #           <outDir>/<configTable>/<session_name>/
    #   -> Each config will create a folder in this directory
    #
    now = datetime.datetime.now()
    sec = now.second
    timestamp = now.strftime("%Y-%m-%d_%H.%M.") + str(sec)
    sql_datetime_str = now.strftime("%Y-%m-%d %H:%M:") + str(sec)

    session_name = name
    if not session_name:
        session_name = timestamp

    testBatchDir = outdir
    if ctip.CREATE_DIR_STRUCTURE:
        testBatchDir = os.path.join(outdir, table, session_name)
        try:
            os.makedirs(testBatchDir)
        except OSError:
            if not os.path.isdir(testBatchDir):
                raise RuntimeError('Could not create the output directory')

    # Store text file snapshot of config table at root session dir
    snapshotPath = os.path.join(testBatchDir, table + ".csv")
    with open(snapshotPath, 'w') as sf:
        writeConfigCsv(sf, table, colnames, configs)

    # Add this session info to the sessions table
    session_id = manager.newSession(table, name, sql_datetime_str, whereClause)

    # Call the test_function for each config
    jobs = []
    id_queue = Queue()
    for config in configs:
        if qsub:
            p = Process(target=test_func,
                        args=(config, id_queue, testBatchDir, qsub))
        else:
            p = Process(target=test_func,
                        args=(config, id_queue, testBatchDir))
        p.start()
        jobs.append(p)
    for p in jobs:
        p.join()

    job_ids = []
    try:
        while True:
            job_ids.append(id_queue.get_nowait())
    except QueueEmpty:
        pass

    for id, cfg_id in job_ids:
        manager.addJobToSession(session_id, cfg_id, id)
示例#17
0
def list(args):
    db = DatabaseManager()
    db.printTable(args.table_name, ' '.join(args.where_clause))
示例#18
0
def tables(args):
    db = DatabaseManager()
    db.listConfigTables()
示例#19
0
def clean(args):
    db = DatabaseManager()
    if args.session_id:
        db.deleteSession(args.session_id)
    else:
        db.deleteFinishedSessions()
示例#20
0
def log_end(args):
    db = DatabaseManager()
    db.endJob(args.job_id)
示例#21
0
def update_status(args):
    db = DatabaseManager()
    db.updateJobStatus(args.job_id, args.new_status)
示例#22
0
def tables(args):
    db = DatabaseManager()
    db.listConfigTables()
示例#23
0
def log_start(args):
    db = DatabaseManager()
    db.startJob(args.job_id)
示例#24
0
def update_id(args):
    db = DatabaseManager()
    db.updateJobId(args.job_id, args.new_id)
示例#25
0
def log_resume(args):
    db = DatabaseManager()
    db.resumeJob(args.job_id)
示例#26
0
def log_resume(args):
    db = DatabaseManager()
    db.resumeJob(args.job_id)
示例#27
0
def clean(args):
    db = DatabaseManager()
    if args.session_id:
        db.deleteSession(args.session_id)
    else:
        db.deleteFinishedSessions()
示例#28
0
def initTestSession(test_func, table, whereClause="", outdir="", qsub=None, name=None):
    """
    Initialize a test session of all configs in 'table' that satisfy
    the 'whereClause'.
    """

    manager = DatabaseManager()

    # Get configs from the database
    colnames, configs = manager.getRecords(table, whereClause)

    #
    # Create a folder for this test session based on the date and time
    #   -> The directory being created is:
    #           <outDir>/<configTable>/<date_time>/
    #   -> If a session_name is specified, the directory is:
    #           <outDir>/<configTable>/<session_name>/
    #   -> Each config will create a folder in this directory
    #
    now = datetime.datetime.now()
    sec = now.second
    timestamp = now.strftime("%Y-%m-%d_%H.%M.") + str(sec)
    sql_datetime_str = now.strftime("%Y-%m-%d %H:%M:") + str(sec)

    session_name = name
    if not session_name:
        session_name = timestamp

    testBatchDir = outdir
    if ctip.CREATE_DIR_STRUCTURE:
        testBatchDir = os.path.join(outdir, table, session_name)
        try:
            os.makedirs(testBatchDir)
        except OSError:
            if not os.path.isdir(testBatchDir):
                raise RuntimeError("Could not create the output directory")

    # Store text file snapshot of config table at root session dir
    snapshotPath = os.path.join(testBatchDir, table + ".csv")
    with open(snapshotPath, "w") as sf:
        writeConfigCsv(sf, table, colnames, configs)

    # Add this session info to the sessions table
    session_id = manager.newSession(table, name, sql_datetime_str, whereClause)

    # Call the test_function for each config
    jobs = []
    id_queue = Queue()
    for config in configs:
        if qsub:
            p = Process(target=test_func, args=(config, id_queue, testBatchDir, qsub))
        else:
            p = Process(target=test_func, args=(config, id_queue, testBatchDir))
        p.start()
        jobs.append(p)
    for p in jobs:
        p.join()

    job_ids = []
    try:
        while True:
            job_ids.append(id_queue.get_nowait())
    except QueueEmpty:
        pass

    for id, cfg_id in job_ids:
        manager.addJobToSession(session_id, cfg_id, id)
示例#29
0
def list(args):
    db = DatabaseManager()
    db.printTable(args.table_name, ' '.join(args.where_clause))
示例#30
0
def update_status(args):
    db = DatabaseManager()
    db.updateJobStatus(args.job_id, args.new_status)