Ejemplo n.º 1
0
    def run_to_db(self, reverseMapT, runT):
        """
    Store the "run" data into the database.
    @param: reverseMapT: The map between directories and modules
    @param: runT:        The run data stored in a table
    """

        nameA = [
            'num_cores', 'num_nodes', 'account', 'job_id', 'queue',
            'submit_host'
        ]
        query = ""
        try:
            conn = self.connect()
            query = "USE " + self.db()
            conn.query(query)
            query = "START TRANSACTION"
            conn.query(query)

            translate(nameA, runT['envT'], runT['userT'])
            XALT_Stack.push("SUBMIT_HOST: " + runT['userT']['submit_host'])

            dateTimeStr = time.strftime(
                "%Y-%m-%d %H:%M:%S",
                time.localtime(float(runT['userT']['start_time'])))
            uuid = runT['xaltLinkT'].get('Build.UUID')
            if (uuid):
                uuid = "'" + uuid + "'"
            else:
                uuid = "NULL"

            #print( "Looking for run_uuid: ",runT['userT']['run_uuid'])

            query = "SELECT run_id FROM xalt_run WHERE run_uuid='%s'" % runT[
                'userT']['run_uuid']
            conn.query(query)

            result = conn.store_result()
            if (result.num_rows() > 0):
                #print("found")
                row = result.fetch_row()
                run_id = int(row[0][0])
                if (runT['userT']['end_time'] > 0):
                    query = "UPDATE xalt_run SET run_time='%.2f', end_time='%.2f' WHERE run_id='%d'" % (
                        runT['userT']['run_time'], runT['userT']['end_time'],
                        run_id)
                    conn.query(query)
                    query = "COMMIT"
                    conn.query(query)
                v = XALT_Stack.pop()
                carp("SUBMIT_HOST", v)

                return
            else:
                #print("not found")
                moduleName = obj2module(runT['userT']['exec_path'],
                                        reverseMapT)
                exit_status = runT['userT'].get('exit_status', 0)
                query = "INSERT INTO xalt_run VALUES (NULL,'%s','%s','%s', '%s',%s,'%s', '%s','%s','%.2f', '%.2f','%.2f','%d', '%d','%d','%s', '%d','%s','%s', '%s','%s') " % (
                    runT['userT']['job_id'], runT['userT']['run_uuid'],
                    dateTimeStr, runT['userT']['syshost'], uuid,
                    runT['hash_id'], runT['userT']['account'],
                    runT['userT']['exec_type'], runT['userT']['start_time'],
                    runT['userT']['end_time'], runT['userT']['run_time'],
                    runT['userT']['num_cores'], runT['userT']['num_nodes'],
                    runT['userT']['num_threads'], runT['userT']['queue'],
                    exit_status, runT['userT']['user'],
                    runT['userT']['exec_path'], moduleName,
                    runT['userT']['cwd'])
                conn.query(query)
                run_id = conn.insert_id()

            self.load_objects(conn, runT['libA'], reverseMapT,
                              runT['userT']['syshost'], "join_run_object",
                              run_id)

            # loop over env. vars.
            for key in runT['envT']:
                # use the single quote pattern to protect all the single quotes in env vars.
                value = patSQ.sub(r"\\'", runT['envT'][key])
                query = "SELECT env_id FROM xalt_env_name WHERE env_name='%s'" % key
                conn.query(query)
                result = conn.store_result()
                if (result.num_rows() > 0):
                    row = result.fetch_row()
                    env_id = int(row[0][0])
                    found = True
                else:
                    query = "INSERT INTO xalt_env_name VALUES(NULL, '%s')" % key
                    conn.query(query)
                    env_id = conn.insert_id()
                    found = False

                query = "INSERT INTO join_run_env VALUES (NULL, '%d', '%d', '%s')" % (
                    env_id, run_id, value.encode("ascii", "ignore"))
                try:
                    conn.query(query)
                except Exception as e:
                    query = "INSERT INTO join_run_env VALUES (NULL, '%d', '%d', '%s')" % (
                        env_id, run_id, "XALT_ILLEGAL_VALUE")
                    conn.query(query)

            v = XALT_Stack.pop()
            carp("SUBMIT_HOST", v)
            query = "COMMIT"
            conn.query(query)
            conn.close()

        except Exception as e:
            print(XALT_Stack.contents())
            print(query.encode("ascii", "ignore"))
            print("run_to_db(): ", e)
            sys.exit(1)
Ejemplo n.º 2
0
  def run_to_db(self, reverseMapT, runT):
    """
    Store the "run" data into the database.
    @param: reverseMapT: The map between directories and modules
    @param: runT:        The run data stored in a table
    """
    
    nameA = [ 'num_cores', 'num_nodes', 'account', 'job_id', 'queue' , 'submit_host']
    query = ""
    try:
      conn   = self.connect()
      cursor = conn.cursor()
      query  = "USE "+self.db()
      conn.query(query)
      query  = "START TRANSACTION"
      conn.query(query)

      
      translate(nameA, runT['envT'], runT['userT']);
      XALT_Stack.push("SUBMIT_HOST: "+ runT['userT']['submit_host'])

      runTime     = "%.2f" % float(runT['userT']['run_time'])
      endTime     = "%.2f" % float(runT['userT']['end_time'])
      dateTimeStr = time.strftime("%Y-%m-%d %H:%M:%S",
                                  time.localtime(float(runT['userT']['start_time'])))
      uuid        = runT['xaltLinkT'].get('Build.UUID',None)
      #print( "Looking for run_uuid: ",runT['userT']['run_uuid'])

      query = "SELECT run_id FROM xalt_run WHERE run_uuid=%s"
      cursor.execute(query,[runT['userT']['run_uuid']])

      if (cursor.rowcount > 0):
        #print("found")
        row    = cursor.fetchone()
        run_id = int(row[0])
        if (runT['userT']['end_time'] > 0):
          query  = "UPDATE xalt_run SET run_time=%s, end_time=%s WHERE run_id=%s" 
          cursor.execute(query,(runTime, endTime, run_id))
          query = "COMMIT"
          conn.query(query)
        v = XALT_Stack.pop()
        carp("SUBMIT_HOST",v)

        return
      else:
        #print("not found")
        moduleName    = obj2module(runT['userT']['exec_path'], reverseMapT)
        exit_status   = convertToTinyInt(runT['userT'].get('exit_status',0))
        num_threads   = convertToTinyInt(runT['userT'].get('num_threads',0))
        usr_cmdline   = runT.get('cmdlineA', "UNKNOWN")

        job_num_cores = int(runT['userT'].get('job_num_cores',0))
        startTime     = "%.f" % float(runT['userT']['start_time'])
        query  = "INSERT INTO xalt_run VALUES (NULL, %s,%s,%s, %s,%s,%s, %s,%s,%s, %s,%s,%s, %s,%s,%s, %s,%s,%s, %s,%s,%s, COMPRESS(%s))"
        cursor.execute(query, (runT['userT']['job_id'],      runT['userT']['run_uuid'],    dateTimeStr,
                               runT['userT']['syshost'],     uuid,                         runT['hash_id'],
                               runT['userT']['account'][:20],runT['userT']['exec_type'],   startTime,
                               endTime,                      runTime,                      runT['userT']['num_cores'],
                               job_num_cores,                runT['userT']['num_nodes'],   num_threads,
                               runT['userT']['queue'],       exit_status,                  runT['userT']['user'],
                               runT['userT']['exec_path'],   moduleName,                   runT['userT']['cwd'],
                               usr_cmdline))
        run_id   = cursor.lastrowid


      self.load_objects(conn, runT['libA'], reverseMapT, runT['userT']['syshost'],
                        "join_run_object", run_id)

      envT    = runT['envT']
      jsonStr = json.dumps(envT)
      query   = "INSERT INTO xalt_total_env VALUES(NULL, %s, COMPRESS(%s))"
      cursor.execute(query, [run_id, jsonStr])
      
      # loop over env. vars.
      for key in envT:
        if (not keep_env_var(key)):
          continue
        value = envT[key]
        query = "SELECT env_id FROM xalt_env_name WHERE env_name=%s"
        cursor.execute(query,[key])
        if (cursor.rowcount > 0):
          row    = cursor.fetchone()
          env_id = int(row[0])
          found  = True
        else:
          query  = "INSERT INTO xalt_env_name VALUES(NULL, %s)"
          cursor.execute(query,[key[:64]])
          env_id = cursor.lastrowid
          found  = False
        
        query = "INSERT INTO join_run_env VALUES (NULL, %s, %s, %s)"
        cursor.execute(query,(env_id, run_id, value))
          
      v = XALT_Stack.pop()
      carp("SUBMIT_HOST",v)
      query = "COMMIT"
      conn.query(query)
      conn.close()

    except Exception as e:
      print(XALT_Stack.contents())
      print(query.encode("ascii","ignore"))
      print ("run_to_db(): %s" % e)
Ejemplo n.º 3
0
    def run_to_db(self, reverseMapT, runT):
        """
    Store the "run" data into the database.
    @param: reverseMapT: The map between directories and modules
    @param: runT:        The run data stored in a table
    """

        nameA = [
            'num_cores', 'num_nodes', 'account', 'job_id', 'queue',
            'submit_host'
        ]
        query = ""
        try:
            conn = self.connect()
            cursor = conn.cursor()
            query = "USE " + self.db()
            conn.query(query)
            query = "START TRANSACTION"
            conn.query(query)

            translate(nameA, runT['envT'], runT['userT'])
            XALT_Stack.push("SUBMIT_HOST: " + runT['userT']['submit_host'])

            runTime = "%.2f" % float(runT['userT']['run_time'])
            endTime = "%.2f" % float(runT['userT']['end_time'])
            dateTimeStr = time.strftime(
                "%Y-%m-%d %H:%M:%S",
                time.localtime(float(runT['userT']['start_time'])))
            uuid = runT['xaltLinkT'].get('Build.UUID', None)
            #print( "Looking for run_uuid: ",runT['userT']['run_uuid'])

            query = "SELECT run_id FROM xalt_run WHERE run_uuid=%s"
            cursor.execute(query, [runT['userT']['run_uuid']])

            if (cursor.rowcount > 0):
                #print("found")
                row = cursor.fetchone()
                run_id = int(row[0])
                if (runT['userT']['end_time'] > 0):
                    query = "UPDATE xalt_run SET run_time=%s, end_time=%s WHERE run_id=%s"
                    cursor.execute(query, (runTime, endTime, run_id))
                    query = "COMMIT"
                    conn.query(query)
                v = XALT_Stack.pop()
                carp("SUBMIT_HOST", v)

                return
            else:
                #print("not found")
                moduleName = obj2module(runT['userT']['exec_path'],
                                        reverseMapT)
                exit_status = convertToTinyInt(runT['userT'].get(
                    'exit_status', 0))
                num_threads = convertToTinyInt(runT['userT'].get(
                    'num_threads', 0))
                usr_cmdline = json.dumps(runT['cmdlineA'])

                job_num_cores = int(runT['userT'].get('job_num_cores', 0))
                startTime = "%.f" % float(runT['userT']['start_time'])
                query = "INSERT INTO xalt_run VALUES (NULL, %s,%s,%s, %s,%s,%s, %s,%s,%s, %s,%s,%s, %s,%s,%s, %s,%s,%s, %s,%s,%s, COMPRESS(%s))"
                cursor.execute(
                    query, (runT['userT']['job_id'], runT['userT']['run_uuid'],
                            dateTimeStr, runT['userT']['syshost'], uuid,
                            runT['hash_id'], runT['userT']['account'][:20],
                            runT['userT']['exec_type'], startTime, endTime,
                            runTime, runT['userT']['num_cores'], job_num_cores,
                            runT['userT']['num_nodes'], num_threads,
                            runT['userT']['queue'], exit_status,
                            runT['userT']['user'], runT['userT']['exec_path'],
                            moduleName, runT['userT']['cwd'], usr_cmdline))
                run_id = cursor.lastrowid

            self.load_objects(conn, runT['libA'], reverseMapT,
                              runT['userT']['syshost'], "join_run_object",
                              run_id)

            envT = runT['envT']
            jsonStr = json.dumps(envT)
            query = "INSERT INTO xalt_total_env VALUES(NULL, %s, COMPRESS(%s))"
            cursor.execute(query, [run_id, jsonStr])

            # loop over env. vars.
            for key in envT:
                if (not keep_env_var(key)):
                    continue
                value = envT[key]
                query = "SELECT env_id FROM xalt_env_name WHERE env_name=%s"
                cursor.execute(query, [key])
                if (cursor.rowcount > 0):
                    row = cursor.fetchone()
                    env_id = int(row[0])
                    found = True
                else:
                    query = "INSERT INTO xalt_env_name VALUES(NULL, %s)"
                    cursor.execute(query, [key[:64]])
                    env_id = cursor.lastrowid
                    found = False

                query = "INSERT INTO join_run_env VALUES (NULL, %s, %s, %s)"
                cursor.execute(query, (env_id, run_id, value))

            v = XALT_Stack.pop()
            carp("SUBMIT_HOST", v)
            query = "COMMIT"
            conn.query(query)
            conn.close()

        except Exception as e:
            print(XALT_Stack.contents())
            print(query.encode("ascii", "ignore"))
            print("run_to_db(): %s" % e)
            sys.exit(1)
Ejemplo n.º 4
0
  def run_to_db(self, reverseMapT, runT):
    """
    Store the "run" data into the database.
    @param: reverseMapT: The map between directories and modules
    @param: runT:        The run data stored in a table
    """
    
    nameA = [ 'num_cores', 'num_nodes', 'account', 'job_id', 'queue' , 'submit_host']
    try:
      conn   = self.connect()
      query  = "USE "+self.db()
      conn.query(query)

      translate(nameA, runT['envT'], runT['userT']);
      XALT_Stack.push("SUBMIT_HOST: "+ runT['userT']['submit_host'])

      dateTimeStr = time.strftime("%Y-%m-%d %H:%M:%S",
                                  time.localtime(float(runT['userT']['start_time'])))
      uuid        = runT['xaltLinkT'].get('Build.UUID')
      if (uuid):
        uuid = "'" + uuid + "'"
      else:
        uuid = "NULL"

      #print( "Looking for run_uuid: ",runT['userT']['run_uuid'])

      query = "SELECT run_id FROM xalt_run WHERE run_uuid='%s'" % runT['userT']['run_uuid']
      conn.query(query)

      result = conn.store_result()
      if (result.num_rows() > 0):
        #print("found")
        row    = result.fetch_row()
        run_id = int(row[0][0])
        query  = "UPDATE xalt_run SET run_time='%.2f', end_time='%.2f' WHERE run_id='%d'" % (
          runT['userT']['run_time'], runT['userT']['end_time'], run_id)
        conn.query(query)
        v = XALT_Stack.pop()
        carp("SUBMIT_HOST",v)
        return
      else:
        #print("not found")
        moduleName = obj2module(runT['userT']['exec_path'], reverseMapT)
        query  = "INSERT INTO xalt_run VALUES (NULL,'%s','%s','%s', '%s',%s,'%s', '%s','%s','%.2f', '%.2f','%.2f','%d', '%d','%d','%s', '%s','%s',%s,'%s') " % (
          runT['userT']['job_id'],      runT['userT']['run_uuid'],    dateTimeStr,
          runT['userT']['syshost'],     uuid,                         runT['hash_id'],
          runT['userT']['account'],     runT['userT']['exec_type'],   runT['userT']['start_time'],
          runT['userT']['end_time'],    runT['userT']['run_time'],    runT['userT']['num_cores'],
          runT['userT']['num_nodes'],   runT['userT']['num_threads'], runT['userT']['queue'],
          runT['userT']['user'],        runT['userT']['exec_path'],   moduleName,
          runT['userT']['cwd'])
        conn.query(query)
        run_id   = conn.insert_id()

      self.load_objects(conn, runT['libA'], reverseMapT, runT['userT']['syshost'],
                        "join_run_object", run_id) 

      # loop over env. vars.
      for key in runT['envT']:
        # use the single quote pattern to protect all the single quotes in env vars.
        value = patSQ.sub(r"\\'", runT['envT'][key])
        query = "SELECT env_id FROM xalt_env_name WHERE env_name='%s'" % key
        conn.query(query)
        result = conn.store_result()
        if (result.num_rows() > 0):
          row    = result.fetch_row()
          env_id = int(row[0][0])
          found  = True
        else:
          query  = "INSERT INTO xalt_env_name VALUES(NULL, '%s')" % key
          conn.query(query)
          env_id = conn.insert_id()
          found  = False
        #print("env_id: ", env_id, ", found: ",found)

        
        query = "INSERT INTO join_run_env VALUES (NULL, '%d', '%d', '%s')" % (
          env_id, run_id, value.encode("ascii","ignore"))
        conn.query(query)
      v = XALT_Stack.pop()
      carp("SUBMIT_HOST",v)
    except Exception as e:
      print(XALT_Stack.contents())
      print(query.encode("ascii","ignore"))
      print ("run_to_db(): ",e)
      sys.exit (1)