Ejemplo n.º 1
0
def master_plot(file,mode='lines',threshold=False,
                output_dir='.',prefix='graph',mintime=3600,wayness=16,
                header='Master',lariat_dict=None,wide=False,job_stats=None):
  k1={'amd64' :
      ['amd64_core','amd64_core','amd64_sock','lnet','lnet',
       'ib_sw','ib_sw','cpu'],
      'intel' : ['intel_pmc3', 'intel_pmc3', 'intel_pmc3', 
                 'lnet', 'lnet', 'ib_ext','ib_ext','cpu','mem','mem'],
      'intel_snb' : ['intel_snb_imc', 'intel_snb_imc', 'intel_snb', 
                     'lnet', 'lnet', 'ib_sw','ib_sw','cpu',
                     'intel_snb', 'intel_snb', 'mem', 'mem'],
      }
  
  k2={'amd64':
      ['SSE_FLOPS','DCSF','DRAM','rx_bytes','tx_bytes',
       'rx_bytes','tx_bytes','user'],
      'intel' : ['MEM_LOAD_RETIRED_L1D_HIT', 'FP_COMP_OPS_EXE_X87', 
                 'INSTRUCTIONS_RETIRED', 'rx_bytes','tx_bytes', 
                 'port_recv_data','port_xmit_data','user', 'MemUsed', 'AnonPages'],
      'intel_snb' : ['CAS_READS', 'CAS_WRITES', 'LOAD_L1D_ALL',
                     'rx_bytes','tx_bytes', 'rx_bytes','tx_bytes','user',
                     'SSE_D_ALL', 'SIMD_D_256', 'MemUsed', 'AnonPages'],
      }

  try:
    print file
    ts=tspl.TSPLSum(file,k1,k2,job_stats)
  except tspl.TSPLException as e:
    return 
  
  ignore_qs=[]#'gpu','gpudev','vis','visdev']
  if not tspl_utils.checkjob(ts,mintime,wayness,ignore_qs):
    return

  if lariat_dict == None:
    ld=lariat_utils.LariatData(ts.j.id,end_epoch=ts.j.end_time,daysback=3,directory=analyze_conf.lariat_path)
  elif lariat_dict == "pass": ld = lariat_utils.LariatData(ts.j.id)
  else:
    ld=lariat_utils.LariatData(ts.j.id,olddata=lariat_dict)

    

  wayness=ts.wayness
  if ld.wayness != -1 and ld.wayness < ts.wayness:
    wayness=ld.wayness

  if wide:
    fig,ax=plt.subplots(6,2,figsize=(15.5,12),dpi=110)

    # Make 2-d array into 1-d, and reorder so that the left side is blank
    ax=my_utils.flatten(ax)
    ax_even=ax[0:12:2]
    ax_odd =ax[1:12:2]
    ax=ax_odd + ax_even
    
    for a in ax_even:
      a.axis('off')
  else:
    fig,ax=plt.subplots(6,1,figsize=(8,12),dpi=110)

  if mode == 'hist':
    plot=plot_thist
  elif mode == 'percentile':
    plot=plot_mmm
  else:
    plot=plot_lines

  if ts.pmc_type == 'intel_snb' :
    # Plot key 1
    plot(ax[0],ts,[8,9],3600.,1e9,
         ylabel='Total AVX +\nSSE Ginst/s')
    
    # Plot key 2
    plot(ax[1],ts,[0,1],3600.,1.0/64.0*1024.*1024.*1024.,
         ylabel='Total Mem BW GB/s')

    #Plot key 3
    #plot(ax[2],ts,[2],3600.,1.0/64.0*1e9, ylabel='L1 BW GB/s')
    plot(ax[2],ts,[10,-11],3600.,1024.0*1024.0*1024.0, ylabel='Memory Usage GB',
         do_rate=False)
  elif ts.pmc_type == 'intel':
    plot(ax[0],ts,[1],3600.,1e9,ylabel='FP Ginst/s')
    plot(ax[2],ts,[8,-9],3600.,1024.0*1024.0*1024.0, ylabel='Memory Usage GB',do_rate=False)
  else: 
    #Fix this to support the old amd plots
    print ts.pmc_type + ' not supported'
    return 

  # Plot lnet sum rate
  plot(ax[3],ts,[3,4],3600.,1024.**2,ylabel='Total lnet MB/s')

  # Plot remaining IB sum rate
  if ts.pmc_type == 'intel_snb' :
    plot(ax[4],ts,[5,6,-3,-4],3600.,1024.**2,ylabel='Total (ib_sw-lnet) MB/s') 
  elif ts.pmc_type == 'intel' :
    plot(ax[4],ts,[5,6,-3,-4],3600.,1024.**2,ylabel='Total (ib_ext-lnet) MB/s') 

  #Plot CPU user time
  plot(ax[5],ts,[7],3600.,wayness*100.,
       xlabel='Time (hr)',
       ylabel='Total cpu user\nfraction')
  
  print ts.j.id + ': '
  
  plt.subplots_adjust(hspace=0.35)
  if wide:
    left_text=header+'\n'+my_utils.summary_text(ld,ts)
    text_len=len(left_text.split('\n'))
    fontsize=ax[0].yaxis.label.get_size()
    linespacing=1.2
    fontrate=float(fontsize*linespacing)/72./15.5
    yloc=.8-fontrate*(text_len-1) # this doesn't quite work. fontrate is too
                                  # small by a small amount
    plt.figtext(.05,yloc,left_text,linespacing=linespacing)
    fname='_'.join([prefix,ts.j.id,ts.owner,'wide_master'])
  elif header != None:
    title=header+'\n'+ts.title
    if threshold:
      title+=', V: %(v)-6.1f' % {'v': threshold}
    title += '\n' + ld.title()
    plt.suptitle(title)
    fname='_'.join([prefix,ts.j.id,ts.owner,'master'])
  else:
    fname='_'.join([prefix,ts.j.id,ts.owner,'master'])

  if mode == 'hist':
    fname+='_hist'
  elif mode == 'percentile':
    fname+='_perc'
    

  plt.close()

  return fig, fname
Ejemplo n.º 2
0
 def test_net(self, writer=None):
     total_metrics = [0 for _ in range(len(self.metrics_names))]
     total_loss = 0
     predictions = None
     labels = None
     for test_i in range(0, self.test_iter):
         batch = self.test_batch_iter.next()
         if predictions is None:
             if self.mode == 'regression':
                 predictions = np.zeros(([
                     0,
                 ] + self.prediction_op.shape.as_list()[1:]))
             else:  # segmentation
                 predictions = np.zeros(((0, ) + batch[1].shape[1:]))
         if labels is None:
             if self.mode == 'regression':
                 labels = np.zeros(([
                     0,
                 ] + self.prediction_op.shape.as_list()[1:]))
             else:  # segmentation
                 labels = np.zeros(((0, ) + batch[1].shape[1:]))
         res = self.sess.run(
             self.metrics_ops + [self.loss_op, self.prediction_op],
             feed_dict=self.create_feed_dict(batch, phase=0))
         for i in range(len(total_metrics)):
             total_metrics[i] += res[i]
         total_loss += res[len(total_metrics)]
         predictions = np.concatenate(
             [predictions, res[len(total_metrics) + 1]], axis=0)
         if self.mode == 'regression':
             labels = np.concatenate(
                 [labels, [flatten(b) for b in batch[1]]], axis=0)
         else:
             labels = np.concatenate([labels, batch[1]], axis=0)
     for i in range(len(total_metrics)):
         total_metrics[i] = total_metrics[i] * 1. / self.test_iter
     total_loss = total_loss * 1. / self.test_iter
     # add loss and metrics to tensorboard
     if self.mode == 'regression':
         img = distance_plot_for_tensorboard(labels[:, 0].flatten(),
                                             predictions[:, 0].flatten(),
                                             self.eval_image_shape)
         if 'coord' in self.labels:
             img_coords = coords_xyz_plot_for_tensorboard(
                 np.concatenate([labels[:, 1:4], labels[:, 4:7]], axis=0),
                 np.concatenate([predictions[:, 1:4], predictions[:, 4:7]],
                                axis=0), self.eval_coords_image_shape)
     elif self.mode == 'segmentation':
         img = confusion_matrix_for_tensorboard(labels[:, 0].flatten(),
                                                predictions[:, 0].flatten(),
                                                self.labels,
                                                self.eval_image_shape)
     feed_dict = {
         var: value
         for var, value in zip(self.test_metrics_vars, total_metrics)
     }
     feed_dict.update({
         self.test_loss_var: total_loss,
         self.eval_image_var: img
     })
     if 'coord' in self.labels:
         feed_dict.update({self.eval_coords_image_var: img_coords})
     feed_dict[K.learning_phase()] = 0
     if self.show_activation is not False:
         # feed activation images four times to deal with multi-gpu (up to 4)
         # if net has multiple inputs, need to feed multiple activation images
         if not isinstance(self.show_activation, tuple):
             images = (self.show_activation, )
         else:
             images = self.show_activation
         for i, img in enumerate(images):
             feed_dict.update(
                 {self.net.inputs[i]: np.concatenate((img, img, img, img))})
     summary = self.sess.run(self.test_summary_op, feed_dict=feed_dict)
     writer.add_summary(summary, self.global_step_var.eval())
     del img
     return total_metrics, total_loss
Ejemplo n.º 3
0
def master_plot(file,mode='lines',threshold=False,
                output_dir='.',prefix='graph',mintime=3600,wayness=16,
                header='Master'):
  k1={'amd64' :
      ['amd64_core','amd64_core','amd64_sock','lnet','lnet',
       'ib_sw','ib_sw','cpu'],
      'intel' : ['intel_pmc3', 'intel_pmc3', 'intel_pmc3', 
                 'lnet', 'lnet', 'ib_sw','ib_sw','cpu']
      }
  
  k2={'amd64':
      ['SSE_FLOPS','DCSF','DRAM','rx_bytes','tx_bytes',
       'rx_bytes','tx_bytes','user'],
      'intel' : ['PMC3', 'PMC2', 'FIXED_CTR0',
                 'rx_bytes','tx_bytes', 'rx_bytes','tx_bytes','user']
      }

  try:
    print file
    ts=tspl.TSPLSum(file,k1,k2)
  except tspl.TSPLException as e:
    return

  if not tspl_utils.checkjob(ts,mintime,wayness):
    return

  fig,ax=plt.subplots(6,1,figsize=(8,12),dpi=80)
  ax=my_utils.flatten(ax)

  if mode == 'hist':
    plot=plot_thist
  elif mode == 'percentile':
    plot=plot_mmm
  else:
    plot=plot_lines
  
  # Plot SSE FLOPS
  plot(ax[0],ts,[0],3600.)
    
  # Plot DCSF rate
  plot(ax[1],ts,[1],3600.,1e9)

  #Plot DRAM rate
  plot(ax[2],ts,[2],3600.,1e9)
  
  # Plot lnet sum rate
  plot(ax[3],ts,[3,4],3600.,1024.**2,ylabel='Total lnet MB/s')

  # Plot remaining IB sum rate
  plot(ax[4],ts,[5,6,-3,-4],3600.,1024.**2,ylabel='Total (ib_sw-lnet) MB/s') 

  #Plot CPU user time
  plot(ax[5],ts,[7],3600.,ts.wayness*100.,
       xlabel='Time (hr)',
       ylabel='Total cpu user\nfraction')
  
  print ts.j.id + ': '
  print 'cc'
  
  title=header+'\n'+ts.title
  if threshold:
    title+=', V: %(v)-8.3f' % {'v': threshold}
  ld=lariat_utils.LariatData(ts.j.id,ts.j.end_time,'/scratch/projects/lariatData')
  title += '\n' + ld.title()
  print 'dd'

  plt.suptitle(title)
  plt.subplots_adjust(hspace=0.35)

  fname='_'.join([prefix,ts.j.id,ts.owner,'master'])
  if mode == 'hist':
    fname+='_hist'
  elif mode == 'percentile':
    fname+='_perc'
    
  fig.savefig(output_dir+'/'+fname)
  plt.close()
Ejemplo n.º 4
0
def master_plot(file,
                mode='lines',
                threshold=False,
                output_dir='.',
                prefix='graph',
                mintime=3600,
                wayness=16,
                header='Master',
                lariat_dict=None,
                wide=False,
                job_stats=None):
    k1 = {
        'amd64': [
            'amd64_core', 'amd64_core', 'amd64_sock', 'lnet', 'lnet', 'ib_sw',
            'ib_sw', 'cpu'
        ],
        'intel': [
            'intel_pmc3', 'intel_pmc3', 'intel_pmc3', 'lnet', 'lnet', 'ib_ext',
            'ib_ext', 'cpu', 'mem', 'mem'
        ],
        'intel_snb': [
            'intel_snb_imc', 'intel_snb_imc', 'intel_snb', 'lnet', 'lnet',
            'ib_sw', 'ib_sw', 'cpu', 'intel_snb', 'intel_snb', 'mem', 'mem'
        ],
    }

    k2 = {
        'amd64': [
            'SSE_FLOPS', 'DCSF', 'DRAM', 'rx_bytes', 'tx_bytes', 'rx_bytes',
            'tx_bytes', 'user'
        ],
        'intel': [
            'MEM_LOAD_RETIRED_L1D_HIT', 'FP_COMP_OPS_EXE_X87',
            'INSTRUCTIONS_RETIRED', 'rx_bytes', 'tx_bytes', 'port_recv_data',
            'port_xmit_data', 'user', 'MemUsed', 'AnonPages'
        ],
        'intel_snb': [
            'CAS_READS', 'CAS_WRITES', 'LOAD_L1D_ALL', 'rx_bytes', 'tx_bytes',
            'rx_bytes', 'tx_bytes', 'user', 'SSE_D_ALL', 'SIMD_D_256',
            'MemUsed', 'AnonPages'
        ],
    }

    try:
        print file
        ts = tspl.TSPLSum(file, k1, k2, job_stats)
    except tspl.TSPLException as e:
        return

    ignore_qs = []  #'gpu','gpudev','vis','visdev']
    if not tspl_utils.checkjob(ts, mintime, wayness, ignore_qs):
        return

    if lariat_dict == None:
        ld = lariat_utils.LariatData(ts.j.id,
                                     end_epoch=ts.j.end_time,
                                     daysback=3,
                                     directory=analyze_conf.lariat_path)
    elif lariat_dict == "pass":
        ld = lariat_utils.LariatData(ts.j.id)
    else:
        ld = lariat_utils.LariatData(ts.j.id, olddata=lariat_dict)

    wayness = ts.wayness
    if ld.wayness != -1 and ld.wayness < ts.wayness:
        wayness = ld.wayness

    if wide:
        fig, ax = plt.subplots(6, 2, figsize=(15.5, 12), dpi=110)

        # Make 2-d array into 1-d, and reorder so that the left side is blank
        ax = my_utils.flatten(ax)
        ax_even = ax[0:12:2]
        ax_odd = ax[1:12:2]
        ax = ax_odd + ax_even

        for a in ax_even:
            a.axis('off')
    else:
        fig, ax = plt.subplots(6, 1, figsize=(8, 12), dpi=110)

    if mode == 'hist':
        plot = plot_thist
    elif mode == 'percentile':
        plot = plot_mmm
    else:
        plot = plot_lines

    if ts.pmc_type == 'intel_snb':
        # Plot key 1
        plot(ax[0], ts, [8, 9], 3600., 1e9, ylabel='Total AVX +\nSSE Ginst/s')

        # Plot key 2
        plot(ax[1],
             ts, [0, 1],
             3600.,
             1.0 / 64.0 * 1024. * 1024. * 1024.,
             ylabel='Total Mem BW GB/s')

        #Plot key 3
        #plot(ax[2],ts,[2],3600.,1.0/64.0*1e9, ylabel='L1 BW GB/s')
        plot(ax[2],
             ts, [10, -11],
             3600.,
             1024.0 * 1024.0 * 1024.0,
             ylabel='Memory Usage GB',
             do_rate=False)
    elif ts.pmc_type == 'intel':
        plot(ax[0], ts, [1], 3600., 1e9, ylabel='FP Ginst/s')
        plot(ax[2],
             ts, [8, -9],
             3600.,
             1024.0 * 1024.0 * 1024.0,
             ylabel='Memory Usage GB',
             do_rate=False)
    else:
        #Fix this to support the old amd plots
        print ts.pmc_type + ' not supported'
        return

    # Plot lnet sum rate
    plot(ax[3], ts, [3, 4], 3600., 1024.**2, ylabel='Total lnet MB/s')

    # Plot remaining IB sum rate
    if ts.pmc_type == 'intel_snb':
        plot(ax[4],
             ts, [5, 6, -3, -4],
             3600.,
             1024.**2,
             ylabel='Total (ib_sw-lnet) MB/s')
    elif ts.pmc_type == 'intel':
        plot(ax[4],
             ts, [5, 6, -3, -4],
             3600.,
             1024.**2,
             ylabel='Total (ib_ext-lnet) MB/s')

    #Plot CPU user time
    plot(ax[5],
         ts, [7],
         3600.,
         wayness * 100.,
         xlabel='Time (hr)',
         ylabel='Total cpu user\nfraction')

    print ts.j.id + ': '

    plt.subplots_adjust(hspace=0.35)
    if wide:
        left_text = header + '\n' + my_utils.summary_text(ld, ts)
        text_len = len(left_text.split('\n'))
        fontsize = ax[0].yaxis.label.get_size()
        linespacing = 1.2
        fontrate = float(fontsize * linespacing) / 72. / 15.5
        yloc = .8 - fontrate * (text_len - 1
                                )  # this doesn't quite work. fontrate is too
        # small by a small amount
        plt.figtext(.05, yloc, left_text, linespacing=linespacing)
        fname = '_'.join([prefix, ts.j.id, ts.owner, 'wide_master'])
    elif header != None:
        title = header + '\n' + ts.title
        if threshold:
            title += ', V: %(v)-6.1f' % {'v': threshold}
        title += '\n' + ld.title()
        plt.suptitle(title)
        fname = '_'.join([prefix, ts.j.id, ts.owner, 'master'])
    else:
        fname = '_'.join([prefix, ts.j.id, ts.owner, 'master'])

    if mode == 'hist':
        fname += '_hist'
    elif mode == 'percentile':
        fname += '_perc'

    plt.close()

    return fig, fname