示例#1
0
    def test_split_connection(self):
        cm = None
        M = None
        S = None
        try:
            cm = Cm.CM("test_split_connection")
            cm.create_workspace()

            # M (master) S(slave)
            M = _init_pgs(0, 'localhost', 1900, cm.dir)
            M.smr.role_master(M.id, 1, 0)
            S = _init_pgs(1, 'localhost', 1910, cm.dir)
            S.smr.role_slave(S.id, 'localhost', M.base_port, 0)
            # M: confset "slave_idle_timeout_msec" to 100
            M.smr.fi_once('__no_free_client_conn')
            M.smr.confset('slave_idle_timeout_msec', 100)
            S.smr.wait_role(Smr.SMR.LCONN)
            M.smr.confset('slave_idle_timeout_msec', 18000)
            # S: role slave again
            ps = S.smr.getseq_log()
            S.smr.role_slave(S.id, 'localhost', M.base_port, ps['max'])
            S.smr.wait_role(Smr.SMR.SLAVE)
            S.be.init_conn()
            # check be at S works
            # Util.tstop('before check it!')
            r = S.be.set(0, '100')
            assert r >= 0
        finally:
            if M is not None:
                M.kill()
            if S is not None:
                S.kill()
            if cm is not None:
                cm.remove_workspace()
示例#2
0
    def test_slave_keep_alive(self):
        cm = None
        pgs1 = None
        pgs2 = None
        pgs3 = None
        try:
            cm = Cm.CM("test_pgs")
            cm.create_workspace()
            pg = Pg.PG(0)

            # pgs1 --> master
            pgs1 = Pgs.PGS(0, 'localhost', 1900, cm.dir)
            pg.join(pgs1, start=True)
            pgs1.smr.wait_role(Smr.SMR.MASTER)

            # pgs2 --> slave
            pgs2 = Pgs.PGS(1, 'localhost', 1910, cm.dir)
            pg.join(pgs2, start=True)
            pgs2.smr.wait_role(Smr.SMR.SLAVE)

            # send command
            r = pgs2.be.set(0, '100')
            assert r >= 0

            # check master idle time grows close upto 1 second.
            for i in range(0, 9):
                res = pgs1.smr.info('slave')
                line = res['slave']['slave_' + str(pgs2.id)]
                assert line != None
                idle_msec = int(line[(line.find('idle_msec=') +
                                      len('idle_msec=')):])
                assert idle_msec < i * 100 + 500
                time.sleep(0.1)  # sleep 100 msec

            # make pgs3 and force join
            pgs3 = Pgs.PGS(1, 'localhost', 1920, cm.dir)
            pg.join(pgs3, start=True, Force=True)
            pgs3.smr.wait_role(Smr.SMR.SLAVE)
            pgs2.smr.wait_role(Smr.SMR.LCONN)

            # check
            r2 = pgs3.be.get(0)
            assert r == r2

            # current configuration (pgs1, pgs3)
            pgs1.smr.confset('slave_idle_timeout_msec', 100)
            pgs3.smr.wait_role(Smr.SMR.LCONN)

        finally:
            if pgs1 is not None:
                pgs1.kill_smr()
                pgs1.kill_be()
            if pgs2 is not None:
                pgs2.kill_smr()
                pgs2.kill_be()
            if pgs3 is not None:
                pgs3.kill_smr()
                pgs3.kill_be()
            if cm is not None:
                cm.remove_workspace()
示例#3
0
  def _test_pgs(self):
    cm = None
    pgs = None
    try:
      cm = Cm.CM("test_pgs")
      assert cm is not None

      cm.create_workspace()
      assert cm.dir != None

      pgs = Pgs.PGS(0, 'localhost', 1900, cm.dir)
      assert pgs is not None

      pgs.start_smr()
      assert pgs.smr != None

      role = pgs.smr.get_role()
      assert role == Smr.SMR.NONE

      pgs.start_be()
      assert pgs.be != None

      pgs.smr.wait_role(Smr.SMR.LCONN)
    except:
      Util.tstop('Exception Occurred')
      traceback.print_exc()
      e = sys.exc_info()[0]
    finally:
      if pgs is not None:
	pgs.kill_be()
	pgs.kill_smr()
      if cm is not None:
	cm.remove_workspace()
示例#4
0
  def _test_smr(self):
    cm = None
    pgs = None
    try:
      cm = Cm.CM("test_pgs")
      assert cm is not None

      cm.create_workspace()
      assert cm.dir != None

      pgs = Pgs.PGS(0, 'localhost', 1900, cm.dir)
      assert pgs is not None

      pgs.start_smr()
      assert pgs.smr != None

      # send confset with empty arguments
      self.assertRaisesRegexp(Exception, '-ERR bad number of token:0',
              pgs.smr.confset, '', '')

    finally:
      if pgs is not None:
	pgs.kill_smr()
      if cm is not None:
	cm.remove_workspace()
示例#5
0
    def test_deadline(self):
        cm = None
        pgs = None
        dummy_peer = None
        try:
            cm = Cm.CM("test_pgs")
            cm.create_workspace()
            pg = Pg.PG(0)

            pgs = Pgs.PGS(0, 'localhost', 1900, cm.dir)
            pg.join(pgs, start=True)
            pgs.smr.wait_role(Smr.SMR.MASTER)

            dummy_peer = DummyPeer('localhost', 1902, 3)
            dummy_peer.start()
            time.sleep(0.5)

            st = int(round(time.time() * 1000))
            seqs = pgs.smr.getseq_log()
            et = int(round(time.time() * 1000))
            assert et - st < 1000

        finally:
            # Util.tstop('Check output!')
            if pgs is not None:
                pgs.kill_smr()
                pgs.kill_be()
            if dummy_peer is not None:
                dummy_peer.join()
            if cm is not None:
                cm.remove_workspace()
示例#6
0
 def __init__(self, name, members):
   self.name = name
   self.members = members
   self.cm = Cm.CM(self.name)
   self.cm.create_workspace()
   self.pg = None
   self.pgs_ary = []
   self.clients = []
示例#7
0
 def __init__(self, name, copy):
     self.name = name
     self.copy = copy
     self.cm = Cm.CM(self.name)
     self.cm.create_workspace()
     self.pg = None
     self.pgs_ary = []
     self.clients = []
示例#8
0
  def _test_be(self):
    cm = None
    pgs = None
    try:
      cm = Cm.CM("test_be")
      cm.create_workspace()
      pg = Pg.PG(0)

      # pgs --> master
      pgs = Pgs.PGS(0, 'localhost', 1900, cm.dir)
      pg.join(pgs, start=True)
      pgs.smr.wait_role(Smr.SMR.MASTER)

      # Test basic op
      old = pgs.be.set(0, '100')
      assert old >= 0

      r = pgs.be.reset()
      assert r == 0

      new = pgs.be.set(0, '100')
      assert old == new

      r = pgs.be.ping()
      assert r == 0

      # do checkpoint
      r = pgs.be.ckpt()
      assert r == 0

      # restart
      pg.leave(pgs.id, kill=True)
      pg.join(pgs, start=True)
      pgs.smr.wait_role(Smr.SMR.MASTER)

      # check crc of key = 0
      new = pgs.be.get(0)
      assert old == new
    except:
      Util.tstop('Exception Occurred')
      traceback.print_exc()
      e = sys.exc_info()[0]
    finally:
      if pgs is not None:
	pgs.kill_smr()
	pgs.kill_be()
      if cm is not None:
	cm.remove_workspace()
示例#9
0
 def test_quorum_membership_commands(self):
     cm = None
     M = None
     try:
         cm = Cm.CM("test_quorum")
         cm.create_workspace()
         # M(1), S, S
         M = _init_pgs(0, 'localhost', 1900, cm.dir)
         M.smr.role_master(M.id, 1, 0, [1, 2])
         qm = M.smr.getquorumv()
         assert len(qm) == 3  # quorum and nids
     finally:
         #Util.tstop('Check output!')
         if M is not None:
             M.kill()
         if cm is not None:
             cm.remove_workspace()
示例#10
0
  def _test_cm(self):
    cm = None
    try:
      cm = Cm.CM("test_cm")
      assert cm is not None

      cm.create_workspace()
      dir = cm.dir
      assert os.path.exists(dir)

      cm.remove_workspace()
      assert cm.dir is None
      assert os.path.exists(dir) == False
      cm = None
    finally:
      if cm != None:
	cm.remove_workspace()
示例#11
0
    def test_logdelete(self):
        cm = None
        pgs1 = None
        try:
            cm = Cm.CM("test_portscan")
            cm.create_workspace()
            pg = Pg.PG(0)

            # pgs --> master
            pgs = Pgs.PGS(0, 'localhost', 1900, cm.dir)
            pg.join(pgs, start=True)
            pgs.smr.wait_role(Smr.SMR.MASTER)

            # -----------------------------------------
            # Test bad handshake blocks IP temporarily
            # -----------------------------------------
            for off in range(0, 3):
                # bad handshake
                try:
                    conn = Conn.Conn('localhost', 1900 + off)
                    resp = conn.do_request('ping')
                except:
                    pass
                self._check_conn_blocked()

            # wait for block released
            time.sleep(2.0)  # actually 1.5 sec

            # -------------------------------------------------------
            # Can't connect mgmt port SMR_MAX_MGMT_CLIENTS_PER_IP(50)
            # -------------------------------------------------------
            conns = []
            for i in range(50 - 1):  # -1
                conn = Conn.Conn('localhost', 1903)
                resp = conn.do_request('ping')
                assert (len(resp) == 1 and resp[0].startswith('+OK')), resp
                conns.append(conn)
            self._check_conn_blocked()

        finally:
            if pgs is not None:
                pgs.kill_smr()
                pgs.kill_be()
            if cm is not None:
                cm.remove_workspace()
示例#12
0
  def _test_pg(self):
    cm = None
    pgs1 = None
    pgs2 = None
    try:
      cm = Cm.CM("test_pgs")
      cm.create_workspace()
      pg = Pg.PG(0)

      # pgs1 --> master
      pgs1 = Pgs.PGS(0, 'localhost', 1900, cm.dir)
      pg.join(pgs1, start=True)
      pgs1.smr.wait_role(Smr.SMR.MASTER)

      # pgs2 --> slave
      pgs2 = Pgs.PGS(1, 'localhost', 1910, cm.dir)
      pg.join(pgs2, start=True)
      pgs2.smr.wait_role(Smr.SMR.SLAVE)

      # kill pgs2, check quorum, 
      pg.leave(pgs2.id, kill = True)
      assert pgs1.smr.getquorum() == 0

      # join pgs2
      pg.join(pgs2, start=True)
      pgs2.smr.wait_role(Smr.SMR.SLAVE)

      # kill pgs1 (check pgs1 is master)
      pg.leave(pgs1.id, kill = True)
      pgs2.smr.wait_role(Smr.SMR.MASTER)
      assert pgs2.smr.getquorum() == 0

      # join pgs1
      pg.join(pgs1, start=True)
      pgs1.smr.wait_role(Smr.SMR.SLAVE)

    finally:
      if pgs1 is not None:
	pgs1.kill_smr()
	pgs1.kill_be()
      if pgs2 is not None:
	pgs2.kill_smr()
	pgs2.kill_be()
      if cm is not None:
	cm.remove_workspace()
示例#13
0
  def test_log_recover_idempotent(self):
    cm = None
    pgs = None
    try:
      cm = Cm.CM("test_log_recover_idempotent")
      cm.create_workspace()
      pg = Pg.PG(0)

      # pgs --> master
      pgs = Pgs.PGS(0, 'localhost', self.BASE_PORT, cm.dir)
      pg.join(pgs, start=True)
      pgs.smr.wait_role(Smr.SMR.MASTER)

      # make some logs
      self.make_some_logs(runtime=10)
      time.sleep(1)

      # remember original sequences
      org_seqs = pgs.smr.getseq_log()

      # kill pgs
      pg.leave(pgs.id, kill=True)

      # log_recovery is dempotent
      for i in range(0, 3):
	# due to sucking master election logic in Pg.py we do not launch be currently
	# use PgNew later
	pgs.start_smr()
	seqs = pgs.smr.getseq_log()
	pgs.kill_smr()
	assert seqs['commit'] == org_seqs['commit']
	assert seqs['min'] == org_seqs['min']
	assert seqs['max'] == org_seqs['max']
	
      
    finally:
      # Util.tstop('Check output!')
      if pgs is not None:
	pgs.kill_smr()
	pgs.kill_be()
      if cm is not None:
	cm.remove_workspace() 
示例#14
0
def reliable_failover():
    try:
        gCtx.cm = Cm.CM("reliable_failover")
        gCtx.cm.create_workspace()
        setup_pgs()
        setup_cluster('reliable_failover')
        # wait initial green
        while True:
            time.sleep(0.5)
            allGreen = True
            for pgs in gCtx.pgs_ary:
                role = pgs.smr.get_role()
                if role != Smr.SMR.MASTER and role != Smr.SMR.SLAVE:
                    allGreen = False
                    break
            if allGreen:
                break
        for pgs in gCtx.pgs_ary:
            pgs.be.init_conn()

        print ("\n\n\n============================================================== START")
        start_clients()
        dirty_progress()
        stop_clients()
        print ("============================================================== END\n\n\n")
    except:
        traceback.print_exc()
        e = sys.exc_info()[0]
        print e
        Util.tstop('CHECK EXCEPTION')
    finally:
        try:
            teardown_cluster('reliable_failover', True)
        except:
            pass
        try:
            teardown_pgs()
        except:
            pass
        if gCtx.cm is not None:
            gCtx.cm.remove_workspace()
示例#15
0
    def test_broken_log_recover(self):
        ''' sigkill(9) smr and check recovery works '''
        cm = None
        pgs = None
        try:
            cm = Cm.CM("test_broken_log_recover")
            cm.create_workspace()
            pg = Pg.PG(0)

            # pgs --> master
            pgs = Pgs.PGS(0, 'localhost', self.BASE_PORT, cm.dir)
            pg.join(pgs, start=True)
            pgs.smr.wait_role(Smr.SMR.MASTER)

            # make some logs
            self.make_broken_logs(pgs)

            # check idemponent
            org_seqs = None
            for i in range(0, 3):
                pgs.start_smr()
                seqs = pgs.smr.getseq_log()
                pgs.kill_smr(sigkill=True)
                if org_seqs == None:
                    org_seqs = seqs
                    print '--- ORG_SEQS'
                    print org_seqs
                else:
                    print '--- SEQS'
                    print seqs
                    assert seqs['commit'] == org_seqs['commit']
                    assert seqs['min'] == org_seqs['min']
                    assert seqs['max'] == org_seqs['max']
        finally:
            if pgs is not None:
                pgs.kill_smr()
                pgs.kill_be()
            if cm is not None:
                cm.remove_workspace()
示例#16
0
    def test_be_retry(self):
        cm = None
        M = None
        S = None
        try:
            cm = Cm.CM("test_be_retry")
            cm.create_workspace()

            # M (master) S(slave)
            M = _init_pgs(0, 'localhost', 1900, cm.dir)
            M.smr.role_master(M.id, 1, 0)
            S = _init_pgs(1, 'localhost', 1910, cm.dir)
            S.smr.role_slave(S.id, 'localhost', M.base_port, 0)
            count = 1
            while count > 0:
                S.smr.role_lconn()
                seqs = S.smr.getseq_log()
                # Util.tstop('before fault injection!')
                M.smr.fi_error_client_accept()
                S.smr.role_slave(S.id, 'localhost', M.base_port, seqs['max'])
                time.sleep(0.5)
                info = M.smr.info()
                #print (info)
                assert (('client' in info)
                        and not ('client_1' in info['client']))
                time.sleep(1.0)
                info = M.smr.info()
                #print (info)
                assert (('client' in info) and ('client_1' in info['client']))
                count = count - 1
        finally:
            if M is not None:
                M.kill()
            if S is not None:
                S.kill()
            if cm is not None:
                cm.remove_workspace()
示例#17
0
def sprint(opt):
  if opt.valgrind_smr:
    Conf.VALGRIND_SMR = True
  if opt.use_mem_log:
    Conf.USE_MEM_LOG = True
  if opt.smr_opt_x:
    Conf.SMR_OPT_X = opt.smr_opt_x
  if opt.smr_bin_path:
    Conf.OVERRIDE_SMR_BIN_PATH=opt.smr_bin_path

  cm = None
  pgs = None
  pgs_ary = []
  client_ary = []
  try:
    cm = Cm.CM("test_be")
    cm.create_workspace()
    pg = Pg.PG(0)

    base_port = opt.base_port

    # master
    master = Pgs.PGS(0, 'localhost', base_port, cm.dir)
    pg.join(master, start=True)
    master.smr.wait_role(Smr.SMR.MASTER)
    pgs_ary.append(master);

    # slaves
    id = 1
    for i in range(opt.num_slaves):
      slave = Pgs.PGS(id, 'localhost', base_port + id * 10, cm.dir)
      pg.join(slave, start=True)
      slave.smr.wait_role(Smr.SMR.SLAVE)
      id = id + 1
      pgs_ary.append(slave)

    slotid = 0

    # set option and make clients
    for pgs in pgs_ary:
      try:
	if opt.pin:
	  if opt.pin == "be":
	    pgs.be.pin('arc_ftrace', 'be.ftrace')
	  elif opt.pin == "smr":
	    pgs.smr.pin('arc_ftrace', 'smr.ftrace')
      except:
	traceback.print_exc()
	e = sys.exc_info()[0]
	print e
	pass
      if not opt.no_client:
	client = Client.Client()
	client.slotid = slotid
	client.add(slotid, 'localhost', pgs.base_port + 9)
	client.size(opt.size, slotid)
	client.tps(opt.tps, slotid)
	client_ary.append(client)
	pgs.client = client;
	slotid = slotid + 1

    ##Util.tstop('DEBUG POINT')
    print '=====> Start!'
    for client in client_ary:
      client.start(client.slotid)

    # sleep and checkpoint
    runtime = 0
    ckpt_interval = 0
    while runtime < opt.runtime:
      time.sleep(1)
      runtime = runtime + 1
      ckpt_interval = ckpt_interval + 1
      if ckpt_interval >= opt.ckpt_interval:
	for pgs in pgs_ary:
	  print 'ckeckpoint backend %d' % pgs.base_port
	  pgs.be.ckpt()
	ckpt_interval = 0


    print '=====> Done!'
    while len(client_ary) > 0:
      client = client_ary.pop(0)
      client.stop(client.slotid)

    print '======> Clients Stopped'

    print '========================== RESULT==========================' 
    for pgs in pgs_ary:
      print '>>>>>>>>>>>>>>>>>>>>> PGS(%d)' % pgs.id
      print 'seq:', pgs.smr.getseq_log()
      info = pgs.smr.info('all')
      print 'info:\n', json.dumps(info, indent=4, sort_keys=True)
      if hasattr(pgs, 'client'):
	print pgs.client.stat(pgs.client.slotid)

    for pgs in pgs_ary:
      if hasattr(pgs, 'client'):
	print 'PGS(%d)' % pgs.id, pgs.client.stat(pgs.client.slotid)
    Util.tstop('Type to destroy intermediate results')
  except:
    traceback.print_exc()
    e = sys.exc_info()[0]
    print e
  finally:
    while len(client_ary) > 0:
      client = client_ary.pop(0)
      client.kill()

    while len(pgs_ary) > 0:
      pgs = pgs_ary.pop(0)
      pgs.kill_smr()
      pgs.kill_be()

    Util.tstop('CHECK OUTPUT')
    if cm is not None:
      cm.remove_workspace()
示例#18
0
       print ('traces',traces,type(traces))
       print ('traveltime',traveltime,type(traveltime))


#==================================semblance calculation========================================

    t1 = time.time()
    traces = traces.reshape   (1,nostat*minSampleCount)
    traveltime = traveltime.reshape (1,nostat*dimX*dimY)
    USE_C_CODE = True
    try:
        if USE_C_CODE :
            import Cm
            import CTrig
            start_time = time.time()
            k  = Cm.otest (maxp,nostat,nsamp,ntimes,nstep,dimX,dimY,Gmint,new_frequence,
                          minSampleCount,latv,lonv,traveltime,traces)
            print("--- %s seconds ---" % (time.time() - start_time))
        else :
            start_time = time.time()
            k = otest (maxp,nostat,nsamp,ntimes,nstep,dimX,dimY,Gmint,new_frequence,
                      minSampleCount,latv,lonv,traveltime,traces)                       #hs
            print("--- %s seconds ---" % (time.time() - start_time))
    except:
        print "loaded tttgrid has probably wrong dimensions or stations, delete\
                ttgrid or exchange"

    t2 = time.time()


    partSemb = k
示例#19
0
    def test_be_reconfiguration(self):
        cm = None
        M = None
        C = None
        P = None
        try:
            cm = Cm.CM("test_pgs")
            cm.create_workspace()

            # M(1), S, S
            M = _init_pgs(0, 'localhost', 1900, cm.dir)
            M.smr.role_master(M.id, 1, 0)

            C = _init_pgs(1, 'localhost', 1910, cm.dir)
            C.smr.role_slave(C.id, 'localhost', M.base_port, 0)

            P = _init_pgs(2, 'localhost', 1920, cm.dir)
            P.smr.role_slave(P.id, 'localhost', M.base_port, 0)

            _attach_client(M, 1024, 1000)
            _attach_client(C, 1024, 1000)
            _attach_client(P, 1024, 1000)

            M.client.start(M.client.slotid)
            C.client.start(C.client.slotid)
            P.client.start(P.client.slotid)

            # loop
            loopCount = 100
            for i in range(0, loopCount):
                # M(1) S L
                P.smr.role_lconn()

                # wait some time and check progress
                prev_mseqs = M.smr.getseq_log()
                prev_cseqs = C.smr.getseq_log()
                time.sleep(0.5)
                curr_mseqs = M.smr.getseq_log()
                curr_cseqs = C.smr.getseq_log()
                assert prev_mseqs['max'] < curr_mseqs['max']
                assert prev_cseqs['max'] < curr_cseqs['max']

                # M(1) S S
                ps = P.smr.getseq_log()
                P.smr.role_slave(P.id, 'localhost', M.base_port, ps['max'])

                # L L L
                M.smr.role_lconn()
                C.smr.role_lconn()
                P.smr.role_lconn()

                # alternate pgs1 and pgs2
                tmp = M
                M = C
                C = tmp

                # M(1) S S
                ms = M.smr.getseq_log()
                cs = C.smr.getseq_log()
                ps = P.smr.getseq_log()

                M.smr.role_master(M.id, 1, ms['max'])
                C.smr.role_slave(C.id, 'localhost', M.base_port,
                                 min(ms['max'], cs['max']))
                P.smr.role_slave(P.id, 'localhost', M.base_port,
                                 min(ms['max'], ps['max']))

                # check progress and backend sanity
                mstat = M.client.stat(M.client.slotid)
                cstat = C.client.stat(C.client.slotid)
                pstat = P.client.stat(P.client.slotid)
                print "M ====>" + str(mstat)
                print "C ====>" + str(cstat)
                print "P ====>" + str(pstat)
                mresp = int(mstat['resp'])
                cresp = int(cstat['resp'])
                presp = int(pstat['resp'])

                sleepCount = 0
                while True:
                    time.sleep(0.1)
                    mstat = M.client.stat(M.client.slotid)
                    cstat = C.client.stat(C.client.slotid)
                    pstat = P.client.stat(P.client.slotid)
                    assert int(mstat['reconn']) == 0 and int(
                        cstat['reconn']) == 0 and int(pstat['reconn']) == 0
                    mr = int(mstat['resp'])
                    cr = int(cstat['resp'])
                    pr = int(pstat['resp'])
                    if mr > mresp and cr > cresp and pr > presp:
                        break
                    sleepCount = sleepCount + 1
                    assert sleepCount <= 10

            # stop clients
            M.client.stop(M.client.slotid)
            C.client.stop(C.client.slotid)
            P.client.stop(P.client.slotid)

        finally:
            #Util.tstop('Check output!')
            if M is not None:
                M.kill()
            if C is not None:
                C.kill()
            if P is not None:
                P.kill()
            if cm is not None:
                cm.remove_workspace()
示例#20
0
def doCalc(flag, Config, WaveformDict, FilterMetaData, Gmint, Gmaxt,
           TTTGridMap, Folder, Origin, ntimes, switch, ev, arrayfolder,
           syn_in):
    '''
    method for calculating semblance of one station array
    '''
    Logfile.add('PROCESS %d %s' % (flag, ' Enters Semblance Calculation'))
    Logfile.add('MINT  : %f  MAXT: %f Traveltime' % (Gmint, Gmaxt))

    cfg = ConfigObj(dict=Config)
    cfg_f = FilterCfg(Config)

    timeev = util.str_to_time(ev.time)
    dimX = cfg.dimX()  #('dimx')
    dimY = cfg.dimY()  #('dimy')
    winlen = cfg.winlen()  #('winlen')
    step = cfg.step()  #('step')

    new_frequence = cfg.newFrequency()  #('new_frequence')
    forerun = cfg.Int('forerun')
    duration = cfg.Int('duration')

    nostat = len(WaveformDict)
    traveltimes = {}
    recordstarttime = ''
    minSampleCount = 999999999

    ntimes = int((forerun + duration) / step)
    nsamp = int(winlen * new_frequence)
    nstep = int(step * new_frequence)
    from pyrocko import obspy_compat
    from pyrocko import model
    obspy_compat.plant()

    ############################################################################
    calcStreamMap = WaveformDict

    stations = []
    py_trs = []
    lats = []
    lons = []
    for trace in calcStreamMap.iterkeys():
        py_tr = obspy_compat.to_pyrocko_trace(calcStreamMap[trace])
        py_trs.append(py_tr)
        for il in FilterMetaData:
            if str(il) == str(trace):
                szo = model.Station(lat=float(il.lat),
                                    lon=float(il.lon),
                                    station=il.sta,
                                    network=il.net,
                                    channels=py_tr.channel,
                                    elevation=il.ele,
                                    location=il.loc)
                stations.append(szo)
                lats.append(float(il.lat))
                lons.append(float(il.lon))
    array_center = [num.mean(lats), num.mean(lons)]

    #==================================synthetic BeamForming======================

    if cfg.Bool('synthetic_test') is True:
        store_id = syn_in.store()
        engine = LocalEngine(store_superdirs=[syn_in.store_superdirs()])
        recordstarttimes = []
        for tracex in calcStreamMap.iterkeys():
            recordstarttimes.append(
                calcStreamMap[tracex].stats.starttime.timestamp)
            tr_org = obspy_compat.to_pyrocko_trace(calcStreamMap[tracex])
            tmin = tr_org.tmin

        #tmin= num.min(recordstarttimes)
        targets = []
        sources = []
        for st in stations:
            target = Target(lat=st.lat,
                            lon=st.lon,
                            store_id=store_id,
                            codes=(st.network, st.station, st.location, 'BHZ'),
                            tmin=-6900,
                            tmax=6900,
                            interpolation='multilinear',
                            quantity=cfg.quantity())
            targets.append(target)

        if syn_in.nsources() == 1:
            if syn_in.use_specific_stf() is True:
                stf = syn_in.stf()
                exec(stf)
            else:
                stf = STF()
            if syn_in.source() == 'RectangularSource':
                sources.append(
                    RectangularSource(
                        lat=float(syn_in.lat_0()),
                        lon=float(syn_in.lon_0()),
                        east_shift=float(syn_in.east_shift_0()) * 1000.,
                        north_shift=float(syn_in.north_shift_0()) * 1000.,
                        depth=syn_in.depth_syn_0() * 1000.,
                        strike=syn_in.strike_0(),
                        dip=syn_in.dip_0(),
                        rake=syn_in.rake_0(),
                        width=syn_in.width_0() * 1000.,
                        length=syn_in.length_0() * 1000.,
                        nucleation_x=syn_in.nucleation_x_0(),
                        slip=syn_in.slip_0(),
                        nucleation_y=syn_in.nucleation_y_0(),
                        stf=stf,
                        time=util.str_to_time(syn_in.time_0())))
            if syn_in.source() == 'DCSource':
                sources.append(
                    DCSource(lat=float(syn_in.lat_0()),
                             lon=float(syn_in.lon_0()),
                             east_shift=float(syn_in.east_shift_0()) * 1000.,
                             north_shift=float(syn_in.north_shift_0()) * 1000.,
                             depth=syn_in.depth_syn_0() * 1000.,
                             strike=syn_in.strike_0(),
                             dip=syn_in.dip_0(),
                             rake=syn_in.rake_0(),
                             stf=stf,
                             time=util.str_to_time(syn_in.time_0()),
                             magnitude=syn_in.magnitude_0()))

        else:
            for i in range(syn_in.nsources()):
                if syn_in.use_specific_stf() is True:
                    stf = syn_in.stf()
                    exec(stf)

                else:
                    stf = STF()
                if syn_in.source() == 'RectangularSource':
                    sources.append(
                        RectangularSource(
                            lat=float(syn_in.lat_1(i)),
                            lon=float(syn_in.lon_1(i)),
                            east_shift=float(syn_in.east_shift_1(i)) * 1000.,
                            north_shift=float(syn_in.north_shift_1(i)) * 1000.,
                            depth=syn_in.depth_syn_1(i) * 1000.,
                            strike=syn_in.strike_1(i),
                            dip=syn_in.dip_1(i),
                            rake=syn_in.rake_1(i),
                            width=syn_in.width_1(i) * 1000.,
                            length=syn_in.length_1(i) * 1000.,
                            nucleation_x=syn_in.nucleation_x_1(i),
                            slip=syn_in.slip_1(i),
                            nucleation_y=syn_in.nucleation_y_1(i),
                            stf=stf,
                            time=util.str_to_time(syn_in.time_1(i))))

                if syn_in.source() == 'DCSource':
                    sources.append(
                        DCSource(
                            lat=float(syn_in.lat_1(i)),
                            lon=float(syn_in.lon_1(i)),
                            east_shift=float(syn_in.east_shift_1(i)) * 1000.,
                            north_shift=float(syn_in.north_shift_1(i)) * 1000.,
                            depth=syn_in.depth_syn_1(i) * 1000.,
                            strike=syn_in.strike_1(i),
                            dip=syn_in.dip_1(i),
                            rake=syn_in.rake_1(i),
                            stf=stf,
                            time=util.str_to_time(syn_in.time_1(i)),
                            magnitude=syn_in.magnitude_1(i)))
            #source = CombiSource(subsources=sources)
        synthetic_traces = []
        for source in sources:
            response = engine.process(source, targets)
            synthetic_traces_source = response.pyrocko_traces()
            if not synthetic_traces:
                synthetic_traces = synthetic_traces_source
            else:
                for trsource, tr in zip(synthetic_traces_source,
                                        synthetic_traces):
                    tr.add(trsource)
            from pyrocko import trace as trld
            #trld.snuffle(synthetic_traces)
        timeev = util.str_to_time(syn_in.time_0())
        if cfg.Bool('synthetic_test_add_noise') is True:
            from noise_addition import add_noise
            trs_orgs = []
            calcStreamMapsyn = calcStreamMap.copy()
            #from pyrocko import trace
            for tracex in calcStreamMapsyn.iterkeys():
                for trl in synthetic_traces:
                    if str(trl.name()[4:12]) == str(tracex[4:]) or str(
                            trl.name()[3:13]) == str(tracex[3:]) or str(
                                trl.name()[3:11]) == str(tracex[3:]) or str(
                                    trl.name()[3:14]) == str(tracex[3:]):
                        tr_org = obspy_compat.to_pyrocko_trace(
                            calcStreamMapsyn[tracex])
                        tr_org.downsample_to(2.0)
                        trs_orgs.append(tr_org)
            store_id = syn_in.store()
            engine = LocalEngine(store_superdirs=[syn_in.store_superdirs()])
            synthetic_traces = add_noise(trs_orgs,
                                         engine,
                                         source.pyrocko_event(),
                                         stations,
                                         store_id,
                                         phase_def='P')
        trs_org = []
        trs_orgs = []
        from pyrocko import trace
        fobj = os.path.join(arrayfolder, 'shift.dat')
        calcStreamMapsyn = calcStreamMap.copy()
        for tracex in calcStreamMapsyn.iterkeys():
            for trl in synthetic_traces:
                if str(trl.name()[4:12]) == str(tracex[4:]) or str(
                        trl.name()[3:13]) == str(tracex[3:]) or str(
                            trl.name()[3:11]) == str(tracex[3:]) or str(
                                trl.name()[3:14]) == str(tracex[3:]):
                    mod = trl
                    recordstarttime = calcStreamMapsyn[
                        tracex].stats.starttime.timestamp
                    recordendtime = calcStreamMapsyn[
                        tracex].stats.endtime.timestamp
                    tr_org = obspy_compat.to_pyrocko_trace(
                        calcStreamMapsyn[tracex])
                    if switch == 0:
                        tr_org.bandpass(4, cfg_f.flo(), cfg_f.fhi())
                    elif switch == 1:
                        tr_org.bandpass(4, cfg_f.flo2(), cfg_f.fhi2())
                    trs_orgs.append(tr_org)
                    tr_org_add = mod.chop(recordstarttime,
                                          recordendtime,
                                          inplace=False)
                    synthetic_obs_tr = obspy_compat.to_obspy_trace(tr_org_add)
                    calcStreamMapsyn[tracex] = synthetic_obs_tr
                    trs_org.append(tr_org_add)
        calcStreamMap = calcStreamMapsyn

    if cfg.Bool('shift_by_phase_pws') == True:
        calcStreamMapshifted = calcStreamMap.copy()
        from obspy.core import stream
        stream = stream.Stream()
        for trace in calcStreamMapshifted.iterkeys():
            stream.append(calcStreamMapshifted[trace])
        pws_stack = PWS_stack([stream], weight=2, normalize=True)
        for tr in pws_stack:
            for trace in calcStreamMapshifted.iterkeys():
                calcStreamMapshifted[trace] = tr
        calcStreamMap = calcStreamMapshifted

    if cfg.Bool('shift_by_phase_cc') is True:
        from stacking import align_traces
        calcStreamMapshifted = calcStreamMap.copy()
        list_tr = []
        for trace in calcStreamMapshifted.iterkeys():
            tr_org = calcStreamMapshifted[trace]
            list_tr.append(tr_org)
        shifts, ccs = align_traces(list_tr, 10, master=False)
        for shift in shifts:
            for trace in calcStreamMapshifted.iterkeys():
                tr_org = obspy_compat.to_pyrocko_trace(
                    calcStreamMapshifted[trace])
                tr_org.shift(shift)
                shifted = obspy_compat.to_obspy_trace(tr_org)
                calcStreamMapshifted[trace] = shifted
        calcStreamMap = calcStreamMapshifted

    if cfg.Bool('shift_by_phase_onset') is True:
        pjoin = os.path.join
        timeev = util.str_to_time(ev.time)
        trs_orgs = []
        calcStreamMapshifted = calcStreamMap.copy()
        for trace in calcStreamMapshifted.iterkeys():
            tr_org = obspy_compat.to_pyrocko_trace(calcStreamMapshifted[trace])
            trs_orgs.append(tr_org)

        timing = CakeTiming(
            phase_selection='first(p|P|PP|P(cmb)P(icb)P(icb)p(cmb)p)-20',
            fallback_time=100.)
        traces = trs_orgs

        event = model.Event(lat=float(ev.lat),
                            lon=float(ev.lon),
                            depth=ev.depth * 1000.,
                            time=timeev)
        directory = arrayfolder
        bf = BeamForming(stations, traces, normalize=True)
        shifted_traces = bf.process(event=event,
                                    timing=timing,
                                    fn_dump_center=pjoin(
                                        directory, 'array_center.pf'),
                                    fn_beam=pjoin(directory, 'beam.mseed'))
        i = 0
        store_id = syn_in.store()
        engine = LocalEngine(store_superdirs=[syn_in.store_superdirs()])
        for tracex in calcStreamMapshifted.iterkeys():
            for trl in shifted_traces:
                if str(trl.name()[4:12]) == str(tracex[4:]) or str(
                        trl.name()[3:13]) == str(tracex[3:]) or str(
                            trl.name()[3:11]) == str(tracex[3:]) or str(
                                trl.name()[3:14]) == str(tracex[3:]):
                    mod = trl
                    recordstarttime = calcStreamMapshifted[
                        tracex].stats.starttime.timestamp
                    recordendtime = calcStreamMapshifted[
                        tracex].stats.endtime.timestamp
                    tr_org = obspy_compat.to_pyrocko_trace(
                        calcStreamMapshifted[tracex])
                    tr_org_add = mod.chop(recordstarttime,
                                          recordendtime,
                                          inplace=False)
                    shifted_obs_tr = obspy_compat.to_obspy_trace(tr_org_add)
                    calcStreamMapshifted[tracex] = shifted_obs_tr
        calcStreamMap = calcStreamMapshifted

    weight = 1.
    if cfg.Bool('weight_by_noise') is True:
        from noise_analyser import analyse
        pjoin = os.path.join
        timeev = util.str_to_time(ev.time)
        trs_orgs = []
        calcStreamMapshifted = calcStreamMap.copy()
        for trace in calcStreamMapshifted.iterkeys():
            tr_org = obspy_compat.to_pyrocko_trace(calcStreamMapshifted[trace])
            trs_orgs.append(tr_org)

        timing = CakeTiming(
            phase_selection='first(p|P|PP|P(cmb)P(icb)P(icb)p(cmb)p)-20',
            fallback_time=100.)
        traces = trs_orgs
        event = model.Event(lat=float(ev.lat),
                            lon=float(ev.lon),
                            depth=ev.depth * 1000.,
                            time=timeev)
        directory = arrayfolder
        bf = BeamForming(stations, traces, normalize=True)
        shifted_traces = bf.process(event=event,
                                    timing=timing,
                                    fn_dump_center=pjoin(
                                        directory, 'array_center.pf'),
                                    fn_beam=pjoin(directory, 'beam.mseed'))
        i = 0
        store_id = syn_in.store()
        engine = LocalEngine(store_superdirs=[syn_in.store_superdirs()])
        weight = analyse(shifted_traces,
                         engine,
                         event,
                         stations,
                         100.,
                         store_id,
                         nwindows=1,
                         check_events=True,
                         phase_def='P')

    if cfg.Bool('array_response') is True:
        from obspy.signal import array_analysis
        from obspy.core import stream
        ntimesr = int((forerun + duration) / step)
        nsampr = int(winlen)
        nstepr = int(step)
        sll_x = -3.0
        slm_x = 3.0
        sll_y = -3.0
        slm_y = 3.0
        sl_s = 0.03,
        # sliding window properties

        # frequency properties
        frqlow = 1.0,
        frqhigh = 8.0
        prewhiten = 0
        # restrict output
        semb_thres = -1e9
        vel_thres = -1e9
        stime = stime
        etime = etime
        stream_arr = stream.Stream()
        for trace in calcStreamMapshifted.iterkeys():
            stream_arr.append(calcStreamMapshifted[trace])
        results = array_analysis.array_processing(stream_arr, nsamp, nstep,\
                                                  sll_x, slm_x, sll_y, slm_y,\
                                                   sl_s, semb_thres, vel_thres, \
                                                   frqlow, frqhigh, stime, \
                                                   etime, prewhiten)
        timestemp = results[0]
        relative_relpow = results[1]
        absolute_relpow = results[2]

    for trace in calcStreamMap.iterkeys():
        recordstarttime = calcStreamMap[trace].stats.starttime
        d = calcStreamMap[trace].stats.starttime
        d = d.timestamp

        if calcStreamMap[trace].stats.npts < minSampleCount:
            minSampleCount = calcStreamMap[trace].stats.npts

    ###########################################################################

    traces = num.ndarray(shape=(len(calcStreamMap), minSampleCount),
                         dtype=float)
    traveltime = num.ndarray(shape=(len(calcStreamMap), dimX * dimY),
                             dtype=float)

    latv = num.ndarray(dimX * dimY, dtype=float)
    lonv = num.ndarray(dimX * dimY, dtype=float)
    ###########################################################################

    c = 0
    streamCounter = 0

    for key in calcStreamMap.iterkeys():
        streamID = key
        c2 = 0

        for o in calcStreamMap[key]:
            if c2 < minSampleCount:
                traces[c][c2] = o

                c2 += 1

        for key in TTTGridMap.iterkeys():

            if streamID == key:
                traveltimes[streamCounter] = TTTGridMap[key]
            else:
                "NEIN", streamID, key

        if not streamCounter in traveltimes:
            continue  #hs : thread crashed before

        g = traveltimes[streamCounter]
        dimZ = g.dimZ
        mint = g.mint
        gridElem = g.GridArray

        for x in range(dimX):
            for y in range(dimY):
                elem = gridElem[x, y]

                traveltime[c][x * dimY + y] = elem.tt
                latv[x * dimY + y] = elem.lat
                lonv[x * dimY + y] = elem.lon
        #endfor

        c += 1
        streamCounter += 1

    #endfor

    ################ CALCULATE PARAMETER FOR SEMBLANCE CALCULATION ########
    nsamp = winlen * new_frequence

    nstep = step * new_frequence
    migpoints = dimX * dimY

    dimZ = 0
    maxp = int(Config['ncore'])

    Logfile.add('PROCESS %d  NTIMES: %d' % (flag, ntimes))

    if False:
        print('nostat ', nostat, type(nostat))
        print('nsamp ', nsamp, type(nsamp))
        print('ntimes ', ntimes, type(ntimes))
        print('nstep ', nstep, type(nstep))
        print('dimX ', dimX, type(dimX))
        print('dimY ', dimY, type(dimY))
        print('mint ', Gmint, type(mint))
        print('new_freq ', new_frequence, type(new_frequence))
        print('minSampleCount ', minSampleCount, type(minSampleCount))
        print('latv ', latv, type(latv))
        print('traces', traces, type(traces))

#===================compressed sensing=================================
    try:
        cs = cfg.cs()
    except:
        cs = 0
    if cs == 1:
        csmaxvaluev = num.ndarray(ntimes, dtype=float)
        csmaxlatv = num.ndarray(ntimes, dtype=float)
        csmaxlonv = num.ndarray(ntimes, dtype=float)
        folder = Folder['semb']
        fobjcsmax = open(os.path.join(folder, 'csmax_%s.txt' % (switch)), 'w')
        traveltimes = traveltime.reshape(1, nostat * dimX * dimY)
        traveltime2 = toMatrix(traveltimes, dimX * dimY)  # for relstart
        traveltime = traveltime.reshape(dimX * dimY, nostat)
        import matplotlib as mpl
        import scipy.optimize as spopt
        import scipy.fftpack as spfft
        import scipy.ndimage as spimg
        import cvxpy as cvx
        import matplotlib.pyplot as plt
        A = spfft.idct(traveltime, norm='ortho', axis=0)
        n = (nostat * dimX * dimY)
        vx = cvx.Variable(dimX * dimY)
        res = cvx.Variable(1)
        objective = cvx.Minimize(cvx.norm(res, 1))
        back2 = num.zeros([dimX, dimY])
        l = int(nsamp)
        fobj = open(
            os.path.join(folder,
                         '%s-%s_%03d.cs' % (switch, Origin['depth'], l)), 'w')
        for i in range(ntimes):
            ydata = []
            try:
                for tr in traces:
                    relstart = int((dimX * dimY - mint) * new_frequence +
                                   0.5) + i * nstep
                    tr = spfft.idct(tr[relstart + i:relstart + i +
                                       dimX * dimY],
                                    norm='ortho',
                                    axis=0)

                    ydata.append(tr)
                    ydata = num.asarray(ydata)
                    ydata = ydata.reshape(dimX * dimY, nostat)

                    constraints = [
                        res == cvx.sum_entries(0 + num.sum([
                            ydata[:, x] - A[:, x] * vx for x in range(nostat)
                        ]))
                    ]

                    prob = cvx.Problem(objective, constraints)
                    result = prob.solve(verbose=False, max_iters=200)

                    x = num.array(vx.value)
                    x = num.squeeze(x)
                    back1 = x.reshape(dimX, dimY)
                    sig = spfft.idct(x, norm='ortho', axis=0)
                    back2 = back2 + back1
                    xs = num.array(res.value)
                    xs = num.squeeze(xs)
                    max_cs = num.max(back1)
                    idx = num.where(back1 == back1.max())
                    csmaxvaluev[i] = max_cs
                    csmaxlatv[i] = latv[idx[0]]
                    csmaxlonv[i] = lonv[idx[1]]
                    fobj.write('%.5f %.5f %.20f\n' %
                               (latv[idx[0]], lonv[idx[1]], max_cs))
                    fobjcsmax.write('%.5f %.5f %.20f\n' %
                                    (latv[idx[0]], lonv[idx[1]], max_cs))
                fobj.close()
                fobjcsmax.close()

            except:
                pass

#==================================semblance calculation========================================

    t1 = time.time()
    traces = traces.reshape(1, nostat * minSampleCount)

    traveltimes = traveltime.reshape(1, nostat * dimX * dimY)
    USE_C_CODE = False
    #try:
    if USE_C_CODE:
        import Cm
        import CTrig
        start_time = time.time()
        k = Cm.otest(maxp, nostat, nsamp, ntimes, nstep, dimX, dimY, Gmint,
                     new_frequence, minSampleCount, latv, lonv, traveltimes,
                     traces)
        print("--- %s seconds ---" % (time.time() - start_time))
    else:
        start_time = time.time()
        ntimes = int((forerun + duration) / step)
        nsamp = int(winlen)
        nstep = int(step)
        Gmint = cfg.Int('forerun')
        k = otest(maxp, nostat, nsamp, ntimes, nstep, dimX, dimY, Gmint,
                  new_frequence, minSampleCount, latv, lonv, traveltimes,
                  traces, calcStreamMap, timeev)
        print("--- %s seconds ---" % (time.time() - start_time))
    #except ValueError:
    #        k  = Cm.otest(maxp,nostat,nsamp,ntimes,nstep,dimX,dimY,Gmint,new_frequence,
    #                      minSampleCount,latv,lonv,traveltimes,traces)
    #    print "loaded tttgrid has probably wrong dimensions or stations,\
    #                delete ttgrid or exchange is recommended"

    t2 = time.time()

    Logfile.add('%s took %0.3f s' % ('CALC:', (t2 - t1)))

    partSemb = k
    partSemb = partSemb.reshape(ntimes, migpoints)

    return partSemb, weight, array_center
示例#21
0
def  doCalc_syn (flag,Config,WaveformDict,FilterMetaData,Gmint,Gmaxt,TTTGridMap,
                Folder,Origin, ntimes, switch, ev,arrayfolder, syn_in, parameter):
    '''
    method for calculating semblance of one station array
    '''
    Logfile.add ('PROCESS %d %s' % (flag,' Enters Semblance Calculation') )
    Logfile.add ('MINT  : %f  MAXT: %f Traveltime' % (Gmint,Gmaxt))

    cfg = ConfigObj (dict=Config)

    dimX   = cfg.dimX()         # ('dimx')
    dimY   = cfg.dimY()         # ('dimy')
    winlen = cfg.winlen ()      # ('winlen')
    step   = cfg.step()         # ('step')

    new_frequence   = cfg.newFrequency()          #('new_frequence')
    forerun= cfg.Int('forerun')
    duration= cfg.Int('duration')
    gridspacing = cfg.Float('gridspacing')

    nostat = len (WaveformDict)
    traveltimes = {}
    recordstarttime = ''
    minSampleCount  = 999999999

    if cfg.UInt ('forerun')>0:
        ntimes = int ((cfg.UInt ('forerun') + cfg.UInt ('duration') ) / cfg.UInt ('step') )
    else:
        ntimes = int ((cfg.UInt ('duration') ) / cfg.UInt ('step') )
    nsamp  = int (winlen * new_frequence)
    nstep  = int (step   * new_frequence)
    from pyrocko import obspy_compat
    from pyrocko import orthodrome, model
    obspy_compat.plant()

    ############################################################################
    calcStreamMap = WaveformDict

    stations = []
    py_trs = []
    for trace in calcStreamMap.keys():
        py_tr = obspy_compat.to_pyrocko_trace(calcStreamMap[trace])
        py_trs.append(py_tr)
        for il in FilterMetaData:
            if str(il) == str(trace):
                        szo = model.Station(lat=il.lat, lon=il.lon,
                                            station=il.sta, network=il.net,
                                            channels=py_tr.channel,
                                            elevation=il.ele, location=il.loc)
                        stations.append(szo) #right number of stations?

    store_id = syn_in.store()
    engine = LocalEngine(store_superdirs=[syn_in.store_superdirs()])

    targets = []
    for st in stations:
        target = Target(
                lat=st.lat,
                lon=st.lon,
                store_id=store_id,
                codes=(st.network, st.station, st.location, 'BHZ'),
                tmin=-1900,
                tmax=3900,
                interpolation='multilinear',
                quantity=cfg.quantity())
        targets.append(target)

    if syn_in.nsources() == 1:
        if syn_in.use_specific_stf() is True:
            stf = syn_in.stf()
            exec(stf)
        else:
            stf = STF()
        if syn_in.source() == 'RectangularSource':
                source = RectangularSource(
                    lat=float(syn_in.lat_0()),
                    lon=float(syn_in.lon_0()),
                    depth=syn_in.depth_syn_0()*1000.,
                    strike=syn_in.strike_0(),
                    dip=syn_in.dip_0(),
                    rake=syn_in.rake_0(),
                    width=syn_in.width_0()*1000.,
                    length=syn_in.length_0()*1000.,
                    nucleation_x=syn_in.nucleation_x_0(),
                    slip=syn_in.slip_0(),
                    nucleation_y=syn_in.nucleation_y_0(),
                    stf=stf,
                    time=util.str_to_time(syn_in.time_0()))
        if syn_in.source() == 'DCSource':
                source = DCSource(
                    lat=float(syn_in.lat_0()),
                    lon=float(syn_in.lon_0()),
                    depth=syn_in.depth_syn_0()*1000.,
                    strike=syn_in.strike_0(),
                    dip=syn_in.dip_0(),
                    rake=syn_in.rake_0(),
                    stf=stf,
                    time=util.str_to_time(syn_in.time_0()),
                    magnitude=syn_in.magnitude_0())

    else:
        sources = []
        for i in range(syn_in.nsources()):
            if syn_in.use_specific_stf() is True:
                stf = syn_in.stf()
                exec(stf)

            else:
                stf = STF()
            if syn_in.source() == 'RectangularSource':
                    sources.append(RectangularSource(
                        lat=float(syn_in.lat_1(i)),
                        lon=float(syn_in.lon_1(i)),
                        depth=syn_in.depth_syn_1(i)*1000.,
                        strike=syn_in.strike_1(i),
                        dip=syn_in.dip_1(i),
                        rake=syn_in.rake_1(i),
                        width=syn_in.width_1(i)*1000.,
                        length=syn_in.length_1(i)*1000.,
                        nucleation_x=syn_in.nucleation_x_1(i),
                        slip=syn_in.slip_1(i),
                        nucleation_y=syn_in.nucleation_y_1(i),
                        stf=stf,
                        time=util.str_to_time(syn_in.time_1(i))))

            if syn_in.source() == 'DCSource':
                    sources.append(DCSource(
                        lat=float(syn_in.lat_1(i)),
                        lon=float(syn_in.lon_1(i)),
                        depth=syn_in.depth_1(i)*1000.,
                        strike=syn_in.strike_1(i),
                        dip=syn_in.dip_1(i),
                        rake=syn_in.rake_1(i),
                        stf=stf,
                        time=util.str_to_time(syn_in.time_1(i)),
                        magnitude=syn_in.magnitude_1(i)))
        source = CombiSource(subsources=sources)
    response = engine.process(source, targets)

    synthetic_traces = response.pyrocko_traces()
    if cfg.Bool('synthetic_test_add_noise') is True:
        from noise_addition import add_noise
        trs_orgs = []
        calcStreamMapsyn = calcStreamMap.copy()
        #from pyrocko import trace
        for tracex in calcStreamMapsyn.keys():
                for trl in synthetic_traces:
                    if str(trl.name()[4:12]) == str(tracex[4:]):
                        tr_org = obspy_compat.to_pyrocko_trace(calcStreamMapsyn[tracex])
                        tr_org.downsample_to(2.0)
                        trs_orgs.append(tr_org)
        store_id = syn_in.store()
        engine = LocalEngine(store_superdirs=[syn_in.store_superdirs()])
        synthetic_traces = add_noise(trs_orgs, engine, source.pyrocko_event(),
                                     stations,
                                     store_id, phase_def='P')
    trs_org = []
    trs_orgs = []
    fobj = os.path.join(arrayfolder, 'shift.dat')
    xy = num.loadtxt(fobj, usecols=1, delimiter=',')
    calcStreamMapsyn = calcStreamMap.copy()
    #from pyrocko import trace
    for tracex in calcStreamMapsyn.keys():
            for trl in synthetic_traces:
                if str(trl.name()[4:12])== str(tracex[4:]):
                    mod = trl

                    recordstarttime = calcStreamMapsyn[tracex].stats.starttime.timestamp
                    recordendtime = calcStreamMapsyn[tracex].stats.endtime.timestamp
                    tr_org = obspy_compat.to_pyrocko_trace(calcStreamMapsyn[tracex])
                    trs_orgs.append(tr_org)

                    tr_org_add = mod.chop(recordstarttime, recordendtime, inplace=False)
                    synthetic_obs_tr = obspy_compat.to_obspy_trace(tr_org_add)
                    calcStreamMapsyn[tracex] = synthetic_obs_tr
                    trs_org.append(tr_org_add)
    calcStreamMap = calcStreamMapsyn

    if cfg.Bool('shift_by_phase_pws') == True:
        calcStreamMapshifted= calcStreamMap.copy()
        from obspy.core import stream
        stream = stream.Stream()
        for trace in calcStreamMapshifted.keys():
            stream.append(calcStreamMapshifted[trace])
        pws_stack = PWS_stack([stream], weight=2, normalize=True)
        for tr in pws_stack:
            for trace in calcStreamMapshifted.keys():
                    calcStreamMapshifted[trace]=tr
        calcStreamMap = calcStreamMapshifted


    if cfg.Bool('shift_by_phase_onset') == True:
        pjoin = os.path.join
        timeev = util.str_to_time(ev.time)
        trs_orgs= []
        calcStreamMapshifted= calcStreamMap.copy()
        for trace in calcStreamMapshifted.keys():
                tr_org = obspy_compat.to_pyrocko_trace(calcStreamMapshifted[trace])
                trs_orgs.append(tr_org)

        timing = CakeTiming(
           phase_selection='first(p|P|PP|P(cmb)P(icb)P(icb)p(cmb)p)-20',
           fallback_time=100.)
        traces = trs_orgs

        event = model.Event(lat=float(ev.lat), lon=float(ev.lon), depth=ev.depth*1000., time=timeev)
        directory = arrayfolder
        bf = BeamForming(stations, traces, normalize=True)
        shifted_traces = bf.process(event=event,
                  timing=timing,
                  fn_dump_center=pjoin(directory, 'array_center.pf'),
                  fn_beam=pjoin(directory, 'beam.mseed'))
        i = 0
        store_id = syn_in.store()
        engine = LocalEngine(store_superdirs=[syn_in.store_superdirs()])
        for trace in calcStreamMapshifted.keys():
            recordstarttime = calcStreamMapshifted[trace].stats.starttime.timestamp
            recordendtime = calcStreamMapshifted[trace].stats.endtime.timestamp
            mod = shifted_traces[i]
            extracted = mod.chop(recordstarttime, recordendtime, inplace=False)
            shifted_obs_tr = obspy_compat.to_obspy_trace(extracted)
            calcStreamMapshifted[trace]=shifted_obs_tr
            i = i+1

        calcStreamMap = calcStreamMapshifted


    weight = 0.
    if cfg.Bool('weight_by_noise') == True:
        from noise_analyser import analyse
        pjoin = os.path.join
        timeev = util.str_to_time(ev.time)
        trs_orgs= []
        calcStreamMapshifted= calcStreamMap.copy()
        for trace in calcStreamMapshifted.keys():
                tr_org = obspy_compat.to_pyrocko_trace(calcStreamMapshifted[trace])
                trs_orgs.append(tr_org)

        timing = CakeTiming(
           phase_selection='first(p|P|PP|P(cmb)P(icb)P(icb)p(cmb)p)-20',
           fallback_time=100.)
        traces = trs_orgs
        event = model.Event(lat=float(ev.lat), lon=float(ev.lon), depth=ev.depth*1000., time=timeev)
        directory = arrayfolder
        bf = BeamForming(stations, traces, normalize=True)
        shifted_traces = bf.process(event=event,
                  timing=timing,
                  fn_dump_center=pjoin(directory, 'array_center.pf'),
                  fn_beam=pjoin(directory, 'beam.mseed'))
        i = 0
        store_id = syn_in.store()
        engine = LocalEngine(store_superdirs=[syn_in.store_superdirs()])
        weight = analyse(shifted_traces, engine, event, stations,
         100., store_id, nwindows=1,
         check_events=True, phase_def='P')

    for trace in calcStreamMap.keys():
        recordstarttime = calcStreamMap[trace].stats.starttime
        d = calcStreamMap[trace].stats.starttime
        d = d.timestamp

        if calcStreamMap[trace].stats.npts < minSampleCount:
            minSampleCount = calcStreamMap[trace].stats.npts

    ############################################################################
    traces = num.ndarray (shape=(len(calcStreamMap), minSampleCount), dtype=float)
    traveltime = num.ndarray (shape=(len(calcStreamMap), dimX*dimY), dtype=float)
    latv   = num.ndarray (dimX*dimY, dtype=float)
    lonv   = num.ndarray (dimX*dimY, dtype=float)
    ############################################################################


    c=0
    streamCounter = 0

    for key in calcStreamMap.keys():
        streamID = key
        c2   = 0

        for o in calcStreamMap[key]:
            if c2 < minSampleCount:
                traces[c][c2] = o

                c2 += 1


        for key in TTTGridMap.keys():

            if streamID == key:
                traveltimes[streamCounter] = TTTGridMap[key]
            else:
                "NEIN", streamID, key


        if not streamCounter in traveltimes :
           continue                              #hs : thread crashed before

        g = traveltimes[streamCounter]
        dimZ  = g.dimZ
        mint  = g.mint
        maxt  = g.maxt
        Latul = g.Latul
        Lonul = g.Lonul
        Lator = g.Lator
        Lonor = g.Lonor

        gridElem = g.GridArray

        for x in range(dimX):
            for y in range(dimY):
                elem = gridElem[x, y]

                traveltime [c][x * dimY + y] = elem.tt
                latv [x * dimY + y] = elem.lat
                lonv [x * dimY + y] = elem.lon
        #endfor

        c += 1
        streamCounter += 1

    #endfor


    ############################## CALCULATE PARAMETER FOR SEMBLANCE CALCULATION ##################
    nsamp = winlen * new_frequence

    nstep = int (step*new_frequence)
    migpoints = dimX * dimY

    dimZ = 0
    new_frequence = cfg.newFrequency ()              # ['new_frequence']
    maxp = int (Config['ncore'])


    Logfile.add ('PROCESS %d  NTIMES: %d' % (flag,ntimes))

    if False :
       print ('nostat ',nostat,type(nostat))
       print ('nsamp ',nsamp,type(nsamp))
       print ('ntimes ',ntimes,type(ntimes))
       print ('nstep ',nstep,type(nstep))
       print ('dimX ',dimX,type(dimX))
       print ('dimY ',dimY,type(dimY))
       print ('mint ',Gmint,type(mint))
       print ('new_freq ',new_frequence,type(new_frequence))
       print ('minSampleCount ',minSampleCount,type(minSampleCount))
       print ('latv ',latv,type(latv))
       print ('traces',traces,type(traces))
       print ('traveltime',traveltime,type(traveltime))


#==================================semblance calculation========================================

    t1 = time.time()
    traces = traces.reshape   (1,nostat*minSampleCount)
    traveltime = traveltime.reshape (1,nostat*dimX*dimY)
    USE_C_CODE = True
    try:
        if USE_C_CODE :
            import Cm
            import CTrig
            start_time = time.time()
            k  = Cm.otest (maxp,nostat,nsamp,ntimes,nstep,dimX,dimY,Gmint,new_frequence,
                          minSampleCount,latv,lonv,traveltime,traces)
            print("--- %s seconds ---" % (time.time() - start_time))
        else :
            start_time = time.time()
            k = otest (maxp,nostat,nsamp,ntimes,nstep,dimX,dimY,Gmint,new_frequence,
                      minSampleCount,latv,lonv,traveltime,traces)                       #hs
            print("--- %s seconds ---" % (time.time() - start_time))
    except:
        print("loaded tttgrid has probably wrong dimensions or stations, delete\
                ttgrid or exchange")

    t2 = time.time()


    partSemb = k

    partSemb_syn  = partSemb.reshape (ntimes,migpoints)


    return partSemb_syn
示例#22
0
def doCalc(flag, Config, WaveformDict, FilterMetaData, Gmint, Gmaxt,
           TTTGridMap, Folder, Origin, ntimes, switch, ev, arrayfolder,
           syn_in):
    '''
    method for calculating semblance of one station array
    '''
    Logfile.add('PROCESS %d %s' % (flag, ' Enters Semblance Calculation'))
    Logfile.add('MINT  : %f  MAXT: %f Traveltime' % (Gmint, Gmaxt))

    cfg = ConfigObj(dict=Config)

    dimX = cfg.dimX()  # ('dimx')
    dimY = cfg.dimY()  # ('dimy')
    winlen = cfg.winlen()  # ('winlen')
    step = cfg.step()  # ('step')

    new_frequence = cfg.newFrequency()  #('new_frequence')
    forerun = cfg.Int('forerun')
    duration = cfg.Int('duration')
    gridspacing = cfg.Float('gridspacing')

    nostat = len(WaveformDict)
    traveltimes = {}
    recordstarttime = ''
    minSampleCount = 999999999

    ntimes = int((forerun + duration) / step)
    nsamp = int(winlen * new_frequence)
    nstep = int(step * new_frequence)
    from pyrocko import obspy_compat
    from pyrocko import orthodrome, model
    obspy_compat.plant()

    ############################################################################
    calcStreamMap = WaveformDict

    stations = []
    py_trs = []
    for trace in calcStreamMap.iterkeys():
        py_tr = obspy_compat.to_pyrocko_trace(calcStreamMap[trace])
        py_trs.append(py_tr)
        for il in FilterMetaData:
            if str(il) == str(trace):
                szo = model.Station(lat=il.lat,
                                    lon=il.lon,
                                    station=il.sta,
                                    network=il.net,
                                    channels=py_tr.channel,
                                    elevation=il.ele,
                                    location=il.loc)
                stations.append(szo)  #right number of stations?

#==================================synthetic BeamForming=======================================

    if cfg.Bool('shift_by_phase_pws') == True:
        calcStreamMapshifted = calcStreamMap.copy()
        from obspy.core import stream
        stream = stream.Stream()
        for trace in calcStreamMapshifted.iterkeys():
            stream.append(calcStreamMapshifted[trace])
        pws_stack = PWS_stack([stream], weight=2, normalize=True)
        for tr in pws_stack:
            for trace in calcStreamMapshifted.iterkeys():
                calcStreamMapshifted[trace] = tr
        calcStreamMap = calcStreamMapshifted

    if cfg.Bool('shift_by_phase_onset') == True:
        pjoin = os.path.join
        timeev = util.str_to_time(ev.time)
        trs_orgs = []
        calcStreamMapshifted = calcStreamMap.copy()
        for trace in calcStreamMapshifted.iterkeys():
            tr_org = obspy_compat.to_pyrocko_trace(calcStreamMapshifted[trace])
            trs_orgs.append(tr_org)

        timing = CakeTiming(
            phase_selection='first(p|P|PP|P(cmb)P(icb)P(icb)p(cmb)p)-20',
            fallback_time=100.)
        traces = trs_orgs

        event = model.Event(lat=float(ev.lat),
                            lon=float(ev.lon),
                            depth=ev.depth * 1000.,
                            time=timeev)
        directory = arrayfolder
        bf = BeamForming(stations, traces, normalize=True)
        shifted_traces = bf.process(event=event,
                                    timing=timing,
                                    fn_dump_center=pjoin(
                                        directory, 'array_center.pf'),
                                    fn_beam=pjoin(directory, 'beam.mseed'))
        i = 0
        store_id = syn_in.store()
        engine = LocalEngine(store_superdirs=[syn_in.store_superdirs()])
        for trace in calcStreamMapshifted.iterkeys():
            recordstarttime = calcStreamMapshifted[
                trace].stats.starttime.timestamp
            recordendtime = calcStreamMapshifted[trace].stats.endtime.timestamp
            mod = shifted_traces[i]
            extracted = mod.chop(recordstarttime, recordendtime, inplace=False)
            shifted_obs_tr = obspy_compat.to_obspy_trace(extracted)
            calcStreamMapshifted[trace] = shifted_obs_tr
            i = i + 1

        calcStreamMap = calcStreamMapshifted

    weight = 0.
    if cfg.Bool('weight_by_noise') == True:
        from noise_analyser import analyse
        pjoin = os.path.join
        timeev = util.str_to_time(ev.time)
        trs_orgs = []
        calcStreamMapshifted = calcStreamMap.copy()
        for trace in calcStreamMapshifted.iterkeys():
            tr_org = obspy_compat.to_pyrocko_trace(calcStreamMapshifted[trace])
            trs_orgs.append(tr_org)

        timing = CakeTiming(
            phase_selection='first(p|P|PP|P(cmb)P(icb)P(icb)p(cmb)p)-20',
            fallback_time=100.)
        traces = trs_orgs
        event = model.Event(lat=float(ev.lat),
                            lon=float(ev.lon),
                            depth=ev.depth * 1000.,
                            time=timeev)
        directory = arrayfolder
        bf = BeamForming(stations, traces, normalize=True)
        shifted_traces = bf.process(event=event,
                                    timing=timing,
                                    fn_dump_center=pjoin(
                                        directory, 'array_center.pf'),
                                    fn_beam=pjoin(directory, 'beam.mseed'))
        i = 0
        store_id = syn_in.store()
        engine = LocalEngine(store_superdirs=[syn_in.store_superdirs()])
        weight = analyse(shifted_traces,
                         engine,
                         event,
                         stations,
                         100.,
                         store_id,
                         nwindows=1,
                         check_events=True,
                         phase_def='P')

    for trace in calcStreamMap.iterkeys():
        recordstarttime = calcStreamMap[trace].stats.starttime
        d = calcStreamMap[trace].stats.starttime
        d = d.timestamp

        if calcStreamMap[trace].stats.npts < minSampleCount:
            minSampleCount = calcStreamMap[trace].stats.npts

    ############################################################################
    traces = num.ndarray(shape=(len(calcStreamMap), minSampleCount),
                         dtype=float)
    traveltime = num.ndarray(shape=(len(calcStreamMap), dimX * dimY),
                             dtype=float)
    latv = num.ndarray(dimX * dimY, dtype=float)
    lonv = num.ndarray(dimX * dimY, dtype=float)
    ############################################################################

    c = 0
    streamCounter = 0

    for key in calcStreamMap.iterkeys():
        streamID = key
        c2 = 0

        for o in calcStreamMap[key]:
            if c2 < minSampleCount:
                traces[c][c2] = o

                c2 += 1

        for key in TTTGridMap.iterkeys():

            if streamID == key:
                traveltimes[streamCounter] = TTTGridMap[key]
            else:
                "NEIN", streamID, key

        if not streamCounter in traveltimes:
            continue  #hs : thread crashed before

        g = traveltimes[streamCounter]
        dimZ = g.dimZ
        mint = g.mint
        maxt = g.maxt
        Latul = g.Latul
        Lonul = g.Lonul
        Lator = g.Lator
        Lonor = g.Lonor

        gridElem = g.GridArray

        for x in range(dimX):
            for y in range(dimY):
                elem = gridElem[x, y]

                traveltime[c][x * dimY + y] = elem.tt
                latv[x * dimY + y] = elem.lat
                lonv[x * dimY + y] = elem.lon
        #endfor

        c += 1
        streamCounter += 1

    #endfor

    ############################## CALCULATE PARAMETER FOR SEMBLANCE CALCULATION ##################
    nsamp = winlen * new_frequence

    nstep = int(step * new_frequence)
    migpoints = dimX * dimY

    dimZ = 0
    new_frequence = cfg.newFrequency()  # ['new_frequence']
    maxp = int(Config['ncore'])

    Logfile.add('PROCESS %d  NTIMES: %d' % (flag, ntimes))

    if False:
        print('nostat ', nostat, type(nostat))
        print('nsamp ', nsamp, type(nsamp))
        print('ntimes ', ntimes, type(ntimes))
        print('nstep ', nstep, type(nstep))
        print('dimX ', dimX, type(dimX))
        print('dimY ', dimY, type(dimY))
        print('mint ', Gmint, type(mint))
        print('new_freq ', new_frequence, type(new_frequence))
        print('minSampleCount ', minSampleCount, type(minSampleCount))
        print('latv ', latv, type(latv))
        print('traces', traces, type(traces))
        print('traveltime', traveltime, type(traveltime))

    t1 = time.time()
    traces_org = traces.reshape(1, nostat * minSampleCount)
    traveltime_org = traveltime.reshape(1, nostat * dimX * dimY)
    USE_C_CODE = True
    try:
        if USE_C_CODE:
            import Cm
            import CTrig
            start_time = time.time()
            k = Cm.otest(maxp, nostat, nsamp, ntimes, nstep, dimX, dimY, Gmint,
                         new_frequence, minSampleCount, latv, lonv,
                         traveltime_org, traces_org)
            print("--- %s seconds ---" % (time.time() - start_time))
        else:
            start_time = time.time()
            k = otest(maxp, nostat, nsamp, ntimes, nstep, dimX, dimY, Gmint,
                      new_frequence, minSampleCount, latv, lonv,
                      traveltime_org, traces_org)  #hs
            print("--- %s seconds ---" % (time.time() - start_time))
    except:
        print "loaded tttgrid has probably wrong dimensions or stations, delete\
                ttgrid or exchange"

    t2 = time.time()

    Logfile.add('%s took %0.3f s' % ('CALC:', (t2 - t1)))

    partSemb = k

    partSemb_data = partSemb.reshape(ntimes, migpoints)

    return partSemb_data
    def test_ping_during_log_send(self):
        cm = None
        pgs1 = None
        pgs2 = None
        try:
            cm = Cm.CM("test_pgs")
            cm.create_workspace()
            pg = Pg.PG(0)

            # pgs1 --> master
            pgs1 = Pgs.PGS(0, 'localhost', 1900, cm.dir)
            pg.join(pgs1, start=True)
            pgs1.smr.wait_role(Smr.SMR.MASTER)

            # make lots of logs
            clients = []
            num_clients = 20
            for i in range(0, num_clients):
                C = Client.Client()
                clients.append(C)
                C.slotid = i
                C.add(C.slotid, 'localhost', 1909)
                C.size(64 * 1024, C.slotid)  # 1M
                C.tps(100, C.slotid)

            for C in clients:
                C.start(C.slotid)

            runtime_limit = 60
            runtime = 0
            while runtime < runtime_limit:
                time.sleep(1)
                runtime = runtime + 1

            for C in clients:
                C.stop(C.slotid)

            # get log seqeunce of the master
            master_seqs = pgs1.smr.getseq_log()

            # pgs2 --> slave
            pgs2 = Pgs.PGS(1, 'localhost', 9000, cm.dir)
            pg.join(pgs2, start=True)
            pgs2.smr.wait_role(Smr.SMR.SLAVE)

            try_count = 0
            count = 0
            prev_seq = 0
            master_response_times = []
            slave_response_times = []

            while True:
                try_count = try_count + 1

                # master
                st = int(round(time.time() * 1000))
                seqs = pgs1.smr.getseq_log()
                et = int(round(time.time() * 1000))
                master_response_times.append(et - st)

                # slave
                st = int(round(time.time() * 1000))
                seqs = pgs2.smr.getseq_log()
                et = int(round(time.time() * 1000))
                slave_response_times.append(et - st)

                if prev_seq != seqs['max']:
                    count = count + 1
                    prev_seq = seqs['max']
                if master_seqs['max'] <= seqs['max']:
                    break
                time.sleep(0.1)
            print "==========> try_count:%d count:%d" % (try_count, count)
            print "MASTER ==========>", master_response_times
            print "SLAVE ==========>", slave_response_times
            for rt in master_response_times:
                assert rt < 100
            for rt in slave_response_times:
                assert rt < 100

        finally:
            # Util.tstop('Check output!')
            if pgs1 is not None:
                pgs1.kill_smr()
                pgs1.kill_be()
            if pgs2 is not None:
                pgs2.kill_smr()
                pgs2.kill_be()
            if cm is not None:
                cm.remove_workspace()
示例#24
0
def doCalc(flag, Config, WaveformDict, FilterMetaData, Gmint, Gmaxt,
           TTTGridMap, Folder, Origin, ntimes):
    '''
    method for calculating semblance of one station array
    '''
    Logfile.add('PROCESS %d %s' % (flag, ' Enters Semblance Calculation'))
    Logfile.add('MINT  : %f  MAXT: %f Traveltime' % (Gmint, Gmaxt))

    cfg = ConfigObj(dict=Config)

    dimX = cfg.dimX()  # ('dimx')
    dimY = cfg.dimY()  # ('dimy')
    winlen = cfg.winlen()  # ('winlen')
    step = cfg.step()  # ('step')

    new_frequence = cfg.newFrequency()  #  ('new_frequence')
    forerun = cfg.Int('forerun')
    duration = cfg.Int('duration')
    gridspacing = cfg.Float('gridspacing')

    nostat = len(WaveformDict)
    traveltimes = {}
    recordstarttime = ''
    minSampleCount = 999999999

    ntimes = int((forerun + duration) / step)
    nsamp = int(winlen * new_frequence)
    nstep = int(step * new_frequence)

    #for i in WaveformDict.iterkeys():
    #    print i,WaveformDict[i]

    ############################################################################
    calcStreamMap = WaveformDict

    for trace in calcStreamMap.iterkeys():
        recordstarttime = calcStreamMap[trace].stats.starttime
        d = calcStreamMap[trace].stats.starttime
        d = d.timestamp

        if calcStreamMap[trace].stats.npts < minSampleCount:
            minSampleCount = calcStreamMap[trace].stats.npts

    ############################################################################
    traces = np.ndarray(shape=(len(calcStreamMap), minSampleCount),
                        dtype=float)
    traveltime = np.ndarray(shape=(len(calcStreamMap), dimX * dimY),
                            dtype=float)
    latv = np.ndarray(dimX * dimY, dtype=float)
    lonv = np.ndarray(dimX * dimY, dtype=float)
    ############################################################################

    #traces      = np.ndarray (nostat*minSampleCount,dtype=float)
    #traveltimes = np.ndarray (nostat*dimX*dimY,dtype=float)
    #latv        = np.ndarray (dimX*dimY,dtype=float)
    #lonv        = np.ndarray (dimX*dimY,dtype=float)
    #print 'minSC: ',minSampleCount,' LCSM: ',len(calcStreamMap)
    c = 0
    streamCounter = 0

    for key in calcStreamMap.iterkeys():
        streamID = key
        c2 = 0
        #print streamID, len(calcStreamMap[key]),minSampleCount

        for o in calcStreamMap[key]:
            if c2 < minSampleCount:
                traces[c][c2] = o
                #print 'C: ',c,' C2: ',c2,' TRACES:',traces[c][c2]
                c2 += 1
        #endfor

        for key in TTTGridMap.iterkeys():

            if streamID == key:
                traveltimes[streamCounter] = TTTGridMap[key]
            else:
                "NEIN", streamID, key
        #endfor

        if not streamCounter in traveltimes:
            continue  #hs : thread crashed before

        g = traveltimes[streamCounter]
        dimZ = g.dimZ
        mint = g.mint
        maxt = g.maxt
        Latul = g.Latul
        Lonul = g.Lonul
        Lator = g.Lator
        Lonor = g.Lonor

        gridElem = g.GridArray

        for x in range(dimX):
            for y in range(dimY):
                elem = gridElem[x, y]

                traveltime[c][x * dimY + y] = elem.tt
                latv[x * dimY + y] = elem.lat
                lonv[x * dimY + y] = elem.lon
        #endfor

        c += 1
        streamCounter += 1
    #endfor

    ############################## CALCULATE PARAMETER FOR SEMBLANCE CALCULATION ##################
    nsamp = winlen * new_frequence
    nstep = int(step * new_frequence)
    migpoints = dimX * dimY

    dimZ = 0
    new_frequence = cfg.newFrequency()  # ['new_frequence']
    maxp = int(Config['ncore'])
    #maxp = 20                                        #hs

    Logfile.add('PROCESS %d  NTIMES: %d' % (flag, ntimes))

    #k = Csemblance.semb(flag,nostat,nsamp,ntimes,nstep,Gmint,Gmaxt,Lonul,Latul,minSampleCount,
    #                    dimZ,dimX,dimY,new_frequence,ntimesstart,ntimesend,winlen,step,gridspacing,
    #                    latv,lonv,traveltime,traces,backveclen)
    #k = sembPar.semb   (flag,nostat,nsamp,ntimes,nstep,Gmint,Gmaxt,Lonul,Latul,minSampleCount,dimZ,
    #                    dimX,dimY,new_frequence,ntimesstart,ntimesend,winlen,step,gridspacing,latv,
    #                    lonv,traveltime,traces,backveclen)

    if False:
        print('nostat ', nostat, type(nostat))
        print('nsamp ', nsamp, type(nsamp))
        print('ntimes ', ntimes, type(ntimes))
        print('nstep ', nstep, type(nstep))
        print('dimX ', dimX, type(dimX))
        print('dimY ', dimY, type(dimY))
        print('mint ', Gmint, type(mint))
        print('new_freq ', new_frequence, type(new_frequence))
        print('minSampleCount ', minSampleCount, type(minSampleCount))
        print('latv ', latv, type(latv))
        print('traces', traces, type(traces))
        print('traveltime', traveltime, type(traveltime))

    traveltime = traveltime.reshape(1, nostat * dimX * dimY)
    traces = traces.reshape(1, nostat * minSampleCount)
    #print 'traveltime2',traveltime,type(traveltime)

    t1 = time.time()

    if USE_C_CODE:
        k = Cm.otest(maxp, nostat, nsamp, ntimes, nstep, dimX, dimY, Gmint,
                     new_frequence, minSampleCount, latv, lonv, traveltime,
                     traces)
    else:
        k = otest(maxp, nostat, nsamp, ntimes, nstep, dimX, dimY, Gmint,
                  new_frequence, minSampleCount, latv, lonv, traveltime,
                  traces)  #hs

    t2 = time.time()

    Logfile.add('%s took %0.3f s' % ('CALC:', (t2 - t1)))
    #print 'K',k,len(k),' MUST ',ntimes*dimX*dimY,' RES ',k[1]

    partSemb = k

    #partSemb = partSemb.reshape (1,migpoints)
    partSemb = partSemb.reshape(ntimes, migpoints)

    #print 'PARTSEMB FLAG: ',partSemb,type(partSemb),partSemb.ndim

    return partSemb
示例#25
0
    def test_quorum(self):
        cm = None
        M = None
        C = None
        P = None
        try:
            cm = Cm.CM("test_quorum")
            cm.create_workspace()
            # M(1), S, S
            M = _init_pgs(0, 'localhost', 1900, cm.dir)
            M.smr.role_master(M.id, 1, 0)

            C = _init_pgs(1, 'localhost', 1910, cm.dir)
            C.smr.role_slave(C.id, 'localhost', M.base_port, 0)

            P = _init_pgs(2, 'localhost', 1920, cm.dir)
            P.smr.role_slave(P.id, 'localhost', M.base_port, 0)

            _attach_client(M, 1024, 1000)
            _attach_client(C, 1024, 1000)
            _attach_client(P, 1024, 1000)

            M.client.start(M.client.slotid)
            C.client.start(C.client.slotid)
            P.client.start(P.client.slotid)

            # Loop
            loopCount = 20
            for i in range(0, loopCount):
                #  M(1), S, S
                M.smr.setquorum(1)
                _check_progress([M, C, P])
                #  M(1), S, L
                P.smr.role_lconn()
                _check_progress([M, C])
                #  M(C), S, S
                ps = P.smr.getseq_log()
                P.smr.role_slave(P.id, 'localhost', M.base_port, ps['max'])
                _check_progress([M, C, P])
                #  M(C), S, L
                M.smr.setquorum(1, [C.id])
                P.smr.role_lconn()
                _check_progress([M, C])
                #  M(C), S, S
                ps = P.smr.getseq_log()
                P.smr.role_slave(P.id, 'localhost', M.base_port, ps['max'])
                snap = _check_progress([M, C, P])
                #  M(C), L, S
                C.smr.role_lconn()
                _check_progress([M, P], noprogress=True)
                #  M(C), S, S
                cs = C.smr.getseq_log()
                C.smr.role_slave(C.id, 'localhost', M.base_port, cs['max'])
                _check_progress([M, C, P])
                loopCount = loopCount + 1
            # Stop clients
            M.client.stop(M.client.slotid)
            C.client.stop(C.client.slotid)
            P.client.stop(P.client.slotid)
        finally:
            #Util.tstop('Check output!')
            if M is not None:
                M.kill()
            if C is not None:
                C.kill()
            if P is not None:
                P.kill()
            if cm is not None:
                cm.remove_workspace()
示例#26
0
    def test_logdelete(self):
        cm = None
        pgs1 = None
        try:
            cm = Cm.CM("test_logdelete")
            cm.create_workspace()
            pg = Pg.PG(0)

            # pgs --> master
            pgs = Pgs.PGS(0, 'localhost', 1900, cm.dir)
            pg.join(pgs, start=True)
            pgs.smr.wait_role(Smr.SMR.MASTER)

            # make lots of logs
            clients = []
            num_clients = 20
            for i in range(0, num_clients):
                C = Client.Client()
                clients.append(C)
                C.slotid = i
                C.add(C.slotid, 'localhost', 1909)
                C.size(64 * 1024, C.slotid)
                C.tps(100, C.slotid)

            for C in clients:
                C.start(C.slotid)

            runtime_limit = 15
            runtime = 0
            while runtime < runtime_limit:
                time.sleep(1)
                runtime = runtime + 1

            for C in clients:
                C.stop(C.slotid)

            # checkpoint server
            pgs.be.ckpt()
            time.sleep(1.0)

            # delete log
            log_delete_seq = pgs.smr.deletelog(retain=0)
            assert (log_delete_seq > 0), log_delete_seq
            print("deltelog 0 returns", log_delete_seq)

            # log delete interval in idle state is 10 sec. (see bio.c)
            # after than, 1.0 0.9 0.8 ... 0.1, 0.1 ...
            logdel_timeout = 10 + log_delete_seq / (64 * 1024 * 1024)
            deltime = 0
            while deltime < logdel_timeout:
                seqs = pgs.smr.getseq_log()
                # Note background delete spares one log file (see del_proc in bio.c)
                if seqs['min'] + 64 * 1024 * 1024 == log_delete_seq:
                    break
                time.sleep(1)
                deltime = deltime + 1
            assert (deltime < logdel_timeout), logdel_timeout
        finally:
            if pgs is not None:
                pgs.kill_smr()
                pgs.kill_be()
            if cm is not None:
                cm.remove_workspace()
示例#27
0
  def test_mem_disk_change (self):
    saved = Conf.USE_MEM_LOG
    Conf.USE_MEM_LOG = True
    cm = None
    pgs = None
    expected_seqs = {}
    try:
      cm = Cm.CM("tmpTestMemDiskChange")
      cm.create_workspace()
      pg = Pg.PG(0)
      pgs = Pgs.PGS(0, 'localhost', self.BASE_PORT, cm.dir)

      expected_seqs['min'] = 0
      expected_seqs['max'] = 0

      for i in range (0, 5):
	if i % 2 == 0:
	  Log.createlog(pgs.dir)
	else:
	  Log.deletelog(pgs.dir)

	# start replicator 
	pgs.start_smr()
	pgs.smr.wait_role(Smr.SMR.NONE)

	# check seqs
	seqs = pgs.smr.getseq_log()
	print "expected =======>", expected_seqs
	print "seqs     =======>", seqs
	assert seqs['min'] == expected_seqs['min']
	assert seqs['max'] == expected_seqs['max']

	# pgs -> master
	pgs.start_be()
	pgs.smr.wait_role(Smr.SMR.LCONN)
	pg.join(pgs)
	pgs.smr.wait_role(Smr.SMR.MASTER)

	# kill be 
	self.make_some_logs(runtime=3)

	# wait for be to apply all logs
	time.sleep(1)

	# checkpoint
	pgs.be.ckpt()

	# remember original sequences
	expected_seqs = pgs.smr.getseq_log()

	# kill pgs
	pg.leave(pgs.id, kill=True)

    finally:
      #Util.tstop('Check output!')
      if pgs is not None:
	pgs.kill_smr()
	pgs.kill_be()
      if cm is not None:
	cm.remove_workspace() 
      Conf.USE_MEM_LOG = saved
示例#28
0
    def test_logutil(self):
        cm = None
        master = None
        try:
            cm = Cm.CM("test_logutil")
            cm.create_workspace()
            pg = Pg.PG(0)

            # master --> master
            master = Pgs.PGS(0, 'localhost', 1900, cm.dir)
            pg.join(master, start=True)
            master.smr.wait_role(Smr.SMR.MASTER)

            # make some logs
            self.make_logs(master, 5)

            # get file sequences
            seqs = master.smr.getseq_log()
            file_seqs = []
            seq = 0
            while seq < seqs['max']:
                file_seqs.append(seq)
                seq = seq + 64*1024*1024

            # get leading timestamp and sequence from each logs and peek one
            peeks = []
            for seq in file_seqs:
                out, _ = Log.datadump_communicate(master.dir, seq, seq + 5*1024, 'Tsld32')
                lines = out.split('\n')
                assert len(lines) > 0
                peek = lines[len(lines)/2]
                peeks.append(peek)

            # find by time range and check peek is in the lines
            idx = 0
            for seq in file_seqs:
                peek = peeks[idx]
                ts = int(peek.split()[0])
                assert ts > 20160817000000000
                out, _ = Log.dumpbytime_communicate(master.dir, ts, ts+1, 'Tsld32')
                lines = out.split('\n')
                found = False
                for line in lines:
                    if line == peek:
                        found = True
                        break
                assert found == True
                idx = idx + 1

            # decachelog (just for coverage)
            out, _ = Log.decachelog_communicate(master.dir, file_seqs[-1], True)
            assert len(out) == 0

            # mincorelog (just for coverage)
            out, _ = Log.mincorelog_communicate(master.dir)
            assert len(out.split('\n')) >= 5

        finally:
            # Util.tstop('Check output!')
            if master is not None:
                master.kill_smr()
                master.kill_be()
            if cm is not None:
                cm.remove_workspace()