Example #1
0
def load_cached_mouse_hist2d_segments(hist2d_tarf,outlines_tarf,SHAPE,return_sizes=False,check_lengths=True, \
				      **kwargs):
	msize_out = os.path.splitext(hist2d_tarf)[0]+'-mouse_sizes.list'
	if os.path.exists(hist2d_tarf) and (not check_lengths or (len(tarfile.open(hist2d_tarf).getnames()) == len(tarfile.open(outlines_tarf).getnames()))) \
	       and (not return_sizes or os.path.exists(msize_out)):
		mm_hists = Util.tar2ar_all(tarfile.open(hist2d_tarf),SHAPE,int)
		print >> sys.stderr, '%s mouse location histograms loaded, max counts: %s' % (len(mm_hists),max([m.max() for m in mm_hists]) )
		if return_sizes:
			svec = eval(open(msize_out).read())
			print >> sys.stderr, '%s mouse size values loaded; mean %s' % (len(svec),numpy.mean(svec))
	else:
		if return_sizes:
			print >> sys.stderr, 'calculate mouse locations from outlines %s' % outlines_tarf
			mm_hists,svec = load_mouse_hist2d_segments(tarfile.open(outlines_tarf),SHAPE,return_sizes=True,**kwargs)
			print >> sys.stderr, '%s mouse location histograms calculated, max counts: %s' % (len(mm_hists),max([m.max() for m in mm_hists]))
			print >> sys.stderr, '%s mouse size values calculated; mean %s' % (len(svec),numpy.mean(svec))
			open(msize_out,'w').write(svec.__repr__())
		else:
			mm_hists = load_mouse_hist2d_segments(tarfile.open(outlines_tarf),SHAPE,**kwargs)
			print >> sys.stderr, '%s mouse location histograms calculated' % len(mm_hists)
		h2d_tarh = tarfile.open(hist2d_tarf,'w')
		for fname,ar in zip(sorted(tarfile.open(outlines_tarf).getnames()),mm_hists):
			Util.ar2tar(ar,fname,h2d_tarh)
		h2d_tarh.close()
	if return_sizes:
		return mm_hists,svec
	else:
		return mm_hists
def insertRandomRoom(level):
	x = Util.getRandom(0, LEVEL_WIDTH-1)
	y = Util.getRandom(0, LEVEL_HEIGHT-1)
	width = Util.getRandom(3, 6)
	height = Util.getRandom(3, 6)
	room = Level.MakeRoom(width, height)
	level.applyFeature(room, x, y)
Example #3
0
    def resetToDefaults(self, confirm=True):
        """
        Reset the application to the defaults.  At the very least,
        this will delete all waves, tables, and figures.

        Before resetting, this will ask the user if they want to save
        the current project.
        """
        
        Util.debug(2, "App.resetToDefaults", "Resetting application to defaults")
        
        if confirm:
            # Check to see if we want to save the current project
            saveProjectMessage = QMessageBox()
            saveProjectMessage.setText("You are about to load another project.")
            saveProjectMessage.setInformativeText("Do you want to save your current project?")
            saveProjectMessage.setStandardButtons(QMessageBox.Save | QMessageBox.Discard | QMessageBox.Cancel)
            saveProjectResponse = saveProjectMessage.exec_()
            if saveProjectResponse == QMessageBox.Save:
                self.saveProject()
            elif saveProjectResponse == QMessageBox.Cancel:
                return

        # Now reset to a clean slate
        self.setDefaults()
def usedVigenere(inputFile, dictionary):
    best = 0
    key = 0
    percent = 0

    #listChars = ['a','b','c','d']
    listChars = ['1','2','3','4','5','6']

    setDictionary = Util.setWords(dictionary)
    for sizeKey in range(1, len(listChars)+1):
       for i in itertools.product(listChars, repeat=sizeKey):
            print(i)
            tmp = DecriptFunctions.modDecVigenere(i, inputFile[0:240])
            uniqueWords = Util.setWords(tmp)
            nWords = len(uniqueWords & setDictionary)

            if nWords > best:
                best = nWords
                key = i
                countWords = len(uniqueWords)
                percent = (best*100)/countWords

            if percent > 50:
                break    
    print("\nAlgoritmo de Vigenere"
                              "\nChave: " + str(key) +
                              "\nTotal de palavras: " + str(countWords) +
                              "\nTotal de palavras encontradas: " + str(best) +
                              "\n" + str(percent) + "% de acerto" )
Example #5
0
 def setCurrentProject(self, fileName):
     Util.debug(3, "App.setCurrentProject", "Setting current project title")
     self.currentProjectFile = fileName
     if fileName != "":
         self.setWindowTitle("PySciPlot - " + fileName)
     else:
         self.setWindowTitle("PySciPlot")
Example #6
0
def G1DBinaryStringXSinglePoint(genome, **args):
   """ The crossover of 1D Binary String, Single Point

   .. warning:: You can't use this crossover method for binary strings with length of 1.

   """
   sister = None
   brother = None
   gMom = args["mom"]
   gDad = args["dad"]

   if len(gMom) == 1:
      Util.raiseException("The Binary String have one element, can't use the Single Point Crossover method !", TypeError)

   cut = rand_randint(1, len(gMom)-1)

   if args["count"] >= 1:
      sister = gMom.clone()
      sister.resetStats()
      sister[cut:] = gDad[cut:]

   if args["count"] == 2:
      brother = gDad.clone()
      brother.resetStats()
      brother[cut:] = gMom[cut:]

   return (sister, brother)
Example #7
0
def G1DBinaryStringXTwoPoint(genome, **args):
   """ The 1D Binary String crossover, Two Point

   .. warning:: You can't use this crossover method for binary strings with length of 1.

   """
   sister = None
   brother = None
   gMom = args["mom"]
   gDad = args["dad"]
   
   if len(gMom) == 1:
      Util.raiseException("The Binary String have one element, can't use the Two Point Crossover method !", TypeError)

   cuts = [rand_randint(1, len(gMom)-1), rand_randint(1, len(gMom)-1)]

   if cuts[0] > cuts[1]:
      Util.listSwapElement(cuts, 0, 1)

   if args["count"] >= 1:
      sister = gMom.clone()
      sister.resetStats()
      sister[cuts[0]:cuts[1]] = gDad[cuts[0]:cuts[1]]

   if args["count"] == 2:
      brother = gDad.clone()
      brother.resetStats()
      brother[cuts[0]:cuts[1]] = gMom[cuts[0]:cuts[1]]

   return (sister, brother)
Example #8
0
 def process(self):
     Util.log('SUBCLASSES : %s' % self.subclasses, 'debug')
     for cls in self.subclasses.values():
         cls.process()
     Util.log('METHODS : %s' % self.methods, 'debug')
     for meth in self.methods:
         self.select_meth(meth)
Example #9
0
    def __init__(self, methanalysis, this):
        self.memory = {}
        self.method = methanalysis.get_method()
        self.name = self.method.get_name()
        self.lparams = []
        self.variables = Instruction.Variable()
        
        if self.name == '<init>':
            self.name = self.method.get_class_name()[1:-1].split('/')[-1]
        self.basic_blocks = methanalysis.basic_blocks.bb
        code = self.method.get_code()

        access = self.method.get_access()
        self.access = [flag for flag in Util.ACCESS_FLAGS_METHODS if flag & access]
        desc = self.method.get_descriptor()
        self.type = Util.get_type(desc.split(')')[-1])
        self.paramsType = Util.get_params_type(desc)

        #Util.log('Searching loops in method %s' % self.name, 'debug')
        """
        Findloop.loopCounter = 0
        cfg = Findloop.CFG()
        lsg = Findloop.LSG()
        self.cfg = cfg
        self.lsg = lsg
        Findloop.AddEdges(cfg, self)
        nbLoops = Findloop.FindLoops(cfg, lsg)
        
        if nbLoops > 1:
            Util.log('==================', 'debug')
            Util.log('==== DUMP CFG ====', 'debug')
            cfg.dump()
            Util.log('==================', 'debug')
            Findloop.ShowBlocks(lsg, 0)
            Util.log('==== DUMP LSG ====', 'debug')
            lsg.dump()
            Util.log('==================', 'debug')
        """
        lsg = None
        blocks = []
        exceptions = methanalysis.exceptions.exceptions
        print "METHOD :", self.name
        self.graph = Structure.ConstructCFG(lsg, self.basic_blocks, exceptions, blocks)
        print

        #androguard.bytecode.method2png('graphs/%s#%s.png' % (self.method.get_class_name().split('/')[-1][:-1], self.name), methanalysis)

        if code is None:
            Util.log('CODE NONE : %s %s' % (self.name, self.method.get_class_name()), 'debug')
        else:
            start = code.registers_size.get_value() - code.ins_size.get_value()
            # 0x8 == Static case : this is not passed as a parameter
            if 0x8 in self.access:
                self._add_parameters(start)
            else:
                self.memory[start] = Register(this, start)
                #print 'THIS ( %s ) : %d' % (self.name, start)
                self._add_parameters(start + 1)
        self.ins = []
        self.cur = 0
Example #10
0
def loadStoredSettings(app, storedSettings):
    """
    Load stored settings into the application.
    """
    Util.debug(1, "Load", "Loading stored settings from file")
    storedSettingsObj = pickle.loads(str(storedSettings.firstChild.data.strip()))
    app.storedSettings().update(storedSettingsObj)
Example #11
0
def loadFigures(app, figures, stackingOrder):
    """
    Load given figures into the application.
    """
    
    Util.debug(1, "Load", "Loading figures from file")

    figureList = figures.getElementsByTagName("figure")

    for figure in figureList:
        figureAttrs = {} # QT attributes

        # Get attributes for the figure
        for attrName in figure.attributes.keys():
            figureAttrs[attrName] = int(figure.attributes[attrName].value.strip())

        # Get figure from xml
        pickledFigure = figure.firstChild.data.strip()
        figureObj = pickle.loads(str(pickledFigure))
        figureWindow = app.figures().addFigure(figureObj)._figureSubWindow

        # Reset QT widget values
        figureWindow.setWindowState(Qt.WindowState(figureAttrs["windowState"]))
        figureWindow.resize(figureAttrs["width"], figureAttrs["height"])
        figureWindow.move(figureAttrs["xpos"], figureAttrs["ypos"])

        # Add to stackingOrder
        if figureAttrs["stackingOrder"] >= len(stackingOrder):
            stackingOrder.extend([None] * (figureAttrs["stackingOrder"] - len(stackingOrder) + 1))
        stackingOrder[figureAttrs["stackingOrder"]] = figureWindow

        # Set the figure dialog selected to the first figure
        app._loadedModules['EditFigureDialog'].changeCurrentFigure(0)
        app._loadedModules['EditFigureDialog'].updateFigureSelectorIndex(0)
Example #12
0
    def test_channelAnimatorBoundaryAnimation(self):
        driver = self.driver 

        # Open a test image so we have something to animate
        Util.load_image( self, driver, "Default")

        # Find and record the last valid value of the animation
        self._getLastValue( driver, "Channel" )
        lastChannelValue = self._getCurrentValue( driver, "Channel" )

        # Find and click the lower spin box
        lowerBoundText = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.XPATH, "//div[@id='ChannelLowerBoundSpin']/input")))
        driver.execute_script( "arguments[0].scrollIntoView(true);", lowerBoundText)
        lowerBoundText.click()

        # Change the lower bound value
        lowerBoundValue = Util._changeElementText(self, driver, lowerBoundText, 1)

        # Find and click the upper spin box
        upperBoundText = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.XPATH, "//div[@id='ChannelUpperBoundSpin']/input")))
        driver.execute_script( "arguments[0].scrollIntoView(true);", upperBoundText)
        upperBoundText.click()

        # Change the upper bound value 
        upperBoundValue = Util._changeElementText( self, driver, upperBoundText, int(lastChannelValue)-1)

        # Allow test image to animate for 2 seconds
        self._animateForward( driver, "Channel" )
        time.sleep(2)

        # Check that the lower and upper bound values did not change during animation
        lowerBound = lowerBoundText.get_attribute("value")
        upperBound = upperBoundText.get_attribute("value")
        self.assertEqual( int(lowerBound), int(lowerBoundValue), "Lower bound channel value changed during animation")
        self.assertEqual( int(upperBound), int(upperBoundValue), "Upper bound channel value changed during animation")
Example #13
0
 def collectData(self,hostId,sqlServerId, discoverConfigs = 1):
     self.connection.open()
     oshv = ObjectStateHolderVector()
     try:
         oshv.add(self.getServerProperties(sqlServerId,hostId))
         dbMap = self.getDatabases(sqlServerId)
         #get the databases
         oshv.addAll(self.sqlDataBaseProps.getDatabases(dbMap,hostId,discoverConfigs))
         oshv.addAll(self.sqlDataBaseProps.getStoredProcedures(dbMap))
         #get the server configuration:
         logger.debug('discovering configs')
         try:
             oshv.add(self.sqlServerConfig.getServerConfiguration(sqlServerId))
             oshv.add(self.sqlServerConfig.getServerStartup(sqlServerId))
             self.sqlServerConfig.discoverPlans(oshv,sqlServerId,dbMap)
         except:
             logger.debugException(hostId.toString())
         if self.discoveryOptions and self.discoveryOptions.discoverDbUser:
             users = self.getDbUsers(sqlServerId)
             Util.addFromMap(users,oshv)
         else:
             users = LinkedHashMap()
         oshv.addAll(self.getProcesses(hostId,sqlServerId,dbMap,users))
         oshv.addAll(self.clusterConfiguration.collectData(sqlServerId))
         #db configuration:
         oshv.addAll(self.getDbConf(dbMap,hostId,users))
         logger.debug("sql db result for hostid:"+hostId.toString())
     except:
         logger.debugException(hostId.toString())
     self.connection.close()
     return oshv
Example #14
0
    def test_campusLinkLastYears(self):

        self.ui.tabWid.setCurrentIndex(2)
        gear = tests.mkGear('J', 10)
        tests.enterGearInfo(self, gear)
        QTest.mouseClick(self.tGear.updtBut, QtCore.Qt.LeftButton)

        self.ui.tabWid.setCurrentIndex(1)
        self.tMemb.nameSearch.clear()
        memb = tests.mkMember('A', 1, forms=True, campusLink=True,
                              formsDate=Util.convert_date('Qt2DB', QtCore.QDate.currentDate().addDays(1)),
                              campusDate=Util.convert_date('Qt2DB', QtCore.QDate.currentDate().addDays(-10)))
        QTest.keyClicks(self.tMemb.nameSearch, memb['FirstName'] + ' ' + memb['LastName'])
        tests.enterMemberInfo(self, memb)
        self.assertTrue(self.tMemb.Button_addUpdButClick())

        self.ui.tabWid.setCurrentIndex(0)
        setTransGear(self, gear, 'Name')
        setTransMemb(self, memb)

        QTest.mouseClick(self.tTran.payBut, QtCore.Qt.LeftButton)
        QTest.mouseClick(self.tTran.payWind.payBut, QtCore.Qt.LeftButton)
        self.tTran.payWind.close()

        self.tTran.radioOut.click()
        self.assertTrue(self.tTran.trans())
Example #15
0
def RawScoreCriteria(ga_engine):
   """ Terminate the evolution using the **bestrawscore** and **rounddecimal**
   parameter obtained from the individual

   Example:
      >>> genome.setParams(bestrawscore=0.00, rounddecimal=2)
      (...)
      >>> ga_engine.terminationCriteria.set(GSimpleGA.RawScoreCriteria)

   """
   ind = ga_engine.bestIndividual()
   bestRawScore = ind.getParam("bestrawscore")
   roundDecimal = ind.getParam("rounddecimal")

   if bestRawScore is None:
      Util.raiseException("you must specify the bestrawscore parameter", ValueError)

   if ga_engine.getMinimax() == Consts.minimaxType["maximize"]:
      if roundDecimal is not None:
         return round(bestRawScore, roundDecimal) <= round(ind.score, roundDecimal)
      else:
         return bestRawScore <= ind.score
   else:
      if roundDecimal is not None:
         return round(bestRawScore, roundDecimal) >= round(ind.score, roundDecimal)
      else:
         return bestRawScore >= ind.score
Example #16
0
    def _startVm(
        self,
        withLocalNetwork=False,
        requestedIpAddress=None,
        instanceNumber=1,
        noCheckImageUrl=False,
        msgRecipients=None,
        raiseOnFailed=True,
    ):
        self.runner = self._createRunner(withLocalNetwork, requestedIpAddress)
        self.runner.instanceNumber = instanceNumber

        self.runner.noCheckImageUrl = noCheckImageUrl

        if not msgRecipients:
            self.runner.msgRecipients = msgRecipients

        vmIds = self.runner.runInstance()
        self.vmIds.extend(vmIds)

        for vmId in vmIds:
            vmStarted = self.runner.waitUntilVmRunningOrTimeout(vmId, VM_START_TIMEOUT, failOn="Failed")
            if not vmStarted and raiseOnFailed:
                error = "Failed to start VM id: %s" % vmId
                Util.printError(error, exit=False)
                raise OneException(error)

        return self.runner
Example #17
0
 def _umountPDiskAndStopVm(self, runner, pdiskDevice):
     self._umountDisk(runner, pdiskDevice)
     Util.printStep("Stopping VM...")
     self._stopVm(runner)
     self.vmIds = []
     # Wait for the pdisk hook to be executed
     sleep(20)
Example #18
0
def processing():
    mat_name = r'silk'
    ori_folder = r'D:\Datasets\silkA_16bit\16bit'
    res_folder = r'D:\Datasets\PreprocessingData'

    # Read tiff images and dump them into a binary file
#    print 'Read and Dump images begins ...'    
#    volume_path = os.path.join(res_folder, mat_name + '.dat')
#    volume_data = TiffsHandler.readTiffImagesInFolder(ori_folder)  
#    print 'Read images finished ...'
#    
#    print 'Dump volume to (%s) begins ...' % volume_path
#    Util.dumpData(volume_data, volume_path)
#    print 'Dump volume to (%s) finished ...' % volume_path

    # load volume data
    volume_path = os.path.join(res_folder, mat_name + '.dat')
    volume_data = Util.loadData(volume_path)
    
    # Histogram equalize
    print 'Histogram equalize begins ...'
    bins = np.arange(0, 65537)
#    bins = np.linspace(0, 65537, 50, endpoint=True)
    volume_he = Util.histEq(volume_data, bins=bins)
    volume_he = volume_he.astype('float32')
    print 'Histogram equalize finished ...'
    
    volume_he_path = os.path.join(res_folder, mat_name + '_he.dat')
    print 'Dump volume_he to (%s) begins ...' % volume_he_path
    Util.dumpData(volume_he, volume_he_path)
    print 'Dump volume_he to (%s) finished ...' % volume_he_path    
    
    return volume_data, volume_he
Example #19
0
def resample():
    volume = Util.loadData(r'D:\Datasets\PreprocessingData\silk_he.dat')
    volume = volume.swapaxes(0, 2).swapaxes(0, 1)

    m = [25.6207523,
         0.00563881328,
         -7.50831215e-06,
         -0.000110252038,
         5.8775209e-07,
         -4.20923367e-10,
         2.7086295e-07,
         -1.18467307e-09,
         1.03793103e-12]
    m = np.asarray(m)
    
    nx, ny, nz = volume.shape
    x = np.arange(0, nx)
    y = np.arange(0, ny)
    X, Y = np.meshgrid(x, y, indexing='ij')
    Z = Util.polyval2d(X, Y, m)
    
    print Z.shape
    
    Zmin = Z.min()
    Zmax = Z.max()
    
    R = np.zeros((nx, ny, nz + Zmax))
    
    for i, j in zip(X.ravel(), Y.ravel()):
        s = int(Zmax - Z[i, j])
        R[i, j, s:s+nz] = volume[i, j, :]
    
    Util.dumpData(R, r'D:\Datasets\PreprocessingData\silk_rs.dat')
        
    return R
Example #20
0
def denoiseSilk():
    d = np.load(r'')  # [508, 1013, 992], 'uint16', [0, 65535]
    J = np.load(r'') # [1013, 992, 105], 'float32'
    ori = np.load(r'D:\Dataset\silk\orientation.dat') # [1013, 992, 105, 3], 'float32'
    ed = 0.4
    eJ = -1.0

    # preprocess
    d = d.astype(dtype='float32')
    d /= 65535.0
#    print d.shape, d.dtype, d.min(), d.max()

    J = J.swapaxes(0, 2).swapaxes(1, 2)
#    print J.shape, J.dtype, J.min(), J.max()

    ori = ori.swapaxes(0, 2).swapaxes(1, 2)
#    print ori.shape, ori.dtype

    # denoising
    denoise(d, J, ori, ed, eJ)
    print 'finish deonising ...'

    # save
    silk_dn_path = r'D:\Dataset\silk\silk_dn.dat'
    ori_dn_path = r'D:\Dataset\silk\orientation_dn.dat'

    Util.dumpData(d, silk_dn_path)
    Util.dumpData(ori, ori_dn_path)
Example #21
0
def precomputeFilterQ():
    # Precompute kernel 'q' on a set of directions

    # Calculate the directions
    nx = 32
    ny = 32
    nz = 6
    bound_points = generateBoundingPoints(nx + 1, ny + 1, nz + 1)
    center = np.array([nx/2, ny/2, nz/2])
    
    directions = bound_points - center
    directions = np.asarray(directions, dtype='float64')    

    # normalize    
    for i in range(0, directions.shape[0]):
        sqrt_sum = np.sqrt((directions[i]**2).sum())
        directions[i] = directions[i] / sqrt_sum
    
    d_path = r'D:\Datasets\RecoverOrientationData\ds_nx32ny32nz6.dat'
    Util.dumpData(directions, d_path)    
    
    # Calculate 'q', material: silk
    h = 12
    s = 4 #!! some erros in paper about these parameters, s>t!!
    t = 3
    c = np.array([h/2, h/2, h/2], dtype='float64') # center
    d = directions
    
    q = np.zeros((d.shape[0], h, h, h))

    for i in range(0, q.shape[0]):
        calculateFilterQ(q[i], h, s, t, d[i], c)    # pass reference?!!

    qs_path = r'D:\Datasets\RecoverOrientationData\qs_h12s4t3.dat'
    Util.dumpData(q, qs_path) 
Example #22
0
def LinearScaling(pop):
   """ Linear Scaling scheme

   .. warning :: Linear Scaling is only for positive raw scores

   """
   logging.debug("Running linear scaling.")
   pop.statistics()
   c = Consts.CDefScaleLinearMultiplier
   a = b = delta = 0.0

   pop_rawAve = pop.stats["rawAve"]
   pop_rawMax = pop.stats["rawMax"]
   pop_rawMin = pop.stats["rawMin"]
   
   if pop_rawAve == pop_rawMax:
      a = 1.0
      b = 0.0
   elif pop_rawMin > (c * pop_rawAve - pop_rawMax / (c - 1.0)):
      delta = pop_rawMax - pop_rawAve
      a = (c - 1.0) * pop_rawAve / delta
      b = pop_rawAve * (pop_rawMax - (c * pop_rawAve)) / delta
   else:
      delta = pop_rawAve - pop_rawMin
      a = pop_rawAve / delta
      b = -pop_rawMin * pop_rawAve / delta

   for i in xrange(len(pop)):
      f = pop[i].score
      if f < 0.0:
         Util.raiseException("Score %r is negative, linear scaling not supported !" % (f,), ValueError)
      f = f * a + b
      if f < 0:
         f = 0.0
      pop[i].fitness = f
Example #23
0
    def test_histogramLinkRemoval(self):
        driver = self.driver
        browser = selectBrowser._getBrowser()

        # Wait for the image window to be present (ensures browser is fully loaded)
        imageWindow = WebDriverWait(driver, 20).until(EC.presence_of_element_located((By.XPATH, "//div[@qxclass='skel.widgets.Window.DisplayWindowImage']")))

        # Find the Histogram window
        histWindow = self._getHistogramWindow( driver )
        ActionChains(driver).click(histWindow).perform()

        # Open Link settings for the Histogram window
        linkMenuButton = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.XPATH, "//div[@qxclass='qx.ui.toolbar.MenuButton']/div[text()='Links...']")))
        ActionChains(driver).click( linkMenuButton ).perform()

        # Remove link from the Histogram to the main image window
        Util.remove_main_link( self, driver, imageWindow )

        # Load an image
        Util.load_image( self, driver, "Default")

        # Find and select the histogram window
        histWindow = self._getHistogramWindow( driver )
        ActionChains(driver).click( histWindow ).perform()

        # Click the settings button to expose the settings
        self._openHistogramSettings( driver )

        # Check that the histogram values are default values
        newMaxZoomValue = self._getTextValue( driver, "histogramZoomMaxValue")
        self.assertEqual( float(newMaxZoomValue), 1, "Histogram is linked to image after link was removed")
Example #24
0
    def test_histogramAddImage(self):
        driver = self.driver
        timeout = selectBrowser._getSleep()

        # Wait for the image window to be present (ensures browser is fully loaded)
        imageWindow = WebDriverWait(driver, 20).until(EC.presence_of_element_located((By.XPATH, "//div[@qxclass='skel.widgets.Window.DisplayWindowImage']")))

        # Load an image
        Util.load_image(self, driver, "Default")

        # Find and select the histogram
        histWindow = self._getHistogramWindow(driver)
        ActionChains(driver).click( histWindow ).perform()

        # Click the settings button to expose the settings
        self._openHistogramSettings( driver )

        # Get the max zoom value of the first image
        maxZoomValue = self._getTextValue( driver, "histogramZoomMaxValue")
        print "First image maxZoomValue:", maxZoomValue

        # Load a different image in the same window
        Util.load_image(self, driver, "aH.fits")
        time.sleep( timeout )

        # Check that the new max zoom value updates
        newMaxZoomValue = self._getTextValue( driver, "histogramZoomMaxValue")
        self.assertNotEqual(float(newMaxZoomValue), float(maxZoomValue), "The histogram did not update when a new image was loaded.")
        print "Second image maxZoomValue:", newMaxZoomValue
Example #25
0
    def test_histogramRemoveImage(self):
        driver = self.driver
        timeout = selectBrowser._getSleep()

        # Wait for the image window to be present (ensures browser is fully loaded)
        imageWindow = WebDriverWait(driver, 20).until(EC.presence_of_element_located((By.XPATH, "//div[@qxclass='skel.widgets.Window.DisplayWindowImage']")))

        # Load an image
        Util.load_image( self, driver, "Default")

        # Click on the Data->Close->Image button to close the image
        imageWindow = driver.find_element_by_xpath( "//div[@qxclass='skel.widgets.Window.DisplayWindowImage']")
        ActionChains(driver).double_click( imageWindow ).perform()
        dataButton = WebDriverWait(driver, 20).until(EC.presence_of_element_located((By.XPATH, "//div[text()='Data']/..")))
        ActionChains(driver).click( dataButton ).send_keys(Keys.ARROW_DOWN).send_keys(Keys.ARROW_DOWN).send_keys(
            Keys.ARROW_RIGHT).send_keys(Keys.ENTER).perform()

        # Find and select the Histogram window
        histWindow = self._getHistogramWindow( driver )
        ActionChains(driver).click( histWindow ).perform()
        time.sleep( timeout )

        # Click the settings button to expose the settings
        self._openHistogramSettings( driver )

        # Check that the histogram values are restored to default values
        newMaxZoomValue = self._getTextValue( driver, "histogramZoomMaxValue")
        print "Zoom max value=", newMaxZoomValue
        self.assertEqual( float(newMaxZoomValue), 1, "Default values were not restored after image removal")
Example #26
0
    def test_statsAddImage(self):
        driver = self.driver

        # Load a large test image. 
        Util.load_image( self, driver, "aH.fits")

        # Move to the center of the image window so data appears in the Stats Window
        imageWindow = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.XPATH, "//div[@qxclass='skel.widgets.Window.DisplayWindowImage']")))
        ActionChains(driver).move_to_element( imageWindow ).perform()

        # Get the Statistics of the loaded image 
        statsText = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.XPATH, "//div[@qxclass='skel.boundWidgets.Label']")))
        statsText = statsText.get_attribute('textContent')        
        
        # Load a different image in the same window
        Util.load_image( self, driver, "aK.fits")

        # Move to the center of the image window so data appears in the Stats Window
        ActionChains(driver).move_to_element( imageWindow ).perform()

        # Sometimes text does not appear, therefore move mouse cursor
        ActionChains(driver).move_by_offset( 100, 100 ).perform()

        # Get the Statistics of the loaded image 
        newStatsText = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.XPATH, "//div[@qxclass='skel.boundWidgets.Label']")))
        newStatsText = newStatsText.get_attribute('textContent')

        # Check that the Statistics text changed when new image was loaded
        self.assertNotEqual( newStatsText, statsText, "Stats text did not update when new image was loaded in main image window")
Example #27
0
    def test_statsAnimation(self):
        driver = self.driver

        # Load a test image with animation
        Util.load_image( self, driver, "Default")

        # Move to the center of the image window so data appears in the Stats Window
        imageWindow = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.XPATH, "//div[@qxclass='skel.widgets.Window.DisplayWindowImage']")))
        ActionChains(driver).move_to_element( imageWindow ).perform()

        # Get the Statistics of the loaded image 
        statsText = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.XPATH, "//div[@qxclass='skel.boundWidgets.Label']")))
        statsText = statsText.get_attribute('textContent')   

        # Click the forward animation button
        forwardAnimateButton = driver.find_element_by_xpath("//div[@class='qx-toolbar']/div[@class='qx-button'][2]")
        self.assertIsNotNone( forwardAnimateButton, "Could not find forward animation button")
        driver.execute_script( "arguments[0].scrollIntoView(true);", forwardAnimateButton)
        ActionChains(driver).click( forwardAnimateButton ).perform()
        time.sleep(3)

        # Move to the center of the image window so data appears in the Stats Window
        imageWindow = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.XPATH, "//div[@qxclass='skel.widgets.Window.DisplayWindowImage']")))
        ActionChains(driver).move_to_element( imageWindow ).perform()
        
        # Sometimes text does not appear, therefore move mouse cursor
        ActionChains(driver).move_by_offset( 100, 100 ).perform()

        # Get the Statistics of the loaded image 
        newStatsText = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.XPATH, "//div[@qxclass='skel.boundWidgets.Label']")))
        newStatsText = newStatsText.get_attribute('textContent')   

        # Check that the Statistics text changed when image in the image window was changed
        self.assertNotEqual( newStatsText, statsText, "Stats text did not update during animation of the image")
Example #28
0
    def test_histogramChangeLinks(self):
        driver = self.driver
        browser = selectBrowser._getBrowser()

        # Wait for the image window to be present (ensures browser is fully loaded)
        imageWindow = WebDriverWait(driver, 20).until(EC.presence_of_element_located((By.XPATH, "//div[@qxclass='skel.widgets.Window.DisplayWindowImage']")))

        # Find and select the Histogram window
        histWindow = self._getHistogramWindow( driver )
        ActionChains(driver).click( histWindow ).perform()

        # Click the settings button to expose the settings
        self._openHistogramSettings( driver )

         # Open Link settings for the Histogram window
        linkMenuButton = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.XPATH, "//div[@qxclass='qx.ui.toolbar.MenuButton']/div[text()='Links...']")))
        ActionChains(driver).click( linkMenuButton ).perform()

        # Remove the link from the Histogram to the main image window
        Util.remove_main_link( self, driver, imageWindow )

        # Load an image in a separate window
        imageWindow2 = Util.load_image_different_window( self, driver, "aH.fits")

        # Open link settings for the Histogram
        ActionChains(driver).click( histWindow ).perform()
        linkMenuButton = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.XPATH, "//div[@qxclass='qx.ui.toolbar.MenuButton']/div[text()='Links...']")))
        ActionChains(driver).click( linkMenuButton ).perform()

        # Link the Histogram to the second image
        Util.link_second_image( self, driver, imageWindow2)

        # Check that the histogram values are not default values
        newMaxZoomValue = self._getTextValue( driver, "histogramZoomMaxValue")
        self.assertNotEqual( float(newMaxZoomValue), 1, "Histogram did not update to newly linked image")
Example #29
0
    def test_coordinateChange(self):
        driver = self.driver
        timeout = selectBrowser._getSleep()

        #Load an image with a tabular axis
        Util.load_image( self, driver, "aJ.fits")
        time.sleep( 3 )

        #Open the image settings tab
        Util.openSettings( self, driver, "Image", True )
        time.sleep( timeout )
        
         #Get the old coordinateSystem
        systemText = driver.find_element_by_xpath("//div[@id='ImageCoordinateSystem']/input")
        driver.execute_script( "arguments[0].scrollIntoView(true);", systemText )
        oldSystem = systemText.get_attribute('value')
        print "Old system=", oldSystem
        
        #Change the coordinate system
        csCombo = driver.find_element_by_xpath("//div[@id='ImageCoordinateSystem']/div")
        driver.execute_script( "arguments[0].scrollIntoView(true);", csCombo )
        ActionChains(driver).click(csCombo).send_keys( Keys.ARROW_DOWN).send_keys( Keys.ARROW_DOWN
                ).send_keys( Keys.ARROW_DOWN).send_keys( Keys.ENTER).perform()
        time.sleep( timeout )
        
        #Verify the coordinate system is changed
        systemText = driver.find_element_by_xpath("//div[@id='ImageCoordinateSystem']/input")
        driver.execute_script( "arguments[0].scrollIntoView(true);", systemText )
        newSystem = systemText.get_attribute( 'value')
        print "New system=",newSystem
        self.assertTrue( newSystem != oldSystem, "Coordinate system did not change")
def _get_graph_output_dir():
  """Assign an output path for the graph(s)."""
  cfg = Configuration.getConfig()
  graph_dir = cfg.get('Path', 'base-dir') + cfg.get('Path', 'graph-dir')
  graph_dir += 'ActiveUsers/'
  Util.ensure_dir_exist(graph_dir)
  return graph_dir
Example #31
0
 def shift2Local(self):
     self.id  = '6100'+Util.generateRnd(4)
     
Example #32
0
        else:
            embedding_array[i] = np.random.rand(
                Config.embedding_size) * 0.02 - 0.01
    print "Found embeddings: ", foundEmbed, "/", len(knownWords)

    return embedding_array


if __name__ == '__main__':

    wordDict = {}
    posDict = {}
    labelDict = {}
    parsing_system = None

    trainSents, trainTrees = Util.loadConll('train.conll')
    devSents, devTrees = Util.loadConll('dev.conll')
    testSents, _ = Util.loadConll('test.conll')
    genDictionaries(trainSents, trainTrees)

    embedding_filename = 'word2vec.model'

    embedding_array = load_embeddings(embedding_filename, wordDict, posDict,
                                      labelDict)

    labelInfo = []
    for idx in np.argsort(labelDict.values()):
        labelInfo.append(labelDict.keys()[idx])
    parsing_system = ParsingSystem(labelInfo[1:])
    print parsing_system.rootLabel
Example #33
0
class GvimTool(Tool.Tool):
    """A Gvim Tool: edit files with Gvim."""
    def __init__(self, *args, **keyw):
        apply(Tool.Tool.__init__, (self, ) + args, keyw)
        self.server = None
        self.handler = None
        self.prep_start()

        self.sernumPC = 22  # serial number used for PC annotation
        self.sernumLast = 33  # serial number used for annotations

    def prep_start(self):
        """Prepare for starting Gvim.  Done when the tool is created and when
           Gvim has exited."""
        self.gvim_thread = None
        self.gvim_open = 0  # set when gvim starts, reset when it exits.
        self.gvim_start_time = 0
        self.buffers = {}  # buffers used in Netbeans Gvim tool
        self.lastbuffer = 0

    def gvimRunning(self):
        """Check if Gvim is still running."""
        if self.gvim_open:
            if gvimHasNetbeans(self.topmodel):
                # gvim_open flag is set and reset appropriately.
                return 1
            # No netbeans connection: need to check if the process is still
            # there.
            if Tool.stillRunning(self.gvim_open):
                return 1
            self.gvim_open = 0
        return 0

    def gvimWait(self):
        """Can't send a command to Gvim when it is still starting up. Wait a
           couple of seconds to avoid an error or starting another Gvim."""
        wait = self.gvim_start_time + 2 - time.time()
        if wait > 0:
            time.sleep(wait)

    def getProperties(self):
        """Properties that this tool supports."""
        return getProperties(self.topmodel)

    def openItem(self, item, action, lnum=None, col=None, off=None):
        """Open ActyItem "item" in this Tool.  Not actually done until
        foreground() is called."""
        self.addItem(item)

    def reload(self, item, node=None):
        """Reload the "item", it was changed elsewhere."""
        if not node:
            node = item.node
        if not self.gvimRunning():
            self.foreground(item, node)
        elif gvimHasNetbeans(self.topmodel):
            # TODO: What if the file was changed?
            if node:
                self.currentnode = node
                self.gvimOpenFile(node)
        else:
            cmd = ":if exists('*inputsave')|call inputsave()|endif"
            # XXX escape special characters
            # XXX jump to other window if it's edited there.
            cmd = cmd + ('|hide edit %s' % item.fullName())
            cmd = cmd + "|if exists('*inputsave')|call inputrestore()|endif<CR>"
            self.sendkeys(cmd)

    def goto(self, item, node=None, lnum=None, col=None, off=None):
        """goto position "lnum"/"col" or "off" in "item"."""
        self.foreground(item, node)
        if gvimHasNetbeans(self.topmodel):
            if item:
                node = item.node
            buf = self.buffers.get(node)
            if buf and self.handler:
                if off:
                    self.handler.sendCmd(buf.nr, "setDot", " %d" % off)
                elif lnum:
                    if not col:
                        col = 0
                    self.handler.sendCmd(buf.nr, "setDot",
                                         " %d/%d" % (lnum, col))
        else:
            if off:
                cmd = ("%dgo" % (off + 1))
            else:
                cmd = ''
                if lnum:
                    cmd = cmd + ("%dG" % lnum)
                if col:
                    cmd = cmd + ("%d|" % col)
            self.sendkeys(cmd)

    def showPC(self, item, node=None, lnum=0, col=0, off=None, show=1):
        """Put PC marker at position "lnum"/"col" or "off" in "item"."""
        if gvimHasNetbeans(self.topmodel) and self.handler:
            if item:
                node = item.node
            buf = self.buffers.get(node)

            # The Netbeans interface has a very strange way of handling
            # annotations.  An arbitrary name can be used to define it, but
            # it's later placed and unplaced by a number, starting at 1 and
            # rising for each anno used in a node (buffer in Vim terms).

            if not buf.annoPCTypenum:
                # First use in this node: define the annotation type.
                buf.annoLastTypenum = buf.annoLastTypenum + 1
                buf.annoPCTypenum = buf.annoLastTypenum

                # Define a sign with line highlighting of a lightblue
                # background.
                # args: typeNum (not actually used)
                #       typeName (only used to define sign)
                #       tooltip (not used)
                #       glyphFile (name of bitmap for sign, optional)
                #       fg      (fg color number or "none" for line highlight)
                #       bg      (bg color number or "none" for line highlight)
                self.handler.sendCmd(buf.nr, "defineAnnoType",
                                     ' 22 "PC" "" "" none %d' % 0xa0a0ff)

            if show:
                self.addAnno(buf, self.sernumPC, buf.annoPCTypenum, lnum, col,
                             off)
            else:
                self.removeAnno(buf, self.sernumPC)

    def addAnno(self, buf, sernum, typenum, lnum, col, off):
        """Add an annotation with serial number "sernum", type number "typenum"
           at postion "lnum"/"col" or "off"."""
        if off:
            s = "%d" % off
        else:
            s = "%d/%d" % (lnum, col)
        self.handler.sendCmd(buf.nr, "addAnno",
                             " %d %d %s -1" % (sernum, typenum, s))

    def removeAnno(self, buf, sernum):
        """Delete annotation "sernum"."""
        self.handler.sendCmd(buf.nr, "removeAnno", " %d" % sernum)

    def displayBreakpoint(self, what, bp):
        """Update breakpoint "bp" for action "what"."""
        if (bp.item and bp.item.node and self.buffers.get(bp.item.node)
                and self.handler):
            node = bp.item.node
            buf = self.buffers[node]

            if what == "new":
                if bp in buf.breakpoints:
                    # Already in the list, happens when the tool is both in the
                    # "edit" and "view" list of the node.
                    return
                buf.breakpoints.append(bp)
                if not buf.annoBreakETypenum:
                    # First use in this node: define the annotation type.
                    buf.annoLastTypenum = buf.annoLastTypenum + 1
                    buf.annoBreakETypenum = buf.annoLastTypenum
                    buf.annoLastTypenum = buf.annoLastTypenum + 1
                    buf.annoBreakDTypenum = buf.annoLastTypenum

                    # Define a sign with line highlighting of a lightblue
                    # background.
                    # args: typeNum (not actually used)
                    #       typeName (only used to define sign)
                    #       tooltip (not used)
                    #       glyphFile (name of bitmap for sign, optional)
                    #       fg      (fg color number or "none" for line highlight)
                    #       bg      (bg color number or "none" for line highlight)
                    self.handler.sendCmd(
                        buf.nr, "defineAnnoType",
                        ' 23 "BreakE" "" ">>" none %d' % 0xffa0a0)
                    self.handler.sendCmd(
                        buf.nr, "defineAnnoType",
                        ' 24 "BreakD" "" ">>" none %d' % 0xb0b0b0)
                self.sernumLast = self.sernumLast + 1
                self.addAnno(buf, self.sernumLast, buf.annoBreakETypenum,
                             bp.lnum, 0, None)
                bp.sernum = self.sernumLast

            else:  # "del" or "upd"
                self.removeAnno(buf, bp.sernum)
                if what == "upd":
                    if bp.enable:
                        typenum = buf.annoBreakETypenum
                    else:
                        typenum = buf.annoBreakDTypenum
                    self.addAnno(buf, bp.sernum, typenum, bp.lnum, 0, None)

    def sendkeys(self, cmd):
        """Execute "cmd" in the Gvim server.  Caller should check if gvim is
           still running."""
        if gvimHasNetbeans(self.topmodel):
            # safety check for using sendkeys() with netbeans connection.
            print "ERROR: send keys: '%s'" % cmd
        else:
            self.gvimWait()
            os.system(
                gvimProgName(self.topmodel) +
                ' --servername AAPGVIM --remote-send "<C-\\><C-N>' + cmd + '"')

    def escapeStringArg(self, line):
        """Escape special characters in "line" to be send to gvim.  Reverse of
           parseStringArg()."""
        res = ''
        i = 0
        while i < len(line):
            if line[i] == '\n':
                res = res + "\\n"
            elif line[i] == '\t':
                res = res + "\\t"
            elif line[i] == '\r':
                res = res + "\\r"
            elif line[i] == '\\':
                res = res + "\\\\"
            elif line[i] == '"':
                res = res + '\\"'
            else:
                res = res + line[i]
            i = i + 1
        return res

    def close(self, shutdown):
        """Close the tool.
           Return non-zero if closing is OK."""
        if self.gvimRunning():
            self.gvim_open = 0
            if self.gvim_thread:
                # Tell gvim to exit.
                if self.handler:
                    self.handler.sendFunc(0, "saveAndExit")
                    # We don't wait for the answer, Vim may continue to run.
            else:
                # XXX check if there are any changed files in gvim.
                self.sendkeys(":qa<CR>")
            self.topmodel.toollist.delTool(self)
        return 1  # can shutdown

    def foreground(self, item, node=None, lnum=None):
        """Move this Tool to the foreground, with the current activity.
           When "item" is not None edit this item.
           When "node" is not None edit this node."""
        winposarg = ('"+winpos %d %d" "+set lines=30 columns=82" ' %
                     (self.topmodel.view.GetSize().width + 10, 0))
        if item:
            node = item.node

        # Start a Vim server on the file.
        if gvimHasNetbeans(self.topmodel):
            self.gvimRunNetbeans(winposarg)
        if self.handler:
            if node:
                self.currentnode = node
                self.gvimOpenFile(node)
            else:
                self.handler.sendCmd(0, "raise")
        else:
            running = self.gvimRunning()
            cmd = "%s -f --servername AAPGVIM " % gvimProgName(self.topmodel)

            # Jump to the node when it's specified, the user may have started
            # editing another file.
            if not running or node:
                # Position the Vim window next to ours and bring it to the
                # front.
                cmd = cmd + winposarg

                if node:
                    if lnum:
                        pos = "+%d " % lnum
                    else:
                        pos = ''
                    cmd = cmd + ('--remote-silent %s"%s"' %
                                 (pos, node.fullName()))
                if item:
                    self.setCurrentItem(item)
                else:
                    self.currentnode = node
            else:
                cmd = cmd + '--remote-send "<C-\\><C-N>:call foreground()<CR>"'
            if not running:
                self.gvim_start_time = time.time()
                self.gvim_open = Tool.spawn(cmd)
            else:
                self.gvimWait()
                os.system(cmd)

    def gvimRunNetbeans(self, gvimarg):
        """Make sure we have a running Gvim with Netbeans connection."""
        if self.gvim_open:
            return
        # start a new thread to run the server in
        self.gvim_thread = thread.start_new_thread(self.gvimThreadFunc,
                                                   (gvimarg, ))
        self.gvim_open = 1

        # Wait until Gvim is available and finished starting up.  We cannot
        # send commands before this.
        # When opening gvim fails the other thead sets gvim_open to zero.
        # Timeout after ten seconds.
        countdown = 1000
        while countdown > 0:
            time.sleep(0.01)
            if self.handler or self.gvim_open == 0:
                break
            countdown = countdown - 1

        if not self.handler:
            # XXX error handling
            self.gvim_open = 0
            print "Cannot open Gvim connection!"

    def gvimThreadFunc(self, gvimarg):
        """Function that is run in a separate thread.  It starts the socket
           server and runs gvim."""
        # start the server

        # IANA says that port numbers 49152 through 65335 are for dynamic use.
        # Start a bit higher to avoid clashes too often.
        portnr = 49765
        while not self.server:
            try:
                self.server = SocketServer.TCPServer(("", portnr), GvimHandler)
                self.server.gvimtool = self
            except Exception, e:
                if string.find(str(e), "ddress already in use") < 0:
                    print "Could not start socket server: ", e
                    self.gvim_open = 0
                    return
                # Port number already in use, retry with another port number.
                portnr = portnr + 1

        # start gvim and tell it to use the server
        # use a random password for semi-security (it shows up in the process
        # list...)
        vimrc = os.path.join(self.topmodel.rootdir, "startup.vim")
        self.password = str(random.random())
        Util.async_system(
            [], None,
            '{quiet} %s --servername AAPGVIM %s-nb:localhost:%d:%s -S "%s"' %
            (gvimProgName(
                self.topmodel), gvimarg, portnr, self.password, vimrc))

        # handle ONE request
        self.server.handle_request()

        # Gvim was closed, need to restart the server next time.
        self.prep_start()

        # If netbeans doesn't work close the socket server
        if gvim_has_netbeans == 0:
            del self.server
            self.server = None
Example #34
0
def executeShCmd(cmd, cfg, cwd, results):
    if isinstance(cmd, ShUtil.Seq):
        if cmd.op == ';':
            res = executeShCmd(cmd.lhs, cfg, cwd, results)
            return executeShCmd(cmd.rhs, cfg, cwd, results)

        if cmd.op == '&':
            raise NotImplementedError,"unsupported test command: '&'"

        if cmd.op == '||':
            res = executeShCmd(cmd.lhs, cfg, cwd, results)
            if res != 0:
                res = executeShCmd(cmd.rhs, cfg, cwd, results)
            return res
        if cmd.op == '&&':
            res = executeShCmd(cmd.lhs, cfg, cwd, results)
            if res is None:
                return res

            if res == 0:
                res = executeShCmd(cmd.rhs, cfg, cwd, results)
            return res

        raise ValueError,'Unknown shell command: %r' % cmd.op

    assert isinstance(cmd, ShUtil.Pipeline)
    procs = []
    input = subprocess.PIPE
    stderrTempFiles = []
    # To avoid deadlock, we use a single stderr stream for piped
    # output. This is null until we have seen some output using
    # stderr.
    for i,j in enumerate(cmd.commands):
        # Apply the redirections, we use (N,) as a sentinal to indicate stdin,
        # stdout, stderr for N equal to 0, 1, or 2 respectively. Redirects to or
        # from a file are represented with a list [file, mode, file-object]
        # where file-object is initially None.
        redirects = [(0,), (1,), (2,)]
        for r in j.redirects:
            if r[0] == ('>',2):
                redirects[2] = [r[1], 'w', None]
            elif r[0] == ('>>',2):
                redirects[2] = [r[1], 'a', None]
            elif r[0] == ('>&',2) and r[1] in '012':
                redirects[2] = redirects[int(r[1])]
            elif r[0] == ('>&',) or r[0] == ('&>',):
                redirects[1] = redirects[2] = [r[1], 'w', None]
            elif r[0] == ('>',):
                redirects[1] = [r[1], 'w', None]
            elif r[0] == ('>>',):
                redirects[1] = [r[1], 'a', None]
            elif r[0] == ('<',):
                redirects[0] = [r[1], 'r', None]
            else:
                raise NotImplementedError,"Unsupported redirect: %r" % (r,)

        # Map from the final redirections to something subprocess can handle.
        final_redirects = []
        for index,r in enumerate(redirects):
            if r == (0,):
                result = input
            elif r == (1,):
                if index == 0:
                    raise NotImplementedError,"Unsupported redirect for stdin"
                elif index == 1:
                    result = subprocess.PIPE
                else:
                    result = subprocess.STDOUT
            elif r == (2,):
                if index != 2:
                    raise NotImplementedError,"Unsupported redirect on stdout"
                result = subprocess.PIPE
            else:
                if r[2] is None:
                    if kAvoidDevNull and r[0] == '/dev/null':
                        r[2] = tempfile.TemporaryFile(mode=r[1])
                    else:
                        r[2] = open(r[0], r[1])
                    # Workaround a Win32 and/or subprocess bug when appending.
                    if r[1] == 'a':
                        r[2].seek(0, 2)
                result = r[2]
            final_redirects.append(result)

        stdin, stdout, stderr = final_redirects

        # If stderr wants to come from stdout, but stdout isn't a pipe, then put
        # stderr on a pipe and treat it as stdout.
        if (stderr == subprocess.STDOUT and stdout != subprocess.PIPE):
            stderr = subprocess.PIPE
            stderrIsStdout = True
        else:
            stderrIsStdout = False

            # Don't allow stderr on a PIPE except for the last
            # process, this could deadlock.
            #
            # FIXME: This is slow, but so is deadlock.
            if stderr == subprocess.PIPE and j != cmd.commands[-1]:
                stderr = tempfile.TemporaryFile(mode='w+b')
                stderrTempFiles.append((i, stderr))

        # Resolve the executable path ourselves.
        args = list(j.args)
        args[0] = Util.which(args[0], cfg.environment['PATH'])
        if not args[0]:
            raise InternalShellError(j, '%r: command not found' % j.args[0])

        procs.append(subprocess.Popen(args, cwd=cwd,
                                      stdin = stdin,
                                      stdout = stdout,
                                      stderr = stderr,
                                      env = cfg.environment,
                                      close_fds = kUseCloseFDs))

        # Immediately close stdin for any process taking stdin from us.
        if stdin == subprocess.PIPE:
            procs[-1].stdin.close()
            procs[-1].stdin = None

        # Update the current stdin source.
        if stdout == subprocess.PIPE:
            input = procs[-1].stdout
        elif stderrIsStdout:
            input = procs[-1].stderr
        else:
            input = subprocess.PIPE

    # FIXME: There is probably still deadlock potential here. Yawn.
    procData = [None] * len(procs)
    procData[-1] = procs[-1].communicate()

    for i in range(len(procs) - 1):
        if procs[i].stdout is not None:
            out = procs[i].stdout.read()
        else:
            out = ''
        if procs[i].stderr is not None:
            err = procs[i].stderr.read()
        else:
            err = ''
        procData[i] = (out,err)
        
    # Read stderr out of the temp files.
    for i,f in stderrTempFiles:
        f.seek(0, 0)
        procData[i] = (procData[i][0], f.read())

    exitCode = None
    for i,(out,err) in enumerate(procData):
        res = procs[i].wait()
        # Detect Ctrl-C in subprocess.
        if res == -signal.SIGINT:
            raise KeyboardInterrupt

        results.append((cmd.commands[i], out, err, res))
        if cmd.pipe_err:
            # Python treats the exit code as a signed char.
            if res < 0:
                exitCode = min(exitCode, res)
            else:
                exitCode = max(exitCode, res)
        else:
            exitCode = res

    if cmd.negate:
        exitCode = not exitCode

    return exitCode
from WordVector import *
import Util

mContentVectorTrainer = ChineseVectorTrainer()

# read all vocabulary
client = MongoClient("localhost", 27017)
db = client.DeepSearch
mContentVectorTrainer.FeedWords(db.words.find({}))
mContentVectorTrainer.FeedWords(db.probWords.find({}))

lines = []
datafile = open("wiki_cn_out_00", "r")
for i in range(100000):
    lines.append(datafile.readline().replace(" \n", "").split(" "))
mContentVectorTrainer.FeedContent(lines)
# print(mContentVectorTrainer.data)
# mContentVectorTrainer.ShowData(1, 8, 2, 1)
mContentVectorTrainer.InitConfig()
mContentVectorTrainer.TrainModal()

# print(mContentVectorTrainer.weights[1, :])
# print(mContentVectorTrainer.biases.shape)

wordList = mContentVectorTrainer.GetResultWordList()
# for index in range(1):
#     print(wordList[index])

for wordItem in wordList:
    Util.UpdateWord(wordItem)
Example #36
0
#!/usr/bin/python
import sys
import os
sys.path.append("/home/won/WON/")
import Util
from multiprocessing import Pool

F = Util.ReadColumn("Tophat/configure.list", [0, 1])
Title = ["Tophat/%s/accepted_hits.bam.count" % x[0] for x in F]
Column = [x[1] for x in F]
note = 0
pre = ""
for i in xrange(len(Column)):
    if Column[i] != pre:
        note = 1
    pre = Column[i]
    Column[i] = Column[i] + "_%d" % note
    note += 1

cmd = "paste "
for mlist in Title:
    cmd += "%s " % mlist

cmd += "| cut -f 1"
for i in xrange(1, 100):
    cmd += ",%d" % (2 * i)
cmd += " > gene.tmp"
os.system(cmd)

L = Util.ReadList("gene.tmp")
Ens2Gene = Util.ReadDict("mm9.Trans2Gene.dat", [0], [1])
import os
import sys
base_path = os.path.split(os.path.realpath(__file__))[0]
sys.path.append(base_path)
# lib_path = os.path.join(base_path, 'lib')
# sys.path.append(lib_path)
import template_file
import tempfile
import Util
import subprocess
import time
from socket import *
import json

# 日志对象
logger = Util.getLogger('login')
# 操作超时时间
# wait_timeout = int(Util.getConfig('wait_timeout'))
# 自动登录watcher端口
watcher_port = int(Util.getConfig('watcher_port'))
# 堡垒机登录名
# login_name = Util.getConfig('login_name')
# 登录的ip
target_ip = crt.Arguments.GetArg(0)

logger.info('login plsql script start, target ip: ' + target_ip)

tab = crt.GetScriptTab()
tab.Screen.Synchronous = True

try:
Example #38
0
if args.provider == "digital_ocean" and not args.linear:
    logger.warning(
        "Provider '%s' does not support parallel creation of VMs. Linear creation is automatically enabled. See https://github.com/devopsgroup-io/vagrant-digitalocean/pull/230 for further details."
        % args.provider)
    args.linear = True

if len(args.databases) > 1 and (args.nodestroy or args.noshutdown):
    logger.warning(
        "The arguments --noshutdown and --nodestroy do not work with multiple databases at one run. Both are automatically disabled."
    )
    args.nodestroy = False
    args.noshutdown = False

# File checks and deletions (if necessary)
for folder in args.vagrantfolders:
    if not Util.check_folder(folder, logger):
        exit(-1)
for vagrantCredFile in vagrantCredFiles:
    found_cred_file = False
    for folder in args.vagrantfolders:
        if Util.check_file_exists(os.path.join(folder, vagrantCredFile)):
            found_cred_file = True
    if not found_cred_file:
        logger.error("%s not found in any of the given vagrantfolders (%s)." %
                     (vagrantCredFile, args.vagrantfolders))
        exit(-1)
if not Util.check_folder(args.tmpfolder, logger):
    exit(-1)
if not Util.delete_file(logFile, logger, True):
    exit(-1)
Example #39
0
def train(sqn, save_model_path):
    print('Starting train...')

    # Hyper parameters
    epochs = 1
    batch_size = 128
    keep_probability = 0.7
    learning_rate = 0.001
    n_batches = 5  #CIFAR10 dataset in the python version has 5 batches

    x = tf.placeholder(tf.float32, shape=(None, 32, 32, 3), name='input_x')
    y = tf.placeholder(tf.float32, shape=(None, 10), name='output_y')
    keep_prob = tf.placeholder(tf.float32, name='keep_prob')

    logits = sqn.inference(x)

    # Loss and Optimizer
    cost = tf.reduce_mean(
        tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=y))
    l2Loss = sum(list(map(lambda x: tf.nn.l2_loss(x), sqn.all_weights)))
    beta = 1e-5
    cost = cost + beta * l2Loss
    optimizer = tf.train.AdamOptimizer(
        learning_rate=learning_rate).minimize(cost)

    # Accuracy
    correct_pred = tf.equal(tf.argmax(logits, 1), tf.argmax(y, 1))
    accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32),
                              name='accuracy')

    valid_features, valid_labels = Util.load_preprocess_validation_data()
    testing_features, testing_labels = Util.load_preprocess_testing_data()

    print('Training now...')
    with tf.Session() as sess:
        # Initializing the variables
        sess.run(tf.global_variables_initializer())

        # Training cycle
        for epoch in range(epochs):
            # Loop over all batches
            for batch_i in range(1, n_batches + 1):
                for batch_features, batch_labels in Util.load_preprocess_training_batch(
                        batch_i, batch_size):
                    sess.run(optimizer,
                             feed_dict={
                                 x: batch_features,
                                 y: batch_labels,
                                 keep_prob: keep_probability
                             })

                print('Epoch {:>2}, CIFAR-10 Batch {}:  '.format(
                    epoch + 1, batch_i),
                      end='')

                # Print stats
                loss = sess.run(cost,
                                feed_dict={
                                    x: batch_features,
                                    y: batch_labels,
                                    keep_prob: keep_probability
                                })
                train_acc = sess.run(accuracy,
                                     feed_dict={
                                         x: batch_features,
                                         y: batch_labels,
                                         keep_prob: keep_probability
                                     })
                valid_acc = sess.run(accuracy,
                                     feed_dict={
                                         x: valid_features,
                                         y: valid_labels,
                                         keep_prob: keep_probability
                                     })
                testing_acc = sess.run(accuracy,
                                       feed_dict={
                                           x: testing_features,
                                           y: testing_labels,
                                           keep_prob: keep_probability
                                       })
                print(
                    'Loss: {:>10.4f} Train Acc: {:.6f} Validation Accuracy: {:.6f} Testing Acc: {:.6f}'
                    .format(loss, train_acc, valid_acc, testing_acc))

            if (epoch % 10 == 0):
                # Save Model
                saver = tf.train.Saver()
                save_path = saver.save(sess, save_model_path)
Example #40
0
 def shift2AnotherCity(self):
     self.id = Util.generateRnd(6)
Example #41
0
    def __init__(self,*args):
        if not os.path.sep in __file__:
            scriptdir = os.getcwd()
        else:
            scriptdir = os.path.dirname(__file__)

        #n3file = os.path.sep.join([scriptdir,"etc","sfs-extra.n3"])
        #n3url = "file://" + n3file.replace("\\","/")

        #print "scriptdir: %s" % scriptdir
        #print "n3file: %s" % n3file
        #print "n3url: %s" % n3url

        self.graph = Graph()
        n3file = Util.relpath(scriptdir + "/etc/sfs-extra.n3")
        # print "loading n3file %s" % n3file
        self.graph.load(n3file, format="n3")
        self.roots = []
        self.uriformatter = {}
        self.decl = ""
        self.namedlaws = {}
        self.load_ebnf(scriptdir+"/etc/base.ebnf")

        self.args = args
        if self.LAGRUM in args:
            productions = self.load_ebnf(scriptdir+"/etc/lagrum.ebnf")
            for p in productions:
                self.uriformatter[p] = self.sfs_format_uri
            self.namedlaws.update(self.get_relations(RDFS.label))
            self.roots.append("sfsrefs")
            self.roots.append("sfsref")

        if self.KORTLAGRUM in args:
            # om vi inte redan laddat lagrum.ebnf måste vi göra det
            # nu, eftersom kortlagrum.ebnf beror på produktioner som
            # definerats där
            if not self.LAGRUM in args:
                self.load_ebnf(scriptdir+"/etc/lagrum.ebnf")
                
            productions = self.load_ebnf(scriptdir+"/etc/kortlagrum.ebnf")
            for p in productions:
                self.uriformatter[p] = self.sfs_format_uri
            DCT = Namespace("http://purl.org/dc/terms/")
            d = self.get_relations(DCT['alternate'])
            self.namedlaws.update(d)
            lawlist = [x.encode(SP_CHARSET) for x in d.keys()]
            # Make sure longer law abbreviations come before shorter
            # ones (so that we don't mistake "3 § MBL" for "3 § MB"+"L")
            lawlist.sort(cmp=lambda x,y:len(y)-len(x))
            self.decl += "LawAbbreviation ::= ('%s')\n" % "'/'".join(lawlist)
            self.roots.insert(0,"kortlagrumref")

        if self.EGLAGSTIFTNING in args:
            productions = self.load_ebnf(scriptdir+"/etc/eglag.ebnf")
            for p in productions:
                self.uriformatter[p] = self.eglag_format_uri
            self.roots.append("eglagref")
        if self.FORARBETEN in args:
            productions = self.load_ebnf(scriptdir+"/etc/forarbeten.ebnf")
            for p in productions:
                self.uriformatter[p] = self.forarbete_format_uri
            self.roots.append("forarbeteref")
        if self.RATTSFALL in args:
            productions = self.load_ebnf(scriptdir+"/etc/rattsfall.ebnf")
            for p in productions:
                self.uriformatter[p] = self.rattsfall_format_uri
            self.roots.append("rattsfallref")
        if self.EGRATTSFALL in args:
            productions = self.load_ebnf(scriptdir+"/etc/egratt.ebnf")
            for p in productions:
                self.uriformatter[p] = self.egrattsfall_format_uri
            self.roots.append("ecjcaseref")
            
        self.decl += "root ::= (%s/plain)+\n" % "/".join(self.roots)
        # pprint(productions)
        # print self.decl.decode(SP_CHARSET,'ignore')

        self.parser = Parser(self.decl, "root")
        self.tagger = self.parser.buildTagger("root")
        # print "tagger length: %d" % len(repr(self.tagger))
        self.verbose = False
        self.depth = 0

        # SFS-specifik kod
        self.currentlaw     = None
        self.currentchapter = None
        self.currentsection = None
        self.currentpiece   = None
        self.lastlaw        = None
        self.currentlynamedlaws = {}
Example #42
0
def main():
    scalingFac = 12
    findAccOrArgMaxOrPredVal = 0
    restoreWeights = True
    onlysavegraph = False
    save_model_path = './TrainedModel/model'
    doTraining = False

    inp = None
    if (len(sys.argv) > 1):
        inp = sys.argv[1]
        if (inp == 'train'):
            doTraining = True
        elif (inp == 'savegraph'):
            findAccOrArgMaxOrPredVal = 1
            restoreWeights = False
            onlysavegraph = True
            testing_features, testing_labels = Util.get_sample_points(
                2, 4555, 4556)
        elif (inp == 'testSingleTestInp'):
            testBatchInpNum = int(sys.argv[2])
            findAccOrArgMaxOrPredVal = 1
            restoreWeights = True
            onlysavegraph = False
            all_testing_features, all_testing_labels = Util.load_preprocess_testing_data(
            )
            testing_features, testing_labels = all_testing_features[
                testBatchInpNum:testBatchInpNum +
                1], all_testing_labels[testBatchInpNum:testBatchInpNum + 1]
        elif (inp == 'testSingleTestInpAndSaveData'):
            testBatchInpNum = int(sys.argv[2])
            findAccOrArgMaxOrPredVal = int(sys.argv[3])
            restoreWeights = True
            onlysavegraph = False
            all_testing_features, all_testing_labels = Util.load_preprocess_testing_data(
            )
            testing_features, testing_labels = all_testing_features[
                testBatchInpNum:testBatchInpNum +
                1], all_testing_labels[testBatchInpNum:testBatchInpNum + 1]
            # testing_features, testing_labels = numpy.full((1,32,32,3),0.01), numpy.full((1,10),0.01)
        elif (inp == 'savegraphAndDataBatch'):
            batchNum = int(sys.argv[2])
            imgStartNum = int(sys.argv[3])
            imgEndNum = int(sys.argv[4])
            findAccOrArgMaxOrPredVal = 1
            restoreWeights = False
            onlysavegraph = True
            testing_features, testing_labels = Util.get_sample_points(
                batchNum, imgStartNum, imgEndNum)
        elif (inp == 'testBatchInp'):
            imgStartNum = int(sys.argv[2])
            imgEndNum = int(sys.argv[3])
            findAccOrArgMaxOrPredVal = 1
            restoreWeights = True
            onlysavegraph = False
            all_testing_features, all_testing_labels = Util.load_preprocess_testing_data(
            )
            testing_features, testing_labels = all_testing_features[
                imgStartNum:imgEndNum], all_testing_labels[
                    imgStartNum:imgEndNum]
        elif (inp == 'findAndSaveCorrectTestImg'):
            findAccOrArgMaxOrPredVal = 2
            restoreWeights = True
            onlysavegraph = False
            testing_features, testing_labels = Util.load_preprocess_testing_data(
            )
            testing_features, testing_labels = testing_features[
                0:100], testing_labels[0:100]
        else:
            if (inp != ""):
                print("WARNING : Given option didn't match any known value.")
            testing_features, testing_labels = Util.load_preprocess_testing_data(
            )

    sqn = SqueezeNet1(use_cons_init=onlysavegraph)
    if doTraining:
        train(sqn, save_model_path)
    else:
        with tf.Session() as sess:
            pred = infer(sqn,
                         sess,
                         testing_features,
                         testing_labels,
                         save_model_path,
                         findAccOrArgMaxOrPredVal=findAccOrArgMaxOrPredVal,
                         restoreWeights=restoreWeights,
                         onlysavegraph=onlysavegraph)
            if findAccOrArgMaxOrPredVal == 1 and not (onlysavegraph):
                print("Actual labels = ", testing_labels)
                print("ArgMax in actual label : ",
                      numpy.argmax(testing_labels, axis=1))

            if (inp == 'findAndSaveCorrectTestImg'):
                print('Running ' + inp)
                print(pred[0].shape)
                findAndSaveCorrectTestImg(pred, testing_features,
                                          testing_labels,
                                          './testPred/CorrectImg/',
                                          './testPred/IncorrectImg/',
                                          './testPred/TestInputs/', sess, sqn,
                                          scalingFac)

            if (inp == 'savegraphAndDataBatch'
                    or inp == 'testSingleTestInpAndSaveData'):
                imgFileName = 'SqNet_CIFAR_img.inp'
                weightsFileName = 'SqNet_CIFAR_weights.inp'
                for ii, curFeature in enumerate(testing_features):
                    if ii == 0:
                        DumpTFMtData.dumpImageDataInt(curFeature, imgFileName,
                                                      scalingFac, 'w')
                    else:
                        DumpTFMtData.dumpImageDataInt(curFeature, imgFileName,
                                                      scalingFac, 'a')
                DumpTFMtData.dumpTrainedWeightsInt(sess, sqn.all_weights,
                                                   weightsFileName, scalingFac,
                                                   'w')
Example #43
0
                    else:
                        # respond to the original requester
                        try:
                            self.transaction_map[transaction]["socket"].wfile.write(line + "\n")
                        except socket.error, e:
                            logger.warning("It can't write on requestor socket...")

                        # remove from map if end of transaction
                        if Util.get_var("(ackend)", line) != "":
                            logger.debug("Closing transaction: %s" % transaction)
                            if self.transaction_map.has_key(transaction):
                                del self.transaction_map[transaction]

                # assume we are a command request to an agent
                else:
                    id = Util.get_var("id=\"([^\"]+)\"", line)

                    if id == "" or id == "all":
                        logger.debug("Broadcasting to all ...")

                        if len(self.control_agents) == 0:
                            response = line + ' errno="-1" error="No agents available." ackend\n'

                        else:
                            # send line to each control agent
                            for key in self.control_agents:

                                # add this connection to the transaction map
                                transaction = self.__transaction_id_get()
                                self.transaction_map[transaction] = {'socket':requestor, 'time':time.time()}
Example #44
0
    npUrl = ""
    source = {''}
    SAnews = []

    for t in type_:
        logFile = logPath.format(t, datetime.now().strftime('%Y%m%d'))
        payload["channel_id"] = t

        DateStr = '2020-12-01'
        startDate = datetime.strptime(DateStr, '%Y-%m-%d')

        for i in range(0, 1):
            #nextDate = startDate + relativedelta(months=1)
            nextDate = startDate + relativedelta(days=1)
            Util.writeLog(
                logFile, "StartTime:{0} & EndTime:{1}".format(
                    startDate.strftime('%Y-%m-%d'),
                    nextDate.strftime('%Y-%m-%d')))

            payload['startDate'] = startDate.strftime('%Y-%m-%d')
            payload['endDate'] = nextDate.strftime('%Y-%m-%d')
            payload['page'] = 1

            res = requests.post('https://api.infominer.io/search',
                                json=payload,
                                headers=headers,
                                verify=False,
                                timeout=30)
            jd = res.json()
            pageNum = math.ceil(jd["total_count"] / 100)

            data = jd["data"]
Example #45
0
def parse(path, block_index, tx_index):

    print('Parsing tx to JSON file', path, '. . .')
    time_start = time.time()  # get starting time

    with open(path,
              'w') as json_file:  # open the output (json) file in writing mode

        with open(
                block_index.path, 'rb'
        ) as block_file:  # open the input (.dat) file in reading binary mode

            tx_ins = []
            tx_outs = []

            in_prev_hash = []
            in_prev_index = []

            out_address = []
            out_index = []
            out_value = []

            current_block = -1

            for i in range(0, len(tx_index.byte_index)
                           ):  # iterate over all the tx in the tx index
                # clearing
                tx_ins.clear()
                tx_outs.clear()

                in_prev_hash.clear()
                in_prev_index.clear()

                out_address.clear()
                out_value.clear()

                # parsing
                if current_block != tx_index.block_num[
                        i]:  # parsing timestamp (same timestamp for tx in the same block)
                    current_block = tx_index.block_num[i]
                    block_file.seek(
                        block_index.byte_index[current_block] + 76
                    )  # magic num : 4, block size : 4, version : 4, previous hash : 32, merkle root : 32
                    byte_time = block_file.read(4)  # block timestamp : 4
                    tx_timestamp = int.from_bytes(byte_time,
                                                  byteorder='little')

                block_file.seek(tx_index.byte_index[i])  # parsing tx hash
                byte_tx = block_file.read(tx_index.tx_size[i])
                h_sha256 = hashlib.sha256()
                h_sha256.update(byte_tx)
                h_bytes = h_sha256.digest()
                h_sha256 = hashlib.sha256()
                h_sha256.update(h_bytes)
                tx_id_hash = h_sha256.hexdigest()
                tx_id = Util.formatHashString(tx_id_hash, True, True)
                if len(tx_id) != 66: print(tx_id_hash, tx_id)

                block_file.seek(
                    tx_index.byte_index[i] +
                    4)  # parsing number of tx inputs, tx version : 4
                byte_in_count = block_file.read(1)  # reading the INPUT COUNT

                # check if varInt is on 1, 2, 4, or 8 bytes
                if (byte_in_count[0] == 253
                    ):  # varInt is on 2 bytes AFTER the prefix
                    byte_in_count = block_file.read(2)

                elif (byte_in_count[0] == 254
                      ):  # varInt is on 4 bytes AFTER the prefix
                    byte_in_count = block_file.read(4)

                elif (byte_in_count[0] == 255
                      ):  # varInt is on 8 bytes AFTER the prefix
                    byte_in_count = block_file.read(8)

                # else: # varInt was on 1 bytes, nothing to do

                in_count = int.from_bytes(byte_in_count, byteorder='little')
                ins = 0
                while ins < in_count:
                    byte_hash = block_file.read(32)  # previous tx hash : 32
                    first_hash = hex(int.from_bytes(byte_hash,
                                                    byteorder='big'))
                    prev_hash = Util.formatPrevHashString(
                        first_hash[2:], True, True)
                    if (len(prev_hash) != 66 and len(prev_hash) != 2):
                        print(byte_hash, first_hash, prev_hash)
                    in_prev_hash.append(prev_hash)

                    byte_prev_index = block_file.read(
                        4)  # previous tx index : 4
                    prev_index = int.from_bytes(byte_prev_index,
                                                byteorder='little')
                    in_prev_index.append(prev_index)

                    byte_script_len = block_file.read(
                        1)  # reading the INPUT COUNT

                    # check if varInt is on 1, 2, 4, or 8 bytes
                    if (byte_script_len[0] == 253
                        ):  # varInt is on 2 bytes AFTER the prefix
                        byte_script_len = block_file.read(2)

                    elif (byte_script_len[0] == 254
                          ):  # varInt is on 4 bytes AFTER the prefix
                        byte_script_len = block_file.read(4)

                    elif (byte_script_len[0] == 255
                          ):  # varInt is on 8 bytes AFTER the prefix
                        byte_script_len = block_file.read(8)

                    # else: # varInt was on 1 bytes, nothing to do

                    script_len = int.from_bytes(byte_script_len,
                                                byteorder='little')
                    block_file.read(script_len)  # script : script_len
                    block_file.read(4)  # sequence : 4
                    ins += 1

                byte_out_count = block_file.read(1)  # reading the OUTPUT COUNT

                # check if varInt is on 1, 2, 4, or 8 bytes
                if (byte_out_count[0] == 253
                    ):  # varInt is on 2 bytes AFTER the prefix
                    byte_out_count = block_file.read(2)

                elif (byte_out_count[0] == 254
                      ):  # varInt is on 4 bytes AFTER the prefix
                    byte_out_count = block_file.read(4)

                elif (byte_out_count[0] == 255
                      ):  # varInt is on 8 bytes AFTER the prefix
                    byte_out_count = block_file.read(8)

                # else: # varInt was on 1 bytes, nothing to do

                out_count = int.from_bytes(byte_out_count, byteorder='little')
                outs = 0
                while outs < out_count:
                    byte_value = block_file.read(8)  # value : 8
                    value = int.from_bytes(byte_value, byteorder='little')
                    out_value.append(value)

                    out_index.append(outs)

                    byte_script_len = block_file.read(
                        1)  # reading the INPUT COUNT

                    # check if varInt is on 1, 2, 4, or 8 bytes
                    if (byte_script_len[0] == 253
                        ):  # varInt is on 2 bytes AFTER the prefix
                        byte_script_len = block_file.read(2)

                    elif (byte_script_len[0] == 254
                          ):  # varInt is on 4 bytes AFTER the prefix
                        byte_script_len = block_file.read(4)

                    elif (byte_script_len[0] == 255
                          ):  # varInt is on 8 bytes AFTER the prefix
                        byte_script_len = block_file.read(8)

                    # else: # varInt was on 1 bytes, nothing to do

                    script_len = int.from_bytes(byte_script_len,
                                                byteorder='little')
                    byte_script = block_file.read(
                        script_len)  # script : script_len
                    script = Util.intToHexString(
                        int.from_bytes(byte_script, byteorder='big'), False,
                        False)

                    address_hex = Util.getDataFromHexStringScript(script)
                    if (len(address_hex) == 130):
                        address_hex = Util.pubKStringToAddress(address_hex)
                    elif (len(address_hex) == 40):
                        address_hex = Util.base58Check(address_hex)
                    else:
                        address_hex = 'UNABLE_TO_PARSE_ADDRESS'

                    out_address.append(address_hex)
                    outs += 1

                tx_total_value = 0
                for outs in range(0, len(out_value)):
                    tx_total_value += out_value[outs]  # sum of outs values

                for ins in range(0, len(in_prev_hash)):
                    if in_prev_hash[ins] == '0x':
                        tx_ins.append(
                            JsonTx.JsonTxIn('MINING_REWARD', in_prev_hash[ins],
                                            -1, tx_total_value))
                    elif len(in_prev_hash) == 1:
                        tx_ins.append(
                            JsonTx.JsonTxIn('NO_ADDRESS', in_prev_hash[ins],
                                            in_prev_index[ins], tx_total_value)
                        )  # if there is only 1 input its value is the total value of the tx
                    else:
                        if len(in_prev_hash[ins]) != 66:
                            print(in_prev_hash[ins], len(in_prev_hash[ins]))
                        tx_ins.append(
                            JsonTx.JsonTxIn('NO_ADDRESS', in_prev_hash[ins],
                                            in_prev_index[ins], 'NO_VALUE'))
                for outs in range(0, len(out_value)):
                    tx_outs.append(
                        JsonTx.JsonTxOut(out_address[outs], out_index[outs],
                                         out_value[outs]))
                tx = JsonTx.JsonTx('NO_X_RATE', tx_timestamp, tx_total_value,
                                   tx_ins, tx_outs, tx_id)

                json_str = json.dumps(tx, default=jsonDefault)
                json_file.write(json_str + '\n')

        block_file.closed  # close the file

    json_file.closed  # close the file
    time_end = time.time()
    print('end of parsing in', time_end - time_start, 's')
Example #46
0
    def process(self, requestor, command, line):
        """Process all the requests coming from the webgui
        """
        self.__mutexRquest.acquire()
        try:
            response = ""
            action = Util.get_var("action=\"([^\"]+)\"", line)

            if action == "connect":
                id = Util.get_var("id=\"([^\"]+)\"", line)
                sensor_id = Util.get_var("sensor_id=\"([0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12})\"", line)
                if id != "":
                    requestor.set_id(id)
                else:
                    requestor.set_id("%s_%i" % (requestor.client_address))
                requestor.set_sensorID(sensor_id)
                logger.debug("Adding control agent %s to the list., with key %s [%s]" % (id, requestor.getRequestorIP(),requestor.get_sensorID()))
                # add this connection to our control agent collection
                self.control_agents[requestor.getRequestorIP()] = requestor
                # indicate we're good to go
                response = 'ok id="%s"\n' % id
                self.__ntop_apache_manager.refreshDefaultNtopConfiguration(first_sensor_name=id, must_reload=True)
                timer = Timer(5.0, self.refreshAgentCache, (requestor, requestor.getRequestorIP(), id,))
                timer.start()
                timer = Timer(10.0, self.refreshAgentInventoryTasks, (requestor, requestor.getRequestorIP(), id,))
                timer.start()

            elif action == "getconnectedagents":

                # set up response
                response = "control getconnectedagents"

                # indicate the number of agents connected
                keys = self.control_agents.keys()
                response += ' count="%d"' % len(keys)

                # build the connected list
                if keys != None:
                    # sort list before sending
                    keys.sort()
                    names = ""
                    for key in keys:
                        names += "%s=%s|" % (self.control_agents[key].getRequestorID(), key)
                    names = names[:-1]
                else:
                    names = ""

                response += ' names="%s" errno="0" error="Success." ackend\n' % names

            elif action == "gettotalagentsconnected":

                # set up response
                response = "control gettotalagentsconnected"

                # indicate the number of agents connected
                keys = self.control_agents.keys()
                response += ' count="%d" errno="0" error="Success." ackend\n' % len(keys)

            elif action == "getconnectedagents_p":

                # set up response
                response = "control getconnectedagents_p"
                begin_index = 0 
                end_index = 0 
                try:
                    begin_index = int(Util.get_var("begin_index=\"([^\"]+)\"", line))
                except:
                    begin_index = 0
                try:
                    end_index = int(Util.get_var("end_index=\"([^\"]+)\"", line))
                except:
                    end_index = 0

                pag_size = end_index - begin_index
                real_size = 0
                if pag_size > 0:
                    # indicate the number of agents connected
                    keys = self.control_agents.keys()
                    response += ' count="%d"' % len(keys)
                    # build the connected list
                    if keys != None:
                        # sort list before sending
                        keys.sort()
                        if end_index >= len(keys):
                            page_keys = keys[begin_index:]
                        else:
                            page_keys = keys[begin_index:end_index]
                        real_size = len(page_keys)
                        names = ""
                        for key in page_keys:
                            names += "%s=%s|" % (self.control_agents[key].getRequestorID(), key)
                        names = names[:-1]
                        #names = "|".join(page_keys)
                    else:
                        names = ""
                    response += ' page_size="%d" names="%s" errno="0" error="Success." ackend\n' % (real_size, names)
                else:
                    response += 'errno="-1" error="Invalid page size requested." ackend\n' 
            elif action == "refresh_asset_list":
                for agent_id in self.control_agents.keys():
                    self.refreshAgentCache(self.control_agents[agent_id], self.control_agents[agent_id].getRequestorIP(), self.control_agents[agent_id].getRequestorID())
            elif action == "refresh_inventory_task":
                for agent_id in self.control_agents.keys():
                    self.refreshAgentInventoryTasks(self.control_agents[agent_id], self.control_agents[agent_id].getRequestorIP(), self.control_agents[agent_id].getRequestorID())
                response="%s ok\n" % line
            elif action == "getOCSInventory":
                logger.info("Request for ocs inventory")
                response = self.getOCSInventoryData()
            elif action == "getNagiosInventory":
                logger.info("Request for ocs inventory")
                response = self.getNagiosInventory()
            else:
                # check if we are a transaction
                transaction = Util.get_var("transaction=\"([^\"]+)\"", line)
                if transaction != "":
                    if transaction not in self.transaction_map:
                        logger.error("Transaction %s has no apparent originator!", transaction)

                    else:
                        # respond to the original requester
                        try:
                            self.transaction_map[transaction]["socket"].wfile.write(line + "\n")
                        except socket.error, e:
                            logger.warning("It can't write on requestor socket...")

                        # remove from map if end of transaction
                        if Util.get_var("(ackend)", line) != "":
                            logger.debug("Closing transaction: %s" % transaction)
                            if self.transaction_map.has_key(transaction):
                                del self.transaction_map[transaction]

                # assume we are a command request to an agent
                else:
Example #47
0
def MultiPO(Board, Moderators, DaysAgo):

    global author_list
    global ip_list
    global new_line
    global ptt_bot
    global publish
    global ask
    global mail

    Util.PTTBot = PTTBot
    Util.post_search = None
    Util.current_board = Board
    Util.Moderators = Moderators

    global publish_content
    if PublishContent is None:

        PublishContent = '此內容由 IP 分析程式產生' + NewLine
        PublishContent += '由 CodingMan 透過 PTT Library 開發,' + NewLine * 2

        PublishContent += 'PTT Library: https://github.com/Truth0906/PTTLibrary' + NewLine
        PublishContent += '開發手冊: https://hackmd.io/@CodingMan/PTTLibraryManual' + NewLine
        PublishContent += 'IP 分析程式: https://github.com/Truth0906/PTTModeratorTool' + NewLine

    StartTime = time.time()
    AuthorList = dict()
    IPList = dict()
    CurrentDate = Util.get_date(DaysAgo)

    PTTBot.log(f'開始 {Board} 板 IP 分析')
    PTTBot.log(f'從 {CurrentDate} 開始分析')

    Start, _ = Util.find_post_rrange(DaysAgo, show=False)
    NewestIndex = PTTBot.getNewestIndex(
        PTT.IndexType.Board,
        Board=Board,
    )
    End = NewestIndex
    PTTBot.log('編號範圍 ' + str(Start) + ' ~ ' + str(End))

    ErrorPostList, DeleteCount = PTTBot.crawlBoard(
        PostHandler,
        Util.current_board,
        StartIndex=Start,
        EndIndex=End,
    )

    MultiPOResult = ''
    for Suspect, TitleAuthorList in AuthorList.items():

        if len(TitleAuthorList) <= 1:
            continue
        # print('=' * 5 + ' ' + Suspect + ' ' + '=' * 5)

        if MultiPOResult != '':
            MultiPOResult += NewLine
        for Title in TitleAuthorList:
            MultiPOResult += CurrentDate + ' ' + \
                Suspect + ' □ ' + Title + NewLine

    IPResult = ''
    for IP, SuspectList in IPList.items():
        # print('len:', len(SuspectList))
        if len(SuspectList) <= 1:
            continue

        # print('IP:', IP)
        IPResult += 'IP: ' + IP + NewLine

        for Line in SuspectList:
            # print('>   ' + CurrentDate + ' ' + Line)
            IPResult += CurrentDate + ' ' + Line + NewLine

    EndTime = time.time()

    Title = CurrentDate + f' {Board} 板 IP 分析結果'

    PublishContent += NewLine
    PublishContent += '◆ ' + CurrentDate + f' {Board} 板 IP 分析結果'

    Time = math.ceil(EndTime - StartTime)
    Min = int(Time / 60)
    Sec = int(Time % 60)

    Content = '此內容由自動抓多 PO 程式產生' + NewLine

    Content += '共耗時'
    PublishContent += '共耗時'
    if Min > 0:
        Content += f' {Min} 分'
        PublishContent += f' {Min} 分'
    Content += f' {Sec} 秒執行完畢' + NewLine * 2
    PublishContent += f' {Sec} 秒執行完畢' + NewLine * 2

    Content += '此程式是由 CodingMan 透過 PTT Library 開發,' + NewLine * 2
    Content += f'蒐集範圍為 ALLPOST 搜尋 ({Board}) 情況下編號 ' + \
        str(Start) + ' ~ ' + str(End) + NewLine
    Content += f'共 {End - Start + 1} 篇文章' + NewLine * 2

    PublishContent += f'    蒐集範圍為 ALLPOST 搜尋 ({Board}) 情況下編號 ' + \
        str(Start) + ' ~ ' + str(End) + NewLine
    PublishContent += f'    共 {End - Start + 1} 篇文章' + NewLine * 2

    if MultiPOResult != '':
        Content += MultiPOResult

        MultiPOResult = MultiPOResult.strip()
        for line in MultiPOResult.split(NewLine):
            PublishContent += '    ' + line + NewLine
    else:
        Content += '◆ ' + CurrentDate + ' 無人違反多 PO 板規' + NewLine
        PublishContent += '    ' + '◆ ' + CurrentDate + ' 無人違反多 PO 板規' + NewLine

    if IPResult != '':
        Content += IPResult
        IPResult = IPResult.strip()
        for line in IPResult.split(NewLine):
            PublishContent += '    ' + line + NewLine
    else:
        Content += NewLine + f'◆ 沒有發現特定 IP 有 {MaxPost + 1} 篇以上文章' + NewLine
        PublishContent += NewLine + \
            f'    ◆ 沒有發現特定 IP 有 {MaxPost + 1} 篇以上文章' + NewLine

    Content += NewLine + '內容如有失準,歡迎告知。' + NewLine
    Content += '此訊息同步發送給 ' + ' '.join(Util.Moderators) + NewLine
    Content += NewLine
    Content += ID

    print(Title)
    print(Content)

    if Ask:
        Choise = input('要發佈嗎? [Y]').lower()
        Publish = (Choise == 'y') or (Choise == '')

    if Mail:
        for Moderator in Util.Moderators:
            PTTBot.mail(Moderator, Title, Content, 0)
            PTTBot.log('寄信給 ' + Moderator + ' 成功')
    else:
        PTTBot.log('取消寄信')
Example #48
0
def apply(x):
    if x < 3:
        mode = 0
        while mode != 1 and mode != 2:
            mode = int(
                input("Mode de (dé)chiffrement : \n\t1. ECB\n\t2. CBC\n"))
        key_len = 0
        while (key_len != 256) and (key_len != 512) and (key_len != 1024):
            key_len = int(input("Taille de la clé en bits(256/512/1024): "))
        file_key = input("Chemin du fichier de la clé: ")
        passwd_user = input("Mot de passe pour chiffrer la clé: ")
        file_path = input("Chemin du fichier à (dé)chiffrer : ")
        word_len = 64
        num_words = int(key_len / word_len)
        word_len_bytes = int(word_len / 8)

        if x == 1:
            file_data = IO.readfile(file_path, word_len, 1)
            file_data_list = Util.organize_data_list(file_data, num_words)
            encrypted_file = ThreeFish.threefish_chiffrement(
                file_data_list, mode, key_len, passwd_user, file_key)
            IO.write_2D_list(file_path, encrypted_file, word_len_bytes)
            IO.rename_file(file_path, 0)

            print("Chiffrement terminé.")

        elif x == 2:
            ciph_data = IO.readfile(file_path, word_len, 0)
            ciph_data_list = Util.organize_data_list(ciph_data, num_words)
            clear_file_data, valeur_pad = ThreeFish.threefish_dechiffrement(
                ciph_data_list, mode, key_len, word_len, passwd_user, file_key)
            IO.write_file_list_pad(file_path, clear_file_data, word_len_bytes,
                                   valeur_pad)
            IO.rename_file(file_path, 1)

            print("Déchiffrement terminé.")

    elif x == 3:
        filepath = input("Chemin du fichier à chiffrer:")
        file_data = IO.read_bytes(filepath)
        ans = ''
        while ans != 'y' and ans != 'n' and ans != 'Y' and ans != 'N':
            ans = input("Avez-vous un fichier de clé publique ? (y/n) ")
        if ans == 'y' or ans == 'Y':
            keypath = input("Chemin de la clé publique: ")
            ciph_data = CramerShoup.encode_with_key(file_data, keypath)
        else:
            k = int(input("Taille de clé souhaitée en bits: "))
            password = input("Mot de passe pour chiffrer la clé privée: ")
            keypath = input("Chemin du répertoire des clés: ")
            ciph_data = CramerShoup.encode_no_key(file_data, keypath, k,
                                                  password)

        ciph_bytes = Conversions.int_list2bytes(ciph_data, 8)
        IO.write_bytes(filepath, ciph_bytes)
        IO.rename_file(filepath, 0)

        print("Chiffrement terminé.")

    elif x == 4:
        filepath = input("Chemin du fichier à déchiffrer: ")
        file_data = IO.read_bytes(filepath)
        keypath = input("Chemin de la clé privée: ")
        password = input("Mot de passe: ")

        ciph_data = Conversions.bytes2int_list(file_data, 8)
        clear_data = CramerShoup.decode(ciph_data, keypath, password)
        IO.write_bytes(filepath, clear_data)
        IO.rename_file(filepath, 1)

        print("Déchiffrement terminé.")

    elif x == 5:
        hash_len = 0
        while hash_len != 256 and hash_len != 512 and hash_len != 1 and hash_len != 2:
            hash_len = int(input("1. BLAKE-256\n2. BLAKE-512\n"))
        if hash_len < 256:
            hash_len *= 32
        else:
            hash_len //= 8

        filepath = input("Chemin du fichier: ")
        with open(filepath, 'r') as rfile:
            file_data = rfile.read()

        ans = ''
        while ans != 'y' and ans != 'n' and ans != 'Y' and ans != 'N':
            ans = input("Protéger l'empreinte ? (y/n) ")

        key = ''
        if ans == 'y' or ans == 'Y':
            key = input("Mot de passe: ")

        h = hex(Hash.blake_hash(file_data, hash_len, key)) + '\n'

        # Rename file : "name.ext" -> "name~hash.ext"
        path = filepath.split('/')
        last = len(path) - 1
        filename = path[last].split('.')
        filename[0] += '~hash'
        path[last] = '.'.join(filename)
        filepath = '/'.join(path)

        with open(filepath, 'w') as wfile:
            wfile.write(h)

    elif x == 6:
        hash_len = 0
        while hash_len != 256 and hash_len != 512 and hash_len != 1 and hash_len != 2:
            hash_len = int(input("1. BLAKE-256\n2. BLAKE-512\n"))
        if hash_len < 256:
            hash_len *= 32
        else:
            hash_len //= 8

        filepath = input("Chemin du fichier: ")
        with open(filepath, 'r') as rfile:
            file_data = rfile.read()

        hashpath = input("Chemin du hash: ")
        with open(hashpath, 'r') as hfile:
            hash = hfile.read()

        ans = ''
        while ans != 'y' and ans != 'n' and ans != 'Y' and ans != 'N':
            ans = input("Empreinte protégée ?(y/n) ")

        key = ''
        if ans == 'y' or ans == 'Y':
            key = input("Mot de passe: ")

        h = hex(Hash.blake_hash(file_data, hash_len, key)) + '\n'

        if h == hash:
            print("Les empreintes sont égales.")
        else:
            print("Les empreintes ne correspondent pas.")
Example #49
0
    "D:\\Andre\\GitHub\\alabliuk\\Alura\\Data Science\\A coleta de dados\\data\\ufo.csv"
)

# Linhas contendo NaN nas colunas City, State ou Time devem ser eliminadas
df_ufo.dropna(axis=0,
              how='any',
              subset=['City', 'State', 'Time'],
              inplace=True)
# Linhas contendo NaN em ambas colunas Colors Reported e Shape Reported devem ser eliminadas
df_ufo.dropna(axis=0,
              how='all',
              subset=['Colors Reported', 'Shape Reported'],
              inplace=True)

# Conexão:
cliente_MongoDB_db_clima = Util.fnc_Conecta_Base_Documentos(
    '', '', '192.168.0.113', '27017', 'dbclima')
db_clima = cliente_MongoDB_db_clima.dbclima

# Inicia Caixa de Areia, destino das linhas cujos valores (cidade, estado, instante) não constam na base clima
caixa_de_areia = []

for index, row in df_ufo.iterrows():
    cidade = row["City"]
    estado = row["State"]
    instante = row["Time"]
    # Esmiuçando "Time" em: mês, dia, ano, hora e minuto. Exemplo de data: 8/21/2004 00:05
    # Temos que garantir o tamanho 2 para cada item.
    # Exemplo: se ler dia "8", precisamos transformá-lo em "08"
    # Para melhorar o percentual de busca, vamos desconsiderar minutos
    x = re.findall('\d+', instante)  # Extraímos os 5 números
    mes = x[0].zfill(2)
#coding: utf-8
__author__ = 'ssm'

import numpy as np
import datetime
from sklearn.cluster import AffinityPropagation, KMeans
from sklearn.metrics.pairwise import cosine_similarity
import Util
import redis

#redis
r = redis.StrictRedis(host='10.2.1.7', port=6379, db=0)

#get 1st subject words
words = Util.cluster_2nd_load_1st_words("")
#get vecs
vecs = Util.cluster_2nd_load_1st_vecs(r, words)

print len(vecs)


def test_ap_clustering(wrds):
    words = wrds[:-len(wrds) / 5]
    #get similarities
    similarities = [[0 for i in range(len(words))] for j in range(len(words))]
    for ci in range(len(words)):
        for cj in range(ci, len(words)):
            similarities[ci][cj] = cosine_similarity([vecs[ci]],
                                                     [vecs[cj]])[0][0]
            similarities[cj][ci] = similarities[ci][cj]
        if ci % 100 == 0:
Example #51
0
 def axisAngle(axis, angle):
     '''Creates a rotation which rotates angle degrees around axis.'''
     return Quaternion.axisAngleInRadian(axis, Util.degreeToRadian(angle))
Example #52
0
        # ('Gossiping', ['Bignana'], 1),
        ('Wanted', ['gogin'], 1),
        ('HatePolitics', ['Neptunium', 'mark2165', 'kero2377'], 1),
    ]

    try:
        with open('Account.txt') as AccountFile:
            Account = json.load(AccountFile)
            ID = Account['ID']
            Password = Account['Password']
    except FileNotFoundError:
        ID = input('請輸入帳號: ')
        Password = getpass.getpass('請輸入密碼: ')

    ptt_bot.login(ID, Password, KickOtherLogin=True)

    for (current_board, ModeratorList, MaxPost) in SearchList:
        MultiPO(current_board, ModeratorList, MaxPost)

    publish_content += new_line + '內容如有失準,歡迎告知。' + new_line
    publish_content += 'CodingMan'

    if publish:
        CurrentDate = Util.get_date(1)

        ptt_bot.post('Test', CurrentDate + ' 多 PO 結果', publish_content, 1, 0)
        ptt_bot.log('在 Test 板發文成功')
    else:
        ptt_bot.log('取消備份')
    ptt_bot.logout()
Example #53
0
    def evict(
        self, leaf
    ):  # returns list of the blocks that go in each node on the path as a 2d list, should compare IDs and return if found as well
        numLevels = Util.levelNumber(leaf) + 1
        result = [0] * numLevels
        for i in range(numLevels):
            result[i] = [Block.Block(0, 0, b"")] * self._z

        stashIter = 0
        full = math.pow(
            2, numLevels
        ) - 1  # "full vector" which has numLevel digits, and 1 means bucket has room for more
        pathVec = [0] * numLevels  # holds number of blocks in bucket occupied

        while stashIter < len(
                self._nodes
        ):  # put nodes in the list where 0th element is 0th level, etc.
            if self._oldStash == False:
                """if leaf == 1:
                #print()
                #print("root")
                bucketPos = pathVec[0]
                result[0][bucketPos] = self._nodes[stashIter]
                self.deleteNode(stashIter)
                pathVec[0] += 1
                if pathVec[0] == self._z:
                    full -= math.pow(2, 0)

                else:"""
                legalLevels = Util.getMaxLevel(
                    leaf, self._nodes[stashIter].getLeaf(
                    ))  # gets the number that tells which levels are allowed
                #print(str(leaf) + ", " + str(self._nodes[stashIter].getLeaf()))
                #print("legalLevels: "  + str(legalLevels))
                availBuc = int(
                    full
                ) & legalLevels  # gets number that shows which buckets are available and allowed
                #print("availBuc: " + str(availBuc))
                if availBuc == 0:  # if it's 0, then there are no available ones, we move on
                    print("no available")
                    stashIter += 1
                else:
                    print("available")
                    bucketLevel = int(math.log(availBuc, 2))
                    #print("bucketLevel: " + str(bucketLevel))
                    bucketPos = pathVec[bucketLevel]
                    result[bucketLevel][bucketPos] = self._nodes[stashIter]
                    self.deleteNode(stashIter)
                    pathVec[bucketLevel] += 1
                    if pathVec[bucketLevel] == self._z:
                        full -= math.pow(2, bucketLevel)

            else:
                curLevel = Util.getMaxLevel(leaf,
                                            self._nodes[stashIter].getLeaf())
                while curLevel > -1:
                    if pathVec[curLevel] < self._z:
                        result[curLevel][
                            pathVec[curLevel]] = self._nodes[stashIter]
                        self.deleteNode(stashIter)
                        pathVec[curLevel] += 1
                        break
                    curLevel -= 1
                if curLevel == -1:
                    stashIter += 1

        return result
Example #54
0
  def __init__(self, fn_in, time_begin, time_end, overloaded):
    self.overloaded = overloaded

    # Unzip when the file is not there
    if not os.path.exists(fn_in):
      fn_zipped = "%s.bz2" % fn_in
      if not os.path.exists(fn_zipped):
        raise RuntimeError("Unexpected: %s" % fn_in)
      Util.RunSubp("cd %s && bzip2 -dk %s > /dev/null" % (os.path.dirname(fn_zipped), os.path.basename(fn_zipped)))
    if not os.path.exists(fn_in):
      raise RuntimeError("Unexpected")
    #Cons.P(fn_in)

    mo_list = []
    line_params = None
    line_run = None
    with open(fn_in) as fo:
      for line in fo:
        #Cons.P(line)
        # 2017-10-13 20:41:01:258 2 sec: 34 operations; 34 current ops/sec; est completion in 68 days 1 hours [READ: Count=28, Max=46943, Min=33,
        # Avg=32239.54, 90=45343, 99=46943, 99.9=46943, 99.99=46943] [INSERT: Count=8, Max=9343, Min=221, Avg=4660.88, 90=8695, 99=9343, 99.9=9343,
        # 99.99=9343]
        mo = re.match(r"\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d:\d\d\d (?P<rel_time>\d+) sec: \d+ operations; " \
            "(?P<db_iops>(\d|\.)+) current ops\/sec; .*" \
            "\[READ: Count=(?P<r_cnt>\d+), Max=(?P<r_max>\d+), Min=(?P<r_min>\d+), Avg=(?P<r_avg>(\d|\.)+)," \
            " 90=(?P<r_90>\d+), 99=(?P<r_99>\d+), 99.9=(?P<r_999>\d+), 99.99=(?P<r_9999>\d+)\] " \
            "\[INSERT: Count=(?P<w_cnt>\d+), Max=(?P<w_max>\d+), Min=(?P<w_min>\d+), Avg=(?P<w_avg>(\d|\.)+)," \
            " 90=(?P<w_90>\d+), 99=(?P<w_99>\d+), 99.9=(?P<w_999>\d+), 99.99=(?P<w_9999>\d+)\] " \
            , line)
        if mo is None:
          continue

        total_seconds = int(mo.group("rel_time"))
        s = total_seconds % 60
        total_seconds -= s
        total_mins = total_seconds / 60
        m = total_mins % 60
        total_mins -= m
        h = total_mins / 60
        rel_time = "%02d:%02d:%02d" % (h, m, s)
        if (time_begin <= rel_time) and (rel_time <= time_end):
          mo_list.append((rel_time, mo))
    if len(mo_list) == 0:
      raise RuntimeError("Unexpected. Check file [%s]" % fn_in)

    cnt = 0
    db_iops = []
    r_cnt = 0
    r_avg = 0.0
    r_min = 0
    r_max = 0
    r_90  = 0
    r_99  = 0
    r_999 = 0
    r_9999 = 0
    w_cnt = 0
    w_avg = 0.0
    w_min = 0
    w_max = 0
    w_90  = 0
    w_99  = 0
    w_999 = 0
    w_9999 = 0
    for e in mo_list:
      rel_time = e[0]
      mo = e[1]
      db_iops.append(float(mo.group("db_iops")))
      r_cnt   += int(mo.group("r_cnt"))
      r_avg   += float(mo.group("r_avg"))
      r_min   += int(mo.group("r_min"))
      r_max   += int(mo.group("r_max"))
      r_90    += int(mo.group("r_90"))
      r_99    += int(mo.group("r_99"))
      r_999   += int(mo.group("r_999"))
      r_9999  += int(mo.group("r_9999"))
      w_cnt   += int(mo.group("w_cnt"))
      w_avg   += float(mo.group("w_avg"))
      w_min   += int(mo.group("w_min"))
      w_max   += int(mo.group("w_max"))
      w_90    += int(mo.group("w_90"))
      w_99    += int(mo.group("w_99"))
      w_999   += int(mo.group("w_999"))
      w_9999  += int(mo.group("w_9999"))
      cnt += 1

    self.db_iops_stat = Stat.Gen(db_iops)

    self.r_cnt  = r_cnt
    self.r_avg  = (float(r_avg ) / cnt)
    self.r_min  = (float(r_min ) / cnt)
    self.r_max  = (float(r_max ) / cnt)
    self.r_90   = (float(r_90  ) / cnt)
    self.r_99   = (float(r_99  ) / cnt)
    self.r_999  = (float(r_999 ) / cnt)
    self.r_9999 = (float(r_9999) / cnt)
    self.w_cnt  = (float(w_cnt ) / cnt)
    self.w_avg  = (float(w_avg ) / cnt)
    self.w_min  = (float(w_min ) / cnt)
    self.w_max  = (float(w_max ) / cnt)
    self.w_90   = (float(w_90  ) / cnt)
    self.w_99   = (float(w_99  ) / cnt)
    self.w_999  = (float(w_999 ) / cnt)
    self.w_9999 = (float(w_9999) / cnt)
    def get_seqs_bfr_aft_trgt_idx(self, data_dict, init):
        chr_path = init[0]
        max_len = init[1]
        win_len = init[2]
        result_dict = {}

        util = Util.Utils()
        for chr_key, val_list in data_dict.items():
            result_dict.update({chr_key: {}})

            n3_after_strd_dict = {}
            n_len_after_strd_dict = {}
            n_mx_len_after_strd_dict = {}
            with open(chr_path + chr_key + ".fa", "r") as f:
                header = f.readline()  # header ignored : >chr19
                print("header : " + header)

                idx = 1
                full_cover_p_str = ""
                full_cover_m_str = ""

                while True:
                    c = f.read(1)
                    if c == "":
                        print(str(idx))
                        break
                    if "\n" in c:
                        continue
                    elif "\r" in c:
                        continue

                    full_cover_p_str = full_cover_p_str + c.upper()
                    full_cover_m_str = full_cover_m_str + self.get_complementary(
                        c.upper())
                    p_m_str_n3_list = self.get_p_m_str_list_for_n3(
                        full_cover_p_str, full_cover_m_str, max_len, win_len)
                    p_m_str_list, full_cover_p_str, full_cover_m_str, n_len_str = self.get_p_m_str_list(
                        full_cover_p_str, full_cover_m_str, max_len, win_len)

                    # add forward seq into result_dict
                    result_dict = self.add_in_frm_forwrd_seq(
                        idx, chr_key, n3_after_strd_dict, result_dict,
                        p_m_str_n3_list)
                    result_dict = self.add_in_frm_forwrd_seq(
                        idx, chr_key, n_mx_len_after_strd_dict, result_dict,
                        p_m_str_list)
                    result_dict = self.add_out_frm_forwrd_seq(
                        idx, chr_key, n_len_after_strd_dict, result_dict,
                        p_m_str_list, n_len_str)

                    if idx in val_list:
                        for tmp_val in val_list[idx]:
                            Cellline = tmp_val[0]
                            Genome_Change = tmp_val[1]
                            tmp_key = Cellline + "^" + Genome_Change  # HCC827^g.chr1:915845C>A
                            Genomesequence = tmp_val[3]
                            cDNA_change = tmp_val[4]
                            Codon_Change = tmp_val[
                                6]  # Codon_Change : c.(223-225)Ggt>Tgt
                            Codon_Change_arr = Codon_Change.split(
                                ">")  # ['c.(223-225)Ggt', 'Tgt']

                            # check +/- strand
                            strnd = self.check_p_m_strand(
                                Genome_Change, cDNA_change)

                            if len(Codon_Change_arr) > 1:
                                mut_seq = Codon_Change_arr[1]  # mut_seq : Tgt
                                mut_char_arr = Genome_Change.split(">")
                                if len(mut_seq) == 3 and len(mut_char_arr) > 1:
                                    # TODO del
                                    mut_char = mut_char_arr[1]  # A
                                    # idx_mut_char = mut_seq.find(mut_char)

                                    if strnd == "-":
                                        forward, frwrd_idx, bckwrd_idx = self.get_frwrd_str_bckwrd_idx(
                                            mut_seq, p_m_str_n3_list, strnd)

                                        result_dict[chr_key].update({
                                            tmp_key:
                                            [strnd, bckwrd_idx + 3, forward]
                                        })

                                        if idx + 3 + max_len - frwrd_idx in n3_after_strd_dict:
                                            n3_after_strd_dict[
                                                idx + 3 + max_len -
                                                frwrd_idx].append(tmp_key)
                                        else:
                                            n3_after_strd_dict.update({
                                                idx + 3 + max_len - frwrd_idx:
                                                [tmp_key]
                                            })
                                    else:
                                        forward, frwrd_idx, bckwrd_idx = self.get_frwrd_str_bckwrd_idx(
                                            mut_seq, p_m_str_n3_list, strnd)

                                        result_dict[chr_key].update({
                                            tmp_key:
                                            [strnd, bckwrd_idx, forward]
                                        })

                                        if idx + 3 + max_len - frwrd_idx in n3_after_strd_dict:
                                            n3_after_strd_dict[
                                                idx + 3 + max_len -
                                                frwrd_idx].append(tmp_key)
                                        else:
                                            n3_after_strd_dict.update({
                                                idx + 3 + max_len - frwrd_idx:
                                                [tmp_key]
                                            })
                                else:
                                    # mut_seq's len is not 3
                                    end_seq = int(
                                        re.findall(
                                            r'\d+',
                                            Genome_Change[Genome_Change.
                                                          index(":"):])[1])
                                    # if Genomesequence has 'In_Frame'
                                    if 'In_Frame' in Genomesequence:
                                        idx_p_m_str_list = 0
                                        if strnd == "-":
                                            idx_p_m_str_list = 3
                                        forward_str = p_m_str_list[
                                            idx_p_m_str_list]
                                        adder = 1
                                        if 'ins'.upper(
                                        ) in Genome_Change.upper():
                                            adder = 0
                                            forward_str = full_cover_p_str[
                                                -max_len:]
                                            if strnd == '-':
                                                forward_str = full_cover_m_str[
                                                    -max_len:]

                                        result_dict[chr_key].update({
                                            tmp_key: [
                                                strnd, idx_p_m_str_list,
                                                forward_str
                                            ]
                                        })
                                        if end_seq + max_len + adder in n_mx_len_after_strd_dict:
                                            n_mx_len_after_strd_dict[
                                                end_seq + max_len +
                                                adder].append(tmp_key)
                                        else:
                                            n_mx_len_after_strd_dict.update({
                                                end_seq + max_len + adder:
                                                [tmp_key]
                                            })

                                    else:
                                        idx_p_m_str_list = 0
                                        if strnd == "-":
                                            idx_p_m_str_list = 3

                                        forward_str = n_len_str + p_m_str_list[
                                            idx_p_m_str_list]
                                        adder = 1
                                        if 'ins'.upper(
                                        ) in Genome_Change.upper():
                                            adder = 0
                                            forward_str = full_cover_p_str[
                                                -max_len + win_len:]
                                            if strnd == '-':
                                                forward_str = full_cover_m_str[
                                                    -max_len + win_len:]

                                        result_dict[chr_key].update({
                                            tmp_key: [
                                                strnd, idx_p_m_str_list,
                                                forward_str
                                            ]
                                        })
                                        if end_seq + max_len + win_len + adder in n_len_after_strd_dict:
                                            n_len_after_strd_dict[
                                                end_seq + max_len + win_len +
                                                adder].append(tmp_key)
                                        else:
                                            n_len_after_strd_dict.update({
                                                end_seq + max_len + win_len + adder:
                                                [tmp_key]
                                            })
                            elif "_" in Genome_Change:
                                # deli with '_'
                                end_seq = int(
                                    re.findall(
                                        r'\d+', Genome_Change[Genome_Change.
                                                              index(":"):])[1])
                                # if Genomesequence has 'In_Frame'
                                if 'In_Frame' in Genomesequence:
                                    idx_p_m_str_list = 0
                                    if strnd == "-":
                                        idx_p_m_str_list = 3
                                    forward_str = p_m_str_list[
                                        idx_p_m_str_list]
                                    adder = 1
                                    if 'ins'.upper() in Genome_Change.upper():
                                        adder = 0
                                        forward_str = full_cover_p_str[
                                            -max_len:]
                                        if strnd == '-':
                                            forward_str = full_cover_m_str[
                                                -max_len:]

                                    result_dict[chr_key].update({
                                        tmp_key:
                                        [strnd, idx_p_m_str_list, forward_str]
                                    })
                                    if end_seq + max_len + adder in n_mx_len_after_strd_dict:
                                        n_mx_len_after_strd_dict[
                                            end_seq + max_len +
                                            adder].append(tmp_key)
                                    else:
                                        n_mx_len_after_strd_dict.update({
                                            end_seq + max_len + adder:
                                            [tmp_key]
                                        })

                                else:
                                    idx_p_m_str_list = 0
                                    if strnd == "-":
                                        idx_p_m_str_list = 3

                                    forward_str = n_len_str + p_m_str_list[
                                        idx_p_m_str_list]
                                    adder = 1
                                    if 'ins'.upper() in Genome_Change.upper():
                                        adder = 0
                                        forward_str = full_cover_p_str[
                                            -max_len - win_len:]
                                        if strnd == "-":
                                            forward_str = full_cover_m_str[
                                                -max_len - win_len:]

                                    result_dict[chr_key].update({
                                        tmp_key:
                                        [strnd, idx_p_m_str_list, forward_str]
                                    })
                                    if end_seq + max_len + win_len + adder in n_len_after_strd_dict:
                                        n_len_after_strd_dict[
                                            end_seq + max_len + win_len +
                                            adder].append(tmp_key)
                                    else:
                                        n_len_after_strd_dict.update({
                                            end_seq + max_len + win_len + adder:
                                            [tmp_key]
                                        })

                            else:
                                # out of logic
                                end_seq = idx  # + 1
                                idx_p_m_str_list = 0
                                if strnd == "-":
                                    idx_p_m_str_list = 3

                                forward_str = n_len_str + p_m_str_list[
                                    idx_p_m_str_list]
                                adder = 1
                                if 'ins'.upper() in Genome_Change.upper():
                                    adder = 0
                                    forward_str = full_cover_p_str[-max_len -
                                                                   win_len:]
                                    if strnd == "-":
                                        forward_str = full_cover_m_str[
                                            -max_len - win_len:]

                                result_dict[chr_key].update({
                                    tmp_key:
                                    [strnd, idx_p_m_str_list, forward_str]
                                })
                                if end_seq + max_len + win_len + adder in n_len_after_strd_dict:
                                    n_len_after_strd_dict[
                                        end_seq + max_len + win_len +
                                        adder].append(tmp_key)
                                else:
                                    n_len_after_strd_dict.update({
                                        end_seq + max_len + win_len + adder:
                                        [tmp_key]
                                    })

                    idx = idx + 1

        return result_dict
Example #56
0
 def toAxisAngle(self):
     '''Converts a rotation to axis-angle representation(angle in degree).'''
     axis, angle = self.toAxisAngleInRadian()
     return axis, Util.radianToDegree(angle)
def _GenLocalSsdData(fn):
	fn_plot_data = "plot-data/%s" % fn
	fn_raw = "../log/ioping/%s" % fn

	# Uncompress the raw files if not exist
	if not os.path.isfile(fn_raw):
		with Cons.MeasureTime("file %s doesn't exist. uncompressing from the archive" % fn_raw):
			cmd = "7z x -o../log/ioping %s.7z" % fn_raw
			Util.RunSubp(cmd)

	# Generate one if not exist. Skip for now. It's fast enough.
	#if os.path.isfile(fn_plot_data):
	#	Cons.P("%s already exists" % fn_plot_data)
	#	return

	if not os.path.exists("plot-data"):
		os.makedirs("plot-data")

	with Cons.MeasureTime("Generating plot data %s" % fn_plot_data):
		with open(fn_raw) as fo_r, open(fn_plot_data, "w") as fo_w:
			fmt = "%13s" \
					" %5.1f" \
					" %3d %3d %5d %4d" \
					" %3d %3d %5d %3d" \
					" %6d"
			fo_w.write("%s\n" % Util.BuildHeader(fmt,
				"datetime" \
				" blocks_used_percent" \
				" rr_4k_min rr_4k_avg rr_4k_max rr_4k_sd" \
				" rw_4k_min rw_4k_avg rw_4k_max rw_4k_sd" \
				" seqw_100m"
				))

			# 160425-011752
			# 100 17430 5737 23499713 170 174 211 5
			# 100 14822 6747 27634597 134 148 231 17
			# 1 216553 5 484212179 216553 216553 216553 0
			# Filesystem     1K-blocks    Used Available Use% Mounted on
			# /dev/xvda1       8115168 2545256   5134636  34% /
			# /dev/xvdb      769018760  173080 768829296   1% /mnt/local-ssd1

			i = 0
			dt = None
			block_used_percent_prev = None
			blocks_total = None
			blocks_used  = None
			rr_4k_min = None
			rr_4k_avg = None
			rr_4k_max = None
			rr_4k_sd  = None
			rw_4k_min = None
			rw_4k_avg = None
			rw_4k_max = None
			rw_4k_sd  = None
			seqw_100m = None
			for line in fo_r.readlines():
				#Cons.P(line)
				if i == 0:
					dt = line.strip()
				# 4k rand read
				elif i == 1:
					t = line.split()
					if len(t) != 8:
						raise RuntimeError("Unexpected format [%s]" % line)
					# All in us
					rr_4k_min = int(t[4])
					rr_4k_avg = int(t[5])
					rr_4k_max = int(t[6])
					rr_4k_sd  = int(t[7])
				elif i == 2:
					t = line.split()
					if len(t) != 8:
						raise RuntimeError("Unexpected format [%s]" % line)
					# All in us
					rw_4k_min = int(t[4])
					rw_4k_avg = int(t[5])
					rw_4k_max = int(t[6])
					rw_4k_sd  = int(t[7])
				elif i == 3:
					t = line.split()
					if len(t) != 8:
						raise RuntimeError("Unexpected format [%s]" % line)
					# All of the min, avg, max are the same since there is only 1 request
					# at a time.
					seqw_100m = int(t[4])
				elif i == 6:
					t = line.split()
					if (len(t) != 6) or (t[0] != "/dev/xvdb"):
						raise RuntimeError("Unexpected format [%s]" % line)
					blocks_total = int(t[1])
					blocks_used  = int(t[2])

				i += 1
				if i == 7:
					block_used_percent = round(100.0 * blocks_used / blocks_total, 1)
					if block_used_percent_prev != block_used_percent:
						fo_w.write((fmt + "\n") % (dt
							, (100.0 * blocks_used / blocks_total)
							, rr_4k_min, rr_4k_avg, rr_4k_max, rr_4k_sd
							, rw_4k_min, rw_4k_avg, rw_4k_max, rw_4k_sd
							, seqw_100m
							))
					block_used_percent_prev = block_used_percent
					i = 0
					dt = None
					blocks_total = None
					blocks_used  = None
					rr_4k_min = None
					rr_4k_avg = None
					rr_4k_max = None
					rr_4k_sd  = None
					rw_4k_min = None
					rw_4k_avg = None
					rw_4k_max = None
					rw_4k_sd  = None
					seqw_100m = None
		Cons.P("Created %s %d" % (fn_plot_data, os.path.getsize(fn_plot_data)))
Example #58
0
 def correctLeaves(self, treeSize):
     for node in self._nodes:
         newLeaf = Util.correctLeaf(node.getLeaf(), treeSize,
                                    node.getSegID() % 2)
         if newLeaf != None:
             node.setLeaf(newLeaf)
def _GenEbsSsdData(fn0, fn1):
	fn_plot_data = "plot-data/%s" % fn1
	fn_raw0 = "../log/ioping/%s" % fn0
	fn_raw1 = "../log/ioping/%s" % fn1

	# Generate one if not exist. Skip for now. It's fast enough.
	#if os.path.isfile(fn_plot_data):
	#	Cons.P("%s already exists" % fn_plot_data)
	#	return

	# Uncompress the raw files if not exist
	for fn in [fn_raw0, fn_raw1]:
		if not os.path.isfile(fn):
			with Cons.MeasureTime("file %s doesn't exist. uncompressing from the archive" % fn):
				cmd = "7z x -o../log/ioping %s.7z" % fn
				Util.RunSubp(cmd)

	if not os.path.exists("plot-data"):
		os.makedirs("plot-data")

	with Cons.MeasureTime("Generating plot data %s" % fn_plot_data):
		with open(fn_plot_data, "w") as fo_w, open(fn_raw0) as fo_r0, open(fn_raw1) as fo_r1:
			fmt = "%13s" \
					" %5.1f" \
					" %3d %4d %5d %4d" \
					" %3d %4d %5d %4d" \
					" %6d"
			fo_w.write("%s\n" % Util.BuildHeader(fmt,
				"datetime" \
				" blocks_used_percent" \
				" rr_4k_min rr_4k_avg rr_4k_max rr_4k_sd" \
				" rw_4k_min rw_4k_avg rw_4k_max rw_4k_sd" \
				" seqw_100m"
				))

			#160423-195525
			#100 68474 1460 5981833 230 685 1631 568
			#100 74883 1335 5469866 495 749 4723 585
			#1 1690975 1 62010142 1690975 1690975 1690975 0
			#Filesystem       1K-blocks       Used   Available Use% Mounted on
			#/dev/xvda1         8115168    2676196     5003696  35% /
			#/dev/xvdc      17111375772 1565491700 15545867688  10% /mnt/ebs-ssd1

			i = 0
			dt = None
			block_used_percent_prev = None
			blocks_total = None
			blocks_used  = None
			rr_4k_min = None
			rr_4k_avg = None
			rr_4k_max = None
			rr_4k_sd  = None
			rw_4k_min = None
			rw_4k_avg = None
			rw_4k_max = None
			rw_4k_sd  = None
			seqw_100m = None

			for fo_r in [fo_r0, fo_r1]:
				for line in fo_r.readlines():
					#Cons.P("%d [%s]" % (i, line.strip()))
					if i == 0:
						dt = line.strip()
					# 4k rand read
					elif i == 1:
						t = line.split()
						if len(t) != 8:
							raise RuntimeError("Unexpected format [%s]" % line)
						# All in us
						rr_4k_min = int(t[4])
						rr_4k_avg = int(t[5])
						rr_4k_max = int(t[6])
						rr_4k_sd  = int(t[7])
					elif i == 2:
						t = line.split()
						if len(t) != 8:
							raise RuntimeError("Unexpected format [%s]" % line)
						# All in us
						rw_4k_min = int(t[4])
						rw_4k_avg = int(t[5])
						rw_4k_max = int(t[6])
						rw_4k_sd  = int(t[7])
					elif i == 3:
						t = line.split()
						if len(t) != 8:
							raise RuntimeError("Unexpected format [%s]" % line)
						# All of the min, avg, max are the same since there is only 1 request
						# at a time.
						seqw_100m = int(t[4])
					elif i == 6:
						t = line.split()
						if (len(t) != 6) or (t[0] != "/dev/xvdc"):
							raise RuntimeError("Unexpected format [%s]" % line)
						blocks_total = int(t[1])
						blocks_used  = int(t[2])

					i += 1
					if i == 7:
						block_used_percent = round(100.0 * blocks_used / blocks_total, 1)
						if block_used_percent_prev != block_used_percent:
							fo_w.write((fmt + "\n") % (dt
								, (100.0 * blocks_used / blocks_total)
								, rr_4k_min, rr_4k_avg, rr_4k_max, rr_4k_sd
								, rw_4k_min, rw_4k_avg, rw_4k_max, rw_4k_sd
								, seqw_100m
								))
						block_used_percent_prev = block_used_percent
						i = 0
						dt = None
						blocks_total = None
						blocks_used  = None
						rr_4k_min = None
						rr_4k_avg = None
						rr_4k_max = None
						rr_4k_sd  = None
						rw_4k_min = None
						rw_4k_avg = None
						rw_4k_max = None
						rw_4k_sd  = None
						seqw_100m = None
		Cons.P("Created %s %d" % (fn_plot_data, os.path.getsize(fn_plot_data)))
Example #60
0
  def add_file(self, filename):
    """
    Setups data:
      self.seq_lengths
      self.file_index
      self.file_start
      self.file_seq_start
    Use load_seqs() to load the actual data.
    :type filename: str
    """
    if self._use_cache_manager:
      filename = Util.cf(filename)
    fin = h5py.File(filename, "r")
    if 'targets' in fin:
      self.labels = {
        k: [self._decode(item) for item in fin["targets/labels"][k][...].tolist()]
        for k in fin['targets/labels']}
    if not self.labels:
      labels = [item.split('\0')[0] for item in fin["labels"][...].tolist()]; """ :type: list[str] """
      self.labels = {'classes': labels}
      assert len(self.labels['classes']) == len(labels), "expected " + str(len(self.labels['classes'])) + " got " + str(len(labels))
    self.files.append(filename)
    print("parsing file", filename, file=log.v5)
    if 'times' in fin:
      if self.timestamps is None:
        self.timestamps = fin[attr_times][...]
      else:
        self.timestamps = numpy.concatenate([self.timestamps, fin[attr_times][...]], axis=0)
    seq_lengths = fin[attr_seqLengths][...]
    if 'targets' in fin:
      self.target_keys = sorted(fin['targets/labels'].keys())
    else:
      self.target_keys = ['classes']

    if len(seq_lengths.shape) == 1:
      seq_lengths = numpy.array(zip(*[seq_lengths.tolist() for i in range(len(self.target_keys)+1)]))

    if len(self._seq_lengths) == 0:
      self._seq_lengths = numpy.array(seq_lengths)
    else:
      self._seq_lengths = numpy.concatenate((self._seq_lengths, seq_lengths), axis=0)
    if not self._seq_start:
      self._seq_start = [numpy.zeros((seq_lengths.shape[1],), 'int64')]
    seq_start = numpy.zeros((seq_lengths.shape[0] + 1, seq_lengths.shape[1]), dtype="int64")
    numpy.cumsum(seq_lengths, axis=0, dtype="int64", out=seq_start[1:])
    self._tags += fin["seqTags"][...].tolist()
    self.file_seq_start.append(seq_start)
    nseqs = len(seq_start) - 1
    self._num_seqs += nseqs
    self.file_index.extend([len(self.files) - 1] * nseqs)
    self.file_start.append(self.file_start[-1] + nseqs)
    self._num_timesteps += numpy.sum(seq_lengths[:, 0])
    if self._num_codesteps is None:
      self._num_codesteps = [0 for i in range(1, len(seq_lengths[0]))]
    for i in range(1, len(seq_lengths[0])):
      self._num_codesteps[i - 1] += numpy.sum(seq_lengths[:, i])
    if 'maxCTCIndexTranscriptionLength' in fin.attrs:
      self.max_ctc_length = max(self.max_ctc_length, fin.attrs['maxCTCIndexTranscriptionLength'])
    if len(fin['inputs'].shape) == 1:  # sparse
      num_inputs = [fin.attrs[attr_inputPattSize], 1]
    else:
      num_inputs = [fin['inputs'].shape[1], len(fin['inputs'].shape)] #fin.attrs[attr_inputPattSize]
    if self.num_inputs == 0:
      self.num_inputs = num_inputs[0]
    assert self.num_inputs == num_inputs[0], "wrong input dimension in file %s (expected %s got %s)" % (
                                             filename, self.num_inputs, num_inputs[0])
    if 'targets/size' in fin:
      num_outputs = {}
      for k in fin['targets/size'].attrs:
        if numpy.isscalar(fin['targets/size'].attrs[k]):
          num_outputs[k] = (fin['targets/size'].attrs[k], len(fin['targets/data'][k].shape))
        else:  # hdf_dump will give directly as tuple
          assert fin['targets/size'].attrs[k].shape == (2,)
          num_outputs[k] = tuple(fin['targets/size'].attrs[k])
    else:
      num_outputs = {'classes': fin.attrs[attr_numLabels]}
    num_outputs["data"] = num_inputs
    if not self.num_outputs:
      self.num_outputs = num_outputs
    assert self.num_outputs == num_outputs, "wrong dimensions in file %s (expected %s got %s)" % (
                                            filename, self.num_outputs, num_outputs)
    if 'ctcIndexTranscription' in fin:
      if self.ctc_targets is None:
        self.ctc_targets = fin['ctcIndexTranscription'][...]
      else:
        tmp = fin['ctcIndexTranscription'][...]
        pad_width = self.max_ctc_length - tmp.shape[1]
        tmp = numpy.pad(tmp, ((0,0),(0,pad_width)), 'constant', constant_values=-1)
        pad_width = self.max_ctc_length - self.ctc_targets.shape[1]
        self.ctc_targets = numpy.pad(self.ctc_targets, ((0,0),(0,pad_width)), 'constant', constant_values=-1)
        self.ctc_targets = numpy.concatenate((self.ctc_targets, tmp))
      self.num_running_chars = numpy.sum(self.ctc_targets != -1)
    if 'targets' in fin:
      for name in fin['targets/data']:
        tdim = 1 if len(fin['targets/data'][name].shape) == 1 else fin['targets/data'][name].shape[1]
        self.data_dtype[name] = str(fin['targets/data'][name].dtype) if tdim > 1 else 'int32'
        self.targets[name] = None
    else:
      self.targets = { 'classes' : numpy.zeros((self._num_timesteps,), dtype=theano.config.floatX)  }
      self.data_dtype['classes'] = 'int32'
    self.data_dtype["data"] = str(fin['inputs'].dtype)
    assert len(self.target_keys) == len(self._seq_lengths[0]) - 1
    fin.close()