def load_cached_mouse_hist2d_segments(hist2d_tarf,outlines_tarf,SHAPE,return_sizes=False,check_lengths=True, \ **kwargs): msize_out = os.path.splitext(hist2d_tarf)[0]+'-mouse_sizes.list' if os.path.exists(hist2d_tarf) and (not check_lengths or (len(tarfile.open(hist2d_tarf).getnames()) == len(tarfile.open(outlines_tarf).getnames()))) \ and (not return_sizes or os.path.exists(msize_out)): mm_hists = Util.tar2ar_all(tarfile.open(hist2d_tarf),SHAPE,int) print >> sys.stderr, '%s mouse location histograms loaded, max counts: %s' % (len(mm_hists),max([m.max() for m in mm_hists]) ) if return_sizes: svec = eval(open(msize_out).read()) print >> sys.stderr, '%s mouse size values loaded; mean %s' % (len(svec),numpy.mean(svec)) else: if return_sizes: print >> sys.stderr, 'calculate mouse locations from outlines %s' % outlines_tarf mm_hists,svec = load_mouse_hist2d_segments(tarfile.open(outlines_tarf),SHAPE,return_sizes=True,**kwargs) print >> sys.stderr, '%s mouse location histograms calculated, max counts: %s' % (len(mm_hists),max([m.max() for m in mm_hists])) print >> sys.stderr, '%s mouse size values calculated; mean %s' % (len(svec),numpy.mean(svec)) open(msize_out,'w').write(svec.__repr__()) else: mm_hists = load_mouse_hist2d_segments(tarfile.open(outlines_tarf),SHAPE,**kwargs) print >> sys.stderr, '%s mouse location histograms calculated' % len(mm_hists) h2d_tarh = tarfile.open(hist2d_tarf,'w') for fname,ar in zip(sorted(tarfile.open(outlines_tarf).getnames()),mm_hists): Util.ar2tar(ar,fname,h2d_tarh) h2d_tarh.close() if return_sizes: return mm_hists,svec else: return mm_hists
def insertRandomRoom(level): x = Util.getRandom(0, LEVEL_WIDTH-1) y = Util.getRandom(0, LEVEL_HEIGHT-1) width = Util.getRandom(3, 6) height = Util.getRandom(3, 6) room = Level.MakeRoom(width, height) level.applyFeature(room, x, y)
def resetToDefaults(self, confirm=True):
"""
Reset the application to the defaults. At the very least,
this will delete all waves, tables, and figures.
Before resetting, this will ask the user if they want to save
the current project.
"""
Util.debug(2, "App.resetToDefaults", "Resetting application to defaults")
if confirm:
# Check to see if we want to save the current project
saveProjectMessage = QMessageBox()
saveProjectMessage.setText("You are about to load another project.")
saveProjectMessage.setInformativeText("Do you want to save your current project?")
saveProjectMessage.setStandardButtons(QMessageBox.Save | QMessageBox.Discard | QMessageBox.Cancel)
saveProjectResponse = saveProjectMessage.exec_()
if saveProjectResponse == QMessageBox.Save:
self.saveProject()
elif saveProjectResponse == QMessageBox.Cancel:
return
# Now reset to a clean slate
self.setDefaults()
def usedVigenere(inputFile, dictionary):
best = 0
key = 0
percent = 0
#listChars = ['a','b','c','d']
listChars = ['1','2','3','4','5','6']
setDictionary = Util.setWords(dictionary)
for sizeKey in range(1, len(listChars)+1):
for i in itertools.product(listChars, repeat=sizeKey):
print(i)
tmp = DecriptFunctions.modDecVigenere(i, inputFile[0:240])
uniqueWords = Util.setWords(tmp)
nWords = len(uniqueWords & setDictionary)
if nWords > best:
best = nWords
key = i
countWords = len(uniqueWords)
percent = (best*100)/countWords
if percent > 50:
break
print("\nAlgoritmo de Vigenere"
"\nChave: " + str(key) +
"\nTotal de palavras: " + str(countWords) +
"\nTotal de palavras encontradas: " + str(best) +
"\n" + str(percent) + "% de acerto" )
def setCurrentProject(self, fileName):
Util.debug(3, "App.setCurrentProject", "Setting current project title")
self.currentProjectFile = fileName
if fileName != "":
self.setWindowTitle("PySciPlot - " + fileName)
else:
self.setWindowTitle("PySciPlot")
def G1DBinaryStringXSinglePoint(genome, **args):
""" The crossover of 1D Binary String, Single Point
.. warning:: You can't use this crossover method for binary strings with length of 1.
"""
sister = None
brother = None
gMom = args["mom"]
gDad = args["dad"]
if len(gMom) == 1:
Util.raiseException("The Binary String have one element, can't use the Single Point Crossover method !", TypeError)
cut = rand_randint(1, len(gMom)-1)
if args["count"] >= 1:
sister = gMom.clone()
sister.resetStats()
sister[cut:] = gDad[cut:]
if args["count"] == 2:
brother = gDad.clone()
brother.resetStats()
brother[cut:] = gMom[cut:]
return (sister, brother)
def G1DBinaryStringXTwoPoint(genome, **args):
""" The 1D Binary String crossover, Two Point
.. warning:: You can't use this crossover method for binary strings with length of 1.
"""
sister = None
brother = None
gMom = args["mom"]
gDad = args["dad"]
if len(gMom) == 1:
Util.raiseException("The Binary String have one element, can't use the Two Point Crossover method !", TypeError)
cuts = [rand_randint(1, len(gMom)-1), rand_randint(1, len(gMom)-1)]
if cuts[0] > cuts[1]:
Util.listSwapElement(cuts, 0, 1)
if args["count"] >= 1:
sister = gMom.clone()
sister.resetStats()
sister[cuts[0]:cuts[1]] = gDad[cuts[0]:cuts[1]]
if args["count"] == 2:
brother = gDad.clone()
brother.resetStats()
brother[cuts[0]:cuts[1]] = gMom[cuts[0]:cuts[1]]
return (sister, brother)
def process(self):
Util.log('SUBCLASSES : %s' % self.subclasses, 'debug')
for cls in self.subclasses.values():
cls.process()
Util.log('METHODS : %s' % self.methods, 'debug')
for meth in self.methods:
self.select_meth(meth)
def __init__(self, methanalysis, this):
self.memory = {}
self.method = methanalysis.get_method()
self.name = self.method.get_name()
self.lparams = []
self.variables = Instruction.Variable()
if self.name == '<init>':
self.name = self.method.get_class_name()[1:-1].split('/')[-1]
self.basic_blocks = methanalysis.basic_blocks.bb
code = self.method.get_code()
access = self.method.get_access()
self.access = [flag for flag in Util.ACCESS_FLAGS_METHODS if flag & access]
desc = self.method.get_descriptor()
self.type = Util.get_type(desc.split(')')[-1])
self.paramsType = Util.get_params_type(desc)
#Util.log('Searching loops in method %s' % self.name, 'debug')
"""
Findloop.loopCounter = 0
cfg = Findloop.CFG()
lsg = Findloop.LSG()
self.cfg = cfg
self.lsg = lsg
Findloop.AddEdges(cfg, self)
nbLoops = Findloop.FindLoops(cfg, lsg)
if nbLoops > 1:
Util.log('==================', 'debug')
Util.log('==== DUMP CFG ====', 'debug')
cfg.dump()
Util.log('==================', 'debug')
Findloop.ShowBlocks(lsg, 0)
Util.log('==== DUMP LSG ====', 'debug')
lsg.dump()
Util.log('==================', 'debug')
"""
lsg = None
blocks = []
exceptions = methanalysis.exceptions.exceptions
print "METHOD :", self.name
self.graph = Structure.ConstructCFG(lsg, self.basic_blocks, exceptions, blocks)
print
#androguard.bytecode.method2png('graphs/%s#%s.png' % (self.method.get_class_name().split('/')[-1][:-1], self.name), methanalysis)
if code is None:
Util.log('CODE NONE : %s %s' % (self.name, self.method.get_class_name()), 'debug')
else:
start = code.registers_size.get_value() - code.ins_size.get_value()
# 0x8 == Static case : this is not passed as a parameter
if 0x8 in self.access:
self._add_parameters(start)
else:
self.memory[start] = Register(this, start)
#print 'THIS ( %s ) : %d' % (self.name, start)
self._add_parameters(start + 1)
self.ins = []
self.cur = 0
def loadStoredSettings(app, storedSettings):
"""
Load stored settings into the application.
"""
Util.debug(1, "Load", "Loading stored settings from file")
storedSettingsObj = pickle.loads(str(storedSettings.firstChild.data.strip()))
app.storedSettings().update(storedSettingsObj)
def loadFigures(app, figures, stackingOrder):
"""
Load given figures into the application.
"""
Util.debug(1, "Load", "Loading figures from file")
figureList = figures.getElementsByTagName("figure")
for figure in figureList:
figureAttrs = {} # QT attributes
# Get attributes for the figure
for attrName in figure.attributes.keys():
figureAttrs[attrName] = int(figure.attributes[attrName].value.strip())
# Get figure from xml
pickledFigure = figure.firstChild.data.strip()
figureObj = pickle.loads(str(pickledFigure))
figureWindow = app.figures().addFigure(figureObj)._figureSubWindow
# Reset QT widget values
figureWindow.setWindowState(Qt.WindowState(figureAttrs["windowState"]))
figureWindow.resize(figureAttrs["width"], figureAttrs["height"])
figureWindow.move(figureAttrs["xpos"], figureAttrs["ypos"])
# Add to stackingOrder
if figureAttrs["stackingOrder"] >= len(stackingOrder):
stackingOrder.extend([None] * (figureAttrs["stackingOrder"] - len(stackingOrder) + 1))
stackingOrder[figureAttrs["stackingOrder"]] = figureWindow
# Set the figure dialog selected to the first figure
app._loadedModules['EditFigureDialog'].changeCurrentFigure(0)
app._loadedModules['EditFigureDialog'].updateFigureSelectorIndex(0)
def test_channelAnimatorBoundaryAnimation(self):
driver = self.driver
# Open a test image so we have something to animate
Util.load_image( self, driver, "Default")
# Find and record the last valid value of the animation
self._getLastValue( driver, "Channel" )
lastChannelValue = self._getCurrentValue( driver, "Channel" )
# Find and click the lower spin box
lowerBoundText = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.XPATH, "//div[@id='ChannelLowerBoundSpin']/input")))
driver.execute_script( "arguments[0].scrollIntoView(true);", lowerBoundText)
lowerBoundText.click()
# Change the lower bound value
lowerBoundValue = Util._changeElementText(self, driver, lowerBoundText, 1)
# Find and click the upper spin box
upperBoundText = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.XPATH, "//div[@id='ChannelUpperBoundSpin']/input")))
driver.execute_script( "arguments[0].scrollIntoView(true);", upperBoundText)
upperBoundText.click()
# Change the upper bound value
upperBoundValue = Util._changeElementText( self, driver, upperBoundText, int(lastChannelValue)-1)
# Allow test image to animate for 2 seconds
self._animateForward( driver, "Channel" )
time.sleep(2)
# Check that the lower and upper bound values did not change during animation
lowerBound = lowerBoundText.get_attribute("value")
upperBound = upperBoundText.get_attribute("value")
self.assertEqual( int(lowerBound), int(lowerBoundValue), "Lower bound channel value changed during animation")
self.assertEqual( int(upperBound), int(upperBoundValue), "Upper bound channel value changed during animation")
def collectData(self,hostId,sqlServerId, discoverConfigs = 1):
self.connection.open()
oshv = ObjectStateHolderVector()
try:
oshv.add(self.getServerProperties(sqlServerId,hostId))
dbMap = self.getDatabases(sqlServerId)
#get the databases
oshv.addAll(self.sqlDataBaseProps.getDatabases(dbMap,hostId,discoverConfigs))
oshv.addAll(self.sqlDataBaseProps.getStoredProcedures(dbMap))
#get the server configuration:
logger.debug('discovering configs')
try:
oshv.add(self.sqlServerConfig.getServerConfiguration(sqlServerId))
oshv.add(self.sqlServerConfig.getServerStartup(sqlServerId))
self.sqlServerConfig.discoverPlans(oshv,sqlServerId,dbMap)
except:
logger.debugException(hostId.toString())
if self.discoveryOptions and self.discoveryOptions.discoverDbUser:
users = self.getDbUsers(sqlServerId)
Util.addFromMap(users,oshv)
else:
users = LinkedHashMap()
oshv.addAll(self.getProcesses(hostId,sqlServerId,dbMap,users))
oshv.addAll(self.clusterConfiguration.collectData(sqlServerId))
#db configuration:
oshv.addAll(self.getDbConf(dbMap,hostId,users))
logger.debug("sql db result for hostid:"+hostId.toString())
except:
logger.debugException(hostId.toString())
self.connection.close()
return oshv
def test_campusLinkLastYears(self):
self.ui.tabWid.setCurrentIndex(2)
gear = tests.mkGear('J', 10)
tests.enterGearInfo(self, gear)
QTest.mouseClick(self.tGear.updtBut, QtCore.Qt.LeftButton)
self.ui.tabWid.setCurrentIndex(1)
self.tMemb.nameSearch.clear()
memb = tests.mkMember('A', 1, forms=True, campusLink=True,
formsDate=Util.convert_date('Qt2DB', QtCore.QDate.currentDate().addDays(1)),
campusDate=Util.convert_date('Qt2DB', QtCore.QDate.currentDate().addDays(-10)))
QTest.keyClicks(self.tMemb.nameSearch, memb['FirstName'] + ' ' + memb['LastName'])
tests.enterMemberInfo(self, memb)
self.assertTrue(self.tMemb.Button_addUpdButClick())
self.ui.tabWid.setCurrentIndex(0)
setTransGear(self, gear, 'Name')
setTransMemb(self, memb)
QTest.mouseClick(self.tTran.payBut, QtCore.Qt.LeftButton)
QTest.mouseClick(self.tTran.payWind.payBut, QtCore.Qt.LeftButton)
self.tTran.payWind.close()
self.tTran.radioOut.click()
self.assertTrue(self.tTran.trans())
def RawScoreCriteria(ga_engine):
""" Terminate the evolution using the **bestrawscore** and **rounddecimal**
parameter obtained from the individual
Example:
>>> genome.setParams(bestrawscore=0.00, rounddecimal=2)
(...)
>>> ga_engine.terminationCriteria.set(GSimpleGA.RawScoreCriteria)
"""
ind = ga_engine.bestIndividual()
bestRawScore = ind.getParam("bestrawscore")
roundDecimal = ind.getParam("rounddecimal")
if bestRawScore is None:
Util.raiseException("you must specify the bestrawscore parameter", ValueError)
if ga_engine.getMinimax() == Consts.minimaxType["maximize"]:
if roundDecimal is not None:
return round(bestRawScore, roundDecimal) <= round(ind.score, roundDecimal)
else:
return bestRawScore <= ind.score
else:
if roundDecimal is not None:
return round(bestRawScore, roundDecimal) >= round(ind.score, roundDecimal)
else:
return bestRawScore >= ind.score
def _startVm(
self,
withLocalNetwork=False,
requestedIpAddress=None,
instanceNumber=1,
noCheckImageUrl=False,
msgRecipients=None,
raiseOnFailed=True,
):
self.runner = self._createRunner(withLocalNetwork, requestedIpAddress)
self.runner.instanceNumber = instanceNumber
self.runner.noCheckImageUrl = noCheckImageUrl
if not msgRecipients:
self.runner.msgRecipients = msgRecipients
vmIds = self.runner.runInstance()
self.vmIds.extend(vmIds)
for vmId in vmIds:
vmStarted = self.runner.waitUntilVmRunningOrTimeout(vmId, VM_START_TIMEOUT, failOn="Failed")
if not vmStarted and raiseOnFailed:
error = "Failed to start VM id: %s" % vmId
Util.printError(error, exit=False)
raise OneException(error)
return self.runner
def _umountPDiskAndStopVm(self, runner, pdiskDevice):
self._umountDisk(runner, pdiskDevice)
Util.printStep("Stopping VM...")
self._stopVm(runner)
self.vmIds = []
# Wait for the pdisk hook to be executed
sleep(20)
def processing():
mat_name = r'silk'
ori_folder = r'D:\Datasets\silkA_16bit\16bit'
res_folder = r'D:\Datasets\PreprocessingData'
# Read tiff images and dump them into a binary file
# print 'Read and Dump images begins ...'
# volume_path = os.path.join(res_folder, mat_name + '.dat')
# volume_data = TiffsHandler.readTiffImagesInFolder(ori_folder)
# print 'Read images finished ...'
#
# print 'Dump volume to (%s) begins ...' % volume_path
# Util.dumpData(volume_data, volume_path)
# print 'Dump volume to (%s) finished ...' % volume_path
# load volume data
volume_path = os.path.join(res_folder, mat_name + '.dat')
volume_data = Util.loadData(volume_path)
# Histogram equalize
print 'Histogram equalize begins ...'
bins = np.arange(0, 65537)
# bins = np.linspace(0, 65537, 50, endpoint=True)
volume_he = Util.histEq(volume_data, bins=bins)
volume_he = volume_he.astype('float32')
print 'Histogram equalize finished ...'
volume_he_path = os.path.join(res_folder, mat_name + '_he.dat')
print 'Dump volume_he to (%s) begins ...' % volume_he_path
Util.dumpData(volume_he, volume_he_path)
print 'Dump volume_he to (%s) finished ...' % volume_he_path
return volume_data, volume_he
def resample():
volume = Util.loadData(r'D:\Datasets\PreprocessingData\silk_he.dat')
volume = volume.swapaxes(0, 2).swapaxes(0, 1)
m = [25.6207523,
0.00563881328,
-7.50831215e-06,
-0.000110252038,
5.8775209e-07,
-4.20923367e-10,
2.7086295e-07,
-1.18467307e-09,
1.03793103e-12]
m = np.asarray(m)
nx, ny, nz = volume.shape
x = np.arange(0, nx)
y = np.arange(0, ny)
X, Y = np.meshgrid(x, y, indexing='ij')
Z = Util.polyval2d(X, Y, m)
print Z.shape
Zmin = Z.min()
Zmax = Z.max()
R = np.zeros((nx, ny, nz + Zmax))
for i, j in zip(X.ravel(), Y.ravel()):
s = int(Zmax - Z[i, j])
R[i, j, s:s+nz] = volume[i, j, :]
Util.dumpData(R, r'D:\Datasets\PreprocessingData\silk_rs.dat')
return R
def denoiseSilk():
d = np.load(r'') # [508, 1013, 992], 'uint16', [0, 65535]
J = np.load(r'') # [1013, 992, 105], 'float32'
ori = np.load(r'D:\Dataset\silk\orientation.dat') # [1013, 992, 105, 3], 'float32'
ed = 0.4
eJ = -1.0
# preprocess
d = d.astype(dtype='float32')
d /= 65535.0
# print d.shape, d.dtype, d.min(), d.max()
J = J.swapaxes(0, 2).swapaxes(1, 2)
# print J.shape, J.dtype, J.min(), J.max()
ori = ori.swapaxes(0, 2).swapaxes(1, 2)
# print ori.shape, ori.dtype
# denoising
denoise(d, J, ori, ed, eJ)
print 'finish deonising ...'
# save
silk_dn_path = r'D:\Dataset\silk\silk_dn.dat'
ori_dn_path = r'D:\Dataset\silk\orientation_dn.dat'
Util.dumpData(d, silk_dn_path)
Util.dumpData(ori, ori_dn_path)
def precomputeFilterQ():
# Precompute kernel 'q' on a set of directions
# Calculate the directions
nx = 32
ny = 32
nz = 6
bound_points = generateBoundingPoints(nx + 1, ny + 1, nz + 1)
center = np.array([nx/2, ny/2, nz/2])
directions = bound_points - center
directions = np.asarray(directions, dtype='float64')
# normalize
for i in range(0, directions.shape[0]):
sqrt_sum = np.sqrt((directions[i]**2).sum())
directions[i] = directions[i] / sqrt_sum
d_path = r'D:\Datasets\RecoverOrientationData\ds_nx32ny32nz6.dat'
Util.dumpData(directions, d_path)
# Calculate 'q', material: silk
h = 12
s = 4 #!! some erros in paper about these parameters, s>t!!
t = 3
c = np.array([h/2, h/2, h/2], dtype='float64') # center
d = directions
q = np.zeros((d.shape[0], h, h, h))
for i in range(0, q.shape[0]):
calculateFilterQ(q[i], h, s, t, d[i], c) # pass reference?!!
qs_path = r'D:\Datasets\RecoverOrientationData\qs_h12s4t3.dat'
Util.dumpData(q, qs_path)
def LinearScaling(pop):
""" Linear Scaling scheme
.. warning :: Linear Scaling is only for positive raw scores
"""
logging.debug("Running linear scaling.")
pop.statistics()
c = Consts.CDefScaleLinearMultiplier
a = b = delta = 0.0
pop_rawAve = pop.stats["rawAve"]
pop_rawMax = pop.stats["rawMax"]
pop_rawMin = pop.stats["rawMin"]
if pop_rawAve == pop_rawMax:
a = 1.0
b = 0.0
elif pop_rawMin > (c * pop_rawAve - pop_rawMax / (c - 1.0)):
delta = pop_rawMax - pop_rawAve
a = (c - 1.0) * pop_rawAve / delta
b = pop_rawAve * (pop_rawMax - (c * pop_rawAve)) / delta
else:
delta = pop_rawAve - pop_rawMin
a = pop_rawAve / delta
b = -pop_rawMin * pop_rawAve / delta
for i in xrange(len(pop)):
f = pop[i].score
if f < 0.0:
Util.raiseException("Score %r is negative, linear scaling not supported !" % (f,), ValueError)
f = f * a + b
if f < 0:
f = 0.0
pop[i].fitness = f
def test_histogramLinkRemoval(self):
driver = self.driver
browser = selectBrowser._getBrowser()
# Wait for the image window to be present (ensures browser is fully loaded)
imageWindow = WebDriverWait(driver, 20).until(EC.presence_of_element_located((By.XPATH, "//div[@qxclass='skel.widgets.Window.DisplayWindowImage']")))
# Find the Histogram window
histWindow = self._getHistogramWindow( driver )
ActionChains(driver).click(histWindow).perform()
# Open Link settings for the Histogram window
linkMenuButton = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.XPATH, "//div[@qxclass='qx.ui.toolbar.MenuButton']/div[text()='Links...']")))
ActionChains(driver).click( linkMenuButton ).perform()
# Remove link from the Histogram to the main image window
Util.remove_main_link( self, driver, imageWindow )
# Load an image
Util.load_image( self, driver, "Default")
# Find and select the histogram window
histWindow = self._getHistogramWindow( driver )
ActionChains(driver).click( histWindow ).perform()
# Click the settings button to expose the settings
self._openHistogramSettings( driver )
# Check that the histogram values are default values
newMaxZoomValue = self._getTextValue( driver, "histogramZoomMaxValue")
self.assertEqual( float(newMaxZoomValue), 1, "Histogram is linked to image after link was removed")
def test_histogramAddImage(self):
driver = self.driver
timeout = selectBrowser._getSleep()
# Wait for the image window to be present (ensures browser is fully loaded)
imageWindow = WebDriverWait(driver, 20).until(EC.presence_of_element_located((By.XPATH, "//div[@qxclass='skel.widgets.Window.DisplayWindowImage']")))
# Load an image
Util.load_image(self, driver, "Default")
# Find and select the histogram
histWindow = self._getHistogramWindow(driver)
ActionChains(driver).click( histWindow ).perform()
# Click the settings button to expose the settings
self._openHistogramSettings( driver )
# Get the max zoom value of the first image
maxZoomValue = self._getTextValue( driver, "histogramZoomMaxValue")
print "First image maxZoomValue:", maxZoomValue
# Load a different image in the same window
Util.load_image(self, driver, "aH.fits")
time.sleep( timeout )
# Check that the new max zoom value updates
newMaxZoomValue = self._getTextValue( driver, "histogramZoomMaxValue")
self.assertNotEqual(float(newMaxZoomValue), float(maxZoomValue), "The histogram did not update when a new image was loaded.")
print "Second image maxZoomValue:", newMaxZoomValue
def test_histogramRemoveImage(self):
driver = self.driver
timeout = selectBrowser._getSleep()
# Wait for the image window to be present (ensures browser is fully loaded)
imageWindow = WebDriverWait(driver, 20).until(EC.presence_of_element_located((By.XPATH, "//div[@qxclass='skel.widgets.Window.DisplayWindowImage']")))
# Load an image
Util.load_image( self, driver, "Default")
# Click on the Data->Close->Image button to close the image
imageWindow = driver.find_element_by_xpath( "//div[@qxclass='skel.widgets.Window.DisplayWindowImage']")
ActionChains(driver).double_click( imageWindow ).perform()
dataButton = WebDriverWait(driver, 20).until(EC.presence_of_element_located((By.XPATH, "//div[text()='Data']/..")))
ActionChains(driver).click( dataButton ).send_keys(Keys.ARROW_DOWN).send_keys(Keys.ARROW_DOWN).send_keys(
Keys.ARROW_RIGHT).send_keys(Keys.ENTER).perform()
# Find and select the Histogram window
histWindow = self._getHistogramWindow( driver )
ActionChains(driver).click( histWindow ).perform()
time.sleep( timeout )
# Click the settings button to expose the settings
self._openHistogramSettings( driver )
# Check that the histogram values are restored to default values
newMaxZoomValue = self._getTextValue( driver, "histogramZoomMaxValue")
print "Zoom max value=", newMaxZoomValue
self.assertEqual( float(newMaxZoomValue), 1, "Default values were not restored after image removal")
def test_statsAddImage(self):
driver = self.driver
# Load a large test image.
Util.load_image( self, driver, "aH.fits")
# Move to the center of the image window so data appears in the Stats Window
imageWindow = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.XPATH, "//div[@qxclass='skel.widgets.Window.DisplayWindowImage']")))
ActionChains(driver).move_to_element( imageWindow ).perform()
# Get the Statistics of the loaded image
statsText = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.XPATH, "//div[@qxclass='skel.boundWidgets.Label']")))
statsText = statsText.get_attribute('textContent')
# Load a different image in the same window
Util.load_image( self, driver, "aK.fits")
# Move to the center of the image window so data appears in the Stats Window
ActionChains(driver).move_to_element( imageWindow ).perform()
# Sometimes text does not appear, therefore move mouse cursor
ActionChains(driver).move_by_offset( 100, 100 ).perform()
# Get the Statistics of the loaded image
newStatsText = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.XPATH, "//div[@qxclass='skel.boundWidgets.Label']")))
newStatsText = newStatsText.get_attribute('textContent')
# Check that the Statistics text changed when new image was loaded
self.assertNotEqual( newStatsText, statsText, "Stats text did not update when new image was loaded in main image window")
def test_statsAnimation(self):
driver = self.driver
# Load a test image with animation
Util.load_image( self, driver, "Default")
# Move to the center of the image window so data appears in the Stats Window
imageWindow = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.XPATH, "//div[@qxclass='skel.widgets.Window.DisplayWindowImage']")))
ActionChains(driver).move_to_element( imageWindow ).perform()
# Get the Statistics of the loaded image
statsText = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.XPATH, "//div[@qxclass='skel.boundWidgets.Label']")))
statsText = statsText.get_attribute('textContent')
# Click the forward animation button
forwardAnimateButton = driver.find_element_by_xpath("//div[@class='qx-toolbar']/div[@class='qx-button'][2]")
self.assertIsNotNone( forwardAnimateButton, "Could not find forward animation button")
driver.execute_script( "arguments[0].scrollIntoView(true);", forwardAnimateButton)
ActionChains(driver).click( forwardAnimateButton ).perform()
time.sleep(3)
# Move to the center of the image window so data appears in the Stats Window
imageWindow = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.XPATH, "//div[@qxclass='skel.widgets.Window.DisplayWindowImage']")))
ActionChains(driver).move_to_element( imageWindow ).perform()
# Sometimes text does not appear, therefore move mouse cursor
ActionChains(driver).move_by_offset( 100, 100 ).perform()
# Get the Statistics of the loaded image
newStatsText = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.XPATH, "//div[@qxclass='skel.boundWidgets.Label']")))
newStatsText = newStatsText.get_attribute('textContent')
# Check that the Statistics text changed when image in the image window was changed
self.assertNotEqual( newStatsText, statsText, "Stats text did not update during animation of the image")
def test_histogramChangeLinks(self):
driver = self.driver
browser = selectBrowser._getBrowser()
# Wait for the image window to be present (ensures browser is fully loaded)
imageWindow = WebDriverWait(driver, 20).until(EC.presence_of_element_located((By.XPATH, "//div[@qxclass='skel.widgets.Window.DisplayWindowImage']")))
# Find and select the Histogram window
histWindow = self._getHistogramWindow( driver )
ActionChains(driver).click( histWindow ).perform()
# Click the settings button to expose the settings
self._openHistogramSettings( driver )
# Open Link settings for the Histogram window
linkMenuButton = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.XPATH, "//div[@qxclass='qx.ui.toolbar.MenuButton']/div[text()='Links...']")))
ActionChains(driver).click( linkMenuButton ).perform()
# Remove the link from the Histogram to the main image window
Util.remove_main_link( self, driver, imageWindow )
# Load an image in a separate window
imageWindow2 = Util.load_image_different_window( self, driver, "aH.fits")
# Open link settings for the Histogram
ActionChains(driver).click( histWindow ).perform()
linkMenuButton = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.XPATH, "//div[@qxclass='qx.ui.toolbar.MenuButton']/div[text()='Links...']")))
ActionChains(driver).click( linkMenuButton ).perform()
# Link the Histogram to the second image
Util.link_second_image( self, driver, imageWindow2)
# Check that the histogram values are not default values
newMaxZoomValue = self._getTextValue( driver, "histogramZoomMaxValue")
self.assertNotEqual( float(newMaxZoomValue), 1, "Histogram did not update to newly linked image")
def test_coordinateChange(self):
driver = self.driver
timeout = selectBrowser._getSleep()
#Load an image with a tabular axis
Util.load_image( self, driver, "aJ.fits")
time.sleep( 3 )
#Open the image settings tab
Util.openSettings( self, driver, "Image", True )
time.sleep( timeout )
#Get the old coordinateSystem
systemText = driver.find_element_by_xpath("//div[@id='ImageCoordinateSystem']/input")
driver.execute_script( "arguments[0].scrollIntoView(true);", systemText )
oldSystem = systemText.get_attribute('value')
print "Old system=", oldSystem
#Change the coordinate system
csCombo = driver.find_element_by_xpath("//div[@id='ImageCoordinateSystem']/div")
driver.execute_script( "arguments[0].scrollIntoView(true);", csCombo )
ActionChains(driver).click(csCombo).send_keys( Keys.ARROW_DOWN).send_keys( Keys.ARROW_DOWN
).send_keys( Keys.ARROW_DOWN).send_keys( Keys.ENTER).perform()
time.sleep( timeout )
#Verify the coordinate system is changed
systemText = driver.find_element_by_xpath("//div[@id='ImageCoordinateSystem']/input")
driver.execute_script( "arguments[0].scrollIntoView(true);", systemText )
newSystem = systemText.get_attribute( 'value')
print "New system=",newSystem
self.assertTrue( newSystem != oldSystem, "Coordinate system did not change")
def _get_graph_output_dir():
"""Assign an output path for the graph(s)."""
cfg = Configuration.getConfig()
graph_dir = cfg.get('Path', 'base-dir') + cfg.get('Path', 'graph-dir')
graph_dir += 'ActiveUsers/'
Util.ensure_dir_exist(graph_dir)
return graph_dir
def shift2Local(self):
self.id = '6100'+Util.generateRnd(4)
else:
embedding_array[i] = np.random.rand(
Config.embedding_size) * 0.02 - 0.01
print "Found embeddings: ", foundEmbed, "/", len(knownWords)
return embedding_array
if __name__ == '__main__':
wordDict = {}
posDict = {}
labelDict = {}
parsing_system = None
trainSents, trainTrees = Util.loadConll('train.conll')
devSents, devTrees = Util.loadConll('dev.conll')
testSents, _ = Util.loadConll('test.conll')
genDictionaries(trainSents, trainTrees)
embedding_filename = 'word2vec.model'
embedding_array = load_embeddings(embedding_filename, wordDict, posDict,
labelDict)
labelInfo = []
for idx in np.argsort(labelDict.values()):
labelInfo.append(labelDict.keys()[idx])
parsing_system = ParsingSystem(labelInfo[1:])
print parsing_system.rootLabel
class GvimTool(Tool.Tool):
"""A Gvim Tool: edit files with Gvim."""
def __init__(self, *args, **keyw):
apply(Tool.Tool.__init__, (self, ) + args, keyw)
self.server = None
self.handler = None
self.prep_start()
self.sernumPC = 22 # serial number used for PC annotation
self.sernumLast = 33 # serial number used for annotations
def prep_start(self):
"""Prepare for starting Gvim. Done when the tool is created and when
Gvim has exited."""
self.gvim_thread = None
self.gvim_open = 0 # set when gvim starts, reset when it exits.
self.gvim_start_time = 0
self.buffers = {} # buffers used in Netbeans Gvim tool
self.lastbuffer = 0
def gvimRunning(self):
"""Check if Gvim is still running."""
if self.gvim_open:
if gvimHasNetbeans(self.topmodel):
# gvim_open flag is set and reset appropriately.
return 1
# No netbeans connection: need to check if the process is still
# there.
if Tool.stillRunning(self.gvim_open):
return 1
self.gvim_open = 0
return 0
def gvimWait(self):
"""Can't send a command to Gvim when it is still starting up. Wait a
couple of seconds to avoid an error or starting another Gvim."""
wait = self.gvim_start_time + 2 - time.time()
if wait > 0:
time.sleep(wait)
def getProperties(self):
"""Properties that this tool supports."""
return getProperties(self.topmodel)
def openItem(self, item, action, lnum=None, col=None, off=None):
"""Open ActyItem "item" in this Tool. Not actually done until
foreground() is called."""
self.addItem(item)
def reload(self, item, node=None):
"""Reload the "item", it was changed elsewhere."""
if not node:
node = item.node
if not self.gvimRunning():
self.foreground(item, node)
elif gvimHasNetbeans(self.topmodel):
# TODO: What if the file was changed?
if node:
self.currentnode = node
self.gvimOpenFile(node)
else:
cmd = ":if exists('*inputsave')|call inputsave()|endif"
# XXX escape special characters
# XXX jump to other window if it's edited there.
cmd = cmd + ('|hide edit %s' % item.fullName())
cmd = cmd + "|if exists('*inputsave')|call inputrestore()|endif<CR>"
self.sendkeys(cmd)
def goto(self, item, node=None, lnum=None, col=None, off=None):
"""goto position "lnum"/"col" or "off" in "item"."""
self.foreground(item, node)
if gvimHasNetbeans(self.topmodel):
if item:
node = item.node
buf = self.buffers.get(node)
if buf and self.handler:
if off:
self.handler.sendCmd(buf.nr, "setDot", " %d" % off)
elif lnum:
if not col:
col = 0
self.handler.sendCmd(buf.nr, "setDot",
" %d/%d" % (lnum, col))
else:
if off:
cmd = ("%dgo" % (off + 1))
else:
cmd = ''
if lnum:
cmd = cmd + ("%dG" % lnum)
if col:
cmd = cmd + ("%d|" % col)
self.sendkeys(cmd)
def showPC(self, item, node=None, lnum=0, col=0, off=None, show=1):
"""Put PC marker at position "lnum"/"col" or "off" in "item"."""
if gvimHasNetbeans(self.topmodel) and self.handler:
if item:
node = item.node
buf = self.buffers.get(node)
# The Netbeans interface has a very strange way of handling
# annotations. An arbitrary name can be used to define it, but
# it's later placed and unplaced by a number, starting at 1 and
# rising for each anno used in a node (buffer in Vim terms).
if not buf.annoPCTypenum:
# First use in this node: define the annotation type.
buf.annoLastTypenum = buf.annoLastTypenum + 1
buf.annoPCTypenum = buf.annoLastTypenum
# Define a sign with line highlighting of a lightblue
# background.
# args: typeNum (not actually used)
# typeName (only used to define sign)
# tooltip (not used)
# glyphFile (name of bitmap for sign, optional)
# fg (fg color number or "none" for line highlight)
# bg (bg color number or "none" for line highlight)
self.handler.sendCmd(buf.nr, "defineAnnoType",
' 22 "PC" "" "" none %d' % 0xa0a0ff)
if show:
self.addAnno(buf, self.sernumPC, buf.annoPCTypenum, lnum, col,
off)
else:
self.removeAnno(buf, self.sernumPC)
def addAnno(self, buf, sernum, typenum, lnum, col, off):
"""Add an annotation with serial number "sernum", type number "typenum"
at postion "lnum"/"col" or "off"."""
if off:
s = "%d" % off
else:
s = "%d/%d" % (lnum, col)
self.handler.sendCmd(buf.nr, "addAnno",
" %d %d %s -1" % (sernum, typenum, s))
def removeAnno(self, buf, sernum):
"""Delete annotation "sernum"."""
self.handler.sendCmd(buf.nr, "removeAnno", " %d" % sernum)
def displayBreakpoint(self, what, bp):
"""Update breakpoint "bp" for action "what"."""
if (bp.item and bp.item.node and self.buffers.get(bp.item.node)
and self.handler):
node = bp.item.node
buf = self.buffers[node]
if what == "new":
if bp in buf.breakpoints:
# Already in the list, happens when the tool is both in the
# "edit" and "view" list of the node.
return
buf.breakpoints.append(bp)
if not buf.annoBreakETypenum:
# First use in this node: define the annotation type.
buf.annoLastTypenum = buf.annoLastTypenum + 1
buf.annoBreakETypenum = buf.annoLastTypenum
buf.annoLastTypenum = buf.annoLastTypenum + 1
buf.annoBreakDTypenum = buf.annoLastTypenum
# Define a sign with line highlighting of a lightblue
# background.
# args: typeNum (not actually used)
# typeName (only used to define sign)
# tooltip (not used)
# glyphFile (name of bitmap for sign, optional)
# fg (fg color number or "none" for line highlight)
# bg (bg color number or "none" for line highlight)
self.handler.sendCmd(
buf.nr, "defineAnnoType",
' 23 "BreakE" "" ">>" none %d' % 0xffa0a0)
self.handler.sendCmd(
buf.nr, "defineAnnoType",
' 24 "BreakD" "" ">>" none %d' % 0xb0b0b0)
self.sernumLast = self.sernumLast + 1
self.addAnno(buf, self.sernumLast, buf.annoBreakETypenum,
bp.lnum, 0, None)
bp.sernum = self.sernumLast
else: # "del" or "upd"
self.removeAnno(buf, bp.sernum)
if what == "upd":
if bp.enable:
typenum = buf.annoBreakETypenum
else:
typenum = buf.annoBreakDTypenum
self.addAnno(buf, bp.sernum, typenum, bp.lnum, 0, None)
def sendkeys(self, cmd):
"""Execute "cmd" in the Gvim server. Caller should check if gvim is
still running."""
if gvimHasNetbeans(self.topmodel):
# safety check for using sendkeys() with netbeans connection.
print "ERROR: send keys: '%s'" % cmd
else:
self.gvimWait()
os.system(
gvimProgName(self.topmodel) +
' --servername AAPGVIM --remote-send "<C-\\><C-N>' + cmd + '"')
def escapeStringArg(self, line):
"""Escape special characters in "line" to be send to gvim. Reverse of
parseStringArg()."""
res = ''
i = 0
while i < len(line):
if line[i] == '\n':
res = res + "\\n"
elif line[i] == '\t':
res = res + "\\t"
elif line[i] == '\r':
res = res + "\\r"
elif line[i] == '\\':
res = res + "\\\\"
elif line[i] == '"':
res = res + '\\"'
else:
res = res + line[i]
i = i + 1
return res
def close(self, shutdown):
"""Close the tool.
Return non-zero if closing is OK."""
if self.gvimRunning():
self.gvim_open = 0
if self.gvim_thread:
# Tell gvim to exit.
if self.handler:
self.handler.sendFunc(0, "saveAndExit")
# We don't wait for the answer, Vim may continue to run.
else:
# XXX check if there are any changed files in gvim.
self.sendkeys(":qa<CR>")
self.topmodel.toollist.delTool(self)
return 1 # can shutdown
def foreground(self, item, node=None, lnum=None):
"""Move this Tool to the foreground, with the current activity.
When "item" is not None edit this item.
When "node" is not None edit this node."""
winposarg = ('"+winpos %d %d" "+set lines=30 columns=82" ' %
(self.topmodel.view.GetSize().width + 10, 0))
if item:
node = item.node
# Start a Vim server on the file.
if gvimHasNetbeans(self.topmodel):
self.gvimRunNetbeans(winposarg)
if self.handler:
if node:
self.currentnode = node
self.gvimOpenFile(node)
else:
self.handler.sendCmd(0, "raise")
else:
running = self.gvimRunning()
cmd = "%s -f --servername AAPGVIM " % gvimProgName(self.topmodel)
# Jump to the node when it's specified, the user may have started
# editing another file.
if not running or node:
# Position the Vim window next to ours and bring it to the
# front.
cmd = cmd + winposarg
if node:
if lnum:
pos = "+%d " % lnum
else:
pos = ''
cmd = cmd + ('--remote-silent %s"%s"' %
(pos, node.fullName()))
if item:
self.setCurrentItem(item)
else:
self.currentnode = node
else:
cmd = cmd + '--remote-send "<C-\\><C-N>:call foreground()<CR>"'
if not running:
self.gvim_start_time = time.time()
self.gvim_open = Tool.spawn(cmd)
else:
self.gvimWait()
os.system(cmd)
def gvimRunNetbeans(self, gvimarg):
"""Make sure we have a running Gvim with Netbeans connection."""
if self.gvim_open:
return
# start a new thread to run the server in
self.gvim_thread = thread.start_new_thread(self.gvimThreadFunc,
(gvimarg, ))
self.gvim_open = 1
# Wait until Gvim is available and finished starting up. We cannot
# send commands before this.
# When opening gvim fails the other thead sets gvim_open to zero.
# Timeout after ten seconds.
countdown = 1000
while countdown > 0:
time.sleep(0.01)
if self.handler or self.gvim_open == 0:
break
countdown = countdown - 1
if not self.handler:
# XXX error handling
self.gvim_open = 0
print "Cannot open Gvim connection!"
def gvimThreadFunc(self, gvimarg):
"""Function that is run in a separate thread. It starts the socket
server and runs gvim."""
# start the server
# IANA says that port numbers 49152 through 65335 are for dynamic use.
# Start a bit higher to avoid clashes too often.
portnr = 49765
while not self.server:
try:
self.server = SocketServer.TCPServer(("", portnr), GvimHandler)
self.server.gvimtool = self
except Exception, e:
if string.find(str(e), "ddress already in use") < 0:
print "Could not start socket server: ", e
self.gvim_open = 0
return
# Port number already in use, retry with another port number.
portnr = portnr + 1
# start gvim and tell it to use the server
# use a random password for semi-security (it shows up in the process
# list...)
vimrc = os.path.join(self.topmodel.rootdir, "startup.vim")
self.password = str(random.random())
Util.async_system(
[], None,
'{quiet} %s --servername AAPGVIM %s-nb:localhost:%d:%s -S "%s"' %
(gvimProgName(
self.topmodel), gvimarg, portnr, self.password, vimrc))
# handle ONE request
self.server.handle_request()
# Gvim was closed, need to restart the server next time.
self.prep_start()
# If netbeans doesn't work close the socket server
if gvim_has_netbeans == 0:
del self.server
self.server = None
def executeShCmd(cmd, cfg, cwd, results):
if isinstance(cmd, ShUtil.Seq):
if cmd.op == ';':
res = executeShCmd(cmd.lhs, cfg, cwd, results)
return executeShCmd(cmd.rhs, cfg, cwd, results)
if cmd.op == '&':
raise NotImplementedError,"unsupported test command: '&'"
if cmd.op == '||':
res = executeShCmd(cmd.lhs, cfg, cwd, results)
if res != 0:
res = executeShCmd(cmd.rhs, cfg, cwd, results)
return res
if cmd.op == '&&':
res = executeShCmd(cmd.lhs, cfg, cwd, results)
if res is None:
return res
if res == 0:
res = executeShCmd(cmd.rhs, cfg, cwd, results)
return res
raise ValueError,'Unknown shell command: %r' % cmd.op
assert isinstance(cmd, ShUtil.Pipeline)
procs = []
input = subprocess.PIPE
stderrTempFiles = []
# To avoid deadlock, we use a single stderr stream for piped
# output. This is null until we have seen some output using
# stderr.
for i,j in enumerate(cmd.commands):
# Apply the redirections, we use (N,) as a sentinal to indicate stdin,
# stdout, stderr for N equal to 0, 1, or 2 respectively. Redirects to or
# from a file are represented with a list [file, mode, file-object]
# where file-object is initially None.
redirects = [(0,), (1,), (2,)]
for r in j.redirects:
if r[0] == ('>',2):
redirects[2] = [r[1], 'w', None]
elif r[0] == ('>>',2):
redirects[2] = [r[1], 'a', None]
elif r[0] == ('>&',2) and r[1] in '012':
redirects[2] = redirects[int(r[1])]
elif r[0] == ('>&',) or r[0] == ('&>',):
redirects[1] = redirects[2] = [r[1], 'w', None]
elif r[0] == ('>',):
redirects[1] = [r[1], 'w', None]
elif r[0] == ('>>',):
redirects[1] = [r[1], 'a', None]
elif r[0] == ('<',):
redirects[0] = [r[1], 'r', None]
else:
raise NotImplementedError,"Unsupported redirect: %r" % (r,)
# Map from the final redirections to something subprocess can handle.
final_redirects = []
for index,r in enumerate(redirects):
if r == (0,):
result = input
elif r == (1,):
if index == 0:
raise NotImplementedError,"Unsupported redirect for stdin"
elif index == 1:
result = subprocess.PIPE
else:
result = subprocess.STDOUT
elif r == (2,):
if index != 2:
raise NotImplementedError,"Unsupported redirect on stdout"
result = subprocess.PIPE
else:
if r[2] is None:
if kAvoidDevNull and r[0] == '/dev/null':
r[2] = tempfile.TemporaryFile(mode=r[1])
else:
r[2] = open(r[0], r[1])
# Workaround a Win32 and/or subprocess bug when appending.
if r[1] == 'a':
r[2].seek(0, 2)
result = r[2]
final_redirects.append(result)
stdin, stdout, stderr = final_redirects
# If stderr wants to come from stdout, but stdout isn't a pipe, then put
# stderr on a pipe and treat it as stdout.
if (stderr == subprocess.STDOUT and stdout != subprocess.PIPE):
stderr = subprocess.PIPE
stderrIsStdout = True
else:
stderrIsStdout = False
# Don't allow stderr on a PIPE except for the last
# process, this could deadlock.
#
# FIXME: This is slow, but so is deadlock.
if stderr == subprocess.PIPE and j != cmd.commands[-1]:
stderr = tempfile.TemporaryFile(mode='w+b')
stderrTempFiles.append((i, stderr))
# Resolve the executable path ourselves.
args = list(j.args)
args[0] = Util.which(args[0], cfg.environment['PATH'])
if not args[0]:
raise InternalShellError(j, '%r: command not found' % j.args[0])
procs.append(subprocess.Popen(args, cwd=cwd,
stdin = stdin,
stdout = stdout,
stderr = stderr,
env = cfg.environment,
close_fds = kUseCloseFDs))
# Immediately close stdin for any process taking stdin from us.
if stdin == subprocess.PIPE:
procs[-1].stdin.close()
procs[-1].stdin = None
# Update the current stdin source.
if stdout == subprocess.PIPE:
input = procs[-1].stdout
elif stderrIsStdout:
input = procs[-1].stderr
else:
input = subprocess.PIPE
# FIXME: There is probably still deadlock potential here. Yawn.
procData = [None] * len(procs)
procData[-1] = procs[-1].communicate()
for i in range(len(procs) - 1):
if procs[i].stdout is not None:
out = procs[i].stdout.read()
else:
out = ''
if procs[i].stderr is not None:
err = procs[i].stderr.read()
else:
err = ''
procData[i] = (out,err)
# Read stderr out of the temp files.
for i,f in stderrTempFiles:
f.seek(0, 0)
procData[i] = (procData[i][0], f.read())
exitCode = None
for i,(out,err) in enumerate(procData):
res = procs[i].wait()
# Detect Ctrl-C in subprocess.
if res == -signal.SIGINT:
raise KeyboardInterrupt
results.append((cmd.commands[i], out, err, res))
if cmd.pipe_err:
# Python treats the exit code as a signed char.
if res < 0:
exitCode = min(exitCode, res)
else:
exitCode = max(exitCode, res)
else:
exitCode = res
if cmd.negate:
exitCode = not exitCode
return exitCode
from WordVector import *
import Util
mContentVectorTrainer = ChineseVectorTrainer()
# read all vocabulary
client = MongoClient("localhost", 27017)
db = client.DeepSearch
mContentVectorTrainer.FeedWords(db.words.find({}))
mContentVectorTrainer.FeedWords(db.probWords.find({}))
lines = []
datafile = open("wiki_cn_out_00", "r")
for i in range(100000):
lines.append(datafile.readline().replace(" \n", "").split(" "))
mContentVectorTrainer.FeedContent(lines)
# print(mContentVectorTrainer.data)
# mContentVectorTrainer.ShowData(1, 8, 2, 1)
mContentVectorTrainer.InitConfig()
mContentVectorTrainer.TrainModal()
# print(mContentVectorTrainer.weights[1, :])
# print(mContentVectorTrainer.biases.shape)
wordList = mContentVectorTrainer.GetResultWordList()
# for index in range(1):
# print(wordList[index])
for wordItem in wordList:
Util.UpdateWord(wordItem)
#!/usr/bin/python
import sys
import os
sys.path.append("/home/won/WON/")
import Util
from multiprocessing import Pool
F = Util.ReadColumn("Tophat/configure.list", [0, 1])
Title = ["Tophat/%s/accepted_hits.bam.count" % x[0] for x in F]
Column = [x[1] for x in F]
note = 0
pre = ""
for i in xrange(len(Column)):
if Column[i] != pre:
note = 1
pre = Column[i]
Column[i] = Column[i] + "_%d" % note
note += 1
cmd = "paste "
for mlist in Title:
cmd += "%s " % mlist
cmd += "| cut -f 1"
for i in xrange(1, 100):
cmd += ",%d" % (2 * i)
cmd += " > gene.tmp"
os.system(cmd)
L = Util.ReadList("gene.tmp")
Ens2Gene = Util.ReadDict("mm9.Trans2Gene.dat", [0], [1])
import os
import sys
base_path = os.path.split(os.path.realpath(__file__))[0]
sys.path.append(base_path)
# lib_path = os.path.join(base_path, 'lib')
# sys.path.append(lib_path)
import template_file
import tempfile
import Util
import subprocess
import time
from socket import *
import json
# 日志对象
logger = Util.getLogger('login')
# 操作超时时间
# wait_timeout = int(Util.getConfig('wait_timeout'))
# 自动登录watcher端口
watcher_port = int(Util.getConfig('watcher_port'))
# 堡垒机登录名
# login_name = Util.getConfig('login_name')
# 登录的ip
target_ip = crt.Arguments.GetArg(0)
logger.info('login plsql script start, target ip: ' + target_ip)
tab = crt.GetScriptTab()
tab.Screen.Synchronous = True
try:
if args.provider == "digital_ocean" and not args.linear:
logger.warning(
"Provider '%s' does not support parallel creation of VMs. Linear creation is automatically enabled. See https://github.com/devopsgroup-io/vagrant-digitalocean/pull/230 for further details."
% args.provider)
args.linear = True
if len(args.databases) > 1 and (args.nodestroy or args.noshutdown):
logger.warning(
"The arguments --noshutdown and --nodestroy do not work with multiple databases at one run. Both are automatically disabled."
)
args.nodestroy = False
args.noshutdown = False
# File checks and deletions (if necessary)
for folder in args.vagrantfolders:
if not Util.check_folder(folder, logger):
exit(-1)
for vagrantCredFile in vagrantCredFiles:
found_cred_file = False
for folder in args.vagrantfolders:
if Util.check_file_exists(os.path.join(folder, vagrantCredFile)):
found_cred_file = True
if not found_cred_file:
logger.error("%s not found in any of the given vagrantfolders (%s)." %
(vagrantCredFile, args.vagrantfolders))
exit(-1)
if not Util.check_folder(args.tmpfolder, logger):
exit(-1)
if not Util.delete_file(logFile, logger, True):
exit(-1)
def train(sqn, save_model_path):
print('Starting train...')
# Hyper parameters
epochs = 1
batch_size = 128
keep_probability = 0.7
learning_rate = 0.001
n_batches = 5 #CIFAR10 dataset in the python version has 5 batches
x = tf.placeholder(tf.float32, shape=(None, 32, 32, 3), name='input_x')
y = tf.placeholder(tf.float32, shape=(None, 10), name='output_y')
keep_prob = tf.placeholder(tf.float32, name='keep_prob')
logits = sqn.inference(x)
# Loss and Optimizer
cost = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=y))
l2Loss = sum(list(map(lambda x: tf.nn.l2_loss(x), sqn.all_weights)))
beta = 1e-5
cost = cost + beta * l2Loss
optimizer = tf.train.AdamOptimizer(
learning_rate=learning_rate).minimize(cost)
# Accuracy
correct_pred = tf.equal(tf.argmax(logits, 1), tf.argmax(y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32),
name='accuracy')
valid_features, valid_labels = Util.load_preprocess_validation_data()
testing_features, testing_labels = Util.load_preprocess_testing_data()
print('Training now...')
with tf.Session() as sess:
# Initializing the variables
sess.run(tf.global_variables_initializer())
# Training cycle
for epoch in range(epochs):
# Loop over all batches
for batch_i in range(1, n_batches + 1):
for batch_features, batch_labels in Util.load_preprocess_training_batch(
batch_i, batch_size):
sess.run(optimizer,
feed_dict={
x: batch_features,
y: batch_labels,
keep_prob: keep_probability
})
print('Epoch {:>2}, CIFAR-10 Batch {}: '.format(
epoch + 1, batch_i),
end='')
# Print stats
loss = sess.run(cost,
feed_dict={
x: batch_features,
y: batch_labels,
keep_prob: keep_probability
})
train_acc = sess.run(accuracy,
feed_dict={
x: batch_features,
y: batch_labels,
keep_prob: keep_probability
})
valid_acc = sess.run(accuracy,
feed_dict={
x: valid_features,
y: valid_labels,
keep_prob: keep_probability
})
testing_acc = sess.run(accuracy,
feed_dict={
x: testing_features,
y: testing_labels,
keep_prob: keep_probability
})
print(
'Loss: {:>10.4f} Train Acc: {:.6f} Validation Accuracy: {:.6f} Testing Acc: {:.6f}'
.format(loss, train_acc, valid_acc, testing_acc))
if (epoch % 10 == 0):
# Save Model
saver = tf.train.Saver()
save_path = saver.save(sess, save_model_path)
def shift2AnotherCity(self):
self.id = Util.generateRnd(6)
def __init__(self,*args):
if not os.path.sep in __file__:
scriptdir = os.getcwd()
else:
scriptdir = os.path.dirname(__file__)
#n3file = os.path.sep.join([scriptdir,"etc","sfs-extra.n3"])
#n3url = "file://" + n3file.replace("\\","/")
#print "scriptdir: %s" % scriptdir
#print "n3file: %s" % n3file
#print "n3url: %s" % n3url
self.graph = Graph()
n3file = Util.relpath(scriptdir + "/etc/sfs-extra.n3")
# print "loading n3file %s" % n3file
self.graph.load(n3file, format="n3")
self.roots = []
self.uriformatter = {}
self.decl = ""
self.namedlaws = {}
self.load_ebnf(scriptdir+"/etc/base.ebnf")
self.args = args
if self.LAGRUM in args:
productions = self.load_ebnf(scriptdir+"/etc/lagrum.ebnf")
for p in productions:
self.uriformatter[p] = self.sfs_format_uri
self.namedlaws.update(self.get_relations(RDFS.label))
self.roots.append("sfsrefs")
self.roots.append("sfsref")
if self.KORTLAGRUM in args:
# om vi inte redan laddat lagrum.ebnf måste vi göra det
# nu, eftersom kortlagrum.ebnf beror på produktioner som
# definerats där
if not self.LAGRUM in args:
self.load_ebnf(scriptdir+"/etc/lagrum.ebnf")
productions = self.load_ebnf(scriptdir+"/etc/kortlagrum.ebnf")
for p in productions:
self.uriformatter[p] = self.sfs_format_uri
DCT = Namespace("http://purl.org/dc/terms/")
d = self.get_relations(DCT['alternate'])
self.namedlaws.update(d)
lawlist = [x.encode(SP_CHARSET) for x in d.keys()]
# Make sure longer law abbreviations come before shorter
# ones (so that we don't mistake "3 § MBL" for "3 § MB"+"L")
lawlist.sort(cmp=lambda x,y:len(y)-len(x))
self.decl += "LawAbbreviation ::= ('%s')\n" % "'/'".join(lawlist)
self.roots.insert(0,"kortlagrumref")
if self.EGLAGSTIFTNING in args:
productions = self.load_ebnf(scriptdir+"/etc/eglag.ebnf")
for p in productions:
self.uriformatter[p] = self.eglag_format_uri
self.roots.append("eglagref")
if self.FORARBETEN in args:
productions = self.load_ebnf(scriptdir+"/etc/forarbeten.ebnf")
for p in productions:
self.uriformatter[p] = self.forarbete_format_uri
self.roots.append("forarbeteref")
if self.RATTSFALL in args:
productions = self.load_ebnf(scriptdir+"/etc/rattsfall.ebnf")
for p in productions:
self.uriformatter[p] = self.rattsfall_format_uri
self.roots.append("rattsfallref")
if self.EGRATTSFALL in args:
productions = self.load_ebnf(scriptdir+"/etc/egratt.ebnf")
for p in productions:
self.uriformatter[p] = self.egrattsfall_format_uri
self.roots.append("ecjcaseref")
self.decl += "root ::= (%s/plain)+\n" % "/".join(self.roots)
# pprint(productions)
# print self.decl.decode(SP_CHARSET,'ignore')
self.parser = Parser(self.decl, "root")
self.tagger = self.parser.buildTagger("root")
# print "tagger length: %d" % len(repr(self.tagger))
self.verbose = False
self.depth = 0
# SFS-specifik kod
self.currentlaw = None
self.currentchapter = None
self.currentsection = None
self.currentpiece = None
self.lastlaw = None
self.currentlynamedlaws = {}
def main():
scalingFac = 12
findAccOrArgMaxOrPredVal = 0
restoreWeights = True
onlysavegraph = False
save_model_path = './TrainedModel/model'
doTraining = False
inp = None
if (len(sys.argv) > 1):
inp = sys.argv[1]
if (inp == 'train'):
doTraining = True
elif (inp == 'savegraph'):
findAccOrArgMaxOrPredVal = 1
restoreWeights = False
onlysavegraph = True
testing_features, testing_labels = Util.get_sample_points(
2, 4555, 4556)
elif (inp == 'testSingleTestInp'):
testBatchInpNum = int(sys.argv[2])
findAccOrArgMaxOrPredVal = 1
restoreWeights = True
onlysavegraph = False
all_testing_features, all_testing_labels = Util.load_preprocess_testing_data(
)
testing_features, testing_labels = all_testing_features[
testBatchInpNum:testBatchInpNum +
1], all_testing_labels[testBatchInpNum:testBatchInpNum + 1]
elif (inp == 'testSingleTestInpAndSaveData'):
testBatchInpNum = int(sys.argv[2])
findAccOrArgMaxOrPredVal = int(sys.argv[3])
restoreWeights = True
onlysavegraph = False
all_testing_features, all_testing_labels = Util.load_preprocess_testing_data(
)
testing_features, testing_labels = all_testing_features[
testBatchInpNum:testBatchInpNum +
1], all_testing_labels[testBatchInpNum:testBatchInpNum + 1]
# testing_features, testing_labels = numpy.full((1,32,32,3),0.01), numpy.full((1,10),0.01)
elif (inp == 'savegraphAndDataBatch'):
batchNum = int(sys.argv[2])
imgStartNum = int(sys.argv[3])
imgEndNum = int(sys.argv[4])
findAccOrArgMaxOrPredVal = 1
restoreWeights = False
onlysavegraph = True
testing_features, testing_labels = Util.get_sample_points(
batchNum, imgStartNum, imgEndNum)
elif (inp == 'testBatchInp'):
imgStartNum = int(sys.argv[2])
imgEndNum = int(sys.argv[3])
findAccOrArgMaxOrPredVal = 1
restoreWeights = True
onlysavegraph = False
all_testing_features, all_testing_labels = Util.load_preprocess_testing_data(
)
testing_features, testing_labels = all_testing_features[
imgStartNum:imgEndNum], all_testing_labels[
imgStartNum:imgEndNum]
elif (inp == 'findAndSaveCorrectTestImg'):
findAccOrArgMaxOrPredVal = 2
restoreWeights = True
onlysavegraph = False
testing_features, testing_labels = Util.load_preprocess_testing_data(
)
testing_features, testing_labels = testing_features[
0:100], testing_labels[0:100]
else:
if (inp != ""):
print("WARNING : Given option didn't match any known value.")
testing_features, testing_labels = Util.load_preprocess_testing_data(
)
sqn = SqueezeNet1(use_cons_init=onlysavegraph)
if doTraining:
train(sqn, save_model_path)
else:
with tf.Session() as sess:
pred = infer(sqn,
sess,
testing_features,
testing_labels,
save_model_path,
findAccOrArgMaxOrPredVal=findAccOrArgMaxOrPredVal,
restoreWeights=restoreWeights,
onlysavegraph=onlysavegraph)
if findAccOrArgMaxOrPredVal == 1 and not (onlysavegraph):
print("Actual labels = ", testing_labels)
print("ArgMax in actual label : ",
numpy.argmax(testing_labels, axis=1))
if (inp == 'findAndSaveCorrectTestImg'):
print('Running ' + inp)
print(pred[0].shape)
findAndSaveCorrectTestImg(pred, testing_features,
testing_labels,
'./testPred/CorrectImg/',
'./testPred/IncorrectImg/',
'./testPred/TestInputs/', sess, sqn,
scalingFac)
if (inp == 'savegraphAndDataBatch'
or inp == 'testSingleTestInpAndSaveData'):
imgFileName = 'SqNet_CIFAR_img.inp'
weightsFileName = 'SqNet_CIFAR_weights.inp'
for ii, curFeature in enumerate(testing_features):
if ii == 0:
DumpTFMtData.dumpImageDataInt(curFeature, imgFileName,
scalingFac, 'w')
else:
DumpTFMtData.dumpImageDataInt(curFeature, imgFileName,
scalingFac, 'a')
DumpTFMtData.dumpTrainedWeightsInt(sess, sqn.all_weights,
weightsFileName, scalingFac,
'w')
else:
# respond to the original requester
try:
self.transaction_map[transaction]["socket"].wfile.write(line + "\n")
except socket.error, e:
logger.warning("It can't write on requestor socket...")
# remove from map if end of transaction
if Util.get_var("(ackend)", line) != "":
logger.debug("Closing transaction: %s" % transaction)
if self.transaction_map.has_key(transaction):
del self.transaction_map[transaction]
# assume we are a command request to an agent
else:
id = Util.get_var("id=\"([^\"]+)\"", line)
if id == "" or id == "all":
logger.debug("Broadcasting to all ...")
if len(self.control_agents) == 0:
response = line + ' errno="-1" error="No agents available." ackend\n'
else:
# send line to each control agent
for key in self.control_agents:
# add this connection to the transaction map
transaction = self.__transaction_id_get()
self.transaction_map[transaction] = {'socket':requestor, 'time':time.time()}
npUrl = ""
source = {''}
SAnews = []
for t in type_:
logFile = logPath.format(t, datetime.now().strftime('%Y%m%d'))
payload["channel_id"] = t
DateStr = '2020-12-01'
startDate = datetime.strptime(DateStr, '%Y-%m-%d')
for i in range(0, 1):
#nextDate = startDate + relativedelta(months=1)
nextDate = startDate + relativedelta(days=1)
Util.writeLog(
logFile, "StartTime:{0} & EndTime:{1}".format(
startDate.strftime('%Y-%m-%d'),
nextDate.strftime('%Y-%m-%d')))
payload['startDate'] = startDate.strftime('%Y-%m-%d')
payload['endDate'] = nextDate.strftime('%Y-%m-%d')
payload['page'] = 1
res = requests.post('https://api.infominer.io/search',
json=payload,
headers=headers,
verify=False,
timeout=30)
jd = res.json()
pageNum = math.ceil(jd["total_count"] / 100)
data = jd["data"]
def parse(path, block_index, tx_index):
print('Parsing tx to JSON file', path, '. . .')
time_start = time.time() # get starting time
with open(path,
'w') as json_file: # open the output (json) file in writing mode
with open(
block_index.path, 'rb'
) as block_file: # open the input (.dat) file in reading binary mode
tx_ins = []
tx_outs = []
in_prev_hash = []
in_prev_index = []
out_address = []
out_index = []
out_value = []
current_block = -1
for i in range(0, len(tx_index.byte_index)
): # iterate over all the tx in the tx index
# clearing
tx_ins.clear()
tx_outs.clear()
in_prev_hash.clear()
in_prev_index.clear()
out_address.clear()
out_value.clear()
# parsing
if current_block != tx_index.block_num[
i]: # parsing timestamp (same timestamp for tx in the same block)
current_block = tx_index.block_num[i]
block_file.seek(
block_index.byte_index[current_block] + 76
) # magic num : 4, block size : 4, version : 4, previous hash : 32, merkle root : 32
byte_time = block_file.read(4) # block timestamp : 4
tx_timestamp = int.from_bytes(byte_time,
byteorder='little')
block_file.seek(tx_index.byte_index[i]) # parsing tx hash
byte_tx = block_file.read(tx_index.tx_size[i])
h_sha256 = hashlib.sha256()
h_sha256.update(byte_tx)
h_bytes = h_sha256.digest()
h_sha256 = hashlib.sha256()
h_sha256.update(h_bytes)
tx_id_hash = h_sha256.hexdigest()
tx_id = Util.formatHashString(tx_id_hash, True, True)
if len(tx_id) != 66: print(tx_id_hash, tx_id)
block_file.seek(
tx_index.byte_index[i] +
4) # parsing number of tx inputs, tx version : 4
byte_in_count = block_file.read(1) # reading the INPUT COUNT
# check if varInt is on 1, 2, 4, or 8 bytes
if (byte_in_count[0] == 253
): # varInt is on 2 bytes AFTER the prefix
byte_in_count = block_file.read(2)
elif (byte_in_count[0] == 254
): # varInt is on 4 bytes AFTER the prefix
byte_in_count = block_file.read(4)
elif (byte_in_count[0] == 255
): # varInt is on 8 bytes AFTER the prefix
byte_in_count = block_file.read(8)
# else: # varInt was on 1 bytes, nothing to do
in_count = int.from_bytes(byte_in_count, byteorder='little')
ins = 0
while ins < in_count:
byte_hash = block_file.read(32) # previous tx hash : 32
first_hash = hex(int.from_bytes(byte_hash,
byteorder='big'))
prev_hash = Util.formatPrevHashString(
first_hash[2:], True, True)
if (len(prev_hash) != 66 and len(prev_hash) != 2):
print(byte_hash, first_hash, prev_hash)
in_prev_hash.append(prev_hash)
byte_prev_index = block_file.read(
4) # previous tx index : 4
prev_index = int.from_bytes(byte_prev_index,
byteorder='little')
in_prev_index.append(prev_index)
byte_script_len = block_file.read(
1) # reading the INPUT COUNT
# check if varInt is on 1, 2, 4, or 8 bytes
if (byte_script_len[0] == 253
): # varInt is on 2 bytes AFTER the prefix
byte_script_len = block_file.read(2)
elif (byte_script_len[0] == 254
): # varInt is on 4 bytes AFTER the prefix
byte_script_len = block_file.read(4)
elif (byte_script_len[0] == 255
): # varInt is on 8 bytes AFTER the prefix
byte_script_len = block_file.read(8)
# else: # varInt was on 1 bytes, nothing to do
script_len = int.from_bytes(byte_script_len,
byteorder='little')
block_file.read(script_len) # script : script_len
block_file.read(4) # sequence : 4
ins += 1
byte_out_count = block_file.read(1) # reading the OUTPUT COUNT
# check if varInt is on 1, 2, 4, or 8 bytes
if (byte_out_count[0] == 253
): # varInt is on 2 bytes AFTER the prefix
byte_out_count = block_file.read(2)
elif (byte_out_count[0] == 254
): # varInt is on 4 bytes AFTER the prefix
byte_out_count = block_file.read(4)
elif (byte_out_count[0] == 255
): # varInt is on 8 bytes AFTER the prefix
byte_out_count = block_file.read(8)
# else: # varInt was on 1 bytes, nothing to do
out_count = int.from_bytes(byte_out_count, byteorder='little')
outs = 0
while outs < out_count:
byte_value = block_file.read(8) # value : 8
value = int.from_bytes(byte_value, byteorder='little')
out_value.append(value)
out_index.append(outs)
byte_script_len = block_file.read(
1) # reading the INPUT COUNT
# check if varInt is on 1, 2, 4, or 8 bytes
if (byte_script_len[0] == 253
): # varInt is on 2 bytes AFTER the prefix
byte_script_len = block_file.read(2)
elif (byte_script_len[0] == 254
): # varInt is on 4 bytes AFTER the prefix
byte_script_len = block_file.read(4)
elif (byte_script_len[0] == 255
): # varInt is on 8 bytes AFTER the prefix
byte_script_len = block_file.read(8)
# else: # varInt was on 1 bytes, nothing to do
script_len = int.from_bytes(byte_script_len,
byteorder='little')
byte_script = block_file.read(
script_len) # script : script_len
script = Util.intToHexString(
int.from_bytes(byte_script, byteorder='big'), False,
False)
address_hex = Util.getDataFromHexStringScript(script)
if (len(address_hex) == 130):
address_hex = Util.pubKStringToAddress(address_hex)
elif (len(address_hex) == 40):
address_hex = Util.base58Check(address_hex)
else:
address_hex = 'UNABLE_TO_PARSE_ADDRESS'
out_address.append(address_hex)
outs += 1
tx_total_value = 0
for outs in range(0, len(out_value)):
tx_total_value += out_value[outs] # sum of outs values
for ins in range(0, len(in_prev_hash)):
if in_prev_hash[ins] == '0x':
tx_ins.append(
JsonTx.JsonTxIn('MINING_REWARD', in_prev_hash[ins],
-1, tx_total_value))
elif len(in_prev_hash) == 1:
tx_ins.append(
JsonTx.JsonTxIn('NO_ADDRESS', in_prev_hash[ins],
in_prev_index[ins], tx_total_value)
) # if there is only 1 input its value is the total value of the tx
else:
if len(in_prev_hash[ins]) != 66:
print(in_prev_hash[ins], len(in_prev_hash[ins]))
tx_ins.append(
JsonTx.JsonTxIn('NO_ADDRESS', in_prev_hash[ins],
in_prev_index[ins], 'NO_VALUE'))
for outs in range(0, len(out_value)):
tx_outs.append(
JsonTx.JsonTxOut(out_address[outs], out_index[outs],
out_value[outs]))
tx = JsonTx.JsonTx('NO_X_RATE', tx_timestamp, tx_total_value,
tx_ins, tx_outs, tx_id)
json_str = json.dumps(tx, default=jsonDefault)
json_file.write(json_str + '\n')
block_file.closed # close the file
json_file.closed # close the file
time_end = time.time()
print('end of parsing in', time_end - time_start, 's')
def process(self, requestor, command, line):
"""Process all the requests coming from the webgui
"""
self.__mutexRquest.acquire()
try:
response = ""
action = Util.get_var("action=\"([^\"]+)\"", line)
if action == "connect":
id = Util.get_var("id=\"([^\"]+)\"", line)
sensor_id = Util.get_var("sensor_id=\"([0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12})\"", line)
if id != "":
requestor.set_id(id)
else:
requestor.set_id("%s_%i" % (requestor.client_address))
requestor.set_sensorID(sensor_id)
logger.debug("Adding control agent %s to the list., with key %s [%s]" % (id, requestor.getRequestorIP(),requestor.get_sensorID()))
# add this connection to our control agent collection
self.control_agents[requestor.getRequestorIP()] = requestor
# indicate we're good to go
response = 'ok id="%s"\n' % id
self.__ntop_apache_manager.refreshDefaultNtopConfiguration(first_sensor_name=id, must_reload=True)
timer = Timer(5.0, self.refreshAgentCache, (requestor, requestor.getRequestorIP(), id,))
timer.start()
timer = Timer(10.0, self.refreshAgentInventoryTasks, (requestor, requestor.getRequestorIP(), id,))
timer.start()
elif action == "getconnectedagents":
# set up response
response = "control getconnectedagents"
# indicate the number of agents connected
keys = self.control_agents.keys()
response += ' count="%d"' % len(keys)
# build the connected list
if keys != None:
# sort list before sending
keys.sort()
names = ""
for key in keys:
names += "%s=%s|" % (self.control_agents[key].getRequestorID(), key)
names = names[:-1]
else:
names = ""
response += ' names="%s" errno="0" error="Success." ackend\n' % names
elif action == "gettotalagentsconnected":
# set up response
response = "control gettotalagentsconnected"
# indicate the number of agents connected
keys = self.control_agents.keys()
response += ' count="%d" errno="0" error="Success." ackend\n' % len(keys)
elif action == "getconnectedagents_p":
# set up response
response = "control getconnectedagents_p"
begin_index = 0
end_index = 0
try:
begin_index = int(Util.get_var("begin_index=\"([^\"]+)\"", line))
except:
begin_index = 0
try:
end_index = int(Util.get_var("end_index=\"([^\"]+)\"", line))
except:
end_index = 0
pag_size = end_index - begin_index
real_size = 0
if pag_size > 0:
# indicate the number of agents connected
keys = self.control_agents.keys()
response += ' count="%d"' % len(keys)
# build the connected list
if keys != None:
# sort list before sending
keys.sort()
if end_index >= len(keys):
page_keys = keys[begin_index:]
else:
page_keys = keys[begin_index:end_index]
real_size = len(page_keys)
names = ""
for key in page_keys:
names += "%s=%s|" % (self.control_agents[key].getRequestorID(), key)
names = names[:-1]
#names = "|".join(page_keys)
else:
names = ""
response += ' page_size="%d" names="%s" errno="0" error="Success." ackend\n' % (real_size, names)
else:
response += 'errno="-1" error="Invalid page size requested." ackend\n'
elif action == "refresh_asset_list":
for agent_id in self.control_agents.keys():
self.refreshAgentCache(self.control_agents[agent_id], self.control_agents[agent_id].getRequestorIP(), self.control_agents[agent_id].getRequestorID())
elif action == "refresh_inventory_task":
for agent_id in self.control_agents.keys():
self.refreshAgentInventoryTasks(self.control_agents[agent_id], self.control_agents[agent_id].getRequestorIP(), self.control_agents[agent_id].getRequestorID())
response="%s ok\n" % line
elif action == "getOCSInventory":
logger.info("Request for ocs inventory")
response = self.getOCSInventoryData()
elif action == "getNagiosInventory":
logger.info("Request for ocs inventory")
response = self.getNagiosInventory()
else:
# check if we are a transaction
transaction = Util.get_var("transaction=\"([^\"]+)\"", line)
if transaction != "":
if transaction not in self.transaction_map:
logger.error("Transaction %s has no apparent originator!", transaction)
else:
# respond to the original requester
try:
self.transaction_map[transaction]["socket"].wfile.write(line + "\n")
except socket.error, e:
logger.warning("It can't write on requestor socket...")
# remove from map if end of transaction
if Util.get_var("(ackend)", line) != "":
logger.debug("Closing transaction: %s" % transaction)
if self.transaction_map.has_key(transaction):
del self.transaction_map[transaction]
# assume we are a command request to an agent
else:
def MultiPO(Board, Moderators, DaysAgo):
global author_list
global ip_list
global new_line
global ptt_bot
global publish
global ask
global mail
Util.PTTBot = PTTBot
Util.post_search = None
Util.current_board = Board
Util.Moderators = Moderators
global publish_content
if PublishContent is None:
PublishContent = '此內容由 IP 分析程式產生' + NewLine
PublishContent += '由 CodingMan 透過 PTT Library 開發，' + NewLine * 2
PublishContent += 'PTT Library: https://github.com/Truth0906/PTTLibrary' + NewLine
PublishContent += '開發手冊: https://hackmd.io/@CodingMan/PTTLibraryManual' + NewLine
PublishContent += 'IP 分析程式: https://github.com/Truth0906/PTTModeratorTool' + NewLine
StartTime = time.time()
AuthorList = dict()
IPList = dict()
CurrentDate = Util.get_date(DaysAgo)
PTTBot.log(f'開始 {Board} 板 IP 分析')
PTTBot.log(f'從 {CurrentDate} 開始分析')
Start, _ = Util.find_post_rrange(DaysAgo, show=False)
NewestIndex = PTTBot.getNewestIndex(
PTT.IndexType.Board,
Board=Board,
)
End = NewestIndex
PTTBot.log('編號範圍 ' + str(Start) + ' ~ ' + str(End))
ErrorPostList, DeleteCount = PTTBot.crawlBoard(
PostHandler,
Util.current_board,
StartIndex=Start,
EndIndex=End,
)
MultiPOResult = ''
for Suspect, TitleAuthorList in AuthorList.items():
if len(TitleAuthorList) <= 1:
continue
# print('=' * 5 + ' ' + Suspect + ' ' + '=' * 5)
if MultiPOResult != '':
MultiPOResult += NewLine
for Title in TitleAuthorList:
MultiPOResult += CurrentDate + ' ' + \
Suspect + ' □ ' + Title + NewLine
IPResult = ''
for IP, SuspectList in IPList.items():
# print('len:', len(SuspectList))
if len(SuspectList) <= 1:
continue
# print('IP:', IP)
IPResult += 'IP: ' + IP + NewLine
for Line in SuspectList:
# print('> ' + CurrentDate + ' ' + Line)
IPResult += CurrentDate + ' ' + Line + NewLine
EndTime = time.time()
Title = CurrentDate + f' {Board} 板 IP 分析結果'
PublishContent += NewLine
PublishContent += '◆ ' + CurrentDate + f' {Board} 板 IP 分析結果'
Time = math.ceil(EndTime - StartTime)
Min = int(Time / 60)
Sec = int(Time % 60)
Content = '此內容由自動抓多 PO 程式產生' + NewLine
Content += '共耗時'
PublishContent += '共耗時'
if Min > 0:
Content += f' {Min} 分'
PublishContent += f' {Min} 分'
Content += f' {Sec} 秒執行完畢' + NewLine * 2
PublishContent += f' {Sec} 秒執行完畢' + NewLine * 2
Content += '此程式是由 CodingMan 透過 PTT Library 開發，' + NewLine * 2
Content += f'蒐集範圍為 ALLPOST 搜尋 ({Board}) 情況下編號 ' + \
str(Start) + ' ~ ' + str(End) + NewLine
Content += f'共 {End - Start + 1} 篇文章' + NewLine * 2
PublishContent += f' 蒐集範圍為 ALLPOST 搜尋 ({Board}) 情況下編號 ' + \
str(Start) + ' ~ ' + str(End) + NewLine
PublishContent += f' 共 {End - Start + 1} 篇文章' + NewLine * 2
if MultiPOResult != '':
Content += MultiPOResult
MultiPOResult = MultiPOResult.strip()
for line in MultiPOResult.split(NewLine):
PublishContent += ' ' + line + NewLine
else:
Content += '◆ ' + CurrentDate + ' 無人違反多 PO 板規' + NewLine
PublishContent += ' ' + '◆ ' + CurrentDate + ' 無人違反多 PO 板規' + NewLine
if IPResult != '':
Content += IPResult
IPResult = IPResult.strip()
for line in IPResult.split(NewLine):
PublishContent += ' ' + line + NewLine
else:
Content += NewLine + f'◆ 沒有發現特定 IP 有 {MaxPost + 1} 篇以上文章' + NewLine
PublishContent += NewLine + \
f' ◆ 沒有發現特定 IP 有 {MaxPost + 1} 篇以上文章' + NewLine
Content += NewLine + '內容如有失準，歡迎告知。' + NewLine
Content += '此訊息同步發送給 ' + ' '.join(Util.Moderators) + NewLine
Content += NewLine
Content += ID
print(Title)
print(Content)
if Ask:
Choise = input('要發佈嗎? [Y]').lower()
Publish = (Choise == 'y') or (Choise == '')
if Mail:
for Moderator in Util.Moderators:
PTTBot.mail(Moderator, Title, Content, 0)
PTTBot.log('寄信給 ' + Moderator + ' 成功')
else:
PTTBot.log('取消寄信')
def apply(x):
if x < 3:
mode = 0
while mode != 1 and mode != 2:
mode = int(
input("Mode de (dé)chiffrement : \n\t1. ECB\n\t2. CBC\n"))
key_len = 0
while (key_len != 256) and (key_len != 512) and (key_len != 1024):
key_len = int(input("Taille de la clé en bits(256/512/1024): "))
file_key = input("Chemin du fichier de la clé: ")
passwd_user = input("Mot de passe pour chiffrer la clé: ")
file_path = input("Chemin du fichier à (dé)chiffrer : ")
word_len = 64
num_words = int(key_len / word_len)
word_len_bytes = int(word_len / 8)
if x == 1:
file_data = IO.readfile(file_path, word_len, 1)
file_data_list = Util.organize_data_list(file_data, num_words)
encrypted_file = ThreeFish.threefish_chiffrement(
file_data_list, mode, key_len, passwd_user, file_key)
IO.write_2D_list(file_path, encrypted_file, word_len_bytes)
IO.rename_file(file_path, 0)
print("Chiffrement terminé.")
elif x == 2:
ciph_data = IO.readfile(file_path, word_len, 0)
ciph_data_list = Util.organize_data_list(ciph_data, num_words)
clear_file_data, valeur_pad = ThreeFish.threefish_dechiffrement(
ciph_data_list, mode, key_len, word_len, passwd_user, file_key)
IO.write_file_list_pad(file_path, clear_file_data, word_len_bytes,
valeur_pad)
IO.rename_file(file_path, 1)
print("Déchiffrement terminé.")
elif x == 3:
filepath = input("Chemin du fichier à chiffrer:")
file_data = IO.read_bytes(filepath)
ans = ''
while ans != 'y' and ans != 'n' and ans != 'Y' and ans != 'N':
ans = input("Avez-vous un fichier de clé publique ? (y/n) ")
if ans == 'y' or ans == 'Y':
keypath = input("Chemin de la clé publique: ")
ciph_data = CramerShoup.encode_with_key(file_data, keypath)
else:
k = int(input("Taille de clé souhaitée en bits: "))
password = input("Mot de passe pour chiffrer la clé privée: ")
keypath = input("Chemin du répertoire des clés: ")
ciph_data = CramerShoup.encode_no_key(file_data, keypath, k,
password)
ciph_bytes = Conversions.int_list2bytes(ciph_data, 8)
IO.write_bytes(filepath, ciph_bytes)
IO.rename_file(filepath, 0)
print("Chiffrement terminé.")
elif x == 4:
filepath = input("Chemin du fichier à déchiffrer: ")
file_data = IO.read_bytes(filepath)
keypath = input("Chemin de la clé privée: ")
password = input("Mot de passe: ")
ciph_data = Conversions.bytes2int_list(file_data, 8)
clear_data = CramerShoup.decode(ciph_data, keypath, password)
IO.write_bytes(filepath, clear_data)
IO.rename_file(filepath, 1)
print("Déchiffrement terminé.")
elif x == 5:
hash_len = 0
while hash_len != 256 and hash_len != 512 and hash_len != 1 and hash_len != 2:
hash_len = int(input("1. BLAKE-256\n2. BLAKE-512\n"))
if hash_len < 256:
hash_len *= 32
else:
hash_len //= 8
filepath = input("Chemin du fichier: ")
with open(filepath, 'r') as rfile:
file_data = rfile.read()
ans = ''
while ans != 'y' and ans != 'n' and ans != 'Y' and ans != 'N':
ans = input("Protéger l'empreinte ? (y/n) ")
key = ''
if ans == 'y' or ans == 'Y':
key = input("Mot de passe: ")
h = hex(Hash.blake_hash(file_data, hash_len, key)) + '\n'
# Rename file : "name.ext" -> "name~hash.ext"
path = filepath.split('/')
last = len(path) - 1
filename = path[last].split('.')
filename[0] += '~hash'
path[last] = '.'.join(filename)
filepath = '/'.join(path)
with open(filepath, 'w') as wfile:
wfile.write(h)
elif x == 6:
hash_len = 0
while hash_len != 256 and hash_len != 512 and hash_len != 1 and hash_len != 2:
hash_len = int(input("1. BLAKE-256\n2. BLAKE-512\n"))
if hash_len < 256:
hash_len *= 32
else:
hash_len //= 8
filepath = input("Chemin du fichier: ")
with open(filepath, 'r') as rfile:
file_data = rfile.read()
hashpath = input("Chemin du hash: ")
with open(hashpath, 'r') as hfile:
hash = hfile.read()
ans = ''
while ans != 'y' and ans != 'n' and ans != 'Y' and ans != 'N':
ans = input("Empreinte protégée ?(y/n) ")
key = ''
if ans == 'y' or ans == 'Y':
key = input("Mot de passe: ")
h = hex(Hash.blake_hash(file_data, hash_len, key)) + '\n'
if h == hash:
print("Les empreintes sont égales.")
else:
print("Les empreintes ne correspondent pas.")
"D:\\Andre\\GitHub\\alabliuk\\Alura\\Data Science\\A coleta de dados\\data\\ufo.csv"
)
# Linhas contendo NaN nas colunas City, State ou Time devem ser eliminadas
df_ufo.dropna(axis=0,
how='any',
subset=['City', 'State', 'Time'],
inplace=True)
# Linhas contendo NaN em ambas colunas Colors Reported e Shape Reported devem ser eliminadas
df_ufo.dropna(axis=0,
how='all',
subset=['Colors Reported', 'Shape Reported'],
inplace=True)
# Conexão:
cliente_MongoDB_db_clima = Util.fnc_Conecta_Base_Documentos(
'', '', '192.168.0.113', '27017', 'dbclima')
db_clima = cliente_MongoDB_db_clima.dbclima
# Inicia Caixa de Areia, destino das linhas cujos valores (cidade, estado, instante) não constam na base clima
caixa_de_areia = []
for index, row in df_ufo.iterrows():
cidade = row["City"]
estado = row["State"]
instante = row["Time"]
# Esmiuçando "Time" em: mês, dia, ano, hora e minuto. Exemplo de data: 8/21/2004 00:05
# Temos que garantir o tamanho 2 para cada item.
# Exemplo: se ler dia "8", precisamos transformá-lo em "08"
# Para melhorar o percentual de busca, vamos desconsiderar minutos
x = re.findall('\d+', instante) # Extraímos os 5 números
mes = x[0].zfill(2)
#coding: utf-8
__author__ = 'ssm'
import numpy as np
import datetime
from sklearn.cluster import AffinityPropagation, KMeans
from sklearn.metrics.pairwise import cosine_similarity
import Util
import redis
#redis
r = redis.StrictRedis(host='10.2.1.7', port=6379, db=0)
#get 1st subject words
words = Util.cluster_2nd_load_1st_words("")
#get vecs
vecs = Util.cluster_2nd_load_1st_vecs(r, words)
print len(vecs)
def test_ap_clustering(wrds):
words = wrds[:-len(wrds) / 5]
#get similarities
similarities = [[0 for i in range(len(words))] for j in range(len(words))]
for ci in range(len(words)):
for cj in range(ci, len(words)):
similarities[ci][cj] = cosine_similarity([vecs[ci]],
[vecs[cj]])[0][0]
similarities[cj][ci] = similarities[ci][cj]
if ci % 100 == 0:
def axisAngle(axis, angle):
'''Creates a rotation which rotates angle degrees around axis.'''
return Quaternion.axisAngleInRadian(axis, Util.degreeToRadian(angle))
# ('Gossiping', ['Bignana'], 1),
('Wanted', ['gogin'], 1),
('HatePolitics', ['Neptunium', 'mark2165', 'kero2377'], 1),
]
try:
with open('Account.txt') as AccountFile:
Account = json.load(AccountFile)
ID = Account['ID']
Password = Account['Password']
except FileNotFoundError:
ID = input('請輸入帳號: ')
Password = getpass.getpass('請輸入密碼: ')
ptt_bot.login(ID, Password, KickOtherLogin=True)
for (current_board, ModeratorList, MaxPost) in SearchList:
MultiPO(current_board, ModeratorList, MaxPost)
publish_content += new_line + '內容如有失準，歡迎告知。' + new_line
publish_content += 'CodingMan'
if publish:
CurrentDate = Util.get_date(1)
ptt_bot.post('Test', CurrentDate + ' 多 PO 結果', publish_content, 1, 0)
ptt_bot.log('在 Test 板發文成功')
else:
ptt_bot.log('取消備份')
ptt_bot.logout()
def evict(
self, leaf
): # returns list of the blocks that go in each node on the path as a 2d list, should compare IDs and return if found as well
numLevels = Util.levelNumber(leaf) + 1
result = [0] * numLevels
for i in range(numLevels):
result[i] = [Block.Block(0, 0, b"")] * self._z
stashIter = 0
full = math.pow(
2, numLevels
) - 1 # "full vector" which has numLevel digits, and 1 means bucket has room for more
pathVec = [0] * numLevels # holds number of blocks in bucket occupied
while stashIter < len(
self._nodes
): # put nodes in the list where 0th element is 0th level, etc.
if self._oldStash == False:
"""if leaf == 1:
#print()
#print("root")
bucketPos = pathVec[0]
result[0][bucketPos] = self._nodes[stashIter]
self.deleteNode(stashIter)
pathVec[0] += 1
if pathVec[0] == self._z:
full -= math.pow(2, 0)
else:"""
legalLevels = Util.getMaxLevel(
leaf, self._nodes[stashIter].getLeaf(
)) # gets the number that tells which levels are allowed
#print(str(leaf) + ", " + str(self._nodes[stashIter].getLeaf()))
#print("legalLevels: " + str(legalLevels))
availBuc = int(
full
) & legalLevels # gets number that shows which buckets are available and allowed
#print("availBuc: " + str(availBuc))
if availBuc == 0: # if it's 0, then there are no available ones, we move on
print("no available")
stashIter += 1
else:
print("available")
bucketLevel = int(math.log(availBuc, 2))
#print("bucketLevel: " + str(bucketLevel))
bucketPos = pathVec[bucketLevel]
result[bucketLevel][bucketPos] = self._nodes[stashIter]
self.deleteNode(stashIter)
pathVec[bucketLevel] += 1
if pathVec[bucketLevel] == self._z:
full -= math.pow(2, bucketLevel)
else:
curLevel = Util.getMaxLevel(leaf,
self._nodes[stashIter].getLeaf())
while curLevel > -1:
if pathVec[curLevel] < self._z:
result[curLevel][
pathVec[curLevel]] = self._nodes[stashIter]
self.deleteNode(stashIter)
pathVec[curLevel] += 1
break
curLevel -= 1
if curLevel == -1:
stashIter += 1
return result
def __init__(self, fn_in, time_begin, time_end, overloaded):
self.overloaded = overloaded
# Unzip when the file is not there
if not os.path.exists(fn_in):
fn_zipped = "%s.bz2" % fn_in
if not os.path.exists(fn_zipped):
raise RuntimeError("Unexpected: %s" % fn_in)
Util.RunSubp("cd %s && bzip2 -dk %s > /dev/null" % (os.path.dirname(fn_zipped), os.path.basename(fn_zipped)))
if not os.path.exists(fn_in):
raise RuntimeError("Unexpected")
#Cons.P(fn_in)
mo_list = []
line_params = None
line_run = None
with open(fn_in) as fo:
for line in fo:
#Cons.P(line)
# 2017-10-13 20:41:01:258 2 sec: 34 operations; 34 current ops/sec; est completion in 68 days 1 hours [READ: Count=28, Max=46943, Min=33,
# Avg=32239.54, 90=45343, 99=46943, 99.9=46943, 99.99=46943] [INSERT: Count=8, Max=9343, Min=221, Avg=4660.88, 90=8695, 99=9343, 99.9=9343,
# 99.99=9343]
mo = re.match(r"\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d:\d\d\d (?P<rel_time>\d+) sec: \d+ operations; " \
"(?P<db_iops>(\d|\.)+) current ops\/sec; .*" \
"\[READ: Count=(?P<r_cnt>\d+), Max=(?P<r_max>\d+), Min=(?P<r_min>\d+), Avg=(?P<r_avg>(\d|\.)+)," \
" 90=(?P<r_90>\d+), 99=(?P<r_99>\d+), 99.9=(?P<r_999>\d+), 99.99=(?P<r_9999>\d+)\] " \
"\[INSERT: Count=(?P<w_cnt>\d+), Max=(?P<w_max>\d+), Min=(?P<w_min>\d+), Avg=(?P<w_avg>(\d|\.)+)," \
" 90=(?P<w_90>\d+), 99=(?P<w_99>\d+), 99.9=(?P<w_999>\d+), 99.99=(?P<w_9999>\d+)\] " \
, line)
if mo is None:
continue
total_seconds = int(mo.group("rel_time"))
s = total_seconds % 60
total_seconds -= s
total_mins = total_seconds / 60
m = total_mins % 60
total_mins -= m
h = total_mins / 60
rel_time = "%02d:%02d:%02d" % (h, m, s)
if (time_begin <= rel_time) and (rel_time <= time_end):
mo_list.append((rel_time, mo))
if len(mo_list) == 0:
raise RuntimeError("Unexpected. Check file [%s]" % fn_in)
cnt = 0
db_iops = []
r_cnt = 0
r_avg = 0.0
r_min = 0
r_max = 0
r_90 = 0
r_99 = 0
r_999 = 0
r_9999 = 0
w_cnt = 0
w_avg = 0.0
w_min = 0
w_max = 0
w_90 = 0
w_99 = 0
w_999 = 0
w_9999 = 0
for e in mo_list:
rel_time = e[0]
mo = e[1]
db_iops.append(float(mo.group("db_iops")))
r_cnt += int(mo.group("r_cnt"))
r_avg += float(mo.group("r_avg"))
r_min += int(mo.group("r_min"))
r_max += int(mo.group("r_max"))
r_90 += int(mo.group("r_90"))
r_99 += int(mo.group("r_99"))
r_999 += int(mo.group("r_999"))
r_9999 += int(mo.group("r_9999"))
w_cnt += int(mo.group("w_cnt"))
w_avg += float(mo.group("w_avg"))
w_min += int(mo.group("w_min"))
w_max += int(mo.group("w_max"))
w_90 += int(mo.group("w_90"))
w_99 += int(mo.group("w_99"))
w_999 += int(mo.group("w_999"))
w_9999 += int(mo.group("w_9999"))
cnt += 1
self.db_iops_stat = Stat.Gen(db_iops)
self.r_cnt = r_cnt
self.r_avg = (float(r_avg ) / cnt)
self.r_min = (float(r_min ) / cnt)
self.r_max = (float(r_max ) / cnt)
self.r_90 = (float(r_90 ) / cnt)
self.r_99 = (float(r_99 ) / cnt)
self.r_999 = (float(r_999 ) / cnt)
self.r_9999 = (float(r_9999) / cnt)
self.w_cnt = (float(w_cnt ) / cnt)
self.w_avg = (float(w_avg ) / cnt)
self.w_min = (float(w_min ) / cnt)
self.w_max = (float(w_max ) / cnt)
self.w_90 = (float(w_90 ) / cnt)
self.w_99 = (float(w_99 ) / cnt)
self.w_999 = (float(w_999 ) / cnt)
self.w_9999 = (float(w_9999) / cnt)
def get_seqs_bfr_aft_trgt_idx(self, data_dict, init):
chr_path = init[0]
max_len = init[1]
win_len = init[2]
result_dict = {}
util = Util.Utils()
for chr_key, val_list in data_dict.items():
result_dict.update({chr_key: {}})
n3_after_strd_dict = {}
n_len_after_strd_dict = {}
n_mx_len_after_strd_dict = {}
with open(chr_path + chr_key + ".fa", "r") as f:
header = f.readline() # header ignored : >chr19
print("header : " + header)
idx = 1
full_cover_p_str = ""
full_cover_m_str = ""
while True:
c = f.read(1)
if c == "":
print(str(idx))
break
if "\n" in c:
continue
elif "\r" in c:
continue
full_cover_p_str = full_cover_p_str + c.upper()
full_cover_m_str = full_cover_m_str + self.get_complementary(
c.upper())
p_m_str_n3_list = self.get_p_m_str_list_for_n3(
full_cover_p_str, full_cover_m_str, max_len, win_len)
p_m_str_list, full_cover_p_str, full_cover_m_str, n_len_str = self.get_p_m_str_list(
full_cover_p_str, full_cover_m_str, max_len, win_len)
# add forward seq into result_dict
result_dict = self.add_in_frm_forwrd_seq(
idx, chr_key, n3_after_strd_dict, result_dict,
p_m_str_n3_list)
result_dict = self.add_in_frm_forwrd_seq(
idx, chr_key, n_mx_len_after_strd_dict, result_dict,
p_m_str_list)
result_dict = self.add_out_frm_forwrd_seq(
idx, chr_key, n_len_after_strd_dict, result_dict,
p_m_str_list, n_len_str)
if idx in val_list:
for tmp_val in val_list[idx]:
Cellline = tmp_val[0]
Genome_Change = tmp_val[1]
tmp_key = Cellline + "^" + Genome_Change # HCC827^g.chr1:915845C>A
Genomesequence = tmp_val[3]
cDNA_change = tmp_val[4]
Codon_Change = tmp_val[
6] # Codon_Change : c.(223-225)Ggt>Tgt
Codon_Change_arr = Codon_Change.split(
">") # ['c.(223-225)Ggt', 'Tgt']
# check +/- strand
strnd = self.check_p_m_strand(
Genome_Change, cDNA_change)
if len(Codon_Change_arr) > 1:
mut_seq = Codon_Change_arr[1] # mut_seq : Tgt
mut_char_arr = Genome_Change.split(">")
if len(mut_seq) == 3 and len(mut_char_arr) > 1:
# TODO del
mut_char = mut_char_arr[1] # A
# idx_mut_char = mut_seq.find(mut_char)
if strnd == "-":
forward, frwrd_idx, bckwrd_idx = self.get_frwrd_str_bckwrd_idx(
mut_seq, p_m_str_n3_list, strnd)
result_dict[chr_key].update({
tmp_key:
[strnd, bckwrd_idx + 3, forward]
})
if idx + 3 + max_len - frwrd_idx in n3_after_strd_dict:
n3_after_strd_dict[
idx + 3 + max_len -
frwrd_idx].append(tmp_key)
else:
n3_after_strd_dict.update({
idx + 3 + max_len - frwrd_idx:
[tmp_key]
})
else:
forward, frwrd_idx, bckwrd_idx = self.get_frwrd_str_bckwrd_idx(
mut_seq, p_m_str_n3_list, strnd)
result_dict[chr_key].update({
tmp_key:
[strnd, bckwrd_idx, forward]
})
if idx + 3 + max_len - frwrd_idx in n3_after_strd_dict:
n3_after_strd_dict[
idx + 3 + max_len -
frwrd_idx].append(tmp_key)
else:
n3_after_strd_dict.update({
idx + 3 + max_len - frwrd_idx:
[tmp_key]
})
else:
# mut_seq's len is not 3
end_seq = int(
re.findall(
r'\d+',
Genome_Change[Genome_Change.
index(":"):])[1])
# if Genomesequence has 'In_Frame'
if 'In_Frame' in Genomesequence:
idx_p_m_str_list = 0
if strnd == "-":
idx_p_m_str_list = 3
forward_str = p_m_str_list[
idx_p_m_str_list]
adder = 1
if 'ins'.upper(
) in Genome_Change.upper():
adder = 0
forward_str = full_cover_p_str[
-max_len:]
if strnd == '-':
forward_str = full_cover_m_str[
-max_len:]
result_dict[chr_key].update({
tmp_key: [
strnd, idx_p_m_str_list,
forward_str
]
})
if end_seq + max_len + adder in n_mx_len_after_strd_dict:
n_mx_len_after_strd_dict[
end_seq + max_len +
adder].append(tmp_key)
else:
n_mx_len_after_strd_dict.update({
end_seq + max_len + adder:
[tmp_key]
})
else:
idx_p_m_str_list = 0
if strnd == "-":
idx_p_m_str_list = 3
forward_str = n_len_str + p_m_str_list[
idx_p_m_str_list]
adder = 1
if 'ins'.upper(
) in Genome_Change.upper():
adder = 0
forward_str = full_cover_p_str[
-max_len + win_len:]
if strnd == '-':
forward_str = full_cover_m_str[
-max_len + win_len:]
result_dict[chr_key].update({
tmp_key: [
strnd, idx_p_m_str_list,
forward_str
]
})
if end_seq + max_len + win_len + adder in n_len_after_strd_dict:
n_len_after_strd_dict[
end_seq + max_len + win_len +
adder].append(tmp_key)
else:
n_len_after_strd_dict.update({
end_seq + max_len + win_len + adder:
[tmp_key]
})
elif "_" in Genome_Change:
# deli with '_'
end_seq = int(
re.findall(
r'\d+', Genome_Change[Genome_Change.
index(":"):])[1])
# if Genomesequence has 'In_Frame'
if 'In_Frame' in Genomesequence:
idx_p_m_str_list = 0
if strnd == "-":
idx_p_m_str_list = 3
forward_str = p_m_str_list[
idx_p_m_str_list]
adder = 1
if 'ins'.upper() in Genome_Change.upper():
adder = 0
forward_str = full_cover_p_str[
-max_len:]
if strnd == '-':
forward_str = full_cover_m_str[
-max_len:]
result_dict[chr_key].update({
tmp_key:
[strnd, idx_p_m_str_list, forward_str]
})
if end_seq + max_len + adder in n_mx_len_after_strd_dict:
n_mx_len_after_strd_dict[
end_seq + max_len +
adder].append(tmp_key)
else:
n_mx_len_after_strd_dict.update({
end_seq + max_len + adder:
[tmp_key]
})
else:
idx_p_m_str_list = 0
if strnd == "-":
idx_p_m_str_list = 3
forward_str = n_len_str + p_m_str_list[
idx_p_m_str_list]
adder = 1
if 'ins'.upper() in Genome_Change.upper():
adder = 0
forward_str = full_cover_p_str[
-max_len - win_len:]
if strnd == "-":
forward_str = full_cover_m_str[
-max_len - win_len:]
result_dict[chr_key].update({
tmp_key:
[strnd, idx_p_m_str_list, forward_str]
})
if end_seq + max_len + win_len + adder in n_len_after_strd_dict:
n_len_after_strd_dict[
end_seq + max_len + win_len +
adder].append(tmp_key)
else:
n_len_after_strd_dict.update({
end_seq + max_len + win_len + adder:
[tmp_key]
})
else:
# out of logic
end_seq = idx # + 1
idx_p_m_str_list = 0
if strnd == "-":
idx_p_m_str_list = 3
forward_str = n_len_str + p_m_str_list[
idx_p_m_str_list]
adder = 1
if 'ins'.upper() in Genome_Change.upper():
adder = 0
forward_str = full_cover_p_str[-max_len -
win_len:]
if strnd == "-":
forward_str = full_cover_m_str[
-max_len - win_len:]
result_dict[chr_key].update({
tmp_key:
[strnd, idx_p_m_str_list, forward_str]
})
if end_seq + max_len + win_len + adder in n_len_after_strd_dict:
n_len_after_strd_dict[
end_seq + max_len + win_len +
adder].append(tmp_key)
else:
n_len_after_strd_dict.update({
end_seq + max_len + win_len + adder:
[tmp_key]
})
idx = idx + 1
return result_dict
def toAxisAngle(self):
'''Converts a rotation to axis-angle representation(angle in degree).'''
axis, angle = self.toAxisAngleInRadian()
return axis, Util.radianToDegree(angle)
def _GenLocalSsdData(fn):
fn_plot_data = "plot-data/%s" % fn
fn_raw = "../log/ioping/%s" % fn
# Uncompress the raw files if not exist
if not os.path.isfile(fn_raw):
with Cons.MeasureTime("file %s doesn't exist. uncompressing from the archive" % fn_raw):
cmd = "7z x -o../log/ioping %s.7z" % fn_raw
Util.RunSubp(cmd)
# Generate one if not exist. Skip for now. It's fast enough.
#if os.path.isfile(fn_plot_data):
# Cons.P("%s already exists" % fn_plot_data)
# return
if not os.path.exists("plot-data"):
os.makedirs("plot-data")
with Cons.MeasureTime("Generating plot data %s" % fn_plot_data):
with open(fn_raw) as fo_r, open(fn_plot_data, "w") as fo_w:
fmt = "%13s" \
" %5.1f" \
" %3d %3d %5d %4d" \
" %3d %3d %5d %3d" \
" %6d"
fo_w.write("%s\n" % Util.BuildHeader(fmt,
"datetime" \
" blocks_used_percent" \
" rr_4k_min rr_4k_avg rr_4k_max rr_4k_sd" \
" rw_4k_min rw_4k_avg rw_4k_max rw_4k_sd" \
" seqw_100m"
))
# 160425-011752
# 100 17430 5737 23499713 170 174 211 5
# 100 14822 6747 27634597 134 148 231 17
# 1 216553 5 484212179 216553 216553 216553 0
# Filesystem 1K-blocks Used Available Use% Mounted on
# /dev/xvda1 8115168 2545256 5134636 34% /
# /dev/xvdb 769018760 173080 768829296 1% /mnt/local-ssd1
i = 0
dt = None
block_used_percent_prev = None
blocks_total = None
blocks_used = None
rr_4k_min = None
rr_4k_avg = None
rr_4k_max = None
rr_4k_sd = None
rw_4k_min = None
rw_4k_avg = None
rw_4k_max = None
rw_4k_sd = None
seqw_100m = None
for line in fo_r.readlines():
#Cons.P(line)
if i == 0:
dt = line.strip()
# 4k rand read
elif i == 1:
t = line.split()
if len(t) != 8:
raise RuntimeError("Unexpected format [%s]" % line)
# All in us
rr_4k_min = int(t[4])
rr_4k_avg = int(t[5])
rr_4k_max = int(t[6])
rr_4k_sd = int(t[7])
elif i == 2:
t = line.split()
if len(t) != 8:
raise RuntimeError("Unexpected format [%s]" % line)
# All in us
rw_4k_min = int(t[4])
rw_4k_avg = int(t[5])
rw_4k_max = int(t[6])
rw_4k_sd = int(t[7])
elif i == 3:
t = line.split()
if len(t) != 8:
raise RuntimeError("Unexpected format [%s]" % line)
# All of the min, avg, max are the same since there is only 1 request
# at a time.
seqw_100m = int(t[4])
elif i == 6:
t = line.split()
if (len(t) != 6) or (t[0] != "/dev/xvdb"):
raise RuntimeError("Unexpected format [%s]" % line)
blocks_total = int(t[1])
blocks_used = int(t[2])
i += 1
if i == 7:
block_used_percent = round(100.0 * blocks_used / blocks_total, 1)
if block_used_percent_prev != block_used_percent:
fo_w.write((fmt + "\n") % (dt
, (100.0 * blocks_used / blocks_total)
, rr_4k_min, rr_4k_avg, rr_4k_max, rr_4k_sd
, rw_4k_min, rw_4k_avg, rw_4k_max, rw_4k_sd
, seqw_100m
))
block_used_percent_prev = block_used_percent
i = 0
dt = None
blocks_total = None
blocks_used = None
rr_4k_min = None
rr_4k_avg = None
rr_4k_max = None
rr_4k_sd = None
rw_4k_min = None
rw_4k_avg = None
rw_4k_max = None
rw_4k_sd = None
seqw_100m = None
Cons.P("Created %s %d" % (fn_plot_data, os.path.getsize(fn_plot_data)))
def correctLeaves(self, treeSize):
for node in self._nodes:
newLeaf = Util.correctLeaf(node.getLeaf(), treeSize,
node.getSegID() % 2)
if newLeaf != None:
node.setLeaf(newLeaf)
def _GenEbsSsdData(fn0, fn1):
fn_plot_data = "plot-data/%s" % fn1
fn_raw0 = "../log/ioping/%s" % fn0
fn_raw1 = "../log/ioping/%s" % fn1
# Generate one if not exist. Skip for now. It's fast enough.
#if os.path.isfile(fn_plot_data):
# Cons.P("%s already exists" % fn_plot_data)
# return
# Uncompress the raw files if not exist
for fn in [fn_raw0, fn_raw1]:
if not os.path.isfile(fn):
with Cons.MeasureTime("file %s doesn't exist. uncompressing from the archive" % fn):
cmd = "7z x -o../log/ioping %s.7z" % fn
Util.RunSubp(cmd)
if not os.path.exists("plot-data"):
os.makedirs("plot-data")
with Cons.MeasureTime("Generating plot data %s" % fn_plot_data):
with open(fn_plot_data, "w") as fo_w, open(fn_raw0) as fo_r0, open(fn_raw1) as fo_r1:
fmt = "%13s" \
" %5.1f" \
" %3d %4d %5d %4d" \
" %3d %4d %5d %4d" \
" %6d"
fo_w.write("%s\n" % Util.BuildHeader(fmt,
"datetime" \
" blocks_used_percent" \
" rr_4k_min rr_4k_avg rr_4k_max rr_4k_sd" \
" rw_4k_min rw_4k_avg rw_4k_max rw_4k_sd" \
" seqw_100m"
))
#160423-195525
#100 68474 1460 5981833 230 685 1631 568
#100 74883 1335 5469866 495 749 4723 585
#1 1690975 1 62010142 1690975 1690975 1690975 0
#Filesystem 1K-blocks Used Available Use% Mounted on
#/dev/xvda1 8115168 2676196 5003696 35% /
#/dev/xvdc 17111375772 1565491700 15545867688 10% /mnt/ebs-ssd1
i = 0
dt = None
block_used_percent_prev = None
blocks_total = None
blocks_used = None
rr_4k_min = None
rr_4k_avg = None
rr_4k_max = None
rr_4k_sd = None
rw_4k_min = None
rw_4k_avg = None
rw_4k_max = None
rw_4k_sd = None
seqw_100m = None
for fo_r in [fo_r0, fo_r1]:
for line in fo_r.readlines():
#Cons.P("%d [%s]" % (i, line.strip()))
if i == 0:
dt = line.strip()
# 4k rand read
elif i == 1:
t = line.split()
if len(t) != 8:
raise RuntimeError("Unexpected format [%s]" % line)
# All in us
rr_4k_min = int(t[4])
rr_4k_avg = int(t[5])
rr_4k_max = int(t[6])
rr_4k_sd = int(t[7])
elif i == 2:
t = line.split()
if len(t) != 8:
raise RuntimeError("Unexpected format [%s]" % line)
# All in us
rw_4k_min = int(t[4])
rw_4k_avg = int(t[5])
rw_4k_max = int(t[6])
rw_4k_sd = int(t[7])
elif i == 3:
t = line.split()
if len(t) != 8:
raise RuntimeError("Unexpected format [%s]" % line)
# All of the min, avg, max are the same since there is only 1 request
# at a time.
seqw_100m = int(t[4])
elif i == 6:
t = line.split()
if (len(t) != 6) or (t[0] != "/dev/xvdc"):
raise RuntimeError("Unexpected format [%s]" % line)
blocks_total = int(t[1])
blocks_used = int(t[2])
i += 1
if i == 7:
block_used_percent = round(100.0 * blocks_used / blocks_total, 1)
if block_used_percent_prev != block_used_percent:
fo_w.write((fmt + "\n") % (dt
, (100.0 * blocks_used / blocks_total)
, rr_4k_min, rr_4k_avg, rr_4k_max, rr_4k_sd
, rw_4k_min, rw_4k_avg, rw_4k_max, rw_4k_sd
, seqw_100m
))
block_used_percent_prev = block_used_percent
i = 0
dt = None
blocks_total = None
blocks_used = None
rr_4k_min = None
rr_4k_avg = None
rr_4k_max = None
rr_4k_sd = None
rw_4k_min = None
rw_4k_avg = None
rw_4k_max = None
rw_4k_sd = None
seqw_100m = None
Cons.P("Created %s %d" % (fn_plot_data, os.path.getsize(fn_plot_data)))
def add_file(self, filename):
"""
Setups data:
self.seq_lengths
self.file_index
self.file_start
self.file_seq_start
Use load_seqs() to load the actual data.
:type filename: str
"""
if self._use_cache_manager:
filename = Util.cf(filename)
fin = h5py.File(filename, "r")
if 'targets' in fin:
self.labels = {
k: [self._decode(item) for item in fin["targets/labels"][k][...].tolist()]
for k in fin['targets/labels']}
if not self.labels:
labels = [item.split('\0')[0] for item in fin["labels"][...].tolist()]; """ :type: list[str] """
self.labels = {'classes': labels}
assert len(self.labels['classes']) == len(labels), "expected " + str(len(self.labels['classes'])) + " got " + str(len(labels))
self.files.append(filename)
print("parsing file", filename, file=log.v5)
if 'times' in fin:
if self.timestamps is None:
self.timestamps = fin[attr_times][...]
else:
self.timestamps = numpy.concatenate([self.timestamps, fin[attr_times][...]], axis=0)
seq_lengths = fin[attr_seqLengths][...]
if 'targets' in fin:
self.target_keys = sorted(fin['targets/labels'].keys())
else:
self.target_keys = ['classes']
if len(seq_lengths.shape) == 1:
seq_lengths = numpy.array(zip(*[seq_lengths.tolist() for i in range(len(self.target_keys)+1)]))
if len(self._seq_lengths) == 0:
self._seq_lengths = numpy.array(seq_lengths)
else:
self._seq_lengths = numpy.concatenate((self._seq_lengths, seq_lengths), axis=0)
if not self._seq_start:
self._seq_start = [numpy.zeros((seq_lengths.shape[1],), 'int64')]
seq_start = numpy.zeros((seq_lengths.shape[0] + 1, seq_lengths.shape[1]), dtype="int64")
numpy.cumsum(seq_lengths, axis=0, dtype="int64", out=seq_start[1:])
self._tags += fin["seqTags"][...].tolist()
self.file_seq_start.append(seq_start)
nseqs = len(seq_start) - 1
self._num_seqs += nseqs
self.file_index.extend([len(self.files) - 1] * nseqs)
self.file_start.append(self.file_start[-1] + nseqs)
self._num_timesteps += numpy.sum(seq_lengths[:, 0])
if self._num_codesteps is None:
self._num_codesteps = [0 for i in range(1, len(seq_lengths[0]))]
for i in range(1, len(seq_lengths[0])):
self._num_codesteps[i - 1] += numpy.sum(seq_lengths[:, i])
if 'maxCTCIndexTranscriptionLength' in fin.attrs:
self.max_ctc_length = max(self.max_ctc_length, fin.attrs['maxCTCIndexTranscriptionLength'])
if len(fin['inputs'].shape) == 1: # sparse
num_inputs = [fin.attrs[attr_inputPattSize], 1]
else:
num_inputs = [fin['inputs'].shape[1], len(fin['inputs'].shape)] #fin.attrs[attr_inputPattSize]
if self.num_inputs == 0:
self.num_inputs = num_inputs[0]
assert self.num_inputs == num_inputs[0], "wrong input dimension in file %s (expected %s got %s)" % (
filename, self.num_inputs, num_inputs[0])
if 'targets/size' in fin:
num_outputs = {}
for k in fin['targets/size'].attrs:
if numpy.isscalar(fin['targets/size'].attrs[k]):
num_outputs[k] = (fin['targets/size'].attrs[k], len(fin['targets/data'][k].shape))
else: # hdf_dump will give directly as tuple
assert fin['targets/size'].attrs[k].shape == (2,)
num_outputs[k] = tuple(fin['targets/size'].attrs[k])
else:
num_outputs = {'classes': fin.attrs[attr_numLabels]}
num_outputs["data"] = num_inputs
if not self.num_outputs:
self.num_outputs = num_outputs
assert self.num_outputs == num_outputs, "wrong dimensions in file %s (expected %s got %s)" % (
filename, self.num_outputs, num_outputs)
if 'ctcIndexTranscription' in fin:
if self.ctc_targets is None:
self.ctc_targets = fin['ctcIndexTranscription'][...]
else:
tmp = fin['ctcIndexTranscription'][...]
pad_width = self.max_ctc_length - tmp.shape[1]
tmp = numpy.pad(tmp, ((0,0),(0,pad_width)), 'constant', constant_values=-1)
pad_width = self.max_ctc_length - self.ctc_targets.shape[1]
self.ctc_targets = numpy.pad(self.ctc_targets, ((0,0),(0,pad_width)), 'constant', constant_values=-1)
self.ctc_targets = numpy.concatenate((self.ctc_targets, tmp))
self.num_running_chars = numpy.sum(self.ctc_targets != -1)
if 'targets' in fin:
for name in fin['targets/data']:
tdim = 1 if len(fin['targets/data'][name].shape) == 1 else fin['targets/data'][name].shape[1]
self.data_dtype[name] = str(fin['targets/data'][name].dtype) if tdim > 1 else 'int32'
self.targets[name] = None
else:
self.targets = { 'classes' : numpy.zeros((self._num_timesteps,), dtype=theano.config.floatX) }
self.data_dtype['classes'] = 'int32'
self.data_dtype["data"] = str(fin['inputs'].dtype)
assert len(self.target_keys) == len(self._seq_lengths[0]) - 1
fin.close()