def __call__(self):
print "Ingest files directory: " + filesDir
# Don't report to the cosole until we verify the result
grinder.statistics.delayReports = 1
# Ingest all the files one by one in directory $filesDir
for f in os.listdir(filesDir):
# Skit those hidden files like .DS_Store
if (f.startswith('.') == False):
cType, encoding = mimetypes.guess_type(f)
if cType is None or encoding is not None:
cType = 'application/octet-stream'
headers = (NVPair("Content-Type", cType), )
inFile = io.FileInputStream(filesDir + f)
# Call the version of POST that takes a byte array.
result = request1.POST(self.requestUrl, inFile, headers)
print "\nThread #" + str(
self.threadNum
) + " ingested " + f + ": " + self.requestUrl + " " + str(
result.statusCode)
inFile.close()
if result.statusCode == 201:
# Report to the console
grinder.statistics.forLastTest.setSuccess(1)
else:
print "\nThread #" + str(
self.threadNum
) + " error: POST " + self.requestUrl + " " + str(
result.statusCode)
def createOverlayBridgeData(fileName):
overlayBridgeID=getOverlayBridgeID()
p = util.Properties()
propertiesFile =javaio.FileInputStream(fileName)
p.load( propertiesFile)
remoteCellName=p.getProperty("cellName")
cellAccessPoint=getRemoteCellAccessPoint(remoteCellName,overlayBridgeID)
dmgrBootAnchorHost=p.getProperty("dmgrBootAnchorHost")
if(dmgrBootAnchorHost!=None and dmgrBootAnchorHost!=""):
dmgrBootAnchorPortString=p.getProperty("dmgrBootAnchorPort")
dmgrBootAnchorPort=-1
try:
dmgrBootAnchorPort=int(dmgrBootAnchorPortString)
except(ValueError):
print "Skipping entry "+dmgrBootAnchorHost+":"+dmgrBootAnchorPortString+" due to invalid port value"
if(dmgrBootAnchorPort>-1):
getOverlayEndpoint(cellAccessPoint,dmgrBootAnchorHost,dmgrBootAnchorPort)
i=0
while(1):
agentBootAnchorHost=p.getProperty("nodeAgentBootAnchorHost."+str(i))
if(agentBootAnchorHost==None or agentBootAnchorHost==""):
break
agentBootAnchorPortString=p.getProperty("nodeAgentBootAnchorPort."+str(i))
agentBootAnchorPort=-1
try:
agentBootAnchorPort=int(agentBootAnchorPortString)
except(ValueError):
print "Skipping entry "+agentBootAnchorHost+":"+agentBootAnchorPortString+" due to invalid port value"
if(agentBootAnchorPort>-1):
getOverlayEndpoint(cellAccessPoint,agentBootAnchorHost,agentBootAnchorPort)
i=i+1
def loadClick(self, e):
returnVal = self._fc.showOpenDialog(self._splitpane)
if returnVal == JFileChooser.APPROVE_OPTION:
warning = """
CAUTION:
Loading a saved configuration deserializes data.
This action may pose a security threat to the application.
Only proceed when the source and contents of this file is trusted.
Load Selected File?
"""
result = JOptionPane.showOptionDialog(self._splitpane, warning,
"Caution",
JOptionPane.YES_NO_OPTION,
JOptionPane.WARNING_MESSAGE,
None, ["OK", "Cancel"], "OK")
if result != JOptionPane.YES_OPTION:
return
f = self._fc.getSelectedFile()
fileName = f.getPath()
ins = io.ObjectInputStream(io.FileInputStream(fileName))
dbData = ins.readObject()
ins.close()
self._db.load(dbData, self._callbacks, self._helpers)
self._userTable.redrawTable()
self._messageTable.redrawTable()
def unserialize_object(filename):
#print "reading serialized object from", filename
file = io.FileInputStream(filename)
objstream = org.python.util.PythonObjectInputStream(file)
obj = objstream.readObject()
objstream.close()
file.close()
def test_serialization(self):
object1 = 42
object2 = ['a', 1, 1.0]
object3 = Foo()
object3.baz = 99
object4 = awt.Color(1, 2, 3)
#writing
fout = io.ObjectOutputStream(io.FileOutputStream(self.sername))
#Python int
fout.writeObject(object1)
#Python list
fout.writeObject(object2)
#Python instance
fout.writeObject(object3)
#Java instance
fout.writeObject(object4)
fout.close()
fin = io.ObjectInputStream(io.FileInputStream(self.sername))
#reading
iobject1 = fin.readObject()
iobject2 = fin.readObject()
iobject3 = fin.readObject()
iobject4 = fin.readObject()
fin.close()
self.assertEqual(iobject1, object1)
self.assertEqual(iobject2, object2)
self.assertEqual(iobject3.baz, 99)
self.assertEqual(iobject3.bar(), 'bar')
self.assertEqual(iobject3.__class__, Foo)
self.assertEqual(iobject4, object4)
def removeOverlayBridgeData(fileName):
overlayBridgeID=getOverlayBridgeID()
p = util.Properties()
propertiesFile =javaio.FileInputStream(fileName)
p.load( propertiesFile)
remoteCellName=p.getProperty("cellName")
cellAccessPoint=getRemoteCellAccessPoint(remoteCellName,overlayBridgeID)
AdminConfig.remove(cellAccessPoint)
def loadObject(fName, check=True, gzip=False):
if gzip: fName = fName + ".gz"
if os.path.isfile(fName):
from java import io
fs = io.FileInputStream(fName)
if gzip: fs = java.util.zip.GZIPInputStream(fs)
ins = io.ObjectInputStream(fs)
x = ins.readObject()
ins.close()
return x
else:
if check: raise Exception("File not found" + fName)
else: return None
def toInputStream(o):
if isinstance(o, (io.InputStream, io.Reader, file)):
return o
f = toFile(o)
if isinstance(f, io.File) and f.exists():
return io.FileInputStream(f)
if isinstance(o, (str, unicode)):
return io.ByteArrayInputStream(lang.String(o).getBytes())
if type(o).__name__ == 'array':
return io.ByteArrayInputStream(o)
def createOverlayBridgeData(fileName):
overlayBridgeID = getOverlayBridgeID()
p = util.Properties()
propertiesFile = javaio.FileInputStream(fileName)
p.load(propertiesFile)
remoteCellName = p.getProperty("cellName")
# remove exisiting OverlayData associated with the cell - won't throw exception if empty
removeOverlayBridgeData(fileName)
cellAccessPoint = getRemoteCellAccessPoint(remoteCellName, overlayBridgeID)
# do not allow duplicate cells
checkForDuplicates(getCellName(), overlayBridgeID)
dmgrBootAnchorHost = p.getProperty("dmgrBootAnchorHost")
if (dmgrBootAnchorHost != None and dmgrBootAnchorHost != ""):
dmgrBootAnchorPortString = p.getProperty("dmgrBootAnchorPort")
dmgrBootAnchorPort = -1
try:
dmgrBootAnchorPort = int(dmgrBootAnchorPortString)
except (ValueError):
print "\tSkipping entry " + dmgrBootAnchorHost + ":" + dmgrBootAnchorPortString + " due to invalid port value"
if (dmgrBootAnchorPort > -1):
getOverlayEndpoint(cellAccessPoint, dmgrBootAnchorHost,
dmgrBootAnchorPort)
i = 0
while (1):
agentBootAnchorHost = p.getProperty("nodeAgentBootAnchorHost." +
str(i))
if (agentBootAnchorHost == None or agentBootAnchorHost == ""):
break
agentBootAnchorPortString = p.getProperty("nodeAgentBootAnchorPort." +
str(i))
agentBootAnchorPort = -1
try:
agentBootAnchorPort = int(agentBootAnchorPortString)
except (ValueError):
print "\tSkipping entry " + agentBootAnchorHost + ":" + agentBootAnchorPortString + " due to invalid port value"
if (agentBootAnchorPort > -1):
getOverlayEndpoint(cellAccessPoint, agentBootAnchorHost,
agentBootAnchorPort)
i = i + 1
def read(filename):
dbf = jparse.DocumentBuilderFactory.newInstance()
db = dbf.newDocumentBuilder()
fis = io.FileInputStream(filename)
doc = db.parse(fis)
fis.close()
doc.normalizeDocument()
leo_file = doc.getDocumentElement()
print leo_file
print "-------"
#de.normalizeDocument()
print "-------"
cnodes = leo_file.getChildNodes()
nodes = {}
for z in xrange(cnodes.length):
node = cnodes.item(z)
print node
nodes[node.getNodeName()] = node
tnodes = nodes['tnodes']
tchildren = tnodes.getChildNodes()
tnodes2 = {}
for z in xrange(tchildren.length):
tnode = tchildren.item(z)
if tnode.getNodeName() == 't':
#print tnode
print tnode.getTextContent()
atts = tnode.getAttributes()
print atts.getNamedItem("tx").getTextContent()
tx = atts.getNamedItem("tx").getTextContent()
tnodes2[tx] = tnode.getTextContent()
print tnodes2
#do vnodes
vnodes = nodes['vnodes']
vchildren = vnodes.getChildNodes()
for z in xrange(vchildren.length):
#print vchildren.item( z )
item = vchildren.item(z)
if item.getNodeName() == 'v':
recursiveWalkV(item, tnodes2)
os.path.walk(path, clb2.walk, None) #must find the jazzy-core.jar file...
#we import the resources via this classloader because just adding the jar file to the sys.path is inadequate
clb2.importClass("SpellCheckListener",
"com.swabunga.spell.event.SpellCheckListener")
clb2.importClass("SpellChecker", "com.swabunga.spell.event.SpellChecker")
clb2.importClass("SpellDictionaryHashMap",
"com.swabunga.spell.engine.SpellDictionaryHashMap")
#clb2.importClass( "SpellDictionaryCachedDichoDisk", "com.swabunga.spell.engine.SpellDictionaryCachedDichoDisk" )
clb2.importClass("StringWordTokenizer",
"com.swabunga.spell.event.StringWordTokenizer")
proppath = g.os_path_join(
g.app.loadDir, "..", "plugins", "spellingdicts",
"which.txt") #we start to determine which dictionary to use
fis = io.FileInputStream(io.File(proppath))
properties = util.Properties()
properties.load(fis)
fis.close()
fis = None
lfile = properties.getProperty("dict")
dpath = g.os_path_join(g.app.loadDir, "..", "plugins", "spellingdicts",
lfile)
dictionary = SpellDictionaryHashMap(io.File(dpath))
import org.python.core.Py as Py #now we descend into the Jython internals...
sstate = Py.getSystemState()
cloader = sstate.getClassLoader()
sstate.setClassLoader(
clb2
) #we do this so the JyLeoSpellChecker class can implement SpellCheckListener, otherwise it fails
sys.exit()
tenantCode = sys.argv[1]
startDate = sys.argv[2]
endDate = sys.argv[3]
paramFile = sys.argv[4]
minObjectId = globalVars.dateToObjectId(startDate)
maxObjectId = globalVars.dateToObjectId(endDate)
Pig.registerJar("/usr/hdp/current/phoenix-client/phoenix-client.jar")
Pig.registerJar("../lib/yucca-phoenix-pig.jar")
try:
props = util.Properties()
propertiesfis = javaio.FileInputStream("mongo_parameters_prod.txt")
props.load(propertiesfis)
except:
print "Errore leggendo mongo_parameters_prod.txt: ", sys.exc_info()[0]
sys.exit(1)
mongo1 = props.getProperty('mongoHost') + ":" + props.getProperty(
'mongoPort') + "/DB_SUPPORT"
mongo2 = " -u " + props.getProperty('mongoUsr')
mongo3 = " -p " + props.getProperty(
'mongoPwd') + ''' --authenticationDatabase admin --quiet --eval "'''
callResult = call("mongo " + mongo1 + " " + mongo2 + " " + mongo3 +
" var param1='" + tenantCode + "' " +
''' " ../list_tenant_defaults.js > tenant.json''',
shell=True)
#! /bin/env jython
import org.jdom as jdom
import java.io as io
def printDocument(jdomDoc):
print "Printing..."
rootElement = jdomDoc.rootElement
iter = rootElement.getDescendants()
print dir(iter)
while iter.hasNext():
element = iter.next()
#print type( element )
#print dir( element )
if isinstance(element, jdom.Text):
print "textTrim=" + element.textTrim
if isinstance(element, jdom.Element):
#print "element=" + str( dir( element ) )
print "element=" + element.getName()
if __name__ == "__main__":
builder = jdom.input.SAXBuilder()
doc = builder.build(
io.FileInputStream(
"/nfs/home4/dmpapp/appd4ec/tmp/QAMP_UAT0000000_090504130000i.xml"))
printDocument(doc)
print 'Recursive assignment to list slices handled incorrectly #1'
x = [1, 2, 3, 4, 5]
x[1:] = x
assert x == [1, 1, 2, 3, 4, 5]
print 'sys.platform should be javax.y.z not jdkx.y.z #4'
import sys
assert sys.platform[:4] == 'java'
print 'java.io.IOExceptions are mangled into IOErrors #5'
from java import io, lang
try:
io.FileInputStream("doesnotexist")
raise TestFailed
except io.FileNotFoundException:
pass
try:
io.FileInputStream("doesnotexist")
raise TestFailed
except IOError:
pass
print 'java.util.Vector\'s can\'t be used in for loops #7'
from java.util import Vector
vec = Vector()
class Test:
text = Data()
class Factory:
def createTest(x):
return Test()
factory = Factory()
foo = factory.createTest()
from java import io
import sys
filename = "test101.out"
fout = io.ObjectOutputStream(io.FileOutputStream(filename))
fout.writeObject(foo)
fout.close()
fin = io.ObjectInputStream(io.FileInputStream(filename))
foo = fin.readObject()
fin.close()
support.compare(foo, "<(__main__|test101).Test instance")
support.compare(foo.text, "<(__main__|test101).Data instance")
support.compare(foo.text.data, "Hello World")
paramFile = sys.argv[3]
# mode = REPLACE --> delete all record marked as datalake
# mode = APPEND --> only add new record (using lastId)
#TO DO gestire bda_id non presente
#TO DO completare modalita append (filtrare dati in Pig)
pid = os.getpid()
# read properties file
import java.util as util
import java.io as javaio
try:
props = util.Properties()
propertiesfis = javaio.FileInputStream(paramFile)
props.load(propertiesfis)
except:
print "Errore leggendo " + paramFile + ": ", sys.exc_info()[0]
sys.exit(1)
mongoConn = (props.getProperty('mongoHost') + ":" +
props.getProperty('mongoPort') + "/DB_SUPPORT" + " -u " +
props.getProperty('mongoUsr') + " -p " +
props.getProperty('mongoPwd') +
" --authenticationDatabase admin --quiet ")
mongoParam = ''' --eval "''' + "var param1='" + tenantCode + "' " + ''' " '''
Pig.registerJar("/usr/hdp/current/phoenix-client/phoenix-client.jar")
Pig.registerJar("../lib/yucca-phoenix-pig.jar")
print_test('writing', 2)
sername = os.path.join(sys.prefix, "test.ser")
fout = io.ObjectOutputStream(io.FileOutputStream(sername))
print_test('Python int', 3)
fout.writeObject(object1)
print_test('Python list', 3)
fout.writeObject(object2)
print_test('Python instance', 3)
fout.writeObject(object3)
print_test('Java instance', 3)
fout.writeObject(object4)
fout.close()
fin = io.ObjectInputStream(io.FileInputStream(sername))
print_test('reading', 2)
iobject1 = fin.readObject()
iobject2 = fin.readObject()
iobject3 = fin.readObject()
iobject4 = fin.readObject()
fin.close()
#print iobject1, iobject2, iobject3, iobject3.__class__, iobject4
print_test('Python int', 3)
assert iobject1 == object1
print_test('Python list', 3)
assert iobject2 == object2
def get_instances(serialized_file, type):
inFile = io.FileInputStream(serialized_file)
inStream = util.PythonObjectInputStream(inFile)
return inStream.readObject()[type]
def loadObject(fname="jython.bin"):
ins=io.ObjectInputStream(io.FileInputStream(fname))
x=ins.readObject()
ins.close()
return x
def open(self, id):
f = io.File(self.pfx, id.replace('/', io.File.separator))
if f.file:
## print "path-open:",f # ?? dbg
return io.BufferedInputStream(io.FileInputStream(f))
return None
