def saveHM(filename):
# Create new HTTP client
client = HttpClient()
client.getParams().setAuthenticationPreemptive(True)
defaultcreds = UsernamePasswordCredentials(user, password)
client.getState().setCredentials(AuthScope.ANY, defaultcreds)
# Get data across HTTP
url = (
"http://"
+ host
+ ":"
+ str(port)
+ "/admin/savedataview.egi?type="
+ type
+ "&data_format=NEXUSHDF5_LZW_6&data_saveopen_action=OPEN_ONLY"
)
getMethod = GetMethod(url)
getMethod.setDoAuthentication(True)
client.executeMethod(getMethod)
# Save locally
file = File(directory + "/" + filename)
out = FileOutputStream(file)
out.write(getMethod.getResponseBody())
out.close()
# Clean up
getMethod.releaseConnection()
def extractZip(zip, dest):
"extract zip archive to dest directory"
logger.info("Begin extracting:" + zip + " --> " +dest)
mkdir_p(dest)
zipfile = ZipFile(zip)
entries = zipfile.entries()
while entries.hasMoreElements():
entry = entries.nextElement()
if entry.isDirectory():
mkdir_p(os.path.join(dest, entry.name))
else:
newFile = File(dest, entry.name)
mkdir_p(newFile.parent)
zis = zipfile.getInputStream(entry)
fos = FileOutputStream(newFile)
nread = 0
buffer = ByteBuffer.allocate(1024)
while True:
nread = zis.read(buffer.array(), 0, 1024)
if nread <= 0:
break
fos.write(buffer.array(), 0, nread)
fos.close()
zis.close()
logger.info("End extracting:" + str(zip) + " --> " + str(dest))
def unzip_atp_wallet(wallet_file, location):
if not os.path.exists(location):
os.mkdir(location)
buffer = jarray.zeros(1024, "b")
fis = FileInputStream(wallet_file)
zis = ZipInputStream(fis)
ze = zis.getNextEntry()
while ze:
fileName = ze.getName()
newFile = File(location + File.separator + fileName)
File(newFile.getParent()).mkdirs()
fos = FileOutputStream(newFile)
len = zis.read(buffer)
while len > 0:
fos.write(buffer, 0, len)
len = zis.read(buffer)
fos.close()
zis.closeEntry()
ze = zis.getNextEntry()
zis.closeEntry()
zis.close()
fis.close()
def get_file(self, src_file, dst_file):
sftp = SFTPv3Client(self.client)
dst_fd = None
src_fd = None
try:
dst_file = os.path.abspath(dst_file.replace('/', os.sep))
dst_dir = os.path.dirname(dst_file)
if not os.path.exists(dst_dir):
os.makedirs(dst_dir)
dst_fd = FileOutputStream(dst_file)
stats = sftp.stat(src_file)
src_size = stats.size
src_fd = sftp.openFileRO(src_file)
size = 0
arraysize = SSHClient.BUFFER_SIZE
data = jarray.zeros(arraysize, 'b')
while True:
moredata = sftp.read(src_fd, size, data, 0, arraysize)
datalen = len(data)
if moredata == -1:
break
if src_size - size < arraysize:
datalen = src_size - size
dst_fd.write(data, 0, datalen)
size += datalen
finally:
if src_fd:
sftp.closeFile(src_fd)
if dst_fd:
dst_fd.flush()
dst_fd.close()
sftp.close()
def syncNmapPortConfigFile(agentPath):
'''
Sync nmap port config with global probe's "port number to port name" mapping
'''
logger.debug('synchronizing nmap port config file')
portConfigFilename = agentPath + CollectorsParameters.getDiscoveryConfigFolder(
) + CollectorsParameters.FILE_SEPARATOR + 'portNumberToPortName.xml'
mamservice = File(portConfigFilename)
nmapservice = File(agentPath +
CollectorsParameters.getDiscoveryResourceFolder() +
CollectorsParameters.FILE_SEPARATOR + 'nmap-services')
if nmapservice.lastModified() > mamservice.lastModified():
return
nmapFile = FileOutputStream(nmapservice)
document = SAXBuilder(0).build(mamservice)
# document = parse(portConfigFilename)
ports = XmlWrapper(document.getRootElement().getChildren('portInfo'))
for port in ports:
if int(port.getAttributeValue("discover")):
portNumber = port.getAttributeValue("portNumber")
portName = port.getAttributeValue("portName")
portProtocol = port.getAttributeValue("portProtocol")
nmapFile.write("%s\t%s/%s\r\n" %
(portName, portNumber, portProtocol))
nmapFile.close()
def extract_obj(self, file_path, res, offset):
try:
f = File(file_path)
# check same name file.
counter = 0
while True:
# The same file name is not exists.
if not f.exists():
break
# Count up the file name.
counter += 1
stem = u"".join(file_path.split(u".")[:-1])
ex = file_path.split(u".")[-1]
_file_path = u"{}({}).{}".format(stem, counter, ex)
f = File(_file_path)
fos = FileOutputStream(f)
fos.write(res[offset:])
self._stdout.printf("save as \"%s\".\n\n", f.getPath())
fos.close()
except Exception as e:
self._stderr.println("[!] In extract_obj.")
self._stderr.println(e)
def extractZip(zip, dest):
"extract zip archive to dest directory"
logger.info("Begin extracting:" + zip + " --> " + dest)
mkdir_p(dest)
zipfile = ZipFile(zip)
entries = zipfile.entries()
while entries.hasMoreElements():
entry = entries.nextElement()
if entry.isDirectory():
mkdir_p(os.path.join(dest, entry.name))
else:
newFile = File(dest, entry.name)
mkdir_p(newFile.parent)
zis = zipfile.getInputStream(entry)
fos = FileOutputStream(newFile)
nread = 0
buffer = ByteBuffer.allocate(1024)
while True:
nread = zis.read(buffer.array(), 0, 1024)
if nread <= 0:
break
fos.write(buffer.array(), 0, nread)
fos.close()
zis.close()
logger.info("End extracting:" + str(zip) + " --> " + str(dest))
def exportAll(exportConfigFile, generalConfigFile):
try:
print "Loading export config from :", exportConfigFile
exportConfigProp = loadProps(exportConfigFile,generalConfigFile)
adminUrl = exportConfigProp.get("adminUrl")
user = exportConfigProp.get("user")
passwd = exportConfigProp.get("password")
jarFileName = exportConfigProp.get("jarFileName")
customFile = exportConfigProp.get("customizationFile")
passphrase = exportConfigProp.get("passphrase")
project = exportConfigProp.get("project")
connectToServer(user, passwd, adminUrl)
print 'connected'
ALSBConfigurationMBean = findService("ALSBConfiguration", "com.bea.wli.sb.management.configuration.ALSBConfigurationMBean")
print "ALSBConfiguration MBean found"
print project
if project == None :
ref = Ref.DOMAIN
collection = Collections.singleton(ref)
if passphrase == None :
print "Export the config"
theBytes = ALSBConfigurationMBean.export(collection, true, None)
else :
print "Export and encrypt the config"
theBytes = ALSBConfigurationMBean.export(collection, true, passphrase)
else :
ref = Ref.makeProjectRef(project);
print "Export the project", project
collection = Collections.singleton(ref)
theBytes = ALSBConfigurationMBean.exportProjects(collection, passphrase)
print 'fileName',jarFileName
aFile = File(jarFileName)
print 'file',aFile
out = FileOutputStream(aFile)
out.write(theBytes)
out.close()
print "ALSB Configuration file: "+ jarFileName + " has been exported"
if customFile != None:
print collection
query = EnvValueQuery(None, Collections.singleton(EnvValueTypes.WORK_MANAGER), collection, false, None, false)
customEnv = FindAndReplaceCustomization('Set the right Work Manager', query, 'Production System Work Manager')
print 'EnvValueCustomization created'
customList = ArrayList()
customList.add(customEnv)
print customList
aFile = File(customFile)
out = FileOutputStream(aFile)
Customization.toXML(customList, out)
out.close()
# print "ALSB Dummy Customization file: "+ customFile + " has been created"
except:
raise
def save_example(self, label, image_byte_array):
output_file_name = label + "_" + str(java.lang.System.currentTimeMillis()) + ".png"
save_path = File(self.dir_path, output_file_name).getCanonicalPath()
fileos = FileOutputStream(save_path)
for byte in image_byte_array:
fileos.write(byte)
fileos.flush()
fileos.close()
def writeToFile(text, testId):
filename = "log/%s-%s-page-%d.html" % (grinder.processName,
testId,
grinder.runNumber)
# Use Java FileOutputStream since it has better Unicode handling
os = FileOutputStream(filename)
os.write(text)
os.close()
def run(string, args=[], callback=None, callbackOnErr=False):
def out (exit, call, inp, err):
return {
"exitCode": exit,
"callbackReturn": call,
"inputArray": inp,
"errorArray": err
}
tmp = File.createTempFile('tmp', None)
tmp.setExecutable(True)
writer = FileOutputStream(tmp);
writer.write(string)
writer.flush()
writer.close()
try:
process = Runtime.getRuntime().exec([tmp.getAbsolutePath()] + ([str(i) for i in args] or []))
process.waitFor()
inp = BufferedReader(InputStreamReader(process.getInputStream()))
err = BufferedReader(InputStreamReader(process.getErrorStream()))
errFlag = False
inputArray = []
errorArray = []
holder = inp.readLine()
while holder != None:
print holder
inputArray += [holder]
holder = inp.readLine()
holder = err.readLine()
while holder != None:
errFlag = True
errorArray += [holder]
holder = err.readLine()
tmp.delete()
if errFlag:
if callback and callbackOnErr: return out(1, callback(out(1, None, inputArray, errorArray)), inputArray, errorArray)
else: return out(1, None, inputArray, errorArray)
else:
if callback: return out(0, callback(out(0, None, inputArray, [])), inputArray, [])
else: return out(0, None, inputArray, [])
except Exception as e:
print str(e)
tmp.delete()
if callback and callbackOnErr: return out(3, callback(out(3, None, [], str(e).split("\n"))), [], str(e).split("\n"))
else: return out(3, None, [], str(e).split("\n"))
def exportAll():
try:
ALSBConfigurationMBean = findService(
"ALSBConfiguration",
"com.bea.wli.sb.management.configuration.ALSBConfigurationMBean")
print "ALSBConfiguration MBean found"
print project
if project == "None":
ref = Ref.DOMAIN
collection = Collections.singleton(ref)
if passphrase == None:
print "Export the config"
theBytes = ALSBConfigurationMBean.export(
collection, true, None)
else:
print "Export and encrypt the config"
theBytes = ALSBConfigurationMBean.export(
collection, true, passphrase)
else:
ref = Ref.makeProjectRef(project)
print "Export the project", project
collection = Collections.singleton(ref)
theBytes = ALSBConfigurationMBean.exportProjects(
collection, passphrase)
aFile = File(exportJar)
out = FileOutputStream(aFile)
out.write(theBytes)
out.close()
print "ALSB Configuration file: " + exportJar + " has been exported"
if customFile != "None":
print collection
query = EnvValueQuery(
None, Collections.singleton(EnvValueTypes.WORK_MANAGER),
collection, false, None, false)
customEnv = FindAndReplaceCustomization(
'Set the right Work Manager', query,
'Production System Work Manager')
print 'EnvValueCustomization created'
customList = ArrayList()
customList.add(customEnv)
print customList
aFile = File(customFile)
out = FileOutputStream(aFile)
Customization.toXML(customList, out)
out.close()
print "ALSB Dummy Customization file: " + customFile + " has been created"
except:
raise
def unzipFile(zipFile, outputFolder):
zis = ZipInputStream(FileInputStream(zipFile))
ze = zis.getNextEntry()
while ze != None:
fileName = ze.getName()
newFile = File(outputFolder + File.separator + fileName)
print ("file unzip : " + str(newFile.getAbsoluteFile()))
File(newFile.getParent()).mkdirs()
fos = FileOutputStream(newFile)
len = zis.read()
while len > 0:
fos.write(len)
len = zis.read()
fos.close()
ze = zis.getNextEntry()
zis.closeEntry()
zis.close()
def unzipFile(zipFile, outputFolder):
zis = ZipInputStream(FileInputStream(zipFile))
ze = zis.getNextEntry()
while ze != None:
fileName = ze.getName()
newFile = File(outputFolder + File.separator + fileName)
print("file unzip : " + str(newFile.getAbsoluteFile()))
File(newFile.getParent()).mkdirs()
fos = FileOutputStream(newFile)
len = zis.read()
while len > 0:
fos.write(len)
len = zis.read()
fos.close()
ze = zis.getNextEntry()
zis.closeEntry()
zis.close()
def exportAll():
try:
ALSBConfigurationMBean = findService("ALSBConfiguration", "com.bea.wli.sb.management.configuration.ALSBConfigurationMBean")
print "ALSBConfiguration MBean found"
print project
if project == "None" :
ref = Ref.DOMAIN
collection = Collections.singleton(ref)
if passphrase == None :
print "Export the config"
theBytes = ALSBConfigurationMBean.export(collection, true, None)
else :
print "Export and encrypt the config"
theBytes = ALSBConfigurationMBean.export(collection, true, passphrase)
else :
ref = Ref.makeProjectRef(project);
print "Export the project", project
collection = Collections.singleton(ref)
theBytes = ALSBConfigurationMBean.exportProjects(collection, passphrase)
aFile = File(exportJar)
out = FileOutputStream(aFile)
out.write(theBytes)
out.close()
print "ALSB Configuration file: "+ exportJar + " has been exported"
if customFile != "None":
print collection
query = EnvValueQuery(None, Collections.singleton(EnvValueTypes.WORK_MANAGER), collection, false, None, false)
customEnv = FindAndReplaceCustomization('Set the right Work Manager', query, 'Production System Work Manager')
print 'EnvValueCustomization created'
customList = ArrayList()
customList.add(customEnv)
print customList
aFile = File(customFile)
out = FileOutputStream(aFile)
Customization.toXML(customList, out)
out.close()
print "ALSB Dummy Customization file: "+ customFile + " has been created"
except:
raise
def getNagiosStatus(self, name=""):
# os io functions not implented in wsadmin jython, use java equivalents
while 1:
try:
fifo=FileOutputStream(self.__nagiosFifo)
logger.debug("Opened fifo for output : %s" % self.__nagiosFifo)
statuslines=[]
perflines=[]
code=-1
statusMessage="WAS-STATUS UNKNOWN"
try:
for a in self.getWasObjects('Server') + self.getWasObjects('DB2DataSource') + self.getWasObjects('AS400DataSource'):
for s in a.getNagiosStatus():
if (s.getPerformanceData()!=""):
perflines.append("%s%s" % (a.getContainmentPath(),s.getPerformanceData()))
statuslines.append("%s %s-%s" % (a.getConfigType(),a.getName(),s.getMessage()))
if s.getCode()>code:
code=s.getCode()
if code==NagiosStat.OK:
statusmessage="WAS-OK"
else:
statusmessage="WAS-%s" % s.getMessage()
except:
logger.error("Error querying PMI state")
logger.error(str(sys.exc_info()[1]) + "\n")
logger.debug(sys.exc_info()[2].dumpStack() + "\n")
if len(perflines) > 0:
statuslines[-1]="%s|%s" % (statuslines[-1],perflines[0])
if len(perflines[1:]) > 1:
statuslines+=perflines[1:]
if code==-1: code=NagiosStat.UNKNOWN
fifo.write(java.lang.String("%s\n" % code).getBytes())
fifo.write(java.lang.String("%s\n" % statusmessage).getBytes())
for s in statuslines:
fifo.write(java.lang.String(s).getBytes())
fifo.write(java.lang.String("\n").getBytes())
# Send EOT to end the message
fifo.write(java.lang.String("\004").getBytes())
fifo.close()
logger.debug("Closed fifo for output : %s" % self.__nagiosFifo)
except:
fifo.close()
logger.error("Interupted syscall, closed fifo for output : %s" % self.__nagiosFifo)
time.sleep(1)
def extract_tools_data_from_jar_files(self):
"""Create tools directories from JAR files with tools data
Read directories from tools/data and create jar files in tools/jar
"""
jarDir = File(self.jarDir)
for jarFileName in jarDir.list():
toolDir = File.separator.join([self.toolsDir, jarFileName[:-4]])
self.delete_old_tool_directory(File(toolDir))
jar = JarFile(File(self.jarDir, jarFileName))
for entry in jar.entries():
f = File(File.separator.join([self.toolsDir, entry.getName()]))
if entry.isDirectory():
f.mkdir()
continue
inputStream = jar.getInputStream(entry)
fos = FileOutputStream(f)
while inputStream.available() > 0:
fos.write(inputStream.read())
fos.close()
inputStream.close()
def writeMovie(bytes, fname):
from org.openlaszlo.iv.flash.api import FlashFile, Script
from org.openlaszlo.iv.flash.api.action import DoAction, Program
file = FlashFile.newFlashFile()
file.version = 5
file.mainScript = Script(1)
frame = file.mainScript.newFrame()
program = Program(bytes, 0, len(bytes))
frame.addFlashObject(DoAction(program))
istr = file.generate().inputStream
from jarray import zeros
bytes = zeros(istr.available(), 'b')
istr.read(bytes)
from java.io import FileOutputStream
ostr = FileOutputStream(fname)
try:
ostr.write(bytes)
finally:
ostr.close()
return
def _get_file(self, source, dest):
localfile = FileOutputStream(dest)
tempstats = self._client.stat(source)
remotefilesize = tempstats.size
remotefile = self._client.openFileRO(source)
size = 0
arraysize = 4096
data = jarray.zeros(arraysize, "b")
while True:
moredata = self._client.read(remotefile, size, data, 0, arraysize)
datalen = len(data)
if moredata == -1:
break
if remotefilesize - size < arraysize:
datalen = remotefilesize - size
localfile.write(data, 0, datalen)
size += datalen
self._client.closeFile(remotefile)
localfile.flush()
localfile.close()
def extract_tools_data_from_jar_files(self):
"""Create tools directories from JAR files with tools data
Read directories from tools/data and create jar files in tools/jar
"""
jarDir = File(self.jarDir)
for jarFileName in jarDir.list():
toolDir = File.separator.join([self.toolsDir, jarFileName[:-4]])
self.delete_old_tool_directory(File(toolDir))
jar = JarFile(File(self.jarDir, jarFileName))
for entry in jar.entries():
f = File(File.separator.join([self.toolsDir, entry.getName()]))
if entry.isDirectory():
f.mkdir()
continue
inputStream = jar.getInputStream(entry)
fos = FileOutputStream(f)
while inputStream.available() > 0:
fos.write(inputStream.read())
fos.close()
inputStream.close()
def _get_file(self, remote_path, local_path):
local_file = FileOutputStream(local_path)
remote_file_size = self._client.stat(remote_path).size
remote_file = self._client.openFileRO(remote_path)
array_size_bytes = 4096
data = jarray.zeros(array_size_bytes, 'b')
offset = 0
while True:
read_bytes = self._client.read(remote_file, offset, data, 0,
array_size_bytes)
data_length = len(data)
if read_bytes == -1:
break
if remote_file_size - offset < array_size_bytes:
data_length = remote_file_size - offset
local_file.write(data, 0, data_length)
offset += data_length
self._client.closeFile(remote_file)
local_file.flush()
local_file.close()
def _get_file(self, remote_path, local_path):
local_file = FileOutputStream(local_path)
remote_file_size = self._client.stat(remote_path).size
remote_file = self._client.openFileRO(remote_path)
array_size_bytes = 4096
data = jarray.zeros(array_size_bytes, 'b')
offset = 0
while True:
read_bytes = self._client.read(remote_file, offset, data, 0,
array_size_bytes)
data_length = len(data)
if read_bytes == -1:
break
if remote_file_size - offset < array_size_bytes:
data_length = remote_file_size - offset
local_file.write(data, 0, data_length)
offset += data_length
self._client.closeFile(remote_file)
local_file.flush()
local_file.close()
def syncNmapPortConfigFile(agentPath):
'''
Sync nmap port config with global probe's "port number to port name" mapping
'''
logger.debug('synchronizing nmap port config file')
portConfigFilename = agentPath + CollectorsParameters.getDiscoveryConfigFolder() + CollectorsParameters.FILE_SEPARATOR + 'portNumberToPortName.xml'
mamservice = File(portConfigFilename)
nmapservice = File(agentPath + CollectorsParameters.getDiscoveryResourceFolder() + CollectorsParameters.FILE_SEPARATOR + 'nmap-services')
if nmapservice.lastModified() > mamservice.lastModified():
return
nmapFile = FileOutputStream(nmapservice)
document = SAXBuilder(0).build(mamservice)
# document = parse(portConfigFilename)
ports = XmlWrapper(document.getRootElement().getChildren('portInfo'))
for port in ports:
if int(port.getAttributeValue("discover")):
portNumber = port.getAttributeValue("portNumber")
portName = port.getAttributeValue("portName")
portProtocol = port.getAttributeValue("portProtocol")
nmapFile.write("%s\t%s/%s\r\n" % (portName, portNumber, portProtocol))
nmapFile.close()
def __init__(self):
dataDir = Settings.dataDir + 'quickstart/'
# Open the stream. Read only access is enough for Aspose.Words to load a document.
stream = FileInputStream(dataDir + 'Document.doc')
# Load the entire document into memory.
doc = Document(stream)
# You can close the stream now, it is no longer needed because the document is in memory.
stream.close()
# ... do something with the document
# Convert the document to a different format and save to stream.
dstStream = ByteArrayOutputStream()
doc.save(dstStream, SaveFormat.RTF)
output = FileOutputStream(dataDir + "Document Out.rtf")
output.write(dstStream.toByteArray())
output.close()
print "Document loaded from stream and then saved successfully."
def __init__(self):
dataDir = Settings.dataDir + 'quickstart/'
# Open the stream. Read only access is enough for Aspose.Words to load a document.
stream = FileInputStream(dataDir + 'Document.doc')
# Load the entire document into memory.
doc = Document(stream)
# You can close the stream now, it is no longer needed because the document is in memory.
stream.close()
# ... do something with the document
# Convert the document to a different format and save to stream.
dstStream = ByteArrayOutputStream()
doc.save(dstStream, SaveFormat.RTF)
output = FileOutputStream(dataDir + "Document Out.rtf")
output.write(dstStream.toByteArray())
output.close()
print "Document loaded from stream and then saved successfully."
def dom_document_from_file(f):
if hasattr(f, 'name'):
documentURI = os.path.abspath(f.name)
else:
documentURI = None
bytestring = f.read()
temp = File.createTempFile('someprefix', 'tmp')
outputstream = FileOutputStream(temp)
try:
outputstream.write(bytestring)
finally:
outputstream.close()
inputstream = FileInputStream(temp)
try:
dom_doc = dom_document_from_inputstream(inputstream)
dom_doc.setDocumentURI(documentURI)
return dom_doc
finally:
inputstream.close()
def dom_document_from_file(f):
if hasattr(f, 'name'):
documentURI = os.path.abspath(f.name)
else:
documentURI = None
bytestring = f.read()
temp = File.createTempFile('someprefix', 'tmp')
outputstream = FileOutputStream(temp)
try:
outputstream.write(bytestring)
finally:
outputstream.close()
inputstream = FileInputStream(temp)
try:
dom_doc = dom_document_from_inputstream(inputstream)
dom_doc.setDocumentURI(documentURI)
return dom_doc
finally:
inputstream.close()
e)
resp.setMessage(msg)
return
attachedFilename = oup.getAttachedFilename()
attachedFile = oup.getAttachedFile()
if attachedFilename and attachedFile:
# spaces will screw up the command line string
attachedFilename = attachedFilename.replace(" ", "")
# dealing with a java byte[] so write it out with java
from java.io import File, FileOutputStream
attachedFilename = createTargetFile("",
OUT_DIR + '/' + attachedFilename)
f = File(attachedFilename)
fos = FileOutputStream(f)
fos.write(attachedFile)
fos.flush()
fos.close()
if test:
try:
os.remove(awipsPathname)
except EnvironmentError:
pass # ignore
if attachedFilename:
try:
os.remove(attachedFilename)
except EnvironmentError:
pass # ignore
resp.setSendLocalSuccess(True)
def exportAll():
try:
ALSBConfigurationMBean = findService(
"ALSBConfiguration",
"com.bea.wli.sb.management.configuration.ALSBConfigurationMBean")
print "ALSBConfiguration MBean found"
print project
if project == "None":
ref = Ref.DOMAIN
collection = Collections.singleton(ref)
if passphrase == None:
print "Export the config"
theBytes = ALSBConfigurationMBean.export(
collection, true, None)
else:
print "Export and encrypt the config"
theBytes = ALSBConfigurationMBean.export(
collection, true, passphrase)
else:
ref = Ref.makeProjectRef(project)
print "Export the project", project
collection = Collections.singleton(ref)
theBytes = ALSBConfigurationMBean.exportProjects(
collection, passphrase)
aFile = File(exportJar)
out = FileOutputStream(aFile)
out.write(theBytes)
out.close()
print "ALSB Configuration file: " + exportJar + " has been exported"
if customFile != "None":
print collection
# see com.bea.wli.sb.util.EnvValueTypes in sb-kernel-api.jar for the values
#EnvValueQuery evquery =
# new EnvValueQuery(
# null, // search across all resource types
# Collections.singleton(EnvValueTypes.URI_ENV_VALUE_TYPE), // search only the URIs
# null, // search across all projects and folders.
# true, // only search across resources that are
# // actually modified/imported in this session
# "localhost", // the string we want to replace
# false // not a complete match of URI. any URI
# // that has "localhost" as substring will match
# );
refTypes = HashSet()
refTypes.add(EnvValueTypes.SERVICE_URI_TABLE)
refTypes.add(EnvValueTypes.SERVICE_URI)
query = EnvValueQuery(
Collections.singleton(Refs.BUSINESS_SERVICE_TYPE), refTypes,
collection, false, "search string", false)
# query = EnvValueQuery(None, Collections.singleton(EnvValueTypes.SERVICE_URI_TABLE), collection, false, "search string", false)
customEnv = FindAndReplaceCustomization('new endpoint url', query,
'replace string')
# object = QualifiedEnvValue(Refs.makeBusinessSvcRef(ref,'file'), Refs.BUSINESS_SERVICE_TYPE, "XSDvalidation/file", "aaa")
# objects = ArrayList()
# objects.add(object)
# customEnv2 = EnvValueCustomization('Set the right endpoints', objects)
print 'EnvValueCustomization created'
customList = ArrayList()
customList.add(customEnv)
# customList.add(customEnv2)
print customList
aFile = File(customFile)
out = FileOutputStream(aFile)
Customization.toXML(customList, out)
out.close()
print "ALSB Dummy Customization file: " + customFile + " has been created"
except:
raise
def exportAll():
try:
ALSBConfigurationMBean = findService("ALSBConfiguration", "com.bea.wli.sb.management.configuration.ALSBConfigurationMBean")
print "ALSBConfiguration MBean found"
print project
if project == "None" :
ref = Ref.DOMAIN
collection = Collections.singleton(ref)
if passphrase == None :
print "Export the config"
theBytes = ALSBConfigurationMBean.export(collection, true, None)
else :
print "Export and encrypt the config"
theBytes = ALSBConfigurationMBean.export(collection, true, passphrase)
else :
ref = Ref.makeProjectRef(project);
print "Export the project", project
collection = Collections.singleton(ref)
theBytes = ALSBConfigurationMBean.exportProjects(collection, passphrase)
aFile = File(exportJar)
out = FileOutputStream(aFile)
out.write(theBytes)
out.close()
print "ALSB Configuration file: "+ exportJar + " has been exported"
if customFile != "None":
print collection
# see com.bea.wli.sb.util.EnvValueTypes in sb-kernel-api.jar for the values
#EnvValueQuery evquery =
# new EnvValueQuery(
# null, // search across all resource types
# Collections.singleton(EnvValueTypes.URI_ENV_VALUE_TYPE), // search only the URIs
# null, // search across all projects and folders.
# true, // only search across resources that are
# // actually modified/imported in this session
# "localhost", // the string we want to replace
# false // not a complete match of URI. any URI
# // that has "localhost" as substring will match
# );
refTypes = HashSet()
refTypes.add(EnvValueTypes.SERVICE_URI_TABLE)
refTypes.add(EnvValueTypes.SERVICE_URI)
query = EnvValueQuery(Collections.singleton(Refs.BUSINESS_SERVICE_TYPE), refTypes, collection, false, "search string", false)
# query = EnvValueQuery(None, Collections.singleton(EnvValueTypes.SERVICE_URI_TABLE), collection, false, "search string", false)
customEnv = FindAndReplaceCustomization('new endpoint url', query, 'replace string')
# object = QualifiedEnvValue(Refs.makeBusinessSvcRef(ref,'file'), Refs.BUSINESS_SERVICE_TYPE, "XSDvalidation/file", "aaa")
# objects = ArrayList()
# objects.add(object)
# customEnv2 = EnvValueCustomization('Set the right endpoints', objects)
print 'EnvValueCustomization created'
customList = ArrayList()
customList.add(customEnv)
# customList.add(customEnv2)
print customList
aFile = File(customFile)
out = FileOutputStream(aFile)
Customization.toXML(customList, out)
out.close()
print "ALSB Dummy Customization file: "+ customFile + " has been created"
except:
raise
def download_request(url, file_path, user_agent, referer_url, timeout,
max_redirects, log):
import jarray
from java.net import URL, HttpURLConnection
from java.io import FileOutputStream
try:
input_stream = None
file_output_stream = FileOutputStream(file_path)
HttpURLConnection.setFollowRedirects(True)
first_request = True
is_redirect = False
cookies = None
redirect_counter = 0
while is_redirect or first_request:
http_url_connection = URL(url).openConnection()
http_url_connection.setFollowRedirects(True)
http_url_connection.setInstanceFollowRedirects(True)
http_url_connection.setRequestProperty("Accept-Language",
"en-US,en;q=0.8")
http_url_connection.setConnectTimeout(timeout)
http_url_connection.setReadTimeout(timeout)
http_url_connection.setRequestMethod("GET")
http_url_connection.setRequestProperty("User-Agent",
user_agent)
http_url_connection.setRequestProperty("Accept-Language",
"en-US,en;q=0.8")
if cookies != None and len(cookies) > 0:
http_url_connection.setRequestProperty("Cookie", cookies)
if referer_url != None:
# Note: Referer not Referrer! (see: Wikipedia)
# ^ ^^
http_url_connection.setRequestProperty(
"Referer", referer_url)
http_url_connection.connect()
first_request = False
# check for redirect
is_redirect = False
status_code = http_url_connection.getResponseCode()
if status_code == HttpURLConnection.HTTP_NOT_FOUND:
raise ScratchtobatHTTP404Error(
"HTTP 404 NOT FOUND for URL: " + url)
if status_code != HttpURLConnection.HTTP_OK:
if status_code == HttpURLConnection.HTTP_MOVED_TEMP \
or status_code == HttpURLConnection.HTTP_MOVED_PERM \
or status_code == HttpURLConnection.HTTP_SEE_OTHER:
redirect_counter += 1
if redirect_counter > max_redirects:
raise ScratchtobatError("Maximum number of HTTP redirects " \
"{} reached!".format(max_redirects))
is_redirect = True
referer_url = url
# set redirect URL from "location" header field as new URL
url = http_url_connection.getHeaderField("Location")
cookies = http_url_connection.getHeaderField(
"Set-Cookie")
log.debug("Redirecting to URL: {}".format(url))
input_stream = http_url_connection.getInputStream()
byte_buffer = jarray.zeros(4096, "b")
length = input_stream.read(byte_buffer)
while length > 0:
file_output_stream.write(byte_buffer, 0, length)
length = input_stream.read(byte_buffer)
finally:
try:
if input_stream != None:
input_stream.close()
except:
if file_output_stream != None:
file_output_stream.close()
from java.io import FileInputStream as FI
from java.io import FileOutputStream as FO
file_read = FI("file1.txt")
file_write = FO("file2.txt")
while True:
ch = file_read.read()
if ch != -1:
file_write.write(chr(ch))
else:
break
file_read.close()
file_write.close()
def exportAll(exportConfigFile):
try:
print "Loading export config from :", exportConfigFile
exportConfigProp = loadProps(exportConfigFile)
adminUrl = exportConfigProp.get("adminUrl")
exportUser = exportConfigProp.get("exportUser")
exportPasswd = exportConfigProp.get("exportPassword")
exportJar = exportConfigProp.get("exportJar")
customFile = exportConfigProp.get("customizationFile")
passphrase = exportConfigProp.get("passphrase")
project = exportConfigProp.get("project")
connectToServer(exportUser, exportPasswd, adminUrl)
ALSBConfigurationMBean = findService(
"ALSBConfiguration",
"com.bea.wli.sb.management.configuration.ALSBConfigurationMBean")
print "ALSBConfiguration MBean found"
print project
if project == None:
ref = Ref.DOMAIN
collection = Collections.singleton(ref)
if passphrase == None:
print "Export the config"
theBytes = ALSBConfigurationMBean.export(
collection, true, None)
else:
print "Export and encrypt the config"
theBytes = ALSBConfigurationMBean.export(
collection, true, passphrase)
else:
ref = Ref.makeProjectRef(project)
print "Export the project", project
collection = Collections.singleton(ref)
theBytes = ALSBConfigurationMBean.exportProjects(
collection, passphrase)
aFile = File(exportJar)
out = FileOutputStream(aFile)
out.write(theBytes)
out.close()
print "ALSB Configuration file: " + exportJar + " has been exported"
if customFile != None:
print collection
customList = ArrayList()
query = EnvValueQuery(
None, Collections.singleton(EnvValueTypes.WORK_MANAGER),
collection, false, None, false)
customEnv = FindAndReplaceCustomization(
'Set the right Work Manager', query,
'Production System Work Manager')
customList.add(customEnv)
# Uncomment the next three lines to update the server and port to the
# alsb30_prod environment correctly
#query = EnvValueQuery(None, Collections.singleton(EnvValueTypes.SERVICE_URI), collection, false, 'localhost:7001', false)
#customEnv = FindAndReplaceCustomization('Update to the correct server and port number', query, 'localhost:7101')
#customList.add(customEnv)
print 'EnvValueCustomization created'
print customList
aFile = File(customFile)
out = FileOutputStream(aFile)
Customization.toXML(customList, out)
out.close()
print "ALSB Customization file: " + customFile + " has been created"
except:
raise
def download_file():
'''
FUNC FOR DOWNLOADING DATA FROM MVD.RF WEBSITE
'''
global buffer_size, downloaded
ins_log('Create proxy settings', 'using ' + str(ODIAgent))
if ODIAgent == 'Internal':
proxy_address = TEST_FILE_DOWNLOADS.ProdProxyHost
proxy_port = TEST_FILE_DOWNLOADS.ProdProxyPort
proxy_user = '******'
proxy_passe = "70320DB646F3C6740262E9224E8A88C7"
proxy_domain = "BANKEXP"
proxy_pass = cryptor.decrypt(proxy_passe)
else:
proxy_address = FILE_DOWNLOADS.ProdProxyHost
proxy_port = FILE_DOWNLOADS.ProdProxyPort
proxy_user = "******"
proxy_passe = "32A47DEE17B2F967BA6094BB609ABF8E"
proxy_domain = "BANKEXP"
proxy_pass = cryptor.decrypt(PROXY_PASSE)
ins_log("Downloading...", url)
builder = OkHttpClient.Builder()
#builder.followRedirects(False).followSslRedirects(False);
builder.connectTimeout(5, TimeUnit.MINUTES).writeTimeout(
5, TimeUnit.MINUTES).readTimeout(5, TimeUnit.MINUTES)
httpClient = builder.proxy(
Proxy(Proxy.Type.HTTP, InetSocketAddress(
proxy_address,
proxy_port))).proxyAuthenticator(proxy_authenticator).build()
call = httpClient.newCall(Request.Builder().url(url).get().build())
#//Call to server
response = call.execute()
#//
ins_log('Call to web server', str(response))
#print(response.code())
if (response.code() == 200): #//Check Response code
inputStream = None
outputStream = None
target = response.body().contentLength()
try:
inputStream = response.body().byteStream()
#//Get stream of bytes
buffer = zeros(buffer_size, 'b')
#//Creating buffer bytes(1024*4) #bytearray(4096)
outputStream = FileOutputStream(File(archived_file))
print_download_process(0, target)
while (downloaded < target):
readed = inputStream.read(buffer)
if (readed == -1):
break
else:
outputStream.write(buffer, 0, readed)
#//write buff
downloaded += readed
print_download_process(downloaded, target)
except Exception:
ins_log("Downloading Error", str(Exception))
finally:
if (inputStream != None):
inputStream.close()
elif (outputStream != None):
outputStream.close()
ins_log("File downloaded!",
str(url) + ' filename:' + str(archived_file))
def deployToTarget(envConfigProp, mavenGroupId, mavenArtifactId, mavenVersionId, serviceGroupName, serviceType, serviceName, interfaceVersion, jarFileLocation, environmentName, domainName, pwd):
print 'Deploying ' + mavenGroupId + ':' + mavenArtifactId + ' ' + interfaceVersion + ' (' + mavenVersionId + ') to ' + domainName + ' in ' + environmentName
auditAction('Attempting deployment of ' + mavenGroupId + '::' + mavenArtifactId + ' version ' + mavenVersionId + "(" + interfaceVersion + ") to domain " + domainName + " in " + environmentName + " environment")
try:
SessionMBean = None
targetName = environmentName + "." + domainName
admin_host = envConfigProp.get("env." + targetName + ".admin_host")
admin_port = envConfigProp.get("env." + targetName + ".admin_port")
adminUrl = "t3://" + admin_host + ":" + admin_port
importUser = envConfigProp.get("env." + targetName + ".username")
importPassword = envConfigProp.get("env." + targetName + ".password")
if pwd != None:
importPassword = pwd
print 'Target url: ', adminUrl
print 'Importing :', jarFileLocation
if importUser == None:
print 'Connecting with config files'
connectToServerWithConfig(configFile, keyFile, adminUrl)
else:
print 'Connecting with username/pwd'
connectToServer(importUser, importPassword, adminUrl)
print 'Attempting to import :', jarFileLocation, " on OSB Admin Server listening on :", adminUrl
theBytes = readBinaryFile(jarFileLocation)
print 'Read file', jarFileLocation
sessionName = createSessionName()
print 'Created session', sessionName
SessionMBean = getSessionManagementMBean(sessionName)
print 'SessionMBean started session'
ALSBConfigurationMBean = findService(String("ALSBConfiguration.").concat(sessionName), "com.bea.wli.sb.management.configuration.ALSBConfigurationMBean")
listOfAutoBackupEnabledEnvironments = envConfigProp.get("autobackupOnDeployment")
if (listOfAutoBackupEnabledEnvironments != None):
if (environmentName in listOfAutoBackupEnabledEnvironments):
#take backup from domain
print "Backing up artifact resources including the common bits"
allRefs = None
if (serviceGroupName == 'Common'):
#step 1: find version specific resources e.g. Common_CanonicalDataModel/v1
projectResourcesQuery = ResourceQuery(None)
projectResourcesQuery.setPath(serviceGroupName + '_' + serviceName + '/' + interfaceVersion + '/*')
artifactRefs = ALSBConfigurationMBean.getRefs(projectResourcesQuery)
allRefs = artifactRefs
else:
#take backup from domain
print "Backing up artifact resources including the common bits"
#step 1: find project's own resources e.g. QueryServices/data/Parcel/v1
projectResourcesQuery = ResourceQuery(None)
projectResourcesQuery.setPath(serviceGroupName + '/' + serviceType + '/' + serviceName + "/" + interfaceVersion + "/*")
artifactRefs = ALSBConfigurationMBean.getRefs(projectResourcesQuery)
#step 2: find common group project resources e.g. QueryServices/common/v1
commonGroupResourcesQuery = ResourceQuery(None)
commonGroupResourcesQuery.setPath(serviceGroupName + '/common/' + interfaceVersion + "/*")
commonGroupRefs = ALSBConfigurationMBean.getRefs(commonGroupResourcesQuery)
#combine all the refs
allRefs = commonGroupRefs
allRefs.addAll(artifactRefs)
exportJarBytes = ALSBConfigurationMBean.export(allRefs, false, None)
todaysFormattedDate = java.text.SimpleDateFormat("yyyyMMdd").format(java.util.Date())
exportFolder = os.environ["HOME"] + "/backups/" + environmentName
File(exportFolder).mkdirs()
if System.getenv('BUILD_TAG') is not None:
exportFileName = todaysFormattedDate + "_" + System.getenv('BUILD_TAG') + "_" + domainName + ".jar"
else:
exportFileName = todaysFormattedDate + "_" + domainName + ".jar"
exportFile = File(exportFolder, exportFileName)
out = FileOutputStream(exportFile)
out.write(exportJarBytes)
out.close()
print "Automatic backup taken to: "+ exportFileName
print 'Uploading jar file'
ALSBConfigurationMBean.uploadJarFile(theBytes)
print 'Jar file uploaded'
print 'Performing deployment'
alsbJarInfo = ALSBConfigurationMBean.getImportJarInfo()
#23/05/2016 Commenting this out as it is causing a deployment issue
# alsbImportPlan = alsbJarInfo.getDefaultImportPlan()
# alsbImportPlan.setPassphrase(passphrase)
# alsbImportPlan.setPreserveExistingEnvValues(true)
importResult = ALSBConfigurationMBean.importUploaded(None) #alsbImportPlan)
if importResult.getFailed().isEmpty() == false:
print 'One or more resources could not be imported properly:'
printDiagMap(importResult.getImportDiagnostics())
raise
customizationFile = getCustomizationFilePath(environmentName, serviceGroupName, serviceName, interfaceVersion, domainName)
print 'Applying customization file: ', customizationFile
ALSBConfigurationMBean.customize(Customization.fromXML(FileInputStream(customizationFile)))
groupCustomizationFile = getGroupCustomizationFilePath(environmentName, serviceGroupName, interfaceVersion, domainName)
if (os.path.isfile(groupCustomizationFile)):
print 'Applying group level customization file: ', groupCustomizationFile
ALSBConfigurationMBean.customize(Customization.fromXML(FileInputStream(groupCustomizationFile)))
print 'Activating change session'
buildTag = System.getenv('BUILD_TAG')
if buildTag == None:
buildTag = ""
else:
buildTag = ". Jenkins build tag: " + buildTag
SessionMBean.activateSession(sessionName, "Scripted import of " + serviceGroupName + ' : ' + serviceName + ' ' + interfaceVersion + ' version ' + mavenVersionId + buildTag)
print "Deployment of : " + serviceGroupName + ' : ' + serviceName + ' ' + interfaceVersion + ' version ' + mavenVersionId + " successful"
auditAction("Service " + serviceGroupName + ' : ' + serviceName + ' ' + interfaceVersion + ' version ' + mavenVersionId + " deployed to domain " + domainName + " in " + environmentName + " environment")
SessionMBean = None
except:
auditAction("Failed to deploy service " + serviceGroupName + ' : ' + serviceName + ' ' + interfaceVersion + ' version ' + mavenVersionId + " to domain " + domainName + " in " + environmentName + " environment")
print "Unexpected error:", sys.exc_info()[0]
dumpStack()
if SessionMBean != None:
SessionMBean.discardSession(sessionName)
raise
def download_request(url, file_path, user_agent, referer_url, timeout, max_redirects, log):
import jarray
from java.net import URL, HttpURLConnection
from java.io import FileOutputStream
try:
input_stream = None
file_output_stream = FileOutputStream(file_path)
HttpURLConnection.setFollowRedirects(True)
first_request = True
is_redirect = False
cookies = None
redirect_counter = 0
while is_redirect or first_request:
http_url_connection = URL(url).openConnection()
http_url_connection.setFollowRedirects(True)
http_url_connection.setInstanceFollowRedirects(True)
http_url_connection.setRequestProperty("Accept-Language", "en-US,en;q=0.8")
http_url_connection.setConnectTimeout(timeout)
http_url_connection.setReadTimeout(timeout)
http_url_connection.setRequestMethod("GET")
http_url_connection.setRequestProperty("User-Agent", user_agent)
http_url_connection.setRequestProperty("Accept-Language", "en-US,en;q=0.8")
if cookies != None and len(cookies) > 0:
http_url_connection.setRequestProperty("Cookie", cookies)
if referer_url != None:
# Note: Referer not Referrer! (see: Wikipedia)
# ^ ^^
http_url_connection.setRequestProperty("Referer", referer_url);
http_url_connection.connect()
first_request = False
# check for redirect
is_redirect = False
status_code = http_url_connection.getResponseCode()
if status_code == HttpURLConnection.HTTP_NOT_FOUND:
raise ScratchtobatHTTP404Error("HTTP 404 NOT FOUND for URL: " + url)
if status_code != HttpURLConnection.HTTP_OK:
if status_code == HttpURLConnection.HTTP_MOVED_TEMP \
or status_code == HttpURLConnection.HTTP_MOVED_PERM \
or status_code == HttpURLConnection.HTTP_SEE_OTHER:
redirect_counter += 1
if redirect_counter > max_redirects:
raise ScratchtobatError("Maximum number of HTTP redirects " \
"{} reached!".format(max_redirects))
is_redirect = True
referer_url = url
# set redirect URL from "location" header field as new URL
url = http_url_connection.getHeaderField("Location")
cookies = http_url_connection.getHeaderField("Set-Cookie")
log.debug("Redirecting to URL: {}".format(url))
input_stream = http_url_connection.getInputStream()
byte_buffer = jarray.zeros(4096, "b")
length = input_stream.read(byte_buffer)
while length > 0:
file_output_stream.write(byte_buffer, 0, length)
length = input_stream.read(byte_buffer)
finally:
try:
if input_stream != None:
input_stream.close()
except:
if file_output_stream != None:
file_output_stream.close()
except Exception, e:
msg = 'Product ' + awipsWanPil + ' failed to be ingested and archived properly. Reason:\n' + str(e)
resp.setMessage(msg)
return
attachedFilename = oup.getAttachedFilename()
attachedFile = oup.getAttachedFile()
if attachedFilename and attachedFile:
# spaces will screw up the command line string
attachedFilename = attachedFilename.replace(" ", "")
# dealing with a java byte[] so write it out with java
from java.io import File, FileOutputStream
attachedFilename = createTargetFile("", OUT_DIR + '/' + attachedFilename)
f = File(attachedFilename)
fos = FileOutputStream(f)
fos.write(attachedFile)
fos.flush()
fos.close()
messageIdToAcknowledge = None
#----------
# Check if product should be distributed over WAN via NCF
#----------
wmoID = contents[0:6]
splitAddr = address.split(',')
for addr in splitAddr:
if addr != '000': # 000 is local only
_Logger.info("Addressee is " + addr)
#----------
# Check if product should be sent to the NWWS for uplink
#----------