def _sapJmxConnect(credId, ip, Framework):
r'''@types: str, str, Framework -> Result
'''
message = None
status = False
for version in SAPJmxAgent.getAvailableVersions():
props = Properties()
props.setProperty(AgentConstants.VERSION_PROPERTY, version)
logger.debug('Trying to connect to ip=%s by \'%s\' '\
'protocol (assuming version %s)...' % (ip,
_PROTOCOL_NAME,
version))
try:
sap = Framework.getAgent(AgentConstants.SAP_JMX_AGENT,
ip, credId, props)
sap.connect()
except:
message = 'error: %s' % sys.exc_info()[1]
logger.debug('connection to ip=%s by \'%s\' '\
'protocol failed. %s' % (ip,
_PROTOCOL_NAME, message))
continue
else:
logger.debug('connection to ip=%s by \'%s\' '\
'protocol is successful' % (ip, _PROTOCOL_NAME))
status = True
break
finally:
sap and sap.disconnect()
return Result(status, message)
def DiscoveryMain(Framework):
OSHVResult = ObjectStateHolderVector()
CmdbOIDFactory = CmdbObjectID.Factory
hostId = CmdbOIDFactory.restoreObjectID(Framework.getDestinationAttribute('hostId'))
sqlServerId = CmdbOIDFactory.restoreObjectID(Framework.getDestinationAttribute('id'))
try:
props = Properties()
instance_name = Framework.getDestinationAttribute('instanceName')
if instance_name and instance_name != 'NA' and instance_name.find('\\') != -1:
props.setProperty('sqlprotocol_dbsid', instance_name[instance_name.find('\\')+1:])
mssqlClient = Framework.createClient(props)
connection = SqlServerConnection.ClientSqlServerConnection(mssqlClient)
logger.debug("got connection")
discoveryOptions = SqlServerDiscoveryOptions()
discoveryOptions.discoverConfigs = Boolean.parseBoolean(Framework.getParameter('discoverConfigs'))
discoveryOptions.discoverDbUser = Boolean.parseBoolean(Framework.getParameter('discoverDbUser'))
discoveryOptions.discoverSqlFile = Boolean.parseBoolean(Framework.getParameter('discoverSqlFile'))
discoveryOptions.discoverSqlJob = Boolean.parseBoolean(Framework.getParameter('discoverSqlJob'))
discoveryOptions.discoverProcedures = Boolean.parseBoolean(Framework.getParameter('discoverStoredProcedures'))
discoveryOptions.discoverInternalProcedures = Boolean.parseBoolean(Framework.getParameter('discoverInternalProcedures'))
sqlServer = SqlServer.SqlServer(connection, discoveryOptions)
OSHVResult.addAll(sqlServer.collectData(hostId, sqlServerId, discoveryOptions.discoverConfigs))
mssqlClient.close()
except JavaException, ex:
strException = ex.getMessage()
errormessages.resolveAndReport(strException, ClientsConsts.SQL_PROTOCOL_NAME, Framework)
def getShellUtils(Framework, shellName, credentials, ip, codepage, warningsList, errorsList):
failedPorts = []
#check which credential is good for the shell:
client = None
for credentialId in credentials:
try:
port = None
props = Properties()
props.setProperty(BaseAgent.ENCODING, codepage)
props.setProperty(CollectorsConstants.ATTR_CREDENTIALS_ID, credentialId)
logger.debug('try credential %s' % credentialId)
client = Framework.createClient(props)
shellUtils = shellutils.ShellUtils(client, None, shellName)
if shellUtils:
return (shellUtils, credentialId)
except IOException, ioEx:
strException = str(ioEx.getMessage())
errormessages.resolveAndAddToObjectsCollections(strException, shellName, warningsList, errorsList)
if client:
client.close()
# we failed to connect - add the problematic port to failedPorts list
if port:
failedPorts.append(port)
except (UnsupportedEncodingException, UnsupportedCharsetException), ex:
strException = str(ex.getClass().getName())
shouldStop = errormessages.resolveAndAddToObjectsCollections(strException, shellName, warningsList, errorsList)
if client:
client.close()
if shouldStop:
return (None, None)
def connectToDb(localFramework, ipAddress, dbPort):
try:
theDbClient = None
## Get protocols
protocols = localFramework.getAvailableProtocols(ipAddress, ClientsConsts.SQL_PROTOCOL_NAME)
for protocolID in protocols:
## If this protocol entry is not for a Sybase DB, ignore it
if localFramework.getProtocolProperty(protocolID, CollectorsConstants.SQL_PROTOCOL_ATTRIBUTE_DBTYPE) != 'Sybase':
debugPrint(5, '[' + SCRIPT_NAME + ':DiscoveryMain] Ignoring non Sybase protocol entry...')
continue
## Don't bother reconnecting if a connection has already been established
if not theDbClient:
## Set DB properties
dbConnectionProperties = Properties()
dbConnectionProperties.setProperty(CollectorsConstants.PROTOCOL_ATTRIBUTE_PORT, dbPort)
# Establish JDBC connection
debugPrint(5, '[' + SCRIPT_NAME + ':connectToDb] Attempting connection to CiscoWorks database at port <%s>...' % dbPort)
try:
theDbClient = localFramework.createClient(protocolID, dbConnectionProperties)
except:
theDbClient and theDBClient.close()
return theDbClient
except:
excInfo = logger.prepareJythonStackTrace('')
logger.warn('[' + SCRIPT_NAME + ':connectToDb] Exception: <%s>' % excInfo)
pass
def __createConnection(self):
"create database"
self.__connection = None
try:
logger.info("RegisterJDBC driver: com.mysql.jdbc.Driver")
driver = com.mysql.jdbc.Driver.newInstance()
if driver and driver.acceptsURL(self.__rootJdbcUrl):
props = Properties()
props.setProperty("user", "root")
props.setProperty("password", self.__rootPwd)
logger.info("Create JDBC connection:" + self.__rootJdbcUrl)
self.__connection = driver.connect(self.__rootJdbcUrl, props)
if self.__connection:
logger.info("Create db select prepared statement")
self.__dbstmt = self.__connection.prepareStatement("show databases;")
rs = self.__dbstmt.executeQuery()
self.__databases = []
while (rs.next()):
db = rs.getString(1)
self.__databases.append(db)
except:
self.__connection = None
type, value, traceback = sys.exc_info()
logger.severe("create connection error:" + `value`)
def DiscoveryMain(Framework):
Framework = jee_connection.EnhancedFramework(Framework)
isAppResourcesDiscoveryEnabled = _asBoolean(Framework.getParameter('discoverAppResources'))
isJmsResourcesDiscoveryEnabled = _asBoolean(Framework.getParameter('discoverJMSResources'))
platform = jee.Platform.JBOSS
try:
r'''In addition to the credentials we have to specify port number and
version of the platform.
Credentials may be defined without such information that is very important
for establishing connections
'''
port = entity.WeakNumeric(int)
port.set(Framework.getDestinationAttribute('port'))
version = Framework.getDestinationAttribute('version')
properties = Properties()
properties.put(CollectorsConstants.PROTOCOL_ATTRIBUTE_PORT, str(port.value()))
properties.put(AgentConstants.VERSION_PROPERTY, version)
client = Framework.createClient(properties)
jmxProvider = jmx.Provider(client)
except (Exception, JException), exc:
logger.warnException("Failed to establish connection")
jee_connection.reportError(Framework, str(exc), platform.getName())
def DiscoveryMain(Framework):
OSHVResult = ObjectStateHolderVector()
ipAddress = Framework.getDestinationAttribute('ip_address')
credentialsId = Framework.getDestinationAttribute('credentialsId')
hostId = Framework.getDestinationAttribute('hostId')
hostOsh = ms_exchange_utils.restoreHostById(hostId)
hostName = Framework.getDestinationAttribute('hostName')
if not hostName or hostName == 'N/A':
hostName = ms_exchange_utils.getHostNameFromWmi(Framework)
if not hostName:
errobj = errorobject.createError(errorcodes.FAILED_GETTING_INFORMATION_NO_PROTOCOL, ['host name'], 'Failed to obtain host name')
logger.reportErrorObject(errobj)
return
props = Properties()
props.put(AgentConstants.PROP_WMI_NAMESPACE, WMI_NAMESPACE)
try:
wmiClient = Framework.createClient(props)
wmiAgent = WmiAgent(wmiClient, Framework)
try:
discoverExchangeServer(wmiAgent, ipAddress, credentialsId, OSHVResult, Framework, hostOsh, hostName)
finally:
wmiClient.close()
except Exception, ex:
message = ex.getMessage()
if (re.search("Invalid\sclass", message)):
message = 'Unable to get Exchange data from WMI'
logger.debugException(message)
errormessages.resolveAndReport(message, WMI_PROTOCOL, Framework)
def main():
# Create the new layer
newProjection = currentView().getProjectionCode()
newLayerSchema = createSchema()
newLayerSchema.append("ID","INTEGER")
newLayerSchema.append("GEOMETRY","GEOMETRY")
geometryType = getGeometryType(POINT,D2)
newLayerSchema.get("GEOMETRY").setGeometryType(geometryType)
newLayerName = "/APLICACIONES/GIS/gvSIG-desktop-2.2.0/mynewlayer.shp"
newLayer = createShape(newLayerSchema,newLayerName,CRS=newProjection,geometryType=POINT)
# Connect to the database
props = Properties()
props.put("user","YOUR_USER")
props.put("password","YOUR_PASSWD")
db = Driver().connect("jdbc:postgresql://localhost/YOUR_DB", props)
# Get geometry info from database and insert it into the new layer
c = db.createStatement()
rs = c.executeQuery("select table.id, ST_X(table.coordinatesxy),ST_Y(table.coordenatesxy) from YOUR_TABLES where YOUR_WHERE_CLAUSE")
data = {}
while rs.next():
id = rs.getInt(1)
newX = rs.getObject(2)
newY = rs.getObject(3)
print(id,newX,newY)
newGeom = createPoint(newX,newY)
dbValues = {"ID":id,"GEOMETRY":newGeom}
newLayer.append(dbValues)
rs.close()
c.close()
db.close()
currentView().addLayer(newLayer)
newLayer.commit()
def createSQLServerConnection(serverName, port, schemaName, userName, password):
try:
url = (
"jdbc:mercury:sqlserver://"
+ serverName
+ ":"
+ str(port)
+ ";DatabaseName="
+ schemaName
+ ";allowPortWithNamedInstance=true"
)
logger.info("URL: ", url)
driverName = "com.mercury.jdbc.sqlserver.SQLServerDriver"
props = Properties()
props.put("user", userName)
props.put("password", password)
cl = Class.forName(driverName, 1, Thread.currentThread().getContextClassLoader())
jdbcDriver = cl.newInstance()
conn = jdbcDriver.connect(url, props)
unlockConnection(conn)
return conn
except Exception, e:
logger.error("setConnection: ")
logger.error(e)
raise Exception(e)
def _load_config(self):
""" Returns the default jclouds client configuration. """
endpoint = "http://" + self.__config.address + "/api"
props = Properties()
props.put("abiquo.endpoint", endpoint)
[props.put(name, value)
for (name, value) in self.__config.client_config]
return props
def _createSshClient(framework, cred_id, port=1000):
try:
from java.util import Properties
props = Properties()
props.put(Protocol.PROTOCOL_ATTRIBUTE_PORT, str(port))
return framework.createClient(cred_id, props)
except JException, je:
raise ovm_flow.ConnectionException(je.getMessage())
def load_redback_registry():
reg = Properties()
reg_file = 'redback.properties'
if os.path.isfile(reg_file):
reg_fd = FileInputStream(reg_file)
reg.load(reg_fd)
reg_fd.close()
return reg
def _buildConnectionProps(instNr, connClient=None):
props = Properties()
logger.debug('Connecting to a SAP instance number:', str(instNr))
props.setProperty(Protocol.SAP_PROTOCOL_ATTRIBUTE_SYSNUMBER, instNr)
if connClient:
logger.debug('Connecting to a SAP system with client: %s' % connClient)
props.setProperty(Protocol.SAP_PROTOCOL_ATTRIBUTE_CLIENT, connClient)
return props
def loadProperties(fileName):
properties = Properties()
input = FileInputStream(fileName)
properties.load(input)
input.close()
result= {}
for entry in properties.entrySet(): result[entry.key] = entry.value
return result
def read_versions(self):
versionsProperties = Properties()
fin = FileInputStream(self.versionsFileName)
versionsProperties.load(fin)
scriptVersion = versionsProperties.getProperty("script")
toolsVersion = versionsProperties.getProperty("tools")
fin.close()
return scriptVersion, toolsVersion
def intialize():
global domainLocation;
global jvmLocation;
global domainTemplate;
global domainProps;
global adminUserName;
global adminPassword;
global userConfigFile;
global userKeyFile;
# test arguments
if len(sys.argv) != 6:
print 'Usage: createDomain.sh <template-file> <default.properties_file> <property_file> <wls_username> <wls_password>';
exit();
print 'Starting the initialization process';
domainTemplate = sys.argv[1];
print 'Using Domain Template: ' + domainTemplate;
try:
domainProps = Properties()
# load DEFAULT properties
# print 'Reading default properties from '+sys.argv[2];
input = FileInputStream(sys.argv[2])
domainProps.load(input)
input.close()
# load properties and overwrite defaults
input = FileInputStream(sys.argv[3])
domainProps.load(input)
input.close()
adminUserName = sys.argv[4];
adminPassword = sys.argv[5];
# Files for generating secret key
userConfigFile = File(sys.argv[3]).getParent()+'/'+domainProps.getProperty('domainName')+'.userconfig'
print userConfigFile
userKeyFile = File(sys.argv[3]).getParent()+'/'+domainProps.getProperty('domainName')+'.userkey'
print userKeyFile
domainLocation = domainProps.getProperty('domainsDirectory') + pathSeparator + domainProps.getProperty('domainName');
print 'Domain Location: ' + domainLocation;
if len(domainProps.getProperty('jvmLocation')) == 0:
print 'JVM location property not defined - cancel creation !';
exit();
print 'JVM Location: ' + domainProps.getProperty('jvmLocation');
except:
dumpStack()
print 'Cannot initialize creation process !';
exit();
print 'Initialization completed';
def get_file_properties(filename):
if filename not in deploy_files_properties:
data = get_file_data(filename)
input_stream = ByteArrayInputStream(data)
properties = Properties()
properties.load(input_stream)
deploy_files_properties[filename] = properties
return deploy_files_properties[filename]
def connect(self, framework):
credentialsId = framework.getDestinationAttribute('credentialsId')#user selects the creds in the UI so we already know them
connection_ip = framework.getDestinationAttribute('ip_address')
self._credentialsId = credentialsId
try:
props = Properties()
props.setProperty('ip_address', connection_ip)
except Exception, e:
pass
def loadProperties(propFileName):
from java.io import FileInputStream
from java.util import Properties
fileprop = Properties()
fileStream = FileInputStream(propFileName)
fileprop.load(fileStream)
return fileprop
def DiscoveryMain(Framework):
# General variables
OSHVResult = ObjectStateHolderVector()
properties = Properties()
protocol = Framework.getDestinationAttribute('Protocol')
processToPortDict = {} ## {PID:[processName, listeningPort, ipAddress, path, version, status, processCommandline]}
databaseDict = {} ## {instanceName/SID:[dbType, listeningPort, ipAddress, installPath, version, status]}
database_ip_service_endpoints ={} ## {instanceName:[ip_address:port]}
client = None
secondClient = None
isWindows = 'true'
# Attempt to create a client
try:
client = Framework.createClient()
## We have a client
## Get processes running on the box
if(client):
dbconnect_utils.debugPrint(3, '[' + SCRIPT_NAME + ':DiscoveryMain] Got client <%s>' % client.getClientType())
if client.getClientType() == 'wmi':
## Open a second client connection to the DEFAULT namespace for registry access
props = Properties()
props.setProperty(AgentConstants.PROP_WMI_NAMESPACE, 'root\\DEFAULT')
secondClient = Framework.createClient(props)
processToPortDict = dbconnect_agentutils.getProcListByWMI(client, secondClient)
# elif client.getClientType() == 'snmp':
# processToPortDict = dbconnect_agentutils.getProcListBySNMP(client)
else:
Framework.reportWarning('Unable to connect using WMI')
## We have process and port infromation
## Find databases, if any
if processToPortDict != None and len(processToPortDict) > 0:
for pid in processToPortDict.keys():
logger.debug('dddd: ', '[' + SCRIPT_NAME + ':DiscoveryMain] Got process/service/software <%s> listening on port <%s:%s>' % ((processToPortDict[pid])[dbconnect_utils.PROCESSNAME_INDEX], (processToPortDict[pid])[dbconnect_utils.IP_INDEX], (processToPortDict[pid])[dbconnect_utils.PORT_INDEX]))
if Framework.getParameter('discover_oracle').strip().lower() == 'true':
dbconnect_oracle.findDatabases(client, processToPortDict, databaseDict, isWindows, secondClient)
if Framework.getParameter('discover_mssql').strip().lower() == 'true':
dbconnect_mssql.findDatabases(client, processToPortDict, databaseDict, database_ip_service_endpoints, isWindows, secondClient)
if databaseDict != None and len(databaseDict) > 0:
for dbName in databaseDict.keys():
dbconnect_utils.debugPrint('Found <%s> instance <%s> (%s) with listener port <%s:%s> and installed in <%s>' % ((databaseDict[dbName])[dbconnect_utils.DBTYPE_INDEX], dbName, (databaseDict[dbName])[dbconnect_utils.STATUS_INDEX], (databaseDict[dbName])[dbconnect_utils.IP_INDEX], (databaseDict[dbName])[dbconnect_utils.PORT_INDEX], (databaseDict[dbName])[dbconnect_utils.PATH_INDEX]))
OSHVResult.addAll(dbconnect_utils.makeDbOSHs(databaseDict))
else:
Framework.reportWarning('No databases found')
else:
## If we're here, we couldn't find any processes, service, or software
## and we have no data to search for databases
Framework.reportWarning('Unable to get a list or processes, services, or installed software')
else:
dbconnect_utils.debugPrint('Unable to connect using WMI')
except Exception, ex:
exInfo = ex.getMessage()
errormessages.resolveAndReport(exInfo, protocol, Framework)
def create_default_client(framework, cred_id, ip_address):
client = None
properties = Properties()
properties.setProperty("ip_address", ip_address)
try:
client = framework.createClient(cred_id, properties)
except(Exception, JavaException), e:
msg = e.getMessage() if hasattr(e, 'getMessage') else str(e)
raise ConnectionException(msg or 'Connection failed')
def read_tools_list(tmpToolsListFile):
"""Read the tools names and tools data urls
from the downlaoded tools list
"""
properties = Properties()
fin = FileInputStream(tmpToolsListFile)
properties.load(fin)
toolsRefs = properties.getProperty("tools.list").split("|")
fin.close()
return toolsRefs
def _establish_connection(framework, config):
properties = Properties()
creds_id = config.credentialsId
inst_nr = config.instance_number
client_nr = config.connection_client
properties.setProperty('credentialsId', creds_id)
if inst_nr:
properties.setProperty(Protocol.SAP_PROTOCOL_ATTRIBUTE_SYSNUMBER, inst_nr)
if client_nr:
properties.setProperty(Protocol.SAP_PROTOCOL_ATTRIBUTE_CLIENT, client_nr)
try:
return framework.createClient(properties)
except (NoClassDefFoundError, MissingJarsException, ExceptionInInitializerError):
errormsg = 'SAP JCo drivers are missing'
logger.debugException(errormsg)
raise flow.ConnectionException(errormsg)
except JcoException, jco:
errormsg = jco.getMessage()
logger.debugException(errormsg)
if config.ip_address and config.instance_number:
cred_args = sap_flow.getCredentialId(framework)
logger.debug('cred_args:', cred_args)
if not cred_args:
raise flow.ConnectionException(errormsg)
for args in cred_args:
logger.debug('args:', args)
properties.setProperty('credentialsId', args[0])
try:
return framework.createClient(properties)
except:
logger.debugException('')
raise flow.ConnectionException(errormsg)
def addPropertiesFromFile(props, filename, site_home):
addProps = Properties()
input = FileInputStream(filename)
addProps.load(input)
input.close()
baseFileList = addProps.getProperty('base')
if not baseFileList is None:
baseFiles = baseFileList.split(",")
for baseFile in baseFiles:
baseFileResolved=getBaseFile(baseFile, site_home)
if baseFileResolved=='':
log.error('Configuration inherits from properties file that does not exist: ' + baseFile)
log.info('Suggested Fix: Update the base property within ' + filename + ' to the correct file path')
sys.exit()
log.debug('Attempting to load properties from ' + baseFile)
addPropertiesFromFile(addProps, baseFileResolved, site_home)
addProps.remove('base')
enum = addProps.keys()
while enum.hasMoreElements():
key = enum.nextElement()
if props.getProperty(key) is None:
props.setProperty(key, addProps.getProperty(key))
addDataLinage(key,filename,addProps.getProperty(key))
def loadPropertiesFromFile(filename,props):
newprops=Properties()
inputstr = FileInputStream(filename)
newprops.load(inputstr)
inputstr.close()
enum = newprops.keys()
while enum.hasMoreElements():
key = enum.nextElement()
if props.getProperty(key) is None:
props.setProperty(key, newprops.getProperty(key))
return props
def loadProperties(self, filename):
f = File("%s" % filename)
props = Properties()
if f.exists():
print 'Chargement de la configuration depuis ' + filename
fis = FileInputStream( f )
props.load( fis )
else:
print 'ERROR: Impossible de charger le fichier de configuration ' + filename
return props
def DiscoveryMain(Framework):
properties = Properties()
SITE_ID = Framework.getDestinationAttribute('SITE_ID')
instance_number = Framework.getDestinationAttribute('instance_number')
connection_client = Framework.getDestinationAttribute('connection_client')
logger.debug('Connecting to a SAP instance number:', str(instance_number))
properties.setProperty(Protocol.SAP_PROTOCOL_ATTRIBUTE_SYSNUMBER, instance_number)
if (connection_client is not None) and (connection_client != 'NA'):
logger.debug('Connecting to a SAP system with client:', str(connection_client))
properties.setProperty(Protocol.SAP_PROTOCOL_ATTRIBUTE_CLIENT, connection_client)
discoverScenarioProcesses = Boolean.parseBoolean(Framework.getParameter('discoverScenarioProcesses'))
GET_PROCESS_STEPS = Boolean.parseBoolean(Framework.getParameter('getProcessSteps'))
reportCIsInChunks = Boolean.parseBoolean(Framework.getParameter('reportCIsInChunks'))
errormsg = ''
client = None
try:
try:
client = Framework.createClient(properties)
solman = saputils.SapSolman(client)
except (NoClassDefFoundError, MissingJarsException, ExceptionInInitializerError):
errormsg = 'SAP drivers are missing'
logger.debugException(errormsg)
except:
errormsg = 'Connection failed'
logger.debugException(errormsg)
else:
try:
sVector, sysToOshPairs = discoverSystems(solman)
Framework.sendObjects(sVector)
sVector.clear()
sysNames = [system.getName() for system, _ in sysToOshPairs]
sys_name_to_system = {}
for system, _ in sysToOshPairs:
sys_name_to_system[system.getName()] = system
component2system = logicalComponents(sysNames, solman)
businessProcesses(Framework, solman, component2system,
sys_name_to_system, SITE_ID, GET_PROCESS_STEPS, reportCIsInChunks,
discoverScenarioProcesses)
except:
strmsg = str(sys.exc_info()[1])
if (strmsg.upper().find('TABLE_NOT_AVAILABLE') > -1):
errmsg = 'No solution manager found'
logger.debugException(errmsg)
Framework.reportError(errmsg)
else:
# unknown exception caught
raise
except JException, ex:
ex_info = ex.getMessage()
errormessages.resolveAndReport(ex_info, 'SAP JCO', Framework)
def _connectToEc2Service(framework, credentialsId):
"""Connection method for the **Elastic Cloud Service**
@types: Framework, str -> Ec2Client
"""
from com.hp.ucmdb.discovery.library.clients.cloud.aws import ServiceType
from com.hp.ucmdb.discovery.library.clients.cloud.aws import Client
properties = Properties()
properties.setProperty(Client.AWS_SERVICE_TYPE_PROPERTY, ServiceType.EC2.name()) # @UndefinedVariable
properties.setProperty("credentialsId", credentialsId)
return framework.createClient(properties).getService()
def _connectToIamService(framework, credentialsId):
"""Connection method for the **Identity and Access Management**
@types: Framework, str -> IamClient
"""
from com.hp.ucmdb.discovery.library.clients.cloud.aws import ServiceType
from com.hp.ucmdb.discovery.library.clients.cloud.aws import Client
properties = Properties()
properties.setProperty("credentialsId", credentialsId)
properties.setProperty(Client.AWS_SERVICE_TYPE_PROPERTY, ServiceType.IAM.name()) # @UndefinedVariable
return framework.createClient(properties).getService()
def DiscoveryMain(Framework):
OSHVResult = ObjectStateHolderVector()
ip = Framework.getDestinationAttribute("ip_address")
sid = Framework.getDestinationAttribute("sid")
port = Framework.getDestinationAttribute("port")
hostId = Framework.getDestinationAttribute("hostId")
if (ip == None) or (ip == "NA"):
# checked and pass all IPs of given host
ips = Framework.getTriggerCIDataAsList("host_ips")
else:
ips = [ip]
for currIP in ips:
if len(currIP) == 0:
continue
logger.debug("Checking sqlserver with no user on ipaddress:", currIP)
protocols = Framework.getAvailableProtocols(currIP, ClientsConsts.SQL_PROTOCOL_NAME)
for protocol in protocols:
dbClient = None
try:
try:
if dbutils.protocolMatch(
Framework, protocol, "microsoftsqlserver", sid, port
) or dbutils.protocolMatch(Framework, protocol, "microsoftsqlserverntlm", sid, port):
props = Properties()
props.setProperty("ip_address", currIP)
dbClient = Framework.createClient(protocol, props)
hostOSH = modeling.createOshByCmdbIdString("host", hostId)
oracleOSH = modeling.createDatabaseOSH(
"sqlserver",
sid,
str(dbClient.getPort()),
dbClient.getIpAddress(),
hostOSH,
protocol,
None,
dbClient.getTimeout(),
dbClient.getDbVersion(),
dbClient.getAppVersion(),
)
logger.debug("Successfully connected to sqlserver object ", sid, " on ", currIP)
OSHVResult.add(oracleOSH)
# since this is knownn oracle and we found credentials for it we can finish execution
return OSHVResult
except:
if logger.isDebugEnabled():
logger.debugException("Unexpected CreateClient() for sqlserver client Exception:")
finally:
if dbClient != None:
dbClient.close()
Framework.reportWarning("Failed to connect using all protocols")
return OSHVResult
def createEmpty():
return Properties()
if __name__ == '__main__':
from wlstModule import * # @UnusedWildImport
def usage():
print "Usage:"
print "ShutdownDomain.py -t domainDir -p <properties>"
#=======================================================================================
# get domain properties.
#=======================================================================================
props = Properties()
domainDir = ""
properties = ""
nodemgrHomeDir = ""
try:
opts, args = getopt.getopt(sys.argv[1:], "p:t:")
except getopt.GetoptError, err:
print str(err)
usage()
sys.exit(2)
for opt, arg in opts:
if opt == "-t":
domainDir = arg
elif opt == "-p":
#
# Each worker thread:
# - Creates a queue session
# - Sends ten messages
# - Closes the queue session
from net.grinder.script.Grinder import grinder
from net.grinder.script import Test
from jarray import zeros
from java.util import Properties, Random
from javax.jms import Session
from javax.naming import Context, InitialContext
from weblogic.jndi import WLInitialContextFactory
# Look up connection factory and queue in JNDI.
properties = Properties()
properties[Context.PROVIDER_URL] = "t3://localhost:7001"
properties[Context.INITIAL_CONTEXT_FACTORY] = WLInitialContextFactory.name
initialContext = InitialContext(properties)
connectionFactory = initialContext.lookup(
"weblogic.examples.jms.QueueConnectionFactory")
queue = initialContext.lookup("weblogic.examples.jms.exampleQueue")
initialContext.close()
# Create a connection.
connection = connectionFactory.createQueueConnection()
connection.start()
random = Random()
def getS(self):
outputProperties = Properties()
outputProperties.put(INDENT, "yes")
return self.__builder.root().asString(outputProperties)
from java.sql import DriverManager
from java.lang import Class
from java.util import Properties
url = 'jdbc:splice://localhost:1527/splicedb'
driver = 'org.apache.derby.jdbc.ClientDriver'
props = Properties()
props.setProperty('user', 'APP')
props.setProperty('password', 'APP')
jcc = Class.forName(driver).newInstance()
conn = DriverManager.getConnection(url, props)
stmt = conn.createStatement()
rs = stmt.executeQuery("select * from sys.systables")
rowCount = 0
while (rs.next() and rowCount < 10) :
rowCount += 1
print "Record=[" + str(rowCount) + \
"] id=[" + rs.getString('TABLEID') + \
"] name=[" + rs.getString('TABLENAME') + \
"] type=[" + rs.getString('TABLETYPE') + "]"
rs.close()
stmt.close()
conn.close()
try:
import com.ibm.db2.jcc.DB2Driver as Driver
except:
print "\n\tNo jdbc driver available, start wsadmin with\n"
print "\t\"-javaoption -Dcom.ibm.ws.scripting.classpath=path/db2jcc4.jar\"\n"
print "\tSee http://scripting101.org/resources/installing-the-scripts/ for details\n"
# Get configuration from properties file
configParser = ConfigParser.ConfigParser()
configFilePath = r'ibmcnx/ibmcnx.properties'
configParser.read(configFilePath)
# Change User and Password
props = Properties()
props.put('user', configParser.get('Database', 'dbUser'))
props.put('password', configParser.get('Database', 'dbPassword'))
jdbcPath = 'jdbc:db2://' + configParser.get(
'Database', 'dbHost') + ':' + configParser.get(
'Database', 'dbPort') + '/' + configParser.get('Database', 'dbName')
conn = Driver().connect(jdbcPath, props)
stmt = conn.createStatement()
email = raw_input(
"\n\tMail address of profile you want to deactivate: ").lower()
sql = 'select PROF_UID,PROF_MAIL,PROF_MAIL_LOWER,PROF_GUID from empinst.employee where PROF_MAIL_LOWER = \'' + \
def __createWMIClient(self, framework, namespace):
''' Create WMI client with needed namespace '''
props = Properties()
props.setProperty(AgentConstants.PROP_WMI_NAMESPACE, namespace)
return framework.createClient(props)
mam_utils.info('########## port: ', port)
mam_utils.info('########## host_id: ', host_id)
mam_utils.info('########## oracle_id: ', oracle_id)
mam_utils.info('########## ip: ', ip)
mam_utils.info('########## credential_id: ', credential_id)
mam_utils.info('########## view_name: ', view_name)
mam_utils.info('########## request_type_name: ', request_type_name)
mam_utils.info('########## oracle_id: ', oracle_id)
mam_utils.info('########## business_id: ', business_id)
mam_utils.info('########## ci_type: ', ci_type)
#######################################################
### Stateholder for the business CI as virtual object
#######################################################
businessOSH = ObjectStateHolder('business', 6, business_id,
AppilogTypes.CMDB_OBJECT_ID)
properties = Properties()
properties.setProperty(Protocol.PROTOCOL_ATTRIBUTE_PORT, port)
properties.setProperty(Protocol.SQL_PROTOCOL_ATTRIBUTE_DBSID, sid)
#######################################################
### select statement for getting the objects
#######################################################
dbObj = "select request_id,description, created_by_username, status_name from %s where request_type_name='%s' " % (
view_name, request_type_name)
## dbLink ="SELECT request_id FROM mitg.SAPCCM_DEPL_TASKS"
def parseDbObjRes(dbObjRes):
if dbObjRes:
rows = dbObjRes.getRowCount()
cols = dbObjRes.getColumnCount()
if mam_utils.isDebugEnabled():
def _jdbc_connect_jpype(jclassname, url, driver_args, java_opts, jars, libs):
import jpype
if not jpype.isJVMStarted():
args = []
class_path = []
if jars:
class_path.extend(jars)
class_path.extend(_get_classpath())
if class_path:
args.append('-Djava.class.path=%s' %
os.path.pathsep.join(class_path))
if java_opts:
for arg in java_opts:
args.append(arg)
if libs:
# path to shared libraries
libs_path = os.path.pathsep.join(libs)
args.append('-Djava.library.path=%s' % libs_path)
# jvm_path = ('/usr/lib/jvm/java-6-openjdk'
# '/jre/lib/i386/client/libjvm.so')
jvm_path = jpype.getDefaultJVMPath()
global old_jpype
if hasattr(jpype, '__version__'):
try:
ver_match = re.match('\d+\.\d+', jpype.__version__)
if ver_match:
jpype_ver = float(ver_match.group(0))
if jpype_ver < 0.7:
old_jpype = True
except ValueError:
pass
if old_jpype:
jpype.startJVM(jvm_path, *args)
else:
jpype.startJVM(jvm_path,
*args,
ignoreUnrecognized=True,
convertStrings=True)
if not jpype.isThreadAttachedToJVM():
jpype.attachThreadToJVM()
if _jdbc_name_to_const is None:
types = jpype.java.sql.Types
types_map = {}
for i in types.__javaclass__.getClassFields():
if old_jpype:
const = i.getStaticAttribute()
else:
const = i.__get__(i)
types_map[i.getName()] = const
_init_types(types_map)
global _java_array_byte
if _java_array_byte is None:
def _java_array_byte(data):
return jpype.JArray(jpype.JByte, 1)(data)
# register driver for DriverManager
jpype.JClass(jclassname)
if isinstance(driver_args, dict):
Properties = jpype.java.util.Properties
info = Properties()
for k, v in driver_args.items():
info.setProperty(k, v)
dargs = [info]
else:
dargs = driver_args
return jpype.java.sql.DriverManager.getConnection(url, *dargs)
# Copyright (C) 2013 MH-EnterpriseConsulting, All rights reserved.
#
#
#
###########################################################################
# Stop all managed servers of a domain
###########################################################################
# ................ import ........................
import sys
from java.util import Properties
from java.io import FileInputStream
from java.io import File
domainProps = Properties()
userConfigFile = ''
userKeyFile = ''
###################################################################
# Load properties
###################################################################
def intialize():
global domainProps
global userConfigFile
global userKeyFile
# test arguments
if len(sys.argv) != 3:
print 'Usage: stopDomain.sh <default.properties_file> <property_file>'
def DiscoveryMain(Framework):
properties = Properties()
properties.setProperty('timeoutDiscover',
Framework.getParameter('timeoutDiscover'))
properties.setProperty('retryDiscover',
Framework.getParameter('retryDiscover'))
properties.setProperty('pingProtocol',
Framework.getParameter('pingProtocol'))
properties.setProperty('threadPoolSize',
Framework.getParameter('threadPoolSize'))
excludePatterns = icmp_utils.preparePatterns(
Framework.getParameter('excludePatternsList'))
virtualMode = Framework.getParameter(
'virtualModeDiscover').lower() == "true"
rangeString = Framework.getParameter('range') or 'NA'
probeName = Framework.getDestinationAttribute('probeName')
ignoreClientType = getGlobalSetting().getPropertyStringValue(
'pingClientTypeIp', "False").lower() == "false"
maxAllowedIPv6CountPerRange = long(
getGlobalSetting().getPropertyStringValue(
'maxPingIPv6CountPerRange',
str(DEFAULT_MAX_PING_IPV6_COUNT_PER_RANGE)))
logger.debug("Max allowed IPv6 range size:", maxAllowedIPv6CountPerRange)
isPingIPv4, isPingIPv6 = getIPSupport(Framework)
try:
client = Framework.createClient(ClientsConsts.ICMP_PROTOCOL_NAME,
properties)
try:
totalReportedIps = 0
selectedRangeList = _convertToRanges(rangeString)
probeRanges = icmp_utils.getProbeRanges(selectedRangeList,
probeName, Framework,
isPingIPv4, isPingIPv6)
logger.info('Start working on total probe ranges: ',
len(probeRanges))
logger.info('ignoreClientType = ', ignoreClientType)
#probeRanges = getDataCenterIPRanges(probeRanges)
for probeRange in probeRanges:
rangeSize = long(probeRange.getRangeSize())
if rangeSize > maxAllowedIPv6CountPerRange:
logger.reportWarning(
"The size of IPv6 range (%s) is %d, exceeds the max range size %d, will skip it."
% (probeRange.toRangeString(), rangeSize,
maxAllowedIPv6CountPerRange))
continue
totalReportedIps += icmp_utils.pingIPsInRange(
Framework,
client,
probeRange,
virtualMode,
excludePatterns=excludePatterns,
ignoreClientType=ignoreClientType)
Framework.saveState(probeRange.toRangeString())
logger.debug('Total reported IPs %s ' % totalReportedIps)
logger.info('Finished working on all Probes Ranges..')
Framework.clearState()
if not totalReportedIps:
logger.reportWarning("No live IPs found in probe ranges")
finally:
client.close()
except:
msg = logger.prepareJythonStackTrace('')
errormessages.resolveAndReport(msg, ClientsConsts.ICMP_PROTOCOL_NAME,
Framework)
return ObjectStateHolderVector()
package com.lsw.springcloud.config;
import java.util.Properties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import com.github.pagehelper.PageHelper;
@Configuration
public class PageHelperConfig {
/**
* 注入pagehelper配置
*
* @return
*/
@Bean
public PageHelper getPageHelper() {
PageHelper pageHelper = new PageHelper();
Properties properties = new Properties();
properties.setProperty("helperDialect", "mysql");
properties.setProperty("reasonable", "true");
properties.setProperty("supportMethodsArguments", "true");
properties.setProperty("params", "count=countSql");
pageHelper.setProperties(properties);
return pageHelper;
}
}
def createWmiClientForRegistry(self):
props = Properties()
props.setProperty(AgentConstants.PROP_WMI_NAMESPACE, 'root\\DEFAULT')
return self.framework.createClient(props)
from java.io import FileOutputStream
from java import io
from java.lang import Exception
from java.lang import Throwable
import os.path
import sys
envproperty = ""
if (len(sys.argv) > 1):
envproperty = sys.argv[1]
else:
print "Environment Property file not specified"
sys.exit(2)
propInputStream = FileInputStream(envproperty)
configProps = Properties()
configProps.load(propInputStream)
##########################################
# Create JMS Moudle will take the
# arguments as name, subdeployment name
# target can be on admin or managed server or cluster
##########################################
def createJMSModule(jms_module_name, adm_name, subdeployment_name):
cd('/JMSServers')
jmssrvlist = ls(returnMap='true')
print jmssrvlist
cd('/')
module = create(jms_module_name, "JMSSystemResource")
#cluster = getMBean("Clusters/"+cluster_target_name)
def getCurrentProfileName(domainConfigurationDirectory):
from oracle.fabric.profiles.impl import ProfileConstants;
fileName = domainConfigurationDirectory + File.separatorChar + 'server-profile-mbean-config.xml'
profileProperties = Properties()
profileProperties.loadFromXML(FileInputStream(fileName))
return profileProperties.getProperty(ProfileConstants.CURRENT_SOA_PROFILE_PROPERTY_NAME)
import com.alibaba.druid.pool.DruidDataSource;
import com.alibaba.druid.pool.DruidDataSourceFactory;
import com.alibaba.druid.support.http.StatViewServlet;
import com.alibaba.druid.support.http.WebStatFilter;
@Configuration
public class DruidConfig {
private static final String DB_PREFIX = "spring.datasource.";
@Autowired
private Environment environment;
@Bean
@ConfigurationProperties(prefix = DB_PREFIX)
public DataSource druidDataSource() {
Properties dbProperties = new Properties();
Map<String, Object> map = new HashMap<>();
for (PropertySource<?> propertySource : ((AbstractEnvironment) environment).getPropertySources()) {
getPropertiesFromSource(propertySource, map);
}
dbProperties.putAll(map);
DruidDataSource dds;
try {
dds = (DruidDataSource) DruidDataSourceFactory.createDataSource(dbProperties);
dds.init();
} catch (Exception e) {
throw new RuntimeException("load datasource error, dbProperties is :" + dbProperties, e);
}
return dds;
}
from java.lang import String
from java.util import Properties,Random
from javax.naming import Context,InitialContext
from net.grinder.script.Grinder import grinder
from net.grinder.script import Test
from weblogic.jndi import WLInitialContextFactory
tests = {
"home" : Test(1, "TraderHome"),
"trade" : Test(2, "Trader buy/sell"),
"query" : Test(3, "Trader getBalance"),
}
# Initial context lookup for EJB home.
p = Properties()
p[Context.INITIAL_CONTEXT_FACTORY] = WLInitialContextFactory.name
home = InitialContext(p).lookup("ejb20-statefulSession-TraderHome")
homeTest = tests["home"].wrap(home)
random = Random()
class TestRunner:
def __call__(self):
log = grinder.logger.output
trader = homeTest.create()
tradeTest = tests["trade"].wrap(trader)
def DiscoveryMain(Framework):
OSHVResult = ObjectStateHolderVector()
ip_address = Framework.getTriggerCIDataAsList('ip_address')
ip_domain = Framework.getTriggerCIDataAsList('ip_domain')
credentialIds = Framework.getAvailableProtocols(
ip_address[0], ProtocolManager.UDDI_REGISTRY)
logger.debug('Len of credentials: %s' % str(len(credentialIds)))
logger.debug('Start on Address:', ip_address[0], ', Domain:',
ip_domain[0])
if credentialIds.__len__() == 0:
msg = errormessages.makeErrorMessage(
ProtocolManager.UDDI_REGISTRY,
pattern=errormessages.ERROR_NO_CREDENTIALS)
errobj = errorobject.createError(
errorcodes.NO_CREDENTIALS_FOR_TRIGGERED_IP,
[ProtocolManager.UDDI_REGISTRY], msg)
logger.reportErrorObject(errobj)
ip_domain = Framework.getDestinationAttribute("ip_domain")
for credentialId in credentialIds:
url = Framework.getProtocolProperty(
credentialId, CollectorsConstants.UDDI_PROTOCOL_ATTRIBUTE_URL, '')
if url == '':
Framework.reportError(
'URL attribute is not specified in the UDDI protocol')
continue
# Check the URL
try:
logger.debug("Checking availability of %s" % url)
netutils.doHttpGet(url, 20000, 'header').strip()
except:
Framework.reportWarning(
'Failed to connect to UDDI Registry using URL: ' + url)
logger.debugException("Cannot connect to UDDI server")
else:
properties = Properties()
properties.setProperty(
CollectorsConstants.UDDI_PROTOCOL_ATTRIBUTE_URL, url)
properties.setProperty("ip_domain", ip_domain)
connected = False
version = 0
for uddiVersion in (3, 2):
if connected:
break
try:
logger.debug('Using version UDDIv%d' % uddiVersion)
properties.setProperty('uddi_version', str(uddiVersion))
logger.debug('Try to connect to UDDI Registry using url: ',
url)
Framework.getAgent(AgentConstants.UDDI_AGENT, '',
credentialId, properties)
logger.debug('Connected to UDDI Registry url: ', url)
connected = True
version = uddiVersion
except MissingSdkJarException, ex:
Framework.reportError(
'UDDI SDK jars are missed. Refer documentation for details'
)
logger.debugException(ex.getMessage())
break
except JException, java_exc:
Framework.reportWarning("Cannot connect to UDDI server")
logger.debugException(
'Failed to connect to UDDI Registry: ' +
java_exc.getMessage())
except:
from net.grinder.statistics import ExpressionView, StatisticsIndexMap, StatisticsView
from jarray import array
import sys
from java.lang import *
from java.util import Properties
from org.omg.CORBA import *
from org.omg.CosNotifyChannelAdmin import *
from org.omg.CosNotifyComm import *
from org.omg.CosNotification import *
from thread import *
log = grinder.logger.output
# setup ORB
args = array([], String)
props = Properties()
props.put("org.omg.CORBA.ORBClass", "org.jacorb.orb.ORB")
props.put("org.omg.CORBA.ORBSingletonClass", "org.jacorb.orb.ORBSingleton")
orb = ORB.init(args, props)
poa = orb.resolve_initial_references("RootPOA")
poa.the_POAManager().activate()
# start ORB Thread
def run_orb():
orb.run()
start_new_thread(run_orb, ())
# resolve NotificationService
propName = join(myDir, "java", "seth.property.xml")
sys.path.append(jarName)
# import the java classes
from java.util import Properties
from java.io import FileInputStream, File
from de.hu.berlin.wbi.objects import DatabaseConnection, dbSNP, UniprotFeature
from seth import SETH
print("init'ing seth")
# get some mutations from text
seth = SETH(mutName)
# setup db connection
print("setup mysql connection")
property = Properties()
property.loadFromXML(FileInputStream(File(propName)))
db = DatabaseConnection(property)
db.connect()
dbSNP.init(db, "PSM", "hgvs")
UniprotFeature.init(db, "uniprot")
def test():
gene = 1312
potentialSNPs = dbSNP.getSNP(gene)
features = UniprotFeature.getFeatures(gene)
mutations = seth.findMutations("p.A123T and Val158Met")
for mut in mutations:
start, end = mut.getStart(), mut.getEnd()
wtRes, mutRes = mut.getWtResidue(), mut.getMutResidue()
def DiscoveryMain(Framework):
properties = Properties()
SITE_ID = Framework.getDestinationAttribute('SITE_ID')
instance_number = Framework.getDestinationAttribute('instance_number')
connection_client = Framework.getDestinationAttribute('connection_client')
logger.debug('Connecting to a SAP instance number:', str(instance_number))
properties.setProperty(Protocol.SAP_PROTOCOL_ATTRIBUTE_SYSNUMBER,
instance_number)
if (connection_client is not None) and (connection_client != 'NA'):
logger.debug('Connecting to a SAP system with client:',
str(connection_client))
properties.setProperty(Protocol.SAP_PROTOCOL_ATTRIBUTE_CLIENT,
connection_client)
discoverScenarioProcesses = Boolean.parseBoolean(
Framework.getParameter('discoverScenarioProcesses'))
GET_PROCESS_STEPS = Boolean.parseBoolean(
Framework.getParameter('getProcessSteps'))
reportCIsInChunks = Boolean.parseBoolean(
Framework.getParameter('reportCIsInChunks'))
errormsg = ''
client = None
try:
try:
client = Framework.createClient(properties)
solman = saputils.SapSolman(client)
except (NoClassDefFoundError, MissingJarsException,
ExceptionInInitializerError):
errormsg = 'SAP drivers are missing'
logger.debugException(errormsg)
except:
errormsg = 'Connection failed'
logger.debugException(errormsg)
else:
try:
sVector, sysToOshPairs = discoverSystems(solman)
Framework.sendObjects(sVector)
sVector.clear()
sysNames = [system.getName() for system, _ in sysToOshPairs]
sys_name_to_system = {}
for system, _ in sysToOshPairs:
sys_name_to_system[system.getName()] = system
component2system = logicalComponents(sysNames, solman)
businessProcesses(Framework, solman, component2system,
sys_name_to_system, SITE_ID,
GET_PROCESS_STEPS, reportCIsInChunks,
discoverScenarioProcesses)
except:
strmsg = str(sys.exc_info()[1])
if (strmsg.upper().find('TABLE_NOT_AVAILABLE') > -1):
errmsg = 'No solution manager found'
logger.debugException(errmsg)
Framework.reportError(errmsg)
else:
# unknown exception caught
raise
except JException, ex:
ex_info = ex.getMessage()
errormessages.resolveAndReport(ex_info, 'SAP JCO', Framework)
from org.apache.log4j import Logger
from java.io import FileInputStream
from java.io import FileOutputStream
from java.util import Properties
from java.lang import String
import sys
import re
import os
import redback
execfile('core/commands/password_encrypter.py')
log = Logger.getLogger('config_loader')
data_linage = Properties()
def addPropertiesFromFile(props, filename, site_home):
addProps = Properties()
input = FileInputStream(filename)
addProps.load(input)
input.close()
baseFileList = addProps.getProperty('base')
if not baseFileList is None:
baseFiles = baseFileList.split(",")
for baseFile in baseFiles:
baseFileResolved=getBaseFile(baseFile, site_home)
if baseFileResolved=='':
log.error('Configuration inherits from properties file that does not exist: ' + baseFile)
from com.ziclix.python.sql import zxJDBC
import os
import sys
from java.util import Properties
#add jar to your class path, then import it
sys.path.append('/root/Desktop/mysql-connector-java-5.1.42.jar')
import com.mysql.jdbc.Driver as Driver
props = Properties()
props.put('user','root')
props.put('password','redhat')
mysqlConn = zxJDBC.connect(Driver().connect('jdbc:mysql://localhost/construction',props))
cursor = mysqlConn.cursor()
def dispContractorDetails():
ls = []
#function to display all items
sql = "SELECT * FROM contractor"
try:
cursor.execute(sql)
results = cursor.fetchall()
i = 1
for row in results:
ls.append(str(i)+".")
ls.append("name")
ls.append(str(row[0]))
ls.append("house type")
ls.append(str(row[1]))
ls.append("locality")
ls.append(str(row[2]))
from org.apache.log4j import Logger, PropertyConfigurator
from org.apache.tools.ant import Project, ProjectHelper, DefaultLogger, BuildException
from org.apache.tools.ant.listener import Log4jListener
sys.path.append('core/engine/')
sys.path.append('core/validators/')
sys.path.append('core/commands')
sys.path.append('core/commands/lib')
sys.path.append('custom/commands')
import redback
import config_loader
True = 1
False = 0
data_linage = Properties()
def run_validators():
config_valid = True
validation_logger = Logger.getLogger('validation')
for val_dir in [
'core/validators',
os.path.join(site_home, 'custom/validators')
]:
sys.path.append(val_dir)
for val_name in os.listdir(val_dir):
val_file = os.path.join(val_dir, val_name)
if os.path.isfile(val_file) and is_jython(val_file):
main_logger.debug('Running validator: ' + val_file)
result = call_extension(val_file, validation_logger)
warcsDir = Path(collectionDir, 'warcs')
noindexFile = Path(collectionDir, 'noindex')
parsedDir = Path(collectionDir, 'parsed')
indexesDir = Path(collectionDir, 'indexes')
dedup = Path(collectionDir, 'dedup')
# More paths, for the graph-related info
parsedCaptures = Path(collectionDir, 'graph/parsed-captures')
linkGraph = Path(collectionDir, 'graph/link-graph')
inlinkCountsPage = Path(collectionDir, 'graph/inlink-counts-page')
inlinkCountsDomain = Path(collectionDir, 'graph/inlink-counts-domain')
parsedCaptureInlinkCounts = Path(collectionDir,
'graph/parsed-capture-inlink-counts')
parsedCaptureBoost = Path(collectionDir, 'graph/parsed-capture-boost')
# Properties used for all jobs
props = Properties()
props.put('mapred.fairscheduler.pool', 'default')
# **************************************************
# Move any (w)arc files from 'incoming/' to 'arcs/'.
incoming = fs.globStatus(Path(incomingDir, '*.gz'))
print 'LOG: Incoming %d' % len(incoming)
if len(incoming) > 0:
if not fs.exists(warcsDir):
# print 'DEBUG: Bootstrap warcs dir: %s' % warcsDir
fs.mkdirs(warcsDir)
for newfile in incoming:
# It's not unusual for (w)arc files to be put into the incoming directory
# a second, third, etc. time. The "ingest pipeline" can have various troubles
# Jython version check
minVersion = (2, 5)
thisVersion = (sys.version_info)
if thisVersion < minVersion:
print "ERROR: must be run with at least Jython %(minVersion)s, you are running %(thisVersion)s" % locals(
)
sys.exit(1)
def onMessage(m):
print "received %(m)r" % locals()
time.sleep(2)
properties = Properties()
properties.put("java.naming.factory.initial",
"org.jnp.interfaces.NamingContextFactory")
properties.put("java.naming.provider.url", "jnp://localhost:1099")
properties.put("java.naming.factory.url.pkgs",
"org.jboss.naming:org.jnp.interfaces")
context = InitialContext(properties)
factory = context.lookup("ConnectionFactory")
destination = context.lookup("queue/DemoJMSProviderGW_reply")
connection = factory.createConnection()
session = connection.createSession(False, Session.AUTO_ACKNOWLEDGE)
consumer = session.createConsumer(destination)
consumer.setMessageListener(onMessage)
# There could be a better way of adding the jars; adding to the
# CLASSPATH environment variable could be one alternative.
# Import the Classes needed for Sentiment Analysis
from edu.stanford.nlp.ling import CoreAnnotations
from edu.stanford.nlp.neural.rnn import RNNCoreAnnotations
from edu.stanford.nlp.pipeline import Annotation
from edu.stanford.nlp.pipeline import StanfordCoreNLP
from edu.stanford.nlp.sentiment import SentimentCoreAnnotations
# Import the Properties class from java to set the Properties
# required to create the CoreNLP pipeline
from java.util import Properties
props = Properties()
props.setProperty('annotators', 'tokenize,ssplit,pos,parse,sentiment')
pipeline = StanfordCoreNLP(a)
# Obtain sentiment of argument string passed to Annotation. Please see
# the Java tutorial file packaged with the source code and the links
# mentioned above for specifics and explanations.
annotation = Annotation(
'java is good, python is also good, but Jython is absolutely amazing.')
pipeline.annotate(annotation)
sentences = annotation.get(CoreAnnotations.SentencesAnnotation)
# ps if running interactively in a Jython Session,
# try type(sentences). ;)
def execSshRemoteUsrPwd(hostname,
username,
password,
commandsSemiColonSeperated,
sessionTimeoutSecs=0,
waitForOutput=True):
_hostname = hostname
_username = username
_password = password
_command = commandsSemiColonSeperated
jsch = JSch()
session = jsch.getSession(_username, _hostname, 22)
session.setPassword(_password)
config = Properties()
config.put("StrictHostKeyChecking", "no")
config.put("GSSAPIAuthentication", "no")
config.put("UnknownHostVerification", "no")
#config.put("PreferredAuthentications", "publickey");
session.setConfig(config)
if (sessionTimeoutSecs > 0): session.setTimeout(sessionTimeoutSecs * 10)
print 'Logging into Remote SSH Shell u/p Auth...'
try:
if (sessionTimeoutSecs > 0):
session.connect(sessionTimeoutSecs * 1000)
else:
session.connect()
except:
return 'None'
channel = session.openChannel("exec")
channel.setCommand('source ~/.bash_profile 2>/dev/null; ' + _command)
outputBuffer = StringBuilder()
stdin = channel.getInputStream()
stdinExt = channel.getExtInputStream()
channel.connect(sessionTimeoutSecs * 1000)
if (waitForOutput == True):
while (1):
n = stdin.read()
if n == -1:
break
if (chr(n) == '\n'):
outputBuffer.append('|')
elif (chr(n) == '\r'):
outputBuffer.append('|')
else:
outputBuffer.append(chr(n))
while (1):
n = stdinExt.read()
if n == -1:
break
if (chr(n) == '\n'):
outputBuffer.append('|')
elif (chr(n) == '\r'):
outputBuffer.append('|')
else:
outputBuffer.append(chr(n))
else:
sleep(sessionTimeoutSecs)
print "Command on: " + hostname + " : " + _command
print "\toutput: " + outputBuffer.toString()
channel.disconnect()
session.disconnect()
del channel
del session
return outputBuffer.toString()
def mkprops():
props = Properties()
props.setProperty("java.ext.dirs", String(JARDIR))
props.setProperty("python.security.respectJavaAccessibility",
String("true"))
return props
def loadProperties(cfg_file, extra_props):
props = Properties()
iterated_props = Properties()
inherited_props = Properties()
data_linage = Properties()
redback_reg = redback.load_redback_registry()
site_home = redback_reg.getProperty('site.home')
global_properties = redback_reg.getProperty('global.properties')
if global_properties:
if site_home:
baseFile=getBaseFile(global_properties, site_home)
if baseFile=='':
log.error('Global properties file does not exist: ' + global_properties)
log.info('Suggested Fix: Update the global.properties property within redback.properties to the correct file path')
sys.exit()
global_properties=baseFile
log.info('Loading global configuration from file: ' + global_properties)
addPropertiesFromFile(props, global_properties, site_home)
if cfg_file:
addPropertiesFromFile(props, cfg_file, site_home)
if extra_props:
props.putAll(extra_props)
# resolve property level inheritance and iterations
log.debug('Attempting to resolve property level inheritance')
enum = props.keys()
while enum.hasMoreElements():
key = enum.nextElement()
value = props.getProperty(key)
# Check for property level inheritance
if re.search(r"\.base$", key) is not None:
prop_to_inherit = value
prop_to_extend = key[:-5]
log.debug('Inheriting the properties from the ' + prop_to_inherit + ' section in to the section ' + prop_to_extend)
# Iterate the properties again looking for a match on properties to inherit
enum_inner = props.keys()
while enum_inner.hasMoreElements():
key_inner = enum_inner.nextElement()
value_inner = props.getProperty(key_inner)
log.debug('Checking key_inner [' + key_inner + '] matches ' + prop_to_inherit)
if String.startsWith(String(key_inner),String(prop_to_inherit)):
new_property = prop_to_extend + String.substring(key_inner, len(prop_to_inherit))
# Don't override the property if it has already been defined earlier
if props.getProperty(new_property) is None:
log.debug('Setting inherited property ' + new_property + ' to value ' + value_inner)
inherited_props.setProperty(new_property, value_inner)
addDataLinage(key,cfg_file,value_inner)
# Remove the key that defines the base, just keeps us consistant with the template behaviours
log.debug("About to remove key " + key)
props.remove(key)
props.putAll(inherited_props)
log.debug('Attempting to resolve iterations')
enum = props.keys()
while enum.hasMoreElements():
key = enum.nextElement()
value = props.getProperty(key)
# Check for property set iterations
if re.search(r"\.iterate$", key) is not None:
iteration_key = key
iteration_set = eval(value)
prop_to_iterate = key[:-9]
log.debug('Iterating the properties from the ' + prop_to_iterate + ' section')
# Iterate the properties again looking for a match on properties to iterate
enum_inner = props.keys()
while enum_inner.hasMoreElements():
key_inner = enum_inner.nextElement()
value_inner = props.getProperty(key_inner)
# if the string is part of the set but not the actual iterator then we will check it
if String.startsWith(String(key_inner),String(prop_to_iterate)) and key_inner != iteration_key:
log.debug('Checking if the key [' + key_inner + '] or value = ' + value_inner + ' contains an iterator ')
contains_iterator = 0
iterated_key = String(key_inner)
iterated_value = String(value_inner)
if String.indexOf(String(key_inner),"%") > -1:
log.debug(key_inner + ' contains an iterator, replacing it')
contains_iterator = 1
if String.indexOf(String(value_inner),"%") > -1:
log.debug(value_inner + ' contains an iterator, replacing it')
contains_iterator = 1
for i in iteration_set:
iterated_key = String.replaceAll(String(key_inner),"\%",str(i))
iterated_value = String.replaceAll(String(value_inner),"\%",str(i))
# Don't override the property if it has already been defined earlier
if props.getProperty(iterated_key) is None:
log.debug('Setting iterated property ' + iterated_key + ' to value ' + iterated_value)
iterated_props.setProperty(iterated_key, iterated_value)
addDataLinage(key,cfg_file,iterated_value)
# Remove the key that gets iterated, just keeps us consistant with the template behaviours
if (contains_iterator == 1):
log.debug('About to remove ' + key_inner)
props.remove(key_inner)
# Remove the key that defines the iterations, just keeps us consistant with the template behaviours
props.remove(iteration_key)
# Add the iterated properties back in to the main set
props.putAll(iterated_props)
# resolve embedded references
enum = props.keys()
while enum.hasMoreElements():
key = enum.nextElement()
value = props.getProperty(key)
if not value is None and len(value) > 0:
if re.search(r"\$\{(.*?)\}", value) is not None:
resolvedValue = resolveProperty(key, props)
if resolvedValue is None:
raise Exception('unable to resolve property: ' + key + '=' + value)
else:
props.setProperty(key, resolvedValue)
addDataLinage(key,cfg_file,resolvedValue)
# remove any properties that start with base (assumed that these are baseline properties that have been inherited
enum = props.keys()
while enum.hasMoreElements():
key = enum.nextElement()
if String.startsWith(String(key),'base.'):
props.remove(key)
decryptAllPasswords(cfg_file,props)
return props
