def __call__(self, environ, start_response):
reqparams = {}
for key, value in parse_qs(environ['QUERY_STRING'], True).items():
reqparams[key.lower()] = value[0]
onlineresource = 'http://%s:%s%s?' % (environ['SERVER_NAME'],
environ['SERVER_PORT'],
environ['PATH_INFO'])
try:
if not reqparams.has_key('request'):
raise OGCException('Missing request parameter.')
request = reqparams['request']
del reqparams['request']
if request == 'GetCapabilities' and not reqparams.has_key(
'service'):
raise OGCException('Missing service parameter.')
if request in ['GetMap', 'GetFeatureInfo']:
service = 'WMS'
else:
service = reqparams['service']
if reqparams.has_key('service'):
del reqparams['service']
try:
mapnikmodule = __import__('mapnik2.ogcserver.' + service)
except:
raise OGCException('Unsupported service "%s".' % service)
ServiceHandlerFactory = getattr(mapnikmodule.ogcserver,
service).ServiceHandlerFactory
servicehandler = ServiceHandlerFactory(
self.conf, self.mapfactory, onlineresource,
reqparams.get('version', None))
if reqparams.has_key('version'):
del reqparams['version']
if request not in servicehandler.SERVICE_PARAMS.keys():
raise OGCException('Operation "%s" not supported.' % request,
'OperationNotSupported')
ogcparams = servicehandler.processParameters(request, reqparams)
try:
requesthandler = getattr(servicehandler, request)
except:
raise OGCException('Operation "%s" not supported.' % request,
'OperationNotSupported')
response = requesthandler(ogcparams)
except:
version = reqparams.get('version', None)
if not version:
version = Version('1.3.0')
else:
version = Version(version)
if version >= '1.3.0':
eh = ExceptionHandler130(self.debug)
else:
eh = ExceptionHandler111(self.debug)
response = eh.getresponse(reqparams)
response_headers = [('Content-Type', response.content_type),
('Content-Length', str(len(response.content)))]
if self.max_age:
response_headers.append(('Cache-Control', max_age))
start_response('200 OK', response_headers)
yield response.content
def ServiceHandlerFactory(conf, mapfactory, onlineresource, version):
if not version:
version = Version('1.3.0')
else:
version = Version(version)
if version >= '1.3.0':
return ServiceHandler130(conf, mapfactory, onlineresource)
else:
return ServiceHandler111(conf, mapfactory, onlineresource)
def traceback(self, req):
reqparams = lowerparams(req.params)
version = reqparams.get('version', None)
if not version:
version = Version('1.3.0')
else:
version = Version(version)
if version >= '1.3.0':
eh = ExceptionHandler130(self.debug)
else:
eh = ExceptionHandler111(self.debug)
response = eh.getresponse(reqparams)
req.set_header('Content-Type', response.content_type)
req.write(response.content)
def get_packal_workflows():
"""Return list of workflows available on Packal.org"""
workflows = []
r = web.get(MANIFEST_URL)
r.raise_for_status()
manifest = ET.fromstring(r.content)
# these elements contain multiple, |||-delimited items
list_elements = ('categories', 'tags', 'osx')
for workflow in manifest:
d = {}
for elem in workflow:
if elem.tag in list_elements:
if not elem.text:
d[elem.tag] = []
else:
d[elem.tag] = [s.strip() for s in elem.text.split('|||')]
# text elements
elif elem.text:
d[elem.tag] = elem.text
else:
d[elem.tag] = ''
# convert timestamp to datetime
d['updated'] = datetime.fromtimestamp(float(d['updated']))
d['version'] = Version(d['version'])
workflows.append(d)
log.debug('{} workflows available on Packal.org'.format(len(workflows)))
return workflows
class Check(object):
""" check system envs """
def __init__(self, dbcfgs_json):
self.dbcfgs = json.loads(dbcfgs_json)
self.version = Version()
def check_sudo(self):
""" check sudo access """
run_cmd('sudo -n echo -n "check sudo access" > /dev/null 2>&1')
def check_hbase_xml(self):
""" check if hbase-site.xml file exists """
hbase_xml_file = self.dbcfgs['hbase_xml_file']
if not os.path.exists(hbase_xml_file):
err('HBase xml file is not found')
def check_java(self):
""" check JDK version """
jdk_path = self.dbcfgs['java_home']
jdk_ver = cmd_output('%s/bin/javac -version' % jdk_path)
try:
jdk_ver, sub_ver = re.search(r'javac (\d\.\d).\d_(\d+)', jdk_ver).groups()
except AttributeError:
err('No JDK found')
if self.dbcfgs['req_java8'] == 'Y': # only allow JDK1.8
support_java = '1.8'
else:
support_java = self.version.get_version('java')
if jdk_ver == '1.7' and int(sub_ver) < 65:
err('Unsupported JDK1.7 version, sub version should be higher than 65')
if jdk_ver not in support_java:
err('Unsupported JDK version %s, supported version: %s' % (jdk_ver, support_java))
def versions_from_targets(phab, targets):
builds = get_builds_from_targets(phab, targets)
buildables = get_buildables_from_builds(phab, builds)
diffs = get_diffs_from_buildables(phab, buildables)
return [
Version(target.id, target.phid, diff.id, diff.branch, diff.base)
for target, diff in zip(targets, diffs)
]
def traceback(self, apacheReq, E):
reqparams = lowerparams(util.FieldStorage(apacheReq))
version = reqparams.get('version', None)
if not version:
version = Version('1.3.0')
else:
version = Version(version)
if version >= '1.3.0':
eh = ExceptionHandler130(self.debug)
else:
eh = ExceptionHandler111(self.debug)
response = eh.getresponse(reqparams)
apacheReq.content_type = response.content_type
apacheReq.headers_out.add('Content-Length', str(len(response.content)))
apacheReq.send_http_header()
apacheReq.write(response.content)
return apache.OK
def packal_metadata(xmlpath):
"""Return ``dict`` of metadata in ``package.xml`` file created by Packal"""
tree = ET.parse(xmlpath)
root = tree.getroot()
data = {}
for elem in root:
data[elem.tag] = elem.text
data['version'] = Version(data['version'])
return data
def get_version_from_git_config(git_config):
config = configparser.ConfigParser()
config.read(git_config)
base = config.get('phabricator', option='base')
branch = config.get('phabricator', option='branch')
diff = config.get('phabricator', option='diff')
target = config.get('phabricator', option='target')
targetPHID = config.get('phabricator', option='targetphid')
return Version(target, targetPHID, diff, branch, base)
class Check(object):
""" check system envs """
def __init__(self, dbcfgs_json):
self.dbcfgs = json.loads(dbcfgs_json)
self.version = Version()
def check_sudo(self):
""" check sudo access """
run_cmd('%s echo -n "check sudo access" > /dev/null 2>&1' % get_sudo_prefix())
def check_ssh_pam(self):
""" check if UsePAM is set to yes in sshd_config """
if not cmd_output('grep "^UsePAM yes" %s' % SSH_CONFIG_FILE):
err('\'UsePAM\' should be set to \'yes\' in %s' % SSH_CONFIG_FILE)
def check_ssh_pam(self):
""" check if UsePAM is set to yes in sshd_config """
if not cmd_output('grep "^UsePAM yes" %s' % SSH_CONFIG_FILE):
err('\'UsePAM\' should be set to \'yes\' in %s' % SSH_CONFIG_FILE)
def check_hbase_xml(self):
""" check if hbase-site.xml file exists """
hbase_xml_file = self.dbcfgs['hbase_xml_file']
if not os.path.exists(hbase_xml_file):
err('HBase xml file is not found')
def check_java(self):
""" check JDK version """
jdk_path = self.dbcfgs['java_home']
jdk_ver = cmd_output('%s/bin/javac -version' % jdk_path)
try:
jdk_ver, sub_ver = re.search(r'javac (\d\.\d).\d_(\d+)', jdk_ver).groups()
except AttributeError:
err('No JDK found')
if self.dbcfgs['req_java8'] == 'Y': # only allow JDK1.8
support_java = '1.8'
else:
support_java = self.version.get_version('java')
if jdk_ver == '1.7' and int(sub_ver) < 65:
err('Unsupported JDK1.7 version, sub version should be higher than 65')
if jdk_ver not in support_java:
err('Unsupported JDK version %s, supported version: %s' % (jdk_ver, support_java))
def __init__(self, dbcfgs_json):
self.dbcfgs = json.loads(dbcfgs_json)
self.version = Version()
def __init__(self, dbcfgs):
self.CPUINFO = cmd_output('cat /proc/cpuinfo')
self.MEMINFO = cmd_output('cat /proc/meminfo')
self.SYSCTLINFO = cmd_output('sysctl -a')
self.version = Version()
self.dbcfgs = dbcfgs
class Discover(object):
""" discover functions, to add a new discover function,
simply add a new def with name get_xx and decorated
by 'deco', then return result in string format:
@deco
def get_xx(self):
# do something
return result
"""
def __init__(self, dbcfgs):
self.CPUINFO = cmd_output('cat /proc/cpuinfo')
self.MEMINFO = cmd_output('cat /proc/meminfo')
self.SYSCTLINFO = cmd_output('sysctl -a')
self.version = Version()
self.dbcfgs = dbcfgs
def _parse_string(self, info, string):
try:
info = info.split('\n')
string_line = [line for line in info if string in line][0]
except IndexError:
err('Cannot get %s info' % string)
return string_line
def _get_cpu_info(self, string):
return self._parse_string(self.CPUINFO, string).split(':')[1].strip()
def _get_mem_info(self, string):
return self._parse_string(self.MEMINFO, string).split(':')[1].split()[0]
def _get_sysctl_info(self, string):
return self._parse_string(self.SYSCTLINFO, string).split('=')[1].strip()
@deco
def get_linux(self):
""" get linux version """
os_dist, os_ver = platform.dist()[:2]
if os_dist not in self.version.get_version('linux'):
return NA
else:
if not os_ver.split('.')[0] in self.version.get_version(os_dist):
return NA
return '%s-%s' % (os_dist, os_ver)
@deco
def get_firewall_status(self):
""" get firewall running status """
iptables_stat = cmd_output('iptables -nL|grep -vE "(Chain|target)"').strip()
if iptables_stat:
return 'Running'
else:
return 'Stopped'
@deco
def get_pidmax(self):
""" get kernel pid max setting """
return self._get_sysctl_info('kernel.pid_max')
@deco
def get_default_java(self):
""" get default java version """
jdk_path = glob('/usr/java/*') + \
glob('/usr/jdk64/*') + \
glob('/usr/lib/jvm/java-*-openjdk.x86_64')
jdk_list = {} # {jdk_version: jdk_path}
for path in jdk_path:
jdk_ver = cmd_output('%s/bin/javac -version' % path)
try:
main_ver, sub_ver = re.search(r'(\d\.\d\.\d)_(\d+)', jdk_ver).groups()
# don't support JDK version less than 1.7.0_65
if main_ver == '1.7.0' and int(sub_ver) < 65:
continue
jdk_list[main_ver] = path
except AttributeError:
continue
if not jdk_list:
return NA
else:
# use JDK1.8 first
if jdk_list.has_key('1.8.0'):
return jdk_list['1.8.0']
elif jdk_list.has_key('1.7.0'):
return jdk_list['1.7.0']
@deco
def get_hive(self):
""" get Hive status """
hive_stat = cmd_output('which hive')
if 'no hive' in hive_stat:
return NA
else:
return OK
@deco
def get_secure_hadoop(self):
if self.dbcfgs.has_key('hadoop_home'): # apache distro
CORE_SITE_XML = '%s/etc/hadoop/core-site.xml' % self.dbcfgs['hadoop_home']
else:
CORE_SITE_XML = '/etc/hadoop/conf/core-site.xml'
p = ParseXML(CORE_SITE_XML)
return p.get_property('hadoop.security.authentication')
@deco
def get_hbase(self):
""" get HBase version """
if self.dbcfgs.has_key('hbase_home'): # apache distro
hbase_home = self.dbcfgs['hbase_home']
hbase_ver = cmd_output('%s/bin/hbase version | head -n1' % hbase_home)
else:
hbase_ver = cmd_output('hbase version | head -n1')
support_hbase_ver = self.version.get_version('hbase')
try:
hbase_ver = re.search(r'HBase (\d\.\d)', hbase_ver).groups()[0]
except AttributeError:
return NA
if hbase_ver not in support_hbase_ver:
return NS
return hbase_ver
@deco
def get_cpu_model(self):
""" get CPU model """
return self._get_cpu_info('model name')
@deco
def get_cpu_cores(self):
""" get CPU cores """
return self.CPUINFO.count('processor')
@deco
def get_arch(self):
""" get CPU architecture """
arch = platform.processor()
if not arch:
arch = 'Unknown'
return arch
@deco
def get_mem_total(self):
""" get total memory size """
mem = self._get_mem_info('MemTotal')
memsize = mem.split()[0]
return "%0.1f GB" % round(float(memsize) / (1024 * 1024), 2)
@deco
def get_mem_free(self):
""" get current free memory size """
free = self._get_mem_info('MemFree')
buffers = self._get_mem_info('Buffers')
cached = self._get_mem_info('Cached')
memfree = float(free) + float(buffers) + float(cached)
return "%0.1f GB" % round(memfree / (1024 * 1024), 2)
@deco
def get_ext_interface(self):
""" get external network interface """
return cmd_output('netstat -rn | grep "^0.0.0.0" | awk \'{print $8}\'').strip()
@deco
def get_rootdisk_free(self):
""" get root disk space left """
space = cmd_output('df -h|grep "\/$" | awk \'{print $4}\'')
return space.strip()
@deco
def get_python_ver(self):
""" get python version """
return platform.python_version()
@deco
def get_traf_status(self):
""" get trafodion running status """
mon_process = cmd_output('ps -ef|grep -v grep|grep -c "monitor COLD"')
if int(mon_process) > 0:
return 'Running'
else:
return 'Stopped'
def __init__(self, dbcfgs_json):
self.dbcfgs = json.loads(dbcfgs_json)
self.version = Version()
def __init__(self, dbcfgs):
self.CPUINFO = cmd_output('cat /proc/cpuinfo')
self.MEMINFO = cmd_output('cat /proc/meminfo')
self.SYSCTLINFO = cmd_output('sysctl -a')
self.version = Version()
self.dbcfgs = dbcfgs
class Discover(object):
""" discover functions, to add a new discover function,
simply add a new def with name get_xx and decorated
by 'deco', then return result in string format:
@deco
def get_xx(self):
# do something
return result
"""
def __init__(self, dbcfgs):
self.CPUINFO = cmd_output('cat /proc/cpuinfo')
self.MEMINFO = cmd_output('cat /proc/meminfo')
self.SYSCTLINFO = cmd_output('sysctl -a')
self.version = Version()
self.dbcfgs = dbcfgs
def _parse_string(self, info, string):
try:
info = info.split('\n')
string_line = [line for line in info if string in line][0]
except IndexError:
err('Cannot get %s info' % string)
return string_line
def _get_cpu_info(self, string):
return self._parse_string(self.CPUINFO, string).split(':')[1].strip()
def _get_mem_info(self, string):
return self._parse_string(self.MEMINFO,
string).split(':')[1].split()[0]
def _get_sysctl_info(self, string):
return self._parse_string(self.SYSCTLINFO,
string).split('=')[1].strip()
@deco
def get_linux(self):
""" get linux version """
os_dist, os_ver = platform.dist()[:2]
if os_dist not in self.version.get_version('linux'):
return NA
else:
if not os_ver.split('.')[0] in self.version.get_version(os_dist):
return NA
return '%s-%s' % (os_dist, os_ver)
@deco
def get_firewall_status(self):
""" get firewall running status """
iptables_stat = cmd_output(
'iptables -nL|grep -vE "(Chain|target)"').strip()
if iptables_stat:
return 'Running'
else:
return 'Stopped'
@deco
def get_pidmax(self):
""" get kernel pid max setting """
return self._get_sysctl_info('kernel.pid_max')
@deco
def get_default_java(self):
""" get default java version """
jdk_path = glob('/usr/java/*') + \
glob('/usr/jdk64/*') + \
glob('/usr/lib/jvm/java-*-openjdk.x86_64')
jdk_list = {} # {jdk_version: jdk_path}
for path in jdk_path:
jdk_ver = cmd_output('%s/bin/javac -version' % path)
try:
main_ver, sub_ver = re.search(r'(\d\.\d\.\d)_(\d+)',
jdk_ver).groups()
# don't support JDK version less than 1.7.0_65
if main_ver == '1.7.0' and int(sub_ver) < 65:
continue
jdk_list[main_ver] = path
except AttributeError:
continue
if not jdk_list:
return NA
else:
# use JDK1.8 first
if jdk_list.has_key('1.8.0'):
return jdk_list['1.8.0']
elif jdk_list.has_key('1.7.0'):
return jdk_list['1.7.0']
@deco
def get_hive(self):
""" get Hive status """
hive_stat = cmd_output('which hive')
if 'no hive' in hive_stat:
return NA
else:
return OK
@deco
def get_secure_hadoop(self):
if self.dbcfgs.has_key('hadoop_home'): # apache distro
CORE_SITE_XML = '%s/etc/hadoop/core-site.xml' % self.dbcfgs[
'hadoop_home']
else:
CORE_SITE_XML = '/etc/hadoop/conf/core-site.xml'
p = ParseXML(CORE_SITE_XML)
return p.get_property('hadoop.security.authentication')
@deco
def get_hbase(self):
""" get HBase version """
if self.dbcfgs.has_key('hbase_home'): # apache distro
hbase_home = self.dbcfgs['hbase_home']
hbase_ver = cmd_output('%s/bin/hbase version | head -n1' %
hbase_home)
else:
hbase_ver = cmd_output('hbase version | head -n1')
support_hbase_ver = self.version.get_version('hbase')
try:
hbase_ver = re.search(r'HBase (\d\.\d)', hbase_ver).groups()[0]
except AttributeError:
return NA
if hbase_ver not in support_hbase_ver:
return NS
return hbase_ver
@deco
def get_cpu_model(self):
""" get CPU model """
return self._get_cpu_info('model name')
@deco
def get_cpu_cores(self):
""" get CPU cores """
return self.CPUINFO.count('processor')
@deco
def get_arch(self):
""" get CPU architecture """
arch = platform.processor()
if not arch:
arch = 'Unknown'
return arch
@deco
def get_mem_total(self):
""" get total memory size """
mem = self._get_mem_info('MemTotal')
memsize = mem.split()[0]
return "%0.1f GB" % round(float(memsize) / (1024 * 1024), 2)
@deco
def get_mem_free(self):
""" get current free memory size """
free = self._get_mem_info('MemFree')
buffers = self._get_mem_info('Buffers')
cached = self._get_mem_info('Cached')
memfree = float(free) + float(buffers) + float(cached)
return "%0.1f GB" % round(memfree / (1024 * 1024), 2)
@deco
def get_ext_interface(self):
""" get external network interface """
return cmd_output(
'netstat -rn | grep "^0.0.0.0" | awk \'{print $8}\'').strip()
@deco
def get_rootdisk_free(self):
""" get root disk space left """
space = cmd_output('df -h|grep "\/$" | awk \'{print $4}\'')
return space.strip()
@deco
def get_python_ver(self):
""" get python version """
return platform.python_version()
@deco
def get_traf_status(self):
""" get trafodion running status """
mon_process = cmd_output('ps -ef|grep -v grep|grep -c "monitor COLD"')
if int(mon_process) > 0:
return 'Running'
else:
return 'Stopped'