def create_pkcs12():
"""Creates a pkcs12 keystore"""
cmd = 'openssl pkcs12 -export -in {cp} -inkey {ckp} -out {p12_out} -passout pass:{p12_export} -name {alias}'.format(
cp=get_config_value(lvl, 'cert_path'),
ckp=get_config_value(lvl, 'cert_key_path'),
p12_out=get_config_value(lvl, 'p12_out_filename'),
p12_export=get_config_value(lvl, 'p12_export_pass'),
alias=get_config_value(lvl, 'alias'))
res = subprocess.call(cmd, shell='True')
def install():
version = get_config_value(lvl, 'java')
ans = ''
if silent_mode == 'silent':
ans = 'y'
else:
ans = get_input('Do you want to install/update ' + version +
' ? [y/n] ')
while ans != 'y' and ans != 'n':
ans = get_input('Do you want to install/update ' + version +
' ? [y/n] ')
if (ans == 'n'):
print('Skipping Java installation.')
print('Exiting...')
exit()
check_java = 'rpm -q java-1.8.0-openjdk-devel.x86_64 > /dev/null'
result = subprocess.call(check_java, shell='True')
if result != 0:
print('Attempting ' + version + ' installation...')
sleep(2)
result = subprocess.call('sudo yum -y install ' + version,
shell='True')
if result == 0:
print('Installation successful!')
else:
print('The required version of java is already installed.')
print('Exiting...')
def translate(paragraph, lang='en'):
key = util.get_config_value('googleTranslateApiKey')
r = requests.get('https://www.googleapis.com/language/translate/v2',
params={
'target': lang,
'key': key,
'q': paragraph,
'format': 'text'
})
#print r.status_code
return r
def import_to_keystore():
"""Imports the created pkcs12 keystore into Java's keystore container"""
cmd = 'keytool -importkeystore -deststorepass {dsp} -destkeypass {dkp} -destkeystore {dks} -srckeystore {sks} -srcstoretype pkcs12 -srcstorepass {ssp} -alias {alias} -noprompt'.format(
dsp=get_config_value(lvl, 'dest_keystore_pass'),
dkp=get_config_value(lvl, 'dest_keystore_pass'),
dks=get_config_value(lvl, 'keystore_path'),
sks=get_config_value(lvl, 'p12_out_filename'),
ssp=get_config_value(lvl, 'p12_export_pass'),
alias=get_config_value(lvl, 'alias'))
res = subprocess.call(cmd, shell='True')
def install(mode, c):
try:
war_file = get_config_value(lvl, 'war_file')
except:
war_file = default_war_file
try:
sso_file = get_config_value(lvl, 'ssoconfig_file')
except:
sso_file = default_ssoconfig_file
# make sure Tomcat is running. Abort if Tomcat is not running
print('Using IAM config file: ' + oam_config)
print('Using war file: ' + war_file)
print('Using ssoconfig: ' + sso_file)
if not is_service_running('tomcat'):
print('Existing: Tomcat is not running...')
exit()
else:
print('Checked: Tomcat service is running...')
# update config file
if (mode != 'silent'):
print(
'*** Please UPDATE all entries with the "UPDATE" (8) in the comment and save the changes ***'
)
print('... ')
sleep(4)
call(["vim", "+40", oam_config])
# deploy and configure
chome = grep('../config/tomcat/tomcat.service', 'CATALINA_HOME',
2).rstrip()
print('Tomcat home = ' + chome)
if (chome):
call('cp ' + war_file + ' ' + chome + '/webapps', shell=True)
else:
print('Exiting due to Tomcat Error: CATALINA_HOME is not defined')
exit()
# copy files - sleep until war is deployed
print('Deploying openam ....')
sleep(35)
call('cp ../config/oam/DataStore.xml ' + chome + '/webapps/' + base_name +
'/config/auth/default_en',
shell=True)
call('cp ../config/oam/index.html ' + chome + '/webapps/' + base_name +
'/XUI',
shell=True)
call('cp ../config/oam/ThemeConfiguration.js ' + chome + '/webapps/' +
base_name + '/XUI/config',
shell=True)
call('cp ../config/oam/translation.json ' + chome + '/webapps/' +
base_name + '/XUI/locales/en',
shell=True)
call('cp ../config/oam/FooterTemplate.html ' + chome + '/webapps/' +
base_name + '/XUI/templates/common',
shell=True)
call('cp ../images/login-logo.png ' + chome + '/webapps/' + base_name +
'/XUI/images',
shell=True)
call('cp ../images/logo-horizontal.png ' + chome + '/webapps/' +
base_name + '/XUI/images',
shell=True)
call('cp ../images/favicon.ico ' + chome + '/webapps/' + base_name +
'/XUI',
shell=True)
call('cp ../images/PrimaryProductName.png ' + chome + '/webapps/' +
base_name + '/console/images',
shell=True)
# configure
# get the keystore file location from server.xml
loc = grep('../config/tomcat/server.xml', 'keystoreFile', 1)
loc2 = grep('../config/tomcat/server.xml', 'keystorePass', 1)
# cmd = 'java -Djavax.net.ssl.trustStore=' + loc.rstrip() + ' -jar ./openam-configurator-tool-13.0.0.jar --file ../iam-configs/config.properties'
cmd = 'java -Djavax.net.ssl.trustStore=' + loc.rstrip(
) + ' -Djavax.net.ssl.trustStorePassword='******' -jar ' + sso_file + ' --file ' + oam_config
print('Configuring OpenAM: ' + cmd)
call(cmd, shell=True)
sleep(2)
def update_totalOzone_master_file(self, directory, master_file, date, mode,
heading): # noqa
"""Updates Total Ozone Master File"""
# Initialization
write_output = 1
current_time = (datetime.now()).strftime("%Y_%m_%d")
log_file = open('totalOzone_processing_log_%s' % current_time,
'wb') # noqa
data_file = None
global tmp_filename
tmp_filename = os.path.join(master_file, 'o3tot')
if mode == 'overwrite':
data_file = open(tmp_filename, 'wb+')
else:
data_file = open(tmp_filename, 'ab+')
if heading == 'on':
data_file.write(
'Platform_ID,Year,Month,Day,Start_Hour,Finish_Hour,Wavelength_Pair,Observation_Type,Total_Column_Ozone_Amount,Ozone_Std_Error,Instrument_Type,Instrument_Number\r\n'
) # noqa
# external ftp file
global output_file
output_file = 'Summaries/TotalOzone/Daily_Summary/o3tot.zip' # noqa
# extract zipfile
zip_flag = False
path = directory
if zipfile.is_zipfile(directory):
zip_flag = True
tmpdir = tempfile.mkdtemp()
z = zipfile.ZipFile(directory)
z.extractall(path=tmpdir)
path = tmpdir
# traverse the given directory
for dirname, dirnames, filenames in os.walk(path):
dirnames.sort()
filenames.sort()
for filename in filenames:
try:
# print filename
file_last_modified_date = time.strftime(
"%Y-%m-%d",
time.localtime(
os.path.getmtime(os.path.join(dirname,
filename)))) # noqa
# date comparison
if date is not None and file_last_modified_date <= date: # noqa
log_file.write(
'PROCESSED#%s last modified date: %s\r\n' %
((os.path.join(dirname, filename)),
file_last_modified_date)) # noqa
extCSV = WOUDCextCSVReader(
os.path.join(dirname, filename)) # noqa
if date is not None and file_last_modified_date > date:
continue
if date is None:
log_file.write(
'PROCESSED#%s last modified date: %s\r\n' %
((os.path.join(dirname, filename)),
file_last_modified_date)) # noqa
extCSV = WOUDCextCSVReader(
os.path.join(dirname, filename)) # noqa
# store data into variables
platform_id = ' '
if 'PLATFORM' in extCSV.sections:
p_id = extCSV.sections['PLATFORM']['ID']
if p_id is not None and len(p_id) != 0:
platform_id = p_id
if len(platform_id) == 1:
platform_id = '00%s' % platform_id
if len(platform_id) == 2:
platform_id = '0%s' % platform_id
else:
log_file.write(
'ERROR#E01:Could not find PLATFORM in input file: %s. Data is ignored\r\n'
% os.path.join(dirname, filename)) # noqa
inst_type_id = ' '
inst_number = ' 0'
if 'INSTRUMENT' in extCSV.sections:
inst_name = extCSV.sections['INSTRUMENT']['Name']
inst_model = extCSV.sections['INSTRUMENT']['Model']
if inst_name is not None and len(
inst_name) != 0 and inst_name: # noqa
try:
inst_type_id = util.get_config_value(
'Instrument Type ID', inst_name) # noqa
if inst_model == 'Japanese':
inst_type_id = util.get_config_value(
'Instrument Type ID',
inst_model + ' ' + inst_name) # noqa
if len(inst_type_id) == 1:
inst_type_id = ' %s' % inst_type_id
except Exception, err:
log_file.write(
'ERROR#E02:There is no instrumet type id for \'%s\' in file %s. Data is ignored\r\n'
%
(inst_name, os.path.join(
dirname, filename))) # noqa
write_output = 0
pass
i_num = extCSV.sections['INSTRUMENT']['Number']
if i_num is not None and len(i_num) != 0:
inst_number = i_num
inst_number = re.sub("^0{1,2}", "",
inst_number) # noqa
if len(inst_number) == 1:
inst_number = ' %s' % inst_number
if len(inst_number) == 2:
inst_number = ' %s' % inst_number
if len(inst_number) == 3:
inst_number = ' %s' % inst_number
if i_num == 'na':
inst_number = ' 0'
else:
log_file.write(
'ERROR#E03:Could not find INSTRUMENT in input file: %s. Data is ignored\r\n'
% os.path.join(dirname, filename)) # noqa
if 'DAILY' in extCSV.sections:
data = StringIO(
(extCSV.sections['DAILY']['_raw']).strip()) # noqa
if data is not None:
try:
data_rows = csv.reader(data)
data_rows.next()
except StopIteration:
log_file.write(
'ERROR#E04:Error reading DAILY block in file %s. Data is ignored\r\n'
% os.path.join(dirname, filename)) # noqa
write_output = 0
pass
for row in data_rows:
year = ' '
month = ' '
day = ' '
UTC_Begin = ' '
UTC_End = ' '
WLCode = ' '
ObsCode = ' '
ozone_std_error = ' '
ColumnO3 = ' '
UTC_Mean = ' '
nObs = ' '
if len(row) > 1 and "*" not in row[0]:
if len(row[0]) != 0:
year = row[0].split('-')[0]
month = row[0].split('-')[1]
day = row[0].split('-')[2]
if len(row) >= 2:
if len(row[1]) != 0:
WLCode = row[1]
if len(WLCode) > 1:
try:
WLCode = util.get_config_value(
'WLCode',
WLCode) # noqa
except Exception, err:
log_file.write(
'ERROR#E05:There is no one character WLCode code for \'%s\' in file %s. Data is ignored\r\n'
%
(WLCode,
os.path.join(
dirname, filename)
)) # noqa
write_output = 0
pass
else:
if inst_name == 'Dobson':
WLCode = util.get_config_value(
'WLCode', 'Dobson') # noqa
if inst_name == 'Brewer':
WLCode = util.get_config_value(
'WLCode', 'Brewer') # noqa
if inst_name == 'Filter':
WLCode = util.get_config_value(
'WLCode', 'Filter') # noqa
if inst_name == 'Microtops':
WLCode = util.get_config_value(
'WLCode',
'Microtops') # noqa
if len(row) >= 3:
if len(row[2]) != 0:
ObsCode = row[2]
if util.is_number(
ObsCode) == False and len(
ObsCode) != 1: # noqa
try:
ObsCode = util.get_config_value(
'Obs Code',
ObsCode) # noqa
except Exception, err:
log_file.write(
'ERROR#E06:There is no obs code for \'%s\' in file %s. Data is ignored\r\n'
%
(ObsCode,
os.path.join(
dirname, filename)
)) # noqa
write_output = 0
pass
else:
ObsCode = '9'
if len(row) >= 4:
if len(
row[3]
) != 0 and row[3] != '0.0' and row[
3] != '0' and not "-" in row[
3]: # noqa
try:
ColumnO3 = '%.0f' % round(
float(
re.findall(
"[0-9]*.[0-9]*",
row[3])[0]),
0) # noqa
if ColumnO3 == '0':
write_output = 0
except Exception, err:
log_file.write(
'ERROR#E07:Could not round ColumnO3 value of: %s in file %s. Data ignored.\r\n'
% (ColumnO3,
os.path.join(
dirname,
filename))) # noqa
write_output = 0
if len(ColumnO3) == 1:
ColumnO3 = ' %s' % ColumnO3 # noqa
if len(ColumnO3) == 2:
ColumnO3 = ' %s' % ColumnO3 # noqa
else:
write_output = 0
if len(row) >= 6:
if len(row[5]) != 0:
UTC_Begin = row[5]
if len(
re.findall(
"[0-9]*", UTC_Begin)
[0]) > 2: # noqa
UTC_Begin = UTC_Begin[:2]
elif "-" in UTC_Begin:
if -1.5 >= float(
UTC_Begin): # noqa
UTC_Begin = '-0'
else:
UTC_Begin = '00'
else:
try:
UTC_Begin = '%.0f' % round(
float(UTC_Begin),
0) # noqa
except Exception, err:
log_file.write(
'ERROR#E08:Could not round UTC_Begin value of: %s in file %s. Data ignored.\r\n'
%
(UTC_Begin,
os.path.join(
dirname, filename)
)) # noqa
write_output = 0
if int(UTC_Begin) in range(
10): # noqa
UTC_Begin = '0%s' % UTC_Begin # noqa
if len(row) >= 7:
if len(row[6]) != 0:
UTC_End = row[6]
if len(
re.findall(
"[0-9]*", UTC_End)
[0]) > 2: # noqa
UTC_End = UTC_End[:2]
elif "-" in UTC_End:
if -1.5 >= float(UTC_End):
UTC_End = '-0'
else:
UTC_End = '00'
else:
try:
UTC_End = '%.0f' % round(
float(UTC_End),
0) # noqa
except Exception, err:
log_file.write(
'ERROR#E09:Could not round UTC_End value of: %s in file %s. Data ignored.\r\n'
%
(UTC_End,
os.path.join(
dirname, filename)
)) # noqa
write_output = 0
if int(UTC_End) in range(
10): # noqa
UTC_End = '0%s' % UTC_End # noqa
if len(row) >= 8:
if len(row[7]) != 0:
UTC_Mean = row[7]
if len(row[6]) == 0:
UTC_End = UTC_Mean
if "-" in UTC_End:
if float(
UTC_End
) <= -1.5 and float(
UTC_End
) > -2: # noqa
UTC_End = '-0'
elif float(
UTC_End
) <= -1 and float(
UTC_End
) > -2: # noqa
UTC_End = '00'
elif float(UTC_End
) >= -1: # noqa
UTC_End = '00'
elif float(
UTC_End
) <= -2 and float(
UTC_End
) >= -10: # noqa
UTC_End = '-0'
elif float(
UTC_End
) < -10 and float(
UTC_End
) >= -10.5: # noqa
UTC_End = '-0'
else:
UTC_End = '-1'
else:
try:
UTC_End = '%.0f' % round(
float(UTC_End),
0) # noqa
except Exception, err:
log_file.write(
'ERROR#E09:Could not round UTC_End value of: %s in file %s. Data ignored.\r\n'
% (UTC_End,
os.path.join(
dirname,
filename))
) # noqa
write_output = 0
if int(UTC_End) in range(
10): # noqa
UTC_End = '0%s' % UTC_End # noqa
def __initialize_environment_variables(self):
self.client_name = util.get_config_value('mastodonClientName')
self.email = util.get_config_value('mastodonEmail')
self.password = util.get_config_value('mastodonPassword')
lvl = 'oam'
def is_service_running(name):
with open(os.devnull, 'wb') as hide_output:
exit_code = subprocess.Popen(['service', name, 'status'],
stdout=hide_output,
stderr=hide_output).wait()
return exit_code == 0
if __name__ == '__main__':
# get the users to create with openam
try:
users = get_config_value(lvl, 'users')
except:
print('Could not get users from config file')
print('Make sure they exist')
print('Exiting...')
exit()
# make sure that tomcat/openam is running
if not is_service_running('tomcat'):
print('Tomcat does not appear to be running')
print('Start tomcat and try running this script again')
print('Exiting...')
exit()
# loop through the users array and programmatically create the users for openam
print('Creating users for openam...')
def java_installed():
"""Checks if the specified version of Java is installed"""
cmd = 'rpm -q {java} > /dev/null'.format(
java=get_config_value(common, 'java'))
result = True if subprocess.call(cmd, shell='True') == 0 else False
return result
alias=get_config_value(lvl, 'alias'))
res = subprocess.call(cmd, shell='True')
def java_installed():
"""Checks if the specified version of Java is installed"""
cmd = 'rpm -q {java} > /dev/null'.format(
java=get_config_value(common, 'java'))
result = True if subprocess.call(cmd, shell='True') == 0 else False
return result
if __name__ == '__main__':
# check to make sure both cert and key files are present at their specified paths
try:
cert = open(get_config_value(lvl, 'cert_path'))
key = open(get_config_value(lvl, 'cert_key_path'))
except:
print('cert.pem and/or key.pem do not exist')
print('Exiting...')
exit()
if not java_installed():
print(
'The required version of java does not appear to be installed on your machine which is a requirement for this script to run properly'
)
print('Please execute java_installer.py first')
else:
# create the pkcs12 keystore from cert and key
create_pkcs12()
# import into keystore using java's keytool
import_to_keystore()
def install(mode, c):
# User wants to uninstall/clean up tomcat
if c == 'clean':
cleanup()
exit()
version = get_config_value(common, 'tomcat')
if mode == 'prompt':
ans = get_input('Do you want to install Tomcat ' + version +
'? [y/n] ')
if (ans == 'n'):
print('Skip installing Tomcat')
exit()
# install wget
print('Installing wget...')
subprocess.call('sudo yum install -y wget', shell='True')
print('Installing Tomcat version ' + version)
subprocess.call('groupadd tomcat >/dev/null 2>&1 ', shell=True)
subprocess.call(
'useradd -s /bin/nologin -g tomcat -d /opt/tomcat tomcat >/dev/null 2>&1 ',
shell=True)
major_version = version.split('.')[0]
tar_name = 'apache-tomcat-' + version
subprocess.call('wget http://archive.apache.org/dist/tomcat/tomcat-' +
major_version + '/v' + version + '/bin/' + tar_name +
'.tar.gz',
shell=True)
subprocess.call(' rm -rf /opt/tomcat', shell=True)
subprocess.call(' mkdir -p /opt/tomcat', shell=True)
subprocess.call(' tar -zxvf ' + tar_name + '.tar.gz' +
' -C /opt/tomcat --strip-components=1',
shell=True)
subprocess.call(' chgrp -R tomcat /opt/tomcat/conf ', shell=True)
subprocess.call(' chmod g+rwx /opt/tomcat/conf ', shell=True)
subprocess.call(' chmod g+r /opt/tomcat/conf/* ', shell=True)
subprocess.call(
' chown -R tomcat /opt/tomcat/logs/ /opt/tomcat/temp/ /opt/tomcat/webapps/ /opt/tomcat/work/ ',
shell=True)
subprocess.call(' chgrp -R tomcat /opt/tomcat/bin ', shell=True)
subprocess.call(' chgrp -R tomcat /opt/tomcat/lib ', shell=True)
subprocess.call(' chmod g+rwx /opt/tomcat/bin ', shell=True)
subprocess.call(' chmod g+r /opt/tomcat/bin/* ', shell=True)
# change ownership of top level /opt/tomcat directory to user:tomcat
# this allows the configuration tool to finish without any errors since it can write create the .openamcfg directory without running into permission issues
subprocess.call(' chown tomcat /opt/tomcat/', shell=True)
# update server.xml and tomcat.service
if (mode == 'silent'):
try:
server_config = get_config_value(lvl, 'tomcat_server_config')
except:
print('Using server.xml file at: ' + server_config)
try:
service_config = get_config_value(lvl, 'tomcat_service_config')
except:
print('Using tomcat service config file at: ' + service_config)
else:
print('Please update keystore path. Save changes when you are done..')
sleep(5)
subprocess.call('vim +121 ../config/tomcat/server.xml', shell=True)
print('Please update tomcat.service. Save changes when you are done..')
sleep(5)
subprocess.call('vim +17 ../config/tomcat/tomcat.service', shell=True)
server_config = default_server_config
service_config = default_service_config
# move the files to the right location
print('Moving configs into place ....')
subprocess.call(' cp {server_config} /opt/tomcat/conf'.format(
server_config=server_config),
shell=True)
subprocess.call(
' cp {service_config} /etc/systemd/system/tomcat.service'.format(
service_config=service_config),
shell=True)
subprocess.call(' cp {keystore} /opt/tomcat'.format(
keystore=get_config_value(lvl, 'keystore_path')),
shell=True)
print('Starting tomcat service ....')
subprocess.call(' systemctl daemon-reload', shell=True)
subprocess.call(' systemctl restart tomcat.service', shell=True)
subprocess.call(' systemctl enable tomcat.service', shell=True)
sleep(10)
print('Cleaning up ....')
subprocess.call('rm apach*.tar.gz*', shell=True)
# finish
print('Tomcat installation complete!.')