def parse_XML():
surveyId = 0
surveyInstanceId = 0
surveySession = ""
surveyStatus = ""
surveyCount = 0
surveyLanguage = ""
dateStarted = ""
answer = ""
optionDetails = ''
agency = ''
batchNo = ''
surveyDate = ''
zipCode = ''
language = ''
imagePath = ''
total_record_count = 0
total_survey_count = 0
lastBatchNo = ''
archive_dir = ''
addSurveyDetail = "INSERT INTO demographic_survey_paper_d(id,survey_number,survey_instance_id,survey_status,survey_language,question,answer, answer_is_optional, data_source, survey_file_name, image_file_name) VALUES(:id,:survey_number,:survey_instance_id,:survey_status,:survey_language,:question,:answer, :answer_is_optional, :data_source, :survey_file_name, :image_file_name)"
localDownload_dir = common.read_config_file("sftp", "fileDownload_path")
archive_dir = common.read_config_file("sftp", "archive_dir")
try:
user = common.read_config_file("db", "user")
password = common.read_config_file("db", "passwd")
password = common.decryptPassword(password)
host = common.read_config_file("db", "database")
connection = cx_Oracle.connect(user, password, host)
cursor = connection.cursor()
except Exception, e:
logging.error('Cursor Error: ' + e.message)
def get_files(pCursor):
global fileList
# Find current max survey date
max_survey_date = common.get_max_papersurvey_date(pCursor)
# Get values from the config file
hostStr = common.read_config_file("sftp", "host")
userName = common.read_config_file("sftp", "user_name")
passWord = common.read_config_file("sftp", "password")
ftp_dir = common.read_config_file("sftp", "ftp_dir")
localDownload_dir = common.read_config_file("sftp", "fileDownload_path")
paperSurveyExtension = common.read_config_file("sftp", "file_extension")
imageExtension = common.read_config_file("sftp", "image_extension")
imageDownloadDir = common.read_config_file("sftp", "imageDownload_path")
rerunDate = common.read_config_file(
"sftp", "survey_date") # Used to reload survey from a specific date
loadFileWithoutImages = common.read_config_file("sftp",
"load_without_images")
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect(hostStr, username=userName, password=passWord)
sftp = client.open_sftp()
# Change to the specified FTP directory
try:
sftp.chdir(ftp_dir)
except:
logging.error('Invalid FTP path: ' + ftp_dir)
sys.exit(-1)
rerunSurveyDate = datetime.strptime(rerunDate, '%d %b %Y')
if rerunDate:
# Use config file to set date to pull files
try:
rerunSurveyDate = datetime.strptime(rerunDate, '%d %b %Y')
except:
logging.error('Invalid survey date in config: ' + rerunDate)
else:
# No entry in config. Use last survey date as starting point
rerunSurveyDate = datetime.strptime(max_survey_date, '%d %b %Y')
latestSurveyDate = rerunSurveyDate.strftime('%Y%m%d')
# List the files
for fileName in sftp.listdir(ftp_dir):
#Process each file, get modified date
utime = sftp.stat(fileName).st_mtime
last_modified = datetime.fromtimestamp(utime)
get_file_from_sftp(sftp, fileName, last_modified, localDownload_dir,
rerunSurveyDate, paperSurveyExtension,
imageDownloadDir, imageExtension,
loadFileWithoutImages)
# Close the connection
sftp.close()
return fileList
def setup_application():
config = get_config()
if config is not None:
return
config = read_config_file()
validate_config(config)
set_config(config)
def setup_application():
config = get_config()
if config is not None:
return
config = read_config_file()
validate_config(config)
set_config(config)
def process_config():
cmdline_opts = parse_cmdline()
validate_options(cmdline_opts)
config = read_config_file(cmdline_opts.get('config_file', None))
config = merge_config(config, cmdline_opts)
return config
def process_config():
cmdline_opts = parse_cmdline()
validate_options(cmdline_opts)
config = read_config_file(cmdline_opts.get('config_file', None))
config = merge_config(config, cmdline_opts)
return config
def process_config():
cmdline_opts = parse_cmdline()
config = read_config_file(cmdline_opts.get('config_file', None))
config = merge_config(config, cmdline_opts)
try:
validate_config(config)
except NetKesConfigError, e:
raise e
def process_config():
cmdline_opts = parse_cmdline()
config = read_config_file(cmdline_opts.get('config_file', None))
config = merge_config(config, cmdline_opts)
try:
validate_config(config)
except NetKesConfigError, e:
raise e
def process_config():
cmdline_opts = parse_cmdline()
config = read_config_file(cmdline_opts.get('config_file', None))
config = merge_config(config, cmdline_opts)
if 'groups' not in config:
raise StartupException("Lacking an LDAP mapping group in the config file. Check your docs!")
log = logging.getLogger('process_config')
log.debug('%s' % config['api_root'])
return config
def delete_files_from_sftp(pFileList, pArchiveDirectory):
global imageFileList
# Get values from the config file
hostStr = common.read_config_file("sftp", "host")
userName = common.read_config_file("sftp", "user_name")
passWord = common.read_config_file("sftp", "password")
ftp_dir = common.read_config_file("sftp", "ftp_dir")
localDownload_dir = common.read_config_file("sftp", "fileDownload_path")
imageDownloadDir = common.read_config_file("sftp", "imageDownload_path")
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect(hostStr, username=userName, password=passWord)
sftp = client.open_sftp()
# Change to the specified FTP directory
try:
sftp.chdir(ftp_dir)
except:
logging.error('Invalid FTP path for delete: ' + ftp_dir)
sys.exit(-1)
for fileName in pFileList:
# If the survey XML exists in the archive, delete the SFTP version
if os.path.isfile(pArchiveDirectory + fileName):
sftp.remove(fileName)
for fileName in imageFileList:
# If the survey iamge exists in the archive, delete the SFTP version
if os.path.isfile(imageDownloadDir + fileName):
sftp.remove(fileName)
def main():
# Configure logging
logPath = common.read_config_file("logging", "path")
logFile = common.read_config_file("logging", "file_name")
logFile = logFile + datetime.datetime.now().strftime('%Y%m%d') + '.log'
logLevel = common.read_config_file("logging", "level")
logLevel = common.translate_logging_level(logLevel)
logging.basicConfig(filename=logPath + '/' + logFile, level=logLevel)
# Setup the required proxys
linuxUser = common.read_config_file("linux", "linuxUser")
linuxPassword = common.read_config_file("linux", "linuxPasswd")
linuxPassword = common.decryptPassword(linuxPassword)
proxy = common.read_config_file("linux", "proxy")
os.environ[
'https_proxy'] = 'https://' + linuxUser + ':' + linuxPassword + '@' + proxy
os.environ[
'http_proxy'] = 'http://' + linuxUser + ':' + linuxPassword + '@' + proxy
# Log details and start processing
startTime = datetime.datetime.today()
startTimeStr = startTime.strftime("%Y%m%d %H:%M:%S")
logging.info('Started ' + startTimeStr)
parse_XML()
def check_sftp_counts():
# Confirm that XML and Image file counts match in Axway
# A mismatch indicates a problem. Each XML should have one TIF file.
surveyCount = 0
imageCount = 0
# Get values from the config file
hostStr = common.read_config_file("sftp", "host")
userName = common.read_config_file("sftp", "user_name")
passWord = common.read_config_file("sftp", "password")
ftp_dir = common.read_config_file("sftp", "ftp_dir")
localDownload_dir = common.read_config_file("sftp", "fileDownload_path")
imageDownloadDir = common.read_config_file("sftp", "imageDownload_path")
paperSurveyExtension = common.read_config_file("sftp", "file_extension")
imageExtension = common.read_config_file("sftp", "image_extension")
loadFileWithoutImages = common.read_config_file("sftp",
"load_without_images")
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect(hostStr, username=userName, password=passWord)
sftp = client.open_sftp()
for fileName in sftp.listdir(ftp_dir):
fileFound = re.search(".*\." + str(paperSurveyExtension) + "$",
str(fileName))
if fileFound:
surveyCount += 1
fileFound = re.search(".*\." + str(imageExtension) + "$",
str(fileName))
if fileFound:
imageCount += 1
if surveyCount == 0:
logging.info('No survey files found: ')
return (False)
if loadFileWithoutImages != "Yes":
if surveyCount != imageCount:
logging.error('XML/Image mismatch. Contact Caso.')
return (False)
# XML file and Image file counts match. Continue processing
return (True)
def decorator(environ, start_response):
log = logging.getLogger('login_required')
log.debug("start")
try:
brand_identifier = environ['query_data']['brand_id'][0]
username = environ['query_data']['username'][0]
password = environ['query_data']['password'][0]
crypt_pw = environ['query_data'].get('crypt_pw', ["True"])[0]
except KeyError:
log.error("Got bad request.")
return BadRequest()(environ, start_response)
decoded_user = unquote(username)
# If we get anything OTHER than explicitly "False" in the request, we will assume it's an encrypted password.
if crypt_pw == "False":
plaintext_password = password
else:
try:
plaintext_password = server.read_escrow_data(
brand_identifier, password)
except KeyError:
log.warn("missing identifier %s" % (brand_identifier, ))
return NotFound()(environ, start_response)
except ValueError:
log.warn("bad values for authenticating user %s" %
(decoded_user, ))
return BadRequest()(environ, start_response)
except Exception:
log.exception(
"server.read_escrow_data failed for user %s brand %s" % (
decoded_user,
brand_identifier,
))
return ServerError()(environ, start_response)
if not authenticator(read_config_file(), decoded_user,
plaintext_password):
log.info("Auth failed for %s" % (decoded_user, ))
return Forbidden()(environ, start_response)
log.info("Auth OK for brand %s with user %s" % (
brand_identifier,
decoded_user,
))
return fun(environ, start_response)
def decorator(environ, start_response):
log = logging.getLogger('login_required')
log.debug("start")
try:
brand_identifier = environ['query_data']['brand_id'][0]
username = environ['query_data']['username'][0]
password = environ['query_data']['password'][0]
crypt_pw = environ['query_data'].get('crypt_pw', ["True"])[0]
except KeyError:
log.error("Got bad request.")
return BadRequest()(environ, start_response)
decoded_user = unquote(username)
# If we get anything OTHER than explicitly "False" in the request, we will assume it's an encrypted password.
if crypt_pw == "False":
plaintext_password = password
else:
try:
plaintext_password = server.read_escrow_data(
brand_identifier, password)
except KeyError:
log.warn("missing identifier %s" % (brand_identifier,))
return NotFound()(environ, start_response)
except ValueError:
log.warn("bad values for authenticating user %s" % (decoded_user,))
return BadRequest()(environ, start_response)
except Exception:
log.exception("server.read_escrow_data failed for user %s brand %s"
% (decoded_user, brand_identifier,))
return ServerError()(environ, start_response)
if not authenticator(read_config_file(), decoded_user, plaintext_password):
log.info("Auth failed for %s" % (decoded_user,))
return Forbidden()(environ, start_response)
log.info("Auth OK for brand %s with user %s" % (brand_identifier, decoded_user, ))
return fun(environ, start_response)
def get_file_from_sftp(pSrv, pFileName, pLastModified, pFileDownloadDir,
pLatestSurveyDate, pPaperSurveyExtension,
pImageDownloadDir, pImageExtension, pLoadWithoutImages):
global fileList
global imageFileList
try:
archive_dir = common.read_config_file("sftp", "archive_dir")
file_name = pFileName
if pLastModified <= pLatestSurveyDate:
#Only pull XML files with right extention
xmlFileFound = re.search(".*\." + pPaperSurveyExtension + "$",
file_name)
if xmlFileFound:
if os.path.isfile(pFileDownloadDir + file_name):
logging.error('File ' + file_name + ' already processed.')
else:
fileList.append(file_name)
pSrv.get(file_name, pFileDownloadDir + file_name)
logging.info('Downloaded: ' + file_name)
if pLoadWithoutImages != 'Yes':
#Only pull image files with right extention
imageFileFound = re.search(".*\." + pImageExtension + "$",
file_name)
if imageFileFound:
if os.path.isfile(pImageDownloadDir + file_name):
logging.error('Image ' + file_name +
' already processed.')
else:
imageFileList.append(file_name)
pSrv.get(file_name, pImageDownloadDir + file_name)
logging.info('Downloaded: ' + file_name)
except ValueError:
logging.error('getFileFromFTP invalid date: ' +
str(pFtpObject.st_mtime))
def main():
args = docopt(__doc__)
config_filename = common.full_path(args['<configfile>'])
yaml_config = common.read_config_file(config_filename)
configs = load_nginx_config_table(yaml_config)
template_mgr = common.get_template_mgr_for_location('templates')
config_template = template_mgr.get_template('nginx_config.j2')
# show all the configurations
#
if args['-l'] or args['--list']:
for key in configs.keys():
print(configs[key])
exit(0)
# we can generate without a specific nginx config, iff there's just one
#
output = None
config_name = args['--env']
if args['<configfile>'] and not config_name:
if len(configs.keys()) > 1:
print('Multiple configurations found. Please specify one.')
exit(0)
output = config_template.render(nginx_config=configs.values()[0])
else:
if not configs.get(config_name):
print('No nginx configuration labeled "%s" in %s.' %
(config_name, config_filename))
exit(0)
output = config_template.render(nginx_config=configs[config_name])
print(output)
exit(0)
def main():
args = docopt(__doc__)
config_filename = common.full_path(args['<configfile>'])
yaml_config = common.read_config_file(config_filename)
configs = load_nginx_config_table(yaml_config)
template_mgr = common.get_template_mgr_for_location('templates')
config_template = template_mgr.get_template('nginx_config.j2')
# show all the configurations
#
if args['-l'] or args['--list']:
for key in configs.keys():
print configs[key]
exit(0)
# we can generate without a specific nginx config, iff there's just one
#
output = None
config_name = args['--env']
if args['<configfile>'] and not config_name:
if len(configs.keys()) > 1:
print 'Multiple configurations found. Please specify one.'
exit(0)
output = config_template.render(nginx_config=configs.values()[0])
else:
if not configs.get(config_name):
print 'No nginx configuration labeled "%s" in %s.' % (config_name, config_filename)
exit(0)
output = config_template.render(nginx_config=configs[config_name])
print output
exit(0)
def decorator(request):
log = logging.getLogger('login_required')
log.debug("start")
if valid_auth_session(request):
return fun(request)
else:
try:
brand_identifier = request.POST['brand_id']
username = request.POST['username']
auth = a2b_base64(request.POST['auth'])
serial_sign_key = request.POST['sign_key']
layer_count = int(request.POST['layer_count'])
except KeyError:
log.error("Got bad request.")
return HttpResponseBadRequest()
try:
sign_key = serial.loads(serial_sign_key)
except (serial.EndOfFile,
serial.NotSerializerFileError,
serial.NotSerializableObjectError):
log.error("Got bad request. Unable to load sign key")
return HttpResponseBadRequest()
decoded_user = urllib.unquote(username)
try:
data = server.read_escrow_data(
brand_identifier,
auth,
sign_key=sign_key,
layer_count=layer_count,
)
plaintext_auth = json.loads(data)
if ('challenge' not in plaintext_auth or
'password' not in plaintext_auth):
log.warn("missing auth key %s" % (brand_identifier,))
return HttpResponseBadRequest()
except KeyError:
log.warn("missing identifier %s" % (brand_identifier,))
return HttpResponseNotFound()
except ValueError:
log.warn("bad values for authenticating user %s" % (decoded_user,))
return HttpResponseBadRequest()
except Exception:
log.exception("server.read_escrow_data failed for user %s brand %s"
% (decoded_user, brand_identifier,))
return HttpResponseServerError()
challenge = valid_challenge(request, plaintext_auth['challenge'])
authenticated = authenticator(read_config_file(),
decoded_user,
plaintext_auth['password'])
if not challenge or not authenticated:
log.info("Auth failed for %s" % (decoded_user,))
return HttpResponseForbidden()
session_challenge = get_challenge(request)
secret_box, nonce = create_secret_box(plaintext_auth['password'],
session_challenge[0])
request.session['auth'] = {
'secret_box': secret_box,
'nonce': nonce,
'time': session_challenge[1],
'brand_identifier': brand_identifier,
'sign_key': sign_key,
'layer_count': layer_count,
}
log.info("Auth OK for brand %s with user %s" % (brand_identifier, decoded_user, ))
return fun(request)
def setup_application():
config = read_config_file()
validate_config(config)
def setup_application():
config = read_config_file()
validate_config(config)
def decorator(request):
log = logging.getLogger('login_required')
log.debug("start")
if valid_auth_session(request):
return fun(request)
else:
try:
brand_identifier = request.POST['brand_id']
username = request.POST['username']
auth = a2b_base64(request.POST['auth'])
serial_sign_key = request.POST['sign_key']
layer_count = int(request.POST['layer_count'])
except KeyError:
log.error("Got bad request.")
return HttpResponseBadRequest()
try:
sign_key = serial.loads(serial_sign_key)
except (serial.EndOfFile, serial.NotSerializerFileError,
serial.NotSerializableObjectError):
log.error("Got bad request. Unable to load sign key")
return HttpResponseBadRequest()
decoded_user = urllib.unquote(username)
try:
data = server.read_escrow_data(
brand_identifier,
auth,
sign_key=sign_key,
layer_count=layer_count,
)
plaintext_auth = json.loads(data)
if ('challenge' not in plaintext_auth
or 'password' not in plaintext_auth):
log.warn("missing auth key %s" % (brand_identifier, ))
return HttpResponseBadRequest()
except KeyError:
log.warn("missing identifier %s" % (brand_identifier, ))
return HttpResponseNotFound()
except ValueError:
log.warn("bad values for authenticating user %s" %
(decoded_user, ))
return HttpResponseBadRequest()
except Exception:
log.exception(
"server.read_escrow_data failed for user %s brand %s" % (
decoded_user,
brand_identifier,
))
return HttpResponseServerError()
challenge = valid_challenge(request, plaintext_auth['challenge'])
authenticated = authenticator(read_config_file(), decoded_user,
plaintext_auth['password'])
if not challenge or not authenticated:
log.info("Auth failed for %s" % (decoded_user, ))
return HttpResponseForbidden()
session_challenge = get_challenge(request)
secret_box, nonce = create_secret_box(plaintext_auth['password'],
session_challenge[0])
request.session['auth'] = {
'secret_box': secret_box,
'nonce': nonce,
'time': session_challenge[1],
'brand_identifier': brand_identifier,
'sign_key': sign_key,
'layer_count': layer_count,
}
log.info("Auth OK for brand %s with user %s" % (
brand_identifier,
decoded_user,
))
return fun(request)
