def parse_XML(): surveyId = 0 surveyInstanceId = 0 surveySession = "" surveyStatus = "" surveyCount = 0 surveyLanguage = "" dateStarted = "" answer = "" optionDetails = '' agency = '' batchNo = '' surveyDate = '' zipCode = '' language = '' imagePath = '' total_record_count = 0 total_survey_count = 0 lastBatchNo = '' archive_dir = '' addSurveyDetail = "INSERT INTO demographic_survey_paper_d(id,survey_number,survey_instance_id,survey_status,survey_language,question,answer, answer_is_optional, data_source, survey_file_name, image_file_name) VALUES(:id,:survey_number,:survey_instance_id,:survey_status,:survey_language,:question,:answer, :answer_is_optional, :data_source, :survey_file_name, :image_file_name)" localDownload_dir = common.read_config_file("sftp", "fileDownload_path") archive_dir = common.read_config_file("sftp", "archive_dir") try: user = common.read_config_file("db", "user") password = common.read_config_file("db", "passwd") password = common.decryptPassword(password) host = common.read_config_file("db", "database") connection = cx_Oracle.connect(user, password, host) cursor = connection.cursor() except Exception, e: logging.error('Cursor Error: ' + e.message)
def get_files(pCursor): global fileList # Find current max survey date max_survey_date = common.get_max_papersurvey_date(pCursor) # Get values from the config file hostStr = common.read_config_file("sftp", "host") userName = common.read_config_file("sftp", "user_name") passWord = common.read_config_file("sftp", "password") ftp_dir = common.read_config_file("sftp", "ftp_dir") localDownload_dir = common.read_config_file("sftp", "fileDownload_path") paperSurveyExtension = common.read_config_file("sftp", "file_extension") imageExtension = common.read_config_file("sftp", "image_extension") imageDownloadDir = common.read_config_file("sftp", "imageDownload_path") rerunDate = common.read_config_file( "sftp", "survey_date") # Used to reload survey from a specific date loadFileWithoutImages = common.read_config_file("sftp", "load_without_images") client = paramiko.SSHClient() client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) client.connect(hostStr, username=userName, password=passWord) sftp = client.open_sftp() # Change to the specified FTP directory try: sftp.chdir(ftp_dir) except: logging.error('Invalid FTP path: ' + ftp_dir) sys.exit(-1) rerunSurveyDate = datetime.strptime(rerunDate, '%d %b %Y') if rerunDate: # Use config file to set date to pull files try: rerunSurveyDate = datetime.strptime(rerunDate, '%d %b %Y') except: logging.error('Invalid survey date in config: ' + rerunDate) else: # No entry in config. Use last survey date as starting point rerunSurveyDate = datetime.strptime(max_survey_date, '%d %b %Y') latestSurveyDate = rerunSurveyDate.strftime('%Y%m%d') # List the files for fileName in sftp.listdir(ftp_dir): #Process each file, get modified date utime = sftp.stat(fileName).st_mtime last_modified = datetime.fromtimestamp(utime) get_file_from_sftp(sftp, fileName, last_modified, localDownload_dir, rerunSurveyDate, paperSurveyExtension, imageDownloadDir, imageExtension, loadFileWithoutImages) # Close the connection sftp.close() return fileList
def setup_application(): config = get_config() if config is not None: return config = read_config_file() validate_config(config) set_config(config)
def process_config(): cmdline_opts = parse_cmdline() validate_options(cmdline_opts) config = read_config_file(cmdline_opts.get('config_file', None)) config = merge_config(config, cmdline_opts) return config
def process_config(): cmdline_opts = parse_cmdline() config = read_config_file(cmdline_opts.get('config_file', None)) config = merge_config(config, cmdline_opts) try: validate_config(config) except NetKesConfigError, e: raise e
def process_config(): cmdline_opts = parse_cmdline() config = read_config_file(cmdline_opts.get('config_file', None)) config = merge_config(config, cmdline_opts) if 'groups' not in config: raise StartupException("Lacking an LDAP mapping group in the config file. Check your docs!") log = logging.getLogger('process_config') log.debug('%s' % config['api_root']) return config
def delete_files_from_sftp(pFileList, pArchiveDirectory): global imageFileList # Get values from the config file hostStr = common.read_config_file("sftp", "host") userName = common.read_config_file("sftp", "user_name") passWord = common.read_config_file("sftp", "password") ftp_dir = common.read_config_file("sftp", "ftp_dir") localDownload_dir = common.read_config_file("sftp", "fileDownload_path") imageDownloadDir = common.read_config_file("sftp", "imageDownload_path") client = paramiko.SSHClient() client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) client.connect(hostStr, username=userName, password=passWord) sftp = client.open_sftp() # Change to the specified FTP directory try: sftp.chdir(ftp_dir) except: logging.error('Invalid FTP path for delete: ' + ftp_dir) sys.exit(-1) for fileName in pFileList: # If the survey XML exists in the archive, delete the SFTP version if os.path.isfile(pArchiveDirectory + fileName): sftp.remove(fileName) for fileName in imageFileList: # If the survey iamge exists in the archive, delete the SFTP version if os.path.isfile(imageDownloadDir + fileName): sftp.remove(fileName)
def main(): # Configure logging logPath = common.read_config_file("logging", "path") logFile = common.read_config_file("logging", "file_name") logFile = logFile + datetime.datetime.now().strftime('%Y%m%d') + '.log' logLevel = common.read_config_file("logging", "level") logLevel = common.translate_logging_level(logLevel) logging.basicConfig(filename=logPath + '/' + logFile, level=logLevel) # Setup the required proxys linuxUser = common.read_config_file("linux", "linuxUser") linuxPassword = common.read_config_file("linux", "linuxPasswd") linuxPassword = common.decryptPassword(linuxPassword) proxy = common.read_config_file("linux", "proxy") os.environ[ 'https_proxy'] = 'https://' + linuxUser + ':' + linuxPassword + '@' + proxy os.environ[ 'http_proxy'] = 'http://' + linuxUser + ':' + linuxPassword + '@' + proxy # Log details and start processing startTime = datetime.datetime.today() startTimeStr = startTime.strftime("%Y%m%d %H:%M:%S") logging.info('Started ' + startTimeStr) parse_XML()
def check_sftp_counts(): # Confirm that XML and Image file counts match in Axway # A mismatch indicates a problem. Each XML should have one TIF file. surveyCount = 0 imageCount = 0 # Get values from the config file hostStr = common.read_config_file("sftp", "host") userName = common.read_config_file("sftp", "user_name") passWord = common.read_config_file("sftp", "password") ftp_dir = common.read_config_file("sftp", "ftp_dir") localDownload_dir = common.read_config_file("sftp", "fileDownload_path") imageDownloadDir = common.read_config_file("sftp", "imageDownload_path") paperSurveyExtension = common.read_config_file("sftp", "file_extension") imageExtension = common.read_config_file("sftp", "image_extension") loadFileWithoutImages = common.read_config_file("sftp", "load_without_images") client = paramiko.SSHClient() client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) client.connect(hostStr, username=userName, password=passWord) sftp = client.open_sftp() for fileName in sftp.listdir(ftp_dir): fileFound = re.search(".*\." + str(paperSurveyExtension) + "$", str(fileName)) if fileFound: surveyCount += 1 fileFound = re.search(".*\." + str(imageExtension) + "$", str(fileName)) if fileFound: imageCount += 1 if surveyCount == 0: logging.info('No survey files found: ') return (False) if loadFileWithoutImages != "Yes": if surveyCount != imageCount: logging.error('XML/Image mismatch. Contact Caso.') return (False) # XML file and Image file counts match. Continue processing return (True)
def decorator(environ, start_response): log = logging.getLogger('login_required') log.debug("start") try: brand_identifier = environ['query_data']['brand_id'][0] username = environ['query_data']['username'][0] password = environ['query_data']['password'][0] crypt_pw = environ['query_data'].get('crypt_pw', ["True"])[0] except KeyError: log.error("Got bad request.") return BadRequest()(environ, start_response) decoded_user = unquote(username) # If we get anything OTHER than explicitly "False" in the request, we will assume it's an encrypted password. if crypt_pw == "False": plaintext_password = password else: try: plaintext_password = server.read_escrow_data( brand_identifier, password) except KeyError: log.warn("missing identifier %s" % (brand_identifier, )) return NotFound()(environ, start_response) except ValueError: log.warn("bad values for authenticating user %s" % (decoded_user, )) return BadRequest()(environ, start_response) except Exception: log.exception( "server.read_escrow_data failed for user %s brand %s" % ( decoded_user, brand_identifier, )) return ServerError()(environ, start_response) if not authenticator(read_config_file(), decoded_user, plaintext_password): log.info("Auth failed for %s" % (decoded_user, )) return Forbidden()(environ, start_response) log.info("Auth OK for brand %s with user %s" % ( brand_identifier, decoded_user, )) return fun(environ, start_response)
def decorator(environ, start_response): log = logging.getLogger('login_required') log.debug("start") try: brand_identifier = environ['query_data']['brand_id'][0] username = environ['query_data']['username'][0] password = environ['query_data']['password'][0] crypt_pw = environ['query_data'].get('crypt_pw', ["True"])[0] except KeyError: log.error("Got bad request.") return BadRequest()(environ, start_response) decoded_user = unquote(username) # If we get anything OTHER than explicitly "False" in the request, we will assume it's an encrypted password. if crypt_pw == "False": plaintext_password = password else: try: plaintext_password = server.read_escrow_data( brand_identifier, password) except KeyError: log.warn("missing identifier %s" % (brand_identifier,)) return NotFound()(environ, start_response) except ValueError: log.warn("bad values for authenticating user %s" % (decoded_user,)) return BadRequest()(environ, start_response) except Exception: log.exception("server.read_escrow_data failed for user %s brand %s" % (decoded_user, brand_identifier,)) return ServerError()(environ, start_response) if not authenticator(read_config_file(), decoded_user, plaintext_password): log.info("Auth failed for %s" % (decoded_user,)) return Forbidden()(environ, start_response) log.info("Auth OK for brand %s with user %s" % (brand_identifier, decoded_user, )) return fun(environ, start_response)
def get_file_from_sftp(pSrv, pFileName, pLastModified, pFileDownloadDir, pLatestSurveyDate, pPaperSurveyExtension, pImageDownloadDir, pImageExtension, pLoadWithoutImages): global fileList global imageFileList try: archive_dir = common.read_config_file("sftp", "archive_dir") file_name = pFileName if pLastModified <= pLatestSurveyDate: #Only pull XML files with right extention xmlFileFound = re.search(".*\." + pPaperSurveyExtension + "$", file_name) if xmlFileFound: if os.path.isfile(pFileDownloadDir + file_name): logging.error('File ' + file_name + ' already processed.') else: fileList.append(file_name) pSrv.get(file_name, pFileDownloadDir + file_name) logging.info('Downloaded: ' + file_name) if pLoadWithoutImages != 'Yes': #Only pull image files with right extention imageFileFound = re.search(".*\." + pImageExtension + "$", file_name) if imageFileFound: if os.path.isfile(pImageDownloadDir + file_name): logging.error('Image ' + file_name + ' already processed.') else: imageFileList.append(file_name) pSrv.get(file_name, pImageDownloadDir + file_name) logging.info('Downloaded: ' + file_name) except ValueError: logging.error('getFileFromFTP invalid date: ' + str(pFtpObject.st_mtime))
def main(): args = docopt(__doc__) config_filename = common.full_path(args['<configfile>']) yaml_config = common.read_config_file(config_filename) configs = load_nginx_config_table(yaml_config) template_mgr = common.get_template_mgr_for_location('templates') config_template = template_mgr.get_template('nginx_config.j2') # show all the configurations # if args['-l'] or args['--list']: for key in configs.keys(): print(configs[key]) exit(0) # we can generate without a specific nginx config, iff there's just one # output = None config_name = args['--env'] if args['<configfile>'] and not config_name: if len(configs.keys()) > 1: print('Multiple configurations found. Please specify one.') exit(0) output = config_template.render(nginx_config=configs.values()[0]) else: if not configs.get(config_name): print('No nginx configuration labeled "%s" in %s.' % (config_name, config_filename)) exit(0) output = config_template.render(nginx_config=configs[config_name]) print(output) exit(0)
def main(): args = docopt(__doc__) config_filename = common.full_path(args['<configfile>']) yaml_config = common.read_config_file(config_filename) configs = load_nginx_config_table(yaml_config) template_mgr = common.get_template_mgr_for_location('templates') config_template = template_mgr.get_template('nginx_config.j2') # show all the configurations # if args['-l'] or args['--list']: for key in configs.keys(): print configs[key] exit(0) # we can generate without a specific nginx config, iff there's just one # output = None config_name = args['--env'] if args['<configfile>'] and not config_name: if len(configs.keys()) > 1: print 'Multiple configurations found. Please specify one.' exit(0) output = config_template.render(nginx_config=configs.values()[0]) else: if not configs.get(config_name): print 'No nginx configuration labeled "%s" in %s.' % (config_name, config_filename) exit(0) output = config_template.render(nginx_config=configs[config_name]) print output exit(0)
def decorator(request): log = logging.getLogger('login_required') log.debug("start") if valid_auth_session(request): return fun(request) else: try: brand_identifier = request.POST['brand_id'] username = request.POST['username'] auth = a2b_base64(request.POST['auth']) serial_sign_key = request.POST['sign_key'] layer_count = int(request.POST['layer_count']) except KeyError: log.error("Got bad request.") return HttpResponseBadRequest() try: sign_key = serial.loads(serial_sign_key) except (serial.EndOfFile, serial.NotSerializerFileError, serial.NotSerializableObjectError): log.error("Got bad request. Unable to load sign key") return HttpResponseBadRequest() decoded_user = urllib.unquote(username) try: data = server.read_escrow_data( brand_identifier, auth, sign_key=sign_key, layer_count=layer_count, ) plaintext_auth = json.loads(data) if ('challenge' not in plaintext_auth or 'password' not in plaintext_auth): log.warn("missing auth key %s" % (brand_identifier,)) return HttpResponseBadRequest() except KeyError: log.warn("missing identifier %s" % (brand_identifier,)) return HttpResponseNotFound() except ValueError: log.warn("bad values for authenticating user %s" % (decoded_user,)) return HttpResponseBadRequest() except Exception: log.exception("server.read_escrow_data failed for user %s brand %s" % (decoded_user, brand_identifier,)) return HttpResponseServerError() challenge = valid_challenge(request, plaintext_auth['challenge']) authenticated = authenticator(read_config_file(), decoded_user, plaintext_auth['password']) if not challenge or not authenticated: log.info("Auth failed for %s" % (decoded_user,)) return HttpResponseForbidden() session_challenge = get_challenge(request) secret_box, nonce = create_secret_box(plaintext_auth['password'], session_challenge[0]) request.session['auth'] = { 'secret_box': secret_box, 'nonce': nonce, 'time': session_challenge[1], 'brand_identifier': brand_identifier, 'sign_key': sign_key, 'layer_count': layer_count, } log.info("Auth OK for brand %s with user %s" % (brand_identifier, decoded_user, )) return fun(request)
def setup_application(): config = read_config_file() validate_config(config)
def decorator(request): log = logging.getLogger('login_required') log.debug("start") if valid_auth_session(request): return fun(request) else: try: brand_identifier = request.POST['brand_id'] username = request.POST['username'] auth = a2b_base64(request.POST['auth']) serial_sign_key = request.POST['sign_key'] layer_count = int(request.POST['layer_count']) except KeyError: log.error("Got bad request.") return HttpResponseBadRequest() try: sign_key = serial.loads(serial_sign_key) except (serial.EndOfFile, serial.NotSerializerFileError, serial.NotSerializableObjectError): log.error("Got bad request. Unable to load sign key") return HttpResponseBadRequest() decoded_user = urllib.unquote(username) try: data = server.read_escrow_data( brand_identifier, auth, sign_key=sign_key, layer_count=layer_count, ) plaintext_auth = json.loads(data) if ('challenge' not in plaintext_auth or 'password' not in plaintext_auth): log.warn("missing auth key %s" % (brand_identifier, )) return HttpResponseBadRequest() except KeyError: log.warn("missing identifier %s" % (brand_identifier, )) return HttpResponseNotFound() except ValueError: log.warn("bad values for authenticating user %s" % (decoded_user, )) return HttpResponseBadRequest() except Exception: log.exception( "server.read_escrow_data failed for user %s brand %s" % ( decoded_user, brand_identifier, )) return HttpResponseServerError() challenge = valid_challenge(request, plaintext_auth['challenge']) authenticated = authenticator(read_config_file(), decoded_user, plaintext_auth['password']) if not challenge or not authenticated: log.info("Auth failed for %s" % (decoded_user, )) return HttpResponseForbidden() session_challenge = get_challenge(request) secret_box, nonce = create_secret_box(plaintext_auth['password'], session_challenge[0]) request.session['auth'] = { 'secret_box': secret_box, 'nonce': nonce, 'time': session_challenge[1], 'brand_identifier': brand_identifier, 'sign_key': sign_key, 'layer_count': layer_count, } log.info("Auth OK for brand %s with user %s" % ( brand_identifier, decoded_user, )) return fun(request)