def sendmail(attach): '''发送测试报告''' subject = '接口测试结果:%s' % time.strftime('%Y/%m/%d %H:%M:%S') content = "接口测试已完成,测试详情请查看附件测试报告!" mail_server = conf.get_conf('mail','server') mail_user = conf.get_conf('mail','user') mail_passwd = conf.get_conf('mail','passwd') mail_to = eval(conf.get_conf('mail','to')) mail_cc = eval(conf.get_conf('mail','cc')) sendmail = SendMail(mail_server,mail_user,mail_passwd) sendmail.send(to=mail_to,cc=mail_cc,subject=subject,content=content,attach=[attach])
def process_request(self, request): lang = request.GET.get(conf.get_conf(conf.LANG_PARAM), '') if not lang: lang = request.META.get( "HTTP_%s" % conf.get_conf(conf.LANG_PARAM).upper(), '') if not lang: lang = request.COOKIES.get(conf.get_conf(conf.LANG_PARAM), '') if lang: translation.activate(lang) request.LANGUAGE_CODE = translation.get_language()
def get_session(connection='preview'): conf = get_conf('database', 'MYSQL').get(connection) connection = 'mysql+pymysql://{user}:{password}@{host}:{port}/{db}?charset=utf8'.format( **conf) engine = create_engine(connection) session = sessionmaker(bind=engine) return session()
def __init__(self): try: self.conf = get_conf() except IOError as e: raise FileError(e.filename, hint='Are you in the right directory?') try: self.name = self.conf['name'] except KeyError: raise ClickException('No name in skipper.yml') self.services = [] for name, details in self.conf['services'].items(): try: self.services.append( self.make_service(name=name, **details)) except (TypeError, ValueError) as e: raise ClickException("%s: %s" % (name, e.message)) self.host = get_host(self.conf.get('host', 'aws')) self.host.creds = creds self.host.project = self self.groups = [] for name, details in self.conf['groups'].items(): try: self.groups.append( self.host.make_group( name=name, region=self.conf.get('region'), **details)) except (TypeError, ValueError) as e: raise ClickException("%s: %s" % (name, e.message))
def connection_str(connection='base'): registry.register('clickhouse', 'clickhouse_sqlalchemy.drivers.base', 'dialect') conf = get_conf('database', 'CLICKHOUSE').get(connection) connection = 'clickhouse://{user}:{password}@{host}:{port}/{db}'.format( **conf) return connection
def main(): """Create the sockets and start listening. """ selector = selectors.DefaultSelector() log_level = get_conf("LOG_LEVEL", logging.WARNING, t=int) logging.basicConfig(level=log_level) tcp_socket = tcp.get_socket(HOST, PORT) tcp_handler = tcp.get_handler( query_dns_fn=lambda data: tls.query_dns(data, DNS_HOST, DNS_PORT), selector=selector, event=selectors.EVENT_READ) udp_socket = udp.get_socket(HOST, PORT) udp_handler = udp.get_handler( query_dns_fn=lambda data: tls.query_dns(data, DNS_HOST, DNS_PORT)) selector.register(tcp_socket, selectors.EVENT_READ, tcp_handler) logging.info("Listening on %s:%s/tcp", HOST, PORT) selector.register(udp_socket, selectors.EVENT_READ, udp_handler) logging.info("Listening on %s:%s/udp", HOST, PORT) try: while True: events = selector.select() for key, _ in events: callback = key.data callback(key.fileobj) except KeyboardInterrupt: logging.debug('Closing sockets... (press Ctrl+C again to force)') tcp_socket.close() udp_socket.close()
def main(): # get options from console. options = args() # get configuration from file. config = get_conf(options['config_file']) # create ES connection to hosts. connections.create_connection(hosts=config['elasticsearch']['hosts'], timeout=30) # create the searcher instance to find alarms, given the options from # console. searcher = Searcher(options['from'], options['query'], ttime=options['to'], per_page=500, min_priority=options['min_priority']) buckets = [ PathClassBucket( utils.build_url(config['kibana']['host'], config['kibana']['secure'])) ] # manually fetch all alarms from the searcher and pass it to every bucket. for alarm in searcher.pages(): for bucket in buckets: bucket.cherry_pick(alarm) # dump all buckets, this will print out all buckets. for bucket in buckets: bucket.dump()
def main(): ''' Run! Prints results to the screen. ''' # Parse args and define some basic params opts = conf.get_conf() args = parse_args() doc_dir = os.path.join(opts['SALT_REPO_PATH'], 'doc') man_build_dir = os.path.join(doc_dir, '_build', 'man') man_dir = os.path.join(doc_dir, 'man') old = args.old_version print('Building man pages in directory: {0}'.format(doc_dir)) # Check out master branch _cmd_run(['make', 'man', '-C', doc_dir]) print('Copying new man files from {0} to {1}'.format( man_build_dir, man_dir)) for file_ in os.listdir(man_build_dir): file_path = os.path.join(man_build_dir, file_) print('Copying file: {0}'.format(file_path)) shutil.copy(file_path, man_dir) for file_ in os.listdir(man_dir): _replace_txt(os.path.join(man_dir, file_), old='"{0}*"'.format(old), new='"{0}" '.format(args.version), regex=True) print('Adding Salt Version {0} to file: {1}'.format( args.version, file_))
def import_button_click(self): """ Import button click handler """ cfg = get_conf(askopenfilename()) for k in self.input_variables: if k in cfg: self.input_variables[k].set(cfg[k])
def get_objects(project,fixver=None,obj_id=None,fields=None): cnf = get_conf('JIRA') server = cnf['server_url'] user_name = cnf['username'] pwd = cnf['password'] objs=[] obj={} try: if obj_id: jql_str ='issue = "{}" and issuetype in ("Story","Bug") '.format(obj_id) else: jql_str ='project = "{}" and issuetype in ("Story","Bug") and fixVersion = "{}" '.format(project,fixver) jira = JIRA(server=server,basic_auth=(user_name,pwd)) if not fields: fields='summary,reporter,issuetype,project,fixVersions,customfield_10401,customfield_10402' else: if fields.find('fixVersions') <0: fields+=',fixVersions' object_list = jira.search_issues(jql_str=jql_str,fields=fields,maxResults=200) print(len(object_list)) for i in range(0,len(object_list)): fix_ver = object_list[i].fields.fixVersions if (fix_ver and str(fix_ver[0]).strip()==str(fixver).strip()) or (obj_id): obj['object_id']=object_list[i].key obj['object_desc']=object_list[i].fields.summary obj['object_rel']=str(fix_ver[0]).strip() if object_list[i].fields.project: obj['object_track']=object_list[i].fields.project.name if object_list[i].fields.customfield_10401: lst = object_list[i].fields.customfield_10401 tmp = '' for k in lst: tmp+=k.displayName+' | ' obj['object_dev'] = tmp else: obj['object_dev']=None if object_list[i].fields.customfield_10402: lst = object_list[i].fields.customfield_10402 tmp = '' for k in lst: tmp+=k.displayName+' | ' obj['object_qa'] = tmp else: obj['object_qa']=None if object_list[i].fields.issuetype: obj['object_type']=object_list[i].fields.issuetype.name objs.append(obj.copy()) obj.clear() except Exception as e: print(e) finally: jira.close() return objs
def __init__(self, http_method, url, **kwargs): self.__retry_times = 1 self.__retry_max = get_conf('base', 'BASE').get('curl_retry_max', 3) self.res = None self.reset_retry() self._http_method = http_method self._url = url kwargs['verify'] = False self._kwargs = kwargs self._data = kwargs.get('json', kwargs.get('data'))
def main(): ''' Run! Prints results to the screen. ''' # Parse args and define some basic params opts = conf.get_conf() args = parse_args() new_stable = args.new_latest old_stable = args.old_latest new_prev = args.new_previous old_prev = args.old_previous git_dir = '--git-dir={0}/.git'.format(opts['SALT_REPO_PATH']) work_tree = '--work-tree={0}'.format(opts['SALT_REPO_PATH']) file_name = '{0}/doc/conf.py'.format(opts['SALT_REPO_PATH']) for branch in opts['SALT_BRANCHES']: print('Updating release version for {0}'.format(branch)) # Check out base branch _cmd_run(['git', git_dir, work_tree, 'checkout', branch]) # Create a new branch branch_name = 'update_version_doc_{0}'.format(branch) _cmd_run(['git', git_dir, work_tree, 'checkout', '-b', branch_name]) print('New branch: {0}'.format(branch_name)) # Update release version for "latest" if new_stable: print('Replacing {0} with {1} in branch {2}'.format( old_stable, new_stable, branch)) _replace_txt(file_name, old_stable, new_stable) # Update release version for "previous" if new_prev: print('Replacing {0} with {1} in branch {2}'.format( old_prev, new_prev, branch)) _replace_txt(file_name, old_prev, new_prev) # Set the commit title commit_msg = 'Update release versions for the {0} branch'.format( branch) # Add files to git _cmd_run(['git', git_dir, work_tree, 'add', 'doc/conf.py']) print('Committing change and pushing branch {0} to {1}\n'.format( branch_name, opts['USER_REMOTE'])) _cmd_run(['git', git_dir, work_tree, 'commit', '-m', commit_msg]) _cmd_run([ 'git', git_dir, work_tree, 'push', opts['USER_REMOTE'], branch_name ])
def main(): global exp_score hadoop_conf = conf.get_conf('../conf/hadoop.conf', '=') hadoop_bin = hadoop_conf['HADOOP_BIN'] pig_bin = hadoop_conf['PIG_BIN'] hadoop_jar = hadoop_conf['HADOOP_JAR'] hadoop_kpi = hadoop_conf['HADOOP_KPI'] hadoop_log = hadoop_conf['HADOOP_LOG'] hadoop_log_file_name = hadoop_conf['HADOOP_LOG_FILE_NAME'] t_day = (datetime.datetime.now() - datetime.timedelta(days=2)).strftime('%Y%m%d') #t_day = time.strftime('%Y%m%d',time.localtime(time.time())) input_path = hadoop_log + '/' + t_day output_path = hadoop_kpi + '/' + t_day + '/' + t_day domain_cmd = pig_bin + ' -p inputPU=' + output_path + '/BasicField' + ' -p inputPT=' + output_path + '/PageTimeLength' + \ ' -p time=' + t_day + ' kpi_domain_d.pig' page_cmd = pig_bin + ' -p inputPU=' + output_path + '/BasicField' + ' -p inputPT=' + output_path + '/PageTimeLength' + \ ' -p time=' + t_day + ' kpi_page_d.pig' source_analysis_cmd = pig_bin + ' -p inputPU=' + output_path + '/BasicField' + ' -p inputPT=' + output_path + '/PageTimeLength' + \ ' -p time=' + t_day + ' kpi_source_analysis.pig' region_cmd = pig_bin + ' -p inputPU=' + output_path + '/BasicField' + ' -p inputPT=' + output_path + '/PageTimeLength' + \ ' -p time=' + t_day + ' kpi_region_distribute.pig' hot_cmd = pig_bin + ' -p inputHeat=' + output_path + '/HeatMap' + ' -p inputHotLink=' + output_path + '/HotLink' + \ ' -p time=' + t_day + ' kpi_heatmap_d.pig' ads_cmd = pig_bin + ' -p inputAds=' + output_path + '/Ads' + ' -p inputAdsPT=' + output_path + '/AdsTimeLength' + \ ' -p time=' + t_day + ' kpi_ads_d.pig' loops_cmd = [domain_cmd, page_cmd, source_analysis_cmd, region_cmd, hot_cmd, ads_cmd] threads = [] nloops= range(len(loops_cmd)) for i in nloops: t = threading.Thread(target=loop, args=(i+1, loops_cmd[i], time.time())) threads.append(t) for i in nloops:#开始线程 threads[i].setDaemon(True) threads[i].start() for i in nloops: #等待所有的子线程执行结束(最多等待3秒,默认为None,无限等待) threads[i].join(timeout=None) print('loop',i+1,'is alive',threads[i].isAlive()) print('loop',i+1,'is daemon',threads[i].isDaemon())
def get_from_file(filename): """get_md5 - calculates md5 sum for a file""" # system_encoding = sys.getfilesystemencoding() # f = open(filename.decode(system_encoding), "rb") f = open(filename, mode='rb') m = hashlib.md5() block_size = get_conf()['checksum']['md5']['block_size'] while True: # Don't read the entire file at once... data = f.read(block_size) if len(data) == 0: break m.update(data) res = m.hexdigest() return res
def main(): ''' Run! ''' # Parse args and define some basic params opts = conf.get_conf() args = parse_args() if args.list_msg: _list_msgs(opts) send_email(args.msg, args.salt_ver, opts, args, sender=args.sender, receiver=args.receiver)
def main(): # pre process check_python_version() conf = get_conf(get_conf_path()) # main process opts = get_args() if opts.plugin == "normal": handler = get_handler(opts) else: plugin_translator, handler = get_plugin(opts) if plugin_translator: TRANSLATE_API[opts.plugin] = plugin_translator opts.api = opts.api if opts.api else opts.plugin try: t = TRANSLATE_API[opts.api](opts.lang_from, opts.lang_to, handler) t.set_parameter_from_conf(conf) t.call_method_with_handler() except IkazuchiError as err: print err
def run(self, ctx): # 还原上次挂起 data_source = self.__data_source data_source.get_handler() # 自动数秒 wait_seconds = get_conf('base', 'BASE').get('source_wait_second', 10) if not self.boot_conf.get('source_topic'): raise ValueError('ERROR SOURCE TOPIC') try: while self._running: position = data_source.mount(ctx) if position: data_source.set_position(position) # break else: time.sleep(wait_seconds) except Exception as err: logger().error('Generator raise error: {}, job exited'.format(err)) finally: logger().info('Job finished.\r\n')
# -*- coding: utf-8 -*- #Actualización de la tabla Inciden por primera vez, llena todos los registros del año actual. import psycopg2 from conf import get_conf conn, cr, path = get_conf() #Borrar los registros del mes cr.execute( "delete from asistmil_inciden where fecha >= date_trunc('year', now())") #Son registros incompletos todos los que su campo horario empiecen con un dígito, y les corresponde el tipo 0355 cr.execute("""insert into asistmil_inciden(empleado,fecha,tipo,secuencia) select empleado,fecha,'0355',0 from asistmil_incidencias where substring(horario from 1 for 1) in ('1','2','3','4','5','6','7','8','9') and registros=1 and extract(year from fecha) = extract(year from now()) and fecha <= now()""") #Los registros de la tabla Autorizaciones entran con Tipo igual a la clave del justificante cr.execute( """insert into asistmil_inciden(empleado,fecha,tipo,tiempo,secuencia) select empleado,fecha,lpad(justificante::text, 4, '0'),0,0 from asistmil_autorizaciones where extract(year from fecha) = extract(year from now()) and fecha <= now()""") #Los retardos tienen tipo 0599 y se guarda el tiempo cr.execute( """insert into asistmil_inciden(empleado,fecha,tipo,tiempo,secuencia)
import conf debug = False def run_cmd(cmd): global debug print(cmd) if debug: pass else: os.system(cmd) if __name__ == '__main__': hadoop_conf = conf.get_conf('../conf/hadoop.conf', '=') hadoop_bin = hadoop_conf['HADOOP_BIN'] pig_bin = hadoop_conf['PIG_BIN'] hadoop_jar = hadoop_conf['HADOOP_JAR'] hadoop_kpi = hadoop_conf['HADOOP_KPI'] hadoop_log = hadoop_conf['HADOOP_LOG'] hadoop_log_file_name = hadoop_conf['HADOOP_LOG_FILE_NAME'] # t_day = time.strftime('%Y%m%d',time.localtime(time.time())) t_day = (datetime.datetime.now() - datetime.timedelta(days=0)).strftime('%Y%m%d') t_time = (datetime.datetime.now() - datetime.timedelta(days=0, hours=2)).strftime('%Y%m%d%H') input_path = hadoop_log + '/' + t_day + '/' + hadoop_log_file_name + t_time output_path = hadoop_kpi + '/' + t_day + '/' + t_time
#coding:utf-8 from conf import get_conf import monitor import sched import time def run_monitor(conf): monitor.MachineMonitor(conf).scan() if __name__ == '__main__': conf = get_conf('monitor.conf') scheduler = sched.scheduler(time.time, time.sleep) scan_interval = int(conf.get('monitor', 'scan_interval')) def monitor_action(): scheduler.enterabs(time.time() + scan_interval, 0, monitor_action, ()) run_monitor(conf) monitor_action() scheduler.run()
import random import hashlib import json import time import uuid from urllib import request, parse from conf import get_conf __youdao_fanyi_api_base_url = 'https://openapi.youdao.com/api' __youdao_conf = get_conf('youdao') def truncate(q): if q is None: return None size = len(q) return q if size <= 20 else q[0:10] + str(size) + q[size - 10:size] def youdao_fanyi_query(q, f, t): if __youdao_conf is None: return app_key = __youdao_conf['fanyi_app_key'] secret_key = __youdao_conf['fanyi_secret_key'] salt = str(uuid.uuid1()) curtime = str(int(time.time())) sign = app_key + truncate(q) + salt + curtime + secret_key
def __init__(self, conf='base'): """初始化构造函数""" self.__rabbitmq_conf = get_conf('base', 'RABBITMQ').get(conf) self.connection = None
def main(t_day): hadoop_conf = conf.get_conf('../conf/hadoop.conf', '=') hadoop_bin = hadoop_conf['HADOOP_BIN'] pig_bin = hadoop_conf['PIG_BIN'] hadoop_jar = hadoop_conf['HADOOP_JAR'] hadoop_kpi = hadoop_conf['HADOOP_KPI'] hadoop_log = hadoop_conf['HADOOP_LOG'] hadoop_harlog = hadoop_conf['HADOOP_HARLOG'] input_path = hadoop_log + '/' + t_day output_path = hadoop_kpi + '/' + t_day + '/' + t_day harlog_name = t_day + '.har' harlog_path = hadoop_harlog + '/' + t_day + '.har' + '/part*' ## delete the output_path cmd = hadoop_bin + ' fs -rm -r ' + output_path run_cmd(cmd) ## hadoop archive 压缩 cmd = hadoop_bin + ' archive -archiveName ' + harlog_name + ' -p ' + input_path + ' ' + hadoop_harlog try: if run_cmd(cmd) == 0: pass #run_cmd(hadoop_bin + ' fs -rm -r ' + input_path) else: raise Exception(cmd) except Exception as ex: logger.error( '\033[91m hadoop archive error/exception occurred.\033[0m') logger.error("\033[91m Exception:%s\033[0m" % str(ex)) sys.exit(1) ## execute the mapredcue jar cmd = '{0} jar {1} com.jobs.kpi.mapred.KPIMapRed {2} {3}'.format( hadoop_bin, hadoop_jar, harlog_path, output_path) try: ctime = time.time() a = run_cmd(cmd) if a == 0: logger.info('%s success!!' % cmd) logger.info('{0} done takes: {1:.2f} seconds.'.format( cmd, time.time() - ctime)) else: raise Exception(cmd) except Exception as ex: logger.error('\nSome error/exception occurred.') logger.error("\033[91m Exception:%s\033[0m" % str(ex)) sys.exit(1) ## Determine the hdfs file exists hdfs_outdirs = frozenset( ['BasicField', 'PageTimeLength', 'Ads', 'AdsTimeLength', 'HotLink']) # 确认目录存在不管是否为空 assert_outdir_exist(hadoop_bin, output_path, hdfs_outdirs) PIG_CMD_TEMPLATE1 = '{0} -p inputPU={1} -p inputPT={2} -p time={3} {{0}}'.format( pig_bin, output_path + '/BasicField', output_path + '/PageTimeLength', t_day) PIG_CMD_TEMPLATE2 = '{0} -p inputHeat={1} -p inputHotLink={2} -p time={3} {{0}}'.format( pig_bin, output_path + '/HeatMap', output_path + '/HotLink', t_day) PIG_CMD_TEMPLATE3 = '{0} -p inputAds={1} -p inputAdsPT={2} -p time={3} {{0}}'.format( pig_bin, output_path + '/Ads', output_path + '/AdsTimeLength', t_day) domain_cmd = PIG_CMD_TEMPLATE1.format('kpi_domain_d.pig') page_cmd = PIG_CMD_TEMPLATE1.format('kpi_page_d.pig') source_analysis_cmd = PIG_CMD_TEMPLATE1.format('kpi_source_analysis.pig') region_cmd = PIG_CMD_TEMPLATE1.format('kpi_region_distribute.pig') hot_cmd = PIG_CMD_TEMPLATE2.format('kpi_heatmap_d.pig') ads_cmd = PIG_CMD_TEMPLATE3.format('kpi_ads_d.pig') loops_cmd = [ domain_cmd, page_cmd, source_analysis_cmd, region_cmd, hot_cmd, ads_cmd ] nloops = range(len(loops_cmd)) cmd_queue = multiprocessing.JoinableQueue() for i in nloops: worker = Worker(cmd_queue, i + 1, time.time()) worker.daemon = True worker.start() for i in nloops: cmd_queue.put(loops_cmd[i]) cmd_queue.join()
if __name__ == '__main__': if len(sys.argv) > 1: if not os.path.exists(sys.argv[1]): exit('error pro dir --%s--' % sys.argv[1]) file = sys.argv[1] else: file = 'app/stream/main.py' Printer.empty( '#############################################################') Printer.info('RUNNING \'[FANG]PYFLINK-FRAMEWORK\' BY FANG') Printer.empty( '#############################################################') packages = load_packages() package_str = ' '.join(packages) boot_conf_list = get_conf('boot', 'BOOT') Printer.info( '+-------------------------------------------------------------------------------+' ) for boot_conf in boot_conf_list: Printer.info( 'BOOT \'{name}\' USE MODULE \'{module}\' ALREADY SUBMIT'.format( **boot_conf)) Printer.info( '+-------------------------------------------------------------------------------+' ) stream_boot_join(file, package_str, boot_conf)
def main(t_day): hadoop_conf = conf.get_conf('../conf/hadoop.conf', '=') hadoop_bin = hadoop_conf['HADOOP_BIN'] pig_bin = hadoop_conf['PIG_BIN'] hadoop_jar = hadoop_conf['HADOOP_JAR'] hadoop_kpi = hadoop_conf['HADOOP_KPI'] hadoop_log = hadoop_conf['HADOOP_LOG'] hadoop_harlog = hadoop_conf['HADOOP_HARLOG'] input_path = hadoop_log + '/' + t_day output_path = hadoop_kpi + '/' + t_day + '/' + t_day harlog_name = t_day + '.har' harlog_path = hadoop_harlog + '/' + t_day + '.har' + '/part*' ## delete the output_path cmd = hadoop_bin + ' fs -rm -r ' + output_path run_cmd(cmd) ## hadoop archive 压缩 cmd = hadoop_bin + ' archive -archiveName ' + harlog_name + ' -p ' + input_path + ' ' + hadoop_harlog try: if run_cmd(cmd) == 0: pass #run_cmd(hadoop_bin + ' fs -rm -r ' + input_path) else: raise Exception(cmd) except Exception as ex: logger.error('\033[91m hadoop archive error/exception occurred.\033[0m') logger.error("\033[91m Exception:%s\033[0m" %str(ex)) sys.exit(1) ## execute the mapredcue jar cmd = '{0} jar {1} com.jobs.kpi.mapred.KPIMapRed {2} {3}'.format( hadoop_bin, hadoop_jar, harlog_path, output_path) try: ctime = time.time() a = run_cmd(cmd) if a == 0: logger.info('%s success!!'%cmd) logger.info('{0} done takes: {1:.2f} seconds.'.format(cmd, time.time() - ctime)) else: raise Exception(cmd) except Exception as ex: logger.error('\nSome error/exception occurred.') logger.error("\033[91m Exception:%s\033[0m" %str(ex)) sys.exit(1) ## Determine the hdfs file exists hdfs_outdirs = frozenset(['BasicField', 'PageTimeLength', 'Ads', 'AdsTimeLength', 'HotLink']) # 确认目录存在不管是否为空 assert_outdir_exist(hadoop_bin, output_path, hdfs_outdirs) PIG_CMD_TEMPLATE1 = '{0} -p inputPU={1} -p inputPT={2} -p time={3} {{0}}'.format( pig_bin, output_path+'/BasicField', output_path+'/PageTimeLength', t_day) PIG_CMD_TEMPLATE2 = '{0} -p inputHeat={1} -p inputHotLink={2} -p time={3} {{0}}'.format( pig_bin, output_path+'/HeatMap', output_path+'/HotLink', t_day) PIG_CMD_TEMPLATE3 = '{0} -p inputAds={1} -p inputAdsPT={2} -p time={3} {{0}}'.format( pig_bin, output_path+'/Ads', output_path+'/AdsTimeLength', t_day) domain_cmd = PIG_CMD_TEMPLATE1.format('kpi_domain_d.pig') page_cmd = PIG_CMD_TEMPLATE1.format('kpi_page_d.pig') source_analysis_cmd = PIG_CMD_TEMPLATE1.format('kpi_source_analysis.pig') region_cmd = PIG_CMD_TEMPLATE1.format('kpi_region_distribute.pig') hot_cmd = PIG_CMD_TEMPLATE2.format('kpi_heatmap_d.pig') ads_cmd = PIG_CMD_TEMPLATE3.format('kpi_ads_d.pig') loops_cmd = [domain_cmd, page_cmd, source_analysis_cmd, region_cmd, hot_cmd, ads_cmd] nloops = range(len(loops_cmd)) cmd_queue = multiprocessing.JoinableQueue() for i in nloops: worker = Worker(cmd_queue, i+1, time.time()) worker.daemon = True worker.start() for i in nloops: cmd_queue.put(loops_cmd[i]) cmd_queue.join()
def validate(self, token): return token == get_conf("WX_CONF")
def __combine_url(uri): host = get_conf('base', 'URL').get('gateway_domain') url = '{}{}'.format(host, uri) return url
#!/usr/bin/env python3 """Main module. Gathers all the configuration from the environment, and sets up both TCP and UDP sockets. """ import logging import selectors import udp import tcp import tls from conf import get_conf HOST = get_conf("HOST") PORT = get_conf("PORT", t=int) DNS_HOST = get_conf("DNS_HOST") DNS_PORT = get_conf("DNS_PORT", t=int) def main(): """Create the sockets and start listening. """ selector = selectors.DefaultSelector() log_level = get_conf("LOG_LEVEL", logging.WARNING, t=int) logging.basicConfig(level=log_level) tcp_socket = tcp.get_socket(HOST, PORT) tcp_handler = tcp.get_handler( query_dns_fn=lambda data: tls.query_dns(data, DNS_HOST, DNS_PORT), selector=selector,
# File: power_test.py # Author: Jayanth M ([email protected]) # Created: 3/6/2018 6:12 PM # Project: gtpower # Description: import unittest import requests import conf # ALWAYS USE DEV CONFIGURATION FOR TESTING config = conf.get_conf("dev") # RUN THE TESTS class TestPlacesApiUsingRequests(unittest.TestCase): def test_CheckUser(self): response = requests.get(config['TEST_Url'] + '/checkuser') self.assertEqual(response.status_code, 200) response = requests.get(config['TEST_Url'] + '/checkuser') # check for unauthorized here self.assertEqual(response.status_code, 403) def test_Energy(self): response = requests.get( config['TEST_Url'] + '/facilities/energy/026?start=2016-09-01 00:00:00&stop=2016-09-03 23:59:59' ) self.assertEqual(response.status_code, 200) response = requests.get(config['TEST_Url'] + '/facilities/energy/026') self.assertEqual(response.status_code, 400)
#!/usr/bin/python import sys sys.path.append("../../python-testclient") from conf import get_conf print "#ifndef MQTT_CONSTANTS_H" print "#define MQTT_CONSTANTS_H" for (k, v) in get_conf().iteritems(): if not isinstance(v, int): v = '"%s"' % v print """#define MQTT_%s %s """ % (k.upper(), v) print "#endif"
from pypxlib import Table import psycopg2 import datetime from conf import get_conf conn, cr, tpath = get_conf() table = Table(tpath + "Asignacion.DB") tup = [] for i, row in enumerate(table): if not row.Inicio: continue anio, semana, dia = row.Inicio.isocalendar() cr.execute( "select * from asistmil_asignaciones where emp=%s and semana=%s and anio=%s", (row.Registro, semana, anio)) if not cr.fetchall(): tup.append([i + 1, row.Registro, semana, anio, row.Secuencia]) if tup: args_str = ','.join(cr.mogrify("(%s,%s,%s,%s,%s)", x) for x in tup) cr.execute( "insert into asistmil_asignaciones(id, emp, semana, anio, secuencia) values " + args_str) conn.commit() conn.close()
import time import conf debug = False def run_cmd(cmd): global debug print(cmd) if debug: pass else: res = os.system(cmd) return res if __name__ == '__main__': hadoop_conf = conf.get_conf('../conf/hadoop.conf', '=') hadoop_bin = hadoop_conf['HADOOP_BIN'] pig_bin = hadoop_conf['PIG_BIN'] hadoop_jar = hadoop_conf['HADOOP_JAR'] hadoop_kpi = hadoop_conf['HADOOP_KPI'] hadoop_log = hadoop_conf['HADOOP_LOG'] hadoop_log_file_name = hadoop_conf['HADOOP_LOG_FILE_NAME'] # t_day = (datetime.datetime.now() - datetime.timedelta(days=0)).strftime('%Y%m%d') t_day = time.strftime('%Y%m%d',time.localtime(time.time())) input_path = hadoop_log + '/' + t_day output_path = hadoop_kpi + '/' + t_day + '/' + t_day ## delete the output_path cmd = hadoop_bin + ' fs -rm -r ' + output_path
def main(): ''' Run! Prints results to the screen. ''' # Parse args and define some basic params opts = conf.get_conf() args = parse_args() # replace versions repo_dir = opts['REPO_SALTSTACK_DIR'] file_dir = os.path.join(repo_dir, 'content', 'repo') if args.replace: branch = '.'.join(args.version.split('.')[:-1]) pre_ver = branch + '.' + str(int(args.version[-1]) - 1) fed_pre_ver = branch + '.' + str(int(args.version[-1]) - 2) if branch == '2017.7': file = '2017.7.md' elif branch == '2018.3': file = 'index.md' edit_file = os.path.join(file_dir, file) # now commit and push changes git_dir = '--git-dir={0}/.git'.format(opts['SSE_DIR']) work_tree = '--work-tree={0}'.format(opts['SSE_DIR']) git_cmd = ['git', git_dir, work_tree] # Check out master branch _cmd_run(git_cmd + ['checkout', 'master']) branch_name = 'repo_update_{0}-2'.format(args.version) _cmd_run(git_cmd + ['checkout', '-b', branch_name]) print('New branch: {0}'.format(branch_name)) # new date if args.date: _replace_txt(edit_file, old=pre_ver, new=args.date, replace_line='set release_date') _replace_txt(edit_file, old=pre_ver, new=args.version) # replace fedora versions for latest if 'index' in file: _replace_txt(edit_file, old=fed_pre_ver, new=pre_ver) commit_msg = 'Update release version and date for repo landing to {0}'.format( args.version) _cmd_run(git_cmd + ['add', edit_file]) print('Committing change and pushing branch {0} to {1}\n'.format( branch_name, opts['FORK_REMOTE'])) _cmd_run(git_cmd + ['commit', '-m', commit_msg]) _cmd_run(git_cmd + ['push', opts['FORK_REMOTE'], branch_name]) if args.build: print('Building Docs') print(_cmd_run(['acrylamid', 'co'], cwd=repo_dir)) if args.staging: print('Pushing to staging') upload = opts['REPO_UPLOAD_SCRIPT'] # check we have the appropriate command ret = _cmd_run(['bash', upload]) # deploy ssh_cmd = [ 'ssh', '-i', opts['REPO_DEPLOY_KEY'], '{0}@{1}'.format(opts['REPO_DEPLOY_SRV_USR'], opts['REPO_DEPLOY_SRV']), ] ret = _cmd_run(ssh_cmd + ["/root/deploy_staging_only.sh"])
def main(): ''' Run! Prints results to the screen. ''' # Parse args and define some basic params opts = conf.get_conf() args = parse_args() new_stable = args.new_latest old_stable = args.old_latest update_stable = args.latest_branch new_prev = args.new_previous old_prev = args.old_previous update_previous = args.previous_branch git_dir = '--git-dir={0}/.git'.format(opts['DOC_BUILDS_PATH']) work_tree = '--work-tree={0}'.format(opts['DOC_BUILDS_PATH']) file_name = '{0}/builddocs/print.sls'.format(opts['DOC_BUILDS_PATH']) git_cmd = ['git', git_dir, work_tree] print('Updating release version for in builddocs/print.sls') # Check out master branch _cmd_run(git_cmd + ['checkout', 'master']) # Create a new branch branch_name = 'update_print_version' _cmd_run(git_cmd + ['checkout', '-b', branch_name]) print('New branch: {0}'.format(branch_name)) # Update release version for "latest" if new_stable: print('Replacing latest version {0} with {1}'.format( old_stable, new_stable)) _replace_txt(file_name, old_stable, new_stable) # Update the base branch for "latest" if update_stable: new_base = new_stable.rsplit('.', 1)[0] old_base = old_stable.rsplit('.', 1)[0] print('Updating latest stable branch {0} with {1}'.format( old_base, new_base)) _replace_txt(file_name, old_base, new_base) # Update release version for "previous" if new_prev: print('Replacing previous version {0} with {1}'.format( old_prev, new_prev)) _replace_txt(file_name, old_prev, new_prev) # Update the base branch for "previous" if update_previous: new_base = new_prev.rsplit('.', 1)[0] old_base = old_prev.rsplit('.', 1)[0] print('Updating previous stable branch {0} with {1}'.format( old_base, new_base)) _replace_txt(file_name, old_base, new_base) # Set the commit title commit_msg = 'Update release version in print.sls file' # Add files to git _cmd_run(git_cmd + ['add', 'builddocs/print.sls']) # Commit changes and push up the branch print('Committing change and pushing branch {0} to {1}\n'.format( branch_name, opts['FORK_REMOTE'])) _cmd_run(git_cmd + ['commit', '-m', commit_msg]) _cmd_run(git_cmd + ['push', opts['FORK_REMOTE'], branch_name])
from sqlalchemy import Integer, Float, String, Text, DateTime, text from sqlalchemy.sql import select, and_ from sqlalchemy.ext.declarative import declarative_base from flasgger import Swagger import flask from flask_cas import CAS, login_required import flask_restful import conf # all configurations are stored here, change individually for development and release configurations. # Import the right configuration from conf.py, based on if it is the development environment or release environment # Run 'python3 power_api.py release' for deployment to release, 'python3 power_api.py dev' or 'python3 power_api.py' will deploy to development environment if __name__ == '__main__': env = sys.argv[1] if len( sys.argv) > 2 else 'dev' # always fall back to dev environment config = conf.get_conf(env) swagger_template = { "swagger": "2.0", "info": { "title": config['SWAGGER_Title'], "description": config['SWAGGER_Description'], "contact": { "responsibleOrganization": "GT-RNOC", "responsibleDeveloper": "RNOC Lab Staff", "email": "*****@*****.**", "url": "http://rnoc.gatech.edu/" }, # "termsOfService": "http://me.com/terms", "version": "2.0" },
import json from tencentcloud.common import credential from tencentcloud.common.profile.client_profile import ClientProfile from tencentcloud.common.profile.http_profile import HttpProfile from tencentcloud.common.exception.tencent_cloud_sdk_exception import TencentCloudSDKException from tencentcloud.tmt.v20180321 import tmt_client, models from conf import get_conf __tencent_conf = get_conf('tencent') def tencent_fanyi_query(q, f, t): if __tencent_conf is None: return app_key = __tencent_conf['fanyi_app_key'] secret_key = __tencent_conf['fanyi_secret_key'] try: cred = credential.Credential(app_key, secret_key) httpProfile = HttpProfile() httpProfile.endpoint = "tmt.tencentcloudapi.com" clientProfile = ClientProfile() clientProfile.httpProfile = httpProfile client = tmt_client.TmtClient(cred, "ap-shanghai", clientProfile) req = models.TextTranslateRequest() params = json.dumps({ 'SourceText': q, 'Source': f,