def cp_rewrite_toDB(): r = redisfactory.getDB(15) logger.debug("cp_rewrite_toDB begining...") rewrites = [] for key in r.keys(): d = {"CHANNEL": key, "REWRITE": r.get(key)} rewrites.append(d) print rewrites database.db_session().rewrite.insert(rewrites) logger.debug("cp_rewrite_toDB end.")
def __init__(self): dbcache = db_session() self.statistical = dbcache.statistical self.cache_user_channels = dbcache.cache_user_channels self.cache_user_channels.create_index('username') self.cache_channel_devices = dbcache.cache_channel_devices self.cache_channel_devices.create_index('channel_code') self.cache_channel_firstlayer_devices = dbcache.cache_channel_firstlayer_devices self.cache_channel_firstlayer_devices.create_index('channel_code') self.cache_user_channels_portal = dbcache.cache_user_channels_portal self.cache_user_channels_portal.create_index('username')
def get_HPCC_num(act_time, end_time): all_num = 0 all_err_num = 0 # host = "223.202.203.93" # db_name = 'bermuda' # db_ber = get_host(host, db_name) db_ber = database.db_session() connect_day = db_ber['devices_status_day'] device_one = connect_day.find_one( {'date': { '$gte': act_time, '$lt': end_time }}) dev_dict = device_one.get("HPCC") return dev_dict.get('200', 1)
def __init__(self, batch_size=3000, package_size=30): self.preload_api_size = 100 self.batch_size = batch_size self.package_size = package_size logger.debug('router start. batch_size:%s package_size:%s' % (self.batch_size, self.package_size)) self.dt = datetime.now() self.merged_urls = {} self.high_merged_urls = {} self.merged_cert = {} self.merged_cert_query = {} self.merged_transfer_cert = {} self.physical_urls = {} self.db = database.db_session() self.s1_db = database.s1_db_session()
def main(): agent = Agent(db_session()) agent.setName('mq_agent') agent.setDaemon(True) agent.run()
Returns: ''' uid = str(url.pop("_id")) url['firstLayer'] = url.pop('is_multilayer') url['layer_type'] = "two" if url.get('firstLayer') else "one" url['r_id'] = str(url.get("r_id")) url['recev_host'] = RECEIVER_HOST url['id'] = uid del url['created_time'] if username == 'sina_t' or username == 'sina_weibo' or username == 'autohome' or username == 'meipai': url['layer_type'] = "three" return url db = database.db_session() db_s1 = database.s1_db_session() @task(ignore_result=True, default_retry_delay=10, max_retries=3) def submit(refresh_task, urls): ''' 提交任务到消息队列 Parameters ---------- refresh_task : 任务 ignore_result 设置任务存储状态,如果为True,不存状态,也查询不了返回值 default_retry_delay 设置重试提交到消息队列间隔时间,默认10 分钟,单位为秒 max_retries 设置重试次数,默认为3
def main(): beat = Beat(db_session(), query_db_session()) beat.setName('retry_beat') beat.setDaemon(True) beat.run()
import datetime, sys import logging import traceback, json from core import database from core import sendEmail LOG_FILENAME = '/Application/bermuda3/logs/fail_details_user.log' logging.basicConfig( filename=LOG_FILENAME, format= '%(asctime)s - %(name)s - %(levelname)s - %(process)d - Line:%(lineno)d - %(message)s', level=logging.INFO) logger = logging.getLogger('fail_url_task') conn = database.db_session() class details_fails_urls(object): def __init__(self, conn, begin_date, end_date, username, to_add): super(details_fails_urls, self).__init__() self.begin_date = begin_date self.end_date = end_date self.connect = conn self.username = username self.to_add = to_add self.collection_url = '' self.collection_dev = '' def get_fail_urls(self): find_url_dic = {
from core import database import sys from core.database import db_session import os LOG_FILENAME = '/Application/bermuda3/logs/fail_details.log' logging.basicConfig(filename=LOG_FILENAME, format='%(asctime)s - %(name)s - %(levelname)s - %(process)d - Line:%(lineno)d - %(message)s', level=logging.INFO) logger = logging.getLogger('fail_url_task') logger.setLevel(logging.DEBUG) # conn = database.db_session() # conn = ReplicaSetConnection('%s:27017,%s:27017,%s:27017' % ('172.16.12.136', '172.16.12.135', '172.16.12.134'), replicaSet ='bermuda_db')['bermuda'] conn =db_session() # conn = Connection("223.202.52.135", 27017)['bermuda'] class details_fails_urls(object): """TTL: db.ref_err.ensureIndex( { "datetime": -1 }, { expireAfterSeconds: 60*60*24*7 }) db.ref_err.ensureIndex({"uid":1 },{"unique":true}) db.ref_err.ensureIndex({"channel_code":1}) db.ref_err.ensureIndex({"failed.code":1}) db.ref_err.ensureIndex({"username":1}) """ def __init__(self, conn, begin_date, end_date): super(details_fails_urls, self).__init__() self.begin_date = begin_date self.end_date = end_date self.connect = conn self.collection_url = ''
from core.config import config # try: # from pymongo import ReplicaSetConnection as MongoClient # except: # from pymongo import MongoClient as MongoClient import logging import traceback from core.database import db_session import os file_path = config.get('success_definition_strategy', 'basic_info_file') # uri ='mongodb://*****:*****@%s:27017,%s:27017,%s:27017/bermuda?replicaSet=bermuda_db' % ('172.16.12.136', '172.16.12.135', '172.16.12.134') # uri = 'mongodb://*****:*****@%s:29017/bermuda' %('172.16.21.205') # conn_url = MongoClient(uri)['bermuda'] conn_url =db_session() LOG_FILENAME = '/Application/bermuda3/logs/basic_info.log' logging.basicConfig(filename=LOG_FILENAME, format='%(asctime)s - %(name)s - %(levelname)s - %(process)d - Line:%(lineno)d - %(message)s', level=logging.INFO) logger = logging.getLogger('basic_info') logger.setLevel(logging.DEBUG) def get_basic_info(): fp = codecs.open(file_path,'r',encoding='utf-8') basic_info=[] key_list = ['Province','City','Zone','Provinc'] for line in fp: info_dic ={}
}).count(): verify.verify(task.get("urls"), db, 'FINISHED') else: db.error_task.save(task) if task.get('retryCount') == 1: logger.debug( "retryCount:3, status:FAILED,task_id : %s dev_id : %s..." % (task.get("_id"), task.get("dev_id"))) verify.verify(task.get("urls"), db, 'FAILED') queue.put_json2("error_task", [get_queue(task)]) def get_queue(task): queue = { "_id": str(task.get("_id")), "channel_code": task.get("urls")[0].get("channel_code"), 'host': task.get("host") } queue['urls'] = [{ "url": u.get("url"), "username": u.get("username") } for u in task.get("urls")] return queue if __name__ == "__main__": beat = Beat(database.db_session(), database.query_db_session()) beat.setName('retry_beat') beat.setDaemon(True) beat.run()
for device in devs_dict.keys(): mailContent += device + ":\n" for url in devs_dict.get(device): mailContent += url.get("url") + "\n" mailContent += u"\n" logger.debug(mailContent) return mailContent def getCustomerUrl(self, username): return [ url for url in self.db.url.find({ "username": username, "status": 'FINISHED', "isMonitor": None, 'created_time': { '$gte': datetime.now() - timedelta(minutes=20) } }) ] def updateUrl(self, url): url["isMonitor"] = 0 self.db.url.save(url) if __name__ == "__main__": monitor = Monitor(database.db_session()) monitor.setName('customer_monitor') monitor.setDaemon(True) monitor.run()
def run(): global DEVICES global MAX_COUNT hour = datetime.datetime.now().hour failed_devices = make_data() logger.debug('failed_devices %s' % (failed_devices)) bak_data() if hour != 0 and not failed_devices: #print hour #print 'no failed devices' #os._exit(0) return #邮件列表配置 all_info = db_session().email_management.find( {'failed_type': 'alarm_link_failed_devices'}) for _info in all_info: #print '_info', _info DEVICES = _info.get('devices') if not DEVICES: #print 'c1-----------------' continue MAX_COUNT = _info.get('threshold') if not MAX_COUNT: #print 'c2-----------------' continue ads_to = _info.get('email_address', []) if not ads_to: #print 'c3-----------------' continue rate = _info.get('rate') if not rate: #print 'c4-----------------' continue can_email_hour = range(0, 23 + 1, int(rate)) if hour not in can_email_hour: #print 'c5-----------------' #print can_email_hour continue if rate == 24: #汇总邮件 _day_key = (datetime.datetime.now() - datetime.timedelta(days=1)).strftime('%Y-%m-%d') all_failed_devices = make_all_data(_day_key) _f = filter(filter_data, all_failed_devices.items()) else: _f = filter(filter_data, failed_devices.items()) # print 'DEVICES', DEVICES # print 'MAX_COUNT', MAX_COUNT # print 'rate', rate if not _f: #print 'c6-----------------' continue #print '_f', _f _content = make_content(dict(_f)) logger.debug('ads_to %s' % (ads_to)) logger.debug('_content %s' % (_content)) send_mail(ads_from, ads_to, '刷新下发设备异常告警', _content) os._exit(0)
def __init__(self, batch_size=100): self.batch_size = batch_size logger.debug('timer start. batch_size:%s !' % self.batch_size) self.dt = datetime.now() self.query_db = database.query_db_session() self.db = database.db_session()
def main(): scanOverTimeTrace(database.db_session())
@contact: [email protected] @site: @software: PyCharm @file: preload_channel_script.py @time: 16-11-8 上午11:18 """ import xlrd import logging import traceback from core import redisfactory, rcmsapi from core.database import query_db_session, db_session from datetime import datetime import sys PRELOAD_DEVS = redisfactory.getDB(5) db = db_session() # LOG_FILENAME = '/Application/bermuda/logs/autodesk_postal.log' LOG_FILENAME = '/home/rubin/logs/preload_channel_script.log' # logging.basicConfig(filename=LOG_FILENAME, # format='%(asctime)s - %(name)s - %(levelname)s - %(process)d - Line:%(lineno)d - %(message)s', # level=logging.INFO) formatter = logging.Formatter( "%(asctime)s - %(name)s - %(levelname)s - %(process)d - Line:%(lineno)d - %(message)s" ) fh = logging.FileHandler(LOG_FILENAME) fh.setFormatter(formatter) logger = logging.getLogger('preload_channel_script') logger.addHandler(fh) logger.setLevel(logging.DEBUG)
def __init__(self): self.dt = datetime.now() self.query_db = database.query_db_session() self.db = database.db_session()