lists.clear() # 记录mongodb的数据总数 dict_count[index_name] = count if __name__ == '__main__': """ mongo 写入 es """ if len(sys.argv) <= 1: raise Exception("params error. you need add config file path") path = sys.argv[1] conf = config.init_config(path) date = datetime.datetime.today() + datetime.timedelta(days=-1) date_str = datetime.datetime(date.year, date.month, date.day).strftime("%Y-%m-%d") if len(sys.argv) >= 3: date_str = sys.argv[2] host = conf.get("mongodb").get("host") port = conf.get("mongodb").get("port") user = conf.get("mongodb").get("user") password = conf.get("mongodb").get("password") db = conf.get("mongodb").get("db") collection_names = conf.get("mongodb").get("collection") buffer_size = int(conf.get("mongodb").get("buffer_size"))
from web3 import Web3, HTTPProvider, IPCProvider from config import CASTLE_ABI, CASTLE_ADDRESS from pymemcache.client.base import Client as MemcacheClient from django.db import IntegrityError from common.daemon import Daemon from common.logger import log from common import dbmodel from etheremon_lib import config as etheremon_config from etheremon_lib.config import DATABASES from common import config as common_config from common import context common_config.init_config(etheremon_config) context.init_django('../', 'etheremon_lib.config') from etheremon_lib.models import EtheremonDB import time import json import traceback web3 = None memcache = None def update_battle(): castle_class = web3.eth.contract(abi=CASTLE_ABI) castle_contract = castle_class(CASTLE_ADDRESS) total_battle = castle_contract.call().totalBattle() # print "total_battle: ", total_battle log.info('update_battle|total_battle=%s', total_battle)
sys.path.append(sys.path[0]) if __name__ == '__main__': """ 按月归档的脚本 """ if len(sys.argv) <= 2: raise Exception("参数错误.需要传入配置文件路径和年月. 例如:./common/dev_config.ini 201901") config_path = sys.argv[1] month = sys.argv[2] conf = config.init_config(config_path) size = conf.get("es").get("max_bulk_size") min_bulk_size = conf.get("es").get("min_bulk_size") min_doc_type = conf.get("es").get("min_doc_type") es = EsHelper(conf) # 获取指定的month的index result = es.cat_indices_list(month=month) res = {} month_indexs = set() # 记录month_index的索引,用来计算插入后的count数 for i in result: month_index = i[:-2] month_indexs.add(month_index) # 指定的doc_type,设置最小的buffer size if utils.get_doc_type(i) in min_doc_type: