def calculate_time(*args, **kw):
began_time = time.time()
callfunc = func(*args, **kw)
end_time = time.time()
log.debug(func.__name__ + " run time = " +
str(end_time - began_time))
return callfunc
def __init__(self, address = ADDRESS):
try:
print(address.split(','))
self._es = Elasticsearch(address.split(','))
except Exception as e:
raise
else:
log.debug('连接到Elasticsearch')
def __init__(self, ip=IP, port=PORT, db=DB):
super(MongoDB, self).__init__()
if not hasattr(self, '_db'):
try:
with pymongo.MongoClient(ip, port) as client:
self._db = client[db]
except Exception as e:
raise
else:
log.debug('连接到数据库 %s' % db)
def delete_by_id(self, table, data_id, doc_type = ''):
"""
根据给定的id,删除文档
:return:
"""
try:
self._es.delete(index = table, doc_type = doc_type or table, id = data_id)
except Exception as e:
log.debug(e)
return False
return True
def del_file(path, ignore=[]):
files = get_file_list(path, ignore)
for file in files:
try:
os.remove(file)
except Exception as e:
log.error('''
删除出错: %s
Exception : %s
''' % (file, str(e)))
else:
log.debug(file + " 删除成功")
finally:
pass
def log_function_time(func):
try:
@functools.wraps(func) # 将函数的原来属性付给新函数
def calculate_time(*args, **kw):
began_time = time.time()
callfunc = func(*args, **kw)
end_time = time.time()
log.debug(func.__name__ + " run time = " +
str(end_time - began_time))
return callfunc
return calculate_time
except:
log.debug('求取时间无效 因为函数参数不符')
return func
def update_by_id(self, table, data_id, data, doc_type = ''):
'''
@summary:
---------
@param table:
@param data_id:
@param data: {"TITLE":"xxx"} 更新的字段及值
@param doc_type:
---------
@result:
'''
try:
self._es.update(index = table, doc_type = doc_type or table, body = {"doc": data}, id = data_id)
except Exception as e:
log.debug(e)
return False
return True
def download_file(url, base_path, filename='', call_func=''):
file_path = base_path + filename
directory = os.path.dirname(file_path)
mkdir(directory)
# 进度条
def progress_callfunc(blocknum, blocksize, totalsize):
'''回调函数
@blocknum : 已经下载的数据块
@blocksize : 数据块的大小
@totalsize: 远程文件的大小
'''
percent = 100.0 * blocknum * blocksize / totalsize
if percent > 100:
percent = 100
# print ('进度条 %.2f%%' % percent, end = '\r')
sys.stdout.write('进度条 %.2f%%' % percent + "\r")
sys.stdout.flush()
if url:
try:
log.debug('''
正在下载 %s
存储路径 %s
''' % (url, file_path))
request.urlretrieve(url, file_path, progress_callfunc)
log.debug('''
下载完毕 %s
文件路径 %s
''' % (url, file_path))
call_func and call_func()
return 1
except Exception as e:
log.error(e)
return 0
else:
return 0