def ToDateTime(dateStr="",formatStr="")->DateTime:
if len(formatStr)!=len(dateStr):
raise Exception()
_year=1970
_month=1
_day=1
_hour=0
_minute=0
_second=0
if "yyyy" in formatStr:
pos=formatStr.index("yyyy")
_year=int(dateStr[pos:pos+4])
if "MM" in formatStr:
pos=formatStr.index("MM")
_month=int(dateStr[pos:pos+2])
if "dd" in formatStr:
pos=formatStr.index("dd")
_day=int(dateStr[pos:pos+2])
if "HH" in formatStr:
pos=formatStr.index("HH")
_hour=int(dateStr[pos:pos+2])
if "hh" in formatStr:
pos=formatStr.index("hh")
_hour=int(dateStr[pos:pos+2])
if "mm" in formatStr:
pos=formatStr.index("mm")
_minute=int(dateStr[pos:pos+2])
if "ss" in formatStr:
pos=formatStr.index("ss")
_second=int(dateStr[pos:pos+2])
return DateTime(_year,_month,_day,_hour,_minute,_second)
async def setCache(task_params: dict,
cacheType: FLAG,
cacheData: List[ICacheItem],
_state: Dict[str, CrawledStateMonth] = None) -> bool:
"""
储存缓存数据
task_params:任务参数
cacheType:缓存类型(CallRecordParam:emum)
cacheData:具体需要缓存的所有数据
_state:获得缓存数据时返回的已缓存数据状态
"""
try:
nowStr = DateTime.Now().ToString("yyyyMM")
nowStrOffSet = DateTime.Now().AddDays(-3).ToString("yyyyMM")
if not _state:
state = {}
else:
state = changeStateDictToObj(_state)
cacheModels: List[EsCacheModel] = []
es = EsCache()
#遍历数据，按月进行分组，并且清洗所有DateTime不正确的格式
_cmonths = state.keys()
for item in cacheData:
#判断是否开发者有漏写month(必要)
if not item.month:
#忽略记录缓存
print("检测到没有month,此条缓存记录无效")
continue
#判断如果当前月份的当前页已在state里面，说明已经缓存过，不存储
if (item.month in _cmonths):
if (item.page in state[item.month].crawledPages):
continue
#判断如果数据月份为当月，不储存当月数据
if (item.month == nowStr):
continue
if (item.month == nowStrOffSet):
continue
#分组数据
_r = ExObject(
list(
filter(
lambda x: x.month == item.month and x.page == item.
page, cacheModels)))["?0"]
item = createEsDataModel(item, cacheType)
if not item:
return False
if _r:
_r.ToOriginal().items.append(item)
else:
_item = EsCacheModel()
_item.apiClass = task_params['Area']
_item.phoneNo = task_params['UserName']
_item.dataType = cacheType.value
_item.month = item['month']
_item.page = item['page']
_item.maxPages = item['maxPages']
_item.items = []
_item.items.append(item)
_item.updateTime = DateTime().Now().TimeStampLong()
cacheModels.append(_item)
#判断数据中的所有子数据，如果有一条清洗失败，则整条不写缓存
rCacheModel = []
for cacheModel in cacheModels:
if checkData(cacheModel.items[0], cacheType):
rCacheModel.append(cacheModel)
if len(rCacheModel) == 0:
return False
r = await es.setBulkCache(rCacheModel)
if r['errors']:
errObj = ExObject(r)
reason = errObj['?items']['?0']['?index']['?error'][
'?reason'].ToString()
print("写入错误 理由:" + reason)
return False
return True
except:
print(traceback.format_exc())
print('失败')
return False
def CleanDateTime(str):
"""
字符串转DateTime对象
"""
if not str:
return DateTime()
#region 包含完整时间
r=re.search("([0-9]{4})\\-([0-9]{1,2})\\-([0-9]{1,2}) (\\d{1,2}):(\\d{1,2}):(\\d{1,2})",str)
if r:
_dt="{}-{}-{} {}:{}:{}".format(
r.group(1)
,r.group(2).zfill(2)
,r.group(3).zfill(2)
,r.group(4).zfill(2)
,r.group(5).zfill(2)
,r.group(6).zfill(2)
)
return Convert.ToDateTime(_dt,"yyyy-MM-dd HH:mm:ss")
r=re.search("([0-9]{4})\\/([0-9]{1,2})\\/([0-9]{1,2}) (\\d{1,2}):(\\d{1,2}):(\\d{1,2})",str)
if r:
_dt="{}-{}-{} {}:{}:{}".format(
r.group(1)
,r.group(2).zfill(2)
,r.group(3).zfill(2)
,r.group(4).zfill(2)
,r.group(5).zfill(2)
,r.group(6).zfill(2)
)
return Convert.ToDateTime(_dt,"yyyy-MM-dd HH:mm:ss")
r=re.search("([0-9]{4})年([0-9]{1,2})月([0-9]{1,2})日 (\\d{1,2}):(\\d{1,2}):(\\d{1,2})",str)
if r:
_dt="{}-{}-{} {}:{}:{}".format(
r.group(1)
,r.group(2).zfill(2)
,r.group(3).zfill(2)
,r.group(4).zfill(2)
,r.group(5).zfill(2)
,r.group(6).zfill(2)
)
return Convert.ToDateTime(_dt,"yyyy-MM-dd HH:mm:ss")
r=re.search("([0-9]{4})([0-9]{1,2})([0-9]{1,2}) (\\d{1,2}):(\\d{1,2}):(\\d{1,2})",str)
if r:
_dt="{}-{}-{} {}:{}:{}".format(
r.group(1)
,r.group(2).zfill(2)
,r.group(3).zfill(2)
,r.group(4).zfill(2)
,r.group(5).zfill(2)
,r.group(6).zfill(2)
)
return Convert.ToDateTime(_dt,"yyyy-MM-dd HH:mm:ss")
r=re.search("([1-2][0-9]{3})([0-1][0-9])([0-3][0-9])([0-2][0-9])([0-5][0-9])([0-5][0-9])",str)
if r:
_dt="{}-{}-{} {}:{}:{}".format(
r.group(1)
,r.group(2).zfill(2)
,r.group(3).zfill(2)
,r.group(4).zfill(2)
,r.group(5).zfill(2)
,r.group(6).zfill(2)
)
return Convert.ToDateTime(_dt,"yyyy-MM-dd HH:mm:ss")
#endregion
#region 只包含年月日
r=re.search("([0-9]{4})\\-([0-9]{1,2})\\-([0-9]{1,2})",str)
if r:
_dt="{}-{}-{} 00:00:00".format(
r.group(1)
,r.group(2).zfill(2)
,r.group(3).zfill(2)
)
return Convert.ToDateTime(_dt,"yyyy-MM-dd HH:mm:ss")
r=re.search("([0-9]{4})\\/([0-9]{1,2})\\/([0-9]{1,2})",str)
if r:
_dt="{}-{}-{} 00:00:00".format(
r.group(1)
,r.group(2).zfill(2)
,r.group(3).zfill(2)
)
return Convert.ToDateTime(_dt,"yyyy-MM-dd HH:mm:ss")
r=re.search("([0-9]{4})年([0-9]{1,2})月([0-9]{1,2})日",str)
if r:
_dt="{}-{}-{} 00:00:00".format(
r.group(1)
,r.group(2).zfill(2)
,r.group(3).zfill(2)
)
return Convert.ToDateTime(_dt,"yyyy-MM-dd HH:mm:ss")
r=re.search("([1-2][0-9]{3})([0-1][0-9])([0-3][0-9])",str)
if r:
_dt="{}-{}-{} 00:00:00".format(
r.group(1)
,r.group(2).zfill(2)
,r.group(3).zfill(2)
)
return Convert.ToDateTime(_dt,"yyyy-MM-dd HH:mm:ss")
#endregion
#region 只包含年月
r=re.search("([0-9]{4})\\-([0-9]{1,2})",str)
if r:
_dt="{}-{}-01 00:00:00".format(
r.group(1)
,r.group(2).zfill(2)
)
return Convert.ToDateTime(_dt,"yyyy-MM-dd HH:mm:ss")
r=re.search("([0-9]{4})\\/([0-9]{1,2})",str)
if r:
_dt="{}-{}-01 00:00:00".format(
r.group(1)
,r.group(2).zfill(2)
)
return Convert.ToDateTime(_dt,"yyyy-MM-dd HH:mm:ss")
r=re.search("([0-9]{4})年([0-9]{1,2})月",str)
if r:
_dt="{}-{}-01 00:00:00".format(
r.group(1)
,r.group(2).zfill(2)
)
return Convert.ToDateTime(_dt,"yyyy-MM-dd HH:mm:ss")
r=re.search("([1-2][0-9]{3})([0-1][0-9])",str)
if r:
_dt="{}-{}-01 00:00:00".format(
r.group(1)
,r.group(2).zfill(2)
)
return Convert.ToDateTime(_dt,"yyyy-MM-dd HH:mm:ss")
async def StartBill(self,month:DateTime=None):
if month:
await self.AWrite(self.logText["bill"][self.language].format(month.ToString("yyyyMM")))
else:
await self.AWrite(self.logText["bill"][self.language].format(""))
async def SmsFailed(self,month:DateTime):
await self.AWrite(self.logText["sms_f"][self.language].format(month.ToString("yyyyMM")))
async def CallSuccess(self,month:DateTime):
await self.AWrite(self.logText["call_s"][self.language].format(month.ToString("yyyyMM")))