def findCurrFunction(self, fileContent, idx):
pattern4Function = re.compile(r'\s*(function|modifier|contract|constructor)\(*\W')
pattern4SingleLine = re.compile(r'\W\sfunction\W.*\;')
pattern4MultiLine = re.compile(r'\W\sfunction\s')
block = []
currLine = {'line': fileContent[idx], 'idx': idx}
result4SingleLine = pattern4SingleLine.findall(currLine['line'])
result4MultiLine = pattern4MultiLine.findall(currLine['line'])
if result4SingleLine:
block.append(currLine)
return block
elif result4MultiLine and not result4SingleLine:
pattern4LeftParenthesis = re.compile(r'(\{)')
result4LeftParenthesis = pattern4LeftParenthesis.findall(fileContent[idx])
while not result4LeftParenthesis:
block.append({'line': fileContent[idx], 'idx': idx})
idx += 1
result4LeftParenthesis = pattern4LeftParenthesis.findall(fileContent[idx])
block2 = Infos().getCurrBlock(fileContent, idx)
for line in block2:
block.append(line)
return block
block = Infos().getCurrBlock(fileContent, idx)
result4Function = pattern4Function.findall(block[0]['line'])
while not result4Function:
block = Infos().getCurrBlock(fileContent, idx)
result4Function = pattern4Function.findall(block[0]['line'])
idx = block[0]['idx'] - 1
return block
def __init__(self):
self._lock = threading.Lock()
self.live_infos = Infos()
self.base_path = os.path.join(
os.path.dirname(os.path.dirname(os.path.realpath(__file__))),
'history')
self.init()
logger.debug('History模块初始化完成')
def __init__(self):
self.console = Console(force_terminal=True, color_system='truecolor')
self.console._environ['TERM'] = 'SMART'
self._lock = threading.Lock()
self.live_infos = Infos()
self.last_time = datetime.datetime.now()
self.last_net_sent = 0.0
self.last_net_recv = 0.0
def __init__(self):
self._lock = threading.Lock()
self._lock2 = threading.Lock()
self.config = Config()
self.infos = Infos()
self.queue = []
self.qname = ''
self.func = lambda x:time.sleep(400)
self.threadRecorder = threadRecorder()
self.base_num = 0
def __init__(self):
self.config = Config()
self.cookies = login()
logger.info(self.cookies)
self.base_path = os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))),
self.config.config['output']['path'])
self.live_infos = Infos()
self.display = Display()
self.decoder = Decoder()
self.uploader = Upload()
self.threadRecorder = threadRecorder()
logger.info('基路径:%s' % (self.base_path))
self.load_room_info()
self.get_live_url()
logger.info('初始化完成')
def getVarDict(self, contractDict, contractFunctionDict, statementDict, fileContent):
globalVarDict, funcVarDict = {}, {}
pattern4Struct = re.compile(r'\Astruct\s+(\w+)\s*\{')
#pattern4Function = re.compile(r'\Afunction\W+([^\(]*)')
pattern4Variables = re.compile(r'(\W|\A)(address\s+payable|address|bool|string|int|int8|int16|int24|int32|int40|int48|int56|int64|int72|int80|int88|int96|int104|int112|int120|int128|int136|int144|int152|int160|int168|int176|int184|int192|int200|int208|int216|int224|int232|int240|int248|int256|uint|uint8|uint16|uint24|uint32|uint40|uint48|uint56|uint64|uint72|uint80|uint88|uint96|uint104|uint112|uint120|uint128|uint136|uint144|uint152|uint160|uint168|uint176|uint184|uint192|uint200|uint208|uint216|uint224|uint232|uint240|uint248|uint256|byte|bytes|bytes1|bytes2|bytes3|bytes4|bytes5|bytes6|bytes7|bytes8|bytes9|bytes10|bytes11|bytes12|bytes13|bytes14|bytes15|bytes16|bytes17|bytes18|bytes19|bytes20|bytes21|bytes22|bytes23|bytes24|bytes25|bytes26|bytes27|bytes28|bytes29|bytes30|bytes31|bytes32|fixed|(fixed[0-9]+x[0-9]+)|ufixed|(ufixed[0-9]+x[0-9]+))(\[\])?\s+(memory\s+|constant\s+|storage\s+|payable\s+)?(public\s+|private\s+|internal\s+)?(memory\s+|constant\s+|storage\s+|payable\s+)?(\w+)')
getInfo = Infos()
for contract in contractDict.items():
bufferBlock = []
funcVarDict[contract[0]] = []
globalVarDict[contract[0]] = []
for function in contractFunctionDict.items():
if function[0].split('|')[0] == contract[0]:
functionBlock = getInfo.getCurrBlock(fileContent, function[1]['idx'])
bufferBlock = bufferBlock + functionBlock
result4ArgvVar = pattern4Variables.findall(functionBlock[0]['line'].strip())
lengthArgvs = len(result4ArgvVar)
while lengthArgvs:
if result4ArgvVar[0][-1] not in funcVarDict[contract[0]]:
funcVarDict[contract[0]].append(result4ArgvVar[0][-1])
result4ArgvVar = result4ArgvVar[1:]
lengthArgvs = lengthArgvs - 1
for funcLine in functionBlock[1:]:
result4LocalVar = pattern4Variables.findall(funcLine['line'].strip())
lengthLocalVar = len(result4LocalVar)
while lengthLocalVar:
if result4LocalVar[0][-1] not in funcVarDict[contract[0]]:
funcVarDict[contract[0]].append(result4LocalVar[0][-1])
result4LocalVar = result4LocalVar[1:]
lengthLocalVar = lengthLocalVar - 1
if statementDict.__contains__(contract[0]):
for statLine in statementDict[contract[0]]:
result4Struct = pattern4Struct.findall(statLine['line'].strip())
#result4Function = pattern4Function.findall(statLine['line'].strip())
#if result4Struct or result4Function:
if result4Struct:
continue
result4GlobalVar = pattern4Variables.findall(statLine['line'].strip())
lengthGlobalVar = len(result4GlobalVar)
#if result4GlobalVar:
# print(result4GlobalVar, result4GlobalVar[0][-1])
# print(statLine['line'])
while lengthGlobalVar:
if result4GlobalVar[0][-1] not in globalVarDict[contract[0]]:
globalVarDict[contract[0]].append(result4GlobalVar[0][-1])
result4GlobalVar = result4GlobalVar[1:]
lengthGlobalVar = lengthGlobalVar - 1
return globalVarDict, funcVarDict
def saveInstantiated(self, fileContent, reservedList, saveList, contractList, contractDict, mainContract): #print(contractDict) for s in saveList: #print(s) if s != mainContract['contract']: block = Infos().getCurrBlock(fileContent, contractDict[s]['idx']) for b in block: if b not in reservedList: reservedList.append(b) return reservedList
def moveConstructFunc(self, contractFunctionDict, mainContract,
fileContent):
mainBlock = Infos().getCurrBlock(fileContent, mainContract['idx'])
minIdx = mainBlock[-1]['idx']
constructFuncIdx = -1
for item in contractFunctionDict.keys():
if item.split("|")[0] == mainContract['contract']:
if contractFunctionDict[item]['idx'] < minIdx:
minIdx = contractFunctionDict[item]['idx']
if item.split("|")[1] == mainContract['contract']:
constructFuncIdx = contractFunctionDict[item]['idx']
if constructFuncIdx != -1 and minIdx < constructFuncIdx:
funcBlock = Infos().getCurrBlock(fileContent, constructFuncIdx)
for f in funcBlock:
del fileContent[f['idx']]
fileContent.insert(minIdx, f['line'])
minIdx = minIdx + 1
return fileContent
def saveGlobalVar(self, fileContent, reservedList, contractFunctionDict, contractDict):
pattern4Contract = re.compile(r'\Acontract\s(\w*)')
rContractList = []
for rLine in reservedList:
result4Contract = pattern4Contract.findall(rLine['line'].strip())
if result4Contract:
rContractList.append(result4Contract[0])
for contract in rContractList:
bufferList = []
contractBlock = Infos().getCurrBlock(fileContent, contractDict[contract]['idx'])
for function in contractFunctionDict.keys():
funcContract = function.split('|')[0]
if funcContract == contract:
funcBlock = Infos().getCurrBlock(fileContent, contractFunctionDict[function]['idx'])
bufferList.extend(funcBlock)
for line in contractBlock:
if line not in bufferList and line not in reservedList:
reservedList.append(line)
return reservedList
def saveRelContract(self, mainContract, contractDict, fileContent):
saveList = []
saveList.append(mainContract['contract'])
pattern4contractInheritance = re.compile(r'contract\s+' +
mainContract['contract'] +
'(.*)\{')
pattern4Instantiation1 = re.compile(r'\W\s*new\s+(\w*)\s*\(')
pattern4ElementaryTypeName = re.compile(
r'(address|bool|string|int|int8|int16|int24|int32|int40|int48|int56|int64|int72|int80|int88|int96|int104|int112|int120|int128|int136|int144|int152|int160|int168|int176|int184|int192|int200|int208|int216|int224|int232|int240|int248|int256|uint|uint8|uint16|uint24|uint32|uint40|uint48|uint56|uint64|uint72|uint80|uint88|uint96|uint104|uint112|uint120|uint128|uint136|uint144|uint152|uint160|uint168|uint176|uint184|uint192|uint200|uint208|uint216|uint224|uint232|uint240|uint248|uint256|byte|bytes|bytes1|bytes2|bytes3|bytes4|bytes5|bytes6|bytes7|bytes8|bytes9|bytes10|bytes11|bytes12|bytes13|bytes14|bytes15|bytes16|bytes17|bytes18|bytes19|bytes20|bytes21|bytes22|bytes23|bytes24|bytes25|bytes26|bytes27|bytes28|bytes29|bytes30|bytes31|bytes32|fixed|(fixed[0-9]+x[0-9]+)|ufixed|(ufixed[0-9]+x[0-9]+))(\s+|\s*\(|\[)'
)
fileContent[mainContract['idx']] = pattern4contractInheritance.sub(
'contract ' + mainContract['contract'] + ' {',
fileContent[mainContract['idx']])
#print(mainContract)
for line in fileContent:
#if "new " in line:
# print(line)
#print(line)
result4Instantiation1 = pattern4Instantiation1.findall(
line.strip())
result4ElementaryTypeName = pattern4ElementaryTypeName.findall(
line.strip())
if result4Instantiation1 and (
result4Instantiation1[0]
not in saveList) and not result4ElementaryTypeName:
saveList.append(result4Instantiation1[0])
#print(line)
for contract in contractDict.keys():
pattern4Instantiation2 = re.compile(r'' + contract +
'\s+\w+.*\=\s*' +
contract + '\W*\(')
result4Instantiation2 = pattern4Instantiation2.findall(line)
if result4Instantiation2 and contract not in saveList:
#print(line)
saveList.append(contract)
delLineList = []
for contract in contractDict.keys():
if contract not in saveList:
#print(fileContent[contractDict[contract]['idx']])
blockBuffer = Infos().getCurrBlock(
fileContent, contractDict[contract]['idx'])
for bLine in blockBuffer:
if bLine not in delLineList:
delLineList.append(bLine)
delLineList = sorted(delLineList, key=operator.itemgetter('idx'))
#print(delLineList)
for d in reversed(delLineList):
del fileContent[d['idx']]
#print(saveList)
return fileContent, saveList
def eventList(self, fileContent, searchPath, contractDict):
contractEventList = {}
contractEventDict = {}
pattern4Contract = re.compile(r'(contract|interface)\s+(\w*)')
pattern4Event = re.compile(r'event\s+(\w*)')
currContract = ''
currList = []
delList = []
eventLine = []
for index, line in enumerate(fileContent):
result4Contract = pattern4Contract.findall(line)
result4Event = pattern4Event.findall(line)
if result4Contract:
currContract = result4Contract[0][1]
currList = []
if result4Event and currContract:
currList.append(result4Event[0])
eventLine.append({'line': line, 'idx': index})
contractEventList[currContract] = currList
for contract in contractDict.keys():
allPath = []
for inheritC in searchPath[contract]:
if contractEventList.__contains__(inheritC):
path = contractEventList[inheritC]
allPath.extend(path)
contractEventDict[contract] = allPath
contractBlock = Infos().getCurrBlock(fileContent,
contractDict[contract]['idx'])
patternStr = ''
for a in allPath:
patternStr = patternStr + '|' + a
if allPath:
pattern4EventCall = re.compile(r'\W(' + patternStr[1:] + ')\(')
for cLine in contractBlock:
result4EventCall = pattern4EventCall.findall(cLine['line'])
if result4EventCall:
delList.append(cLine)
#print(result4EventCall)
#print(cLine)
delList = delList + eventLine
removeList = []
for fileIdx, fileLine in enumerate(fileContent):
fileItem = {'line': fileLine, 'idx': fileIdx}
if fileItem in delList:
removeList.append(fileIdx)
for r in reversed(removeList):
del fileContent[r]
return fileContent
def findBasicKeywords(self, fileContent, mainContract, contractDict, customTag):
keywords = deque()
reservedList = []
ignoreList = ['msg.sender', 'msg.value', 'this', 'uint256', 'new', 'sub', 'add', 'mul', 'div']
pattern4Lines = re.compile(r'\.(transfer|send|call)\(|(\W*(balance|balanceof|balanceOf|balances|balancesof|balancesOf)\[.*\]\s?\=[^\=])', flags=re.I)
pattern4Keyword = re.compile(r'.*\s([^\(]*\.(transfer|send|call))\((.*)\)')
pattern4Balance = re.compile(r'\Wbalance\[(.*)\]\s?\=[^\=]|\Wbalanceof\[(.*)\]\s?\=[^\=]|\Wbalances\[(.*)\]\s?\=[^\=]|\Wbalancesof\[(.*)\]\s?\=[^\=]', flags=re.I)
if customTag:
customStr4Lines = ''
customStr4Balance = ''
for c in customTag:
customStr4Lines = customStr4Lines + '|' + c
customStr4Balance = customStr4Balance + '|\W?' + c + '\[(.*)\]\s?\=[^\=]'
pattern4Lines = re.compile(r'\.(transfer|send|call)\(|(\W*(balance|balanceof|balanceOf|balances|balancesof|balancesOf' + customStr4Lines + ')\[.*\]\s?\=[^\=])', flags=re.I)
pattern4Balance = re.compile(r'\Wbalance\[(.*)\]\s?\=[^\=]|\Wbalanceof\[(.*)\]\s?\=[^\=]|\Wbalances\[(.*)\]\s?\=[^\=]|\Wbalancesof\[(.*)\]\s?\=[^\=]' + customStr4Balance , flags=re.I)
for contractPair in contractDict.items():
content = Infos().getCurrBlock(fileContent, contractPair[1]['idx'])
for line in content:
result = pattern4Lines.findall(line['line'])
key = pattern4Keyword.findall(line['line'])
balanceKey = pattern4Balance.findall(line['line'])
if result and key:
reservedList.append(line)
a = {'key': key[0][0], 'idx': line['idx'], 'contract': contractPair[0], 'contractIdx': contractPair[1]['idx'], 'searched': []}
if a not in keywords:
if a['key'] not in ignoreList: keywords.append(a)
c = key[0][2].split(', ')
for item in c:
i = {'key': item, 'idx': line['idx'], 'contract': contractPair[0], 'contractIdx': contractPair[1]['idx'], 'searched': []}
if i not in keywords:
if item not in ignoreList: keywords.append(i)
if balanceKey:
reservedList.append(line)
a = {'key': balanceKey[0][2], 'idx': line['idx'], 'contract': contractPair[0], 'contractIdx': contractPair[1]['idx'], 'searched': []}
if a not in keywords:
#print(a)
if a['key'] not in ignoreList: keywords.append(a)
for item in contractDict.keys():
for index, line in enumerate(fileContent):
if "new " + item in line:
addLine = {'line': line, 'idx': index}
if addLine not in reservedList:
reservedList.append(addLine)
return keywords, reservedList
def isIteratedFunc(self, contractFuntionStr, contractFunctionDict,
deduplicatedFile):
pattern4Function = re.compile(r'\Afunction\W+([^\(]*)')
if contractFunctionDict.__contains__(contractFuntionStr):
searchBlock = Infos().getCurrBlock(
deduplicatedFile,
contractFunctionDict[contractFuntionStr]['idx'])
result4Function = pattern4Function.findall(
searchBlock[0]['line'].strip())
currFunc = result4Function[0]
pattern4Iterate = re.compile(r'\W?' + currFunc + '\(')
for line in searchBlock[1:]:
result4Iterate = pattern4Iterate.findall(line['line'])
if result4Iterate:
return True
return False
def locateFunction(self, fileContent, reservedList, contractList):
contractSection = []
for contract in contractList:
currContract = Infos().getCurrBlock(fileContent, contract['idx'])
section = {'contract': contract['contract'], 'start': contract['idx'], 'end': currContract[-1]['idx']}
contractSection.append(section)
contractSaved = []
for contract in contractSection:
for line in reservedList:
if line['idx'] >= contract['start'] and line['idx'] <= contract['end']:
if contract not in contractSaved:
contractSaved.append(contract)
for contract in contractSaved:
start = {'line': fileContent[contract['start']], 'idx': contract['start']}
end = {'line': fileContent[contract['end']], 'idx': contract['end']}
if start not in reservedList:
reservedList.append(start)
if end not in reservedList:
reservedList.append(end)
return reservedList
def mergeReduce(self, filename, fileContent, argvs):
if argvs:
customTag = argvs.split(',')
else:
customTag = []
modifyF = Modify()
fileContent = modifyF.deleteEmit(fileContent)
fileContent = modifyF.deleteMulDiv(fileContent)
# STEP 1 Merge
getInfo = Infos()
contractList, contractDict, libDict, libFunctionDict, mainContract, modDict, contractFunctionDict, contractConstructorDict = getInfo.findAllContracts(fileContent)
#print(mainContract)
isAbnormal = False
if mainContract == "NULLNULLNULL":
isAbnormal = True
if isAbnormal:
return isAbnormal, False
contractGraph = {}
for item in contractList:
contractGraph[item['contract']] = item['inherit']
searchPath = getInfo.inheritChain(contractList, contractGraph)
fileContent = modifyF.replaceSuper(searchPath, contractFunctionDict, fileContent)
modSearchOrder, functionSearchOrder = getInfo.subChain(modDict, contractFunctionDict, searchPath)
fileContent = modifyF.eventList(fileContent, searchPath, contractDict)
#update dict info due to delete of event statement
contractList, contractDict, libDict, libFunctionDict, mainContract, modDict, contractFunctionDict, contractConstructorDict = getInfo.findAllContracts(fileContent)
fileContent = modifyF.moveConstructFunc(contractFunctionDict, mainContract, fileContent)
contractList, contractDict, libDict, libFunctionDict, mainContract, modDict, contractFunctionDict, contractConstructorDict = getInfo.findAllContracts(fileContent)
statementDict, contractGlobalVar = getInfo.varStatement(contractList, contractDict, modDict, contractFunctionDict, contractConstructorDict, fileContent)
isHighRisk = False
checkVar = VarCheck()
isHighRisk = checkVar.check(contractDict, contractFunctionDict, statementDict, searchPath, fileContent)
if isHighRisk:
return isAbnormal, isHighRisk
fileContent = modifyF.EPluribusUnum(modSearchOrder, functionSearchOrder, contractList, contractDict, modDict, contractFunctionDict, mainContract, contractConstructorDict, statementDict, searchPath, fileContent)
#delete constructor
contractList, contractDict, libDict, libFunctionDict, mainContract, modDict, contractFunctionDict, contractConstructorDict = getInfo.findAllContracts(fileContent)
fileContent = modifyF.delConstructor(fileContent, contractConstructorDict, mainContract)
contractList, contractDict, libDict, libFunctionDict, mainContract, modDict, contractFunctionDict, contractConstructorDict = getInfo.findAllContracts(fileContent)
contractElemLibDict = getInfo.libBindingDict(fileContent, libDict, libFunctionDict, contractDict)
replaceF = Replace()
fileContent = replaceF.functionReplace(contractDict, fileContent, modSearchOrder, modDict, contractFunctionDict, searchPath, contractElemLibDict, libFunctionDict)
contractList, contractDict, libDict, libFunctionDict, mainContract, modDict, contractFunctionDict, contractConstructorDict = getInfo.findAllContracts(fileContent)
fileContent, saveList = modifyF.saveRelContract(mainContract, contractDict, fileContent)
contractList, contractDict, libDict, libFunctionDict, mainContract, modDict, contractFunctionDict, contractConstructorDict = getInfo.findAllContracts(fileContent)
# STEP 2 Slicing
Slicer = Slicing()
reservedList = Slicer.reserve(fileContent, mainContract, contractDict, contractList, contractFunctionDict, customTag, saveList)
#step 4
newFile = "merged_" + filename
op = open(newFile, "w+")
# Write
op.write(str(contractList.index(mainContract)) + "\n")
op.write(mainContract['contract']+"\n")
for i in reservedList:
if i['line'].strip():
op.write(i['line'])
#for i in fileContent:
# op.write(i)
#Close files
op.close()
return False, False
class Display():
def __init__(self):
self.console = Console(force_terminal=True, color_system='truecolor')
self.console._environ['TERM'] = 'SMART'
self._lock = threading.Lock()
self.live_infos = Infos()
self.last_time = datetime.datetime.now()
self.last_net_sent = 0.0
self.last_net_recv = 0.0
def generate_info(self, row_id: int, live_info: dict) -> Info:
info = None
while info is None:
try:
if live_info is not None and 'live_status' in live_info:
info = Info(
row_id=row_id,
room_id=live_info['room_id'],
anchor=live_info['uname'],
title=live_info['title'],
live_status=live_info['live_status'],
record_status=live_info['recording'],
start_time=datetime.datetime.fromtimestamp(
live_info['live_start_time']).strftime(
'%Y-%m-%d %H:%M:%S'),
record_start_time=live_info['record_start_time'],
queue_status=live_info['queue_status'],
finish_time=live_info['finish_time'])
else:
break
except Exception as e:
continue
return info
def create_info_table(self, live_infos):
dct = {0: 0, 1: 100, 2: 50, 4: 10}
dct2 = {0: 0, 1: 100, 2: 50, 3: 30}
infos = sorted([
self.generate_info(rid, live_infos[key])
for key, rid in zip(live_infos.keys(), range(len(live_infos)))
],
key=lambda i: dct[i.live_status] * 100 + 100 * dct2[
i.record_status] - i.row_id + i.queue_status,
reverse=True)
table1 = Table("行号",
"房间ID",
"主播",
"直播标题",
"直播状态",
"录制状态",
"开播时间",
"录制时长",
"队列情况",
"完成时间",
title="%s" %
datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
box=box.SIMPLE)
for info in infos:
table1.add_row(str(info.row_id), info.room_id_map, info.anchor_map,
info.title_map, info.live_status_map,
info.record_status_map, info.start_time_map,
info.record_time, info.queue_status_map,
info.finish_time_map)
table2 = Table("CPU", "Memory", "NetSent", "NetRecv", box=box.SIMPLE)
time_now = datetime.datetime.now()
now_recv = psutil.net_io_counters().bytes_recv
now_sent = psutil.net_io_counters().bytes_sent
table2.add_row(
str(psutil.cpu_percent(None)) + '%' + ' %.2fGHz' %
(psutil.cpu_freq().current / 1000.0),
str(psutil.virtual_memory().percent) + '%' + ' %s/%s' %
(bytes2human(psutil.virtual_memory().used),
bytes2human(psutil.virtual_memory().total)),
bytes2human((now_sent - self.last_net_sent) /
(time_now - self.last_time).total_seconds()) + '/s',
bytes2human((now_recv - self.last_net_recv) /
(time_now - self.last_time).total_seconds()) + '/s')
self.last_time = time_now
self.last_net_sent = now_sent
self.last_net_recv = now_recv
return RenderGroup(table1, table2)
def run(self):
# self.console.clear()
with Live(console=self.console, auto_refresh=False) as live:
while True:
try:
live.update(self.create_info_table(self.live_infos.copy()),
refresh=True)
time.sleep(1)
except Exception as e:
logger.critical(e)
continue
class Live():
def __init__(self):
self.config = Config()
self.cookies = login()
logger.info(self.cookies)
self.base_path = os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))),
self.config.config['output']['path'])
self.live_infos = Infos()
self.display = Display()
self.decoder = Decoder()
self.uploader = Upload()
self.threadRecorder = threadRecorder()
logger.info('基路径:%s' % (self.base_path))
self.load_room_info()
self.get_live_url()
logger.info('初始化完成')
def create_duration(self, start_time, end_time):
t = datetime.datetime.now()
tt = t.strftime('%Y%m%d %H%M%S')
if start_time == end_time == '0':
return '0'
tmp = datetime.datetime.strptime(tt.split(' ')[0] + ' %s' % start_time, '%Y%m%d %H%M%S')
if t > tmp:
base_time1 = tt.split(' ')[0]
base_time2 = (t + datetime.timedelta(days=1)).strftime('%Y%m%d %H%M%S').split(' ')[0]
else:
base_time1 = (t - datetime.timedelta(days=1)).strftime('%Y%m%d %H%M%S').split(' ')[0]
base_time2 = tt.split(' ')[0]
if start_time > end_time:
start_time = '%s %s' % (base_time1, start_time)
end_time = '%s %s' % (base_time2, end_time)
else:
start_time = '%s %s' % (tt.split(' ')[0], start_time)
end_time = '%s %s' % (tt.split(' ')[0], end_time)
return '%s-%s' % (start_time, end_time)
def check_live(self, key):
duration = self.live_infos.get(key)['duration']
if duration == '0':
return True
lst = duration.split('-')
now_time = datetime.datetime.now()
if len(lst) == 2:
start_time = datetime.datetime.strptime(lst[0], '%Y%m%d %H%M%S')
end_time = datetime.datetime.strptime(lst[1], '%Y%m%d %H%M%S')
if now_time > start_time and now_time < end_time:
return True
else:
logger.debug('%s[RoomID:%s]不在直播时间段' % (self.live_infos.get(key)['uname'], key))
return False
else:
return False
def load_room_info(self):
live_infos = self.live_infos.copy()
for lst in self.config.config['live']['room_info']:
if lst[0] not in live_infos:
live_info = {}
live_info['record_start_time'] = ''
live_info['queue_status'] = 0
live_info['recording'] = 0
live_info['finish_time'] = ''
else:
live_info = live_infos[lst[0]]
live_info['need_rec'] = lst[1]
live_info['need_mask'] = lst[2]
live_info['maxsecond'] = lst[3]
live_info['need_upload'] = lst[4]
live_info['duration'] = self.create_duration(lst[5], lst[6])
live_info['cookies'] = self.cookies
live_info['base_path'] = self.base_path
self.live_infos.update(lst[0],live_info)
def load_realtime(self):
'''
实时加载配置，更新房间信息
'''
self.config.load_cfg()
# logger.info(self.config.config)
room_lst = [i[0] for i in self.config.config['live']['room_info']]
del_lst = []
for key in self.live_infos.copy():
if key not in room_lst:
del_lst.append(key)
for key in del_lst:
self.live_infos.delete(key)
self.load_room_info()
def judge_in(self, key):
room_lst = [i[0] for i in self.config.config['live']['room_info']]
if key not in room_lst:
return False
return True
def judge_download(self,key):
if not self.judge_in(key):
return False
live_info = self.live_infos.copy()[key]
if live_info['live_status'] != 1:
live_info['recording'] =0
self.live_infos.update(key,live_info)
return False
elif not self.check_live(key) and live_info['live_status'] == 1:
live_info['recording'] =2
self.live_infos.update(key,live_info)
return False
elif self.check_live(key) and live_info['live_status'] == 1 and live_info['need_rec'] == '0':
live_info['recording']=3
self.live_infos.update(key,live_info)
return False
elif live_info['live_status'] == 1 and live_info['need_rec'] == '1':
live_info['recording']=1
self.live_infos.update(key,live_info)
return True
else:
logger.warning('%s[RoomID:%s]进入了未知的分支呢' % (live_info['uname'],key))
live_info['recording'] =0
self.live_infos.update(key,live_info)
return False
def get_live_url(self):
'''
获取所有监听直播间的信息
'''
room_lst = [i[0] for i in self.config.config['live']['room_info']]
for id in room_lst:
info = None
while info is None:
try:
info = live.get_room_info(id, cookies=self.cookies)
time.sleep(0.3)
except:
logger.error('[RoomID:%s]获取信息失败，重新尝试' % (id))
continue
live_info = self.live_infos.copy()[id]
live_info['room_id'] = id
live_info['real_id'] = info['room_info']['room_id']
try:
if live_info['live_status'] != 1 and info['room_info']['live_status'] == 1:
logger.info('%s[RoomID:%s]开播了' % (live_info['uname'], id))
toaster = ToastNotifier()
toaster.show_toast("开播通知",
'%s[RoomID:%s]开播了' % (live_info['uname'], id),
icon_path=None,
duration=3)
except:
pass
try:
live_info['live_status'] = info['room_info']['live_status']
live_info['uid'] = info['room_info']['uid']
live_info['uname'] = info['anchor_info']['base_info']['uname']
live_info['save_name'] = '%s_%s.flv' % (
live_info['uname'], time.strftime("%Y%m%d%H%M%S", time.localtime()))
live_info['title'] = info['room_info']['title']
live_info['live_start_time'] = info['room_info']['live_start_time']
self.live_infos.update(id,live_info)
logger.debug(
'%s[RoomID:%s]直播状态\t%s' % (live_info['uname'], id, live_info['live_status']))
except Exception as e:
logger.critical(e)
logger.error('[RoomID:%s]房间信息更新失败' % (id))
logger.error(info)
# logger.info(self.live_infos.copy())
def get_stream(self, key):
'''
获取直播流
:param key: 房间显示id
:return: stream
'''
if not self.judge_in(key):
return None
live_info = self.live_infos.copy()[key]
logger.info('%s[RoomID:%s]获取直播流' % (live_info['uname'], key))
session = streamlink.Streamlink()
session.set_option("http-cookies", self.cookies)
session.set_option("http-headers", headers)
log_path = os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))), 'log', 'stream.log')
session.set_loglevel("debug")
session.set_logoutput(open(log_path, 'a',encoding='utf-8'))
streams = None
while streams is None:
try:
streams = session.streams('https://live.bilibili.com/%s' % key)
except:
logger.warning('%s[RoomID:%s]获取直播流失败，正在重试' % (live_info['uname'], key))
time.sleep(1)
if streams == {}:
logger.error('%s[RoomID:%s]未获取到直播流，可能是下播或者网络问题' % (live_info['uname'], key))
return None
if 'best' in streams:
logger.info('%s[RoomID:%s]获取到best直播流' % (live_info['uname'], key))
return streams['best']
elif 'source' in streams:
logger.info('%s[RoomID:%s]获取到source直播流' % (live_info['uname'], key))
return streams['source']
elif 'worst' in streams:
logger.info('%s[RoomID:%s]获取到worst直播流' % (live_info['uname'], key))
return streams['worst']
else:
logger.info('%s[RoomID:%s]未获取到直播流' % (live_info['uname'], key))
return None
def unlive(self, key, unlived):
if not self.judge_in(key):
return None
live_info = self.live_infos.copy()[key]
logger.info('%s[RoomID:%s]似乎下播了' % (live_info['uname'], key))
live_info['recording'] = 0
logger.info('%s[RoomID:%s]录制结束，录制了%.2f分钟' % (live_info['uname'], key, (
datetime.datetime.now() - datetime.datetime.strptime(
live_info['record_start_time'],
'%Y-%m-%d %H:%M:%S')).total_seconds() / 60.0))
live_info['record_start_time'] = ''
if unlived:
logger.info('%s[RoomID:%s]确认下播，加入转码上传队列' % (live_info['uname'], key))
# if live_info['need_upload'] == '1':
# live_info['filename'] = live_info['uname'] + live_info['duration'].split('-')[0].split(' ')[0]
# live_info['filepath'] = os.path.join(self.base_path, live_info['uname'], '%s_%s' % (live_info['uname'], live_info['duration'].split('-')[0].split(' ')[0]))
# if live_info['need_mask'] == '1':
# live_info['filepath'] += '_mask.mp4'
# else:
# live_info['filepath'] += '.mp4'
self.decoder.enqueue(key)
self.live_infos.update(key,live_info)
def download_live(self, key):
if not self.judge_in(key):
return None
save_path = os.path.join(self.base_path, self.live_infos.get(key)['uname'], 'recording')
logger.info('%s[RoomID:%s]准备下载直播流,保存在%s' % (self.live_infos.get(key)['uname'], key, save_path))
self.live_infos.get(key)['recording'] = 1
if not os.path.exists(save_path):
os.makedirs(save_path)
stream = self.get_stream(key)
if stream is None:
logger.error('%s[RoomID:%s]获取直播流失败' % (self.live_infos.get(key)['uname'], key))
self.live_infos.get(key)['record_start_time'] = ''
self.live_infos.get(key)['recording'] = 0
return
filename = os.path.join(save_path, self.live_infos.get(key)['save_name'])
self.live_infos.get(key)['record_start_time'] = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
try:
fd = stream.open()
except Exception as e:
self.unlive(key,unlived=False)
logger.critical('%s[RoomID:%s]fd open error' % (self.live_infos.get(key)['uname'], key))
logger.error(e)
return
with open(filename, 'wb') as f:
while self.judge_in(key) and self.live_infos.get(key)['live_status'] == 1 and self.live_infos.get(key)[
'need_rec'] == '1' and self.check_live(key):
try:
data = fd.read(1024 * 8)
if len(data) > 0:
f.write(data)
else:
fd.close()
logger.warning('%s[RoomID:%s]直播流断开,尝试重连' % (self.live_infos.get(key)['uname'], key))
stream = self.get_stream(key)
if stream is None:
logger.warning('%s[RoomID:%s]重连失败' % (self.live_infos.get(key)['uname'], key))
self.unlive(key, True)
return
else:
logger.info('%s[RoomID:%s]重连成功' % (self.live_infos.get(key)['uname'], key))
fd = stream.open()
except Exception as e:
fd.close()
self.unlive(key,unlived=False)
logger.critical('%s[RoomID:%s]遇到了什么问题' % (self.live_infos.get(key)['uname'], key))
logger.error(e)
return
fd.close()
self.unlive(key, True)
def run(self):
while True:
time.sleep(1)
self.load_realtime()
self.get_live_url()
live_infos = self.live_infos.copy()
for key in live_infos:
if live_infos[key]['recording'] != 1 and self.judge_download(key):
self.threadRecorder.add('download_live_%s' % (key),self.download_live,[key,],False)
time.sleep(0.2)
def start(self):
self.threadRecorder.add('display_run',self.display.run,None,False)
self.threadRecorder.add('decoder_run',self.decoder.run,None,False)
self.threadRecorder.add('uploader_run',self.uploader.run,None,False)
self.threadRecorder.add('live_run',self.run,None,False)
def functionReplace(self, contractDict, fileContent, modSearchOrder,
modDict, contractFunctionDict, searchPath,
contractElemLibDict, libFunctionDict):
pattern4Contract = re.compile(r'\Acontract\W+(\w*)')
pattern4Function = re.compile(r'\Afunction\W+([^\(]*)')
pattern4Library = re.compile(r'\Alibrary\s+(\w+)\s*\{')
pattern4Modifier = re.compile(r'\Amodifier\W+([^\(]*)')
pattern4Require = re.compile(r'\A(require|assert)\s*\(')
pattern4ElementaryTypeName = re.compile(
r'(address|bool|string|int|int8|int16|int24|int32|int40|int48|int56|int64|int72|int80|int88|int96|int104|int112|int120|int128|int136|int144|int152|int160|int168|int176|int184|int192|int200|int208|int216|int224|int232|int240|int248|int256|uint|uint8|uint16|uint24|uint32|uint40|uint48|uint56|uint64|uint72|uint80|uint88|uint96|uint104|uint112|uint120|uint128|uint136|uint144|uint152|uint160|uint168|uint176|uint184|uint192|uint200|uint208|uint216|uint224|uint232|uint240|uint248|uint256|byte|bytes|bytes1|bytes2|bytes3|bytes4|bytes5|bytes6|bytes7|bytes8|bytes9|bytes10|bytes11|bytes12|bytes13|bytes14|bytes15|bytes16|bytes17|bytes18|bytes19|bytes20|bytes21|bytes22|bytes23|bytes24|bytes25|bytes26|bytes27|bytes28|bytes29|bytes30|bytes31|bytes32|fixed|(fixed[0-9]+x[0-9]+)|ufixed|(ufixed[0-9]+x[0-9]+))(\s+|\s*\()'
)
pattern4Instantiation1 = re.compile(r'\W\s*new\s+(\w*)\s*\(')
funcDict = {}
contract = ''
saveContractList = []
for line in fileContent:
result4Contract = pattern4Contract.findall(line.strip())
result4Function = pattern4Function.findall(line.strip())
if result4Contract:
contract = result4Contract[0].strip()
funcDict.setdefault(contract, [])
if result4Function and contract:
func = result4Function[0].strip()
funcDict[contract].append(func)
#print(contractElemLibDict)
deduplicatedFile = self.variableDeduplication(fileContent, funcDict,
contractDict,
contractFunctionDict)
dFileCopy = copy.deepcopy(deduplicatedFile)
contractVarDict = Infos().varTypeRecord(contractDict, deduplicatedFile)
#print(contractVarDict)
pattern4CallFunc = re.compile(r'(\w+\([^()]*\))')
pattern4Constructor = re.compile(r'constructor')
#pattern4CallExternalFunction = re.compile(r'([^\w\.\[\]]\w.+)\.(\w+\([^()]*\))')
pattern4CallExternalFunction = re.compile(
r'([\w\.\[\]]+)\.(\w+\([^()]*\))')
callFunctionDict = {}
callModifierDict = {}
currContract = ''
currFunc = ''
calledFunctionList = []
delFunctionList = []
flag = 0
end = 0
while not end:
for idx, line in enumerate(dFileCopy[flag:]):
#print("o:", idx+flag)
#print(line)
varType = ""
result4Instantiation1 = pattern4Instantiation1.findall(
line.strip())
result4CallFunc = pattern4CallFunc.findall(line.strip())
result4Constructor = pattern4Constructor.findall(line.strip())
result4Contract = pattern4Contract.findall(line.strip())
result4Library = pattern4Library.findall(line.strip())
result4Require = pattern4Require.findall(line.strip())
result4CurrFunction = pattern4Function.findall(line.strip())
result4CurrModifier = pattern4Modifier.findall(line.strip())
result4CallExternalFunction = pattern4CallExternalFunction.findall(
line.strip())
result4ElementaryTypeName = pattern4ElementaryTypeName.findall(
line.strip())
if result4Instantiation1 and result4Instantiation1[
0] not in saveContractList:
saveContractList.append(result4Instantiation1[0])
if result4Contract:
currContract = result4Contract[0]
if result4Library:
currContract = ''
if result4CurrFunction and currContract:
currFunc = result4CurrFunction[0]
#print(currFunc)
callFunctionDict.setdefault(currContract + '|' + currFunc,
[])
# generate modifier dict
if modSearchOrder.__contains__(currContract):
for m in modSearchOrder[currContract]:
mod = m.split('|')[1]
patter4ModifierCall = re.compile(
r'function\s(.*)\s' + mod + '\W')
result4ModifierCall = patter4ModifierCall.findall(
line.strip())
if result4ModifierCall:
#print(line)
callModifierDict.setdefault(
currContract + '|' + currFunc, [])
if m not in callModifierDict[currContract +
'|' + currFunc]:
list = []
for i in callModifierDict[currContract +
'|' + currFunc]:
list.append(i.split('|')[1])
if mod not in list:
callModifierDict[currContract + '|' +
currFunc].append(m)
if result4Constructor:
currFunc = result4Constructor[0]
if result4CallExternalFunction and currContract and currFunc and not result4CurrFunction and not result4Contract:
caller = result4CallExternalFunction[0][0].strip()
if caller.count("[") > caller.count("]"):
caller = caller.split("[")[-1]
callFunc = result4CallExternalFunction[0][1].split('(')[0]
NcontractList, contractDict, NlibDict, NlibFunctionDict, NmainContract, NmodDict, NnewContractFunctionDict, NcontractConstructorDict = Infos(
).findAllContracts(dFileCopy)
#contractVarDict = Infos().varTypeRecord(contractDict, dFileCopy)
if caller == 'super':
for target in searchPath[currContract][1:]:
searchStr = target + '|' + callFunc
if contractFunctionDict.__contains__(searchStr):
caller = target
break
else:
pass
else:
searchBlock = Infos().getCurrBlock(
dFileCopy, contractDict[currContract]['idx'])
for k in contractDict.keys():
pattern4ContractRef = re.compile(
r'' + k + '\s+(public\s+)?' + caller + '\s*\;')
for s in searchBlock:
result4ContractRef = pattern4ContractRef.findall(
s['line'])
if result4ContractRef:
caller = k
if k not in saveContractList:
saveContractList.append(k)
if contractDict.__contains__(caller):
argvStr = result4CallExternalFunction[0][1].split(
'(')[1].replace(')', '')
if argvStr:
currArgvs = argvStr.split(',')
for cArgvIdx, cArgv in enumerate(currArgvs):
currArgvs[cArgvIdx] = cArgv.strip()
else:
currArgvs = []
callFunctionDict.setdefault(
currContract + '|' + currFunc, [])
countCall = callFunctionDict[currContract + '|' +
currFunc].count(callFunc)
callFunctionDict[currContract + '|' +
currFunc].append(callFunc)
contractFuntionStr = caller + '|' + callFunc
if contractFunctionDict.__contains__(
contractFuntionStr):
isIterated = False
isIterated = self.isIteratedFunc(
contractFuntionStr, contractFunctionDict,
deduplicatedFile)
if isIterated:
continue
#print(line)
contractVarList = [
cvk for cvk in contractVarDict.keys()
]
for cVar in contractVarList:
if cVar.split("|")[0] == caller:
vName = cVar.split("|")[1]
if not contractVarDict.__contains__(
currContract + "|" + vName):
vType = contractVarDict[cVar]["type"]
contractVarDict[currContract + "|" +
vName] = {
"var": vName,
"type": vType
}
contractElemLibList = [
celk for celk in contractElemLibDict.keys()
]
for cELib in contractElemLibList:
if cELib.split("|")[0] == caller:
ketStr = currContract + "|" + cELib.split(
"|")[1]
if not contractElemLibDict.__contains__(
ketStr):
libFunc = contractElemLibDict[cELib]
contractElemLibDict[ketStr] = libFunc
block = self.argvReplace(line, callFunc,
contractFuntionStr,
currArgvs, countCall,
contractFunctionDict,
deduplicatedFile, varType)
if contractFuntionStr not in calledFunctionList:
calledFunctionList.append(contractFuntionStr)
# indent
stringStart = len(line) - len(line.lstrip())
lineStart = 0
for bIdx, bLine in enumerate(block):
if bLine and not lineStart:
lineStart = len(bLine) - len(
bLine.lstrip())
block[bIdx] = line[0:stringStart] + bLine[
lineStart - 1:]
dFileCopy = dFileCopy[
0:flag + idx] + block + dFileCopy[flag + idx +
1:]
block = []
flag = flag + idx
break
else:
#interface call
delFunc = {
"dFunc": currContract + "|" + currFunc,
"caller": caller
}
if delFunc not in delFunctionList:
delFunctionList.append(delFunc)
else:
nCaller = caller.split("[")[0]
if contractVarDict.__contains__(currContract + "|" +
nCaller):
#library function replacement
libFunctionStr = ""
varType = contractVarDict[currContract + "|" +
nCaller]["type"]
if contractElemLibDict.__contains__(currContract +
"|" + varType):
for lFunc in contractElemLibDict[currContract +
"|" +
varType]:
if lFunc.split("|")[1] == callFunc:
libFunctionStr = lFunc
if libFunctionStr:
argvStr = result4CallFunc[0].split(
'(')[1].replace(')', '')
currArgvs = []
if argvStr:
currArgvs = result4CallFunc[0].split(
'(')[1].replace(')', '').split(',')
for cArgvIdx, cArgv in enumerate(
currArgvs):
currArgvs[cArgvIdx] = cArgv.strip()
else:
pass
currArgvs.insert(0, caller.strip())
callFunctionDict.setdefault(
currContract + '|' + currFunc, [])
countCall = callFunctionDict[
currContract + '|' +
currFunc].count(callFunc)
callFunctionDict[currContract + '|' +
currFunc].append(callFunc)
if libFunctionDict.__contains__(
libFunctionStr):
block = self.argvReplace(
line, callFunc, libFunctionStr,
currArgvs, countCall, libFunctionDict,
deduplicatedFile, varType)
if libFunctionStr not in calledFunctionList:
calledFunctionList.append(
libFunctionStr)
# indent
stringStart = len(line) - len(
line.lstrip())
lineStart = 0
for bIdx, bLine in enumerate(block):
if bLine and not lineStart:
lineStart = len(bLine) - len(
bLine.lstrip())
block[bIdx] = line[
0:stringStart] + bLine[lineStart:]
#print(block)
dFileCopy = dFileCopy[
0:flag +
idx] + block + dFileCopy[flag + idx +
1:]
block = []
flag = flag + idx
break
else:
pass
if result4CallFunc and not result4CallExternalFunction and not result4CurrFunction and not result4CurrModifier and currContract and currFunc and not result4ElementaryTypeName and not result4Contract:
#update index in contractFunctionDict
#print(currContract)
if result4CallFunc[0].split(
'(')[0] in funcDict[currContract]:
callFunc = result4CallFunc[0].split('(')[0]
argvStr = result4CallFunc[0].split('(')[1].replace(
')', '')
if argvStr:
currArgvs = result4CallFunc[0].split(
'(')[1].replace(')', '').split(',')
for cArgvIdx, cArgv in enumerate(currArgvs):
currArgvs[cArgvIdx] = cArgv.strip()
else:
currArgvs = []
callFunctionDict.setdefault(
currContract + '|' + currFunc, [])
countCall = callFunctionDict[currContract + '|' +
currFunc].count(callFunc)
callFunctionDict[currContract + '|' +
currFunc].append(callFunc)
contractFuntionStr = currContract + '|' + callFunc
isIterated = False
isIterated = self.isIteratedFunc(
contractFuntionStr, contractFunctionDict,
deduplicatedFile)
if contractFunctionDict.__contains__(
contractFuntionStr) and not isIterated:
#print(currArgvs)
#print(contractFuntionStr)
block = self.argvReplace(line, callFunc,
contractFuntionStr,
currArgvs, countCall,
contractFunctionDict,
deduplicatedFile, varType)
if contractFuntionStr not in calledFunctionList:
calledFunctionList.append(contractFuntionStr)
# indent
stringStart = len(line) - len(line.lstrip())
lineStart = 0
for bIdx, bLine in enumerate(block):
if bLine:
lineStart = len(bLine) - len(
bLine.lstrip())
block[bIdx] = bLine
lineStart = 0
dFileCopy = dFileCopy[
0:flag + idx] + block + dFileCopy[flag + idx +
1:]
block = []
flag = flag + idx
break
block = []
if idx == len(dFileCopy) - flag - 1:
end = 1
contractList, contractDict, libDict, libFunctionDict, mainContract, modDict, newContractFunctionDict, contractConstructorDict = Infos(
).findAllContracts(dFileCopy)
#print(newContractFunctionDict)
# modifier call replacement
modBufferDict = {}
for pair in callModifierDict.items():
modBlock2 = []
for modItem in pair[1]:
#modifiers without params
if not modDict[modItem]['modParams'] or not modDict[modItem][
'modParams'][0]:
modBlock = Infos().getCurrBlock(dFileCopy,
modDict[modItem]['idx'])
for mIdx, mBlk in enumerate(modBlock):
modBlock[mIdx] = mBlk['line']
modBlock = modBlock[1:-1]
#modifiers with params
else:
funcLine = dFileCopy[newContractFunctionDict[pair[0]]
['idx']]
#print(funcLine)
callMod = modItem.split('|')[1]
pattern4CallMod = re.compile(r'\s' + callMod +
'\(([^()]*)\)')
result4CallMod = pattern4CallMod.findall(funcLine)
if result4CallMod:
#print(result4CallMod)
currArgvs = result4CallMod[0].split(',')
for aIdx, a in enumerate(currArgvs):
currArgvs[aIdx] = a.strip()
countCall = 0
modBlock = self.argvReplace(funcLine, callMod, modItem,
currArgvs, countCall, modDict,
dFileCopy, varType)
modBlock2.append({"content": modBlock})
modBufferDict[pair[0]] = {
'content': modBlock2,
'idx': newContractFunctionDict[pair[0]]['idx']
}
contractList, contractDict, libDict, libFunctionDict, mainContract, modDict, newContractFunctionDict, contractConstructorDict = Infos(
).findAllContracts(dFileCopy)
sortedDict = sorted(modBufferDict.items(),
key=lambda x: x[1]['idx'],
reverse=True)
pattern4Sub = re.compile(r'\A\_\;')
for bPair in sortedDict:
funcIdx = newContractFunctionDict[bPair[0]]['idx']
for insertBlock in bPair[1]['content']:
insertB = insertBlock['content']
funcBlock = Infos().getCurrBlock(dFileCopy, funcIdx)
insertFuncBlock = [f['line'] for f in funcBlock[1:-1]]
for insertIdx, insertL in enumerate(insertB):
result4Sub = pattern4Sub.findall(insertL.strip())
if result4Sub:
insertB = insertB[
0:insertIdx] + insertFuncBlock + insertB[
insertIdx + 1:]
break
dFileCopy = dFileCopy[0:funcIdx + 1] + insertB + dFileCopy[
funcBlock[-1]['idx']:]
#discard replaced modifier & function
contractList, contractDict, libDict, libFunctionDict, mainContract, modDict, newContractFunctionDict, contractConstructorDict = Infos(
).findAllContracts(dFileCopy)
pattern4PublicFunc = re.compile(r'\Wpublic\W')
discardList = []
for dFunc in delFunctionList:
print("Delete " + dFunc["dFunc"].split("|")[1] + " in contract " +
dFunc["dFunc"].split("|")[0] +
" due to calling of interface " + dFunc["caller"] + ".")
if dFunc["dFunc"] not in calledFunctionList:
calledFunctionList.append(dFunc["dFunc"])
for cFunc in calledFunctionList:
if cFunc.split("|")[0] in saveContractList:
continue
dfuncIdx = ""
if newContractFunctionDict.__contains__(cFunc):
dfuncIdx = newContractFunctionDict[cFunc]['idx']
elif libFunctionDict.__contains__(cFunc):
dfuncIdx = libFunctionDict[cFunc]['idx']
if dfuncIdx:
descardBlock = Infos().getCurrBlock(dFileCopy, dfuncIdx)
result4PublicFunc = pattern4PublicFunc.findall(
descardBlock[0]['line'])
if not result4PublicFunc:
discardList.extend(descardBlock)
for cMod in modDict.items():
descardBlock = Infos().getCurrBlock(dFileCopy, cMod[1]['idx'])
discardList.extend(descardBlock)
removeList2 = []
for fileIdx, fileLine in enumerate(dFileCopy):
fileItem = {'line': fileLine, 'idx': fileIdx}
if fileItem in discardList:
removeList2.append(fileIdx)
for r in reversed(removeList2):
del dFileCopy[r]
return dFileCopy
def variableDeduplication(self, fileContent, funcDict, contractDict,
contractFunctionDict):
#print(contractFunctionDict)
fileContentCopy = copy.deepcopy(fileContent)
pattern4Variables = re.compile(
r'(\W|\A)(address|bool|string|int|int8|int16|int24|int32|int40|int48|int56|int64|int72|int80|int88|int96|int104|int112|int120|int128|int136|int144|int152|int160|int168|int176|int184|int192|int200|int208|int216|int224|int232|int240|int248|int256|uint|uint8|uint16|uint24|uint32|uint40|uint48|uint56|uint64|uint72|uint80|uint88|uint96|uint104|uint112|uint120|uint128|uint136|uint144|uint152|uint160|uint168|uint176|uint184|uint192|uint200|uint208|uint216|uint224|uint232|uint240|uint248|uint256|byte|bytes|bytes1|bytes2|bytes3|bytes4|bytes5|bytes6|bytes7|bytes8|bytes9|bytes10|bytes11|bytes12|bytes13|bytes14|bytes15|bytes16|bytes17|bytes18|bytes19|bytes20|bytes21|bytes22|bytes23|bytes24|bytes25|bytes26|bytes27|bytes28|bytes29|bytes30|bytes31|bytes32|fixed|(fixed[0-9]+x[0-9]+)|ufixed|(ufixed[0-9]+x[0-9]+))\s+(public\s+|private\s+|internal\s+)?(\w+)'
)
currVar = {}
for pair in funcDict.items():
#print(pair)
currContract = Infos().getCurrBlock(fileContent,
contractDict[pair[0]]['idx'])
for function in pair[1]:
varList = []
pattern4Function = re.compile(r'function\s' + function +
'\(([^()]*)\)')
for line in currContract:
result4Function = pattern4Function.findall(line['line'])
if result4Function:
funcBlock = Infos().getCurrBlock(
fileContent, line['idx'])
for b in funcBlock:
result4Variables = pattern4Variables.findall(
b['line'])
if result4Variables:
for result in result4Variables:
if result[-1] not in varList:
varList.append(result[-1])
if varList:
function = pair[0] + '|' + function
currVar[function] = varList
contractVarList = []
for funcVarPair in currVar.items():
for i, v in enumerate(funcVarPair[1]):
v = v.replace('_', 'line')
count = contractVarList.count(v)
contractVarList.append(v)
currVar[funcVarPair[0]][i] = {
currVar[funcVarPair[0]][i]: v + "X" + str(count)
}
for funcVar in currVar.items():
functionBlock = Infos().getCurrBlock(
fileContent, contractFunctionDict[funcVar[0]]['idx'])
for code in functionBlock:
for varPair in funcVar[1]:
for p in varPair.items():
l = fileContentCopy[code['idx']]
#print(p)
pattern4Argv = re.compile(r'[^\.\w]' + (p[0]) + '\W+')
result4Argv = pattern4Argv.findall(l)
if result4Argv:
#print(result4Argv)
for r in result4Argv:
keyStart = r.find(p[0])
keyEnd = r.find(p[0]) + len(p[0])
subKey = r[0:keyStart] + p[1] + r[keyEnd:]
l = l[0:l.find(r)] + subKey + l[l.find(r) +
len(r):]
#print(l)
fileContentCopy[code['idx']] = l
deduplicatedFile = fileContentCopy
return deduplicatedFile
def argvReplace(self, matchLine, callFunc, contractFuntionStr, currArgvs,
countCall, contractFunctionDict, fileContent, varType):
pattern4Function = re.compile(r'\A(function|modifier)\W+' + callFunc +
'\s?\(([^()]*)\)')
pattern4IfCallFunction = re.compile(r'(if|assert|require)\s*\(.*\W*(' +
callFunc + '\s*\([^)]*\))')
pattern4MultiFunctionCall = re.compile(r'\W(' + callFunc +
'\s*\([^)]*\))(\.\w+\([^)]*\))')
pattern4Assign = re.compile(r'[^\=]+\=[^\=]+' + callFunc + '\s*\(')
pattern4Return = re.compile(r'return\W')
argvs = []
argvDict = {}
functionBlock = Infos().getCurrBlock(
fileContent, contractFunctionDict[contractFuntionStr]['idx'])
isIfCall = 0
isMultiCall = 0
isAssign = 0
isReturn = 0
for idx, item in enumerate(functionBlock):
functionBlock[idx] = item['line']
result4Function = pattern4Function.findall(item['line'].strip())
if result4Function:
argvs = result4Function[0][1].split(',')
for idx, a in enumerate(argvs):
a = a.strip().split(' ')[-1].strip()
#a = re.sub(re.compile('^\s+|\s+$'), '', a).split(' ')[-1]
if a:
argvs[idx] = a
#print(argvs)
#function call in if statement, assertion, require
result4IfCallFunction = pattern4IfCallFunction.findall(
matchLine.strip())
pattern4CallExternalFunction = re.compile(
r'([\w\.\[\]]+)\.(\w+\([^()]*\))')
#print(matchLine)
if result4IfCallFunction:
isIfCall = 1
typeStr = "bool "
if varType:
typeStr = varType + " "
callStart = matchLine.find(result4IfCallFunction[0][1].strip())
callEnd = callStart + len(result4IfCallFunction[0][1].strip())
if matchLine[callStart - 1] == ".":
result4CallExternalFunction = pattern4CallExternalFunction.findall(
matchLine)
if result4CallExternalFunction:
callStart = matchLine.find(
result4CallExternalFunction[0][0] + "." +
result4CallExternalFunction[0][1])
subString1 = "substituteVariable" + "X" + callFunc + str(countCall)
subString2 = typeStr + subString1 + ";\n"
modifiedIfLine = matchLine[0:callStart] + subString1 + matchLine[
callEnd:] + "\n"
#print(subString2)
newMatchLine = subString1 + " = " + result4IfCallFunction[0][
1].strip()
matchLine = newMatchLine
appendLineBeforeList = []
appendLineAfterList = []
appendLineBeforeList.append(subString2)
appendLineAfterList.append(modifiedIfLine)
#print(matchLine)
#function call in format like "A(b, c).call()"
result4MultiFunctionCall = pattern4MultiFunctionCall.findall(
matchLine.strip())
if result4MultiFunctionCall:
isMultiCall = 1
suffixStr = result4MultiFunctionCall[0][1]
#print(matchLine)
result4Assign = pattern4Assign.findall(matchLine.strip())
if result4Assign:
#print(matchLine)
isAssign = 1
assignPrefix = result4Assign[0].split("=")[0]
result4Return = pattern4Return.findall(matchLine.strip())
if result4Return:
#print(matchLine)
isReturn = 1
#substitution
#print("matchLine", matchLine)
if currArgvs:
if len(argvs) > len(currArgvs):
argvs = argvs[0:len(currArgvs)]
for idx, item in enumerate(argvs):
if item:
argvDict[item] = currArgvs[idx]
#print(argvDict)
pattern4ReturnBool = re.compile(r'return\s+\W*(true|false)\W')
pattern4ReturnLine = re.compile(r'(return)(\W+\s*.+\;)')
pattern4ReturnOneVal = re.compile(r'return\s+(.*)\;')
for index, line in enumerate(functionBlock[1:-1]):
#print(line)
result4ReturnBool = pattern4ReturnBool.findall(line.strip())
for pair in argvDict.items():
#print(pair)
pattern4Argv = re.compile(r'[^\.\w]' + pair[0] + '\W')
result4Argv = pattern4Argv.findall(line.strip())
if result4Argv and (pair[0] != pair[1]):
for singleResult in result4Argv:
l = functionBlock[index + 1]
subGoal = singleResult[0] + pair[1] + singleResult[
-1]
startIdx = l.find(singleResult)
endIdx = startIdx + len(singleResult)
functionBlock[
index +
1] = l[0:startIdx] + subGoal + l[endIdx:]
#print(functionBlock[index+1])
result4ReturnLine = pattern4ReturnLine.findall(
functionBlock[index + 1].strip())
if result4ReturnLine:
if isAssign and not isMultiCall:
#print(result4ReturnLine)
functionBlock[index + 1] = pattern4ReturnLine.sub(
assignPrefix + '=' + result4ReturnLine[0][1],
functionBlock[index + 1])
#print(functionBlock[index+1])
elif isMultiCall:
result4ReturnOneVal = pattern4ReturnOneVal.findall(
line.strip())
if result4ReturnOneVal:
strStart = line.find(line.strip())
functionBlock[
index +
1] = line[0:strStart] + result4ReturnOneVal[
0].strip() + suffixStr + ";\n"
elif isReturn:
pass
else:
functionBlock[index + 1] = ""
else:
pass
#while functionB has been call by functionA for multi-times
if countCall:
pattern4Variables = re.compile(
r'(address|bool|string|int|int8|int16|int24|int32|int40|int48|int56|int64|int72|int80|int88|int96|int104|int112|int120|int128|int136|int144|int152|int160|int168|int176|int184|int192|int200|int208|int216|int224|int232|int240|int248|int256|uint|uint8|uint16|uint24|uint32|uint40|uint48|uint56|uint64|uint72|uint80|uint88|uint96|uint104|uint112|uint120|uint128|uint136|uint144|uint152|uint160|uint168|uint176|uint184|uint192|uint200|uint208|uint216|uint224|uint232|uint240|uint248|uint256|byte|bytes|bytes1|bytes2|bytes3|bytes4|bytes5|bytes6|bytes7|bytes8|bytes9|bytes10|bytes11|bytes12|bytes13|bytes14|bytes15|bytes16|bytes17|bytes18|bytes19|bytes20|bytes21|bytes22|bytes23|bytes24|bytes25|bytes26|bytes27|bytes28|bytes29|bytes30|bytes31|bytes32|fixed|(fixed[0-9]+x[0-9]+)|ufixed|(ufixed[0-9]+x[0-9]+))\s+(\w*)'
)
duplicatedVar = {}
for f in functionBlock[1:-1]:
result4Variables = pattern4Variables.findall(f.strip())
if result4Variables:
charIdx = 96 + countCall
duplicatedVar[result4Variables[0][-1]] = result4Variables[
0][-1] + str(charIdx) + chr(charIdx)
for p in duplicatedVar.items():
pattern4Var = re.compile(r'[^\.\w]' + p[0] + '\W')
for index, line in enumerate(functionBlock[1:-1]):
result4Var = pattern4Var.findall(line.strip())
if result4Var and (p[0] != p[1]):
for singleResult in result4Var:
l = functionBlock[index + 1]
subGoal = singleResult[0] + p[1] + singleResult[-1]
startIdx = l.find(singleResult)
endIdx = startIdx + len(singleResult)
functionBlock[
index +
1] = l[0:startIdx] + subGoal + l[endIdx:]
#print(functionBlock[1])
replacedBlock = functionBlock[1:-1]
if isIfCall:
replacedBlock = appendLineBeforeList + functionBlock[
1:-1] + appendLineAfterList
#print(replacedBlock)
return replacedBlock
def EPluribusUnum(self, modSearchOrder, functionSearchOrder, contractList,
contractDict, modDict, contractFunctionDict,
mainContract, contractConstructorDict, statementDict,
searchPath, fileContent):
contractInsertion = {}
contractFuncInsertion = {}
contractStatInsertion = {}
constructorInsertion = {}
pattern4Constructor = re.compile(r'constructor\s*\(.*\{')
#print(contractFunctionDict)
for contractItem in contractList:
contract = contractItem['contract']
funcList = []
for contractFunc in contractFunctionDict.keys():
if contractFunc.split('|')[0] == contract:
funcList.append(contractFunc.split('|')[1])
if functionSearchOrder[contract]:
contractFuncInsertion.setdefault(contract, [])
for func in functionSearchOrder[contract]:
fName = func.split('|')[1]
if fName not in funcList:
funcList.append(fName)
#print(func)
blockBuffer = Infos().getCurrBlock(
fileContent, contractFunctionDict[func]['idx'])
contractFuncInsertion[
contract] = contractFuncInsertion[
contract] + blockBuffer
#modifier
modList = []
for contractMod in modDict.keys():
if contractMod.split('|')[0] == contract:
modList.append(contractMod.split('|')[1])
if modSearchOrder[contract]:
contractFuncInsertion.setdefault(contract, [])
for mod in modSearchOrder[contract]:
mName = mod.split('|')[1]
if mName not in modList:
modList.append(mName)
blockBuffer = Infos().getCurrBlock(
fileContent, modDict[mod]['idx'])
contractFuncInsertion[
contract] = contractFuncInsertion[
contract] + blockBuffer
#statements
statList = []
consList = []
for searchC in searchPath[contract]:
#print(searchC)
#contractFuncInsertion.setdefault(contract, [])
contractStatInsertion.setdefault(contract, [])
if searchC != contract and statementDict.__contains__(searchC):
statList = statementDict[searchC] + statList
if contractConstructorDict.__contains__(searchC):
constructorInsertion.setdefault(contract, [])
consList = contractConstructorDict[searchC]["context"][
1:-1] + consList
if statList:
contractStatInsertion[
contract] = statList + contractStatInsertion[contract]
if consList:
constructorInsertion[
contract] = consList + constructorInsertion[contract]
#constructor
contractSeq = []
for key in contractList:
contractSeq.append(key['contract'])
for c in reversed(contractSeq):
bBuffer = Infos().getCurrBlock(fileContent, contractDict[c]['idx'])
headInsertNode = bBuffer[0]['idx']
tailInsertNode = bBuffer[-1]['idx']
statementNodeCount = 0
if contractFuncInsertion.__contains__(c):
for line in contractFuncInsertion[c]:
fileContent.insert(tailInsertNode, line['line'])
tailInsertNode = tailInsertNode + 1
if statementDict.__contains__(c):
sInsertNodeAdd = bBuffer[0]['idx']
statementNodeCount = statementNodeCount + len(statementDict[c])
for s in statementDict[c]:
del fileContent[s['idx']]
fileContent.insert(sInsertNodeAdd + 1, s['line'])
sInsertNodeAdd = sInsertNodeAdd + 1
if contractConstructorDict.__contains__(c):
constructorInsertNode = contractConstructorDict[c]['idx'] + 1
if constructorInsertion.__contains__(
c) and constructorInsertion[c]:
constructorNodeCount = len(constructorInsertion[c])
for line in constructorInsertion[c]:
fileContent.insert(constructorInsertNode, line['line'])
constructorInsertNode = constructorInsertNode + 1
if contractStatInsertion[c]:
for line in contractStatInsertion[c]:
fileContent.insert(headInsertNode + 1, line['line'])
headInsertNode = headInsertNode + 1
return fileContent
class Queue():
def __init__(self):
self._lock = threading.Lock()
self._lock2 = threading.Lock()
self.config = Config()
self.infos = Infos()
self.queue = []
self.qname = ''
self.func = lambda x:time.sleep(400)
self.threadRecorder = threadRecorder()
self.base_num = 0
def func_call(self, key):
with self._lock2:
logger.info('%s 开始%s' % (self.infos.copy()[key]['uname'], self.qname))
live_info = self.infos.copy()[key]
live_info['queue_status'] = self.base_num
self.infos.update(key, live_info)
self.func(key)
live_info['queue_status'] = self.base_num + 500
live_info['finish_time'] = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
self.infos.update(key, live_info)
def update_status(self):
for key in self.queue:
live_info = self.infos.copy()[key]
live_info['queue_status'] = self.base_num + self.queue.index(key) + 1
self.infos.update(key, live_info)
def enqueue(self, key):
with self._lock:
live_info = self.infos.copy()[key]
live_info['queue_status'] = self.base_num + len(self.queue) + 1
if key not in self.queue:
self.queue.append(key)
logger.info('%s 进入%s等待队列' % (self.infos.copy()[key]['uname'],self.qname))
else:
self.queue.remove(key)
self.queue.append(key)
logger.info('%s 在%s等待队列中的状态更新了' % (self.infos.copy()[key]['uname'],self.qname))
self.infos.update(key, live_info)
def dequeue(self):
if self._lock2.locked():
return None
with self._lock:
with self._lock2:
if len(self.queue) > 0:
key = self.queue[0]
del self.queue[0]
self.update_status()
logger.info('%s 退出%s等待队列' % (self.infos.copy()[key]['uname'],self.qname))
return key
else:
return None
def run(self):
self.threadRecorder.add('%s heartbeat' % self.qname,self.heartbeat,None,True)
while True:
time.sleep(1)
if len(self.queue) > 0:
key = self.dequeue()
if key is not None:
self.threadRecorder.add('%s_%s' % (self.qname,key),self.func_call,[key,],False)
def heartbeat(self):
while True:
time.sleep(180)
logger.info('当前%s队列情况: %s' % (self.qname, ' '.join([self.infos.copy()[key]['uname'] for key in self.queue])))
class History():
def __init__(self):
self._lock = threading.Lock()
self.live_infos = Infos()
self.base_path = os.path.join(
os.path.dirname(os.path.dirname(os.path.realpath(__file__))),
'history')
self.init()
logger.debug('History模块初始化完成')
def init(self):
if not os.path.exists(self.base_path):
os.mkdir(self.base_path)
for key in self.live_infos.copy():
history_file = os.path.join(
self.base_path,
'%s_%s' % (key, self.live_infos.get(key)['uname']))
if not os.path.exists(history_file):
with open(history_file, 'w', encoding='utf-8') as a:
pass
def add_info(self, key, para, output):
self.init()
history_file = os.path.join(
self.base_path, '%s_%s' % (key, self.live_infos.get(key)['uname']))
with self._lock:
with open(history_file, 'a', encoding='utf-8') as a:
a.write(
'####📢 %s, 当前%s状态为: %s, 当前时间: %s\n' %
(output, para, self.live_infos.get(key)[para],
datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')))
def heartbeat(self):
while True:
try:
time.sleep(600)
self.init()
for key in self.live_infos.copy():
history_file = os.path.join(
self.base_path,
'%s_%s' % (key, self.live_infos.get(key)['uname']))
with self._lock:
with open(history_file, 'a', encoding='utf-8') as a:
a.write('####✨ 当前时间: %s\n' %
(datetime.datetime.now().strftime(
'%Y-%m-%d %H:%M:%S')))
a.write('录制时段: %s\n' %
(self.live_infos.get(key)['duration']))
a.write('直播状态: %s\n' %
(self.live_infos.get(key)['live_status']))
a.write('录制状态: %s\n' %
(self.live_infos.get(key)['recording']))
a.write('是否录制: %s\n' %
(self.live_infos.get(key)['need_rec']))
a.write('是否遮挡: %s\n' %
(self.live_infos.get(key)['need_mask']))
a.write('是否上传: %s\n' %
(self.live_infos.get(key)['need_upload']))
except Exception as e:
logger.critical(e)
continue