def process_upload(name, url, date):
yield Event(BE_MODIFIED, (url, 2))
try:
data = {"url": url, "date": date}
upload("bili_web", name, data)
finally:
yield Event(BE_MODIFIED, args=(url, 0))
def free_upload(self, _urls):
logger.debug(_urls)
event = []
for title, v in engine.links_id.items():
url = v[0]
if self.free(v) and Upload(title).filter_file():
event_d = Event(DOWNLOAD_UPLOAD)
event_d.args = (title, url, 'up')
event.append(event_d)
# self.event_manager.send_event(event_d)
self.url_status[url] = 2
return tuple(event)
def free_upload(self, _urls):
logger.debug(_urls)
for title, v in engine.streamer_url.items():
url = v[0]
if self.free(v) and UploadBase.filter_file(title):
event_manager.send_event(Event(UPLOAD, args=(title, url)))
self.url_status[url] = 2
def singleton_check(self, platform):
plugin = checker[platform]
if isinstance(plugin, BatchCheckBase):
live = batch_check(plugin)
else:
live = singleton_check(plugin)
return Event(TO_MODIFY, args=(live,))
def free_upload(self, _urls):
logger.debug(_urls)
event = []
for title, v in engine.links_id.items():
# names = list(map(find_name, urls))
url = v[0]
# if title not in names and url_status[url] == 0 and Upload(title, url).filter_file():
if self.free(v) and Upload(title).filter_file():
event_d = Event(DOWNLOAD_UPLOAD)
event_d.args = (title, url, 'up')
event.append(event_d)
# self.event_manager.send_event(event_d)
self.url_status[url] = 2
# print('up')
return tuple(event)
def process(name, url, mod):
try:
now = common.time_now()
if mod == 'dl':
p = multiprocessing.Process(target=download, args=(name, url))
p.start()
p.join()
# download(name, url)
Upload(name).start(url, now)
elif mod == 'up':
Upload(name).start(url, now)
else:
return url
finally:
event = Event(BE_MODIFIED)
event.args = (url, )
return event
def process(name, url):
date = common.time_now()
try:
p = multiprocessing.Process(target=download, args=(name, url))
p.start()
p.join()
# download(name, url)
finally:
return Event(UPLOAD, (name, url, date))
def main(event_manager):
# 初始化定时器
timer = Timer(func=event_manager.send_event, args=(Event(CHECK),), interval=40)
# 模块更新自动重启
autoreload(event_manager, timer, interval=15)
event_manager.start()
timer.start()
def process(name, url):
try:
data = {"url": url, "date": common.time_now()}
p = multiprocessing.Process(target=download, args=(name, url))
p.start()
p.join()
# download(name, url)
upload("bili_web", name, data)
finally:
return Event(BE_MODIFIED, args=(url, ))
def modify(self, url):
if not url:
return logger.debug('无人直播')
if self.url_status[url] == 1:
return logger.debug('已开播正在下载')
if self.url_status[url] == 2:
return logger.debug('正在上传稍后下载')
name = self.inverted_index[url]
logger.debug(f'{name}刚刚开播，去下载')
self.url_status[url] = 1
return Event(DOWNLOAD, args=(name, url))
def modify(self, live_m):
live_d = {}
if live_m:
event = []
for live in live_m:
if self.url_status[live] == 1:
logger.debug('已开播正在下载')
else:
name = engine.find_name(live)
logger.debug(name + '刚刚开播，去下载')
event_d = Event(DOWNLOAD_UPLOAD)
event_d.args = (name, live, 'dl')
event.append(event_d)
live_d[live] = 1
self.url_status.update(live_d)
# url_status = {**url_status_base, **live_d}
return tuple(event)
else:
logger.debug('无人直播')
def all_check(self):
live = []
try:
for batch in self.batches:
res = batch.check()
if res:
live.extend(res)
for one in self.onebyone:
for url in one.url_list:
if one('检测' + url, url).check_stream():
live.append(url)
if url != one.url_list[-1]:
logger.debug('歇息会')
time.sleep(15)
except IOError:
logger.exception("IOError")
finally:
event_t = Event(TO_MODIFY)
event_t.args = (live,)
event_u = Event(UPLOAD)
event_u.args = (live,)
return event_u, event_t
def modify(self, live_m):
if not live_m:
return logger.debug('无人直播')
for live in live_m:
if self.url_status[live] == 1:
logger.debug('已开播正在下载')
continue
if self.url_status[live] == 2:
logger.debug('正在上传稍后下载')
continue
name = inverted_index[live]
logger.debug(f'{name}刚刚开播，去下载')
self.url_status[live] = 1
yield Event(DOWNLOAD, args=(name, live))
def modify(self, live_m):
if not live_m:
return logger.debug('无人直播')
live_d = {}
for live in live_m:
if self.url_status[live] == 1:
logger.debug('已开播正在下载')
else:
name = inverted_index[live]
logger.debug(f'{name}刚刚开播，去下载')
event_manager.send_event(Event(DOWNLOAD, args=(name, live)))
live_d[live] = 1
self.url_status.update(live_d)
def evaluate(hourlyActionRates, objectPreference, currentTime, unitTime):
'''
Decide user action performed on object by flipping a coin.
:param hourlyActionRates: list of agent's HourlyActionRate instance, each instance corresponding to one actionType
:param objectPreference: agent's preference over objects she touches.
:return: a list of events
'''
events = []
objectIndexes = [i for i in range(len(objectPreference.objectIds))]
rv = rv_discrete(values=(objectIndexes, objectPreference.probs))
for hourlyActionRate in hourlyActionRates: #Consider each type of actions independently
dailyActivityLevel = hourlyActionRate.activityLevel # How many actions of this type this user may take per day?
if sum(hourlyActionRate.probs
) == 0: #No record on this type of actions.
continue
if dailyActivityLevel == 0:
continue
prob = hourlyActionRate.probs[currentTime % 24]
for count in range(dailyActivityLevel):
if random.random(
) <= prob: # He will adopt an action of this type
agentId = hourlyActionRate.agentId
objectId = objectPreference.objectIds[rv.rvs(
size=1)[0]] # Get 1 sample the distribution
actionType = hourlyActionRate.actionType
event = Event(userID=agentId,
objID=objectId,
eventType=actionType,
timestamp=currentTime)
events.append(event)
return events
def process(name, url):
date = common.time_now()
try:
download(name, url)
finally:
return Event(UPLOAD, (name, url, date))
async def check_timer():
event_manager.send_event(Event(CHECK_UPLOAD))
for k in checker.keys():
event_manager.send_event(Event(CHECK, (k, )))
def singleton_check(self):
live = check(self.urls, "single")
return Event(TO_MODIFY, args=(live, ))
def check_timer(event_manager):
event_manager.send_event(Event(CHECK_UPLOAD))
for k in platforms:
event_manager.send_event(Event(CHECK, (k, )))
def evaluate(userDependency, dependencyLength, objectPreference,
currentTime, unitTime):
'''
Simmulate the user dependent actions according to his dependency relationship and
the actions of his dependent users.
:param userDependency: the dependency relationships of the given user
:param dependencyLength: the length of dependnecy we consider here, a measure on time.
:param dependencyLogger: the log of the simulated events we need to query
:param objectPreference: the preference among objects of this given user.
:param currentTime: current time step
:param unitTime: length of time of each step
:return: the simulated dependent events
'''
events = []
dependentEventLogger = DependentEventLogger.getInstance()
objectIndexes = [i for i in range(len(objectPreference.objectIds))]
rv = rv_discrete(values=(objectIndexes, objectPreference.probs))
if not userDependency.depUserIds: # No dependency for this user
return events
noDenpendentActionProb = 1
dependentEventTypeScore = {}
for eventType in DependentBehaviorModel.eventTypes:
dependentEventTypeScore[eventType] = 0.0
for depUserId in userDependency.depUserIds: # Consider each pairwise dependency independently
dependentProb = userDependency.userDependency[depUserId]
performedAction = False
#First check if the dependent user performed action during the dpendency window.
for timestamp in np.arange(currentTime - dependencyLength,
currentTime, unitTime):
if timestamp < 0:
continue
for eventType in DependentBehaviorModel.eventTypes:
if dependentEventLogger.checkUserEventAtTime(
depUserId, eventType, timestamp):
dependentEventTypeScore[eventType] += dependentProb
if not performedAction:
performedAction = True
if performedAction:
noDenpendentActionProb *= (1 - dependentProb)
if noDenpendentActionProb < 1:
actionProb = 1 - noDenpendentActionProb
if random.random(
) <= actionProb: # He will adopt an action of this type
userId = userDependency.userId # ID of this user, but not the dependent user.
objectId = objectPreference.objectIds[rv.rvs(
size=1)[0]] # Get 1 sample the distribution
#Select the event type with greateset event type score
actionType = max(dependentEventTypeScore,
key=dependentEventTypeScore.get)
event = Event(userID=userId,
objID=objectId,
eventType=actionType,
timestamp=currentTime)
events.append(event)
return events
def singleton_check(self, platform):
plugin = self.checker[platform]
for url in check_url(plugin):
yield Event(TO_MODIFY, args=(url, ))
def process_upload(name, url):
try:
data = {"url": url, "date": common.time_now()}
upload("bili_web", name, data)
finally:
return Event(BE_MODIFIED, args=(url, ))
def free_upload(self):
for title, urls in self.streamer_url.items():
if self.free(urls) and UploadBase.filter_file(title):
yield Event(UPLOAD, args=(title, urls[0], common.time_now()))
def batch_check(self):
live = check(self.urls, "batch")
return Event(CHECK_UPLOAD, args=(live, )), Event(TO_MODIFY,
args=(live, ))