def schedulerOrdersPerf(plant, orderList, testNum):
machines = plant.machines[:]
orders = orderList.orders[:]
pprint("PERF Starting benchmark test " + str(testNum) + " on orders", BLUE)
orderList.orders = []
plant.machines = machines[:]
ordertimes = [0]
for i in range(1, len(orders) + 1):
pprint("PERF Number of orders = " + str(i), BLUE)
orderList.orders = orders[:i]
scheduler = Scheduler(plant, orderList)
t = time()
scheduler.start()
t = time() - t
ordertimes.append(t)
pprint("PERF Time = " + str(t), GREEN)
try:
from thirdparty.CairoPlot import dot_line_plot
except:
pprint("PERF Will not output to graph.", RED)
return
dot_line_plot(
path.join("benchmarks", "orders-" + str(testNum)) + ".ps", ordertimes,
800, 800, (255, 255, 255), 5, True, True, True, None, None, None, None)
def test_inserting_recurring_schedule_sets_message(good_scheduled_message):
body = good_scheduled_message.body
scheduler_under_test = Scheduler()
scheduler_under_test.schedule_message(good_scheduled_message)
message = scheduler_under_test.get_messages()[0]
actual_body = message.body
assert actual_body == body
def schedulerLargeValuesPerf(plant, orderList, testNum):
machines = plant.machines[:]
orders = orderList.orders[:]
pprint(
"PERF Starting benchmark test " + str(testNum) + " with large values",
BLUE)
orderList.orders = orders[:6]
plant.machines = machines[:]
largeValuesTimes = [0]
for i in range(1, 10):
pprint("PERF Large Value = " + str(i * 4), BLUE)
for o in orderList.orders:
o.deadline *= 4
for r in o.recipe.recipe:
r[1] *= 4
scheduler = Scheduler(plant, orderList)
t = time()
scheduler.start()
t = time() - t
largeValuesTimes.append(t)
pprint("PERF Time = " + str(t), GREEN)
try:
from thirdparty.CairoPlot import dot_line_plot
except:
pprint("PERF Will not output to graph.", RED)
return
dot_line_plot(
path.join("benchmarks", "largevalues-" + str(testNum)) + ".ps",
largeValuesTimes, 800, 800, (255, 255, 255), 5, True, True, True, None,
None, None, None)
def __init__(self, test_case_conf, clients: List[Client]):
self.test_case_conf = test_case_conf
self.logger = log.get_logger()
self.clients = {}
for client in clients:
self.clients[client.index] = client
self.scheduler = Scheduler(test_case_conf, clients)
# TC statistics
self.start_chain_height = 0
self.end_chain_height_before_recover = 0
self.end_chain_height_after_recover = 0
self.start_time = time.time()
self.start_recover_time = None
self.end_recover_time = None
self.tx_history_file = TX_HISTORY_FILE.format(self.start_time)
# TX issuing statistics
self.tx_start_chain_height = 0
self.tx_end_chain_height = 0
self.tx_start_time = None
self.tx_end_time = None
self.tx_sent = 0
self.tx_mined = 0
self.balance_mined_by_the_test = 0
self.sender_before_balance = 0
self.receiver_before_balance = 0
def test_inserting_schedule_sets_end_datetime(good_scheduled_message):
end_datetime_in_utc = good_scheduled_message.end_datetime_in_utc
scheduler_under_test = Scheduler()
scheduler_under_test.schedule_message(good_scheduled_message)
message = scheduler_under_test.get_messages()[0]
end_datetime = message.end_datetime_in_utc
assert end_datetime.isoformat() == end_datetime_in_utc.isoformat()
def message_schedule(person_name, message, expiration_datetime, ical,
start_datetime):
try:
click.echo("Scheduling message for person {}".format(person_name))
scheduler = Scheduler()
start_datetime = arrow.utcnow()
if start_datetime is None:
click.echo("Setting start time to now")
else:
start_datetime = arrow.get(start_datetime)
click.echo("Setting custom start time to %s" % start_datetime)
if ical is None:
ical = 'FREQ=DAILY;COUNT=1'
if expiration_datetime is None:
expiration_datetime = start_datetime.replace(hours=1)
message = ScheduledMessage(StartDateTimeInUtc=start_datetime,
ical=ical,
Body=message,
PersonName=person_name,
EndDateTimeInUtc=expiration_datetime)
scheduler.schedule_message(message)
except Exception:
click.echo(traceback.print_exc())
exit(2)
def get_messages(self, **kwargs):
person_name = kwargs.get('PersonName', '')
self.sqs_msgs = []
"""get all available messages off the queue"""
print "Checking message queue"
messages = self.queue.receive_messages(MessageAttributeNames=[
'NoMoreOccurrences', 'ExpirationDateTimeInUtc', 'PersonName',
'Voice', 'UUID'
],
WaitTimeSeconds=20,
MaxNumberOfMessages=10)
if len(messages) > 0:
for m in messages:
qm = QueuedMessage(QueuedMessage=m)
if not qm.person_name == person_name:
continue
if qm.person_name not in self.messages:
logging.info("First message for " + qm.person_name)
self.messages[qm.person_name] = []
logging.info('Adding message:\n')
logging.info(qm.body)
self.messages[qm.person_name].append(qm)
logging.info('Total messages so far: ' +
str(len(self.messages[qm.person_name])))
self.sqs_msgs.append(m)
scheduler = Scheduler()
scheduler.update_queue_status(qm.uuid_key, person_name, False)
def __init__(self,local_mapping_file):
logger.init('./globalhapi.log',logging.INFO)
self.logger=logger.getLogger('global',logging.INFO)
# self.topo = topo
# self.locals = [0,1] #{0:"bala"}
# self.dp_to_local = {1:0,2:1}
self.locals = read_mapping_from_file(local_mapping_file).keys()
self.dp_to_local = get_reverse_mapping_from_file(local_mapping_file)
self.logger.info(self.locals)
self.logger.info(self.dp_to_local)
self.sockets = {}
self.link_bw = {}#outer layer
self.link_ltcy = {}
self.local_to_buf_size = {}
self.dp_to_tcam_size = {}
# self.dp_to_local = {1:0,2:0,3:1,4:1}#for lineartopo2
self.flows = {}# all flows now
self.flows_new = {} #flows updating
self.flows_to_schedule = {} #flow to update
self.status_num = 0 #for start update
# self.flows_new = {"10.0.0.110.0.0.25001":flowdes0}
self.scheduler = Scheduler()
self.tag_flows_temp = {}
self.schedule_result = {}
eventlet.spawn(self.run_fd_server)
def __init__(self, parent=None):
super(MainApp, self).__init__(parent)
# set up GUI
self.setupUi(self)
self.taskListElems.setAlignment(QtCore.Qt.AlignTop)
self.currentTaskElem.setAlignment(QtCore.Qt.AlignTop)
self.currentTask.setStyleSheet("QScrollArea {background-color:transparent;}");
self.currentTaskContents.setStyleSheet("background-color:transparent;");
self.taskList.setStyleSheet("QScrollArea {background-color:transparent;}");
self.taskListContents.setStyleSheet("background-color:transparent;");
self.setStyleSheet("#mainWindow{background-color:white;}")
# connect signals
self.add_task_signal.connect(self.add_task)
self.pop_task_signal.connect(self.pop_task)
self.clear_tasks_signal.connect(self.clear_tasks)
self.update_time_signal.connect(self.update_time)
self.update_position_signal.connect(self.update_position)
# initialize cloud components
self.stt = stt.SpeechToTextManager("google_key.json")
self.nlp = nlp.NaturalProcessingLanguageGoogleCloud()
self.tts = tts.TextToSpeechManager()
th = threading.Thread(target=self.get_task_thread_body, daemon=True)
th.start()
# initialize scheduler
self.time_daemon = TimeDaemon(min_every_sec=1, update_time_signal=self.update_time_signal)
threading.Thread(target=self.time_daemon.run, args=()).start()
# need to set current GPS position here
self.scheduler = Scheduler(current_position="Cecina", time_daemon=self.time_daemon, polling_sec=5,
task_finished_signal=self.pop_task_signal,
update_position_signal=self.update_position_signal)
def __init__(self, config: Config):
self._scheduler = Scheduler()
self._email_notification = EmailNotification(config.get('smtp'), config.get('recipients'))
for site in config.get('sites'):
self._scheduler.every_minutes(Site(site), self.parse)
logger.info(f"Will be parsing {len(config.get('sites'))} site(s)")
def test_inserting_schedule_sets_start_datetime(good_scheduled_message):
start_datetime_in_utc = good_scheduled_message.start_datetime_in_utc
scheduler_under_test = Scheduler()
scheduler_under_test.schedule_message(good_scheduled_message)
message = scheduler_under_test.get_messages()[0]
start_date = message.start_datetime_in_utc
assert start_date.isoformat() == start_datetime_in_utc.isoformat()
def test_inserting_recurring_schedule_sets_ical(good_scheduled_message):
test_ical = good_scheduled_message.ical
scheduler_under_test = Scheduler()
scheduler_under_test.schedule_message(good_scheduled_message)
message = scheduler_under_test.get_messages()[0]
ical = message.ical
assert ical == test_ical
def add_sina_news_tasks(scheduler: Scheduler):
for keyword in config.KEYWORDS:
search_key = urllib.parse.quote(keyword)
url = f'https://search.sina.com.cn/?q={search_key}&range=all&c=news&sort=time'
metadata = {
'keyword': keyword,
}
scheduler.append_request_task(Task(url, '', '', metadata=metadata))
def run_scheduler():
scheduler = Scheduler(taskdb=get_taskdb(), projectdb=get_projectdb(),
newtask_queue=self.newtask_queue, status_queue=self.status_queue,
out_queue=self.scheduler2fetcher, data_path="./test/data/")
scheduler.UPDATE_PROJECT_INTERVAL = 0.05
scheduler.LOOP_INTERVAL = 0.01
run_in_thread(scheduler.xmlrpc_run, port=self.scheduler_xmlrpc_port)
scheduler.run()
def add_weibo_topic_list_tasks(scheduler: Scheduler):
for keyword in config.KEYWORDS:
search_key = urllib.parse.quote(keyword)
url = f'https://s.weibo.com/topic?q={search_key}&pagetype=topic&topic=1&Refer=weibo_topic'
metadata = {
'keyword': keyword,
}
scheduler.append_request_task(Task(url, '', '', metadata=metadata))
def add_gov_tasks(scheduler: Scheduler):
for keyword in config.KEYWORDS:
search_key = urllib.parse.quote(keyword)
url = f'http://sousuo.gov.cn/s.htm?t=govall&q={search_key}'
metadata = {
'keyword': keyword,
}
scheduler.append_request_task(Task(url, '', '', metadata=metadata))
def message_schedule(person_name, message, ical, count, frequency, lexbot,
interval, timezone, start_date, start_time, end_date,
bot_names, ice_breaker, required_bots, introduction,
end_time):
try:
print ice_breaker
print required_bots
click.echo("Scheduling message for person {}".format(person_name))
scheduler = Scheduler()
if not timezone:
timezone = tzlocal.get_localzone().zone
click.echo('Timezone: {}'.format(timezone))
if start_time is None:
start_time = arrow.now(timezone).format('HH:mm')
if start_date is None:
start_date = arrow.now(timezone).format('YYYY-MM-DD')
start_datetime = arrow.get(
'{} {}'.format(start_date, start_time)) \
.replace(tzinfo=tz.gettz(timezone)).to('UTC')
if end_time is None:
end_time = start_time.format('HH:mm')
if end_date is None:
end_date = start_datetime.replace(years=10).format('YYYY-MM-DD')
end_datetime = arrow.get(
'{} {}'.format(end_date, end_time)) \
.replace(tzinfo=tz.gettz(timezone)).to('UTC')
message = ScheduledMessage(StartDateTimeInUtc=start_datetime,
ical=ical,
Body=message,
PersonName=person_name,
Frequency=frequency,
Count=count,
Lexbot=lexbot,
TimeZone=timezone,
Interval=interval,
BotNames=bot_names,
IceBreaker=ice_breaker,
Introduction=introduction,
RequiredBots=required_bots,
EndDateTimeInUtc=end_datetime)
scheduler.schedule_message(message)
click.echo('Start Time: {}'.format(start_datetime))
click.echo('End Time: {}'.format(end_datetime))
if ical:
click.echo('ical:\n{}'.format(ical))
print "Next: {}".format(message.next_occurrence_local)
print message.to_ical()
except Exception:
click.echo(traceback.print_exc())
exit(2)
def message_delete(person_name, message_id, verbose):
log = logging.getLogger('PollexyCli')
if verbose:
os.environ['LOG_LEVEL'] = 'DEBUG'
log.setLevel(logging.DEBUG)
s = Scheduler()
click.echo('Deleting message')
s.delete_message(Key=message_id,
PersonName=person_name)
def add_cnr_search_task(scheduler: Scheduler):
for keyword in config.KEYWORDS:
search_key = urllib.parse.quote(keyword)
url = (f'http://was.cnr.cn/was5/web/search?page=2&channelid=234439&searchword={search_key}'
f'&keyword={search_key}&orderby=LIFO&was_custom_expr=%28{search_key}%29&perpage=10'
f'&outlinepage=1&searchscope=×cope=×copecolumn=&orderby=LIFO&andsen=&total=&orsen=&exclude=')
scheduler.append_request_task(Task(url, '', '', metadata={
'keyword': keyword
}))
def add_weibo_info_tasks(scheduler: Scheduler):
for item in WeiboTopicItem.objects().order_by('title'):
search_key = urllib.parse.quote(item.title)
url = f'https://m.s.weibo.com/ajax_topic/trend?q={search_key}&time=30d'
metadata = {
'keyword': item.keyword,
'title': item.title,
}
scheduler.append_request_task(Task(url, '', '', metadata=metadata))
def add_cctv_tasks(scheduler: Scheduler):
for keyword in config.KEYWORDS:
search_key = urllib.parse.quote(keyword)
url = (f'https://search.cctv.com/search.php?qtext={search_key}&sort=relevance&type=web&vtime='
f'&datepid=1&channel=&page=1')
metadata = {
'keyword': keyword,
}
scheduler.append_request_task(Task(url, '', '', metadata=metadata))
def add_china_cdc_tasks(scheduler: Scheduler):
for keyword in config.KEYWORDS:
search_key = urllib.parse.quote(keyword)
url = (f'http://www.chinacdc.cn/was5/web/search?searchword={search_key}&channelid=233877×cope=&'
f'timescopecolumn=&orderby=-%E6%97%A5%E6%9C%9F&perpage=10&searchscope=')
metadata = {
'keyword': keyword,
}
scheduler.append_request_task(Task(url, '', '', metadata=metadata))
def testScheduler(self): print "testScheduler():" self.pm = ProcessorMonitor(self.y1) self.proc.addMonitor(self.pm) s = Scheduler(self.y1, self.proc) s.run() Utils.dumpSchedule(s.getSchedule(), self.s1) Utils.dumpYaml(self.pm.getResults(), self.e1) print
def main():
""" Scheduler scripts, reads from the database all
schedules and determines if any spider should
be executed at this time.
"""
#path = dirname(realpath(__file__))
project_conf = get_project_conf()
scheduler = Scheduler(project_conf)
scheduler.run_scheduler()
def runAll(self):
# create schedule and run it
input = self.workload['input']
schedulesummary = self.workload['schedulesummary']
schedule = self.workload['schedule']
executions = self.workload['executions']
self.log.info("")
self.log.info("##############################################")
self.log.info("# Running " + input)
self.log.info("##############################################")
self.log.info("")
proc = Processor()
w = Utils.loadYaml(input)
pm = ProcessorMonitor(w)
proc.addMonitor(pm)
sc = Scheduler(w, proc)
sc.run()
Utils.dumpSchedule(sc.getSchedule(), schedulesummary)
Utils.dumpYaml(sc.getSchedule(), schedule)
Utils.dumpYaml(pm.getResults(), executions)
# plot the schedule
p = Plot(sc.getSchedule(), pm.getResults(), sc.getH(), sc.getI(), self.wid)
p.create()
# verify the schedule
v = Verifier(Utils.loadYaml(executions), sc.getH(), input)
v.verifySchedule()
return p.getFigure()
def add_china_news_tasks(scheduler: Scheduler):
for keyword in config.KEYWORDS:
search_key = urllib.parse.quote(keyword)
url = f'http://sou.chinanews.com/search.do'
body = {
'q': search_key,
}
metadata = {
'keyword': keyword,
}
scheduler.append_request_task(Task(url, '', '', method='POST', body=body, metadata=metadata))
def test_new_message_is_ready(good_scheduled_message):
msg = ScheduledMessage(
StartDateTimeInUtc=arrow.get('2012-01-01 01:01 UTC'),
ical="FREQ=DAILY",
Body="Test Message Body",
PersonName="Testperson",
EndDateTimeInUtc=arrow.get('2027-01-01 01:01 UTC'))
scheduler_under_test = Scheduler()
scheduler_under_test.schedule_message(msg)
m = scheduler_under_test.get_messages()[0]
assert m.is_message_ready
def test_sanity(self):
test_scheduler = TestingScheduler(max_jobs=SchedulerTests.MAX_JOBS)
self.assertEqual(test_scheduler.time_unit, TimeUnitEnum.SECONDS)
self.assertEqual(len(test_scheduler), SchedulerTests.MAX_JOBS)
scheduler = Scheduler(max_jobs=SchedulerTests.MAX_JOBS)
self.assertEqual(scheduler.time_unit, TimeUnitEnum.HOURS)
self.assertEqual(len(scheduler), SchedulerTests.MAX_JOBS)
self.assertFalse(scheduler.is_alive())
self.assertFalse(scheduler.daemon)
def test_ignore_expired_messages(good_scheduled_message):
msg = ScheduledMessage(
StartDateTimeInUtc=arrow.get('2012-01-01 01:01 UTC'),
ical="FREQ=DAILY",
Body="Test Message Body",
PersonName="Testperson",
EndDateTimeInUtc=arrow.get('2014-01-01 01:01 UTC'))
compare_date = arrow.utcnow()
scheduler_under_test = Scheduler()
scheduler_under_test.schedule_message(msg)
m = scheduler_under_test.get_messages(compare_date, True)
assert len(m) == 0
def bench(self):
orders = self.orderList.orders[:]
i = self.startValue
while i <= len(orders):
pprint("PERF Number of orders = " + str(i), BLUE)
self.orderList.orders = orders[:i]
scheduler = Scheduler(self.plant, self.orderList)
t = time()
scheduler.start()
t = time() - t
self.addCairoPlotTime(t)
self.addGnuPlotTime(i, t)
i += 1
def fail_speech(self, **kwargs):
logging.info('Speech failed: ' +
kwargs.get('Reason', 'Unknown Reason'))
dont_delete = kwargs.get('DontDelete', False)
if (dont_delete):
logging.info('We are NOT deleting the original SQS messages')
return
for m in self.sqs_msgs:
scheduler = Scheduler()
qm = QueuedMessage(QueuedMessage=m)
logging.info("Setting messages InQueue to False")
scheduler.update_queue_status(qm.uuid_key, qm.person_name, False)
self.delete_sqs_msgs()
def trigger():
if not request.content_type == 'application/json':
return Response('Error: Invalid Content-Type header',
status=400,
mimetype='text/plain')
try:
scheduler = Scheduler(request.json, 420)
scheduler.start()
except SchedulerError as e:
return Response('Error: ' + str(e), status=400, mimetype='text/plain')
return Response(f"Launched scheduler", status=200, mimetype='text/plain')
def schedule(self):
"""
Runs the Scheduler with the OrderList from orderListName on the Plant
with plantName.
"""
scheduler = Scheduler(self.plant, self.orderList)
evaluator = Evaluator.fromXmlFile(self.configFilename, self.plant)
result = scheduler.start()
if result != None:
solutions = parseSolutions(result, self.plant, self.orderList)
evaluator.evaluate(solutions)
best = bestSolution(solutions)
best.unNormalize(self.normValue)
print best
def run_scheduler():
scheduler = Scheduler(taskdb=get_taskdb(), projectdb=get_projectdb(),
newtask_queue=self.newtask_queue, status_queue=self.status_queue,
out_queue=self.scheduler2fetcher, data_path="./test/data/")
scheduler.UPDATE_PROJECT_INTERVAL = 0.1
scheduler.LOOP_INTERVAL = 0.1
scheduler.INQUEUE_LIMIT = 10
scheduler._last_tick = time.time() # not dispatch cronjob
run_in_thread(scheduler.xmlrpc_run, port=self.scheduler_xmlrpc_port)
scheduler.run()
def runTasks():
log = logging.getLogger("main")
# setup
workloads = { 'workload1': { 'input':'input/workload1.yaml', 'schedulesummary':'output/schedule1.txt',
'schedule':'output/schedule1.yaml', 'executions':'output/executions1.yaml' },
'workload2': { 'input':'input/workload2.yaml', 'schedulesummary':'output/schedule2.txt',
'schedule':'output/schedule2.yaml', 'executions':'output/executions2.yaml' } }
for id,workload in workloads.iteritems():
# create schedule and run it
input = workload['input']
schedulesummary = workload['schedulesummary']
schedule = workload['schedule']
executions = workload['executions']
log.info("")
log.info("##############################################")
log.info("# Running " + input)
log.info("##############################################")
log.info("")
proc = Processor()
w = Utils.loadYaml(input)
pm = ProcessorMonitor(w)
proc.addMonitor(pm)
sc = Scheduler(w, proc)
sc.run()
Utils.dumpSchedule(sc.getSchedule(), schedulesummary)
Utils.dumpYaml(sc.getSchedule(), schedule)
Utils.dumpYaml(pm.getResults(), executions)
# plot the schedule
p = Plot(sc.getSchedule(), pm.getResults(), sc.getH(), sc.getI(), id)
p.create()
# verify the schedule
v = Verifier(Utils.loadYaml(executions), sc.getH(), input)
v.verifySchedule()
# show the plot
p.show()
