def schedulerOrdersPerf(plant, orderList, testNum): machines = plant.machines[:] orders = orderList.orders[:] pprint("PERF Starting benchmark test " + str(testNum) + " on orders", BLUE) orderList.orders = [] plant.machines = machines[:] ordertimes = [0] for i in range(1, len(orders) + 1): pprint("PERF Number of orders = " + str(i), BLUE) orderList.orders = orders[:i] scheduler = Scheduler(plant, orderList) t = time() scheduler.start() t = time() - t ordertimes.append(t) pprint("PERF Time = " + str(t), GREEN) try: from thirdparty.CairoPlot import dot_line_plot except: pprint("PERF Will not output to graph.", RED) return dot_line_plot( path.join("benchmarks", "orders-" + str(testNum)) + ".ps", ordertimes, 800, 800, (255, 255, 255), 5, True, True, True, None, None, None, None)
def test_inserting_recurring_schedule_sets_message(good_scheduled_message): body = good_scheduled_message.body scheduler_under_test = Scheduler() scheduler_under_test.schedule_message(good_scheduled_message) message = scheduler_under_test.get_messages()[0] actual_body = message.body assert actual_body == body
def schedulerLargeValuesPerf(plant, orderList, testNum): machines = plant.machines[:] orders = orderList.orders[:] pprint( "PERF Starting benchmark test " + str(testNum) + " with large values", BLUE) orderList.orders = orders[:6] plant.machines = machines[:] largeValuesTimes = [0] for i in range(1, 10): pprint("PERF Large Value = " + str(i * 4), BLUE) for o in orderList.orders: o.deadline *= 4 for r in o.recipe.recipe: r[1] *= 4 scheduler = Scheduler(plant, orderList) t = time() scheduler.start() t = time() - t largeValuesTimes.append(t) pprint("PERF Time = " + str(t), GREEN) try: from thirdparty.CairoPlot import dot_line_plot except: pprint("PERF Will not output to graph.", RED) return dot_line_plot( path.join("benchmarks", "largevalues-" + str(testNum)) + ".ps", largeValuesTimes, 800, 800, (255, 255, 255), 5, True, True, True, None, None, None, None)
def __init__(self, test_case_conf, clients: List[Client]): self.test_case_conf = test_case_conf self.logger = log.get_logger() self.clients = {} for client in clients: self.clients[client.index] = client self.scheduler = Scheduler(test_case_conf, clients) # TC statistics self.start_chain_height = 0 self.end_chain_height_before_recover = 0 self.end_chain_height_after_recover = 0 self.start_time = time.time() self.start_recover_time = None self.end_recover_time = None self.tx_history_file = TX_HISTORY_FILE.format(self.start_time) # TX issuing statistics self.tx_start_chain_height = 0 self.tx_end_chain_height = 0 self.tx_start_time = None self.tx_end_time = None self.tx_sent = 0 self.tx_mined = 0 self.balance_mined_by_the_test = 0 self.sender_before_balance = 0 self.receiver_before_balance = 0
def test_inserting_schedule_sets_end_datetime(good_scheduled_message): end_datetime_in_utc = good_scheduled_message.end_datetime_in_utc scheduler_under_test = Scheduler() scheduler_under_test.schedule_message(good_scheduled_message) message = scheduler_under_test.get_messages()[0] end_datetime = message.end_datetime_in_utc assert end_datetime.isoformat() == end_datetime_in_utc.isoformat()
def message_schedule(person_name, message, expiration_datetime, ical, start_datetime): try: click.echo("Scheduling message for person {}".format(person_name)) scheduler = Scheduler() start_datetime = arrow.utcnow() if start_datetime is None: click.echo("Setting start time to now") else: start_datetime = arrow.get(start_datetime) click.echo("Setting custom start time to %s" % start_datetime) if ical is None: ical = 'FREQ=DAILY;COUNT=1' if expiration_datetime is None: expiration_datetime = start_datetime.replace(hours=1) message = ScheduledMessage(StartDateTimeInUtc=start_datetime, ical=ical, Body=message, PersonName=person_name, EndDateTimeInUtc=expiration_datetime) scheduler.schedule_message(message) except Exception: click.echo(traceback.print_exc()) exit(2)
def get_messages(self, **kwargs): person_name = kwargs.get('PersonName', '') self.sqs_msgs = [] """get all available messages off the queue""" print "Checking message queue" messages = self.queue.receive_messages(MessageAttributeNames=[ 'NoMoreOccurrences', 'ExpirationDateTimeInUtc', 'PersonName', 'Voice', 'UUID' ], WaitTimeSeconds=20, MaxNumberOfMessages=10) if len(messages) > 0: for m in messages: qm = QueuedMessage(QueuedMessage=m) if not qm.person_name == person_name: continue if qm.person_name not in self.messages: logging.info("First message for " + qm.person_name) self.messages[qm.person_name] = [] logging.info('Adding message:\n') logging.info(qm.body) self.messages[qm.person_name].append(qm) logging.info('Total messages so far: ' + str(len(self.messages[qm.person_name]))) self.sqs_msgs.append(m) scheduler = Scheduler() scheduler.update_queue_status(qm.uuid_key, person_name, False)
def __init__(self,local_mapping_file): logger.init('./globalhapi.log',logging.INFO) self.logger=logger.getLogger('global',logging.INFO) # self.topo = topo # self.locals = [0,1] #{0:"bala"} # self.dp_to_local = {1:0,2:1} self.locals = read_mapping_from_file(local_mapping_file).keys() self.dp_to_local = get_reverse_mapping_from_file(local_mapping_file) self.logger.info(self.locals) self.logger.info(self.dp_to_local) self.sockets = {} self.link_bw = {}#outer layer self.link_ltcy = {} self.local_to_buf_size = {} self.dp_to_tcam_size = {} # self.dp_to_local = {1:0,2:0,3:1,4:1}#for lineartopo2 self.flows = {}# all flows now self.flows_new = {} #flows updating self.flows_to_schedule = {} #flow to update self.status_num = 0 #for start update # self.flows_new = {"10.0.0.110.0.0.25001":flowdes0} self.scheduler = Scheduler() self.tag_flows_temp = {} self.schedule_result = {} eventlet.spawn(self.run_fd_server)
def __init__(self, parent=None): super(MainApp, self).__init__(parent) # set up GUI self.setupUi(self) self.taskListElems.setAlignment(QtCore.Qt.AlignTop) self.currentTaskElem.setAlignment(QtCore.Qt.AlignTop) self.currentTask.setStyleSheet("QScrollArea {background-color:transparent;}"); self.currentTaskContents.setStyleSheet("background-color:transparent;"); self.taskList.setStyleSheet("QScrollArea {background-color:transparent;}"); self.taskListContents.setStyleSheet("background-color:transparent;"); self.setStyleSheet("#mainWindow{background-color:white;}") # connect signals self.add_task_signal.connect(self.add_task) self.pop_task_signal.connect(self.pop_task) self.clear_tasks_signal.connect(self.clear_tasks) self.update_time_signal.connect(self.update_time) self.update_position_signal.connect(self.update_position) # initialize cloud components self.stt = stt.SpeechToTextManager("google_key.json") self.nlp = nlp.NaturalProcessingLanguageGoogleCloud() self.tts = tts.TextToSpeechManager() th = threading.Thread(target=self.get_task_thread_body, daemon=True) th.start() # initialize scheduler self.time_daemon = TimeDaemon(min_every_sec=1, update_time_signal=self.update_time_signal) threading.Thread(target=self.time_daemon.run, args=()).start() # need to set current GPS position here self.scheduler = Scheduler(current_position="Cecina", time_daemon=self.time_daemon, polling_sec=5, task_finished_signal=self.pop_task_signal, update_position_signal=self.update_position_signal)
def __init__(self, config: Config): self._scheduler = Scheduler() self._email_notification = EmailNotification(config.get('smtp'), config.get('recipients')) for site in config.get('sites'): self._scheduler.every_minutes(Site(site), self.parse) logger.info(f"Will be parsing {len(config.get('sites'))} site(s)")
def test_inserting_schedule_sets_start_datetime(good_scheduled_message): start_datetime_in_utc = good_scheduled_message.start_datetime_in_utc scheduler_under_test = Scheduler() scheduler_under_test.schedule_message(good_scheduled_message) message = scheduler_under_test.get_messages()[0] start_date = message.start_datetime_in_utc assert start_date.isoformat() == start_datetime_in_utc.isoformat()
def test_inserting_recurring_schedule_sets_ical(good_scheduled_message): test_ical = good_scheduled_message.ical scheduler_under_test = Scheduler() scheduler_under_test.schedule_message(good_scheduled_message) message = scheduler_under_test.get_messages()[0] ical = message.ical assert ical == test_ical
def add_sina_news_tasks(scheduler: Scheduler): for keyword in config.KEYWORDS: search_key = urllib.parse.quote(keyword) url = f'https://search.sina.com.cn/?q={search_key}&range=all&c=news&sort=time' metadata = { 'keyword': keyword, } scheduler.append_request_task(Task(url, '', '', metadata=metadata))
def run_scheduler(): scheduler = Scheduler(taskdb=get_taskdb(), projectdb=get_projectdb(), newtask_queue=self.newtask_queue, status_queue=self.status_queue, out_queue=self.scheduler2fetcher, data_path="./test/data/") scheduler.UPDATE_PROJECT_INTERVAL = 0.05 scheduler.LOOP_INTERVAL = 0.01 run_in_thread(scheduler.xmlrpc_run, port=self.scheduler_xmlrpc_port) scheduler.run()
def add_weibo_topic_list_tasks(scheduler: Scheduler): for keyword in config.KEYWORDS: search_key = urllib.parse.quote(keyword) url = f'https://s.weibo.com/topic?q={search_key}&pagetype=topic&topic=1&Refer=weibo_topic' metadata = { 'keyword': keyword, } scheduler.append_request_task(Task(url, '', '', metadata=metadata))
def add_gov_tasks(scheduler: Scheduler): for keyword in config.KEYWORDS: search_key = urllib.parse.quote(keyword) url = f'http://sousuo.gov.cn/s.htm?t=govall&q={search_key}' metadata = { 'keyword': keyword, } scheduler.append_request_task(Task(url, '', '', metadata=metadata))
def message_schedule(person_name, message, ical, count, frequency, lexbot, interval, timezone, start_date, start_time, end_date, bot_names, ice_breaker, required_bots, introduction, end_time): try: print ice_breaker print required_bots click.echo("Scheduling message for person {}".format(person_name)) scheduler = Scheduler() if not timezone: timezone = tzlocal.get_localzone().zone click.echo('Timezone: {}'.format(timezone)) if start_time is None: start_time = arrow.now(timezone).format('HH:mm') if start_date is None: start_date = arrow.now(timezone).format('YYYY-MM-DD') start_datetime = arrow.get( '{} {}'.format(start_date, start_time)) \ .replace(tzinfo=tz.gettz(timezone)).to('UTC') if end_time is None: end_time = start_time.format('HH:mm') if end_date is None: end_date = start_datetime.replace(years=10).format('YYYY-MM-DD') end_datetime = arrow.get( '{} {}'.format(end_date, end_time)) \ .replace(tzinfo=tz.gettz(timezone)).to('UTC') message = ScheduledMessage(StartDateTimeInUtc=start_datetime, ical=ical, Body=message, PersonName=person_name, Frequency=frequency, Count=count, Lexbot=lexbot, TimeZone=timezone, Interval=interval, BotNames=bot_names, IceBreaker=ice_breaker, Introduction=introduction, RequiredBots=required_bots, EndDateTimeInUtc=end_datetime) scheduler.schedule_message(message) click.echo('Start Time: {}'.format(start_datetime)) click.echo('End Time: {}'.format(end_datetime)) if ical: click.echo('ical:\n{}'.format(ical)) print "Next: {}".format(message.next_occurrence_local) print message.to_ical() except Exception: click.echo(traceback.print_exc()) exit(2)
def message_delete(person_name, message_id, verbose): log = logging.getLogger('PollexyCli') if verbose: os.environ['LOG_LEVEL'] = 'DEBUG' log.setLevel(logging.DEBUG) s = Scheduler() click.echo('Deleting message') s.delete_message(Key=message_id, PersonName=person_name)
def add_cnr_search_task(scheduler: Scheduler): for keyword in config.KEYWORDS: search_key = urllib.parse.quote(keyword) url = (f'http://was.cnr.cn/was5/web/search?page=2&channelid=234439&searchword={search_key}' f'&keyword={search_key}&orderby=LIFO&was_custom_expr=%28{search_key}%29&perpage=10' f'&outlinepage=1&searchscope=×cope=×copecolumn=&orderby=LIFO&andsen=&total=&orsen=&exclude=') scheduler.append_request_task(Task(url, '', '', metadata={ 'keyword': keyword }))
def add_weibo_info_tasks(scheduler: Scheduler): for item in WeiboTopicItem.objects().order_by('title'): search_key = urllib.parse.quote(item.title) url = f'https://m.s.weibo.com/ajax_topic/trend?q={search_key}&time=30d' metadata = { 'keyword': item.keyword, 'title': item.title, } scheduler.append_request_task(Task(url, '', '', metadata=metadata))
def add_cctv_tasks(scheduler: Scheduler): for keyword in config.KEYWORDS: search_key = urllib.parse.quote(keyword) url = (f'https://search.cctv.com/search.php?qtext={search_key}&sort=relevance&type=web&vtime=' f'&datepid=1&channel=&page=1') metadata = { 'keyword': keyword, } scheduler.append_request_task(Task(url, '', '', metadata=metadata))
def add_china_cdc_tasks(scheduler: Scheduler): for keyword in config.KEYWORDS: search_key = urllib.parse.quote(keyword) url = (f'http://www.chinacdc.cn/was5/web/search?searchword={search_key}&channelid=233877×cope=&' f'timescopecolumn=&orderby=-%E6%97%A5%E6%9C%9F&perpage=10&searchscope=') metadata = { 'keyword': keyword, } scheduler.append_request_task(Task(url, '', '', metadata=metadata))
def testScheduler(self): print "testScheduler():" self.pm = ProcessorMonitor(self.y1) self.proc.addMonitor(self.pm) s = Scheduler(self.y1, self.proc) s.run() Utils.dumpSchedule(s.getSchedule(), self.s1) Utils.dumpYaml(self.pm.getResults(), self.e1) print
def main(): """ Scheduler scripts, reads from the database all schedules and determines if any spider should be executed at this time. """ #path = dirname(realpath(__file__)) project_conf = get_project_conf() scheduler = Scheduler(project_conf) scheduler.run_scheduler()
def runAll(self): # create schedule and run it input = self.workload['input'] schedulesummary = self.workload['schedulesummary'] schedule = self.workload['schedule'] executions = self.workload['executions'] self.log.info("") self.log.info("##############################################") self.log.info("# Running " + input) self.log.info("##############################################") self.log.info("") proc = Processor() w = Utils.loadYaml(input) pm = ProcessorMonitor(w) proc.addMonitor(pm) sc = Scheduler(w, proc) sc.run() Utils.dumpSchedule(sc.getSchedule(), schedulesummary) Utils.dumpYaml(sc.getSchedule(), schedule) Utils.dumpYaml(pm.getResults(), executions) # plot the schedule p = Plot(sc.getSchedule(), pm.getResults(), sc.getH(), sc.getI(), self.wid) p.create() # verify the schedule v = Verifier(Utils.loadYaml(executions), sc.getH(), input) v.verifySchedule() return p.getFigure()
def add_china_news_tasks(scheduler: Scheduler): for keyword in config.KEYWORDS: search_key = urllib.parse.quote(keyword) url = f'http://sou.chinanews.com/search.do' body = { 'q': search_key, } metadata = { 'keyword': keyword, } scheduler.append_request_task(Task(url, '', '', method='POST', body=body, metadata=metadata))
def test_new_message_is_ready(good_scheduled_message): msg = ScheduledMessage( StartDateTimeInUtc=arrow.get('2012-01-01 01:01 UTC'), ical="FREQ=DAILY", Body="Test Message Body", PersonName="Testperson", EndDateTimeInUtc=arrow.get('2027-01-01 01:01 UTC')) scheduler_under_test = Scheduler() scheduler_under_test.schedule_message(msg) m = scheduler_under_test.get_messages()[0] assert m.is_message_ready
def test_sanity(self): test_scheduler = TestingScheduler(max_jobs=SchedulerTests.MAX_JOBS) self.assertEqual(test_scheduler.time_unit, TimeUnitEnum.SECONDS) self.assertEqual(len(test_scheduler), SchedulerTests.MAX_JOBS) scheduler = Scheduler(max_jobs=SchedulerTests.MAX_JOBS) self.assertEqual(scheduler.time_unit, TimeUnitEnum.HOURS) self.assertEqual(len(scheduler), SchedulerTests.MAX_JOBS) self.assertFalse(scheduler.is_alive()) self.assertFalse(scheduler.daemon)
def test_ignore_expired_messages(good_scheduled_message): msg = ScheduledMessage( StartDateTimeInUtc=arrow.get('2012-01-01 01:01 UTC'), ical="FREQ=DAILY", Body="Test Message Body", PersonName="Testperson", EndDateTimeInUtc=arrow.get('2014-01-01 01:01 UTC')) compare_date = arrow.utcnow() scheduler_under_test = Scheduler() scheduler_under_test.schedule_message(msg) m = scheduler_under_test.get_messages(compare_date, True) assert len(m) == 0
def bench(self): orders = self.orderList.orders[:] i = self.startValue while i <= len(orders): pprint("PERF Number of orders = " + str(i), BLUE) self.orderList.orders = orders[:i] scheduler = Scheduler(self.plant, self.orderList) t = time() scheduler.start() t = time() - t self.addCairoPlotTime(t) self.addGnuPlotTime(i, t) i += 1
def fail_speech(self, **kwargs): logging.info('Speech failed: ' + kwargs.get('Reason', 'Unknown Reason')) dont_delete = kwargs.get('DontDelete', False) if (dont_delete): logging.info('We are NOT deleting the original SQS messages') return for m in self.sqs_msgs: scheduler = Scheduler() qm = QueuedMessage(QueuedMessage=m) logging.info("Setting messages InQueue to False") scheduler.update_queue_status(qm.uuid_key, qm.person_name, False) self.delete_sqs_msgs()
def trigger(): if not request.content_type == 'application/json': return Response('Error: Invalid Content-Type header', status=400, mimetype='text/plain') try: scheduler = Scheduler(request.json, 420) scheduler.start() except SchedulerError as e: return Response('Error: ' + str(e), status=400, mimetype='text/plain') return Response(f"Launched scheduler", status=200, mimetype='text/plain')
def schedule(self): """ Runs the Scheduler with the OrderList from orderListName on the Plant with plantName. """ scheduler = Scheduler(self.plant, self.orderList) evaluator = Evaluator.fromXmlFile(self.configFilename, self.plant) result = scheduler.start() if result != None: solutions = parseSolutions(result, self.plant, self.orderList) evaluator.evaluate(solutions) best = bestSolution(solutions) best.unNormalize(self.normValue) print best
def run_scheduler(): scheduler = Scheduler(taskdb=get_taskdb(), projectdb=get_projectdb(), newtask_queue=self.newtask_queue, status_queue=self.status_queue, out_queue=self.scheduler2fetcher, data_path="./test/data/") scheduler.UPDATE_PROJECT_INTERVAL = 0.1 scheduler.LOOP_INTERVAL = 0.1 scheduler.INQUEUE_LIMIT = 10 scheduler._last_tick = time.time() # not dispatch cronjob run_in_thread(scheduler.xmlrpc_run, port=self.scheduler_xmlrpc_port) scheduler.run()
def runTasks(): log = logging.getLogger("main") # setup workloads = { 'workload1': { 'input':'input/workload1.yaml', 'schedulesummary':'output/schedule1.txt', 'schedule':'output/schedule1.yaml', 'executions':'output/executions1.yaml' }, 'workload2': { 'input':'input/workload2.yaml', 'schedulesummary':'output/schedule2.txt', 'schedule':'output/schedule2.yaml', 'executions':'output/executions2.yaml' } } for id,workload in workloads.iteritems(): # create schedule and run it input = workload['input'] schedulesummary = workload['schedulesummary'] schedule = workload['schedule'] executions = workload['executions'] log.info("") log.info("##############################################") log.info("# Running " + input) log.info("##############################################") log.info("") proc = Processor() w = Utils.loadYaml(input) pm = ProcessorMonitor(w) proc.addMonitor(pm) sc = Scheduler(w, proc) sc.run() Utils.dumpSchedule(sc.getSchedule(), schedulesummary) Utils.dumpYaml(sc.getSchedule(), schedule) Utils.dumpYaml(pm.getResults(), executions) # plot the schedule p = Plot(sc.getSchedule(), pm.getResults(), sc.getH(), sc.getI(), id) p.create() # verify the schedule v = Verifier(Utils.loadYaml(executions), sc.getH(), input) v.verifySchedule() # show the plot p.show()