def main(args): """ Runs the pipeline on a given input/output pair and dataset. @params args.dataset: subfolder of data/ args.name: name of the image to process, input and output are stored under data/args.dataset/args.name """ # Setup params params = Parameters() processor = Processor(params) # I/O paths outputDir = os.path.join(OUTPUT_DIR, args.dataset) inputDir = os.path.join(DATA_DIR, args.dataset) # Result struct r = Result() r.dataset = args.dataset r.name = args.name r.dataPath = os.path.join(inputDir,args.name) r.outputPath = os.path.join(outputDir,args.name) r.error = "" # Run r = processor.process(r) print "---------------------------" print "* Processed image %s/%s" % (args.dataset, args.name) print " - time\t%.2f s." % r.computation_time print " - PSNR:\t %5.2f dB" % r.psnr print " - input:\t %5.2f %%" % (r.compression_up*100) print " - output:\t %5.2f %%" % (r.compression_down*100)
def __init__(self, master=None): super().__init__(master) self.master = master self.master.title(CONST_APP_TITLE) self.master.geometry('600x400+400+100') self.master.resizable(0, 0) self.config(width=500, height=500) self.place(x=50, y=0) self.application_widgets() self.processor = Processor()
def __init__(self, config): self.executor = ReadFactory(config).get_executor() self.processor = Processor(config) self.writer = WriterFactory().instance_writer(config, self.processor.aggregation_output_struct, self.processor.enumerate_output_aggregation_field) self._isAnalysis = False if ("analysis" in config.content.keys()): self._isAnalysis = True self.analysis = AnalysisFactory(config, self.processor.aggregation_output_struct, self.processor.enumerate_output_aggregation_field)
def show_solution(self): top = tk.Toplevel(self) top.geometry('500x400+400+100') top.title(RESULT_TITLE) scroll = tk.Scrollbar(top) c = tk.Canvas(top, yscrollcommand=scroll.set) scroll.config(command=c.yview) scroll.pack(side='right', fill='y') top_frame = tk.Frame(c) top_frame.config(width=500) c.pack(side='left', fill='both', expand=True) c.create_window(0, 0, window=top_frame, anchor='nw') top.update() c.config(scrollregion=c.bbox('all')) response = self.run_data() self.processor = Processor() tk.Label(top_frame, text=CONST_TOPIC_NAME).grid(padx=(70, 80), pady=(5, 30), row=0, column=0) tk.Label(top_frame, text=CONST_RESULT_PAGES).grid(padx=(70, 80), pady=(5, 30), row=0, column=1) for i in range(len(response[TOPICS_RESPONSE])): if response[INCLUDE_RESPONSE][i] == 'true': tk.Label(top_frame, text=response[TOPICS_RESPONSE][i]).grid(padx=(70, 80), pady=(5, 30), row=i + 1, column=0) tk.Label(top_frame, text=response[PAGES_RESPONSE][i]).grid(padx=(70, 80), pady=(5, 30), row=i + 1, column=1) tk.Label(top_frame, text=CONST_READERS).grid( padx=(70, 80), pady=(5, 30), row=len(response[TOPICS_RESPONSE]) + 1, column=0) tk.Label(top_frame, text=response[READERS_RESPONSE]).grid( padx=(70, 80), pady=(5, 30), row=len(response[TOPICS_RESPONSE]) + 1, column=1)
def main(): """ Usage: python3 app.py input.txt' or 'python3 < input.txt' """ log.info('Begin processing...') t = process_time() processor = Processor() # Accept input from two types of sources: # a filename passed in command line arguments or STDIN. with open(sys.argv[1], 'r') if len(sys.argv) > 1 else sys.stdin as f: for line in f: processor.parse_event(line) log.info('Finished processing in {0:.3f} seconds'.format(process_time() - t)) summary = processor.generate_summary() processor.write_output(summary)
class Dispatcher: def __init__(self, config, file_config): self.executor = ReadFactory(config, file_config).get_executor() self.processor = Processor(config) self.writers = WriterFactory().get_writers( config, self.processor.aggregation_output_struct, self.processor.enumerate_output_aggregation_field) self._isAnalysis = False if "analysis" in config.content.keys(): self._isAnalysis = True self.analysis = AnalysisFactory( config, self.processor.aggregation_output_struct, self.processor.enumerate_output_aggregation_field) def run_pipeline(self): processor_part = self.processor.get_pipeline_processing() write_funcs = [w.get_write_lambda() for w in self.writers] write_func = lambda rdd: [w(rdd) for w in write_funcs] # pipeline = lambda rdd: write_lambda(processor_part(rdd)) if self._isAnalysis: analysis_lambda = self.analysis.get_analysis_lambda() pipeline = lambda rdd: self._all_pipeline( rdd, processor_part, write_func, analysis_lambda) else: analysis_lambda = lambda x: x pipeline = lambda rdd: self._all_pipeline( rdd, processor_part, write_func, analysis_lambda) self.executor.set_pipeline_processing(pipeline) self.executor.run_pipeline() def _all_pipeline(self, rdd, processor_part, write_part, analysis_part): processed = processor_part(rdd) write_part(processed) analysis_part(processed) def stop_pipeline(self): self.executor.stop_pipeline()
def get_parser(add_help=False): # parameter priority: command line > config > default parent_parser = Processor.get_parser(add_help=False) parser = argparse.ArgumentParser( add_help=add_help, parents=[parent_parser], description='Spatial Temporal Graph Convolution Network') # region arguments yapf: disable # evaluation parser.add_argument('--show_topk', type=int, default=[1, 5], nargs='+', help='which Top K accuracy will be shown') # optim parser.add_argument('--base_lr', type=float, default=0.01, help='initial learning rate') parser.add_argument('--step', type=int, default=[], nargs='+', help='the epoch where optimizer reduce the learning rate') parser.add_argument('--optimizer', default='SGD', help='type of optimizer') parser.add_argument('--nesterov', type=str2bool, default=True, help='use nesterov or not') parser.add_argument('--weight_decay', type=float, default=0.0001, help='weight decay for optimizer') # endregion yapf: enable return parser
def test__number__(self): config = Config(CONFIG_PATH_NUM) p = Processor(config) self.assertIsInstance( p.transformation, types.LambdaType, "Processor#transformation should be a lambda object")
import sys import torch from torch.backends import cudnn from processor.processor import Processor torch.backends.cudnn.deterministic = False cudnn.benchmark = True # https://discuss.pytorch.org/t/what-does-torch-backends-cudnn-benchmark-do/5936 torch.cuda.empty_cache() # release cache if __name__ == '__main__': proc = Processor(sys.argv[1:]) proc.start()
import os from processor.processor import Processor if __name__ == "__main__": current_path = os.path.dirname(os.path.abspath(__file__)) data_path = os.path.join(current_path, "data") resource_path = os.path.join(data_path, "resources") input_path = os.path.join(data_path, "inputs", "1984.txt") output_path = os.path.join(data_path, "outputs") processor = Processor(resource_path=resource_path, config={}) processor.process(input_path=input_path, output_path=output_path)
class TestProcessor(unittest.TestCase): projectdb_path = './test/data/project.db' @classmethod def setUpClass(self): shutil.rmtree('./test/data/', ignore_errors=True) os.makedirs('./test/data/') def get_projectdb(): return projectdb.ProjectDB(self.projectdb_path) self.projectdb = get_projectdb() self.in_queue = Queue(10) self.status_queue = Queue(10) self.newtask_queue = Queue(10) self.result_queue = Queue(10) def run_processor(): self.processor = Processor(get_projectdb(), self.in_queue, self.status_queue, self.newtask_queue, self.result_queue) self.processor.CHECK_PROJECTS_INTERVAL = 0.1 self.processor.run() self.process = run_in_thread(run_processor) time.sleep(1) @classmethod def tearDownClass(self): if self.process.is_alive(): self.processor.quit() self.process.join(2) assert not self.process.is_alive() shutil.rmtree('./test/data/', ignore_errors=True) def test_10_update_project(self): self.assertEqual(len(self.processor.projects), 0) self.projectdb.insert( 'test_project', { 'name': 'test_project', 'group': 'group', 'status': 'TODO', 'script': open('libs/sample_handler.py', 'r').read(), 'comments': 'test project', 'rate': 1.0, 'burst': 10, }) task = { "process": { "callback": "on_start" }, "project": "not_exists", "taskid": "data:,on_start", "url": "data:,on_start" } self.in_queue.put((task, {})) time.sleep(1) self.assertTrue(self.status_queue.empty()) self.assertEqual(len(self.processor.projects), 1) def test_30_new_task(self): self.assertTrue(self.status_queue.empty()) self.assertTrue(self.newtask_queue.empty()) task = { "process": { "callback": "on_start" }, "project": "test_project", "taskid": "data:,on_start", "url": "data:,on_start" } fetch_result = { "orig_url": "data:,on_start", "content": "on_start", "headers": {}, "status_code": 200, "url": "data:,on_start", "time": 0, } self.in_queue.put((task, fetch_result)) time.sleep(1) self.assertFalse(self.status_queue.empty()) while not self.status_queue.empty(): self.status_queue.get() self.assertFalse(self.newtask_queue.empty()) def test_40_index_page(self): task = None while not self.newtask_queue.empty(): task = self.newtask_queue.get() self.assertIsNotNone(task) fetch_result = { "orig_url": task['url'], "content": "<html><body><a href='http://binux.me'>binux</a></body></html>", "headers": {}, "status_code": 200, "url": task['url'], "time": 0, } self.in_queue.put((task, fetch_result)) time.sleep(1) self.assertFalse(self.status_queue.empty()) self.assertFalse(self.newtask_queue.empty()) task = self.newtask_queue.get() self.assertEqual(task['url'], 'http://binux.me/')
def run_processor(): self.processor = Processor(get_projectdb(), self.in_queue, self.status_queue, self.newtask_queue, self.result_queue) self.processor.CHECK_PROJECTS_INTERVAL = 0.1 self.processor.run()
class Application(tk.Frame): def __init__(self, master=None): super().__init__(master) self.master = master self.master.title(CONST_APP_TITLE) self.master.geometry('600x400+400+100') self.master.resizable(0, 0) self.config(width=500, height=500) self.place(x=50, y=0) self.application_widgets() self.processor = Processor() def application_widgets(self): self.create_frames() self.create_buttons() self.create_labels() self.create_entrys() def create_frames(self): self.pages_frame = tk.Frame(self, width=500, height=50) self.pages_frame.place(x=0, y=40) self.topics_frame = tk.Frame(self, width=500, height=250) self.topics_frame.place(x=0, y=100) def create_labels(self): tk.Label(self.pages_frame, text=CONST_TOTAL_PAGES).place(x=100, y=10) tk.Label(self.topics_frame, width=10, height=2, justify='center', text=CONST_TOPIC_NAME).place(x=10, y=40) tk.Label(self.topics_frame, height=2, justify='center', text=CONST_MIN_PAGES).place(x=113, y=40) tk.Label(self.topics_frame, height=2, justify='center', text=CONST_MAX_PAGES).place(x=230, y=40) tk.Label(self.topics_frame, height=2, justify='center', text=CONST_READERS).place(x=350, y=40) def create_entrys(self): self.max_pages = tk.Entry(self.pages_frame) self.max_pages.place(x=300, y=10) self.topic_name_entry = tk.Entry(self.topics_frame) self.topic_name_entry.config(width=10, justify='center') self.topic_name_entry.place(x=10, y=100) self.topic_min_entry = tk.Entry(self.topics_frame) self.topic_min_entry.config(width=10, justify='center') self.topic_min_entry.place(x=130, y=100) self.topic_max_entry = tk.Entry(self.topics_frame) self.topic_max_entry.config(width=10, justify='center') self.topic_max_entry.place(x=245, y=100) self.topic_readers_entry = tk.Entry(self.topics_frame) self.topic_readers_entry.config(width=10, justify='center') self.topic_readers_entry.place(x=365, y=100) def create_buttons(self): self.solve_button = tk.Button(self) self.solve_button.config(width=10, height=1, state='disabled') self.solve_button['text'] = CONST_SOLVE self.solve_button['command'] = self.create_data self.solve_button.place(x=200, y=325) #Botón para agregar nuevo tema self.add_topic_button = tk.Button(self.topics_frame) self.add_topic_button.config(width=10, height=1) self.add_topic_button.place(x=270, y=180) self.add_topic_button['text'] = CONST_ADD_TOPIC self.add_topic_button['command'] = self.add_topic ##Botón para consultar temas guardados self.check_topics_button = tk.Button(self.topics_frame) self.check_topics_button.config(width=10, height=1) self.check_topics_button.place(x=110, y=180) self.check_topics_button['text'] = CONST_CHECK_TOPICS self.check_topics_button['command'] = self.create_topics_window def create_topics_window(self): self.top = tk.Toplevel(self) self.top.geometry('550x400+400+100') self.top.title(TEMAS) scroll = tk.Scrollbar(self.top) c = tk.Canvas(self.top, yscrollcommand=scroll.set) scroll.config(command=c.yview) scroll.pack(side='right', fill='y') top_frame = tk.Frame(c) top_frame.config(width=500) c.pack(side='left', fill='both', expand=True) c.create_window(0, 0, window=top_frame, anchor='nw') # texto =tk.Label(top_frame, wraplength=380) # texto.pack() self.top.update() c.config(scrollregion=c.bbox('all')) tk.Label(top_frame, text=CONST_TOPIC_NAME).grid(padx=(10, 10), pady=(5, 30), row=0, column=0) tk.Label(top_frame, text=CONST_MIN_PAGES).grid(padx=(10, 10), pady=(5, 30), row=0, column=1) tk.Label(top_frame, text=CONST_MAX_PAGES).grid(padx=(10, 10), pady=(5, 30), row=0, column=2) tk.Label(top_frame, text=CONST_READERS).grid(padx=(10, 10), pady=(5, 30), row=0, column=3) y = 1 for key in self.processor.info: tk.Label(top_frame, text=key).grid(padx=(10, 10), pady=(5, 30), row=y, column=0) tk.Label(top_frame, text=self.processor.info[key][MIN_KEY]).grid(padx=(10, 10), pady=(5, 30), row=y, column=1) tk.Label(top_frame, text=self.processor.info[key][MAX_KEY]).grid(padx=(10, 10), pady=(5, 30), row=y, column=2) tk.Label(top_frame, text=self.processor.info[key][READERS_KEY]).grid( padx=(10, 10), pady=(5, 30), row=y, column=3) y = y + 1 def valid_entrys(self): # print (self.topic_name_entry.get()) # print (self.topic_min_entry.get()) # print( self.topic_max_entry.get()) # print (self.topic_readers_entry.get()) if self.topic_name_entry.get() == '': True return False def add_topic(self): dic = { READERS_KEY: int(self.topic_readers_entry.get()), MAX_KEY: int(self.topic_max_entry.get()), MIN_KEY: int(self.topic_min_entry.get()), } self.processor.add_topic({self.topic_name_entry.get(): dic}) self.topic_name_entry.delete(0, 100) self.topic_max_entry.delete(0, 100) self.topic_min_entry.delete(0, 100) self.topic_readers_entry.delete(0, 100) self.topic_name_entry.focus() self.max_pages.config(state='disabled') self.solve_button.config(state='normal') def show_solution(self): top = tk.Toplevel(self) top.geometry('500x400+400+100') top.title(RESULT_TITLE) scroll = tk.Scrollbar(top) c = tk.Canvas(top, yscrollcommand=scroll.set) scroll.config(command=c.yview) scroll.pack(side='right', fill='y') top_frame = tk.Frame(c) top_frame.config(width=500) c.pack(side='left', fill='both', expand=True) c.create_window(0, 0, window=top_frame, anchor='nw') top.update() c.config(scrollregion=c.bbox('all')) response = self.run_data() self.processor = Processor() tk.Label(top_frame, text=CONST_TOPIC_NAME).grid(padx=(70, 80), pady=(5, 30), row=0, column=0) tk.Label(top_frame, text=CONST_RESULT_PAGES).grid(padx=(70, 80), pady=(5, 30), row=0, column=1) for i in range(len(response[TOPICS_RESPONSE])): if response[INCLUDE_RESPONSE][i] == 'true': tk.Label(top_frame, text=response[TOPICS_RESPONSE][i]).grid(padx=(70, 80), pady=(5, 30), row=i + 1, column=0) tk.Label(top_frame, text=response[PAGES_RESPONSE][i]).grid(padx=(70, 80), pady=(5, 30), row=i + 1, column=1) tk.Label(top_frame, text=CONST_READERS).grid( padx=(70, 80), pady=(5, 30), row=len(response[TOPICS_RESPONSE]) + 1, column=0) tk.Label(top_frame, text=response[READERS_RESPONSE]).grid( padx=(70, 80), pady=(5, 30), row=len(response[TOPICS_RESPONSE]) + 1, column=1) def run_data(self): return self.processor.run_data() def create_data(self): print('creating datazinc file..') self.topic_name_entry.delete(0, 100) self.topic_max_entry.delete(0, 100) self.topic_min_entry.delete(0, 100) self.topic_readers_entry.delete(0, 100) self.max_pages.config(state='normal') self.max_pages.focus() self.processor.pages = int(self.max_pages.get()) self.max_pages.delete(0, 100) self.solve_button.config(state='disabled') self.processor.create_dzn() self.show_solution() print('Done!')
start_time = time.time() def generator(): case = Case() for cr in range(randrange(0, 19)): x = randrange(-1000, 1000) y = randrange(-1000, 1000) r = randrange(-360, 360) route = Route(x, y, r) for cc in range(randrange(1, 24)): command_type = choice(list(CommandType)) if command_type == CommandType.TURN: command = Command(command_type, randrange(-360, 360)) else: command = Command(command_type, randrange(-1000, 1000)) route.add_command(command) case.add_route(route) yield case for i in reversed(range(99)): g = generator() g_case = g.next() Processor.process(g_case) print("average x: %s, average y: %s, average worst: %s" % (round(g_case.sx, 4), round(g_case.sy, 4), round(g_case.aw, ))) print("--- %s seconds ---" % (time.time() - start_time)) print(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)
class TestProcessor(unittest.TestCase): projectdb_path = './test/data/project.db' @classmethod def setUpClass(self): shutil.rmtree('./test/data/', ignore_errors=True) os.makedirs('./test/data/') def get_projectdb(): return projectdb.ProjectDB(self.projectdb_path) self.projectdb = get_projectdb() self.in_queue = Queue(10) self.status_queue = Queue(10) self.newtask_queue = Queue(10) self.result_queue = Queue(10) def run_processor(): self.processor = Processor(get_projectdb(), self.in_queue, self.status_queue, self.newtask_queue, self.result_queue) self.processor.CHECK_PROJECTS_INTERVAL = 0.1 self.processor.run() self.process = run_in_thread(run_processor) time.sleep(1) @classmethod def tearDownClass(self): if self.process.is_alive(): self.processor.quit() self.process.join(2) assert not self.process.is_alive() shutil.rmtree('./test/data/', ignore_errors=True) def test_10_update_project(self): self.assertEqual(len(self.processor.projects), 0) self.projectdb.insert('test_project', { 'name': 'test_project', 'group': 'group', 'status': 'TODO', 'script': open('libs/sample_handler.py', 'r').read(), 'comments': 'test project', 'rate': 1.0, 'burst': 10, }) task = { "process": { "callback": "on_start" }, "project": "not_exists", "taskid": "data:,on_start", "url": "data:,on_start" } self.in_queue.put((task, {})) time.sleep(1) self.assertTrue(self.status_queue.empty()) self.assertEqual(len(self.processor.projects), 1) def test_30_new_task(self): self.assertTrue(self.status_queue.empty()) self.assertTrue(self.newtask_queue.empty()) task = { "process": { "callback": "on_start" }, "project": "test_project", "taskid": "data:,on_start", "url": "data:,on_start" } fetch_result = { "orig_url": "data:,on_start", "content": "on_start", "headers": {}, "status_code": 200, "url": "data:,on_start", "time": 0, } self.in_queue.put((task, fetch_result)) time.sleep(1) self.assertFalse(self.status_queue.empty()) while not self.status_queue.empty(): self.status_queue.get() self.assertFalse(self.newtask_queue.empty()) def test_40_index_page(self): task = None while not self.newtask_queue.empty(): task = self.newtask_queue.get() self.assertIsNotNone(task) fetch_result = { "orig_url": task['url'], "content": "<html><body><a href='http://binux.me'>binux</a></body></html>", "headers": {}, "status_code": 200, "url": task['url'], "time": 0, } self.in_queue.put((task, fetch_result)) time.sleep(1) self.assertFalse(self.status_queue.empty()) self.assertFalse(self.newtask_queue.empty()) task = self.newtask_queue.get() self.assertEqual(task['url'], 'http://binux.me/')
parser.add_argument('--num_of_vertices', type=int, default=358, help='The number of vertices') parser.add_argument('--gen_config_args', type=dict, default=dict(), help='The config of data generate') return parser if __name__ == '__main__': parser = get_parser() # load arg form config file p = parser.parse_args() if p.config is not None: with open(p.config, 'r') as f: default_arg = yaml.load(f) key = vars(p).keys() for k in default_arg.keys(): if k not in key: print('WRONG ARG: {}'.format(k)) assert (k in key) parser.set_defaults(**default_arg) arg = parser.parse_args() init_seed(0) processor = Processor(arg) processor.start()
class TestProcessor(unittest.TestCase): def setUp(self): self.processor = Processor() # event parsing def test_event_is_string(self): self.assertRaises(ValueError, self.processor.parse_event, 0) def test_parse_event_can_upack_string(self): self.assertRaises(ValueError, self.processor.parse_event, 'Add') def test_parse_event_has_valid_args(self): self.assertRaises(ParseError, self.processor.parse_event, 'Add Tom') def test_parse_dollars_has_valid_number(self): self.assertRaises(ValueError, self.processor.parse_dollars, '$fail') def test_parse_dollars_returns_valid_card_number(self): self.assertTrue( self.processor.parse_dollars('4111111111111111').isdigit()) # luhn def test_card_number_is_numberic_in_luhn_checksum(self): self.assertRaises(ValueError, self.processor.luhn_checksum, 'fail') def test_card_number_is_numberic_in_is_luhn_valid(self): self.assertRaises(ValueError, self.processor.is_luhn_valid, 'fail') def test_luhn_catches_invalid_invalid_numbers(self): self.assertFalse(self.processor.is_luhn_valid('1234567890123456')) # add def test_invalid_card_balance_equals_error(self): self.processor.add('User', '1234567890123456', '$4000') self.assertEqual(self.processor.db['User']['balance'], 'error') def test_balance_type_is_decimal(self): self.processor.add('User', '4111111111111111', '$4000') self.assertIsInstance(self.processor.db['User']['balance'], Decimal) # get account def test_nonexistant_account_name_raises_key_error(self): self.assertRaises(KeyError, self.processor.get_account_details, 'Non-Existent account') def test_missing_param_raises_key_error(self): self.processor.db['User'] = { 'card_number': '4111111111111111', 'limit': '$5000', 'balance': None } self.assertRaises(KeyError, self.processor.get_account_details, 'User') # charge def test_charge_with_bad_params_raises_type_error(self): self.processor.db['User'] = { 'card_number': '4111111111111111', 'limit': Decimal('9000'), 'balance': Decimal('9000') } self.assertRaises(TypeError, self.processor.charge, 'User', '$1000') def test_amount_over_limit_doesnt_charge(self): self.processor.db['User'] = { 'card_number': '4111111111111111', 'limit': Decimal('9000'), 'balance': Decimal('9000') } self.assertEqual(self.processor.charge('User', Decimal('1')), Decimal('9000')) # over 9000 ;) def test_invalid_card_is_not_charged(self): self.processor.add('User', '1234567890123456', Decimal('9000')) self.assertEqual(self.processor.charge('User', Decimal('1')), 'error') def test_credit_correctly_decreases_balance(self): self.processor.add('User', '4111111111111111', Decimal('9000')) self.processor.credit('User', Decimal()) # credit def test_credit_with_bad_params_raises_type_error(self): self.processor.db['User'] = { 'card_number': '4111111111111111', 'limit': Decimal('9000'), 'balance': Decimal('9000') } self.assertRaises(TypeError, self.processor.credit, 'User', '$1000') def test_invalid_card_is_not_credited(self): self.processor.add('User', '1234567890123456', Decimal('9000')) self.assertEqual(self.processor.credit('User', Decimal('1')), 'error')
def setUp(self): self.processor = Processor()