def __init__(self): self.monotonic_time = 0.0 # list of functions that should be patched self.patchers = [ mock_patch('time.monotonic', side_effect=self.monotonic), mock_patch('time.sleep', side_effect=self.sleep), ]
def __init__(self, systemcalls): self.systemcalls = systemcalls # list of functions that should be patched self.patchers = [ mock_patch('socket.socket', side_effect=self.__get_socket), mock_patch('selectors.DefaultSelector', side_effect=self.__get_selector), mock_patch('time.monotonic', side_effect=self.systemcalls.monotonic), mock_patch('time.time', side_effect=self.systemcalls.time), # mock_patch('time.sleep', side_effect=self.systemcalls.sleep), ]
def test_predict(self): #Arrange num_prediction_steps = 1 model, input_data, input_params, image_generation_params, prediction_results = TestPrediction.get_inputs( num_prediction_steps) with mock_patch('operation.utils.imload', side_effect=patch_imload): #Act predictor = Prediction(model, input_params, image_generation_params) predictions = predictor.predict(input_data, num_prediction_steps) num_results = image_generation_params.batch_size * num_prediction_steps self.assertTrue(len(predictions), num_results) self.assertCountEqual( input_data.loc[:num_results - 1, image_generation_params.image_cols[0]].values, predictions.loc[:num_results, image_generation_params.image_cols[0]].values) self.assertCountEqual(list(predictions), [ constants.PANDAS_PREDICTION_COLUMN, constants.PANDAS_MATCH_COLUMN, image_generation_params.image_cols[0], image_generation_params.label_col ]) self.assertCountEqual( [0], predictions[constants.PANDAS_MATCH_COLUMN].unique())
def validate_base_model_creation(self, feature_model_name, module_path): #Arrange base_model = BaseModel(feature_model_name, input_shape) #Act & Assert with mock_patch(module_path) as base_mock: base_model.base_model() base_mock.assert_called_once()
def cnn_validate_base_model_creation(self, feature_model_name, module_path): #Arrange base_model = BaseModel(feature_model_name, input_shape) #Act & Assert with mock_patch(module_path) as base_mock: base_model._prepare_model = MagicMock() base_model.cnn(dimensions) base_mock.assert_called_once()
def test_batch_train_no_lr(self): #Arrange model, input_data, trainer = get_train_args() trainer._training_params.learning_rate = None trainer._training_params.number_of_epochs = 1 trainer._image_generation_params.batch_size = 40 trainer._dropbox_auth = None trainer._dropbox_dir = None #Mock the relevant calls model.train_on_batch = Mock(return_value = [random() for _ in model.metrics_names]) K.set_value = MagicMock() with mock_patch('operation.utils.imload', side_effect = patch_imload): #Act _ = trainer.batch_train(model, input_data) #Assert K.set_value.assert_not_called()
def test_predict_prediction_with_reduced_num_prediction_steps(self): #Arrange num_prediction_steps = 2 num_results = 5 model, input_data, input_params, image_generation_params, _ = TestPrediction.get_inputs( num_prediction_steps, num_results) with mock_patch('operation.utils.imload', side_effect=patch_imload): #Act predictor = Prediction(model, input_params, image_generation_params) predictions = predictor.predict(input_data[:num_results], num_prediction_steps) #Assert model.predict_generator.assert_called_once() _, args = model.predict_generator.call_args_list[0] self.assertEqual( 1, #Modified prediction steps args['steps'])
def batch_train(self, batch_id = 0, epoch_id = 0, number_of_epochs = 1): #Arrange model, input_data, trainer = get_train_args() #Arrange batch size batch_size = 40 trainer._image_generation_params.batch_size = batch_size number_of_batches = ceil(len(input_data) / batch_size) first_epoch_batches = number_of_batches - batch_id total_training_epochs = number_of_epochs - epoch_id total_training_batches = first_epoch_batches + number_of_batches * (total_training_epochs - 1) #Arrange batch and epoch parameters trainer._training_params.batch_id = batch_id trainer._training_params.epoch_id = epoch_id trainer._training_params.number_of_epochs = number_of_epochs #Mock the relevant calls model.fit_generator = Mock(return_value = [random() for _ in model.metrics_names]) model.train_on_batch = Mock(return_value = [random() for _ in model.metrics_names]) trainer._transformer.fit = MagicMock() K.set_value = MagicMock() with mock_patch('operation.utils.imload', side_effect = patch_imload): #Act response = trainer.batch_train(model, input_data) #Assert model.fit_generator.assert_not_called() #Assert train batch calls call_args_list = model.train_on_batch.call_args_list self.assertEqual(len(call_args_list), total_training_batches) #Assert model and learning rate self.assert_response_and_learning_rate(model, response) #Assert checkpoint calls self.assert_checkpoint(trainer._checkpoint_callback, total_training_batches, total_training_epochs)
def test_train(self): #Arrange model, input_data, trainer = get_train_args() #Mock the relevant calls model.fit_generator = MagicMock() model.train_on_batch = MagicMock() K.set_value = MagicMock() with mock_patch('operation.utils.imload', side_effect = patch_imload): #Act response = trainer.train(model, input_data) #Assert model.fit_generator.assert_called_once() model.train_on_batch.assert_not_called() _, args = model.fit_generator.call_args_list[0] self.assertTrue(isinstance(args['generator'], ImageDataIterator)) self.assertTrue(isinstance(args['validation_data'], ImageDataIterator)) self.assertTrue(args['epochs'], trainer._training_params.number_of_epochs) #Assert model and learning rate self.assert_response_and_learning_rate(model, response)
def batch_train_input_data_randomized(self, start_batch_id, is_randomized = True): #Arrange model, input_data, trainer = get_train_args() trainer._training_params.batch_id = start_batch_id trainer._training_params.epoch_id = 0 trainer._training_params.number_of_epochs = 1 trainer._image_generation_params.batch_size = 40 trainer._dropbox_auth = None trainer._dropbox_dir = None #Mock the relevant calls model.train_on_batch = Mock(return_value = [random() for _ in model.metrics_names]) trainer._transformer.fit = MagicMock() input_data.sample = MagicMock() with mock_patch('operation.utils.imload', side_effect = patch_imload): #Act _ = trainer.batch_train(model, input_data) #Assert if is_randomized: input_data.sample.assert_called_once() else: input_data.sample.assert_not_called()
def download_evts(calendar=None, in_loop=False): db = sqlite3.connect('evts.sqlite3') with mock_patch('pickle.dumps'): keys = list(_ev_entry(MagicMock()).keys()) db.execute( f'create table if not exists events (id PRIMARY KEY,{",".join(keys[1:])})' ) db.execute( f'create index if not exists id_date on events (root_id, startdate_index)' ) db.execute( f'''create view if not exists events_recurring as select a.*, count(*) - 1 as local_recurring from events a left join events b on a.root_id = b.root_id and b.startdate_index >= a.startdate_index-14 and b.startdate_index <= a.startdate_index+14 group by a.id ''' ) now = datetime.now(tzlocal()) tries_remaining = 2 if not in_loop else 5 while tries_remaining: try: tries_remaining -= 1 cals = s().calendarList().list().execute()['items'] break except Exception as e: print(e) if not tries_remaining: raise if in_loop: print('retrying in 30 seconds...') sleep(30) obj = { 'calendars': cals, 'timestamp': now, } calmap = {cal['id']: cal for cal in cals} allCals = get_visible_cals(cals) try: old_obj = load_evts(print_warning=False, partial=True) for cal in old_obj['calendars']: calmap[cal['id']]['syncToken'] = cal.get('syncToken') except FileNotFoundError: pass if calendar is not None: calsToDownload = [calendar] else: calsToDownload = allCals.values() for calId in calsToDownload: if not isinstance(calId, str) or '@' not in calId: continue print(f'downloading {calId}...') kwargs = { 'calendarId': calId, 'singleEvents': True, 'maxResults': 2500, 'syncToken': calmap[calId].get('syncToken'), } pagenum = 0 while True: pagenum += 1 if pagenum > 1: print(f' downloading page {pagenum}...') try: r = s().events().list(**kwargs).execute() except HttpError as e: if int(e.resp['status']) == 410: print(" 410'd, redownloading...") db.execute(f'''delete from events where calendar = ?''', (calId,)) calmap[calId].pop('syncToken', None) if 'syncToken' in kwargs: del kwargs['syncToken'] if 'pageToken' in kwargs: del kwargs['pageToken'] pagenum = 0 continue else: raise except Exception as e: print(repr(e)) entries = [] deleting = [] for e in r['items']: if e['status'] == 'cancelled': deleting.append((e['id'],)) else: ev = Event.unpkg(e) ev.calendar = calId entries.append(_ev_entry(ev)) db.executemany( f'''insert into events values ({",".join(f":{key}" for key in keys)}) on conflict(id) do update set {",".join(f"{key}=:{key}" for key in keys[1:])}''', entries, ) db.executemany(f'''delete from events where id = ?''', deleting) if 'nextPageToken' in r: kwargs['pageToken'] = r['nextPageToken'] continue if 'nextSyncToken' in r: calmap[calId]['syncToken'] = r['nextSyncToken'] break db.commit() with open('evts.yaml', 'w') as f: yaml.dump(obj, f, default_flow_style=False) with open('evts.pickle', 'wb') as f: pickle.dump(obj, f, protocol=-1) db.commit() return obj