def execute(language):
language = language
data = Dataset(language)
print("{}: {} training - {} dev".format(language, len(data.trainset),
len(data.devset)))
baseline = Baseline(language)
estimator = SVC(gamma=300)
title = 'Spanish Learning Curves (SVM, γ=300)'
X, y = baseline.train(data.trainset)
plot_learning_curve(estimator,
title,
X,
y,
ylim=None,
n_jobs=1,
train_sizes=np.linspace(.1, 1.0, 5))
predictions = baseline.test(data.devset)
gold_labels = [sent['gold_label'] for sent in data.devset]
target_words = [sent['target_word'] for sent in data.devset]
prediction = []
for i in predictions:
prediction.append(i)
df = pd.DataFrame(columns=['target_word', 'prediction'])
df["target_word"] = target_words
df['gold_label'] = gold_labels
df['prediction'] = prediction
df.to_csv('out_s2.csv')
report_score(gold_labels, predictions)
def __init__(self):
Baseline.__init__(self)
self['config_changes_for_estimation'] = ConfigChangesForEstimation()
## set base_year and years to 2006 for HLCM for the psrc_parcel project
#self['config_changes_for_estimation']['household_location_choice_model'].merge({'base_year': 2006, 'years':(2006, 2006)})
self['cache_directory'] = '/Users/hana/urbansim_cache/psrc/cache_source_parcel'
self['estimation_database_configuration'] = EstimationDatabaseConfiguration(
database_name = 'psrc_2005_parcel_baseyear_change_estimation',
)
self['dataset_pool_configuration'] = DatasetPoolConfiguration(
package_order=['psrc_parcel', 'psrc', 'urbansim_parcel', 'urbansim', 'opus_core'],
)
self['datasets_to_cache_after_each_model'] = []
self["datasets_to_preload"] = {
'zone':{},
'household':{},
'building':{},
'parcel':{'package_name':'urbansim_parcel'},
#'development_template': {'package_name':'urbansim_parcel'},
#'development_template_component': {'package_name':'urbansim_parcel'},
#"job_building_type":{}
'job':{},
'person':{'package_name':'urbansim_parcel'},
"building_type":{'package_name':'urbansim_parcel'},
'travel_data':{},
}
self['base_year'] = 2000
self['years'] = (2000,2000)
#self['low_memory_mode'] = True
if 'models_in_year' in self.keys():
del self['models_in_year']
def __init__(self):
config = Baseline()
config_changes = {
'description': 'baseline travel model fast (no build)',
}
config.replace(config_changes)
from psrc.configs.create_travel_model_configuration import create_travel_model_configuration
travel_model_configuration = create_travel_model_configuration(
'baseline_travel_model_psrc_fast_no_build',
emme2_batch_file='MODELUSim.BAT ..\\triptabs',
mode='full')
config['travel_model_configuration'] = travel_model_configuration
config['travel_model_configuration']['locations_to_disaggregate'] = [
'parcel', 'building'
]
del config['travel_model_configuration'][2000]
##fast model doesn't have bank2 and bank3; disable macros using them
del config['travel_model_configuration']['export_macros'][
'tazvmt2.mac']
del config['travel_model_configuration']['export_macros'][
'tazvmt3.mac']
del config['travel_model_configuration']['matrix_variable_map'][
'bank2']
del config['travel_model_configuration']['matrix_variable_map'][
'bank3']
self.merge(config)
#if __name__ == "__main__":
# config = BaselineTravelModelFast()
def test_trainer(self):
batch_size = 3
length = 5
descriptors = torch.FloatTensor(self.config.nclasses,
self.config.descriptor_dim).normal_()
sender = Sender(self.config)
sender.eval()
receiver = Receiver(self.config)
receiver.eval()
exchange_model = ExchangeModel(self.config)
baseline_sender = Baseline(self.config, 'sender')
baseline_receiver = Baseline(self.config, 'receiver')
exchange = Exchange(exchange_model, sender, receiver, baseline_sender,
baseline_receiver, descriptors)
trainer = Trainer(exchange)
image = torch.FloatTensor(batch_size, self.config.image_in).normal_()
target_dist = F.softmax(torch.FloatTensor(
batch_size, self.config.nclasses).normal_(),
dim=1)
target = target_dist.argmax(dim=1)
trainer_loss = trainer.run_step(image, target)
self.assertEqual(trainer_loss.sender_message_loss.numel(), 1)
self.assertEqual(trainer_loss.receiver_message_loss.numel(), 1)
self.assertEqual(trainer_loss.stop_loss.numel(), 1)
self.assertEqual(trainer_loss.baseline_loss_sender.numel(), 1)
self.assertEqual(trainer_loss.baseline_loss_receiver.numel(), 1)
self.assertEqual(trainer_loss.xent_loss.numel(), 1)
def __init__(self, cfg, inference=False, threshold=0.5):
self.device = torch.device(
"cuda") if cfg.MODEL.DEVICE == 'cuda' else torch.device("cpu")
if not inference:
print('load training data')
self.dataloader, class_num = get_train_loader(cfg)
print('load testing data')
if cfg.TEST.MODE == 'face':
self.agedb_30, self.cfp_fp, self.lfw, self.agedb_30_issame, self.cfp_fp_issame, self.lfw_issame = get_val_data(
self.dataloader.dataset.root.parent)
else:
pairs = read_pairs(
os.path.join(cfg.DATASETS.FOLDER, 'pairs.txt'))
self.data, self.data_issame = get_paths(
os.path.join(cfg.DATASETS.FOLDER, 'test'), pairs)
print('load model')
self.model = Baseline(cfg)
self.model = self.model.to(self.device)
self.load_state(cfg)
if cfg.SOLVER.OPT == 'SGD':
self.optimizer = optim.SGD(
[{
'params': self.model.parameters()
}],
lr=cfg.SOLVER.BASE_LR,
momentum=cfg.SOLVER.MOMENTUM,
weight_decay=cfg.SOLVER.WEIGHT_DECAY)
else:
self.optimizer = optim.Adam(
[{
'params': self.model.parameters()
}],
lr=cfg.SOLVER.BASE_LR,
weight_decay=cfg.SOLVER.WEIGHT_DECAY)
self.scheduler = optim.lr_scheduler.CosineAnnealingLR(
self.optimizer,
T_max=cfg.SOLVER.MAX_EPOCH,
eta_min=cfg.SOLVER.ETA_MIN_LR)
checkpoints = cfg.CHECKPOINT.SAVE_DIR
os.makedirs(checkpoints, exist_ok=True)
self.best_score = 0.
self.best_threshold = 0.
else:
self.device = torch.device(
"cuda") if cfg.TEST.DEVICE == 'cuda' else torch.device("cpu")
print('load model')
self.model = Baseline(cfg)
self.model = self.model.to(self.device)
self.load_state(cfg)
self.threshold = threshold
self.test_transform = trans.Compose([
trans.ToTensor(),
trans.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])
])
def compute_pose(self, view1, view2=None, is_baseline=False):
"""Computes the pose of the new view"""
# procedure for baseline pose estimation
if is_baseline and view2:
match_object = self.matches[(view1.name, view2.name)]
baseline_pose = Baseline(view1, view2, match_object)
view2.R, view2.t = baseline_pose.get_pose(self.K)
rpe1, rpe2 = self.triangulate(view1, view2)
self.errors.append(np.mean(rpe1))
self.errors.append(np.mean(rpe2))
self.done.append(view1)
self.done.append(view2)
# procedure for estimating the pose of all other views
else:
view1.R, view1.t = self.compute_pose_PNP(view1)
errors = []
# reconstruct unreconstructed points from all of the previous views
for i, old_view in enumerate(self.done):
match_object = self.matches[(old_view.name, view1.name)]
_ = remove_outliers_using_F(old_view, view1, match_object)
self.remove_mapped_points(match_object, i)
_, rpe = self.triangulate(old_view, view1)
errors += rpe
self.done.append(view1)
self.errors.append(np.mean(errors))
def test_read_ascii(self):
"""Test reading ASCII tags and compare to known baseline values."""
self.assertEqual(self.image.datetime, Baseline("""2018:03:12 10:12:07"""))
self.assertEqual(self.image.make, Baseline("""Apple"""))
self.assertEqual(self.image.model, Baseline("""iPhone 7"""))
self.assertEqual(self.image.gps_latitude_ref, Baseline("""N"""))
self.assertEqual(self.image.gps_longitude_ref, Baseline("""W"""))
def __init__(self):
config = Baseline()
config_changes = {
'description':'baseline travel model fast (no build)',
}
config.replace(config_changes)
from psrc.configs.create_travel_model_configuration import create_travel_model_configuration
travel_model_configuration = create_travel_model_configuration('baseline_travel_model_psrc_fast_no_build',
emme2_batch_file='MODELUSim.BAT ..\\triptabs',
mode='full')
config['travel_model_configuration'] = travel_model_configuration
config['travel_model_configuration']['locations_to_disaggregate'] = ['parcel', 'building']
del config['travel_model_configuration'][2000]
##fast model doesn't have bank2 and bank3; disable macros using them
del config['travel_model_configuration']['export_macros']['tazvmt2.mac']
del config['travel_model_configuration']['export_macros']['tazvmt3.mac']
del config['travel_model_configuration']['matrix_variable_map']['bank2']
del config['travel_model_configuration']['matrix_variable_map']['bank3']
self.merge(config)
#if __name__ == "__main__":
# config = BaselineTravelModelFast()
def __init__(self):
config = Baseline()
config_changes = {
'description':
'run travel model for baseyear and then extract travel data',
'models': [],
'models_in_year': {
2000: [],
},
'years': (2000, 2001),
}
from psrc.configs.create_travel_model_configuration import create_travel_model_configuration
travel_model_configuration = create_travel_model_configuration(
'baseline_travel_model_psrc_2008_lmwang',
emme2_batch_file='MODEL1-0.BAT',
mode='full',
years_to_run={
2000: '2000_v1.0aTG',
2005: '2006_v1.0aTG',
2010: '2010_v1.0aTG',
2015: '2010_v1.0aTG_2015',
2020: '2020_v1.0aTG'
})
config['travel_model_configuration'] = travel_model_configuration
config.replace(config_changes)
self.merge(config)
def test_read_rational(self):
"""Test reading RATIONAL tags and compare to known baseline values."""
self.assertEqual(str(self.image.gps_altitude)[:13], Baseline("""2189.98969072"""))
self.assertEqual(str(self.image.gps_latitude), Baseline("""(36.0, 3.0, 11.08)"""))
self.assertEqual(str(self.image.gps_longitude), Baseline("""(112.0, 5.0, 4.18)"""))
self.assertEqual(str(self.image.x_resolution), Baseline("""72.0"""))
self.assertEqual(str(self.image.y_resolution), Baseline("""72.0"""))
def run(args=None):
usage = "usage : %prog [options]"
parser = optparse.OptionParser(usage=usage)
parser.add_option("--test", action="store_true", dest="test", default=False)
parser.add_option("--output_dim", dest="output_dim", type="int", default=0)
parser.add_option("--out_file", dest="out_file", type="string",
default="models/baseline.bin")
parser.add_option("--in_file", dest="in_file", type="string",
default="models/baseline.bin")
parser.add_option("--data", dest="data", type="string", default="train")
(opts, args) = parser.parse_args(args)
# Testing
if opts.test:
test(opts.in_file, opts.data)
return
print("Loading data...")
# load training data
trees = load_trees()
if opts.output_dim == 0:
opts.output_dim = len(load_label_map())
baseline = Baseline(opts.output_dim)
baseline.train(trees)
with open(opts.out_file, 'wb') as fid:
pickle.dump(opts, fid)
baseline.to_file(fid)
def __init__(self):
config = Baseline()
if self.multiple_runs:
config.sample_inputs()
config['number_of_runs'] = 99
config['seed'] = 1
config_changes = {
'description':'baseline travel model fast',
}
config.replace(config_changes)
from psrc.configs.create_travel_model_configuration import create_travel_model_configuration
travel_model_configuration = create_travel_model_configuration('baseline_travel_model_psrc_fast_hana',
emme2_batch_file='MODELUSim.BAT ..\\triptabs',
mode='skims',
years_to_run={2005: '2005_06', 2010: '2010_06', 2015: '2010_06'})
config['travel_model_configuration'] = travel_model_configuration
config['travel_model_configuration']['locations_to_disaggregate'] = ['parcel', 'building']
##fast model doesn't have bank2 and bank3; disable macros using them
del config['travel_model_configuration']['export_macros']['tazvmt2.mac']
del config['travel_model_configuration']['export_macros']['tazvmt3.mac']
del config['travel_model_configuration']['matrix_variable_map']['bank2']
del config['travel_model_configuration']['matrix_variable_map']['bank3']
config['travel_model_configuration']['export_macros']['get_link_attributes.mac'] = {'bank':'bank1', 'scenario':-1, 'path':'export_macros'}
config['travel_model_configuration']['node_matrix_variable_map'] = {"bank1": {"attr_on_links.rpt": {"timau": "am_pk_travel_time", "len": "distance"},
"tveham.rpt": {"@tveh": "vehicle_volume"}
}}
#config['travel_model_configuration'][2015]['models'] = list(config['travel_model_configuration'][2015].get('models'))
#config['travel_model_configuration'][2015]['models'].append('opus_emme2.models.restore_trip_tables')
self.merge(config)
def __init__(self):
Baseline.__init__(self)
self['description']='Washtenaw baseline + travel model'
from washtenaw.transcad.create_travel_model_configuration import create_travel_model_configuration
travel_model_configuration = create_travel_model_configuration('C:\\SEMCOG_baseline\\', mode='full')
self['travel_model_configuration'] = travel_model_configuration
def __init__(self):
Baseline.__init__(self)
self['description'] = 'Washtenaw baseline + travel model'
from washtenaw.transcad.create_travel_model_configuration import create_travel_model_configuration
travel_model_configuration = create_travel_model_configuration(
'C:\\SEMCOG_baseline\\', mode='full')
self['travel_model_configuration'] = travel_model_configuration
def __init__(self):
config = Baseline()
if self.multiple_runs:
config.sample_inputs()
config['years'] = (2001, 2005)
config['number_of_runs'] = 50
config['seed'] = 1
config['description'] = 'baseline multiple runs'
self.merge(config)
def __init__(self):
config = Baseline()
if self.multiple_runs:
config.sample_inputs()
config['years'] = (2001, 2005)
config['number_of_runs'] = 50
config['seed'] = 1
config['description'] = 'baseline multiple runs'
self.merge(config)
def test_same_value(self):
"""Test same baselined text for every call on the same line.
Check baseline instantiations on the same line produce the
same instance.
"""
baseline1, baseline2 = Baseline("Hello!"), Baseline("Hello!")
self.assertIs(baseline1, baseline2)
def __init__(self):
Baseline.__init__(self)
self['description']='Washtenaw + travel model'
self['models']= []
self['years'] = (2001, 2002)
#self['creating_baseyear_cache_configuration']['cache_from_database'] = False
#self['creating_baseyear_cache_configuration']['cache_directory_root']=r'C:\urbansim_cache\washtenaw'
#self['creating_baseyear_cache_configuration']['baseyear_cache']['directory_to_cache'] = r'C:\urbansim_cache\washtenaw\cache_source20070129'
from washtenaw.transcad.create_travel_model_configuration import create_travel_model_configuration
travel_model_configuration = create_travel_model_configuration('C:\\SEMCOG_baseline\\', mode='full')
self['travel_model_configuration'] = travel_model_configuration
def __init__(self):
Baseline.__init__(self)
self['config_changes_for_estimation'] = ConfigChangesForEstimation()
self['cache_directory'] = os.path.join(os.environ['OPUS_HOME'], 'data/eugene_gridcell/base_year_data')
self['scenario_database_configuration'] = ScenarioDatabaseConfiguration(database_name = 'eugene_1980_baseyear')
self['estimation_database_configuration'] = EstimationDatabaseConfiguration(database_name = 'eugene_1980_baseyear_estimation_xxx')
self['datasets_to_cache_after_each_model' ] = []
self['low_memory_mode'] = False
self['base_year'] = 1980
self['years'] = (1980,1980)
self['seed'] = 10
def test_modify_rational(self):
"""Verify that modifying RATIONAL tags updates the tag values as expected."""
self.image.gps_altitude = 123.456789
self.assertEqual(str(self.image.gps_altitude),
Baseline("""123.456789"""))
self.image.gps_latitude = (41.0, 36.0, 33.786)
self.assertEqual(str(self.image.gps_latitude),
Baseline("""(41.0, 36.0, 33.786)"""))
segment_hex = self.image._segments['APP1'].get_segment_hex()
self.assertEqual('\n'.join(textwrap.wrap(segment_hex, 90)),
MODIFY_RATIONAL_HEX_BASELINE)
def __init__(self):
Baseline.__init__(self)
self['config_changes_for_estimation'] = ConfigChangesForEstimation()
self['cache_directory'] = paths.get_opus_home_path('data/eugene_zone/base_year_data')
self['scenario_database_configuration'] = ScenarioDatabaseConfiguration(database_name = 'eugene_1980_baseyear_zone')
self['estimation_database_configuration'] = EstimationDatabaseConfiguration(database_name = 'eugene_1980_baseyear_zone')
self['datasets_to_cache_after_each_model' ] = []
self['low_memory_mode'] = False
self['base_year'] = 1980
self['years'] = (1980,1980)
self['seed'] = 1
def __init__(self):
Baseline.__init__(self)
self['config_changes_for_estimation'] = ConfigChangesForEstimation()
self['cache_directory'] = os.path.join(os.environ['OPUS_HOME'], 'data/eugene_zone/base_year_data')
self['scenario_database_configuration'] = ScenarioDatabaseConfiguration(database_name = 'eugene_1980_baseyear_zone')
self['estimation_database_configuration'] = EstimationDatabaseConfiguration(database_name = 'eugene_1980_baseyear_zone')
self['datasets_to_cache_after_each_model' ] = []
self['low_memory_mode'] = False
self['base_year'] = 1980
self['years'] = (1980,1980)
self['seed'] = 1
def __init__(self):
config = Baseline()
config_changes = {
'description':'baseline with skims',
}
config.replace(config_changes)
from psrc.configs.create_travel_model_configuration import create_travel_model_configuration
travel_model_configuration = create_travel_model_configuration('baseline_travel_model_psrc', mode='skims')
config['travel_model_configuration'] = travel_model_configuration
config['travel_model_configuration']['locations_to_disaggregate'] = ['parcel', 'building']
self.merge(config)
def __init__(self):
Baseline.__init__(self)
self['description'] = 'Washtenaw + travel model'
self['models'] = []
self['years'] = (2001, 2002)
#self['creating_baseyear_cache_configuration']['cache_from_database'] = False
#self['creating_baseyear_cache_configuration']['cache_directory_root']=r'C:\urbansim_cache\washtenaw'
#self['creating_baseyear_cache_configuration']['baseyear_cache']['directory_to_cache'] = r'C:\urbansim_cache\washtenaw\cache_source20070129'
from washtenaw.transcad.create_travel_model_configuration import create_travel_model_configuration
travel_model_configuration = create_travel_model_configuration(
'C:\\SEMCOG_baseline\\', mode='full')
self['travel_model_configuration'] = travel_model_configuration
def __init__(self):
Baseline.__init__(self)
self["config_changes_for_estimation"] = ConfigChangesForEstimation()
self["cache_directory"] = paths.get_opus_home_path("data/eugene_gridcell/base_year_data")
self["scenario_database_configuration"] = ScenarioDatabaseConfiguration(database_name="eugene_1980_baseyear")
self["estimation_database_configuration"] = EstimationDatabaseConfiguration(
database_name="eugene_1980_baseyear_estimation_xxx"
)
self["datasets_to_cache_after_each_model"] = []
self["low_memory_mode"] = False
self["base_year"] = 1980
self["years"] = (1980, 1980)
self["seed"] = 10
def test(baseline_file, data_set):
assert baseline_file is not None, "Must give model to test"
trees = load_trees(data_set)
assert trees, "No data found"
with open(baseline_file, 'rb') as fid:
opts = pickle.load(fid)
baseline = Baseline(opts.output_dim)
baseline.from_file(fid)
print("Testing...")
correct, total, pred = baseline.predict(trees)
print("Correct %d/%d, Acc %f" % (correct, total, correct / float(total)))
print_trees('results/gold.txt', trees, 'Labeled')
print_trees('results/pred_baseline.txt', pred, 'Baseline predicted')
def __init__(self):
config = Baseline()
config_changes = {
'description':'baseline with full Visum travel model run',
'years':(1981, 1981),
}
config.replace(config_changes)
from opus_visum.configs.visum_configuration import VisumConfiguration
travel_model_configuration = VisumConfiguration(r'C:/visum/eugene', mode='full')
config['travel_model_configuration'] = travel_model_configuration
self.merge(config)
def test(baseline_file, data_set):
assert baseline_file is not None, "Must give model to test"
trees = load_trees(data_set)
assert trees, "No data found"
with open(baseline_file, 'rb') as fid:
opts = pickle.load(fid)
baseline = Baseline(opts.output_dim)
baseline.from_file(fid)
print("Testing...")
correct, total, pred = baseline.predict(trees)
print("Correct %d/%d, Acc %f" % (correct, total, correct / float(total)))
print_trees('results/gold.txt', trees, 'Labeled')
print_trees('results/pred_baseline.txt', pred, 'Baseline predicted')
def __init__(self):
config = Baseline()
config_changes = {
'description': 'baseline with full Visum travel model run',
'years': (1981, 1981),
}
config.replace(config_changes)
from opus_visum.configs.visum_configuration import VisumConfiguration
travel_model_configuration = VisumConfiguration(r'C:/visum/eugene',
mode='full')
config['travel_model_configuration'] = travel_model_configuration
self.merge(config)
def test_differing_value(self):
"""Test differing baselined text for every call.
Check that exception is raised, that "atexit" registration
did not occur, and that no files were to be updated.
"""
with self.assertRaises(RuntimeError):
Baseline('junk1'), Baseline('junk2')
with self.assertRaises(RuntimeError):
for text in ['junk1', 'junk2']:
Baseline(text)
self.check_updated_files()
def __init__(self, parent=None):
super(CameraPreview, self).__init__(parent)
self.roi_origin = QPoint(0, 0)
self._pixmap: QPixmap = QPixmap(480, 360)
self._double_buffer: QImage = None
self._raw_image: np.ndarray = None
self._image_size = (1, 1)
self._image_size_invalid = True
self._roi_rubber_band = ResizableRubberBand(self)
self._needle_mask = DynamicNeedleMask(self)
self._needle_mask.update_mask_signal.connect(self.update_mask)
self._baseline = Baseline(self)
self._droplet = Droplet()
self._mask = None
logging.debug("initialized camera preview")
def execute_demo(language):
if language == 'english':
word_emb = load_word_embeddings('english')
elif language == 'spanish':
word_emb = load_word_embeddings('spanish')
data = Dataset(language)
print("{}: {} training - {} dev".format(language, len(data.trainset),
len(data.devset)))
#for sent in data.trainset:
# Gold label -> 0 if the word is not complex, 1 if the word is complex.
#print(sent['sentence'], sent['target_word'], sent['gold_label'])
baseline = Baseline(language)
model = Model(language)
model.train(data.trainset, word_emb)
predictions = model.test(data.devset, word_emb)
gold_labels = [sent['gold_label'] for sent in data.devset]
report_score(gold_labels, predictions)
def test_delete_all_tags(self):
"""Verify deleting all EXIF tags from the Image object."""
self.image.delete_all()
segment_hex = (binascii.hexlify(
self.image._segments["APP1"].get_segment_bytes()).decode(
"utf-8").upper())
self.assertEqual("\n".join(textwrap.wrap(segment_hex, 90)),
DELETE_ALL_HEX_BASELINE)
with TemporaryFile("w+b") as temporary_file_stream:
temporary_file_stream.write(self.image.get_file())
temporary_file_stream.seek(0)
reloaded_image = Image(temporary_file_stream)
dunder_dir_text = "\n".join(
textwrap.wrap(repr(sorted(dir(reloaded_image))), 90))
self.assertEqual(
dunder_dir_text,
Baseline("""
['<unknown EXIF tag 59932>', '<unknown EXIF tag 59933>', '_exif_ifd_pointer',
'_gps_ifd_pointer', '_segments', 'delete', 'delete_all', 'exif_version', 'get', 'get_all',
'get_file', 'get_thumbnail', 'has_exif', 'list_all']
"""),
)
def test_no_app1_segment(self):
"""Verify behavior of an image without an APP1 segment marker.
Assert the ``has_exif`` attribute is false. Verify non-EXIF ``dir()`` list contents. Then,
check the ``get_file()`` hexadecimal.
"""
image_path = os.path.join(os.path.dirname(__file__), "no_app1.png")
with open(image_path, "rb") as image_file:
my_image = Image(image_file)
self.assertFalse(my_image.has_exif)
self.assertEqual(
str(dir(my_image)),
Baseline("""
['_segments', 'delete', 'delete_all', 'get', 'get_all', 'get_file', 'get_thumbnail', 'has_exif', 'list_all']
"""),
)
with pytest.raises(RuntimeError,
match="image does not contain thumbnail"):
my_image.get_thumbnail()
self.assertEqual(
"\n".join(textwrap.wrap(str(my_image.get_file()), 90)),
NO_APP1_PNG)
def __init__(self):
config = Baseline()
config_changes = {
'description': 'baseline with skims',
}
config.replace(config_changes)
from psrc.configs.create_travel_model_configuration import create_travel_model_configuration
travel_model_configuration = create_travel_model_configuration(
'baseline_travel_model_psrc', mode='skims')
config['travel_model_configuration'] = travel_model_configuration
config['travel_model_configuration']['locations_to_disaggregate'] = [
'parcel', 'building'
]
self.merge(config)
def test_list_attributes_photo(self):
"""Verify that calling dir() on a camera photo lists the expected EXIF attributes."""
with open(
os.path.join(os.path.dirname(__file__), "grand_canyon.jpg"), "rb"
) as image_file:
image = Image(image_file)
dunder_dir_text = "\n".join(textwrap.wrap(repr(sorted(dir(image))), 90))
self.assertEqual(
dunder_dir_text,
Baseline(
"""
['<unknown EXIF tag 59932>', '<unknown EXIF tag 59933>', '_exif_ifd_pointer',
'_gps_ifd_pointer', '_segments', 'aperture_value', 'brightness_value', 'color_space',
'components_configuration', 'compression', 'datetime', 'datetime_digitized',
'datetime_original', 'delete', 'delete_all', 'exif_version', 'exposure_bias_value',
'exposure_mode', 'exposure_program', 'exposure_time', 'f_number', 'flash',
'flashpix_version', 'focal_length', 'focal_length_in_35mm_film', 'get', 'get_all',
'get_file', 'get_thumbnail', 'gps_altitude', 'gps_altitude_ref', 'gps_datestamp',
'gps_dest_bearing', 'gps_dest_bearing_ref', 'gps_horizontal_positioning_error',
'gps_img_direction', 'gps_img_direction_ref', 'gps_latitude', 'gps_latitude_ref',
'gps_longitude', 'gps_longitude_ref', 'gps_speed', 'gps_speed_ref', 'gps_timestamp',
'has_exif', 'jpeg_interchange_format', 'jpeg_interchange_format_length', 'lens_make',
'lens_model', 'lens_specification', 'list_all', 'make', 'maker_note', 'metering_mode',
'model', 'orientation', 'photographic_sensitivity', 'pixel_x_dimension',
'pixel_y_dimension', 'resolution_unit', 'scene_capture_type', 'scene_type',
'sensing_method', 'shutter_speed_value', 'software', 'subject_area',
'subsec_time_digitized', 'subsec_time_original', 'white_balance', 'x_resolution',
'y_and_c_positioning', 'y_resolution']
"""
),
)
def test_nic_name_map(self, mocked_check_output):
"""Test nic_name_map attribute of NetworkAdapters."""
self.assertEqual(
str(
sorted(self.test_adapters.nic_name_map,
key=self.test_adapters.nic_name_map.get)),
Baseline("""['Ethernet Adapter', 'Wi-Fi Adapter']"""))
def test_index_accessor():
"""Test accessing attributes using index syntax."""
with open(os.path.join(os.path.dirname(__file__), "grand_canyon.jpg"),
"rb") as image_file:
image = Image(image_file)
assert image["datetime"] == Baseline("""2018:03:12 10:12:07""")
def run(dataset):
n_features = len(meta[dataset]['val_name'])
result_online = Result('%s-%s' %(dataset, 'aws-online'), aws=True)
result_baseline = Result('%s-%s' %(dataset, 'aws-baseline'), aws=True)
result_active = Result('%s-%s' %(dataset, 'aws-active'), aws=True)
for repeat in range(0, n_repeat):
print 'Round %d of %d'% (repeat, n_repeat - 1)
ex = AWSOnline(meta[dataset]['model_id'], 1, 0, n_features, meta[dataset]['val_name'], ftype='uniform', error=.1)
test_x, test_y = load_svmlight_file('/Users/Fan/dev/ML/code/binary-classifiers/targets/%s/test.scale' % dataset, n_features)
test_x = test_x.todense()
test_y = [a if a == 1 else 0 for a in test_y]
train_x, train_y = [], []
for i in result_active.index:
q_by_u = result_active.Q_by_U[i]
print 'Active learning with budget %d / %d' % (q_by_u, q_by_u * (n_features + 1))
main = ActiveLearning(ex, (None, None), (test_x, test_y), n_features,
q_by_u * (n_features + 1), 5)
L_unif, L_test = main.do()
result_active.L_unif[i].append(L_unif)
result_active.L_test[i].append(L_test)
result_active.nquery[i].append(ex.get_n_query())
ex = AWSOnline(meta[dataset]['model_id'], 1, 0, n_features, meta[dataset]['val_name'], ftype='uniform', error=.1)
for i in result_online.index:
q_by_u = result_online.Q_by_U[i]
print 'collecting up to budget %d / %d' % (q_by_u, q_by_u * (n_features + 1))
ex.collect_up_to_budget(q_by_u * (n_features + 1))
train_x.extend(ex.pts_near_b)
train_y.extend(ex.pts_near_b_labels)
print 'retraining with %d points' % len(train_y)
# online
e = RBFKernelRetraining(ex.batch_predict, (train_x, train_y), (test_x, test_y), n_features)
L_unif, L_test = e.grid_retrain_in_x()
result_online.L_unif[i].append(L_unif)
result_online.L_test[i].append(L_test)
result_online.nquery[i].append(ex.get_n_query())
# baseline
e = Baseline(ex.batch_predict, (train_x, train_y), (test_x, test_y), n_features)
L_unif, L_test = e.do()
result_baseline.L_unif[i].append(L_unif)
result_baseline.L_test[i].append(L_test)
result_baseline.nquery[i].append(ex.get_n_query())
print result_online
print result_baseline
print result_active
def test_nic_connection_id_map(self, mocked_check_output):
"""Test nic_connection_id_map attribute of NetworkAdapters."""
self.assertEqual(
str(
sorted(self.test_adapters.nic_connection_id_map,
key=self.test_adapters.nic_connection_id_map.get)),
Baseline(
"""['Local Area Connection', 'Wireless Area Connection']"""))
def test_modify_rational(self):
"""Verify that modifying RATIONAL tags updates the tag values as expected."""
self.image.gps_altitude = 123.456789
self.assertEqual(str(self.image.gps_altitude), Baseline("""123.456789"""))
self.image.gps_latitude = (41.0, 36.0, 33.786)
self.assertEqual(
str(self.image.gps_latitude), Baseline("""(41.0, 36.0, 33.786)""")
)
segment_hex = (
binascii.hexlify(self.image._segments["APP1"].get_segment_bytes())
.decode("utf-8")
.upper()
)
self.assertEqual(
"\n".join(textwrap.wrap(segment_hex, 90)), MODIFY_RATIONAL_HEX_BASELINE
)
def test_get_method():
"""Test behavior when accessing tags using the ``get()`` method."""
with open(os.path.join(os.path.dirname(__file__), 'grand_canyon.jpg'), 'rb') as image_file:
image = Image(image_file)
assert image.get('fake_attribute') is None
assert image.get('light_source', default=-1) == -1 # tag not in image
assert image.get('make') == Baseline("""Apple""")
def __init__(self):
config = Baseline()
config_changes = {"description": "baseline land use + baseline travel model"}
config.replace(config_changes)
config["number_of_runs"] = 1
config["seed"] = 1
config["creating_baseyear_cache_configuration"].cache_directory_root = r"/urbansim_cache/psrc_parcel/runs"
# config['creating_baseyear_cache_configuration'].cache_from_database = True
config[
"creating_baseyear_cache_configuration"
].baseyear_cache.existing_cache_to_copy = r"/urbansim_cache/psrc_parcel/base_year_data/baseline"
from psrc.configs.create_travel_model_configuration import create_travel_model_configuration
travel_model_configuration = create_travel_model_configuration(
self.tm_scenario,
emme2_batch_file="model1-0.sh",
mode="full",
years_to_run={
2005: "2006_v1.0bb",
2010: "2010_v1.0bb",
# 2015: '2010_v1.0bb',
2020: "2020_v1.0bb",
# 2025: '2020_v1.0bb',
2030: "2030_v1.0bb",
},
)
config["travel_model_configuration"] = travel_model_configuration
# config['travel_model_configuration']['travel_model_input_file_writer'] = 'psrc_parcel.travel_model_input_file_writer'
config["travel_model_configuration"]["system_command"] = ""
config["travel_model_configuration"]["emme_command"] = "emme-run -ng --set-iks 127.0.0.1"
config["travel_model_configuration"]["locations_to_disaggregate"] = ["parcel", "building"]
# config['travel_model_configuration']['export_macros']['get_link_attributes.mac'] = {'bank':'bank1', 'scenario':-1, 'path':'export_macros'}
# config['travel_model_configuration']['node_matrix_variable_map'] = {"bank1": {"attr_on_links.rpt": {"timau": "am_pk_travel_time", "len": "distance"},
# "tveham.rpt": {"@tveh": "vehicle_volume"}
# }}
self.merge(config)
def __init__(self):
config = Baseline()
config_changes = {
'description':'extract travel data from travel model',
'models':[],
'models_in_year': {2000:[],},
'years': (2005,2005),
}
from psrc.configs.create_travel_model_configuration import create_travel_model_configuration
travel_model_configuration = create_travel_model_configuration('baseline_travel_model_psrc_2008_lmwang',
emme2_batch_file='MODEL1-0.BAT',
mode='skims', years_to_run={2000: '2000_v1.0aTG',
2005: '2006_v1.0aTG',
2010: '2010_v1.0aTG',
2015: '2010_v1.0aTG_2015',
2020: '2020_v1.0aTG'})
config['travel_model_configuration'] = travel_model_configuration
config.replace(config_changes)
self.merge(config)
def __init__(self):
config = Baseline()
config_changes = {
'description':'Aggressive land use + pricing & transit travel model',
}
config.replace(config_changes)
config['number_of_runs'] = 1
config['seed'] = 1
config['creating_baseyear_cache_configuration'].cache_directory_root = r'/urbansim_cache/psrc_parcel/runs'
#config['creating_baseyear_cache_configuration'].cache_from_database = True
config['creating_baseyear_cache_configuration'].baseyear_cache.existing_cache_to_copy = r'/urbansim_cache/psrc_parcel/base_year_data/aggressive'
from psrc.configs.create_travel_model_configuration import create_travel_model_configuration
travel_model_configuration = create_travel_model_configuration(self.tm_scenario,
emme2_batch_file='model1-0.sh',
mode='full',
years_to_run={
2005: '2006_v1.0bb',
2010: '2010_v1.0bb',
#2015: '2010_v1.0bb',
2020: '2020_v1.0bb',
#2025: '2020_v1.0bb',
2030: '2030_v1.0bb'})
config['travel_model_configuration'] = travel_model_configuration
#config['travel_model_configuration']['travel_model_input_file_writer'] = 'psrc_parcel.travel_model_input_file_writer'
config['travel_model_configuration']['system_command'] = ''
config['travel_model_configuration']['emme_command'] = 'emme-run -ng --set-iks 127.0.0.1'
config['travel_model_configuration']['locations_to_disaggregate'] = ['parcel', 'building']
#config['travel_model_configuration']['export_macros']['get_link_attributes.mac'] = {'bank':'bank1', 'scenario':-1, 'path':'export_macros'}
#config['travel_model_configuration']['node_matrix_variable_map'] = {"bank1": {"attr_on_links.rpt": {"timau": "am_pk_travel_time", "len": "distance"},
# "tveham.rpt": {"@tveh": "vehicle_volume"}
# }}
self.merge(config)
import recsys.algorithm
recsys.algorithm.VERBOSE = True
from recsys.evaluation.prediction import RMSE, MAE
from recsys.datamodel.data import Data
from baseline import Baseline #Import the test class we've just created
#Dataset
PERCENT_TRAIN = int(sys.argv[2])
data = Data()
data.load(sys.argv[1], sep='::', format={'col':0, 'row':1, 'value':2, 'ids': int})
#Train & Test data
train, test = data.split_train_test(percent=PERCENT_TRAIN)
baseline = Baseline()
baseline.set_data(train)
baseline.compute() # In this case, it does nothing
# Evaluate
rmse = RMSE()
mae = MAE()
for rating, item_id, user_id in test.get():
try:
pred_rating = baseline.predict(item_id, user_id, user_is_row=False)
rmse.add(rating, pred_rating)
mae.add(rating, pred_rating)
except KeyError:
continue
print 'RMSE=%s' % rmse.compute() # in my case (~80% train, ~20% test set) returns RMSE = 1.036374
def __init__(self):
Baseline.__init__(self)
self["config_changes_for_estimation"] = ConfigChangesForEstimation()
self.merge(self._get_estimation_changes())
