def get_resources_for_dataset(self,
resources=None,
in_storage=None,
out_storage=None,
in_table_name=None,
in_table_name_groups=None,
out_table_name=None,
attributes=None,
id_name=None,
id_name_default=None,
nchunks=None,
debug=None,
):
# Defaults:
in_table_name_default = "plan_type_groups"
attributes_default = AttributeType.PRIMARY
out_table_name_default = "plan_type_groups"
dataset_name = "plan_type_group"
nchunks_default = 1
resources = ResourceFactory().get_resources_for_dataset(
dataset_name,
resources = resources,
in_storage = in_storage,
out_storage = out_storage,
in_table_name_pair = (in_table_name,in_table_name_default),
attributes_pair = (attributes,attributes_default),
out_table_name_pair = (out_table_name, out_table_name_default),
id_name_pair = (id_name,id_name_default),
nchunks_pair = (nchunks,nchunks_default),
debug_pair = (debug,None),
)
resources.merge({"replace_nulls_with":"None"})
return resources
def _get_resources_for_dataset(self,
in_table_name_default,
in_table_name_groups_default,
out_table_name_default,
dataset_name,
resources=None,
in_storage=None,
out_storage=None,
in_table_name=None,
in_table_name_groups=None,
attributes=None,
out_table_name=None,
id_name=None,
id_name_default=None,
debug=None,
):
# Defaults:
attributes_default = AttributeType.PRIMARY
resources = ResourceFactory().get_resources_for_dataset(
dataset_name,
resources = resources,
in_storage = in_storage,
out_storage = out_storage,
in_table_name_pair = (in_table_name,in_table_name_default),
attributes_pair = (attributes,attributes_default),
out_table_name_pair = (out_table_name, out_table_name_default),
id_name_pair = (id_name,id_name_default),
debug_pair = (debug,None),
)
resources.merge_if_not_None({"in_table_name_groups":in_table_name_groups})
resources.merge_with_defaults({"in_table_name_groups":in_table_name_groups_default})
return resources
def get_resources_for_dataset(self,
resources=None,
in_storage=None,
out_storage=None,
in_table_name=None,
in_table_name_groups=None,
out_table_name=None,
attributes=None,
id_name=None,
id_name_default=None,
nchunks=None,
debug=None):
# Defaults:
in_table_name_default = "employment_adhoc_sector_groups"
attributes_default = AttributeType.PRIMARY
out_table_name_default = "employment_adhoc_sector_groups"
dataset_name = "employment_sector_group"
nchunks_default = 1
resources = ResourceFactory().get_resources_for_dataset(
dataset_name,
resources=resources,
in_storage=in_storage,
out_storage=out_storage,
in_table_name_pair=(in_table_name, in_table_name_default),
attributes_pair=(attributes, attributes_default),
out_table_name_pair=(out_table_name, out_table_name_default),
id_name_pair=(id_name, id_name_default),
nchunks_pair=(nchunks, nchunks_default),
debug_pair=(debug, None))
resources.merge({"replace_nulls_with": "None"})
return resources
def __init__(self,
categories=array([
1,
]),
resources=None,
what=None,
attribute_name=None,
data=None,
names=None,
in_storage=None,
out_storage=None,
in_table_name=None,
attributes=None,
out_table_name=None,
id_name=None,
nchunks=None,
debuglevel=0):
"""
'what' must be a string, such as 'residential' or 'commercial'.
"""
debug = DebugPrinter(debuglevel)
debug.print_debug(
"Creating DevelopmentProjectDataset object for %s projects." %
what, 2)
self.categories = categories
self.what = what
self.attribute_name = attribute_name
attributes_default = AttributeType.PRIMARY
dataset_name = "development_project"
nchunks_default = 1
if data <> None:
in_storage = StorageFactory().get_storage('dict_storage')
in_storage.write_table(table_name='development_projects',
table_data=data)
in_table_name = 'development_projects'
resources = ResourceFactory().get_resources_for_dataset(
dataset_name,
resources=resources,
in_storage=in_storage,
out_storage=out_storage,
in_table_name_pair=(in_table_name, None),
out_table_name_pair=(out_table_name, None),
attributes_pair=(attributes, attributes_default),
id_name_pair=(id_name, self.id_name_default),
nchunks_pair=(nchunks, nchunks_default),
debug_pair=(debug, None))
self.category_variable_name = resources.get(
"category_variable_name", self.category_variable_name_default)
Dataset.__init__(self, resources=resources)
def __init__(
self,
categories=array([1]),
resources=None,
what=None,
attribute_name=None,
data=None,
names=None,
in_storage=None,
out_storage=None,
in_table_name=None,
attributes=None,
out_table_name=None,
id_name=None,
nchunks=None,
debuglevel=0,
):
"""
'what' must be a string, such as 'residential' or 'commercial'.
"""
debug = DebugPrinter(debuglevel)
debug.print_debug("Creating DevelopmentProjectDataset object for %s projects." % what, 2)
self.categories = categories
self.what = what
self.attribute_name = attribute_name
attributes_default = AttributeType.PRIMARY
dataset_name = "development_project"
nchunks_default = 1
if data <> None:
in_storage = StorageFactory().get_storage("dict_storage")
in_storage.write_table(table_name="development_projects", table_data=data)
in_table_name = "development_projects"
resources = ResourceFactory().get_resources_for_dataset(
dataset_name,
resources=resources,
in_storage=in_storage,
out_storage=out_storage,
in_table_name_pair=(in_table_name, None),
out_table_name_pair=(out_table_name, None),
attributes_pair=(attributes, attributes_default),
id_name_pair=(id_name, self.id_name_default),
nchunks_pair=(nchunks, nchunks_default),
debug_pair=(debug, None),
)
self.category_variable_name = resources.get("category_variable_name", self.category_variable_name_default)
Dataset.__init__(self, resources=resources)
def setUp(self):
self.start_year = 2001
self.expected_sic_data = array([6,4,7,808,6])
self.job_id = array([1,2,3,4,5])
self.base_cache_dir = tempfile.mkdtemp(prefix='opus_tmp_test_dataset')
self.simulation_state = SimulationState(low_memory_run=True, new_instance=True, base_cache_dir=self.base_cache_dir)
self.dir = self.simulation_state.get_cache_directory()
self.simulation_state.set_current_time(self.start_year)
if not os.path.exists(self.dir):
os.makedirs(self.dir)
self.in_storage = StorageFactory().get_storage('dict_storage')
self.in_storage.write_table(
table_name='jobs',
table_data={
'grid_id':array([10,20,30,40,50]),
'job_id':self.job_id,
},
)
self.out_storage = StorageFactory().get_storage('dict_storage')
self.job_set_resources = ResourceFactory().get_resources_for_dataset(
'job',
in_storage = self.in_storage,
out_storage = self.out_storage,
in_table_name_pair = ('jobs',None),
out_table_name_pair = ('jobs_exported',None),
attributes_pair = (None,AttributeType.PRIMARY),
id_name_pair = ('job_id','job_id'),
nchunks_pair = (1,1),
debug_pair = (1,None)
)
def __init__(self, resources=None, what="household", in_storage=None,
in_table_name=None, out_storage=None, out_table_name=None,
id_name=None, nchunks=None, debuglevel=0):
## TODO remove "what" arguement
debug = DebugPrinter(debuglevel)
debug.print_debug("Creating ControlTotalDataset object for "+what+".",2)
if not self.in_table_name_default:
self.in_table_name_default = "annual_" + what + "_control_totals"
if not self.out_table_name_default:
self.out_table_name_default = "annual_" + what + "_control_totals"
attributes_default = AttributeType.PRIMARY
#dataset_name = "control_total"
nchunks_default = 1
resources = ResourceFactory().get_resources_for_dataset(
self.dataset_name,
resources=resources,
in_storage=in_storage,
out_storage=out_storage,
in_table_name_pair=(in_table_name,self.in_table_name_default),
attributes_pair=(None, attributes_default),
out_table_name_pair=(out_table_name, self.out_table_name_default),
id_name_pair=(id_name,self.id_name_default),
nchunks_pair=(nchunks,nchunks_default),
debug_pair=(debug,None)
)
table_name = resources["in_table_name"]
if resources['id_name'] is None or len(resources['id_name'])== 0:
#if both self.id_name_default and id_name argument in __init__ is unspecified,
#ControlTotalDataset would use all attributes not beginning with "total"
#as id_name
id_names = []
column_names = resources["in_storage"].get_column_names(table_name)
for column_name in column_names:
if not re.search('^total', column_name):
id_names.append(column_name)
resources.merge({"id_name":resources["id_name"] + id_names})
Dataset.__init__(self, resources = resources)
def __init__(self,
resources=None,
data=None,
names=None,
in_storage=None,
out_storage=None,
in_table_name=None,
out_table_name=None,
attributes=None,
id_name=None,
nchunks=None,
debuglevel=0):
debug = DebugPrinter(debuglevel)
debug.print_debug("Creating LandUseDevelopmentDataset object.", 2)
dataset_name = "landuse_development"
nchunks_default = 1
if data <> None:
in_storage = StorageFactory().get_storage('dict_storage')
in_storage.write_table(table_name='landuse_developments',
table_data=data)
in_table_name = 'landuse_developments'
resources = ResourceFactory().get_resources_for_dataset(
dataset_name,
in_storage=in_storage,
out_storage=out_storage,
resources=resources,
in_table_name_pair=(in_table_name, None),
out_table_name_pair=(out_table_name, None),
id_name_pair=(id_name, self.id_name_default),
nchunks_pair=(nchunks, nchunks_default),
debug_pair=(debug, None))
Dataset.__init__(self, resources=resources)
def __init__(self,
resources=None,
in_storage=None,
out_storage=None,
in_table_name=None,
out_table_name=None,
attributes=None,
id_name=None,
nchunks=None,
debuglevel=0):
try:
debug = SessionConfiguration().get('debuglevel', 0)
except:
debug = 0
debug = DebugPrinter(debug)
if debuglevel > debug.flag:
debug.flag = debuglevel
debug.print_debug(
"Creating object %s.%s" %
(self.__class__.__module__, self.__class__.__name__), 2)
resources = ResourceFactory().get_resources_for_dataset(
self.dataset_name,
resources=resources,
in_storage=in_storage,
in_table_name_pair=(in_table_name, self.in_table_name_default),
attributes_pair=(attributes, self.attributes_default),
out_storage=out_storage,
out_table_name_pair=(out_table_name, self.out_table_name_default),
id_name_pair=(id_name, self.id_name_default),
nchunks_pair=(nchunks, self.nchunks_default),
debug_pair=(debug, None),
)
CoreDataset.__init__(self, resources=resources)
def _get_resources_for_dataset(
self,
in_table_name_default,
in_table_name_groups_default,
out_table_name_default,
dataset_name,
resources=None,
in_storage=None,
out_storage=None,
in_table_name=None,
in_table_name_groups=None,
attributes=None,
out_table_name=None,
id_name=None,
id_name_default=None,
debug=None,
):
# Defaults:
attributes_default = AttributeType.PRIMARY
resources = ResourceFactory().get_resources_for_dataset(
dataset_name,
resources=resources,
in_storage=in_storage,
out_storage=out_storage,
in_table_name_pair=(in_table_name, in_table_name_default),
attributes_pair=(attributes, attributes_default),
out_table_name_pair=(out_table_name, out_table_name_default),
id_name_pair=(id_name, id_name_default),
debug_pair=(debug, None),
)
resources.merge_if_not_None(
{"in_table_name_groups": in_table_name_groups})
resources.merge_with_defaults(
{"in_table_name_groups": in_table_name_groups_default})
return resources