def get_resources_for_dataset(self, resources=None, in_storage=None, out_storage=None, in_table_name=None, in_table_name_groups=None, out_table_name=None, attributes=None, id_name=None, id_name_default=None, nchunks=None, debug=None, ): # Defaults: in_table_name_default = "plan_type_groups" attributes_default = AttributeType.PRIMARY out_table_name_default = "plan_type_groups" dataset_name = "plan_type_group" nchunks_default = 1 resources = ResourceFactory().get_resources_for_dataset( dataset_name, resources = resources, in_storage = in_storage, out_storage = out_storage, in_table_name_pair = (in_table_name,in_table_name_default), attributes_pair = (attributes,attributes_default), out_table_name_pair = (out_table_name, out_table_name_default), id_name_pair = (id_name,id_name_default), nchunks_pair = (nchunks,nchunks_default), debug_pair = (debug,None), ) resources.merge({"replace_nulls_with":"None"}) return resources
def _get_resources_for_dataset(self, in_table_name_default, in_table_name_groups_default, out_table_name_default, dataset_name, resources=None, in_storage=None, out_storage=None, in_table_name=None, in_table_name_groups=None, attributes=None, out_table_name=None, id_name=None, id_name_default=None, debug=None, ): # Defaults: attributes_default = AttributeType.PRIMARY resources = ResourceFactory().get_resources_for_dataset( dataset_name, resources = resources, in_storage = in_storage, out_storage = out_storage, in_table_name_pair = (in_table_name,in_table_name_default), attributes_pair = (attributes,attributes_default), out_table_name_pair = (out_table_name, out_table_name_default), id_name_pair = (id_name,id_name_default), debug_pair = (debug,None), ) resources.merge_if_not_None({"in_table_name_groups":in_table_name_groups}) resources.merge_with_defaults({"in_table_name_groups":in_table_name_groups_default}) return resources
def get_resources_for_dataset(self, resources=None, in_storage=None, out_storage=None, in_table_name=None, in_table_name_groups=None, out_table_name=None, attributes=None, id_name=None, id_name_default=None, nchunks=None, debug=None): # Defaults: in_table_name_default = "employment_adhoc_sector_groups" attributes_default = AttributeType.PRIMARY out_table_name_default = "employment_adhoc_sector_groups" dataset_name = "employment_sector_group" nchunks_default = 1 resources = ResourceFactory().get_resources_for_dataset( dataset_name, resources=resources, in_storage=in_storage, out_storage=out_storage, in_table_name_pair=(in_table_name, in_table_name_default), attributes_pair=(attributes, attributes_default), out_table_name_pair=(out_table_name, out_table_name_default), id_name_pair=(id_name, id_name_default), nchunks_pair=(nchunks, nchunks_default), debug_pair=(debug, None)) resources.merge({"replace_nulls_with": "None"}) return resources
def __init__(self, categories=array([ 1, ]), resources=None, what=None, attribute_name=None, data=None, names=None, in_storage=None, out_storage=None, in_table_name=None, attributes=None, out_table_name=None, id_name=None, nchunks=None, debuglevel=0): """ 'what' must be a string, such as 'residential' or 'commercial'. """ debug = DebugPrinter(debuglevel) debug.print_debug( "Creating DevelopmentProjectDataset object for %s projects." % what, 2) self.categories = categories self.what = what self.attribute_name = attribute_name attributes_default = AttributeType.PRIMARY dataset_name = "development_project" nchunks_default = 1 if data <> None: in_storage = StorageFactory().get_storage('dict_storage') in_storage.write_table(table_name='development_projects', table_data=data) in_table_name = 'development_projects' resources = ResourceFactory().get_resources_for_dataset( dataset_name, resources=resources, in_storage=in_storage, out_storage=out_storage, in_table_name_pair=(in_table_name, None), out_table_name_pair=(out_table_name, None), attributes_pair=(attributes, attributes_default), id_name_pair=(id_name, self.id_name_default), nchunks_pair=(nchunks, nchunks_default), debug_pair=(debug, None)) self.category_variable_name = resources.get( "category_variable_name", self.category_variable_name_default) Dataset.__init__(self, resources=resources)
def __init__( self, categories=array([1]), resources=None, what=None, attribute_name=None, data=None, names=None, in_storage=None, out_storage=None, in_table_name=None, attributes=None, out_table_name=None, id_name=None, nchunks=None, debuglevel=0, ): """ 'what' must be a string, such as 'residential' or 'commercial'. """ debug = DebugPrinter(debuglevel) debug.print_debug("Creating DevelopmentProjectDataset object for %s projects." % what, 2) self.categories = categories self.what = what self.attribute_name = attribute_name attributes_default = AttributeType.PRIMARY dataset_name = "development_project" nchunks_default = 1 if data <> None: in_storage = StorageFactory().get_storage("dict_storage") in_storage.write_table(table_name="development_projects", table_data=data) in_table_name = "development_projects" resources = ResourceFactory().get_resources_for_dataset( dataset_name, resources=resources, in_storage=in_storage, out_storage=out_storage, in_table_name_pair=(in_table_name, None), out_table_name_pair=(out_table_name, None), attributes_pair=(attributes, attributes_default), id_name_pair=(id_name, self.id_name_default), nchunks_pair=(nchunks, nchunks_default), debug_pair=(debug, None), ) self.category_variable_name = resources.get("category_variable_name", self.category_variable_name_default) Dataset.__init__(self, resources=resources)
def setUp(self): self.start_year = 2001 self.expected_sic_data = array([6,4,7,808,6]) self.job_id = array([1,2,3,4,5]) self.base_cache_dir = tempfile.mkdtemp(prefix='opus_tmp_test_dataset') self.simulation_state = SimulationState(low_memory_run=True, new_instance=True, base_cache_dir=self.base_cache_dir) self.dir = self.simulation_state.get_cache_directory() self.simulation_state.set_current_time(self.start_year) if not os.path.exists(self.dir): os.makedirs(self.dir) self.in_storage = StorageFactory().get_storage('dict_storage') self.in_storage.write_table( table_name='jobs', table_data={ 'grid_id':array([10,20,30,40,50]), 'job_id':self.job_id, }, ) self.out_storage = StorageFactory().get_storage('dict_storage') self.job_set_resources = ResourceFactory().get_resources_for_dataset( 'job', in_storage = self.in_storage, out_storage = self.out_storage, in_table_name_pair = ('jobs',None), out_table_name_pair = ('jobs_exported',None), attributes_pair = (None,AttributeType.PRIMARY), id_name_pair = ('job_id','job_id'), nchunks_pair = (1,1), debug_pair = (1,None) )
def __init__(self, resources=None, what="household", in_storage=None, in_table_name=None, out_storage=None, out_table_name=None, id_name=None, nchunks=None, debuglevel=0): ## TODO remove "what" arguement debug = DebugPrinter(debuglevel) debug.print_debug("Creating ControlTotalDataset object for "+what+".",2) if not self.in_table_name_default: self.in_table_name_default = "annual_" + what + "_control_totals" if not self.out_table_name_default: self.out_table_name_default = "annual_" + what + "_control_totals" attributes_default = AttributeType.PRIMARY #dataset_name = "control_total" nchunks_default = 1 resources = ResourceFactory().get_resources_for_dataset( self.dataset_name, resources=resources, in_storage=in_storage, out_storage=out_storage, in_table_name_pair=(in_table_name,self.in_table_name_default), attributes_pair=(None, attributes_default), out_table_name_pair=(out_table_name, self.out_table_name_default), id_name_pair=(id_name,self.id_name_default), nchunks_pair=(nchunks,nchunks_default), debug_pair=(debug,None) ) table_name = resources["in_table_name"] if resources['id_name'] is None or len(resources['id_name'])== 0: #if both self.id_name_default and id_name argument in __init__ is unspecified, #ControlTotalDataset would use all attributes not beginning with "total" #as id_name id_names = [] column_names = resources["in_storage"].get_column_names(table_name) for column_name in column_names: if not re.search('^total', column_name): id_names.append(column_name) resources.merge({"id_name":resources["id_name"] + id_names}) Dataset.__init__(self, resources = resources)
def __init__(self, resources=None, data=None, names=None, in_storage=None, out_storage=None, in_table_name=None, out_table_name=None, attributes=None, id_name=None, nchunks=None, debuglevel=0): debug = DebugPrinter(debuglevel) debug.print_debug("Creating LandUseDevelopmentDataset object.", 2) dataset_name = "landuse_development" nchunks_default = 1 if data <> None: in_storage = StorageFactory().get_storage('dict_storage') in_storage.write_table(table_name='landuse_developments', table_data=data) in_table_name = 'landuse_developments' resources = ResourceFactory().get_resources_for_dataset( dataset_name, in_storage=in_storage, out_storage=out_storage, resources=resources, in_table_name_pair=(in_table_name, None), out_table_name_pair=(out_table_name, None), id_name_pair=(id_name, self.id_name_default), nchunks_pair=(nchunks, nchunks_default), debug_pair=(debug, None)) Dataset.__init__(self, resources=resources)
def __init__(self, resources=None, in_storage=None, out_storage=None, in_table_name=None, out_table_name=None, attributes=None, id_name=None, nchunks=None, debuglevel=0): try: debug = SessionConfiguration().get('debuglevel', 0) except: debug = 0 debug = DebugPrinter(debug) if debuglevel > debug.flag: debug.flag = debuglevel debug.print_debug( "Creating object %s.%s" % (self.__class__.__module__, self.__class__.__name__), 2) resources = ResourceFactory().get_resources_for_dataset( self.dataset_name, resources=resources, in_storage=in_storage, in_table_name_pair=(in_table_name, self.in_table_name_default), attributes_pair=(attributes, self.attributes_default), out_storage=out_storage, out_table_name_pair=(out_table_name, self.out_table_name_default), id_name_pair=(id_name, self.id_name_default), nchunks_pair=(nchunks, self.nchunks_default), debug_pair=(debug, None), ) CoreDataset.__init__(self, resources=resources)
def _get_resources_for_dataset( self, in_table_name_default, in_table_name_groups_default, out_table_name_default, dataset_name, resources=None, in_storage=None, out_storage=None, in_table_name=None, in_table_name_groups=None, attributes=None, out_table_name=None, id_name=None, id_name_default=None, debug=None, ): # Defaults: attributes_default = AttributeType.PRIMARY resources = ResourceFactory().get_resources_for_dataset( dataset_name, resources=resources, in_storage=in_storage, out_storage=out_storage, in_table_name_pair=(in_table_name, in_table_name_default), attributes_pair=(attributes, attributes_default), out_table_name_pair=(out_table_name, out_table_name_default), id_name_pair=(id_name, id_name_default), debug_pair=(debug, None), ) resources.merge_if_not_None( {"in_table_name_groups": in_table_name_groups}) resources.merge_with_defaults( {"in_table_name_groups": in_table_name_groups_default}) return resources