def testSelectAttribute(self):
schema = Schema(self.schema)
nodes = schema.SelectAttribute('S1', 'StrA')
self.assertEquals(stringAttribute('S1', 'StrA'),
self.serializeXml(nodes))
def testDeleteAttribute(self):
schema2 = Schema(self.schema2)
schema2.RemoveAttribute('S2', 'EnumY')
result = schema2.SelectAttribute('S2', 'EnumY')
self.assertEqual(0, len(result))
def testDeleteAttribute_doesNotExist(self):
schema2 = Schema(self.schema2)
try:
schema2.RemoveAttribute('BadScope', 'BadAttribute')
self.fail("Expected exception was not thrown")
except Schema.Exception, e:
self.assertEqual("Attribute 'BadScope::BadAttribute' not found",
e.what)
def testInserAttribute_withNonExistingSourceScope(self):
schema1 = Schema(self.schema)
schema2 = Schema(self.schema2)
try:
schema1.InsertAttribute(schema2, 'BadScope', 'BadAttribute', 'S1')
self.fail("Expected exception was not thrown")
except Schema.Exception, e:
self.assertEqual("Attribute 'BadScope::BadAttribute' not found",
e.what)
def check_schema_all(req_data = {}):
""""""
try:
schema = Schema({"header":And({},),"body":And({},)},ignore_extra_keys=True)
return schema.validate(req_data)
except Exception,e:
print traceback.format_exc()
print str(e)
return ""
def testDefaultInit(self):
schema = Schema()
file = cStringIO.StringIO()
schema.Dump(file)
self.assertEqual(
"""\
<?xml version='1.0' encoding='UTF-8'?>
<DescriptionScheme>
<Attributes/>
</DescriptionScheme>
""", file.getvalue())
def QuerySchema(self, descriptors):
(aggregatorScript, sourceIds) = self._AggregatorScriptFor(descriptors)
aggregator = Aggregator(cStringIO.StringIO(aggregatorScript))
result = Schema()
aggregator.run(
result,
[self.sources[sourceId].QuerySchema() for sourceId in sourceIds])
return result
def testInserAttribute(self):
schema1 = Schema(self.schema)
schema2 = Schema(self.schema2)
schema1.InsertAttribute(schema2, 'S2', 'EnumY', 'S2')
result = schema1.SelectAttribute('S2', 'EnumY')
expect = schema2.SelectAttribute('S2', 'EnumY')
self.assertEquals(self.serializeXml(expect), self.serializeXml(result))
def schema_check(req_data =""):
schema_define = Schema(And(Use(json.loads),{
'header':{
"operator":And(str,),
"version":And(Use(str,len),lambda n: n in VERSION_LIST)
},
'body':{
"data":{And({},)}
}
}
))
try:
result = True
msg = ""
schema = schema_define.validate(req_data)
except Exception,e:
import traceback
print traceback.format_exc()
schema = {}
msg = str(e)
result = False
def load_data():
global data_collection
start = time.time()
if os.path.exists(paras.DATA_COL):
with open(paras.DATA_COL, 'rb') as f:
data_collection = pickle.load(f)
else:
vocab = Vocab.Vocab()
schemas = Schema.load_schema()
train_data = Data.load_data(paras.TRAIN_DATA_MERGE, schemas)
test_data = Data.load_data(paras.TEST_DATA, schemas)
train_data.get_indexes(vocab)
test_data.get_indexes(vocab)
print(Data.tot1)
print(Data.tot2)
print('train_data number:', len(train_data.data))
data_collection = DataCollection(vocab, schemas, train_data, test_data)
with open(paras.DATA_COL, 'wb') as f:
pickle.dump(data_collection, f)
end = time.time()
data_collection.vocab.print_info()
data_collection.schemas.print_info()
print('load data time cost:', end - start)
def __init__(self,
path=None,
schemaFile='schema.sc',
poolSuffix=".pool",
extractor=None):
self.path = path
if not schemaFile:
self.schemaFile = os.tmpnam() # TODO: Unsecure
elif self.path is None:
self.schemaFile = schemaFile
else:
self.schemaFile = os.path.join(self.path, schemaFile)
self.idsToRecalculate = []
self.poolSuffix = poolSuffix
# TODO: there should be an InvalidExtractorPath exception check
#if extractor and not os.access(extractor, os.X_OK) :
#raise FileMetadataSource.InvalidExtractorPathException(extractor)
self.extractor = extractor
if extractor and not os.access(self.schemaFile, os.R_OK):
os.system("%s -s %s" % (self.extractor, self.schemaFile))
try:
self.schema = Schema(file(self.schemaFile))
except IOError, e:
raise FileMetadataSource.InvalidSchemaException(self.schemaFile)
# a simple test
dd = {
'application' : ComponentItem(category='applications'),
'backend' : ComponentItem(category='backends'),
'name' : SimpleItem('',comparable=0),
'workdir' : SimpleItem(defvalue=None,type='string',transient=1,protected=1,comparable=0),
'status' : SimpleItem(defvalue='new', protected=1, comparable=0),
'id': SimpleItem(defvalue=None,type='string',protected=1, comparable=0),
'inputbox': FileItem(defvalue=[],sequence=1),
'outputbox': FileItem(defvalue=[],sequence=1),
'overriden_copyable' : SimpleItem(defvalue=None,protected=1,copyable=1),
'plain_copyable' : SimpleItem(defvalue=None,copyable=0)
}
schema = Schema(Version(1,0),dd)
# NOT a public interface: emulate the Ganga Plugin object for test purposes
# Note that pclass MUST be a new-style class in order to support deepcopy
class pclass(object):
_category = 'jobs'
_name = 'Job'
schema._pluginclass = pclass
# end of emulating code
#allSchemas.add(schema)
assert(schema.name == 'Job')
assert(schema.category == 'jobs')
assert(schema.allItems() == dd.items())
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
from Aggregator import *
from Schema import *
import sys
if sys.argv[1] == "-":
script = sys.stdin
else:
script = file(sys.argv[1])
sources = sys.argv[2:]
target = Schema()
aggregator = Aggregator(script)
aggregator.run(target, [Schema(file(source)) for source in sources])
target.Dump(sys.stdout)
def __init__(self, request, action):
super().__init__({{ cookiecutter.ConceptName }}Schema(), request, action, buttons=("submit", ))
#from colander import Length
#from deform.widget import TextAreaWidget, Select2Widget
#from {{ cookiecutter.app_name }}.enums import {{ cookiecutter.ConceptName }}Status
from {{ cookiecutter.app_name }}.helper.contract import Schema, Form #, string_property, enum_property
#from {{ cookiecutter.app_name }}.helper.translation import _
class {{ cookiecutter.ConceptName }}Schema(Schema):
pass
# some field examples
#name = string_property(title=_('name'), validator=Length(min=3, max=255))
#description = string_property(title=_('name'), validator=Length(min=10, max=2000), missing='')
#status = enum_property({{ cookiecutter.ConceptName }}Status, title=_('{{ cookiecutter.concept_name }}status'))
#tags = set_property(title=_('tags'), missing=tuple())
class {{ cookiecutter.ConceptName }}Form(Form):
def __init__(self, request, action):
super().__init__({{ cookiecutter.ConceptName }}Schema(), request, action, buttons=("submit", ))
def prepare_for_render(self, items_for_selects):
widgets = {
# 'description': TextAreaWidget(rows=8),
# 'status': Select2Widget(values=items_for_selects['{{ cookiecutter.concept_name }}_status']),
# 'tags': Select2Widget(multiple=True, tags=True, values=items_for_selects['tags']),
}
self.set_widgets(widgets)
# staging/transform area #
#####################################
# preprocess the data
df_cleaned = DataCleaning.clean_data(df)
df_preprocessed = DataCleaning.process_data(df_cleaned)
df_preprocessed.to_csv(staged_output, header=True, index=False)
#####################################
# transform & load #
#####################################
# Create star schema and load to destination
Schema.create_star_schema(
df_cleaned, **{
"fact_output": fact_output,
"cust_dim_output": cust_dim_output,
"prod_dim_output": prod_dim_output
})
#Extended Star Schema
print(
'Creating an Extended Star Schema, by joining with external Data Sources'
)
#User and Occupation
df_U = pd.read_csv(
r'/Users/sudiptkumarpanda/Downloads/Data Management 2 - Python Codes-20191110/CustDim.csv'
)
df_O = pd.read_csv(
r'/Users/sudiptkumarpanda/Downloads/Data Management 2 - Python Codes-20191110/ETL FRAMEWORK/data/External/Occupation_Categories.csv'
)