def prepare(self):
# before we do anything serious, check if we need to
# bootstrap the schema for schema
if self.schemas.has_key(XMLSchemaNS):
# we're validating a schema with a (purported) schema for schemas,
# or we're running useDTD=0
sfors=self.schemas[XMLSchemaNS]
self.sfors=sfors
if ((not sfors.typeTable.has_key('string')) or
(not sfors.typeTable['string'].__dict__.has_key('basetype')) or
(sfors.typeTable['string'].basetype is None) or
sfors.typeTable['string'].basetype.name is not AST):
# need the ab-initio types
sfors.targetNS=XMLSchemaNS
self.current=sfors
self.targetNS=XMLSchemaNS
sfors.doBuiltIns(self)
else:
self.initSforS()
self.sfors.nsdict={'xs':XMLSchemaNS}
sforsi=Schema(self,None)
self.schemas[XMLSchemaInstanceNS]=sforsi
self.sforsi=sforsi
sforsi.targetNS=XMLSchemaInstanceNS
self.current=sforsi
self.targetNS=XMLSchemaInstanceNS
sforsi.installInstanceAttrs(self)
sforsi.prepared=1
self.current=self.sfors
self.targetNS=XMLSchemaNS
ec=0
for sch in self.schemas.values():
ec=ec+sch.errors
self.prepared=1
return ec
def initFromDump(self,dumpFileName):
self.prepare()
self.rebuilding=1
rds=indRebuild.fromIndFile(dumpFileName,self.psviConstruct)
if rds is None:
sys.stderr.write("couldn't initialize from %s!!!\n"%dumpFileName)
raise error
for nsi in rds.schemaInformation:
if nsi.schemaNamespace:
# xsi is skipped
if self.schemas.has_key(nsi.schemaNamespace):
s=self.schemas[nsi.schemaNamespace]
else:
s=Schema(self,None)
s.targetNS=s.targetNamespace=nsi.schemaNamespace
self.schemas[nsi.schemaNamespace]=s
s.prepared=1
for c in nsi.components:
if c.foundWhere:
if getattr(s,c.foundWhere).has_key(c.name):
if not (isinstance(c,SimpleType) and
c.name in ('ENTITY','NMTOKEN','Name','language')):
# they're allowed as they have the original of various patterns
print ('oops',s,c,c.name)
else:
getattr(s,c.foundWhere)[c.name]=c
self.rebuilding=0
self.current=None
self.targetNS=0
def __calculateOperatorSize(self, operator, schemas):
res = None
if self.__isOperator(operator):
if operator.startswith("SIGMA"):
print("input: ", schemas)
condition = self.__getSub(operator, self.__SquareBrackets)
res = Schema.applySigma(operator, condition, schemas)
elif operator.startswith("PI"):
print("input: ", schemas)
columns = self.__getSub(operator, self.__SquareBrackets)
columns = self.__getColumns(columns)
res = Schema.applyPi(operator, schemas, columns)
elif operator.startswith("CARTESIAN"):
print("input: ", schemas[0], ",", schemas[1])
res = Schema.applyCartesian(schemas[0], schemas[1])
elif operator.startswith("NJOIN"):
print("input: ", schemas[0], ",", schemas[1])
res = Schema.applyJoin(schemas[0], schemas[1])
print("output:", res)
print("------------------------------------------------------")
return res
def initSforS(self): sfors=Schema(self,None) self.schemas[XMLSchemaNS]=sfors self.sfors=sfors sfors.targetNS=XMLSchemaNS self.current=sfors self.targetNS=XMLSchemaNS sfors.doBuiltIns(self)
def checkFlows(self):
for f in self.flows:
result = self.getFlow(f[2])
if result:
schema = Schema()
schema.fromJSON(result,self.getSchema(f[2]))
for original in self.schemas.schemas:
print original.flow["sourcePort"],schema.flow["sourcePort"]
if original.equalsFlow(schema):
self.recentMatches.append(schema)
def checkFlows(self):
for f in self.flows:
result = self.getFlow(f[2])
if result:
schema = Schema()
schema.fromJSON(result, self.getSchema(f[2]))
for original in self.schemas.schemas:
print original.flow["sourcePort"], schema.flow[
"sourcePort"]
if original.equalsFlow(schema):
self.recentMatches.append(schema)
def checkFlows(self):
time.sleep(10)
for f in self.flows:
result = self.getFlow(f[2])
if result:
schema = Schema()
schema.fromJSON(result)
for original in self.schemas.schemas:
if original.equals(schema):
print "sflow match"
self.recentMatches.append(schema)
def anonymizeFile(self, nlfile, anonfile, mapfile, schema, db, stopfile):
schemaObj = Schema(schema, stopfile)
schemaObj.buildIndex('/tmp/' + db + '.index', db)
anonfile = open(anonfile, 'w')
mapfile = open(mapfile, 'w')
for line in open(nlfile, 'r'):
(anon, mp) = self.anonymize(line, schemaObj)
anonfile.write(anon + '\n')
mapfile.write(json.dumps(mp) + '\n')
anonfile.close()
mapfile.close()
def __init__(self, src_schema_file, tgt_schema_file):
with open(src_schema_file, "r") as f:
file = f.read()
src_queries = file.split(';')[:-1]
schema, tables = self.create_tables(
src_queries) #, self.src_db)#, self.src_schema)
self.src_schema = Schema(schema, tables)
with open(tgt_schema_file, "r") as f:
file = f.read()
tgt_queries = file.split(';')[:-1]
schema, tables = self.create_tables(
tgt_queries) #, self.src_db)#, self.src_schema)
self.tgt_schema = Schema(schema, tables)
def createDB(self,sliceName,sliceFieldList,indexName):
# SliceEnv.envSchema is the tables already existed in file, which is a map
newSchema = Schema(sliceName,sliceFieldList,indexName)
if newSchema.getSliceName() not in SliceEnv.envSchema:
# create a new schema in the map
SliceEnv.envSchema[newSchema.getSliceName()] = newSchema
open("Data/" + sliceName + ".slc", 'w')
print "create Sucessful"
else:
# update the existed schema
if not os.path.isfile("Data/" + sliceName + ".slc"):
open("Data/" + sliceName + ".slc", 'w')
print "create Sucessful"
SliceEnv.envSchema[sliceName] = newSchema
pickle.dump(SliceEnv.envSchema, open( "Data/Config.txt", "wb"))
def setup():
return Schema().create('role', [
Blueprint().id('idRole'), {
**Blueprint().string('nameRole'),
**Blueprint().nullable(False)
}
], 1)
def setup():
return Schema().create('user', [
Blueprint().id('idUser'), {
**Blueprint().string('lastnameUser'),
**Blueprint().nullable(False)
}, {
**Blueprint().string('firstnameUser'),
**Blueprint().nullable(False)
}, {
**Blueprint().string('emailUser'),
**Blueprint().nullable(False)
}, {
**Blueprint().string('passwordUser'),
**Blueprint().nullable(False)
}, {
**Blueprint().string('phoneUser'),
**Blueprint().nullable(False)
}, {
**Blueprint().string('lastConnectDate'),
**Blueprint().nullable(True)
}, {
**Blueprint().foreign('idRole'),
**Blueprint().references('idRole'),
**Blueprint().on('role')
}
], 1)
class Args(object):
def __init__(self, schema: str, command: str):
self.command = Command(command)
self.schema = Schema(schema)
def get_value(self, name):
return self.schema.get_value(name, self.command.get_value(name))
def seed():
return Schema().insert('role', [{
'idRole': 1,
'nameRole': 'User'
}, {
'idRole': 2,
'nameRole': 'Admin'
}])
def deanonymizeFile(self, mapfile, sqlfile, schema, stopfile):
schemaObj = Schema(schema, stopfile)
mapfile = open(mapfile, 'r').readlines()
outfile = open(sqlfile + '.deanon', 'w')
sqlfile = open(sqlfile, 'r').readlines()
for m, s in itertools.izip(mapfile, sqlfile):
q = self.deanonymize(s, json.loads(m), schemaObj)
deanon = q.getQuery()
outfile.write(deanon + '\n')
outfile.close()
def generateFromList(grammar, schema, db, ppscale, paraphrase_file, stopfile,
host, user, passwd):
typeMap = {}
paraphraser = PPDB(paraphrase_file, ppscale)
schema = Schema(FLAGS.schema, stopfile)
data = []
for line in grammar:
if len(line) == 0 or line[0] == '#': # ignore empty lines and comments
continue
elif "=>" in line: # A new type
(lhs, rhs) = line.split('=>')
typeMap[lhs.strip()] = [x.strip() for x in rhs.strip().split('|')]
else: # A template
templates = line.split('\t')
sql = templates.pop()
for t in templates:
query = Query(t, sql, host=host, user=user, passwd=passwd)
generate(query, typeMap, schema, db, paraphraser, data)
return data
def __createSchemas(self, i_scheme):
newSchema = Schema()
schema = i_scheme[0]
newSchema.Name = schema[0:schema.find('(')]
variables = schema[schema.find("(") + 1:schema.find(")")].split(",")
for variable in variables:
variableParts = variable.split(":")
variableName = variableParts[0]
variableType = variableParts[1]
newSchema.Columns[variableName] = variableType
vaiablesData = i_scheme[1:]
for data in vaiablesData:
parts = data.split('=')
if parts[0] in "n_" + newSchema.Name:
newSchema.RowCount = parts[1]
elif parts[0].startswith("V"):
columnName = parts[0]
columnName = columnName[columnName.find("(") +
1:columnName.find(")")]
newSchema.ColumnsNumberOfUniqueValues[columnName] = int(
parts[1])
self.__Schemas.append(newSchema)
def test_bool(self):
a = Schema("l:bool,p:int,d:str")
self.assertEqual(a.get_value("l", "true"), True)
self.assertEqual(a.get_value("l", None), False)
def test_str(self):
a = Schema("l:bool,p:int,d:str")
self.assertEqual(a.get_value("d", "usr/log"), "usr/log")
def test_int(self):
a = Schema("l:bool,p:int,d:str")
self.assertEqual(a.get_value("p", "99"), 99)
self.assertEqual(a.get_value("p", "-9"), -9)
self.assertEqual(a.get_value("p", None), 8080)
def initSchema(self): self.queryExec(Schema.tablePerson()) self.queryExec(Schema.tablePhonebook()) self.queryExec(Schema.tableRelation())
def execute(self, rootdir):
if not os.path.isdir(os.path.join(rootdir, self.table_name)):
raise ValueError("Table {} does not exist.".format(
self.table_name))
schema = Schema(os.path.join(rootdir, self.table_name, 'table.json'))
self.all_fields = [
SelectField(field,
result_type=schema.get_field_type(field),
index=schema.get_field_index(field))
for field in schema.get_all_field_names()
]
if self.field_list[0] == "*":
self.field_list = self.all_fields
for field in self.field_list:
field.set_result_type(schema.get_field_type(field.name))
field.index = schema.get_field_index(field.name)
for i, field in enumerate(self.all_fields):
for field2 in self.field_list:
if field.name == field2.name:
self.all_fields[i] = field2
where_func = self.where_to_func(schema)
file_name = '{}.csv'.format(self.table_name)
group = None
if self.group_by_list:
group = GroupBy(rootdir, self.all_fields, self.field_list,
self.table_name, self.group_by_list, self.having)
group.generate_temp_file()
file_name = os.path.join('temp', 'group_by_file')
order = None
if self.order_by_list:
order = OrderBy(rootdir, self.table_name, self.all_fields,
self.order_by_list, file_name)
order.generate_temp_file()
temp_file_name = os.listdir(
os.path.join(rootdir, self.table_name, 'temp'))[0]
table = open(
os.path.join(rootdir, self.table_name, 'temp', temp_file_name),
'r')
else:
table = open(os.path.join(rootdir, self.table_name, file_name),
'r')
reader = csv.reader(table)
if self.outfile_name is None:
i = 0
for row in reader:
if i >= 200:
break
if row and where_func(row):
row = [row[field.index] for field in self.field_list]
print(','.join(row))
i += 1
else:
with open(os.path.join(rootdir, self.outfile_name),
'w',
newline='') as outfile:
writer = csv.writer(outfile)
for row in reader:
if row and where_func(row):
row = [row[field.index] for field in self.field_list]
writer.writerow(row)
table.close()
if order is not None:
shutil.rmtree(order.temp_file_dir)
elif group is not None:
shutil.rmtree(group.temp_file_dir)
def __init__(self, schema: str, command: str):
self.command = Command(command)
self.schema = Schema(schema)
