def generateRules(df, patients, meta=None):
ruleList = []
genes = list(df)
genes.remove('Metastatic')
booleans = [True, False]
for booleanA in booleans: #If true || false
for i in range(len(genes)): #check all of genes in first list
geneA = genes[i]
ruleList.append(Rule.Rule(geneA, booleanA)) #append single gene
for patient in patients: #Append all patients that fit the patient
if (patient.mutations[geneA] == booleanA
and patient.metastatic == meta):
ruleList[-1].patients.append(patient)
for booleanB in booleans:
for j in range(i, len(genes), 1):
geneB = genes[j]
if (geneA != geneB):
ruleList.append(
Rule.Rule(geneA,
booleanA,
gene2=geneB,
value2=booleanB))
for patient in patients:
if (patient.mutations[geneA] == booleanA
and patient.mutations[geneB] == booleanB
and patient.metastatic == meta):
ruleList[-1].patients.append(patient)
return ruleList
def loadRootRules(self, fileName):
ruleFile = open(fileName, 'r')
for line in ruleFile:
# some prepare job
line = line.replace('@', ' ')
line = line.replace(':', ' ')
line = line.replace('/', ' ')
rule = line.split()
# get source IP
srcIPAddr = self.ip2int(rule[0])
srcIPMask = self.generateMask(int(rule[1]), 32)
dstIPAddr = self.ip2int(rule[2])
dstIPMask = self.generateMask(int(rule[3]), 32)
if rule[4] == rule[5]:
srcPortAddr = int(rule[4])
srcPortMask = self.generateMask(16, 16)
else:
srcPortAddr = 0
srcPortMask = self.generateMask(0, 16)
if rule[6] == rule[7]:
dstPortAddr = int(rule[6])
dstPortMask = self.generateMask(16, 16)
else:
dstPortAddr = 0
dstPortMask = self.generateMask(0, 16)
addr = [srcIPAddr, dstIPAddr, srcPortAddr, dstPortAddr]
mask = [srcIPMask, dstIPMask, srcPortMask, dstPortMask]
tmpRule = Rule(0, addr, mask)
self.root.relatedRules.append(tmpRule)
def treeLeafDoubleClick(self, event):
item = event.GetItem()
if self.isLeafItem(item):
(serverText, groupText, itemText) = self.getItemPath(item)
itemValue = self.getItemValue(serverText, groupText, itemText)
if itemValue == None:
return
ruleItem = None
for rule in rules:
if rule.key == (serverText, groupText, itemText):
ruleItem = rule
break
else:
ruleItem = Rule.Rule()
configRuleDialog = CreateRuleDialog.CreateRuleDialog(ruleItem,
rulePath=self.getItemPath(item),
itemValue=itemValue,
rules=rules,
isBool=(type(itemValue[0]) == type(True))
)
self.totalRuleNum = len(rules)
configRuleDialog.Bind(wx.EVT_CLOSE, self.configDialogClose)
configRuleDialog.Bind(wx.EVT_WINDOW_DESTROY, self.configDialogDestroy)
configRuleDialog.ShowModal()
configRuleDialog.Destroy()
event.Skip()
def load_rules_table(self):
sql_fld = "RuleName, RuleID, Topic, Operation, TableName, last_modified"
sql_where = "3i_Rules.Enabled = TRUE"
res = self.env.dbc.select("3i_Rules",
sql_fld,
condition=sql_where,
group_by=None,
limit="10000")
rule_res = res.fetchall()
# Parse Direct Table Write Action Topics to be subscribed to from JSON File and add it to RuleList[]
for aTopic in rule_res:
logging.debug(aTopic)
if aTopic['Operation'] in (1, 2):
w = Actions.WriteTableActionClass(self.env,
aTopic['Operation'],
aTopic['TableName'])
r = RuleClass.Rule(name=aTopic['RuleName'],
topic=aTopic['Topic'],
rule_action=w,
time_stamp=['last_modified'])
self.env.rr.add_rule(r)
else:
erm = "Error: Unimplemented Action Rule Operation %s for rule %s" % (
aTopic['Operation'], aTopic['RuleName'])
logging.critical(erm)
self.env.quit_err()
def btnClick(self, e, grid):
ruleId = 1 if len(self.rules) == 0 else (self.rules[-1].Id + 1)
rule = Rule.Rule(ruleId, self.rulePath, True)
if self.isBool:
rule.dang = (grid.GetCellValue(4, 0) == u'真', grid.GetCellValue(4, 1))
rule.interal = grid.GetCellValue(5, 0)
rule.isBool = True
else:
rule.lower = (grid.GetCellValue(0, 0), grid.GetCellValue(0, 1))
rule.low = (grid.GetCellValue(1, 0), grid.GetCellValue(1, 1))
rule.high = (grid.GetCellValue(2, 0), grid.GetCellValue(2, 1))
rule.higher = (grid.GetCellValue(3, 0), grid.GetCellValue(3, 1))
rule.dang = (grid.GetCellValue(4, 0) == u'真', grid.GetCellValue(4, 1))
rule.interal = grid.GetCellValue(5, 0)
rule.isBool = False
for item in self.rules:
if item.key == self.rulePath:
index = self.rules.index(item)
self.rules[index] = rule
print "rule: {}".format(str(rule))
self.Destroy()
return
self.rules.append(rule)
print "create rule: {}".format(str(rule))
self.Destroy()
pass
def add_rule_from_string(self, l):
"""
Parse a rule in the standard format. We'll do just a very simple parser, defined above
e.g.
f[p*100+r] = f[p]+"\'s\u{e}r" , {"te" , f[r]}
"""
lhs, rhs = full_expr.parseString(l)
r = re.search(r'\s*f\[\s*([0-9]+)\s*\]\s*', lhs)
if r: # if we match a constant
q = int(r.groups(VAR)[0])
p = 1
r = 0
sgn = 1
else: # Now we must extract pqr from lhs. It should be of the form p*q+r or p*q-r
p, q, sgn, r = re.search(
r"([p0-9]+)\s*\*\s*([q0-9]+)\s*(\+|\-)\s*([r0-9]+)\s*",
lhs).groups(VAR)
p = VAR if p == "p" else int(p) # convert these to VAR symbols
q = VAR if q == "q" else int(q)
r = VAR if r == "r" else int(r)
if sgn == "+" or sgn == VAR:
sgn = 1 # convert sign to a multiplier
else:
sgn = -1
# convert structured ParseResults to list
parsed = [
x.asList() if isinstance(x, pyparsing.ParseResults) else x
for x in rhs
]
if any(map(islist, parsed)):
# if we have a sublist, we had brackets in the rule (via our parsing rules)
# so we must remove, and add a rule with r=0
self.add(
Rule(p, q, sgn, 0, filter(lambda x: not islist(x), parsed)))
# and add the normal rule where we unlist
self.add(Rule(p, q, sgn, r, flatten(parsed)))
else:
self.add(Rule(p, q, sgn, r, parsed))
def addresult(self, bug):
rule = Rule.Rule(scanner=self.rule.scanner,
resultwithdirect=True,
severity=bug['severity'],
xtype=bug['xtype'],
xurl=bug['xurl'],
suggestion=bug['suggestion'],
description=bug['description'])
result = self.getresultlist()
result.append(rule)
def generateRules(df, patients): ruleList = [] genes = list(df) booleans = [True, False] for booleanA in booleans: for i in range(len(genes)): geneA = genes[i] ruleList.append(Rule.Rule(geneA, booleanA)) for patient in patients: if (patient.mutations[geneA] == booleanA): ruleList[-1].patients.append(patient) for booleanB in booleans: for j in range(i, len(genes),1): geneB = genes[j] if (geneA != geneB): ruleList.append(Rule.Rule(geneA, booleanA, gene2=geneB, value2=booleanB)) for patient in patients: if (patient.mutations[geneA] == booleanA and patient.mutations[geneB] == booleanB): ruleList[-1].patients.append(patient) return ruleList
def mutate_rule(self, rule):
"""
mutates a rule
:param rule:
:return: a rule
"""
# chance to not mutate the rule
if random.randrange(0, 1000) > self.mutation_probability:
return rule
binary_code = rule.coding
mutated_code = self.mutate_code(binary_code)
new_rule = Rule(mutated_code, False)
return new_rule
def getRule(self):
path = os.path.split(os.path.realpath(__file__))[0]
for root, dirs, files in os.walk(path):
#print root, dirs, files
for filename in files:
filepath = os.path.join(root, filename)
if os.path.splitext(filepath)[1] == '.sb':
#print self.source
rule = Rule.Rule(scanner=self,
rfname=filepath,
program=self.program,
source=self.source)
if rule.load():
self.rules.append(rule)
def read(filename):
"""Read file with all the rule into IDS and return a list of rules"""
with open(filename, 'r') as fp:
cnt=1
errorCount=0
for line in fp:
try:
#print("Rule {} contents--> {}".format(cnt, line))
cnt += 1
rule = Rule(line)
rulelist.append(rule)
except ValueError as err:
errorCount+=1
print(err)
return rulelist, errorCount
def read(filename):
"""Read the input file for rules and return the list of rules and the number of line errors."""
l = list()
with open(filename, 'r') as f:
ruleErrorCount = 0
for line in f:
#rule = parseRule(line)
try:
rule = Rule(line)
l.append(rule)
except ValueError as err:
ruleErrorCount += 1
print err
return l, ruleErrorCount
def make_ga(self, dataset):
"""
initializes a set of GARule objects, by extracting all different persons out of the dataset
(most likely the train set)
:param dataset:
:return:
"""
population = []
answers = []
questions = []
tasks = []
first = True
for person in dataset:
# get the given answers
for task in person:
if 'Task-ID' in task:
quest_id = task['Task-ID']
else:
quest_id = task['TaskID']
ans = task['response']
person_id = person[0]['item'].identifier
answers.append([quest_id, ans, person_id])
# create the questions
quest = self.item_to_question(task['item'])
quest.question_id = quest_id
questions.append(quest)
tasks.append([ans, quest])
if first:
self.nr_of_questions = len(tasks)
print(len(tasks))
first = False
# make subjects
rules = self.get_rules()
for subject in range(0, self.test_subject_number):
rules = deepcopy(self.get_rules())
# make rules
subject_rules = []
for rule_pos in range(0, self.start_rule_number):
subject_rules.append(
Rule(rules[random.randrange(0, len(rules))].coding, False))
population.append(RuleTestPerson(subject_rules, 0, answers))
return GARule(population, tasks, self.get_rules)
def _get_dialog(self, k, refresh):
if k in self.protected_sources:
rules = _rules_per_source(k)
links = []
for rule in rules:
rule_pre = rule.split('!handler')[0]
r = Rule.Rule('%s!match' % (rule_pre))
rule_name = r.GetName()
rule_link = rule_pre.replace('!', '/')
links.append(CTK.consts.LINK_HREF % (rule_link, rule_name))
dialog = CTK.Dialog({
'title': _('Deletion is forbidden'),
'width': 480
})
dialog += CTK.RawHTML(
_('<h2>%s</h2>' % (_("Configuration consistency"))))
dialog += CTK.RawHTML(_(NOTE_FORBID_1))
dialog += CTK.RawHTML('<p>%s: %s</p>' %
(_(NOTE_FORBID_2), ', '.join(links)))
dialog.AddButton(_('Close'), "close")
else:
actions = {'source!%s' % (k): ''}
rule_entries_to_delete = _rules_per_source(k)
for r in rule_entries_to_delete:
actions[r] = ''
dialog = CTK.Dialog({
'title':
_('Do you really want to remove it?'),
'width':
480
})
dialog.AddButton(_('Cancel'), "close")
dialog.AddButton (_('Remove'), CTK.JS.Ajax (URL_APPLY, async=False,
data = actions,
success = dialog.JS_to_close() + \
refresh.JS_to_refresh()))
dialog += CTK.RawHTML(_(NOTE_DELETE_DIALOG))
def __init__(self, rules):
CTK.Container.__init__(self)
table = CTK.Table({'id': 'source-usage'})
self += CTK.RawHTML("<h2>%s</h2>" % (_('Source Usage')))
self += table
table.set_header(1)
table += [CTK.RawHTML(x) for x in (_('Virtual Server'), _('Rule'))]
for rule in rules:
vsrv_num = rule.split('!')[1]
vsrv_name = CTK.cfg.get_val("vserver!%s!nick" % (vsrv_num),
_("Unknown"))
r = Rule.Rule('%s!match' % (rule))
rule_name = r.GetName()
vsrv_link = '/vserver/%s' % (vsrv_num)
rule_link = rule.replace('!', '/')
table += [
CTK.Link(vsrv_link, CTK.RawHTML(vsrv_name)),
CTK.Link(rule_link, CTK.RawHTML(rule_name))
]
class JobSpider(CrawlSpider):
name = 'job'
allowed_domains = ['hr.tencent.com']
start_urls = ['http://hr.tencent.com/position.php']
# url提取规则
rules = [
# 定义了一个url规则,正则,回调函数,是否要跟进连接
Rule(LinkExtractor(allow = r'start=\d+'),callback = 'parseLink',follow = True)
]
# 定义回调
def parseLink(self,response):
trs = response.xpath("//tr[@class='even'] | //tr[@class='odd']")
for i in trs:
job = items.JobItem()
job['name'] = i.xpath("./td/a/text()").extract()
job['href'] = i.xpath("./td/a/@href").extract()
job['type_'] = i.xpath('./td[2]/text()').extract()
job['count'] = i.xpath('./td[3]/text()').extract()
job['address'] = i.xpath('./td[4]/text()').extract()
job['time'] = i.xpath('./td[5]/text()').extract()
yield job
def rewriteDatalog( metarule, cursor ) : logging.debug( " REWRITE DATALOG : running process..." ) logging.debug( " REWRITE DATALOG : metarule.ruleData = " + str( metarule.ruleData ) ) # ------------------------------------------------------ # # dedalus rewrites overwrite the original rules # so, grab the original rid rid = metarule.rid # ------------------------------------------------------ # # initialize new version of meta rule to old version of # meta rule new_metarule_ruleData = metarule.ruleData # ------------------------------------------------------ # # remove goal time arg new_metarule_ruleData[ "goalTimeArg"] = "" # ------------------------------------------------------ # # preserve adjustments by instantiating the new meta rule # as a Rule logging.debug( " REWRITE RULES : overwriting old rule with new ruleData = " + str( new_metarule_ruleData ) ) new_metarule = Rule.Rule( rid, new_metarule_ruleData, cursor ) # ------------------------------------------------------ # # populate rule type new_metarule.rule_type = "datalog" logging.debug( " REWRITE DATALOG : returning new meta rule with rule data = " + str( new_metarule.ruleData ) ) return new_metarule
# 创建 extrator 对象(匹配规则)
extrator = LinkExtractor(allow=('start=\d+',))
# 通过 extract_links 来解析 response 对象,获取Link对象集合
links = extrator.extract_links(response)
for i in links:
print(i) # Link(url='http://hr.tencent.com/position.php?tid=87&start=10#a', text='2', fragment='', nofollow=False)
-----------------------------
Rule |
-----------------------------
* 用于定义怎么去处理匹配到的连接
* 一个 Rule 里面会包含一个匹配规则(LinkExtractor),程序里面可以包含多个 Rules
* 如果多个 Rule 匹配了相同的连接,则根据规则在集合中的定义顺序,第一个将会被使用
* from scrapy.spiders import Rule
* 构造函数
Rule(link_extractor, callback=None, cb_kwargs=None, follow=None, process_links=None, process_request=identity)
link_extractor
* 表示一个匹配规则(LinkExtractor)
callback
* 回调函数名(str),每当从link_extractor中获取到连接的时候,都会作为参数传递给该回调函数
* 注意,该函数名称不能与'parse'冲突
follow
* bool 值,指定了根据该规则提取出来的连接是否要跟进(打开连接,深度提取)
* 如果 callback = None,该值默认为 True,否则该值为 False
process_links
* 以str形式,指定spider中哪个函数将会被调用,从匹配规则(LinkExtractor)中获取到链接列表
* 就是处理提取到的连接的方法,处理完毕后返回连接列表
-----------------------------
Spider-demo |
class YySpider(RedisCrawlSpider):
name = 'yy'
#allowed_domains = ['youyuan.com']
#start_urls = ['http://www.youyuan.com/find/beijing/mm18-25/advance-0-0-0-0-0-0-0/p1/']
redis_key = "yyspider:start_urls"
# 动态域范围获取
def __init__(self, *args, **kwargs):
# Dynamically define the allowed domains list.
domain = kwargs.pop('domain', '')
self.allowed_domains = filter(None, domain.split(','))
super(YySpider, self).__init__(*args, **kwargs)
# 第一级匹配规则：北京市18~25岁女性的每一页链接匹配规则
page_links = LinkExtractor(allow = (r"youyuan.com/find/beijing/mm18-25/advance-0-0-0-0-0-0-0/p\d+/"))
# 第二级匹配规则：每个女性个人主页的匹配规则
profile_links = LinkExtractor(allow = (r"youyuan.com/\d+-profile/"))
rules = (
Rule(page_links),
Rule(profile_links, callback = "parse_item"),
)
def parse_item(self, response):
item = YouyuanItem()
item['username'] = self.get_username(response)
# 年龄
item['age'] = self.get_age(response)
# 头像图片的链接
item['header_url'] = self.get_header_url(response)
# 相册图片的链接
item['images_url'] = self.get_images_url(response)
# 内心独白
item['content'] = self.get_content(response)
# 籍贯
item['place_from'] = self.get_place_from(response)
# 学历
item['education'] = self.get_education(response)
# 兴趣爱好
item['hobby'] = self.get_hobby(response)
# 个人主页
item['source_url'] = response.url
# 数据来源网站
item['sourec'] = "youyuan"
yield item
def get_username(self, response):
username = response.xpath("//dl[@class='personal_cen']//div[@class='main']/strong/text()").extract()
if len(username):
username = username[0]
else:
username = "******"
return username.strip()
def get_age(self, response):
age = response.xpath("//dl[@class='personal_cen']//dd/p/text()").extract()
if len(age):
age = re.findall(u"\d+岁", age[0])[0]
else:
age = "NULL"
return age.strip()
def get_header_url(self, response):
header_url = response.xpath("//dl[@class='personal_cen']/dt/img/@src").extract()
if len(header_url):
header_url = header_url[0]
else:
header_url = "NULL"
return header_url.strip()
def get_images_url(self, response):
images_url = response.xpath("//div[@class='ph_show']/ul/li/a/img/@src").extract()
if len(images_url):
images_url = ", ".join(images_url)
else:
images_url = "NULL"
return images_url
def get_content(self, response):
content = response.xpath("//div[@class='pre_data']/ul/li/p/text()").extract()
if len(content):
content = content[0]
else:
content = "NULL"
return content.strip()
def get_place_from(self, response):
place_from = response.xpath("//div[@class='pre_data']/ul/li[2]//ol[1]/li[1]/span/text()").extract()
if len(place_from):
place_from = place_from[0]
else:
place_from = "NULL"
return place_from.strip()
def get_education(self, response):
education = response.xpath("//div[@class='pre_data']/ul/li[3]//ol[2]/li[2]/span/text()").extract()
if len(education):
education = education[0]
else:
education = "NULL"
return education.strip()
def get_hobby(self, response):
hobby = response.xpath("//dl[@class='personal_cen']//ol/li/text()").extract()
if len(hobby):
hobby = ",".join(hobby).replace(" ","")
else:
hobby = "NULL"
return hobby.strip()
def aggProv(aggRule, nameAppend, cursor):
# create bindings rule (see LDFI paper section 4.1.2)
bindingsRule = regProv(aggRule, nameAppend, cursor)
# generate random ID
rid = tools.getID()
# initialize firings rule
firingsRule = Rule.Rule(rid, cursor)
# goal info
goalName = aggRule.getGoalName() + "_prov" + str(
rid) # allows tables for duplicate names
goalAttList = aggRule.getGoalAttList()
goalTimeArg = ""
rewrittenFlag = 0 # new rules have not yet been rewritten
# check for bugs
if PROVENANCEREWRITE_DEBUG:
print "aggProv: goalName = " + goalName
print "aggProv: goalAttList = " + str(goalAttList)
# subgoal list info
subgoalName = bindingsRule.getGoalName()
subgoalAttList_init = bindingsRule.getGoalAttList()
# check for bugs
if PROVENANCEREWRITE_DEBUG:
print "aggProv: subgoalName = " + subgoalName
print "aggProv: subgoalAttList_init = " + str(subgoalAttList_init)
aggAtts = []
for att in goalAttList:
containsAgg = False
for op in aggOps:
if op in att:
containsAgg = True
if containsAgg:
att = att.split("<")
att = att[1].replace(">", "")
aggAtts.append(att)
subgoalAttList_final = []
for att in subgoalAttList_init:
for a in aggAtts:
if a == att:
subgoalAttList_final.append("_")
else:
subgoalAttList_final.append(att)
# check for bugs
if PROVENANCEREWRITE_DEBUG:
print "aggProv: subgoalAttList_final = " + str(subgoalAttList_final)
sid = tools.getID()
subgoalTimeArg = ""
subgoalAddArgs = ""
# save rule
firingsRule.setGoalInfo(goalName, goalTimeArg, rewrittenFlag)
firingsRule.setGoalAttList(goalAttList)
firingsRule.setSingleSubgoalInfo(sid, subgoalName, subgoalTimeArg)
firingsRule.setSingleSubgoalAttList(sid, subgoalAttList_final)
firingsRule.setSingleSubgoalAddArgs(sid, subgoalAddArgs)
# ----------------------------------------------------------- #
# set goal attribute types for all rules
firingsRule.setAttTypes()
def regProv(regRule, nameAppend, cursor):
if PROVENANCEREWRITE_DEBUG:
print " ... running regProv ..."
#sys.exit( "BREAKPOINT: regRule = " + str(regRule.getSubgoalListStr()) )
# -------------------------------------------------- #
# parse rule
parsedRule = dedalusParser.parse(regRule.display())
#sys.exit( "BREAKPOINT: regRule.display() = " + str( regRule.display() ) + "\nparsedRule = " + str(parsedRule) )
# -------------------------------------------------- #
# generate random ID for new rule
rid = tools.getID()
# -------------------------------------------------- #
# initialize new rule
firingsRule = Rule.Rule(rid, cursor)
# -------------------------------------------------- #
# get goal info
goalName = regRule.getGoalName() + nameAppend + str(rid)
goalTimeArg = ""
rewrittenFlag = 1 # new log rules are not rewritten
# check for bugs
if PROVENANCEREWRITE_DEBUG:
print "regProv: goalName = " + goalName
# -------------------------------------------------- #
# get subgoal array
subgoalArray = extractors.extractSubgoalList(parsedRule[1])
# ................................... #
#if goalName.startswith( "pre_prov" ) :
# tools.bp( __name__, inspect.stack()[0][3], "subgoalArray = " + str(subgoalArray) )
# ................................... #
# check for bugs
if PROVENANCEREWRITE_DEBUG:
print "regProv: subgoalArray = " + str(subgoalArray)
# -------------------------------------------------- #
# collect goal args while inserting subgoals
firstSubgoalAtts = []
goalAttList = []
# -------------------------------------------------- #
# populate with original goal atts first
goalAttList = regRule.getGoalAttList()
if PROVENANCEREWRITE_DEBUG:
print ">>>>>>>> goalAttList init = " + str(goalAttList)
# -------------------------------------------------- #
# then populate with remaining subgoal atts
for subgoal in subgoalArray:
# generate random ID for subgoal
sid = tools.getID()
# -------------------------------------------------- #
# extract info
subgoalName = extractors.extractSubgoalName(subgoal)
subgoalAttList = extractors.extractAttList(subgoal) # returns list
subgoalTimeArg = ""
subgoalAddArgs = extractors.extractAdditionalArgs(
subgoal) # returns list
# check for bugs
if PROVENANCEREWRITE_DEBUG:
print "regProv: subgoal = " + str(subgoal)
print "regProv: subgoalName = " + subgoalName
print "regProv: subgoalAttList = " + str(subgoalAttList)
print "regProv: subgoalAddArgs = " + str(subgoalAddArgs)
# -------------------------------------------------- #
# catch first subgoal att
if len(subgoalAddArgs) == 0:
firstSubgoalAtts.append(subgoalAttList[0])
# -------------------------------------------------- #
# populate goal attribute list
for att in subgoalAttList:
if (not att in goalAttList) and (not att.isdigit()) and (
not att == "_"): # exclude numbers from goal atts
goalAttList.append(att)
# -------------------------------------------------- #
# make sure time attribute appears as rightmost attribute
if not goalAttList == None:
for att in goalAttList:
# make sure time attribute(s) appear in the rightmost columns
if "SndTime" in att:
goalAttList = [x for x in goalAttList if x != att]
goalAttList.append(att)
# -------------------------------------------------- #
# save firings subgoal
firingsRule.setSingleSubgoalInfo(sid, subgoalName, subgoalTimeArg)
firingsRule.setSingleSubgoalAttList(sid, subgoalAttList)
firingsRule.setSingleSubgoalAddArgs(sid, subgoalAddArgs)
if PROVENANCEREWRITE_DEBUG:
print ">>>>>>>> goalAttList final = " + str(goalAttList)
# -------------------------------------------------- #
# get eqn array
eqnArray = regRule.getEquationListArray()
# save firings rule equations
for eqn in eqnArray:
# generate random ID
eid = tools.getID()
# save eqn
firingsRule.setSingleEqn(eid, eqn)
# -------------------------------------------------- #
# save firings rule goal
firingsRule.setGoalInfo(goalName, goalTimeArg, rewrittenFlag)
firingsRule.setGoalAttList(goalAttList)
# ----------------------------------------------------------- #
# set goal attribute types for all rules
firingsRule.setAttTypes()
return firingsRule
def get_random_rule(self):
rand = random.randrange(0, len(self.rules) - 1)
return Rule(self.rule_fct()[rand].coding, False)
import Rule as r
import CFG as c
r1=r.Rule('moein khorasani | avin')
v=set()
rules=set()
alpha=set()
s=''
line=input('Enter your Varibles:\n')
v=line.split(',')
line=input('Enter your alphebet:\n')
alpha=line.split(',')
s=input('Enter your start var:\n')
print('Enter your rules')
while True:
line=input('')
if line.upper()=='END':
break
else:
tmpr=r.Rule(line)
rules.add(tmpr)
c1=c.CFG(v,alpha,rules,s)
b=c1.accept('aaaaaaaaaacccccccc')
for i in range(10):
print('\n')
if b==True:
print('yes')
else:
print('no')
def rewriteInductive( argDict, metarule, cursor ) :
logging.debug( " REWRITE INDUCTIVE : running process..." )
logging.debug( " REWRITE INDUCTIVE : metarule.ruleData = " + str( metarule.ruleData ) )
# ------------------------------------------------------ #
# grab the next rule handling method
try :
NEXT_RULE_HANDLING = tools.getConfig( argDict[ "settings" ], \
"DEFAULT", \
"NEXT_RULE_HANDLING", \
str )
except ConfigParser.NoOptionError :
logging.info( "WARNING : no 'NEXT_RULE_HANDLING' defined " + \
"in 'DEFAULT' section of settings file." )
tools.bp( __name__, inspect.stack()[0][3], "FATAL ERROR : NEXT_RULE_HANDLING parameter not " + \
"specified in DEFAULT section of settings file. use 'USE_AGGS', 'SYNC_ASSUMPTION', or " + \
"'USE_NEXT_CLOCK' only." )
# sanity check next rule handling value
if NEXT_RULE_HANDLING == "USE_AGGS" or \
NEXT_RULE_HANDLING == "SYNC_ASSUMPTION" or \
NEXT_RULE_HANDLING == "USE_NEXT_CLOCK" :
pass
else :
tools.bp( __name__, inspect.stack()[0][3], "FATAL ERROR : " + \
"unrecognized NEXT_RULE_HANDLING value '" + NEXT_RULE_HANDLING + \
"'. use 'USE_AGGS', 'SYNC_ASSUMPTION', or 'USE_NEXT_CLOCK' only." )
# ------------------------------------------------------ #
# dedalus rewrites overwrite the original rules
# so, grab the original rid
rid = metarule.rid
# ------------------------------------------------------ #
# initialize new version of meta rule to old version of
# meta rule
new_metarule_ruleData = metarule.ruleData
# ------------------------------------------------------ #
# add SndTime+1/DelivTime to goal attribute list
if NEXT_RULE_HANDLING == "USE_AGGS" :
new_metarule_ruleData[ "goalAttList"].append( timeAtt_snd+"+1" )
elif NEXT_RULE_HANDLING == "SYNC_ASSUMPTION" :
new_metarule_ruleData[ "goalAttList"].append( timeAtt_deliv )
#new_metarule_ruleData[ "eqnDict" ][ "MRESERVED==NRESERVED+1" ] = { "variableList" : [ "MRESRVED", "NRESERVED" ] }
elif NEXT_RULE_HANDLING == "USE_NEXT_CLOCK" :
new_metarule_ruleData[ "goalAttList"].append( timeAtt_deliv )
# ------------------------------------------------------ #
# remove goal time arg
new_metarule_ruleData[ "goalTimeArg"] = ""
# ------------------------------------------------------ #
# add SndTime (or given numeric time argument)
# to all subgoal attribute lists
for subgoal in new_metarule_ruleData[ "subgoalListOfDicts" ] :
# ------------------------------------------------------ #
# CASE : subgoal time argument in an integer
if subgoal[ "subgoalTimeArg" ].isdigit() :
subgoal[ "subgoalAttList" ].append( subgoal[ "subgoalTimeArg" ] )
subgoal[ "subgoalTimeArg" ] = "" # erase time arg after assignment
# ------------------------------------------------------ #
# CASE : subgoal has no time argument
else :
subgoal[ "subgoalAttList" ].append( timeAtt_snd )
# ------------------------------------------------------ #
# add clock subgoal
# grab the first attribute in a subgoal
# observe the parser ensures the first attributes
# in all inductive rule subgoals
firstAtt = new_metarule_ruleData[ "subgoalListOfDicts" ][0][ "subgoalAttList" ][0]
# build the new clock subgoal dict
# format :
# { subgoalName : 'subgoalNameStr',
# subgoalAttList : [ data1, ... , dataN ],
# polarity : 'notin' OR '',
# subgoalTimeArg : <anInteger> }
if NEXT_RULE_HANDLING == "USE_AGGS" or NEXT_RULE_HANDLING == "SYNC_ASSUMPTION" :
if NEXT_RULE_HANDLING == "USE_AGGS" :
clock_subgoalAttList = [ firstAtt, "_", timeAtt_snd, "_" ]
elif NEXT_RULE_HANDLING == "SYNC_ASSUMPTION" :
clock_subgoalAttList = [ firstAtt, "_", timeAtt_snd, "MRESERVED" ] # only works for synchronous model.
clock_subgoalName = "clock"
clock_polarity = "" # clocks are positive until proven negative.
clock_subgoalTimeArg = ""
elif NEXT_RULE_HANDLING == "USE_NEXT_CLOCK" :
clock_subgoalAttList = [ firstAtt, "_", timeAtt_snd, "MRESERVED" ]
clock_subgoalName = "next_clock"
clock_polarity = "" # clocks are positive until proven negative.
clock_subgoalTimeArg = ""
clock_subgoalDict = {}
clock_subgoalDict[ "subgoalName" ] = clock_subgoalName
clock_subgoalDict[ "subgoalAttList" ] = clock_subgoalAttList
clock_subgoalDict[ "polarity" ] = clock_polarity
clock_subgoalDict[ "subgoalTimeArg" ] = clock_subgoalTimeArg
# ------------------------------------------------------ #
# add the clock subgoal to the subgoal list for this rule
new_metarule_ruleData[ "subgoalListOfDicts" ].append( clock_subgoalDict )
# ------------------------------------------------------ #
# preserve adjustments by instantiating the new meta rule
# as a Rule
new_metarule = Rule.Rule( rid, new_metarule_ruleData, cursor )
# ------------------------------------------------------ #
# populate rule type
new_metarule.rule_type = "inductive"
logging.debug( " REWRITE INDUCTIVE : returning new meta rule with rule data = " + str( new_metarule.ruleData ) )
return new_metarule
def rewriteAsynchronous( metarule, cursor ) :
logging.debug( " REWRITE ASYNCHRONOUS : running process..." )
logging.debug( " REWRITE ASYNCHRONOUS : metarule.ruleData = " + str( metarule.ruleData ) )
# ------------------------------------------------------ #
# dedalus rewrites overwrite the original rules
# so, grab the original rid
rid = metarule.rid
# ------------------------------------------------------ #
# initialize new version of meta rule to old version of
# meta rule
new_metarule_ruleData = metarule.ruleData
# ------------------------------------------------------ #
# add DelivTime to goal attribute list
new_metarule_ruleData[ "goalAttList"].append( timeAtt_deliv )
# ------------------------------------------------------ #
# remove goal time arg
new_metarule_ruleData[ "goalTimeArg"] = ""
# ------------------------------------------------------ #
# add SndTime (or given numeric time argument)
# to all subgoal attribute lists
for subgoal in new_metarule_ruleData[ "subgoalListOfDicts" ] :
# ------------------------------------------------------ #
# CASE : subgoal time argument in an integer
if subgoal[ "subgoalTimeArg" ].isdigit() :
subgoal[ "subgoalAttList" ].append( subgoal[ "subgoalTimeArg" ] )
subgoal[ "subgoalTimeArg" ] = "" # erase time arg after assignment
# ------------------------------------------------------ #
# CASE : subgoal has no time argument
else :
subgoal[ "subgoalAttList" ].append( timeAtt_snd )
# ------------------------------------------------------ #
# add clock subgoal
# assum the first att in all subgoals of
# next rules is the source.
# grab the first attribute in a subgoal
# observe the parser ensures the first attributes
# in all inductive rule subgoals
firstAtt = new_metarule_ruleData[ "subgoalListOfDicts" ][0][ "subgoalAttList" ][0]
# assume the first att in the goal list of
# async rules is the destingation.
# grab the first attribute in the goal att list
# observe the parser ensures the first attributes
# in all async rule subgoals
secondAtt = new_metarule_ruleData[ "goalAttList" ][0]
# build the new clock subgoal dict
# format :
# { subgoalName : 'subgoalNameStr',
# subgoalAttList : [ data1, ... , dataN ],
# polarity : 'notin' OR '',
# subgoalTimeArg : <anInteger> }
clock_subgoalName = "clock"
clock_subgoalAttList = [ firstAtt, secondAtt, timeAtt_snd, timeAtt_deliv ]
clock_polarity = "" # clocks are positive until proven negative.
clock_subgoalTimeArg = ""
clock_subgoalDict = {}
clock_subgoalDict[ "subgoalName" ] = clock_subgoalName
clock_subgoalDict[ "subgoalAttList" ] = clock_subgoalAttList
clock_subgoalDict[ "polarity" ] = clock_polarity
clock_subgoalDict[ "subgoalTimeArg" ] = clock_subgoalTimeArg
# ------------------------------------------------------ #
# add the clock subgoal to the subgoal list for this rule
new_metarule_ruleData[ "subgoalListOfDicts" ].append( clock_subgoalDict )
# ------------------------------------------------------ #
# preserve adjustments by instantiating the new meta rule
# as a Rule
logging.debug( " REWRITE RULES : overwriting old rule with new ruleData = " + str( new_metarule_ruleData ) )
new_metarule = Rule.Rule( rid, new_metarule_ruleData, cursor )
# ------------------------------------------------------ #
# populate rule type
new_metarule.rule_type = "asynchronous"
logging.debug( " REWRITE ASYNCHRONOUS : returning new meta rule with rule data = " + str( new_metarule.ruleData ) )
return new_metarule
def dedToIR( filename, cursor, settings_path ) :
parsedLines = []
parsedLines = dedalusParser.parseDedalus( filename, settings_path ) # program exits here if file cannot be opened.
logging.debug( " DED TO IR : parsedLines = " + str( parsedLines ) )
# collect fact and rule metadata for future dumping.
# these are lists of object pointers.
factMeta = []
ruleMeta = []
# ----------------------------------------------- #
# FACTS #
# ----------------------------------------------- #
# process facts first to establish
# all fact data types.
# iterate over parsed lines
for line in parsedLines :
if line[0] == "fact" :
logging.debug( ">> PROCESSING FACT : " + str( line ) )
# get data for fact
factData = line[1]
# generate random ID for fact
#fid = tools.getID()
fid = tools.getIDFromCounters( "fid" )
logging.debug( " DED TO IR : fid = " + str( fid ) )
# save fact data in persistent DB using IR
newFact = Fact.Fact( fid, factData, cursor )
# save fact metadata (aka object)
factMeta.append( newFact )
# ----------------------------------------------- #
# RULES #
# ----------------------------------------------- #
# process rules after consuming all facts
# iterate over parsed lines
for line in parsedLines :
if line[0] == "rule" : # save rules
logging.debug( ">> PROCESSING RULE : " + str( line ) )
# get data for rule
ruleData = line[1]
# generate a random ID for the rule
#rid = tools.getID()
rid = tools.getIDFromCounters( "rid" )
logging.debug( " DED TO IR : rid = " + str( rid ) )
# save rule goal info
newRule = Rule.Rule( rid, ruleData, cursor )
# save rule metadata (aka object)
ruleMeta.append( newRule )
return [ factMeta, ruleMeta ]
def __init__(self, source=""):
self.source = source
self.rules = []
self.rules += [Rule.Rule("#version", "VERSION_DIRECTIVE")]
self.rules += [Rule.Rule("#define", "DEFINE_DIRECTIVE")]
self.rules += [Rule.Rule("/\*(.|\n)*\*/", "MULTILINE_COMMENT")]
self.rules += [Rule.Rule("//.*\n", "SINGLELINE_COMMENT")]
self.rules += [Rule.Rule("\n", "CRLF")]
self.rules += [Rule.Rule("\d+", "INTEGER_CONSTANT")]
self.rules += [
Rule.Rule("((-?\d+\.\d*)|(-?\d*\.\d+))([eE]([-\+]?)\d+)?",
"FLOAT_CONSTANT")
]
self.rules += [Rule.Rule("if", "IF")]
self.rules += [Rule.Rule("else", "ELSE")]
self.rules += [Rule.Rule("uniform", "UNIFORM")]
self.rules += [Rule.Rule("const", "CONST")]
self.rules += [Rule.Rule("float", "FLOAT")]
self.rules += [Rule.Rule("int", "INT")]
self.rules += [Rule.Rule("vec2", "VEC2")]
self.rules += [Rule.Rule("vec3", "VEC3")]
self.rules += [Rule.Rule("vec4", "VEC4")]
self.rules += [Rule.Rule("mat2", "MAT2")]
self.rules += [Rule.Rule("mat3", "MAT3")]
self.rules += [Rule.Rule("mat4", "MAT4")]
self.rules += [Rule.Rule("sampler2D", "SAMPLER2D")]
self.rules += [Rule.Rule(";", "SEMICOLON")]
self.rules += [Rule.Rule(",", "COLON")]
self.rules += [Rule.Rule("\.", "DOT")]
self.rules += [Rule.Rule("=", "EQUAL")]
self.rules += [Rule.Rule("<=", "LESSEQUAL")]
self.rules += [Rule.Rule(">=", "GREATEREQUAL")]
self.rules += [Rule.Rule("==", "ISEQUAL")]
self.rules += [Rule.Rule("!=", "NOTEQUAL")]
self.rules += [Rule.Rule("<", "LESSTHAN")]
self.rules += [Rule.Rule(">", "GREATERTHAN")]
self.rules += [Rule.Rule("\+=", "PLUSEQUAL")]
self.rules += [Rule.Rule("\*=", "TIMESEQUAL")]
self.rules += [Rule.Rule("/=", "DIVEQUAL")]
self.rules += [Rule.Rule("-=", "MINUSEQUAL")]
self.rules += [Rule.Rule("\*", "TIMES")]
self.rules += [Rule.Rule("/", "DIVBY")]
self.rules += [Rule.Rule("\+", "PLUS")]
self.rules += [Rule.Rule("-", "MINUS")]
self.rules += [Rule.Rule("\(", "LPAREN")]
self.rules += [Rule.Rule("\)", "RPAREN")]
self.rules += [Rule.Rule("\[", "LBRACKET")]
self.rules += [Rule.Rule("\]", "RBRACKET")]
self.rules += [Rule.Rule("{", "LBRACE")]
self.rules += [Rule.Rule("}", "RBRACE")]
self.rules += [Rule.Rule("gl_FragColor", "GL_FRAGCOLOR")]
self.rules += [Rule.Rule("gl_FragCoord", "GL_FRAGCOORD")]
self.rules += [Rule.Rule("in", "IN_QUALIFIER")]
self.rules += [Rule.Rule("out", "OUT_QUALIFIER")]
self.rules += [Rule.Rule("inout", "INOUT_QUALIFIER")]
self.rules += [Rule.Rule("void", "VOID")]
self.rules += [Rule.Rule("texture", "TEXTURE")]
self.rules += [Rule.Rule("main", "MAIN")]
self.rules += [Rule.Rule("sin", "SIN")]
self.rules += [Rule.Rule("cos", "COS")]
self.rules += [Rule.Rule("tan", "TAN")]
self.rules += [Rule.Rule("asin", "ASIN")]
self.rules += [Rule.Rule("acos", "ACOS")]
self.rules += [Rule.Rule("atan", "ATAN")]
self.rules += [Rule.Rule("sinh", "SINH")]
self.rules += [Rule.Rule("cosh", "COSH")]
self.rules += [Rule.Rule("tanh", "TANH")]
self.rules += [Rule.Rule("asinh", "ASINH")]
self.rules += [Rule.Rule("acosh", "ACOSH")]
self.rules += [Rule.Rule("atanh", "ATANH")]
self.rules += [Rule.Rule("[a-zA-Z_]+[a-zA-Z0-9_]*", "IDENTIFIER")]
self.index = 0
self.line = 0
self.column = 0
pass
def rewriteDeductive( metarule, cursor ) :
logging.debug( " REWRITE DEDUCTIVE : running process..." )
logging.debug( " REWRITE DEDUCTIVE : metarule.ruleData = " + str( metarule.ruleData ) )
# ------------------------------------------------------ #
# dedalus rewrites overwrite the original rules
# so, grab the original rid
rid = metarule.rid
# ------------------------------------------------------ #
# initialize new version of meta rule to old version of
# meta rule
new_metarule_ruleData = metarule.ruleData
# ------------------------------------------------------ #
# add SndTime to goal attribute list
new_metarule_ruleData[ "goalAttList"].append( timeAtt_snd )
# ------------------------------------------------------ #
# add SndTime (or given numeric time argument)
# to all subgoal attribute lists
for subgoal in new_metarule_ruleData[ "subgoalListOfDicts" ] :
# ------------------------------------------------------ #
# CASE : subgoal time argument in an integer
if subgoal[ "subgoalTimeArg" ].isdigit() :
subgoal[ "subgoalAttList" ].append( subgoal[ "subgoalTimeArg" ] )
subgoal[ "subgoalTimeArg" ] = "" # erase time arg after assignment
# ------------------------------------------------------ #
# CASE : subgoal has no time argument
else :
subgoal[ "subgoalAttList" ].append( timeAtt_snd )
# ------------------------------------------------------ #
# add clock subgoal
#
# NOTE!!!!
# Deductive rules do NOT need clock subgoals. I'm adding this b/c
# molly adds clock subgoals to deductive rules.
# Semantically, the clock subgoals only contribute self-comms,
# which are ignored later in the LDFI workflow.
# grab the first attribute in a subgoal
# observe the parser ensures the first attributes
# in all inductive rule subgoals
firstAtt = new_metarule_ruleData[ "subgoalListOfDicts" ][0][ "subgoalAttList" ][0]
# build the new clock subgoal dict
# format :
# { subgoalName : 'subgoalNameStr',
# subgoalAttList : [ data1, ... , dataN ],
# polarity : 'notin' OR '',
# subgoalTimeArg : <anInteger> }
clock_subgoalName = "clock"
clock_subgoalAttList = [ firstAtt, firstAtt, timeAtt_snd, "_" ]
clock_polarity = "" # clocks are positive until proven negative.
clock_subgoalTimeArg = ""
clock_subgoalDict = {}
clock_subgoalDict[ "subgoalName" ] = clock_subgoalName
clock_subgoalDict[ "subgoalAttList" ] = clock_subgoalAttList
clock_subgoalDict[ "polarity" ] = clock_polarity
clock_subgoalDict[ "subgoalTimeArg" ] = clock_subgoalTimeArg
# ------------------------------------------------------ #
# add the clock subgoal to the subgoal list for this rule
flag = False
for subgoal in new_metarule_ruleData[ "subgoalListOfDicts" ] :
if subgoal[ "subgoalAttList" ][-1] == "NRESERVED" :
flag = True
# make sure at least one subgoal references NRESERVED.
# need the clock reference otherwise.
if not flag :
new_metarule_ruleData[ "subgoalListOfDicts" ].append( clock_subgoalDict )
# ------------------------------------------------------ #
# preserve adjustments by instantiating the new meta rule
# as a Rule
new_metarule = Rule.Rule( rid, new_metarule_ruleData, cursor )
# ------------------------------------------------------ #
# populate rule type
new_metarule.rule_type = "deductive"
logging.debug( " REWRITE DEDUCTIVE : returning new meta rule with rule data = " + str( new_metarule.ruleData ) )
return new_metarule
def add_rule(self, state1, state2, symbol, action):
rule = Rule(state1, state2, symbol, action)
rule.show_rule()
self.rule_book.append(rule)
return
# Could be something like this
#
# programtype
# >0.9 / \ <0.9
# date fail
# >0.7 / \ <0.7
# time fail
# / | | \
# fail
#
ruleset = {'programtype': Rule('programtype', 'stringmatch', [u'NYHEDSMAGASINET', u'MORGENREDAKTIONEN', u'SØNDAGSAVISEN', u'TV-AVISEN', u'RADIOAVISEN', u'P2-MORGENMAGASINET', u'GO\' MORGEN P3'], 0.8),
'airdate': Rule('airdate', 'stringmatch', [u'Mandag', u'Tirsdag', u'Onsdag', u'Torsdag', u'Fredag', u'Lørdag', u'Søndag'], 0.7),
'date': Rule('date', 'datematch', [], 0.85),
'time': Rule('time', 'timematch', [], 0.75),
}
ruletree = [ {'rule': 'programtype', True: {'rule':'airdate', True: None,
False:None},
False:None},
{'rule': 'date', True: None, False: None},
{'rule': 'time', True: None, False: None} ]
# {'rule': 'airdate', True: None, False: None} ]
def format_block_text(t):
newtxt = re.sub(r'(\w)\n(\w)', r'\1 \2', t, flags=re.UNICODE) # Get rid of "ordinary" line wraps
