def query_AuId_AuId(auId1, auId2, json1, json2):
sys.stderr.write('query_AuId_AuId ' + str(auId1) + ' ' + str(auId2) + '\n')
ans = []
# =========== 2-hop ===========
#print len(json1), len(json2)
# print json2
AFIdSet1 = set()
IdList1 = []
for paper1 in json1:
if paper1.has_key('AA'):
for i in paper1['AA']:
if i.has_key('AfId') and i['AuId'] == auId1:
AFIdSet1.add(i['AfId'])
if paper1.has_key('Id'):
IdList1.append(paper1['Id'])
IdList1.sort()
AFIdSet2 = set()
IdList2 = []
for paper2 in json2:
if paper2.has_key('AA'):
for i in paper2['AA']:
if i.has_key('AfId') and i['AuId'] == auId2:
AFIdSet2.add(i['AfId'])
if paper2.has_key('Id'):
IdList2.append(paper2['Id'])
IdList2.sort()
#AuId-AFId-AuId
if len(AFIdSet1)>0 and len(AFIdSet2)>0:
# AFIdList1.sort()
# AFIdList2.sort()
# jointAFIdList = join(AFIdList1, AFIdList2)
# jointAFIdList = np.intersect1d(AFIdList1, AFIdList2)
jointAFIdSet = AFIdSet1.intersection(AFIdSet2)
# for AFId in jointAFIdList:
map(lambda x: ans.append([auId1, x, auId2]), jointAFIdSet)
#print len(ans)
#AuId-Id-AuId
if len(IdList1)>0 and len(IdList2)>0:
jointIdList = join(IdList1, IdList2)
map(lambda x: ans.append([auId1, x, auId2]), jointIdList)
#print len(ans)
# =========== 3-hop ===========
# AuId-Id-Id-AuId
for paper1 in json1:
if paper1.has_key('RId'):
RIdList = paper1['RId']
RIdList.sort()
jointList = join(RIdList, IdList2)
map(lambda x: ans.append([auId1, paper1['Id'], x, auId2]), jointList)
# paper1['RId'] = dict(zip(paper1['RId'], range(len(paper1['RId']))))
# for paper2 in json2:
# if paper2['Id'] in paper1['RId']:
# ans.append([auId1, paper1['Id'], paper2['Id'], auId2])
#print len(ans)
return ans
def get_niftis(subject_id, data_dir):
from glob import glob
from os.path import join, basename, exists
parent_dir = '/Users/Katie/Dropbox/Data/habenula/'
t1 = join(parent_dir, subject_id, 'anat', '3d.nii')
epi = join(parent_dir, subject_id, 'func', 'rest.nii')
assert exists(t1), "t1 does not exist"
assert exists(epi), "epi does not exist"
return t1, epi
def query_AuId_AuId(auId1, auId2, json1, json2):
sys.stderr.write('query_AuId_AuId ' + str(auId1) + ' ' + str(auId2) + '\n')
ans = []
# =========== 2-hop ===========
AFIdSet1 = set()
IdList1 = []
for paper1 in json1:
if 'AA' in paper1:
for i in paper1['AA']:
if 'AfId' in i and i['AuId'] == auId1:
AFIdSet1.add(i['AfId'])
if 'Id' in paper1:
IdList1.append(paper1['Id'])
IdList1.sort()
AFIdSet2 = set()
IdList2 = []
for paper2 in json2:
if 'AA' in paper2:
for i in paper2['AA']:
if 'AfId' in i and i['AuId'] == auId2:
AFIdSet2.add(i['AfId'])
if 'Id' in paper2:
IdList2.append(paper2['Id'])
IdList2.sort()
#AuId-AFId-AuId
if len(AFIdSet1)>0 and len(AFIdSet2)>0:
# AFIdList1.sort()
# AFIdList2.sort()
# jointAFIdList = join(AFIdList1, AFIdList2)
# jointAFIdList = np.intersect1d(AFIdList1, AFIdList2)
jointAFIdSet = AFIdSet1.intersection(AFIdSet2)
map(lambda x: ans.append([auId1, x, auId2]), jointAFIdSet)
#AuId-Id-AuId
if len(IdList1)>0 and len(IdList2)>0:
jointIdList = join(IdList1, IdList2)
map(lambda x: ans.append([auId1, x, auId2]), jointIdList)
# =========== 3-hop ===========
# AuId-Id-Id-AuId
for paper1 in json1:
if 'RId' in paper1:
RIdList = paper1['RId']
RIdList.sort()
jointList = join(RIdList, IdList2)
map(lambda x: ans.append([auId1, paper1['Id'], x, auId2]), jointList)
return ans
def adddate(dbfn, report = True):
infile = join(inpath, dbfn)
dbdf = featherize(infile)
for col in dbdf:
dateformatted_col = list()
inconsistent_msg = list()
for row_i, val in enumerate(col):
try:
result = datecheck(val)
except AssertionError:
continue # Ignore dateformat failed match
matched_dateformat = result['dateformat']
if len(matched_dateformat) > 1:
errvars = (val, col, row_i, matched_dateformat)
errmsg = 'Value %s in column %s row %d matches the following formats' % errvars
print(errmsg)
if row_i = 1:
continue
consistent_format_condition = matched_dateformat == last_dateformat
constitent_format_msg = 'Warning: Inconsistent dateformat for row %d in column %s' % (row_i, col)
try:
assert consistent_format_condition, consistent_format_msg
except AssertionError:
inconsistent_msg.append(i_row)
## JUST DO TH COLUMNS
last_dateformat = matched_dateformat
if inconsistent, print #some matched others didn't, verify
def load_from_xlsx(self, fnames, compute_scores=False):
""" Load attributes from a given list of xlsx files """
#Load tables one by one
lt = []
for f in fnames:
lt.append(get_table(f))
#Join tables
t = join(*lt)
#Align columns
out = find_matching_columns(t)
#Store data in attributes
[self.tf, self.time, self.ind, self.q] = out
def query_AuId_Id(auId1, id2, json1, json2):
#sys.stderr.write('query_AuId_Id ' + str(auId1) + ' ' + str(id2) + '\n')
print 'query_AuId_Id', auId1, id2
ans = []
now = time.time()
url = 'https://oxfordhk.azure-api.net/academic/v1.0/evaluate?expr=RId=%d&count=100000&orderby=CC:desc&attributes=Id&subscription-key=f7cc29509a8443c5b3a5e56b0e38b5a6'%id2
Id2citedResult = pool.apply_async(lambda url:urlrequest(url)['entities'], (url,))
#now = time.time()
#json2 = getPaperJson(id2, 'F.FId,J.JId,C.CId,AA.AuId,AA.AfId')
#print 'time use2: ', time.time() - now
#url = 'https://oxfordhk.azure-api.net/academic/v1.0/evaluate?expr=Composite(AA.AuId=%d)&count=20000&attributes=Id,RId,F.FId,J.JId,C.CId,AA.AuId&orderby=D:desc&subscription-key=f7cc29509a8443c5b3a5e56b0e38b5a6'%auId1
#json1 = json.loads((urllib.urlopen(url)).read())['entities']
# =========== 1-hop ===========
# AuId-Id
for paper in json1:
if paper['Id'] == id2:
ans.append([auId1, id2])
break
# =========== 2-hop ===========
# AuId-Id-Id
for paper in json1:
if paper.has_key('RId') and (id2 in paper['RId']):
ans.append([auId1, paper['Id'], id2])
# =========== 3-hop ===========
# AuId-Id-F.FId-Id
if json2.has_key('F'):
FIdList2 = map(lambda x:x['FId'], json2['F'])
FIdList2.sort()
for paper in json1:
if paper.has_key('F'):
FIdListTmp = map(lambda x:x['FId'], paper['F'])
FIdListTmp.sort()
jointFIdList = join(FIdListTmp, FIdList2)
map(lambda x: ans.append([auId1, paper['Id'], x, id2]), jointFIdList)
# AuId-Id-C.CId-Id
if json2.has_key('C'):
CId2 = json2['C']['CId']
for paper in json1:
if paper.has_key('C') and paper['C']['CId'] == CId2:
ans.append([auId1, paper['Id'], CId2, id2])
# AuId-Id-J.JId-Id
if json2.has_key('J'):
JId2 = json2['J']['JId']
for paper in json1:
if paper.has_key('J') and paper['J']['JId'] == JId2:
ans.append([auId1, paper['Id'], JId2, id2])
# AuId-Id-AA.AuId-Id
if json2.has_key('AA'):
AuIdList2 = map(lambda x:x['AuId'], json2['AA'])
AuIdList2.sort()
for paper in json1:
if paper.has_key('AA'):
AuIdListTmp = map(lambda x:x['AuId'], paper['AA'])
AuIdListTmp.sort()
jointAuIdList = join(AuIdListTmp, AuIdList2)
map(lambda x: ans.append([auId1, paper['Id'], x, id2]), jointAuIdList)
print 'AuId-Id-AuId-Id finished time =', time.time() - now
# AuId-Id-Id-Id
#now = time.time()
Id2cited = Id2citedResult.get()
print 'Id2cited geted time =', time.time() - now
if len(Id2cited) > 0:
Id2citedList = map(lambda x:x['Id'], Id2cited)
Id2citedList.sort()
for paper in json1:
if paper.has_key('RId'):
RIdListTmp = paper['RId']
RIdListTmp.sort()
jointRIdList = join(RIdListTmp, Id2citedList)
map(lambda x: ans.append([auId1, paper['Id'], x, id2]), jointRIdList)
# AuId-AA.AFId-AA.AuId-Id
AFIdSet1 = set()
for paper in json1:
if paper.has_key('AA'):
for author in paper['AA']:
if author['AuId'] == auId1 and author.has_key('AfId'):
AFIdSet1.add(author['AfId'])
if len(AFIdSet1) > 0 and json2.has_key('AA'):
authorPaperList = getAuthorPaperList(map(lambda x:x['AuId'], json2['AA']), 'AA.AuId,AA.AfId')
authorSet2 = set()
map(lambda x:authorSet2.add(x['AuId']), json2['AA'])
uniqueSet = set()
for paper in authorPaperList:
if paper.has_key('AA'):
for author in paper['AA']:
auId2 = author['AuId']
if (auId2 in authorSet2) and author.has_key('AfId') and (author['AfId'] in AFIdSet1) and not((author['AfId'], auId2) in uniqueSet):
ans.append([auId1, author['AfId'], auId2, id2])
uniqueSet.add((author['AfId'], auId2))
print 'AuId-AfId-AuId-Id finished time =', time.time() - now
#AFIdSet1 = set()
#for paper in json1:
# if paper.has_key('AA'):
# for author in paper['AA']:
# if author['AuId'] == auId1 and author.has_key('AfId'):
# AFIdSet1.add(author['AfId'])
#if json2.has_key('AA'):
# for author in json2['AA']:
# if author.has_key('AfId'):
# if author['AfId'] in AFIdSet1:
# ans.append([auId1, author['AfId'], author['AuId'], id2])
return ans
def query_Id_Id_small(id1, id2, json1, json2):
#sys.stderr.write('query_Id_Id ' + str(id1) + ' ' + str(id2) + '\n')
print 'query_Id_Id_small', id1, id2
ans = []
#print json1
#json1 = getPaperJson(id1, 'RId,F.FId,J.JId,C.CId,AA.AuId')
#json2 = getPaperJson(id2, 'F.FId,J.JId,C.CId,AA.AuId')
#print json1['RId']
#print json2
now = time.time()
url = 'https://oxfordhk.azure-api.net/academic/v1.0/evaluate?expr=RId=%d&count=50000&orderby=CC:desc&attributes=Id,F.FId,J.JId,C.CId,AA.AuId&subscription-key=f7cc29509a8443c5b3a5e56b0e38b5a6'%id2
Id2citedResult = pool.apply_async(lambda url:urlrequest(url)['entities'], (url,))
if json1.has_key('RId'):
urlAttributes = 'Id,RId'
if json2.has_key('F'):
urlAttributes += ',F.FId'
if json2.has_key('J'):
urlAttributes += ',J.JId'
if json2.has_key('C'):
urlAttributes += ',C.CId'
if json2.has_key('AA'):
urlAttributes += ',AA.AuId'
# id1CitePapersInfoResult = pool.map_async(lambda x:getPaperJson(x, urlAttributes), json1['RId'])
id1CitePapersInfo = getPaperJsonList(json1['RId'], urlAttributes)
#print 'time use2: ', time.time() - now
# =========== 1-hop ===========
# Id-Id
if json1.has_key('RId') and id2 in json1['RId']:
ans.append([id1, id2])
# =========== 2-hop ===========
# Id-F.FId-Id
if json1.has_key('F'):
FIdList1 = map(lambda x:x['FId'], json1['F'])
FIdList1.sort()
if json2.has_key('F'):
FIdList2 = map(lambda x:x['FId'], json2['F'])
FIdList2.sort()
if json1.has_key('F') and json2.has_key('F'):
jointFIdList = join(FIdList1, FIdList2)
map(lambda x: ans.append([id1, x, id2]), jointFIdList)
# Id-J.JId-Id
if json1.has_key('J') and json2.has_key('J') and json1['J']['JId'] == json2['J']['JId']:
ans.append([id1, json1['J']['JId'], id2])
# Id-C.CId-Id
if json1.has_key('C') and json2.has_key('C') and json1['C']['CId'] == json2['C']['CId']:
ans.append([id1, json1['C']['CId'], id2])
# Id-AA.AuId-Id
if json1.has_key('AA'):
AuIdList1 = map(lambda x:x['AuId'], json1['AA'])
AuIdList1.sort()
if json2.has_key('AA'):
AuIdList2 = map(lambda x:x['AuId'], json2['AA'])
AuIdList2.sort()
if json1.has_key('AA') and json2.has_key('AA'):
jointAuIdList = join(AuIdList1, AuIdList2)
map(lambda x: ans.append([id1, x, id2]), jointAuIdList)
# Id-Id-Id
Id2cited = Id2citedResult.get()
if json1.has_key('RId'):
RIdList = json1['RId']
Id2citedList = map(lambda x:x['Id'], Id2cited)
RIdList.sort()
Id2citedList.sort()
jointRIdList = join(RIdList, Id2citedList)
map(lambda x: ans.append([id1, x, id2]), jointRIdList)
# =========== 3-hop ===========
# Id-*-Id-Id
if len(Id2cited) != 0:
# Id-F.FId-Id-Id
if json1.has_key('F'):
for paper in Id2cited:
if paper.has_key('F'):
for F_element in paper['F']:
if F_element['FId'] in FIdList1:
ans.append([id1, F_element['FId'], paper['Id'], id2])
# Id-J.JId-Id-Id
if json1.has_key('J'):
for paper in Id2cited:
if paper.has_key('J') and paper['J']['JId'] == json1['J']['JId']:
ans.append([id1, json1['J']['JId'], paper['Id'], id2])
# Id-C.CId-Id-Id
if json1.has_key('C'):
for paper in Id2cited:
if paper.has_key('C') and paper['C']['CId'] == json1['C']['CId']:
ans.append([id1, json1['C']['CId'], paper['Id'], id2])
# Id-AA.AuId-Id-Id
if json1.has_key('AA'):
for paper in Id2cited:
if paper.has_key('AA'):
for AA_element in paper['AA']:
if AA_element['AuId'] in AuIdList1:
ans.append([id1, AA_element['AuId'], paper['Id'], id2])
# Id-Id-*-Id
if json1.has_key('RId'):
#id1CitePapersInfo = id1CitePapersInfoResult.get()
# Id-Id-F.FId-Id
if json2.has_key('F'):
for id1CitePaper in id1CitePapersInfo:
if id1CitePaper.has_key('F'):
FIdListTmp = map(lambda x:x['FId'], id1CitePaper['F'])
FIdListTmp.sort()
jointFIdList = join(FIdListTmp, FIdList2)
map(lambda x: ans.append([id1, id1CitePaper['Id'], x, id2]), jointFIdList)
# Id-Id-J.JId-Id
if json2.has_key('J'):
for id1CitePaper in id1CitePapersInfo:
if id1CitePaper.has_key('J') and id1CitePaper['J']['JId'] == json2['J']['JId']:
ans.append([id1, id1CitePaper['Id'], json2['J']['JId'], id2])
# Id-Id-C.CId-Id
if json2.has_key('C'):
for id1CitePaper in id1CitePapersInfo:
if id1CitePaper.has_key('C') and id1CitePaper['C']['CId'] == json2['C']['CId']:
ans.append([id1, id1CitePaper['Id'], json2['C']['CId'], id2])
# Id-Id-AA.AuId-Id
if json2.has_key('AA'):
for id1CitePaper in id1CitePapersInfo:
if id1CitePaper.has_key('AA'):
AuIdListTmp = map(lambda x:x['AuId'], id1CitePaper['AA'])
AuIdListTmp.sort()
jointAuIdList = join(AuIdListTmp, AuIdList2)
map(lambda x: ans.append([id1, id1CitePaper['Id'], x, id2]), jointAuIdList)
# Id-Id-Id-Id
tmp = len(ans)
for id1CitePaper in id1CitePapersInfo:
if id1CitePaper.has_key('RId'):
RIdListTmp = id1CitePaper['RId']
RIdListTmp.sort()
jointRIdList = join(RIdListTmp, Id2citedList)
map(lambda x: ans.append([id1, id1CitePaper['Id'], x, id2]), jointRIdList)
#print 'len(Id-Id-Id-Id) =', len(ans)-tmp
# return ans
return ans
+from pyqtbuild import PyQtBindings, PyQtProject
+from sipbuild import Option
+import PyQt5
+
+
+class PythonPopplerQt5(PyQtProject):
+ """The Project class."""
+
+ def __init__(self):
+ super().__init__()
+ self.bindings_factories = [PopplerQt5Bindings]
+
+ def update(self, tool):
+ """Allows SIP to find PyQt5 .sip files."""
+ super().update(tool)
+ self.sip_include_dirs.append(join(PyQt5.__path__[0], 'bindings'))
+
+
+class PopplerQt5Bindings(PyQtBindings):
+ """The Poppler-Qt5 Bindings class."""
+
+ def __init__(self, project):
+ super().__init__(project, name='Poppler-Qt5',
+ sip_file='poppler-qt5.sip',
+ qmake_QT=['xml'])
+
+ def get_options(self):
+ """Our custom options that a user can pass to sip-build."""
+ options = super().get_options()
+ options.append(
+ Option('poppler_version',
def dumm():
schedule.every().saturday.at(firstClass).do(join())
def query_Id_Id_small(id1, id2, json1, json2): print 'query_Id_Id_small', id1, id2 ans = [] now = time.time() url = 'https://oxfordhk.azure-api.net/academic/v1.0/evaluate?expr=RId=%d&count=50000&orderby=CC:desc&attributes=Id,F.FId,J.JId,C.CId,AA.AuId&subscription-key=f7cc29509a8443c5b3a5e56b0e38b5a6'%id2 Id2citedResult = pool.apply_async(lambda url:urlrequest(url)['entities'], (url,)) if 'RId' in json1: urlAttributes = 'Id,RId' if 'F' in json2: urlAttributes += ',F.FId' if 'J' in json2: urlAttributes += ',J.JId' if 'C' in json2: urlAttributes += ',C.CId' if 'AA' in json2: urlAttributes += ',AA.AuId' id1CitePapersInfo = getPaperJsonList(json1['RId'], urlAttributes) # =========== 1-hop =========== # Id-Id if 'RId' in json1 and id2 in json1['RId']: ans.append([id1, id2]) # =========== 2-hop =========== # Id-F.FId-Id if 'F' in json1: FIdList1 = map(lambda x:x['FId'], json1['F']) FIdList1.sort() if 'F' in json2: FIdList2 = map(lambda x:x['FId'], json2['F']) FIdList2.sort() if 'F' in json1 and 'F' in json2: jointFIdList = join(FIdList1, FIdList2) map(lambda x: ans.append([id1, x, id2]), jointFIdList) # Id-J.JId-Id if 'J' in json1 and 'J' in json2 and json1['J']['JId'] == json2['J']['JId']: ans.append([id1, json1['J']['JId'], id2]) # Id-C.CId-Id if 'C' in json1 and 'C' in json2 and json1['C']['CId'] == json2['C']['CId']: ans.append([id1, json1['C']['CId'], id2]) # Id-AA.AuId-Id if 'AA' in json1: AuIdList1 = map(lambda x:x['AuId'], json1['AA']) AuIdList1.sort() if 'AA' in json2: AuIdList2 = map(lambda x:x['AuId'], json2['AA']) AuIdList2.sort() if 'AA' in json1 and 'AA' in json2: jointAuIdList = join(AuIdList1, AuIdList2) map(lambda x: ans.append([id1, x, id2]), jointAuIdList) # Id-Id-Id Id2cited = Id2citedResult.get() if 'RId' in json1: RIdList = json1['RId'] Id2citedList = map(lambda x:x['Id'], Id2cited) RIdList.sort() Id2citedList.sort() jointRIdList = join(RIdList, Id2citedList) map(lambda x: ans.append([id1, x, id2]), jointRIdList) # =========== 3-hop =========== # Id-*-Id-Id if len(Id2cited) != 0: # Id-F.FId-Id-Id if 'F' in json1: for paper in Id2cited: if 'F' in paper: for F_element in paper['F']: if F_element['FId'] in FIdList1: ans.append([id1, F_element['FId'], paper['Id'], id2]) # Id-J.JId-Id-Id if 'J' in json1: for paper in Id2cited: if 'J' in paper and paper['J']['JId'] == json1['J']['JId']: ans.append([id1, json1['J']['JId'], paper['Id'], id2]) # Id-C.CId-Id-Id if 'C' in json1: for paper in Id2cited: if 'C' in paper and paper['C']['CId'] == json1['C']['CId']: ans.append([id1, json1['C']['CId'], paper['Id'], id2]) # Id-AA.AuId-Id-Id if 'AA' in json1: for paper in Id2cited: if 'AA' in paper: for AA_element in paper['AA']: if AA_element['AuId'] in AuIdList1: ans.append([id1, AA_element['AuId'], paper['Id'], id2]) # Id-Id-*-Id if 'RId' in json1: if 'F' in json2: for id1CitePaper in id1CitePapersInfo: if 'F' in id1CitePaper: FIdListTmp = map(lambda x:x['FId'], id1CitePaper['F']) FIdListTmp.sort() jointFIdList = join(FIdListTmp, FIdList2) map(lambda x: ans.append([id1, id1CitePaper['Id'], x, id2]), jointFIdList) # Id-Id-J.JId-Id if 'J' in json2: for id1CitePaper in id1CitePapersInfo: if 'J' in id1CitePaper and id1CitePaper['J']['JId'] == json2['J']['JId']: ans.append([id1, id1CitePaper['Id'], json2['J']['JId'], id2]) # Id-Id-C.CId-Id if 'C' in json2: for id1CitePaper in id1CitePapersInfo: if 'C' in id1CitePaper and id1CitePaper['C']['CId'] == json2['C']['CId']: ans.append([id1, id1CitePaper['Id'], json2['C']['CId'], id2]) # Id-Id-AA.AuId-Id if 'AA' in json2: for id1CitePaper in id1CitePapersInfo: if 'AA' in id1CitePaper: AuIdListTmp = map(lambda x:x['AuId'], id1CitePaper['AA']) AuIdListTmp.sort() jointAuIdList = join(AuIdListTmp, AuIdList2) map(lambda x: ans.append([id1, id1CitePaper['Id'], x, id2]), jointAuIdList) # Id-Id-Id-Id tmp = len(ans) for id1CitePaper in id1CitePapersInfo: if 'RId' in id1CitePaper: RIdListTmp = id1CitePaper['RId'] RIdListTmp.sort() jointRIdList = join(RIdListTmp, Id2citedList) map(lambda x: ans.append([id1, id1CitePaper['Id'], x, id2]), jointRIdList) return ans
schedule.every().thursday.at(firstClass).do(joinChem())
schedule.every().thursday.at(secondClass).do(join())
schedule.every().thursday.at(thirdClass).do(joinMaths())
schedule.every().thursday.at(fourthClass).do(joinSS())
schedule.every().thursday.at(fifthClass).do(joinEnglish())
def friday():
schedule.every().friday.at(firstClass).do(joinSS())
schedule.every().friday.at(secondClass).do(joinHindi())
schedule.every().friday.at(thirdClass).do(joinBio())
schedule.every().friday.at(fourthClass).do(joinEnglish())
schedule.every().friday.at(fifthClass).do(joinMaths())
def dumm():
schedule.every().saturday.at(firstClass).do(join())
schedule.every().saturday.at(secondClass).do(join())
schedule.every().saturday.at(thirdClass).do(joinMaths())
schedule.every().saturday.at(fourthClass).do(joinChem())
schedule.every().saturday.at(fifthclass).do(joinHindi())
while True:
schedule.run_pending()
time.sleep(1)#seconds
# Code for Timetable.py ends here
day = (e.strftime("%a"))
print(day)
if day=="Mon":
monday()
-from os import listdir
-from os.path import isfile, join
import importlib
from routersploit.core.exploit.printer import (
@@ -18,6 +16,7 @@ from routersploit.core.exploit.printer i
)
from routersploit.core.exploit.utils import (
+ index_modules,
random_text,
)
@@ -28,14 +27,11 @@ def shell(exploit, architecture="", meth
options = []
if architecture and method:
- path = "routersploit/modules/payloads/{}/".format(architecture)
-
# get all payloads for given architecture
- all_payloads = [f.split(".")[0] for f in listdir(path) if isfile(join(path, f)) and f.endswith(".py") and f != "__init__.py"]
+ all_payloads = [p.lstrip('payloads.').replace('.', '/') for p in index_modules() if "payloads.{}".format(architecture) in p]
- payload_path = path.replace("/", ".")
for p in all_payloads:
- module = getattr(importlib.import_module("{}{}".format(payload_path, p)), 'Payload')
+ module = getattr(importlib.import_module('routersploit.modules.payloads.' + p.replace('/', '.')), 'Payload')
# if method/arch is cmd then filter out payloads
if method is "cmd":
def query_Id_AuId(id1, auId2, json1, json2): print 'query_Id_AuId', id1, auId2 ans = [] now = time.time() paperIdList = map(lambda x:x['Id'], json2) paperIdList.sort() # =========== 1-hop =========== # Id-AuId if 'AA' in json1 and (auId2 in map(lambda x:x['AuId'], json1['AA'])): ans.append([id1, auId2]) # =========== 2-hop =========== # Id-Id-AuId if 'RId' in json1: RIdList = json1['RId'] RIdList.sort() jointRIdList = join(paperIdList, RIdList) map(lambda x: ans.append([id1, x, auId2]), jointRIdList) # =========== 3-hop =========== # Id-F.FId-Id-AuId if 'F' in json1: FIdList1 = map(lambda x:x['FId'], json1['F']) FIdList1.sort() for paper in json2: if 'F' in paper: FIdListTmp = map(lambda x:x['FId'], paper['F']) FIdListTmp.sort() jointFIdList = join(FIdListTmp, FIdList1) map(lambda x: ans.append([id1, x, paper['Id'], auId2]), jointFIdList) # Id-C.CId-Id-AuId if 'C' in json1: CId1 = json1['C']['CId'] for paper in json2: if 'C' in paper and paper['C']['CId'] == CId1: ans.append([id1, CId1, paper['Id'], auId2]) # Id-J.JId-Id-AuId if 'J' in json1: JId1 = json1['J']['JId'] for paper in json2: if 'J' in paper and paper['J']['JId'] == JId1: ans.append([id1, JId1, paper['Id'], auId2]) # Id-AA.AuId-Id-AuId if 'AA' in json1: AuIdList1 = map(lambda x:x['AuId'], json1['AA']) AuIdList1.sort() for paper in json2: if 'AA' in paper: AuIdListTmp = map(lambda x:x['AuId'], paper['AA']) AuIdListTmp.sort() jointAuIdList = join(AuIdListTmp, AuIdList1) map(lambda x: ans.append([id1, x, paper['Id'], auId2]), jointAuIdList) # Id-Id-Id-AuId if 'RId' in json1: citePaperInfoResults = pool.map_async(lambda x:getPaperJson(x, 'RId,Id'), RIdList) citePaperInfos = citePaperInfoResults.get() for citePaperInfo in citePaperInfos: if 'RId' in citePaperInfo: RIdListTmp = citePaperInfo['RId'] RIdListTmp.sort() jointRIdList = join(RIdListTmp, paperIdList) map(lambda x: ans.append([id1, citePaperInfo['Id'], x, auId2]), jointRIdList) # Id-AA.AuId-AA.AfId-AuId AFIdSet2 = set() for paper in json2: if 'AA' in paper: for author in paper['AA']: if author['AuId'] == auId2 and 'AfId' in author: AFIdSet2.add(author['AfId']) if len(AFIdSet2) > 0 and 'AA' in json1: authorPaperList = getAuthorPaperList(map(lambda x:x['AuId'], json1['AA']), 'AA.AuId,AA.AfId') authorSet1 = set() map(lambda x: authorSet1.add(x['AuId']), json1['AA']) uniqueSet = set() for paper in authorPaperList: if 'AA' in paper: for author in paper['AA']: auId1 = author['AuId'] if (auId1 in authorSet1) and 'AfId' in author and (author['AfId'] in AFIdSet2) and not((auId1, author['AfId']) in uniqueSet): ans.append([id1, auId1, author['AfId'], auId2]) uniqueSet.add((auId1, author['AfId'])) return ans
def query_AuId_Id(auId1, id2, json1, json2): print 'query_AuId_Id', auId1, id2 ans = [] now = time.time() url = 'https://oxfordhk.azure-api.net/academic/v1.0/evaluate?expr=RId=%d&count=100000&orderby=CC:desc&attributes=Id&subscription-key=f7cc29509a8443c5b3a5e56b0e38b5a6'%id2 Id2citedResult = pool.apply_async(lambda url:urlrequest(url)['entities'], (url,)) # =========== 1-hop =========== # AuId-Id for paper in json1: if paper['Id'] == id2: ans.append([auId1, id2]) break # =========== 2-hop =========== # AuId-Id-Id for paper in json1: if 'RId' in paper and (id2 in paper['RId']): ans.append([auId1, paper['Id'], id2]) # =========== 3-hop =========== # AuId-Id-F.FId-Id if 'F' in json2: FIdList2 = map(lambda x:x['FId'], json2['F']) FIdList2.sort() for paper in json1: if 'F' in paper: FIdListTmp = map(lambda x:x['FId'], paper['F']) FIdListTmp.sort() jointFIdList = join(FIdListTmp, FIdList2) map(lambda x: ans.append([auId1, paper['Id'], x, id2]), jointFIdList) # AuId-Id-C.CId-Id if 'C' in json2: CId2 = json2['C']['CId'] for paper in json1: if 'C' in paper and paper['C']['CId'] == CId2: ans.append([auId1, paper['Id'], CId2, id2]) # AuId-Id-J.JId-Id if 'J' in json2: JId2 = json2['J']['JId'] for paper in json1: if 'J' in paper and paper['J']['JId'] == JId2: ans.append([auId1, paper['Id'], JId2, id2]) # AuId-Id-AA.AuId-Id if 'AA' in json2: AuIdList2 = map(lambda x:x['AuId'], json2['AA']) AuIdList2.sort() for paper in json1: if 'AA' in paper: AuIdListTmp = map(lambda x:x['AuId'], paper['AA']) AuIdListTmp.sort() jointAuIdList = join(AuIdListTmp, AuIdList2) map(lambda x: ans.append([auId1, paper['Id'], x, id2]), jointAuIdList) print 'AuId-Id-AuId-Id finished time =', time.time() - now # AuId-Id-Id-Id Id2cited = Id2citedResult.get() print 'Id2cited geted time =', time.time() - now if len(Id2cited) > 0: Id2citedList = map(lambda x:x['Id'], Id2cited) Id2citedList.sort() for paper in json1: if 'RId' in paper: RIdListTmp = paper['RId'] RIdListTmp.sort() jointRIdList = join(RIdListTmp, Id2citedList) map(lambda x: ans.append([auId1, paper['Id'], x, id2]), jointRIdList) # AuId-AA.AFId-AA.AuId-Id AFIdSet1 = set() for paper in json1: if 'AA' in paper: for author in paper['AA']: if author['AuId'] == auId1 and 'AfId' in author: AFIdSet1.add(author['AfId']) if len(AFIdSet1) > 0 and 'AA' in json2: authorPaperList = getAuthorPaperList(map(lambda x:x['AuId'], json2['AA']), 'AA.AuId,AA.AfId') authorSet2 = set() map(lambda x:authorSet2.add(x['AuId']), json2['AA']) uniqueSet = set() for paper in authorPaperList: if 'AA' in paper: for author in paper['AA']: auId2 = author['AuId'] if (auId2 in authorSet2) and 'AfId' in author and (author['AfId'] in AFIdSet1) and not((author['AfId'], auId2) in uniqueSet): ans.append([auId1, author['AfId'], auId2, id2]) uniqueSet.add((author['AfId'], auId2)) print 'AuId-AfId-AuId-Id finished time =', time.time() - now return ans
def query_Id_AuId(id1, auId2, json1, json2):
#sys.stderr.write('query_AuId_Id ' + str(id1) + ' ' + str(auId2) + '\n')
print 'query_Id_AuId', id1, auId2
ans = []
#now = time.time()
#json1 = getPaperJson(id1, 'RId,F.FId,J.JId,C.CId,AA.AuId,AA.AfId')
#print 'time use: ', time.time() - now
#now = time.time()
#url = 'https://oxfordhk.azure-api.net/academic/v1.0/evaluate?expr=Composite(AA.AuId=%d)&count=20000&attributes=Id,F.FId,J.JId,C.CId,AA.AuId&orderby=D:asc&subscription-key=f7cc29509a8443c5b3a5e56b0e38b5a6'%auId2
#json2 = json.loads((urllib.urlopen(url)).read())['entities']
paperIdList = map(lambda x:x['Id'], json2)
paperIdList.sort()
# =========== 1-hop ===========
# Id-AuId
if json1.has_key('AA') and (auId2 in map(lambda x:x['AuId'], json1['AA'])):
ans.append([id1, auId2])
# =========== 2-hop ===========
# Id-Id-AuId
if json1.has_key('RId'):
RIdList = json1['RId']
RIdList.sort()
jointRIdList = join(paperIdList, RIdList)
map(lambda x: ans.append([id1, x, auId2]), jointRIdList)
# =========== 3-hop ===========
# Id-F.FId-Id-AuId
if json1.has_key('F'):
FIdList1 = map(lambda x:x['FId'], json1['F'])
FIdList1.sort()
for paper in json2:
if paper.has_key('F'):
FIdListTmp = map(lambda x:x['FId'], paper['F'])
FIdListTmp.sort()
jointFIdList = join(FIdListTmp, FIdList1)
map(lambda x: ans.append([id1, x, paper['Id'], auId2]), jointFIdList)
# Id-C.CId-Id-AuId
if json1.has_key('C'):
CId1 = json1['C']['CId']
for paper in json2:
if paper.has_key('C') and paper['C']['CId'] == CId1:
ans.append([id1, CId1, paper['Id'], auId2])
# Id-J.JId-Id-AuId
if json1.has_key('J'):
JId1 = json1['J']['JId']
for paper in json2:
if paper.has_key('J') and paper['J']['JId'] == JId1:
ans.append([id1, JId1, paper['Id'], auId2])
# Id-AA.AuId-Id-AuId
if json1.has_key('AA'):
AuIdList1 = map(lambda x:x['AuId'], json1['AA'])
AuIdList1.sort()
for paper in json2:
if paper.has_key('AA'):
AuIdListTmp = map(lambda x:x['AuId'], paper['AA'])
AuIdListTmp.sort()
jointAuIdList = join(AuIdListTmp, AuIdList1)
map(lambda x: ans.append([id1, x, paper['Id'], auId2]), jointAuIdList)
# Id-Id-Id-AuId
if json1.has_key('RId'):
citePaperInfoResults = pool.map_async(lambda x:getPaperJson(x, 'RId,Id'), RIdList)
#pool.close()
#pool.join()
citePaperInfos = citePaperInfoResults.get()
for citePaperInfo in citePaperInfos:
if citePaperInfo.has_key('RId'):
RIdListTmp = citePaperInfo['RId']
RIdListTmp.sort()
jointRIdList = join(RIdListTmp, paperIdList)
map(lambda x: ans.append([id1, citePaperInfo['Id'], x, auId2]), jointRIdList)
# Id-AA.AuId-AA.AfId-AuId
AFIdSet2 = set()
for paper in json2:
if paper.has_key('AA'):
for author in paper['AA']:
if author['AuId'] == auId2 and author.has_key('AfId'):
AFIdSet2.add(author['AfId'])
if len(AFIdSet2) > 0 and json1.has_key('AA'):
authorPaperList = getAuthorPaperList(map(lambda x:x['AuId'], json1['AA']), 'AA.AuId,AA.AfId')
authorSet1 = set()
map(lambda x: authorSet1.add(x['AuId']), json1['AA'])
uniqueSet = set()
for paper in authorPaperList:
if paper.has_key('AA'):
for author in paper['AA']:
auId1 = author['AuId']
if (auId1 in authorSet1) and author.has_key('AfId') and (author['AfId'] in AFIdSet2) and not((auId1, author['AfId']) in uniqueSet):
ans.append([id1, auId1, author['AfId'], auId2])
uniqueSet.add((auId1, author['AfId']))
#AFIdSet2 = set()
#for paper in json2:
# if paper.has_key('AA'):
# for author in paper['AA']:
# if author['AuId'] == auId2 and author.has_key('AfId'):
# AFIdSet2.add(author['AfId'])
#if json1.has_key('AA'):
# for author in json1['AA']:
# if author.has_key('AfId'):
# if author['AfId'] in AFIdSet2:
# ans.append([id1, author['AuId'], author['AfId'], auId2])
#print 'time use2: ', time.time() - now
return ans
def thursday():
schedule.every().thursday.at(firstClass).do(joinChem())
schedule.every().thursday.at(secondClass).do(join())
schedule.every().thursday.at(thirdClass).do(joinMaths())
schedule.every().thursday.at(fourthClass).do(joinSS())
schedule.every().thursday.at(fifthClass).do(joinEnglish())
def query_Id_Id_big(id1, id2, json1, json2): print 'query_Id_Id_big', id1, id2 ans = [] now = time.time() url = 'https://oxfordhk.azure-api.net/academic/v1.0/evaluate?expr=And(RId=%d,CC>0)&count=100000&orderby=CC:desc&attributes=Id,F.FId,J.JId,C.CId,AA.AuId&subscription-key=f7cc29509a8443c5b3a5e56b0e38b5a6'%id2 Id2citedResult = pool.apply_async(lambda url:urlrequest(url)['entities'], (url, )) if 'RId' in json1: urlAttributes = 'Id,RId' if 'F' in json2: urlAttributes += ',F.FId' if 'J' in json2: urlAttributes += ',J.JId' if 'C' in json2: urlAttributes += ',C.CId' if 'AA' in json2: urlAttributes += ',AA.AuId' id1CitePapersInfo = getPaperJsonList(json1['RId'], urlAttributes) # =========== 1-hop =========== # Id-Id if 'RId' in json1 and id2 in json1['RId']: ans.append([id1, id2]) # =========== 2-hop =========== # Id-F.FId-Id if 'F' in json1: FIdList1 = map(lambda x:x['FId'], json1['F']) FIdList1.sort() if 'F' in json2: FIdList2 = map(lambda x:x['FId'], json2['F']) FIdList2.sort() if 'F' in json1 and 'F' in json2: jointFIdList = join(FIdList1, FIdList2) map(lambda x: ans.append([id1, x, id2]), jointFIdList) # Id-J.JId-Id if 'J' in json1 and 'J' in json2 and json1['J']['JId'] == json2['J']['JId']: ans.append([id1, json1['J']['JId'], id2]) # Id-C.CId-Id if 'C' in json1 and 'C' in json2 and json1['C']['CId'] == json2['C']['CId']: ans.append([id1, json1['C']['CId'], id2]) # Id-AA.AuId-Id if 'AA' in json1: AuIdList1 = map(lambda x:x['AuId'], json1['AA']) AuIdList1.sort() if 'AA' in json2: AuIdList2 = map(lambda x:x['AuId'], json2['AA']) AuIdList2.sort() if 'AA' in json1 and 'AA' in json2: jointAuIdList = join(AuIdList1, AuIdList2) map(lambda x: ans.append([id1, x, id2]), jointAuIdList) # Id-Id-Id # TODO Id2cited = Id2citedResult.get() if 'RId' in json1: RIdList = json1['RId'] Id2citedList = map(lambda x:x['Id'], Id2cited) RIdList.sort() Id2citedList.sort() jointRIdList = join(RIdList, Id2citedList) map(lambda x: ans.append([id1, x, id2]), jointRIdList) # =========== 3-hop =========== # Id-*-Id-Id if len(Id2cited) != 0: now = time.time() poolResult = [] if 'F' in json1: for FId in FIdList1: url = 'https://oxfordhk.azure-api.net/academic/v1.0/evaluate?expr=And(RId=%d,Composite(F.FId=%d))&count=200000&attributes=Id&subscription-key=f7cc29509a8443c5b3a5e56b0e38b5a6'%(id2, FId) poolResult.append(pool.apply_async(lambda url:urlrequest(url)['entities'], (url, ))) if 'J' in json1: url = 'https://oxfordhk.azure-api.net/academic/v1.0/evaluate?expr=And(RId=%d,Composite(J.JId=%d))&count=200000&attributes=Id&subscription-key=f7cc29509a8443c5b3a5e56b0e38b5a6'%(id2, json1['J']['JId']) poolResult.append(pool.apply_async(lambda url:urlrequest(url)['entities'], (url, ))) if 'C' in json1: url = 'https://oxfordhk.azure-api.net/academic/v1.0/evaluate?expr=And(RId=%d,Composite(C.CId=%d))&count=200000&attributes=Id&subscription-key=f7cc29509a8443c5b3a5e56b0e38b5a6'%(id2, json1['C']['CId']) poolResult.append(pool.apply_async(lambda url:urlrequest(url)['entities'], (url, ))) if 'AA' in json1: for AuId in AuIdList1: url = 'https://oxfordhk.azure-api.net/academic/v1.0/evaluate?expr=And(RId=%d,Composite(AA.AuId=%d))&count=200000&attributes=Id&subscription-key=f7cc29509a8443c5b3a5e56b0e38b5a6'%(id2, AuId) poolResult.append(pool.apply_async(lambda url:urlrequest(url)['entities'], (url, ))) idx = 0 # Id-F.FId-Id-Id if 'F' in json1: for FId in FIdList1: comPaperList = poolResult[idx].get() idx += 1 map(lambda x: ans.append([id1, FId, x['Id'], id2]), comPaperList) # Id-J.JId-Id-Id if 'J' in json1: JId = json1['J']['JId'] comPaperList = poolResult[idx].get() idx += 1 map(lambda x: ans.append([id1, JId, x['Id'], id2]), comPaperList) # Id-C.CId-Id-Id if 'C' in json1: CId = json1['C']['CId'] comPaperList = poolResult[idx].get() idx += 1 map(lambda x: ans.append([id1, CId, x['Id'], id2]), comPaperList) # Id-AA.AuId-Id-Id if 'AA' in json1: for AuId in AuIdList1: comPaperList = poolResult[idx].get() idx += 1 map(lambda x: ans.append([id1, AuId, x['Id'], id2]), comPaperList) # Id-Id-*-Id if 'RId' in json1: # Id-Id-F.FId-Id if 'F' in json2: for id1CitePaper in id1CitePapersInfo: if 'F' in id1CitePaper: FIdListTmp = map(lambda x:x['FId'], id1CitePaper['F']) FIdListTmp.sort() jointFIdList = join(FIdListTmp, FIdList2) map(lambda x: ans.append([id1, id1CitePaper['Id'], x, id2]), jointFIdList) # Id-Id-J.JId-Id if 'J' in json2: for id1CitePaper in id1CitePapersInfo: if 'J' in id1CitePaper and id1CitePaper['J']['JId'] == json2['J']['JId']: ans.append([id1, id1CitePaper['Id'], json2['J']['JId'], id2]) # Id-Id-C.CId-Id if 'C' in json2: for id1CitePaper in id1CitePapersInfo: if 'C' in id1CitePaper and id1CitePaper['C']['CId'] == json2['C']['CId']: ans.append([id1, id1CitePaper['Id'], json2['C']['CId'], id2]) # Id-Id-AA.AuId-Id if 'AA' in json2: for id1CitePaper in id1CitePapersInfo: if 'AA' in id1CitePaper: AuIdListTmp = map(lambda x:x['AuId'], id1CitePaper['AA']) AuIdListTmp.sort() jointAuIdList = join(AuIdListTmp, AuIdList2) map(lambda x: ans.append([id1, id1CitePaper['Id'], x, id2]), jointAuIdList) # Id-Id-Id-Id tmp = len(ans) for id1CitePaper in id1CitePapersInfo: if 'RId' in id1CitePaper: RIdListTmp = id1CitePaper['RId'] RIdListTmp.sort() jointRIdList = join(RIdListTmp, Id2citedList) map(lambda x: ans.append([id1, id1CitePaper['Id'], x, id2]), jointRIdList) return ans