def setUp(self):
self.h1, self.h2, self.h3, self.h4 = XmlHandler(), TxtHandler(), JsonHandler(), CsvHandler()
self.h1.set_next(self.h2)
self.h2.set_next(self.h3)
self.h3.set_next(self.h4)
self.files_dict = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + '/tests/test_data/'
self.output_file = self.files_dict + 'result.res'
open(self.output_file, 'w').close()
def dmirt(self, cell_lines=['GM12878', 'K562']):
from histogram_cl import histogram_gpu
fpath = os.path.join(self.root, 'prediction', 'pred_tss_20_cnn.db')
con = sqlite3.connect(fpath)
for cell_line in cell_lines:
print(cell_line)
df_ref = pd.read_sql("SELECT miRNA, chromosome, start, end, strand FROM "
"(SELECT * FROM 'upstream_pred_{}')".format(cell_line), con)
df_ref[['start', 'end']] = df_ref[['start', 'end']].astype(int)
tss = df_ref[['start', 'end']].mean(axis=1).astype(int)
df_ref['start'] = tss - 500
df_ref['end'] = tss + 500
gro_path = os.path.join(self.root, 'database', 'GRO_cap.db')
hgpu = histogram_gpu(XmlHandler.load_param("user_param.xml"))
df = hgpu.run(df_ref[['chromosome', 'start', 'end', 'strand']], gro_path, '_'.join([cell_line, 'hg19']))
df['miRNA'] = df_ref['miRNA']
nidx = df[df['strand'] == '-'].index
hist_col = list(range(50))
contents = df.loc[nidx, hist_col[::-1]]
contents.columns = hist_col
df.loc[nidx, hist_col] = contents
dirname = os.path.join(self.root, 'evaluation')
if not os.path.exists(dirname):
os.mkdir(dirname)
out_con = sqlite3.connect(os.path.join(dirname, 'eval_peaks.db'))
df.to_sql('DMIRT_GRO_{}'.format(cell_line), out_con, index=None, if_exists='replace')
def cage_tag(self, paper='DMIRT', cell_lines=['GM12878', 'K562']):
from histogram_cl import histogram_gpu
for cell_line in cell_lines:
if paper == 'DMIRT':
fpath = os.path.join(self.root, 'prediction', 'pred_tss_20_cnn.db')
con = sqlite3.connect(fpath)
df_ref = pd.read_sql("SELECT miRNA, chromosome, start, end, strand FROM "
"(SELECT * FROM 'upstream_pred_{}')".format(cell_line), con)
df_ref[['start', 'end']] = df_ref[['start', 'end']].astype(int)
tss = df_ref[['start', 'end']].mean(axis=1).astype(int)
mir_label = 'miRNA'
elif paper == 'HUA':
fpath = os.path.join(self.root, 'database', 'Supplementary file4-alternative_TSS.db')
con = sqlite3.connect(fpath)
df_ref = pd.read_sql("SELECT * FROM 'cell_specific' WHERE cell_lines LIKE '%{}%'".format(cell_line), con)
tss = df_ref['tss'].astype(int)
mir_label = '#MIR'
elif paper == 'PRO':
fpath = os.path.join(self.root, 'database', 'PRO.db')
con = sqlite3.connect(fpath)
df_ref = pd.read_sql("SELECT * FROM '{}'".format(cell_line), con)
tss = df_ref[['tss_start', 'tss_stop']].mean(axis=1).astype(int)
mir_label = 'miRNA'
else:
fpath = os.path.join(self.root, 'prediction', 'pred_tss_20_cnn.db')
con = sqlite3.connect(fpath)
df_ref = pd.read_sql("SELECT miRNA, chromosome, start, end, strand FROM "
"(SELECT * FROM 'upstream_pred_{}')".format(cell_line), con)
df_ref[['start', 'end']] = df_ref[['start', 'end']].astype(int)
tss = df_ref[['start', 'end']].mean(axis=1).astype(int)
mir_label = 'miRNA'
df_ref['start'] = tss - 500
df_ref['end'] = tss + 500
tss = df_ref[['start', 'end']].mean(axis=1).astype(int)
df_ref['start'] = tss - 500
df_ref['end'] = tss + 500
hgpu = histogram_gpu(XmlHandler.load_param("user_param.xml"))
fpath = os.path.join(self.root, "database", "hCAGE_ctss.db")
rsc_tname = '{}_hg19_ctss'.format(cell_line)
df = hgpu.run(df_ref[['chromosome', 'start', 'end', 'strand']], fpath, rsc_tname)
if mir_label:
df['miRNA'] = df_ref[mir_label]
nidx = df[df['strand'] == '-'].index
hist_col = list(range(50))
contents = df.loc[nidx, hist_col[::-1]]
contents.columns = hist_col
df.loc[nidx, hist_col] = contents
out_con = sqlite3.connect(os.path.join(self.root, 'evaluation', 'eval_peaks.db'))
df.to_sql('{}_CAGE_{}'.format(paper, cell_line), out_con, index=None, if_exists='replace')
def processsXml(self, xml):
myData=''
obj=XmlHandler().XmlStringToObj(xml, keyword='events')
if obj:
myData=geoLocationData()
attrs=classUtility.getAttrList(myData)
for attr in obj[0].attributes:
if attr.name in attrs:
setattr(myData, attr.name, attr.value)
return myData
class MainTests(unittest.TestCase):
def setUp(self):
self.h1, self.h2, self.h3, self.h4 = XmlHandler(), TxtHandler(), JsonHandler(), CsvHandler()
self.h1.set_next(self.h2)
self.h2.set_next(self.h3)
self.h3.set_next(self.h4)
self.files_dict = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + '/tests/test_data/'
self.output_file = self.files_dict + 'result.res'
open(self.output_file, 'w').close()
def test_xml(self):
file_name = self.files_dict + 'Test.xml'
self.h1.handle(file_name, self.output_file)
with open(self.output_file, 'r') as f:
content = f.readline()
self.assertTrue('XmlHandler' in content)
def test_json(self):
file_name = self.files_dict + 'Test.json'
self.h1.handle(file_name, self.output_file)
with open(self.output_file, 'r') as f:
content = f.readline()
self.assertTrue('JsonHandler' in content)
def test_txt(self):
file_name = self.files_dict + 'Test.txt'
self.h1.handle(file_name, self.output_file)
with open(self.output_file, 'r') as f:
content = f.readline()
self.assertTrue('TxtHandler' in content)
def test_csv(self):
file_name = self.files_dict + 'Test.csv'
self.h1.handle(file_name, self.output_file)
with open(self.output_file, 'r') as f:
content = f.readline()
self.assertTrue('CsvHandler' in content)
def getModulePreData(self, module):
myPath=upgrade_path % (module+'-'+self.server.replace('.', '_')+'.xml')
if module in obj_name_trans.keys():
fiMod=obj_name_trans[module]
else:
fiMod=module
if fiMod in obj_xml_wrap.keys():
fiKey=obj_xml_wrap[fiMod]
else:
fiKey=generalUtility.getPlural(fiMod)
oriData=XmlHandler().XmlFileToObj(myPath, keyword=fiKey)
indexData={}
for ori in oriData:
if classUtility.getType(ori)!='NoneType':
indexData[ori.attribute['naturalId']]=ori
return indexData
def run(self):
import matplotlib.pyplot as plt
dirname__ = os.path.join(self.root, 'database/Histone ChIP-seq')
filenames = XmlHandler.load_param('histogram_param.xml')
N = len(self.histones)
hgpu = histogram_gpu(fpath='histogram_param.xml')
for tissue, fnames in filenames.items():
df_ref = self.load_ref(tissue)
if df_ref is None:
continue
dfs = {}
for histone, fname in fnames.items():
rsc_path = os.path.join(dirname__, tissue, 'histone.db')
hist_out = self.get_hist(df_ref, rsc_path, fname)
print(hist_out)
def hua(self, cell_lines=['GM12878', 'K562']):
from histogram_cl import histogram_gpu
fpath = os.path.join(self.root, 'database', 'Supplementary file4-alternative_TSS.db')
con = sqlite3.connect(fpath)
for cell_line in cell_lines:
df_ref = pd.read_sql("SELECT * FROM 'cell_specific' WHERE cell_lines LIKE '%{}%'".format(cell_line), con)
df_ref['start'] = df_ref['tss'].astype(int) - 500
df_ref['end'] = df_ref['tss'].astype(int) + 500
gro_path = os.path.join(self.root, 'database', 'GRO_cap.db')
hgpu = histogram_gpu(XmlHandler.load_param("user_param.xml"))
df = hgpu.run(df_ref, gro_path, '_'.join([cell_line, 'hg19']))
dirname = os.path.join(self.root, 'evaluation')
if not os.path.exists(dirname):
os.mkdir(dirname)
out_con = sqlite3.connect(os.path.join(dirname, 'eval_peaks.db'))
df.to_sql('HUA_GRO_{}'.format(cell_line), out_con, if_exists='replace', index=None)
def pro(self, cell_lines=['GM12878', 'K562']):
from histogram_cl import histogram_gpu
fpath = os.path.join(self.root, 'database', 'PRO.db')
con = sqlite3.connect(fpath)
for cell_line in cell_lines:
df_ref = pd.read_sql("SELECT * FROM '{}'".format(cell_line), con)
df_ref['tss'] = df_ref[['tss_start', 'tss_stop']].mean(axis=1).astype(int)
df_ref['start'] = df_ref['tss'].astype(int) - 500
df_ref['end'] = df_ref['tss'].astype(int) + 500
gro_path = os.path.join(self.root, 'database', 'GRO_cap.db')
hgpu = histogram_gpu(XmlHandler.load_param("user_param.xml"))
df = hgpu.run(df_ref, gro_path, '_'.join([cell_line, 'hg19']))
dirname = os.path.join(self.root, 'evaluation')
if not os.path.exists(dirname):
os.mkdir(dirname)
out_con = sqlite3.connect(os.path.join(dirname, 'eval_peaks.db'))
df.to_sql('PRO_GRO_{}'.format(cell_line), out_con, if_exists='replace', index=None)
def getQuery(self, params):
"""This method will get indexed dictinary of query objects."""
indexData = {}
inXml = self.createQueryXml(params)
self.appHandler.getQuery(inXml)
debugInfo = self.appHandler.queryDebug
if self.appHandler.queryResult:
for data in self.appHandler.queryXml:
objList = XmlHandler().XmlStringToObj(data, keyword='events')
if objList:
for subData in objList:
attrList = subData.attributes
attrMap = {}
for attr in subData.attributes:
attrMap[attr.name] = attr.value
delattr(subData, 'attributes')
setattr(subData, 'attributes', attrMap)
indexValue = classUtility.getIndexValue(
subData, obj_index['event'])
if subData.attributes.has_key('incidentRptIp'):
addKey = subData.attributes['incidentRptIp']
else:
addKey = subData.attributes['reptDevIpAddr']
if indexValue is not None:
if addKey is not None:
indexValue += '@' + addKey
else:
indexValue = 'No EventType(not unknown_type)'
if indexValue in indexData.keys():
oldValue = indexData[indexValue]
oldValue.append(subData)
indexData[indexValue] = oldValue
else:
newValue = []
newValue.append(subData)
indexData[indexValue] = newValue
return indexData, debugInfo
class restApiDataHandler:
"""This class will handle Accelops REST API mechnism.
It will start an appServer class instance to get XML data, then return an indexed
object."""
def __init__(self, appServer, user=False, password=False):
self.appServer = appServer
self.appHandler = appHandler(appServer, user=user, password=password)
self.xmlHandler = XmlHandler()
def getData(self,
datatype,
index=True,
pickle=True,
noKey=False,
module='config',
arg={}):
"""This method will call GET to return a dictinary of indexed objects."""
data = ''
if pickle:
data = self.__getData(datatype, module, noKey=noKey, args=arg)
else:
data = self.__getData(datatype, module, pickle=False, args=arg)
if data:
if module == 'namedValue':
datatype = 'entityValue'
if type(data) == types.ListType:
if pickle and index:
indexData = {}
if datatype in TestConstant.obj_name_trans.keys():
fiType = TestConstant.obj_name_trans[datatype]
else:
fiType = datatype
keyItem = testUtility.getKey(fiType)
for item in data:
if item is not None:
if 'Attribute-' in keyItem:
key = item.attribute[keyItem.split('-')[-1]]
elif classUtility.getType(keyItem) == 'list':
values = []
for subKey in keyItem:
subVal = getattr(item, subKey)
if subVal == None:
subVal = ''
values.append(subVal)
key = '@'.join(values)
else:
key = getattr(item, keyItem)
if key not in indexData.keys():
indexData[key] = item
else:
oldData = indexData[key]
Li = []
if classUtility.getType(oldData) == 'list':
Li = oldData
Li.append(item)
else:
Li.append(oldData)
Li.append(item)
indexData[key] = Li
if datatype in TestConstant.rest_special_handling:
finalData = self.__specialHandling(datatype, indexData)
return finalData
else:
return indexData
else:
return data
else:
return data
else:
return data
def __indexVulnerability(self, data):
indexData = {}
affectSwData = self.getData('affectedSoftware')
deviceTypeData = self.getData('deviceType')
idIndexDevType = testUtility.rekeyKey('Attribute-id', deviceTypeData)
for key in data.keys():
asList = []
for sw in affectSwData.keys():
vi = affectSwData[sw].vulnerability.split('@')[-1]
if vi == data[key].attribute['id']:
di = affectSwData[sw].osDeviceType.split('@')[-1]
if di and di in idIndexDevType.keys():
affectSwData[sw].osDeviceType = idIndexDevType[di]
asList.append(affectSwData[sw])
data[key].affectedSoftwares = asList
indexData[key] = data[key]
return indexData
def __indexRule(self, data):
indexData = {}
clearConditions = self.getData('clearCondition')
if clearConditions is not None:
for key in data.keys():
if data[key].attribute['naturalId'] in clearConditions.keys():
setattr(data[key], 'clearCondition',
clearConditions[data[key].attribute['naturalId']])
indexData[key] = data[key]
else:
indexData = data
return indexData
def __indexDomain(self, data):
indexData = {}
collectors = self.getData('eventCollector')
if collectors:
for subkey in data:
if data[subkey].initialized == 'true':
if hasattr(data[subkey], 'collectors'):
data[subkey].collectors = []
for morekey in collectors:
if data[subkey].domainId == collectors[
morekey].attribute['custId']:
data[subkey].collectors.append(
collectors[morekey])
indexData[data[subkey].name] = data[subkey]
else:
indexData = data
for key in indexData.keys():
if key in ['Super', 'service', 'system']:
del indexData[key]
return indexData
def __indexClearCondition(self, data):
indexData = {}
clearEventFilters = self.getData('eventFilter')
for key in data.keys():
mydata = data[key]
if type(mydata.clearEventFilters) != types.NoneType:
filterId = mydata.clearEventFilters.split('@')[-1].strip()
if filterId in clearEventFilters.keys():
setattr(mydata, 'clearEventFilters',
clearEventFilters[filterId])
indexData[key] = mydata
return indexData
tokenDict = {
'vulnerability': __indexVulnerability,
'rule': __indexRule,
'domain': __indexDomain,
'clearCondition': __indexClearCondition,
}
def __specialHandling(self, datatype, data):
if datatype in self.tokenDict.keys():
return self.tokenDict[datatype](self, data)
else:
print '%s: this type is not supported.' % datatype
return {}
def __indexDeviceEventAttribute(self, data):
indexData = {}
deviceTypes = self.getData('deviceType')
for item in data:
deviceId = item.deviceType.split('@')[-1]
for key in deviceTypes.keys():
if deviceId == deviceTypes[key].attribute['id']:
item.deviceType = deviceTypes[key]
item.eventType = item.eventType.split('$')[-1]
indexData[item.eventType] = item
return indexData
def __indexEventCode(self, data):
indexData = {}
keys = CSVHandler.getKeys('eventCode')
for item in data:
values = []
for key in keys:
myVal = getattr(item, key)
if 'EventAttributeType$' in myVal:
values.append(myVal.split('$')[-1])
else:
values.append(myVal)
value = '-'.join(values)
indexData[value] = item
return indexData
def __indexEventParser(self, data):
indexData = {}
parsers = self.__getData('parsers', module='device')
indexParser = {}
for par in parsers:
indexParser[par.name] = par
for item in data:
item.priority = indexParser[item.attribute['name']].priority
indexData[item.attribute['name']] = item
return indexData
def __indexGroup(self, data):
indexData = {}
for item in data:
if item.attribute['type'] in indexData.keys():
map = indexData[item.attribute['type']]
else:
map = {}
if item.parent:
item.parent = item.parent.split('$')[-1]
map[item.attribute['name']] = item
indexData[item.attribute['type']] = map
return indexData
def __indexNotification(self, data):
actions = self.__getData('notificationAction')
conditions = self.__getData('notificationCondition')
indexData = {}
indexData['All'] = []
for item in data:
item.actions = []
for action in actions:
if action.policy.split('@')[-1] == item.attribute['id']:
item.actions.append(action)
item.conditions = []
for condition in conditions:
if condition.policy.split('@')[-1] == item.attribute['id']:
item.conditions.append(condition)
item.attribute['naturalId'] = condition.name
if not len(item.attribute):
indexData['All'].append(item)
else:
indexData[item.attribute['naturalId']] = item
return indexData
def __getData(self, name, module, noKey=False, pickle=True, args={}):
if module == 'namedValue':
myParam = module + '/group?name=' + name
pickname = 'entityValue'
else:
myParam = module + '/' + name
pickname = name
if args:
argStr = ''
argList = []
for key in args.keys():
argList.append(key + '=' + args[key])
argStr = '?' + '&'.join(argList)
myParam += argStr
self.appHandler.getData(myParam)
data = ''
if self.appHandler.xml:
if pickle:
if pickname in TestConstant.obj_name_trans.keys():
keyWord = TestConstant.obj_name_trans[pickname]
else:
keyWord = pickname
if keyWord in TestConstant.obj_xml_wrap.keys():
pickKey = TestConstant.obj_xml_wrap[keyWord]
else:
if noKey:
pickKey = None
else:
pickKey = generalUtility.getPlural(keyWord)
data = self.xmlHandler.XmlStringToObj(self.appHandler.xml,
keyword=pickKey)
else:
data = self.appHandler.xml
return data
def __init__(self, appServer, user=False, password=False):
self.appServer = appServer
self.appHandler = appHandler(appServer, user=user, password=password)
self.xmlHandler = XmlHandler()