def main(option):
if 'i' in option:
temp = Collector.main(option)
journal = temp[0]
vh = temp[1]
allocationFile = temp[2]
extentsFile = temp[3]
catalogFile = temp[4]
attributesFile = temp[5]
else:
f = open(option['j'], 'rb')
journal = f.read()
f.close()
files = []
for i in ['v', 'al', 'e', 'c', 'at']:
if i in option:
f = open(option[i], 'rb')
files.append(f.read())
f.close()
else:
files.append(0)
vh = files[0]
allocationFile = files[1]
extentsFile = files[2]
catalogFile = files[3]
attributesFile = files[4]
print 'Analyzing journal...'
jParseList = journalParser(journal)[0]
#jT = journalTrack(jParseList)
path = 'result{0}'.format(option['id'])
DirectoryCleaning(path)
specialFileAnalyzer(
path, {
'Extents': extentsFile,
'Catalog': catalogFile,
'Attributes': attributesFile
})
rawCSV(path, jParseList)
rawSQLite3(path, jParseList)
if 'r' in option:
recovery(option['l'], path, option['r'], jParseList, vh)
from BloomFilter import *
from rrm import *
from Collector import *
import scipy.stats as stats
import numpy as np
import pylab as plt
p=1/6
q=5/6
f=0.25
MyCollector=Collector(f,p,q)
#print(bf)
x=np.arange(10,30,0.2)
y=np.zeros(100)
for i in range(100):
y[i]=1000
z=np.zeros(100)
for index in range(100):
for i in range(1000):
data=BloomFilter(index,5)
bf=data.getBloomFilter()
response=RRM(bf,f,p,q)
pbf=response.randomer()
MyCollector.receiver(pbf)
#print(pbf)
result=MyCollector.getResult()
#print(result)
for i in range(0,100):
def run(self):
print "Iniciando thread %s " % (self.name)
Collector.MainBoot(self.nome, self.id)
print "Finalizando " + self.nome
import Collector
Collector.start_function('kitten/index_local.html')
from BloomFilter import *
from rrm import *
from Collector import *
import scipy.stats as stats
import numpy as np
import pylab as plt
MyCollector=Collector(0.5,0.5)
#print(bf)
'''
mu=50
sigma=10
x=np.arange(0,101,1)
y=stats.norm.pdf(x,50,10)
'''
x=np.arange(10,30,0.2)
z=np.zeros(100)
y=np.zeros(100)
for i in np.arange(0,100):
y[i]=1000
for index in range(100):
for i in range(int(y[index])):
data=BloomFilter(index,5)
bf=data.getBloomFilter()
start=data.getSecurityDomain1()
end=data.getSecurityDomain2()
response=RRM(bf,0.5,0.5,start,end)
def main():
Utility.DirectoryCleaning('transaction')
Utility.DirectoryCleaning('result')
if raw_input('Is disk connected? [y/n]: ').lower()=='y':
disk=Collector.VolumeFinder()
select = (raw_input('Would you like to create image? [y/n] : ').lower()=='y')
recovery = (raw_input('Would you like to recover the image? [y/n] : ').lower()=='y')
print 'Collecting files...'
volumeHeader=VolumeHeader(Utility.DiskDump(disk,'HFSPlusVolumeHeader',512,2,1,select))
journal=Collector.JournalExtractor(self.journalInfoBlock,disk,volumeHeader.blockSize,select)
allocationFile=Collector.SpecialFileExtractor('AllocationFile',volumeHeader.specialFileFork[0],disk,volumeHeader.blockSize,select)
extentsOverflowFile=Collector.SpecialFileExtractor('ExtentsOverflowFile',volumeHeader.specialFileFork[1],disk,volumeHeader.blockSize,select)
catalogFile=Collector.SpecialFileExtractor('CatalogFile',volumeHeader.specialFileFork[2],disk,volumeHeader.blockSize,select)
else:
path=raw_input('Input path of volume header\n')
f=open(path,'rb')
volumeHeader=VolumeHeader(f.read())
f.close()
path=raw_input('Input path of journal\n')
f=open(path,'rb')
journal=f.read()
f.close()
if raw_input('Input path of catalog file(y/n)\n') == 'y':
path=input()
f=open(path,'rb')
catalogFile=f.read()
f.close()
"""
if raw_input('Input path of allocation file(y/n)') == 'y':
path=raw_input()
f=open(path,'rb')
AllocationFile=f.read()
f.close()
if raw_input('Input path of extents overflow file(y/n)\n') == 'y':
path=raw_input()
f=open(path,'rb')
ExtentsOverflowFile=f.read()
f.close()
"""
recovery=False
print 'Analyzing journal...'
tranBlocks=Analyzer.journalParser(journal)
print 'Printing transactions...'
for i in range(1,tranBlocks[0]+1):
fName='./transaction/transaction{0}_block{1}(journaloffset:{2}_location:{3})'.format(i,j,tranBlocks[i].journalOffset,tranBlocks[i].offset)
f=open(fName,'wb')
f.write(tranBlocks[i].content)
f.close()
print 'Analyzing CatalogFile...'
Analyzer.CatalogFileAnalyzer(CatalogFile)
print 'Analyzing transactions...'
Analyzer.TransactionAnalyzer(tranBlocks,volumeHeader.specialFileFork,volumeHeader.blockSize)
print 'Printing record list...'
fName='./result/recordList.csv'
f=open(fName,'w')
for i in range(1,tranBlocks[0]+1):
if 0<tranBlocks.blockType[0]<4:
for j in range(tranBlocks.content.numRecords):
f.write(str(tranBlocks[i]))
f.write(str(tranBlocks.content.records[j]))
else:
f.write(str(tranBlocks[i]))
f.close()
'''
from BloomFilter import *
from rrm import *
from Collector import *
import scipy.stats as stats
import numpy as np
import pylab as plt
MyCollector = Collector(0.5, 0.5)
#print(bf)
'''
mu=50
sigma=10
x=np.arange(0,101,1)
y=stats.norm.pdf(x,50,10)
'''
lambd = 0.05
x = np.arange(0, 101, 1)
z = np.zeros(101)
y = lambd * np.exp(-lambd * x)
for i in np.arange(0, 101):
y[i] = y[i] * 100000
y[i] = int(y[i])
for index in range(101):
for i in range(int(y[index])):
data = BloomFilter(index, 5)
bf = data.getBloomFilter()
start = data.getSecurityDomain1()
end = data.getSecurityDomain2()
response = RRM(bf, 0.5, 0.5, 0, 127)
pbf = response.randomer()
def run(self):
Collector.CollectNetflow()
def collect_data(organism1, organism2, option, update):
print 'Transferring protein interaction annotations, %s interfaces:\n> ' \
'%s to %s' % (option.upper(), organism1.info, organism2.info)
collector = Collector(organism1, organism2, option)
return collector.run(update=update)
def main():
while True:
logger = None
try:
collector = Collector.Collector(config)
logger = collector.get_logger()
db = collector.get_db()
if not args.skip_weather:
print '\nInitialising import of traffic and weather data...\n'
print '\nRetrieving weather data...'
#todo make option to manually specify cities to process? (maybe)
manual_cities = []
weather_api_client = WeatherAPI.Client(
config['weather']['api']['token'],
config['weather']['api']['endpoint'], logger)
# set the rate limit to 2 seconds per request
weather_api_client.set_rate_limit(2)
weather_collector = WeatherCollect.Collect(
config, weather_api_client, db)
cities = config['weather']['api']['cities']
#if there is a list of cities passed, process them, otherwise use the configured list
if len(manual_cities) > 0:
weather_collector.set_cities(manual_cities)
else:
weather_collector.set_cities(cities)
logger.info(
'Processing the following cities for weather data: %s',
", ".join(cities))
# start collecting the weather data
stats = weather_collector.collect()
# log information regarding collection
logger.info(
'Collected statistics for %d cities successfully, %d failed.'
% (stats['success'], stats['failed']))
else:
print '\nSkipping weather collection...'
if not args.skip_traffic:
logger.info('Starting collection of traffic data')
print '\nRetrieving traffic data...'
traffic_api_client = TrafficAPI.Client(config['traffic'],
logger)
traffic_collector = TrafficCollect.Collect(
config, traffic_api_client, db)
if len(config['traffic']['regions']) > 0:
# add the regions to the collector
for region in config['traffic']['regions']:
region_name = region.keys()[0]
traffic_collector.add_region(region_name, region)
# start collecting the data
stats = traffic_collector.collect()
# log information regarding collection
logger.info(
'Collection for %d regions successfully, %d failed.' %
(stats['success'], stats['failed']))
else:
print '\nNo regions configured, not processing traffic information'
else:
print '\nSkipping traffic collection...'
except Exception as exc:
if logger is not None:
logger.error(exc.message)
raise exc
print '\n Resuming collecion task in %d seconds.' % args.interval
# rest until the interval timer has expired
time.sleep(args.interval)
metavar='N',
type=int,
help="Collect data every N seconds")
p.add_argument('--skip-traffic',
'-s',
action="store_true",
help="skip traffic collection")
p.add_argument('--skip-weather',
'-w',
action="store_true",
help="skip weather collection")
args = p.parse_args()
print args
# get the main configuration
config = Collector.get_config('main.conf')
if not os.access('/var/log/collect', os.W_OK):
print 'Cannot write to /var/log/collect, make sure that you run this script elevated'
exit(1)
def main():
while True:
logger = None
try:
collector = Collector.Collector(config)
logger = collector.get_logger()
import Collector
import json
hostname = 'www2.pref.okinawa.jp'
okinawa = Collector.Collector(
hostname,
'/oki/Gikairep1.nsf/481e05e7edaca1db49256f540004c033/7fb73c2cf7988bc8492579e30024d9b5?OpenDocument'
)
#okinawa = Collector.Collector(hostname,'/oki/Gikairep1.nsf/481e05e7edaca1db49256f540004c033?OpenView')
logs = []
nextpath = ''
while True:
content = okinawa.get(nextpath)
nextpath = content['next']
print(content)
logs.append(content)
if nextpath == '':
break
json.dump(logs, open('output.json', 'w'))
def main():
COL = col.Collector()
tfServ = tenUtil.TensorService(COL, IMG_SIZE)
vectorsAndLables = readFileToNpArray(FILE_PATH)
tfServ.train_neural_network(vectorsAndLables)
pass
p = argparse.ArgumentParser(description='Trafeo collector agent')
p.add_argument('--verbose', '-v', default=None, help="Run in verbose mode")
# todo imlement at some point?
#p.add_argument('--retry', '-r', default=2, metavar='N', type=int, help="If any API fails, retry N times")
p.add_argument('--interval', '-i', default=900, metavar='N', type=int, help="Collect data every N seconds")
p.add_argument('--skip-traffic', '-s', action="store_true", help="skip traffic collection")
p.add_argument('--skip-weather', '-w', action="store_true", help="skip weather collection")
args = p.parse_args()
print args
# get the main configuration
config = Collector.get_config('main.conf')
if not os.access('/var/log/collect', os.W_OK):
print 'Cannot write to /var/log/collect, make sure that you run this script elevated'
exit(1)
def main():
while True:
logger = None
try:
collector = Collector.Collector(config)
logger = collector.get_logger()
db = collector.get_db()
import Collector
# serialNumnCollector = Collector.SerialNumberCollector()
# serialset = serialNumnCollector.findOuter()
# print('已完成全部serialNumber的爬蟲')
# for n in serialset:
# if len(n) < 5:
# continue
# goodsInfoCollector = Collector.GoodsInfoCollector(n)
# goodsInfoCollector.search()
goodsInfoCollector = Collector.GoodsInfoCollector('433245')
goodsInfoCollector.search()
e.append(math.log((d * d - 4 * d + 4) / (d * d), math.e))
print(e)
N = 100000
fo = open('data5.txt', 'a+')
mu = 50
sigma = 10
x = np.arange(0, 101, 1)
y = stats.norm.pdf(x, 50, 10)
z = np.zeros(101)
for i in np.arange(0, 101):
y[i] = y[i] * N
y[i] = int(y[i])
for k in range(10):
f = p[k]
MyCollector = Collector(f, q)
for index in range(101):
for i in range(int(y[index])):
data = BloomFilter(index, 4)
bf = data.getBloomFilter()
start = data.getSecurityDomain1()
end = data.getSecurityDomain2()
response = RRM(bf, f, q, start, end)
pbf = response.randomer()
MyCollector.receiver(pbf, start, end)
# print(pbf)
result = MyCollector.getResult()
for j in range(0, 101):
z[j] = int(result[j])
a = 0
b = 0
import Collector
Collector.start_function('kitten/index_dropbox.html')
def main():
Utility.DirectoryCleaning('transaction')
Utility.DirectoryCleaning('result')
if raw_input('Is disk connected? [y/n]: ').lower() == 'y':
temp = Collector.VolumeFinder()
disk = temp[0]
select = temp[1]
recovery = temp[2]
print 'Collecting files...'
temp = Collector.VolumeHeaderParser(
Utility.DiskDump(disk, 'HFSPlusVolumeHeader', 1024, 1, 1, select))
blockSize = temp[0]
journalInfoBlock = temp[1]
specialFileInfo = temp[2]
AllocationFile = Collector.SpecialFileExtractor(
'AllocationFile', specialFileInfo[0], disk, blockSize, select)
ExtentsOverflowFile = Collector.SpecialFileExtractor(
'ExtentsOverflowFile', specialFileInfo[1], disk, blockSize, select)
CatalogFile = Collector.SpecialFileExtractor('CatalogFile',
specialFileInfo[2], disk,
blockSize, select)
Journal = Collector.JournalExtractor(journalInfoBlock, disk, blockSize,
select)
else:
path = raw_input('Input path of volume header\n')
f = open(path, 'rb')
temp = Collector.VolumeHeaderParser(f.read())
blockSize = temp[0]
journalInfoBlock = temp[1]
specialFileInfo = temp[2]
f.close()
path = raw_input('Input path of journal\n')
f = open(path, 'rb')
Journal = f.read()
f.close()
if raw_input('Input path of catalog file(y/n)\n') == 'y':
path = input()
f = open(path, 'rb')
CatalogFile = f.read()
f.close()
"""
if raw_input('Input path of allocation file(y/n)') == 'y':
path=raw_input()
f=open(path,'rb')
AllocationFile=f.read()
f.close()
if raw_input('Input path of extents overflow file(y/n)\n') == 'y':
path=raw_input()
f=open(path,'rb')
ExtentsOverflowFile=f.read()
f.close()
"""
recovery = False
print 'Analyzing journal...'
temp = Analyzer.JournalParser(Journal)
sectorSize = temp[0]
transaction = temp[1]
print 'Printing transactions...'
for i in range(1, transaction[0] + 1):
for j in range(1, transaction[i][0] + 1):
fName = './transaction/transaction' + str(i) + '_' + str(
j) + '(sector' + hex(transaction[i][j][0]) + ')'
f = open(fName, 'wb')
f.write(transaction[i][j][1])
f.close()
print 'Analyzing CatalogFile...'
nameAndParent = Analyzer.CatalogFileAnalyzer(CatalogFile)
print 'Analyzing transactions...'
Analyzer.TransactionAnalyzer(transaction, specialFileInfo[0],
specialFileInfo[2], nameAndParent, sectorSize,
blockSize)
print 'Printing record list...'
fName = './result/recordList.csv'
f = open(fName, 'w')
for i in range(1, transaction[0] + 1):
for j in range(1, transaction[i][0] + 1):
if transaction[i][j][1][0] == 'c':
if transaction[i][j][1][1] == 'l' or transaction[i][j][1][
1] == 'i':
for k in transaction[i][j][2].keys():
f.write(
'transaction{0}_{1} record{2}\n'.format(i, j, k) +
',')
for l in transaction[i][j][2][k].keys():
if l == 'nodeName':
f.write('{0} : {1}\n'.format(
l, transaction[i][j][2][k][l].encode(
'utf-8')) + ',')
else:
f.write('{0} : {1}\n'.format(
l, transaction[i][j][2][k][l]) + ',')
f.write('\n')
elif transaction[i][j][1][1] == 'h':
f.write('transaction{0}_{1}\n'.format(i, j) + ',')
for k in transaction[i][j][2].keys():
f.write(
'{0} : {1}\n'.format(k, transaction[i][j][2][k]) +
',')
f.write('\n')
f.close()
print 'Deduplicating records...'
deduplicatedRecord = Analyzer.RecordDeduplication(transaction)
print 'Printing deduplicated records...'
keyForType = ['recordType']
keyForString = ['nodeName', 'fullPath']
keyForID = ['parentID', 'CNID', 'ownerID', 'groupID']
keyForDate = [
'createDate', 'contentModDate', 'attributeModDate', 'accessDate'
]
keyForFork = ['dataFork', 'resourceFork']
fName = './result/deduplicatedRecordList.csv'
f = open(fName, 'w')
for i in keyForType:
f.write('transaction' + ',' + i + ',')
for i in keyForString:
f.write(i + ',')
for i in keyForID:
f.write(i + ',')
for i in keyForDate:
f.write(i + ',')
f.write('AllocatedFork' + ',')
f.write('\n')
for i in range(1, deduplicatedRecord[0][0] + 1):
f.write(',')
f.write('\ntransaction{0}\n'.format(i) + ',')
for j in range(1, deduplicatedRecord[0][i][0] + 1):
f.write('\ntransaction{0}_{1}\n'.format(i, j))
for k in range(1, deduplicatedRecord[0][i][j][0] + 1):
if deduplicatedRecord[0][i][j][k]['recordType'] < 3:
for l in keyForType:
if deduplicatedRecord[0][i][j][k][l] == 1:
f.write(',')
f.write('folder')
else:
f.write(',')
f.write('file')
for l in keyForString:
f.write(',')
f.write(
deduplicatedRecord[0][i][j][k][l].encode('utf-8'))
f.write(',')
for l in keyForID:
f.write(hex(deduplicatedRecord[0][i][j][k][l])[2:])
f.write(',')
for l in keyForDate:
f.write((datetime.datetime(1904, 1, 1) +
datetime.timedelta(
seconds=deduplicatedRecord[0][i][j][k][l])
).isoformat(' ') + ',')
if deduplicatedRecord[0][i][j][k]['recordType'] == 2:
duplicated = ''
allocatedFork = 0
for l in keyForFork:
for m in range(
1,
deduplicatedRecord[0][i][j][k][l][0] + 1):
if deduplicatedRecord[0][i][j][k][l][m][
0] == 1:
allocatedFork += 1
elif deduplicatedRecord[0][i][j][k][l][m][
0] == -1:
duplicated = '(duplicated)'
f.write('{0}/{1}{2}'.format(
allocatedFork,
deduplicatedRecord[0][i][j][k]['dataFork'][0] +
deduplicatedRecord[0][i][j][k]['resourceFork'][0],
duplicated) + ',')
f.write('\n')
f.close()
if recovery:
print 'Recoverying deleted files...'
Analyzer.DataRecovery(disk, deduplicatedRecord[2], blockSize)