class GraphPlotter:
dc = DataCollector()
#PLOT FOR AVERAGE NUMBER OF MESSAGES IN NODE INSERTION VS NUMBER OF NODES
def plotForInserts(self):
plotter.plot(self.dc.n, self.dc.avg_insert_messages, color='b', linewidth=2.0)
plotter.ylabel("Average Number of Messages(Node Arrival)")
plotter.xlabel("Number of Nodes(N)")
plotter.show()
#PLOT FOR AVERAGE NUMBER OF MESSAGES IN NODE DELETION VS NUMBER OF NODES
def plotForDeletes(self):
plotter.plot(self.dc.n, self.dc.avg_delete_messages, color='r', linewidth=2.0)
plotter.ylabel("Average Number of Messages(Node Departure)")
plotter.xlabel("Number of Nodes(N)")
plotter.show()
#PLOT FOR AVERAGE NUMBER OF MESSAGES IN KEY LOOKUP VS NUMBER OF NODES
def plotForLookups(self):
plotter.plot(self.dc.n, self.dc.avg_lookup_messages, color='g', linewidth=2.0)
plotter.ylabel("Average Number of Messages(Key Lookup)")
plotter.xlabel("Number of Nodes(N)")
plotter.show()
#PLOT FOR GLOBAL FINGER TABLE HIT RATIO
def plotForFingerTableEfficacy(self):
plotter.plot(self.dc.n, self.dc.finger_table_hitratio, color='g', linewidth=2.0)
plotter.ylabel("Finger Table Hit Ratio")
plotter.xlabel("Number of Nodes(N)")
plotter.show()
def __init__(self,dq,state,freq):
self._running = True
self._display_surf = None
self._image_surf = None
self._apple_surf = None
self.game = Game()
self.player = Player(3,self.windowDimX, self.windowDimY)
self.windowWidth = self.windowDimX*self.player.step
self.windowHeight = self.windowDimY*self.player.step
self.toolbar = Toolbar(self.toolbarWidth, self.windowWidth, self.windowHeight)
self.apple = Apple(randint(0,self.windowDimX-1), randint(0,self.windowDimY-1))
self.dataCollect = DataCollector()
self.displayq=dq
self.state_size = state
self.frequency = freq
def startTask(self, cond, driving='?', blockNum=0):
self.condition = cond
self.fullCondition = driving + '_' + cond
self.run += 1
datacond = '_' + self.roadCond + '_tablet'
if self.doPractice:
datacond = '_prac' + datacond
self.db = DataCollector(
'Tablet DB',
'data/' + globals.participant + datacond + '_tablet.dat', [
'pp', 'cond', 'correct', 'answer', 'conversation', 'question',
'condtime', 'time'
])
self.db.open()
self.started = True
self.pickConversation()
self.startNextClip()
def __init__(self,
cfg,
is_video=True,
multi_camera=False,
enable_attr=False,
enable_action=False,
device='CPU',
run_mode='paddle',
trt_min_shape=1,
trt_max_shape=1280,
trt_opt_shape=640,
trt_calib_mode=False,
cpu_threads=1,
enable_mkldnn=False,
output_dir='output',
draw_center_traj=False,
secs_interval=10,
do_entrance_counting=False):
if enable_attr and not cfg.get('ATTR', False):
ValueError(
'enable_attr is set to True, please set ATTR in config file')
if enable_action and (not cfg.get('ACTION', False)
or not cfg.get('KPT', False)):
ValueError(
'enable_action is set to True, please set KPT and ACTION in config file'
)
self.with_attr = cfg.get('ATTR', False) and enable_attr
self.with_action = cfg.get('ACTION', False) and enable_action
self.with_mtmct = cfg.get('REID', False) and multi_camera
if self.with_attr:
print('Attribute Recognition enabled')
if self.with_action:
print('Action Recognition enabled')
if multi_camera:
if not self.with_mtmct:
print(
'Warning!!! MTMCT enabled, but cannot find REID config in [infer_cfg.yml], please check!'
)
else:
print("MTMCT enabled")
self.is_video = is_video
self.multi_camera = multi_camera
self.cfg = cfg
self.output_dir = output_dir
self.draw_center_traj = draw_center_traj
self.secs_interval = secs_interval
self.do_entrance_counting = do_entrance_counting
self.warmup_frame = self.cfg['warmup_frame']
self.pipeline_res = Result()
self.pipe_timer = PipeTimer()
self.file_name = None
self.collector = DataCollector()
if not is_video:
det_cfg = self.cfg['DET']
model_dir = det_cfg['model_dir']
batch_size = det_cfg['batch_size']
self.det_predictor = Detector(model_dir, device, run_mode,
batch_size, trt_min_shape,
trt_max_shape, trt_opt_shape,
trt_calib_mode, cpu_threads,
enable_mkldnn)
if self.with_attr:
attr_cfg = self.cfg['ATTR']
model_dir = attr_cfg['model_dir']
batch_size = attr_cfg['batch_size']
self.attr_predictor = AttrDetector(model_dir, device, run_mode,
batch_size, trt_min_shape,
trt_max_shape,
trt_opt_shape,
trt_calib_mode, cpu_threads,
enable_mkldnn)
else:
mot_cfg = self.cfg['MOT']
model_dir = mot_cfg['model_dir']
tracker_config = mot_cfg['tracker_config']
batch_size = mot_cfg['batch_size']
self.mot_predictor = SDE_Detector(
model_dir,
tracker_config,
device,
run_mode,
batch_size,
trt_min_shape,
trt_max_shape,
trt_opt_shape,
trt_calib_mode,
cpu_threads,
enable_mkldnn,
draw_center_traj=draw_center_traj,
secs_interval=secs_interval,
do_entrance_counting=do_entrance_counting)
if self.with_attr:
attr_cfg = self.cfg['ATTR']
model_dir = attr_cfg['model_dir']
batch_size = attr_cfg['batch_size']
self.attr_predictor = AttrDetector(model_dir, device, run_mode,
batch_size, trt_min_shape,
trt_max_shape,
trt_opt_shape,
trt_calib_mode, cpu_threads,
enable_mkldnn)
if self.with_action:
kpt_cfg = self.cfg['KPT']
kpt_model_dir = kpt_cfg['model_dir']
kpt_batch_size = kpt_cfg['batch_size']
action_cfg = self.cfg['ACTION']
action_model_dir = action_cfg['model_dir']
action_batch_size = action_cfg['batch_size']
action_frames = action_cfg['max_frames']
display_frames = action_cfg['display_frames']
self.coord_size = action_cfg['coord_size']
self.kpt_predictor = KeyPointDetector(kpt_model_dir,
device,
run_mode,
kpt_batch_size,
trt_min_shape,
trt_max_shape,
trt_opt_shape,
trt_calib_mode,
cpu_threads,
enable_mkldnn,
use_dark=False)
self.kpt_buff = KeyPointBuff(action_frames)
self.action_predictor = ActionRecognizer(
action_model_dir,
device,
run_mode,
action_batch_size,
trt_min_shape,
trt_max_shape,
trt_opt_shape,
trt_calib_mode,
cpu_threads,
enable_mkldnn,
window_size=action_frames)
self.action_visual_helper = ActionVisualHelper(display_frames)
if self.with_mtmct:
reid_cfg = self.cfg['REID']
model_dir = reid_cfg['model_dir']
batch_size = reid_cfg['batch_size']
self.reid_predictor = ReID(model_dir, device, run_mode, batch_size,
trt_min_shape, trt_max_shape,
trt_opt_shape, trt_calib_mode,
cpu_threads, enable_mkldnn)
def __init__(self, args):
self._logger = logging.getLogger('Traniner')
self._checkpoint = {}
config_name = args.config
# hardcode the config path just for convinence.
cfg = Utils.config("./config/" + config_name)
number_of_planes = cfg['GAME'].getint('number_of_planes')
board_size = cfg['GAME'].getint('board_size')
encoder_name = cfg['GAME'].get('encoder_name')
az_mcts_rounds_per_move = cfg['AZ_MCTS'].getint('rounds_per_move')
az_mcts_temperature = cfg['AZ_MCTS'].getfloat('temperature')
c_puct = cfg['AZ_MCTS'].getfloat('C_puct')
basic_mcts_c_puct = cfg['BASIC_MCTS'].getfloat('C_puct')
buffer_size = cfg['TRAIN'].getint('buffer_size')
batch_size = cfg['TRAIN'].getint('batch_size')
epochs = cfg['TRAIN'].getint('epochs')
self._basic_mcts_rounds_per_move = cfg['BASIC_MCTS'].getint(
'rounds_per_move')
self._latest_checkpoint_file = './checkpoints/' + config_name.split(
'.')[0] + '/latest.pth.tar'
self._best_checkpoint_file = './checkpoints/' + config_name.split(
'.')[0] + '/best.pth.tar'
check_number_of_games = cfg['EVALUATE'].getint('number_of_games')
os.makedirs(os.path.dirname(self._latest_checkpoint_file),
exist_ok=True)
os.makedirs(os.path.dirname(self._best_checkpoint_file), exist_ok=True)
use_cuda = torch.cuda.is_available()
devices_ids = []
if use_cuda:
devices_ids = list(map(int, args.gpu_ids.split(',')))
num_devices = torch.cuda.device_count()
if len(devices_ids) > num_devices:
raise Exception(
'#available gpu : {} < --device_ids : {}'.format(
num_devices, len(devices_ids)))
if encoder_name == 'SnapshotEncoder':
encoder = SnapshotEncoder(number_of_planes, board_size)
input_shape = (number_of_planes, board_size, board_size)
if encoder_name == 'DeepMindEncoder':
encoder = DeepMindEncoder(number_of_planes, board_size)
input_shape = (number_of_planes * 2 + 1, board_size, board_size)
if encoder_name == 'BlackWhiteEncoder':
encoder = BlackWhiteEncoder(number_of_planes, board_size)
input_shape = (number_of_planes * 2 + 2, board_size, board_size)
self._model_name = cfg['MODELS'].get('net')
self._model = ResNet8Network(
input_shape, board_size * board_size
) if self._model_name == 'ResNet8Network' else Simple5Network(
input_shape, board_size * board_size)
self._optimizer = Utils.get_optimizer(self._model.parameters(), cfg)
self._experience_buffer = ExpericenceBuffer(buffer_size)
self._check_frequence = cfg['TRAIN'].getint('check_frequence')
self._start_game_index = 1
self._train_number_of_games = cfg['TRAIN'].getint('number_of_games')
# Be aware this is not the first time to run this program
resume = args.resume
if resume:
self._checkpoint = torch.load(self._latest_checkpoint_file,
map_location='cpu')
if self._checkpoint['model_name'] == self._model_name:
if use_cuda:
self._model.to(torch.device('cuda:' + str(devices_ids[0])))
else:
self._model.to(torch.device('cpu'))
self._model.load_state_dict(self._checkpoint['model'])
self._optimizer.load_state_dict(self._checkpoint['optimizer'])
self._basic_mcts_rounds_per_move = self._checkpoint[
'basic_mcts_rounds_per_move']
self._start_game_index = self._checkpoint['game_index']
self._experience_buffer.data = self._checkpoint[
'experience_buffer'].data
self._logger.debug(
'ExpericenceBuffer size is {} when loading from checkpoint'
.format(self._experience_buffer.size()))
writer = SummaryWriter(log_dir='./runs/' + config_name.split('.')[0])
self._data_collector = DataCollector(encoder, self._model,
az_mcts_rounds_per_move, c_puct,
az_mcts_temperature, board_size,
number_of_planes, devices_ids,
use_cuda)
self._policy_improver = PolicyImprover(self._model, batch_size, epochs,
devices_ids, use_cuda,
self._optimizer, writer)
self._policy_checker = PolicyChecker(
devices_ids, use_cuda, encoder, board_size, number_of_planes,
self._model, az_mcts_rounds_per_move, c_puct, az_mcts_temperature,
basic_mcts_c_puct, check_number_of_games, writer)
display.write_message(message)
datafile = "%s/%s.%s" % (dataPath, _getCurrentTime(), logEnding)
loggedBytes += datacollector.write_data_log(datafile,
nbrOfOBDFrames=1000,
messagesPerTimestamp=20)
print "Collected data log..."
gzip.add_file_for_compression(datafile)
if __name__ == "__main__":
## make sure the script is called correctly
if 2 != len(sys.argv):
raise OSError(
"[ERROR] Correct usage:\n python obd2collector <data directory>")
dataPath = sys.argv[1]
print "Starting OBD2 DataCollector"
display = Display(autostart=True)
datacollector = DataCollector(display)
try:
main(dataPath, datacollector, display)
except KeyboardInterrupt:
## close all threads (hopefully)
Thread.shutdown()
os.mkdir('./data/transactions/ponzi/')
os.mkdir('./data/transactions/nonponzi/')
os.mkdir('./data/internal_transactions/ponzi/')
os.mkdir('./data/internal_transactions/nonponzi/')
os.mkdir('./data/contracts/ponzi/')
os.mkdir('./data/contracts/nonponzi/')
# important files
ponzi_contract_csv = './data/ponziContracts.csv'
nonponzi_contract_csv = './data/non_ponziContracts.csv'
opcode_csv = './data/Opcodes.csv'
# Download transaction data
print("Downloading Opcode.csv file...(over 4 GB)")
subprocess.call(['./download_opcodes.sh'])
datacollector = DataCollector(ponzi_contract_csv, nonponzi_contract_csv)
# download opcode data
opcode_collector = OpcodeColector(ponzi_contract_csv, nonponzi_contract_csv,
opcode_csv)
# (RECOMMENDED!) OPTION 2: download the dataset from this link; unzip and place the data folder in the root folder
# comment out the code from line 13 to line 45
# Link: https://drive.google.com/open?id=1izaOs4Mlp6dxdRMtRYQeUfkDhlqLf4Z6
# Extract features
extract_feature_of_all_contract = ExtractFeatureOfAllContract()
# Train models
model_performance = Model()
def startNextState(self, car, traffic, renderer):
state = self.flow[self.flowIdx]
if state.type == 'screen':
renderer.textScreens[self.flow[self.flowIdx].id].start()
elif state.type == 'drive':
self.block += 1
self.blockDuration = state.duration
# reset damage counter after practice
if self.doPractice and self.block == 2:
globals.bonusCounter = globals.STARTING_BONUS
self.radioTask.doPractice = False
self.tabletTask.doPractice = False
print '\n########\nSTARTING BLOCK '+str(self.block-1)+'\n########\n'+state.id+' & '+state.secondary+', length: '+str(self.blockDuration)
print 'Current bonus: '+str(globals.bonusCounter/10.0)
# set up databases for driving
globals.db['blinker'] = DataCollector('Blinker DB', 'data/'+globals.participant+'_'+self.getCondition()+'_blinker.dat',
['pp', 'condition', 'direction', 'block', 'condtime', 'time'])
globals.db['lanetransition'] = DataCollector('Lane Transitions DB', 'data/'+globals.participant+'_'+self.getCondition()+'_transition.dat',
['pp', 'condition', 'blinkerused', 'blinkdir', 'cardir', 'congruent', 'block', 'condtime', 'time'])
globals.db['overtake'] = DataCollector('Overtaken Slow Traffic DB', 'data/'+globals.participant+'_'+self.getCondition()+'_overtake.dat',
['pp', 'condition', 'state', 'carNum', 'carDist', 'block', 'condtime', 'time'])
globals.db['collision'] = DataCollector('Collision DB', 'data/'+globals.participant+'_'+self.getCondition()+'_collision.dat',
['pp', 'condition', 'zone', 'carType', 'carNum', 'block', 'condtime', 'time'])
globals.db['car'] = DataCollector('Car DB', 'data/'+globals.participant+'_'+self.getCondition()+'_car.dat', ['pp', 'condition',
'speed', 'wheelangle', 'yposition', 'deviation', 'accel', 'break', 'xposition', 'block', 'condtime', 'time'])
globals.db['slowcars'] = DataCollector('Slow Car DB', 'data/'+globals.participant+'_'+self.getCondition()+'_slowcars.dat', ['pp', 'condition',
'id', 'speed', 'xposition', 'distance', 'block', 'condtime', 'time'])
globals.db['fastcars'] = DataCollector('Fast Car DB', 'data/'+globals.participant+'_'+self.getCondition()+'_fastcars.dat', ['pp', 'condition',
'id', 'speed', 'xposition', 'distance', 'block', 'condtime', 'time'])
globals.db['blinker'].open()
globals.db['lanetransition'].open()
globals.db['car'].open()
globals.db['overtake'].open()
globals.db['collision'].open()
globals.db['slowcars'].open()
globals.db['fastcars'].open()
# traffic.road = self.roads[self.block]
# car.bindRoad(self.roads[self.block])
traffic.reset(self.roads[self.block])
car.reset(self.roads[self.block])
# populate road with traffic
if state.id == 'complex':
sDiff = abs(globals.EXPECTED_SPEED - globals.SLOW_SPEED)
dist = sDiff * globals.DRIVE_DURATION
overtakes = globals.COMPLEX_OVERTAKES
duration = globals.DRIVE_DURATION
if doPractice and block < 2: # practice blocks
duration = globals.PRACTICE_DURATION
overtakes = globals.PRACTICE_OVERTAKES
interval = dist / overtakes
#print interval
#interval = 20.0
print 'Generating complex traffic'
traffic.genStaticCars(overtakes, globals.EXPECTED_SPEED, globals.SLOW_SPEED, duration, globals.RIGHT_LANE)
else:
sDiff = abs(globals.EXPECTED_SPEED - globals.FAST_SPEED)
dist = sDiff * globals.DRIVE_DURATION
interval = dist / globals.SIMPLE_OVERTAKERS
#print interval
print 'Generating simple traffic'
traffic.genStaticCars(globals.SIMPLE_OVERTAKERS, globals.EXPECTED_SPEED, globals.FAST_SPEED, globals.DRIVE_DURATION, globals.LEFT_LANE)
if state.secondary != 'none':
print 'Starting secondary task'
if state.secondary == 'tablet':
print 'Tablet task selected'
self.secondary = self.tabletTask
else:
print 'Radio task selected'
self.secondary = self.radioTask
if (not self.doPractice or (self.block > 1)) and state.secondary != 'easy':
print 'Loading hard task conditions (quiz or tablet)'
curShow = 0
if self.fixedOrder:
print 'Quiz show order fixed on participant ID'
ppid = globals.participant.split('D') # names have the form SDXX or CDXX with XX between 01 and 24
even = False
if len(ppid) > 1:
ppidx = int(ppid[1])
if (ppidx % 2) == 0:
even = True
if even:
if state.secondary == 'tablet':
curShow = 0
else:
curShow = 1
else: # odd
if state.secondary == 'tablet':
curShow = 1
else:
curShow = 0
else:
print 'Picking quiz show randomly'
# pick a show
idx = random.sample(range(0,len(self.shows)), 1)[0]
curShow = self.shows[idx]
del self.shows[idx]
print 'Playing show '+str(curShow)
self.secondary.setConversation(curShow)
self.secondary.startTask(state.secondary, state.id)
print 'Beginning driving section'
globals.conditionStartTime = helpers.currentTime()
# #########################################
# Main Simulator Module #
# #########################################
from node import Node
from idGenerator import IdGenerator
from operation import Operation
from stats import Stats
from datacollector import DataCollector
from graphplotter import GraphPlotter
nodeIdGenerator = IdGenerator()
nodeIdGenerator.lastAllottedId = property.INITID
nodeIdGenerator.randomRange = property.RANGEID
data_collector = DataCollector()
maxId = 0
########## NODES IN RING OVERLAY #######################
nodes = []
########################################################
############## OPERATIONS AND EXPERIMENTATION ##########
def add_node(newNode, existingnode):
newNode.join(existingnode)
newNode.update_others(Operation.INSERT)
