def test_acquisition_bad_retrieval(): acquisition_obj = Acquisition() try: acquisition_obj.retrieve( "http://s1.twnmm.com/images/en_ca/icons/flags/all/16/ca.png", "ca2.png") assert False except Exception as e: assert True
def __init__(self, config_file_path): """ __init__ - self """ LOGGER.info("Initializing UATU.") self.config_file = config_file_path self.cfg_organizer = ConfigOrganizer(config_file=self.config_file) self.cfg_organizer.read_config_data() self.acq_obj = Acquisition() self.current_max_count() LOGGER.info("Completed initialization.") self.img_processing = ImageProcessing( yolo_path=self.cfg_organizer.config_handler['system']['yolo_dir'])
def main(): inputArgs = sys.argv args = inputArgs[1:] yearsTempList, magnitudeOver, overwrite = Input.getValues(args) years = Database.queryInput(yearsTempList, magnitudeOver, overwrite) StoreData.createFolder() print "Requesting earthquakes data with magnitude over {}, for years: {}".format(magnitudeOver, years) for year in years: print "Processing year: ", year print "Data acquisition starts" firstDate = date(year, 1, 1) lastDate = date(year, 12, 31) for d in dateRange(firstDate, lastDate): start = d.strftime("%Y-%m-%d") + "T00:00:00.000Z" end = (d + timedelta(days=1)).strftime("%Y-%m-%d") + "T00:00:00.000Z" try: eq_list_raw = Acquisition.request(start, end, magnitudeOver) eq_list_temp = Preprocessing.cleanHeaders(eq_list_raw) eq_list = Preprocessing.splitDateTime(eq_list_temp) StoreData.toFile(eq_list, year, d,magnitudeOver) except Exception as error: print "Error while processing a request:", error print "Data acquisition ended" path = HDFS.getPath() HDFS.put('../data/earthquakes{}mag{}.csv'.format(year, magnitudeOver), path)
def parse_acquisitions(self): src_paths = [] for f in self._files: # Check if it is a DICOM try: ds = dicom.read_file(f) except InvalidDicomError: utils.warning('Not a DICOM file: {}'.format(f)) continue # Filter by serie description dsDesc = ds.SeriesDescription if any(_ in dsDesc for _ in self.exclusionCriteria): self._excluded.add(dsDesc) else: src_paths.append(f) groups = dcmstack.parse_and_group(src_paths) for key, group in groups.iteritems(): if self._isFromOneDirectory(group): #utils.ok("Same directory: {}".format(key[2])) stack = dcmstack.stack_group(group) inDir = os.path.dirname(group[0][2]) self._acquisitions.append( Acquisition(inDir, stack, self._session)) else: #TODO: regroup dicoms in a tmp directory utils.new_line() utils.fail( "DICOM of '{}' are not in the same directory.\nThis structure won't be compatible with dcm2niibatch" .format(key[2]))
def main(): awareness = Awareness(BROADCAST_IP, BROADCAST_PORT, HEARBEAT_INTERVAL) awareness.start() acquisition = Acquisition(UNICAST_PORT) acquisition.start() time.sleep(DEFAULT_EXECUTION_TIME) awareness.stop() acquisition.stop()
def __init__(self): super().__init__() self.acquisition = Acquisition() self.preview = CaptureDevicePreview(self.acquisition.captureDevices) self.mainLayout = True self.readyForAcquisition = False # THREAD: Main Thread which works continuously self.stopMainThread = False self.mainThread = threading.Thread(target=self.runMainThread, args=(lambda: self.stopMainThread,)) self.mainThread.start() # THREAD: Acquisition self.createAcquisitionThread()
def main(): Log.info('-----------------------') Log.info('Download process starts') Log.info('-----------------------') inputArgs = sys.argv args = inputArgs[1:] StoreData.createFolder() yearsTempList, magnitudeOver, download_again = Input.getValues(args) path = HDFS.getPath() Log.info("Earthquakes acquisition starts..") years = Database.QueryInput(yearsTempList, magnitudeOver, download_again) Log.info( "Requesting earthquakes data with magnitude over {}, for years: {}". format(magnitudeOver, years)) for year in years: Log.info("Processing year: {}".format(year)) Log.info("Earthquakes acquisition starts.") firstDate = date(year, 1, 1) lastDate = date(year, 12, 31) for d in dateRange(firstDate, lastDate): start = d.strftime("%Y-%m-%d") + "T00:00:00.000Z" end = (d + timedelta(days=1)).strftime("%Y-%m-%d") + "T00:00:00.000Z" try: eq_list_raw = Acquisition.Request(start, end, magnitudeOver) eq_list_no_headers = Preprocessing.cleanHeaders(eq_list_raw) eq_list_split_date_time = Preprocessing.splitDateTime( eq_list_no_headers) eq_list = Preprocessing.checkCountry(eq_list_split_date_time) StoreData.toFile(eq_list, year, d, magnitudeOver) except Exception as error: Log.error("Error while processing a Request:") Log.error(error) Log.info("Earthquakes acquisition for year {} finished".format(year)) HDFS.put( '../../data/earthquakes-history/earthquakes{}mag{}.csv'.format( year, magnitudeOver), path) Log.info('---------------------') Log.info('Download process ends') Log.info('---------------------')
def main(): StoreData.createFolder() try: interval = int(sys.argv[1]) print "Interval parameter passed: ", interval except: interval = 10 print "No interval parameter passed, interval default value ", interval end = datetime.utcnow() - timedelta(minutes=10) start = end - timedelta(minutes=interval) while True: print "Data acquisition starts" print "Requesting earthquakes data" print "from ", start print "to ", end eq_list_raw = Acquisition.request(start, end) eq_list_temp = Preprocessing.cleanHeaders(eq_list_raw) eq_list = Preprocessing.splitDateTime(eq_list_temp) StoreData.toFile(eq_list) print "Data acquisition ended" print "Process starts again in {} minutes".format(interval) time.sleep(interval * 60) end = datetime.utcnow() - timedelta(minutes=10) start = end - timedelta(minutes=interval)
def initialize(self): self.createGroupBox() self.eyeButton = EyeButton(self) self.acquisition = Acquisition(self) self.alarmLayout.addWidget(self.acquisition)
class Module(QGroupBox): def __init__(self, mainWindow, name, devices): QGroupBox.__init__(self) self.mainWindow = mainWindow self.name = name self.devices = devices self.groupBox = [] self.alarmGB = [] self.divcoeff = 5 self.cLayout = QHBoxLayout() self.alarmLayout = QHBoxLayout() self.gbLayout = QGridLayout() self.cLayout.addLayout(self.alarmLayout) self.cLayout.addLayout(self.gbLayout) self.setLayout(self.cLayout) self.initialize() self.waitforData() @property def actors(self): return list(OrderedDict.fromkeys([device.tablename.split('__')[0] for device in self.devices])) @property def isOffline(self): return True if self.mode == 'offline' else False def initialize(self): self.createGroupBox() self.eyeButton = EyeButton(self) self.acquisition = Acquisition(self) self.alarmLayout.addWidget(self.acquisition) def setAlarms(self, alarms): hide = self.cleanAlarms() self.mode = alarms[0].mode if alarms else 'offline' self.setTitle('%s - %s ' % (self.name, self.mode)) for alarm in alarms: widget = AlarmGB(self, alarm) self.alarmGB.append(widget) self.alarmLayout.addWidget(widget) widget.hide() if hide else widget.show() def cleanAlarms(self): hide = False while self.alarmGB: alarm = self.alarmGB[0] hide = alarm.isHidden() self.alarmLayout.removeWidget(alarm) alarm.deleteLater() self.alarmGB.remove(alarm) return hide def moveEye(self): try: self.eyeButton.move(self.width() - 30, 0) except: pass def createGroupBox(self): for i, deviceConf in enumerate(self.devices): groupbox = DeviceGB(self, deviceConf) try: groupbox.testData() self.groupBox.append(groupbox) self.gbLayout.addWidget(groupbox, (i // self.divcoeff) + 1, i % self.divcoeff) except Exception as e: print (e, deviceConf.tablename) def showAll(self, bool): for groupbox in self.groupBox: groupbox.hide() if not bool else groupbox.show() for alarm in self.alarmGB: alarm.show() if not bool else alarm.hide() try: self.acquisition.hide() if bool else self.acquisition.show() except AttributeError: pass self.adjustSize() self.mainWindow.mainWidget.adjustSize() self.mainWindow.adjustSize() def waitforData(self): try: for groupbox in self.groupBox: groupbox.waitforData() for alarm in self.alarmGB: alarm.getValue() self.acquisition.network = True except psycopg2.OperationalError: self.acquisition.network = False self.mainWindow.db.close() def updateMode(self, mode): modes = readMode() for actor in self.actors: modes[actor] = mode writeMode(modes) def getGroupBox(self, tableName): for i, groupbox in enumerate(self.groupBox): if groupbox.tablename == tableName: return self.groupBox[i] def mouseReleaseEvent(self, QMouseEvent): if QMouseEvent.button() == Qt.RightButton: menu = QMenu(self) all_modes = [f[:-4] for f in next(os.walk(self.mainWindow.alarmPath))[-1] if '.cfg' in f] for mode in all_modes: action = QAction(mode, self) action.triggered.connect(partial(self.updateMode, mode)) menu.addAction(action) menu.popup(QCursor.pos()) def resizeEvent(self, QResizeEvent): self.moveEye() QGroupBox.resizeEvent(self, QResizeEvent)
def main(args): task_seq = [ # acquire_method(sub_acquire_method): random(""), no-dete("DAL","BALD"), dete("coreset","entropy",...) # ../../datasets/answer_selection/YahooCQA/data/data-FD/ #evidence, diversity # BiLSTM CNN { "model_name": "BiLSTM", "group_name": "[tkde]BiLSTM+Pets+MRR+160+160", "max_performance": 0.80, "data_path": "../../datasets/answer_selection/YahooCQA/data/data-Pets/", "acquire_method": "no-dete", "sub_acquire_method": "DAL", "unsupervised_method": '', "submodular_k": 1.5, "num_acquisitions_round": 50, "init_question_num": 32, "acquire_question_num_per_round": 32, "warm_start_random_seed": 0, "sample_method": "No-Deterministic+DALLL2Layer+0", }, { "model_name": "BiLSTM", "group_name": "[tkde]BiLSTM+Pets+MRR+160+160", "max_performance": 0.80, "data_path": "../../datasets/answer_selection/YahooCQA/data/data-Pets/", "acquire_method": "no-dete", "sub_acquire_method": "DAL", "unsupervised_method": '', "submodular_k": 1.5, "num_acquisitions_round": 50, "init_question_num": 32, "acquire_question_num_per_round": 32, "warm_start_random_seed": 16, "sample_method": "No-Deterministic+DALLL2Layer+16", }, { "model_name": "BiLSTM", "group_name": "[tkde]BiLSTM+Pets+MRR+160+160", "max_performance": 0.80, "data_path": "../../datasets/answer_selection/YahooCQA/data/data-Pets/", "acquire_method": "no-dete", "sub_acquire_method": "DAL", "unsupervised_method": '', "submodular_k": 1.5, "num_acquisitions_round": 50, "init_question_num": 32, "acquire_question_num_per_round": 32, "warm_start_random_seed": 32, "sample_method": "No-Deterministic+DALLL2Layer+32", }, { "model_name": "BiLSTM", "group_name": "[tkde]BiLSTM+Pets+MRR+160+160", "max_performance": 0.80, "data_path": "../../datasets/answer_selection/YahooCQA/data/data-Pets/", "acquire_method": "no-dete", "sub_acquire_method": "DAL", "unsupervised_method": '', "submodular_k": 1.5, "num_acquisitions_round": 50, "init_question_num": 32, "acquire_question_num_per_round": 32, "warm_start_random_seed": 64, "sample_method": "No-Deterministic+DALLL2Layer+64", }, { "model_name": "BiLSTM", "group_name": "[tkde]BiLSTM+Pets+MRR+160+160", "max_performance": 0.80, "data_path": "../../datasets/answer_selection/YahooCQA/data/data-Pets/", "acquire_method": "no-dete", "sub_acquire_method": "DAL", "unsupervised_method": '', "submodular_k": 1.5, "num_acquisitions_round": 50, "init_question_num": 32, "acquire_question_num_per_round": 32, "warm_start_random_seed": 128, "sample_method": "No-Deterministic+DALLL2Layer+128", }, { "model_name": "BiLSTM", "group_name": "[tkde]BiLSTM+Pets+MRR+160+160", "max_performance": 0.80, "data_path": "../../datasets/answer_selection/YahooCQA/data/data-Pets/", "acquire_method": "random", "sub_acquire_method": "", "unsupervised_method": '', "submodular_k": 1.5, "num_acquisitions_round": 50, "init_question_num": 32, "acquire_question_num_per_round": 32, "warm_start_random_seed": 0, "sample_method": "No-Deterministic+randommm2Layer+0", }, { "model_name": "BiLSTM", "group_name": "[tkde]BiLSTM+Pets+MRR+160+160", "max_performance": 0.80, "data_path": "../../datasets/answer_selection/YahooCQA/data/data-Pets/", "acquire_method": "random", "sub_acquire_method": "", "unsupervised_method": '', "submodular_k": 1.5, "num_acquisitions_round": 50, "init_question_num": 32, "acquire_question_num_per_round": 32, "warm_start_random_seed": 16, "sample_method": "No-Deterministic+randommm2Layer+16", }, { "model_name": "BiLSTM", "group_name": "[tkde]BiLSTM+Pets+MRR+160+160", "max_performance": 0.80, "data_path": "../../datasets/answer_selection/YahooCQA/data/data-Pets/", "acquire_method": "random", "sub_acquire_method": "", "unsupervised_method": '', "submodular_k": 1.5, "num_acquisitions_round": 50, "init_question_num": 32, "acquire_question_num_per_round": 32, "warm_start_random_seed": 32, "sample_method": "No-Deterministic+randommm2Layer+32", }, { "model_name": "BiLSTM", "group_name": "[tkde]BiLSTM+Pets+MRR+160+160", "max_performance": 0.80, "data_path": "../../datasets/answer_selection/YahooCQA/data/data-Pets/", "acquire_method": "random", "sub_acquire_method": "", "unsupervised_method": '', "submodular_k": 1.5, "num_acquisitions_round": 50, "init_question_num": 32, "acquire_question_num_per_round": 32, "warm_start_random_seed": 64, "sample_method": "No-Deterministic+randommm2Layer+64", }, { "model_name": "BiLSTM", "group_name": "[tkde]BiLSTM+Pets+MRR+160+160", "max_performance": 0.80, "data_path": "../../datasets/answer_selection/YahooCQA/data/data-Pets/", "acquire_method": "random", "sub_acquire_method": "", "unsupervised_method": '', "submodular_k": 1.5, "num_acquisitions_round": 50, "init_question_num": 32, "acquire_question_num_per_round": 32, "warm_start_random_seed": 128, "sample_method": "No-Deterministic+randommm2Layer+128", }, ] allMethods_results = [ ] #Record the performance results of each method during active learning for config in task_seq: print("-------------------{}-{}-------------------".format( config["group_name"], config["sample_method"])) ####################################### initial setting ########################################### data_path = config["data_path"] model_name = config[ "model_name"] if "model_name" in config else 'BiLSTM' num_acquisitions_round = config["num_acquisitions_round"] acquire_method = config["acquire_method"] sub_acquire_method = config["sub_acquire_method"] init_question_num = config[ "init_question_num"] if "init_question_num" in config else 160 # number of initial training samples acquire_question_num_per_round = config[ "acquire_question_num_per_round"] if "acquire_question_num_per_round" in config else 20 #Number of samples collected per round warm_start_random_seed = config[ "warm_start_random_seed"] # the random seed for selecting the initial training set sample_method = config["sample_method"] loader = Loader() print('model:', model_name) print('dataset:', data_path) print('acquisition method:', acquire_method, "+", sub_acquire_method) if not os.path.exists(args.result_path): os.makedirs(args.result_path) if not os.path.exists(os.path.join(args.result_path, model_name)): os.makedirs(os.path.join(args.result_path, model_name)) if not os.path.exists( os.path.join(args.result_path, model_name, 'active_checkpoint', acquire_method)): os.makedirs( os.path.join(args.result_path, model_name, 'active_checkpoint', acquire_method)) #### If the data is not compiled, compile; otherwise load directly if (os.path.exists(os.path.join(data_path, 'mappings.pkl')) and os.path.exists(os.path.join(data_path, 'train.pkl')) and os.path.exists(os.path.join(data_path, 'test.pkl'))): mappings = pkl.load( open(os.path.join(data_path, 'mappings.pkl'), 'rb')) train_data = pkl.load( open(os.path.join(data_path, 'train.pkl'), 'rb')) test_data = pkl.load( open(os.path.join(data_path, 'test.pkl'), 'rb')) else: train_data, test_data, mappings = loader.load_yahoo( data_path, args.pretrained_word_embedding, args.word_embedding_dim, args.answer_count) pkl.dump(mappings, open(os.path.join(data_path, 'mappings.pkl'), 'wb')) pkl.dump(train_data, open(os.path.join(data_path, 'train.pkl'), 'wb')) pkl.dump(test_data, open(os.path.join(data_path, 'test.pkl'), 'wb')) #word embedding word_to_id = mappings['word_to_id'] tag_to_id = mappings['tag_to_id'] word_embeds = mappings[ 'word_embeds'] if args.use_pretrained_word_embedding else None word_vocab_size = len(word_to_id) print( ' The total amount of training data:%d\n' % len( train_data ), # Total number of training samples (number of question answer pair) 'The total amount of val data:%d\n' % len(test_data)) acquisition_function = Acquisition(train_data, seed=warm_start_random_seed, answer_count=args.answer_count, cuda_device=args.device[0], batch_size=args.sampling_batch_size, submodular_k=config["submodular_k"]) checkpoint_path = os.path.join(args.result_path, 'active_checkpoint', config["group_name"], sample_method) if not os.path.exists(checkpoint_path): os.makedirs(checkpoint_path) method_result = [ ] # Record the performance results of each method during active learning ####################################### acquire data and retrain ########################################### for i in range(num_acquisitions_round): print("current round:{}".format(i)) #-------------------acquisition--------------------- if i == 0: #first round acq = init_question_num a_m = "random" m_p = "" else: acq = acquire_question_num_per_round a_m = acquire_method m_p = os.path.join(checkpoint_path, 'modelweights') acquisition_function.obtain_data( train_data, model_path=m_p, model_name=model_name, acquire_questions_num=acq, method=a_m, sub_method=sub_acquire_method, unsupervised_method=config["unsupervised_method"]) # -------------------prepare training data--------------------- ''' train_data format: { 'str_words_q': str_words_q, # question word segmentation 'str_words_a': str_words_a, # answer word segmentation 'words_q': words_q, # question word id 'words_a': words_a, # answer word id 'tag': tag, # sample tag id } ''' new_train_index = (acquisition_function.train_index).copy() sorted_train_index = list(new_train_index) sorted_train_index.sort() labeled_train_data = [train_data[i] for i in sorted_train_index] print("Labeled training samples: {}".format( len(acquisition_function.train_index))) # -------------------------------------train-------------------------------------- print('.............Recreate the model...................') if model_name == 'BiLSTM': model = BiLSTM(word_vocab_size, args.word_embedding_dim, args.word_hidden_dim, args.target_size, pretrained=word_embeds, with_sim_features=args.with_sim_feature, cuda_device=args.device[0]) if model_name == 'CNN': model = CNN(word_vocab_size, args.word_embedding_dim, args.word_out_channels, args.target_size, pretrained=word_embeds, cuda_device=args.device[0]) model.cuda(args.device[0]) trainer = Trainer(model, model_name, tag_to_id, answer_count=args.answer_count, cuda_device=args.device[0], sampling_number=args.sampling_number) test_performance = trainer.train_supervisedLearning( args.num_epochs, labeled_train_data, test_data, args.learning_rate, checkpoint_path=checkpoint_path, batch_size=args.batch_size) print('.' * 50) print("Test performance: {}".format(test_performance)) print('*' * 50) #--------------------------Send data for a visual web page------------------------------ max_performance = config[ "max_performance"] if "max_performance" in config else 0 if "group_name" in config: updateLineChart(str(test_performance), sample_method, gp_name=config["group_name"], max=max_performance) else: updateLineChart(str(test_performance), sample_method, max=max_performance) # method_result.append(test_performance) # # print("acquire_method: {},sub_acquire_method: {}, warm_start_random_seed{}" # .format(acquire_method, sub_acquire_method, warm_start_random_seed)) # print(method_result) # allMethods_results.append(method_result) shutil.rmtree(checkpoint_path)
class Uatu: """ Uatu - he who watches """ def __init__(self, config_file_path): """ __init__ - self """ LOGGER.info("Initializing UATU.") self.config_file = config_file_path self.cfg_organizer = ConfigOrganizer(config_file=self.config_file) self.cfg_organizer.read_config_data() self.acq_obj = Acquisition() self.current_max_count() LOGGER.info("Completed initialization.") self.img_processing = ImageProcessing( yolo_path=self.cfg_organizer.config_handler['system']['yolo_dir']) def __repr__(self): return "string" def current_max_count(self): """ current_max_count - returns the current maximum count for the cameras """ namelist = ['name', 'timestamp', 'count'] df = pd.read_csv( self.cfg_organizer.config_handler['system']['csv_location'], index_col=False, names=namelist) df2 = df.fillna(0) camera_names = df['name'].unique() df2.sort_values(by=['name', 'count'], inplace=True) series = df2.groupby('name')['count'].max() self.stored_values = series.to_dict() def producer_images(self, pqueue, cqueue, lock): while True: if not pqueue.empty(): camera_name = pqueue.get() image_name = '/tmp/{}-image.jpg'.format(camera_name) try: self.acq_obj.retrieve( self.cfg_organizer.config_handler[camera_name]['url'], image_name) with lock: LOGGER.debug(f'retrieved image: {camera_name}') cqueue.put((camera_name, image_name)) except Exception as e: with lock: LOGGER.debug(f'exception: {e}') # pqueue.task_done() def consumer_process_image(self, cqueue, lock): counter = 1 while True: if cqueue.empty(): continue camera_name, image_name = cqueue.get() with lock: LOGGER.debug(f'processing camera: {camera_name}') try: with lock: self.img_processing.load_file( f'/tmp/{camera_name}-image.jpg') except IOError: with lock: LOGGER.debug(f'yup - io error - skipping {camera_name}') return with lock: self.img_processing.preprocess_image() with lock: self.img_processing.process_bounding_boxes() processed_image = "/tmp/{}-{}-{}.jpg".format( camera_name, time.time(), self.img_processing.people_count) with lock: self.img_processing.output_adjusted_image( '/tmp/what-{}.jpg'.format(counter)) with lock: print("{},{},{},{}".format(camera_name, time.time(), self.img_processing.people_count, processed_image)) if int(self.img_processing.people_count) > int( self.stored_values[camera_name]): self.img_processing.output_adjusted_image(processed_image) def doit(self): self.current_max_count() pqueue = Queue(maxsize=0) cqueue = Queue(maxsize=0) lock = Lock() for i in range(NUM_WORKER_THREADS): pworker = threading.Thread(target=self.producer_images, args=(pqueue, cqueue, lock)) pworker.daemon = True pworker.start() LOGGER.debug('initiated pworkers') cworker = threading.Thread(target=self.consumer_process_image, args=(cqueue, lock)) cworker.daemon = True cworker.start() LOGGER.debug('initiated workers') cameras = self.cfg_organizer.find_cameras() for camera_name in cameras: LOGGER.debug(f'Loading {camera_name} into pqueue') pqueue.put(camera_name) cworker.join() def debug_dump(self): LOGGER.debug('Building camera information') for camera in self.cfg_organizer.find_cameras(): LOGGER.debug(f'{camera}') LOGGER.debug('\t{}'.format( self.cfg_organizer.config_handler[camera]['url']))
def __init__(self, ws_host="", ws_port=5678): self.daq = Acquisition() self.ws_handler = WebSocketHandler(ws_host, ws_port, self.daq)
def test_acquisition_retrieval(): acquisition_obj = Acquisition() acquisition_obj.retrieve( "http://s1.twnmm.com/images/en_ca/icons/flags/all/16/ca.png", "ca.png") assert os.path.exists("ca.png") == True
def test_acquisition_creation(): acquisition_obj = Acquisition() assert acquisition_obj is not None