Example #1
0
def test_acquisition_bad_retrieval():
    acquisition_obj = Acquisition()
    try:
        acquisition_obj.retrieve(
            "http://s1.twnmm.com/images/en_ca/icons/flags/all/16/ca.png",
            "ca2.png")
        assert False
    except Exception as e:
        assert True
Example #2
0
 def __init__(self, config_file_path):
     """
     __init__ - self
     """
     LOGGER.info("Initializing UATU.")
     self.config_file = config_file_path
     self.cfg_organizer = ConfigOrganizer(config_file=self.config_file)
     self.cfg_organizer.read_config_data()
     self.acq_obj = Acquisition()
     self.current_max_count()
     LOGGER.info("Completed initialization.")
     self.img_processing = ImageProcessing(
         yolo_path=self.cfg_organizer.config_handler['system']['yolo_dir'])
Example #3
0
def main():
    inputArgs = sys.argv
    args = inputArgs[1:]
    yearsTempList, magnitudeOver, overwrite = Input.getValues(args)
    years = Database.queryInput(yearsTempList, magnitudeOver, overwrite)
    StoreData.createFolder()
    print "Requesting earthquakes data with magnitude over {}, for years: {}".format(magnitudeOver, years)
    for year in years:
        print "Processing year: ", year
        print "Data acquisition starts"
        firstDate = date(year, 1, 1)
        lastDate = date(year, 12, 31)
        for d in dateRange(firstDate, lastDate):
            start = d.strftime("%Y-%m-%d") + "T00:00:00.000Z"
            end = (d + timedelta(days=1)).strftime("%Y-%m-%d") + "T00:00:00.000Z"
            try:
                eq_list_raw = Acquisition.request(start, end, magnitudeOver)
                eq_list_temp = Preprocessing.cleanHeaders(eq_list_raw)
                eq_list = Preprocessing.splitDateTime(eq_list_temp)
                StoreData.toFile(eq_list, year, d,magnitudeOver)
            except Exception as error:
                print "Error while processing a request:", error
        print "Data acquisition ended"
        path = HDFS.getPath()
        HDFS.put('../data/earthquakes{}mag{}.csv'.format(year, magnitudeOver), path)
Example #4
0
    def parse_acquisitions(self):
        src_paths = []
        for f in self._files:
            # Check if it is a DICOM
            try:
                ds = dicom.read_file(f)
            except InvalidDicomError:
                utils.warning('Not a DICOM file: {}'.format(f))
                continue
            # Filter by serie description
            dsDesc = ds.SeriesDescription
            if any(_ in dsDesc for _ in self.exclusionCriteria):
                self._excluded.add(dsDesc)
            else:
                src_paths.append(f)
        groups = dcmstack.parse_and_group(src_paths)

        for key, group in groups.iteritems():
            if self._isFromOneDirectory(group):
                #utils.ok("Same directory: {}".format(key[2]))
                stack = dcmstack.stack_group(group)
                inDir = os.path.dirname(group[0][2])
                self._acquisitions.append(
                    Acquisition(inDir, stack, self._session))
            else:
                #TODO: regroup dicoms in a tmp directory
                utils.new_line()
                utils.fail(
                    "DICOM of '{}' are not in the same directory.\nThis structure won't be compatible with dcm2niibatch"
                    .format(key[2]))
Example #5
0
def main():
    awareness = Awareness(BROADCAST_IP, BROADCAST_PORT, HEARBEAT_INTERVAL)
    awareness.start()
    acquisition = Acquisition(UNICAST_PORT)
    acquisition.start()
    time.sleep(DEFAULT_EXECUTION_TIME)
    awareness.stop()
    acquisition.stop()
Example #6
0
    def __init__(self):
        super().__init__()
        self.acquisition = Acquisition()
        self.preview = CaptureDevicePreview(self.acquisition.captureDevices)
        self.mainLayout = True
        self.readyForAcquisition = False

        # THREAD: Main Thread which works continuously
        self.stopMainThread = False
        self.mainThread = threading.Thread(target=self.runMainThread, args=(lambda: self.stopMainThread,))
        self.mainThread.start()

        # THREAD: Acquisition
        self.createAcquisitionThread()
Example #7
0
def main():
    Log.info('-----------------------')
    Log.info('Download process starts')
    Log.info('-----------------------')
    inputArgs = sys.argv
    args = inputArgs[1:]
    StoreData.createFolder()
    yearsTempList, magnitudeOver, download_again = Input.getValues(args)
    path = HDFS.getPath()
    Log.info("Earthquakes acquisition starts..")
    years = Database.QueryInput(yearsTempList, magnitudeOver, download_again)
    Log.info(
        "Requesting earthquakes data with magnitude over {}, for years: {}".
        format(magnitudeOver, years))
    for year in years:
        Log.info("Processing year: {}".format(year))
        Log.info("Earthquakes acquisition starts.")
        firstDate = date(year, 1, 1)
        lastDate = date(year, 12, 31)
        for d in dateRange(firstDate, lastDate):
            start = d.strftime("%Y-%m-%d") + "T00:00:00.000Z"
            end = (d +
                   timedelta(days=1)).strftime("%Y-%m-%d") + "T00:00:00.000Z"
            try:
                eq_list_raw = Acquisition.Request(start, end, magnitudeOver)
                eq_list_no_headers = Preprocessing.cleanHeaders(eq_list_raw)
                eq_list_split_date_time = Preprocessing.splitDateTime(
                    eq_list_no_headers)
                eq_list = Preprocessing.checkCountry(eq_list_split_date_time)
                StoreData.toFile(eq_list, year, d, magnitudeOver)
            except Exception as error:
                Log.error("Error while processing a Request:")
                Log.error(error)
        Log.info("Earthquakes acquisition for  year {} finished".format(year))

        HDFS.put(
            '../../data/earthquakes-history/earthquakes{}mag{}.csv'.format(
                year, magnitudeOver), path)
    Log.info('---------------------')
    Log.info('Download process ends')
    Log.info('---------------------')
def main():
    StoreData.createFolder()
    try:
        interval = int(sys.argv[1])
        print "Interval parameter passed: ", interval
    except:
        interval = 10
        print "No interval parameter passed, interval default value ", interval
    end = datetime.utcnow() - timedelta(minutes=10)
    start = end - timedelta(minutes=interval)
    while True:
        print "Data acquisition starts"
        print "Requesting earthquakes data"
        print "from ", start
        print "to   ", end
        eq_list_raw = Acquisition.request(start, end)
        eq_list_temp = Preprocessing.cleanHeaders(eq_list_raw)
        eq_list = Preprocessing.splitDateTime(eq_list_temp)
        StoreData.toFile(eq_list)
        print "Data acquisition ended"
        print "Process starts again in {} minutes".format(interval)
        time.sleep(interval * 60)
        end = datetime.utcnow() - timedelta(minutes=10)
        start = end - timedelta(minutes=interval)
    def initialize(self):
        self.createGroupBox()

        self.eyeButton = EyeButton(self)
        self.acquisition = Acquisition(self)
        self.alarmLayout.addWidget(self.acquisition)
class Module(QGroupBox):
    def __init__(self, mainWindow, name, devices):
        QGroupBox.__init__(self)

        self.mainWindow = mainWindow
        self.name = name
        self.devices = devices

        self.groupBox = []
        self.alarmGB = []
        self.divcoeff = 5

        self.cLayout = QHBoxLayout()
        self.alarmLayout = QHBoxLayout()
        self.gbLayout = QGridLayout()

        self.cLayout.addLayout(self.alarmLayout)
        self.cLayout.addLayout(self.gbLayout)

        self.setLayout(self.cLayout)

        self.initialize()
        self.waitforData()

    @property
    def actors(self):
        return list(OrderedDict.fromkeys([device.tablename.split('__')[0] for device in self.devices]))

    @property
    def isOffline(self):
        return True if self.mode == 'offline' else False

    def initialize(self):
        self.createGroupBox()

        self.eyeButton = EyeButton(self)
        self.acquisition = Acquisition(self)
        self.alarmLayout.addWidget(self.acquisition)

    def setAlarms(self, alarms):
        hide = self.cleanAlarms()
        self.mode = alarms[0].mode if alarms else 'offline'
        self.setTitle('%s - %s ' % (self.name, self.mode))

        for alarm in alarms:
            widget = AlarmGB(self, alarm)
            self.alarmGB.append(widget)
            self.alarmLayout.addWidget(widget)
            widget.hide() if hide else widget.show()

    def cleanAlarms(self):
        hide = False
        while self.alarmGB:
            alarm = self.alarmGB[0]
            hide = alarm.isHidden()
            self.alarmLayout.removeWidget(alarm)
            alarm.deleteLater()
            self.alarmGB.remove(alarm)

        return hide

    def moveEye(self):

        try:
            self.eyeButton.move(self.width() - 30, 0)
        except:
            pass

    def createGroupBox(self):
        for i, deviceConf in enumerate(self.devices):
            groupbox = DeviceGB(self, deviceConf)
            try:
                groupbox.testData()
                self.groupBox.append(groupbox)
                self.gbLayout.addWidget(groupbox, (i // self.divcoeff) + 1, i % self.divcoeff)
            except Exception as e:
                print (e, deviceConf.tablename)

    def showAll(self, bool):

        for groupbox in self.groupBox:
            groupbox.hide() if not bool else groupbox.show()

        for alarm in self.alarmGB:
            alarm.show() if not bool else alarm.hide()

        try:
            self.acquisition.hide() if bool else self.acquisition.show()
        except AttributeError:
            pass

        self.adjustSize()
        self.mainWindow.mainWidget.adjustSize()
        self.mainWindow.adjustSize()


    def waitforData(self):
        try:
            for groupbox in self.groupBox:
                groupbox.waitforData()
            for alarm in self.alarmGB:
                alarm.getValue()
            self.acquisition.network = True

        except psycopg2.OperationalError:
            self.acquisition.network = False
            self.mainWindow.db.close()

    def updateMode(self, mode):
        modes = readMode()
        for actor in self.actors:
            modes[actor] = mode

        writeMode(modes)

    def getGroupBox(self, tableName):
        for i, groupbox in enumerate(self.groupBox):
            if groupbox.tablename == tableName:
                return self.groupBox[i]

    def mouseReleaseEvent(self, QMouseEvent):
        if QMouseEvent.button() == Qt.RightButton:
            menu = QMenu(self)

            all_modes = [f[:-4] for f in next(os.walk(self.mainWindow.alarmPath))[-1] if '.cfg' in f]
            for mode in all_modes:
                action = QAction(mode, self)
                action.triggered.connect(partial(self.updateMode, mode))
                menu.addAction(action)

            menu.popup(QCursor.pos())

    def resizeEvent(self, QResizeEvent):
        self.moveEye()
        QGroupBox.resizeEvent(self, QResizeEvent)
Example #11
0
def main(args):

    task_seq = [
        # acquire_method(sub_acquire_method): random(""), no-dete("DAL","BALD"), dete("coreset","entropy",...)
        # ../../datasets/answer_selection/YahooCQA/data/data-FD/
        #evidence, diversity # BiLSTM CNN
        {
            "model_name": "BiLSTM",
            "group_name": "[tkde]BiLSTM+Pets+MRR+160+160",
            "max_performance": 0.80,
            "data_path":
            "../../datasets/answer_selection/YahooCQA/data/data-Pets/",
            "acquire_method": "no-dete",
            "sub_acquire_method": "DAL",
            "unsupervised_method": '',
            "submodular_k": 1.5,
            "num_acquisitions_round": 50,
            "init_question_num": 32,
            "acquire_question_num_per_round": 32,
            "warm_start_random_seed": 0,
            "sample_method": "No-Deterministic+DALLL2Layer+0",
        },
        {
            "model_name": "BiLSTM",
            "group_name": "[tkde]BiLSTM+Pets+MRR+160+160",
            "max_performance": 0.80,
            "data_path":
            "../../datasets/answer_selection/YahooCQA/data/data-Pets/",
            "acquire_method": "no-dete",
            "sub_acquire_method": "DAL",
            "unsupervised_method": '',
            "submodular_k": 1.5,
            "num_acquisitions_round": 50,
            "init_question_num": 32,
            "acquire_question_num_per_round": 32,
            "warm_start_random_seed": 16,
            "sample_method": "No-Deterministic+DALLL2Layer+16",
        },
        {
            "model_name": "BiLSTM",
            "group_name": "[tkde]BiLSTM+Pets+MRR+160+160",
            "max_performance": 0.80,
            "data_path":
            "../../datasets/answer_selection/YahooCQA/data/data-Pets/",
            "acquire_method": "no-dete",
            "sub_acquire_method": "DAL",
            "unsupervised_method": '',
            "submodular_k": 1.5,
            "num_acquisitions_round": 50,
            "init_question_num": 32,
            "acquire_question_num_per_round": 32,
            "warm_start_random_seed": 32,
            "sample_method": "No-Deterministic+DALLL2Layer+32",
        },
        {
            "model_name": "BiLSTM",
            "group_name": "[tkde]BiLSTM+Pets+MRR+160+160",
            "max_performance": 0.80,
            "data_path":
            "../../datasets/answer_selection/YahooCQA/data/data-Pets/",
            "acquire_method": "no-dete",
            "sub_acquire_method": "DAL",
            "unsupervised_method": '',
            "submodular_k": 1.5,
            "num_acquisitions_round": 50,
            "init_question_num": 32,
            "acquire_question_num_per_round": 32,
            "warm_start_random_seed": 64,
            "sample_method": "No-Deterministic+DALLL2Layer+64",
        },
        {
            "model_name": "BiLSTM",
            "group_name": "[tkde]BiLSTM+Pets+MRR+160+160",
            "max_performance": 0.80,
            "data_path":
            "../../datasets/answer_selection/YahooCQA/data/data-Pets/",
            "acquire_method": "no-dete",
            "sub_acquire_method": "DAL",
            "unsupervised_method": '',
            "submodular_k": 1.5,
            "num_acquisitions_round": 50,
            "init_question_num": 32,
            "acquire_question_num_per_round": 32,
            "warm_start_random_seed": 128,
            "sample_method": "No-Deterministic+DALLL2Layer+128",
        },
        {
            "model_name": "BiLSTM",
            "group_name": "[tkde]BiLSTM+Pets+MRR+160+160",
            "max_performance": 0.80,
            "data_path":
            "../../datasets/answer_selection/YahooCQA/data/data-Pets/",
            "acquire_method": "random",
            "sub_acquire_method": "",
            "unsupervised_method": '',
            "submodular_k": 1.5,
            "num_acquisitions_round": 50,
            "init_question_num": 32,
            "acquire_question_num_per_round": 32,
            "warm_start_random_seed": 0,
            "sample_method": "No-Deterministic+randommm2Layer+0",
        },
        {
            "model_name": "BiLSTM",
            "group_name": "[tkde]BiLSTM+Pets+MRR+160+160",
            "max_performance": 0.80,
            "data_path":
            "../../datasets/answer_selection/YahooCQA/data/data-Pets/",
            "acquire_method": "random",
            "sub_acquire_method": "",
            "unsupervised_method": '',
            "submodular_k": 1.5,
            "num_acquisitions_round": 50,
            "init_question_num": 32,
            "acquire_question_num_per_round": 32,
            "warm_start_random_seed": 16,
            "sample_method": "No-Deterministic+randommm2Layer+16",
        },
        {
            "model_name": "BiLSTM",
            "group_name": "[tkde]BiLSTM+Pets+MRR+160+160",
            "max_performance": 0.80,
            "data_path":
            "../../datasets/answer_selection/YahooCQA/data/data-Pets/",
            "acquire_method": "random",
            "sub_acquire_method": "",
            "unsupervised_method": '',
            "submodular_k": 1.5,
            "num_acquisitions_round": 50,
            "init_question_num": 32,
            "acquire_question_num_per_round": 32,
            "warm_start_random_seed": 32,
            "sample_method": "No-Deterministic+randommm2Layer+32",
        },
        {
            "model_name": "BiLSTM",
            "group_name": "[tkde]BiLSTM+Pets+MRR+160+160",
            "max_performance": 0.80,
            "data_path":
            "../../datasets/answer_selection/YahooCQA/data/data-Pets/",
            "acquire_method": "random",
            "sub_acquire_method": "",
            "unsupervised_method": '',
            "submodular_k": 1.5,
            "num_acquisitions_round": 50,
            "init_question_num": 32,
            "acquire_question_num_per_round": 32,
            "warm_start_random_seed": 64,
            "sample_method": "No-Deterministic+randommm2Layer+64",
        },
        {
            "model_name": "BiLSTM",
            "group_name": "[tkde]BiLSTM+Pets+MRR+160+160",
            "max_performance": 0.80,
            "data_path":
            "../../datasets/answer_selection/YahooCQA/data/data-Pets/",
            "acquire_method": "random",
            "sub_acquire_method": "",
            "unsupervised_method": '',
            "submodular_k": 1.5,
            "num_acquisitions_round": 50,
            "init_question_num": 32,
            "acquire_question_num_per_round": 32,
            "warm_start_random_seed": 128,
            "sample_method": "No-Deterministic+randommm2Layer+128",
        },
    ]

    allMethods_results = [
    ]  #Record the performance results of each method during active learning

    for config in task_seq:

        print("-------------------{}-{}-------------------".format(
            config["group_name"], config["sample_method"]))

        ####################################### initial setting ###########################################
        data_path = config["data_path"]
        model_name = config[
            "model_name"] if "model_name" in config else 'BiLSTM'
        num_acquisitions_round = config["num_acquisitions_round"]
        acquire_method = config["acquire_method"]
        sub_acquire_method = config["sub_acquire_method"]
        init_question_num = config[
            "init_question_num"] if "init_question_num" in config else 160  # number of initial training samples
        acquire_question_num_per_round = config[
            "acquire_question_num_per_round"] if "acquire_question_num_per_round" in config else 20  #Number of samples collected per round
        warm_start_random_seed = config[
            "warm_start_random_seed"]  # the random seed for selecting the initial training set
        sample_method = config["sample_method"]

        loader = Loader()

        print('model:', model_name)
        print('dataset:', data_path)
        print('acquisition method:', acquire_method, "+", sub_acquire_method)

        if not os.path.exists(args.result_path):
            os.makedirs(args.result_path)

        if not os.path.exists(os.path.join(args.result_path, model_name)):
            os.makedirs(os.path.join(args.result_path, model_name))

        if not os.path.exists(
                os.path.join(args.result_path, model_name, 'active_checkpoint',
                             acquire_method)):
            os.makedirs(
                os.path.join(args.result_path, model_name, 'active_checkpoint',
                             acquire_method))

        #### If the data is not compiled, compile; otherwise load directly
        if (os.path.exists(os.path.join(data_path, 'mappings.pkl'))
                and os.path.exists(os.path.join(data_path, 'train.pkl'))
                and os.path.exists(os.path.join(data_path, 'test.pkl'))):
            mappings = pkl.load(
                open(os.path.join(data_path, 'mappings.pkl'), 'rb'))
            train_data = pkl.load(
                open(os.path.join(data_path, 'train.pkl'), 'rb'))
            test_data = pkl.load(
                open(os.path.join(data_path, 'test.pkl'), 'rb'))
        else:
            train_data, test_data, mappings = loader.load_yahoo(
                data_path, args.pretrained_word_embedding,
                args.word_embedding_dim, args.answer_count)
            pkl.dump(mappings,
                     open(os.path.join(data_path, 'mappings.pkl'), 'wb'))
            pkl.dump(train_data,
                     open(os.path.join(data_path, 'train.pkl'), 'wb'))
            pkl.dump(test_data, open(os.path.join(data_path, 'test.pkl'),
                                     'wb'))

        #word embedding
        word_to_id = mappings['word_to_id']
        tag_to_id = mappings['tag_to_id']
        word_embeds = mappings[
            'word_embeds'] if args.use_pretrained_word_embedding else None

        word_vocab_size = len(word_to_id)

        print(
            ' The total amount of training data:%d\n' % len(
                train_data
            ),  # Total number of training samples (number of question answer pair)
            'The total amount of val data:%d\n' % len(test_data))

        acquisition_function = Acquisition(train_data,
                                           seed=warm_start_random_seed,
                                           answer_count=args.answer_count,
                                           cuda_device=args.device[0],
                                           batch_size=args.sampling_batch_size,
                                           submodular_k=config["submodular_k"])

        checkpoint_path = os.path.join(args.result_path, 'active_checkpoint',
                                       config["group_name"], sample_method)
        if not os.path.exists(checkpoint_path):
            os.makedirs(checkpoint_path)

        method_result = [
        ]  # Record the performance results of each method during active learning
        ####################################### acquire data and retrain ###########################################
        for i in range(num_acquisitions_round):

            print("current round:{}".format(i))

            #-------------------acquisition---------------------
            if i == 0:  #first round
                acq = init_question_num
                a_m = "random"
                m_p = ""
            else:
                acq = acquire_question_num_per_round
                a_m = acquire_method
                m_p = os.path.join(checkpoint_path, 'modelweights')

            acquisition_function.obtain_data(
                train_data,
                model_path=m_p,
                model_name=model_name,
                acquire_questions_num=acq,
                method=a_m,
                sub_method=sub_acquire_method,
                unsupervised_method=config["unsupervised_method"])

            # -------------------prepare training data---------------------
            '''
            train_data format:
            {
                'str_words_q': str_words_q,  # question word segmentation
                'str_words_a': str_words_a,  # answer word segmentation
                'words_q': words_q,  # question word id
                'words_a': words_a,  # answer word id
                'tag': tag,  # sample tag id
            }
            '''

            new_train_index = (acquisition_function.train_index).copy()
            sorted_train_index = list(new_train_index)
            sorted_train_index.sort()
            labeled_train_data = [train_data[i] for i in sorted_train_index]

            print("Labeled training samples: {}".format(
                len(acquisition_function.train_index)))

            # -------------------------------------train--------------------------------------

            print('.............Recreate the model...................')
            if model_name == 'BiLSTM':
                model = BiLSTM(word_vocab_size,
                               args.word_embedding_dim,
                               args.word_hidden_dim,
                               args.target_size,
                               pretrained=word_embeds,
                               with_sim_features=args.with_sim_feature,
                               cuda_device=args.device[0])
            if model_name == 'CNN':
                model = CNN(word_vocab_size,
                            args.word_embedding_dim,
                            args.word_out_channels,
                            args.target_size,
                            pretrained=word_embeds,
                            cuda_device=args.device[0])

            model.cuda(args.device[0])

            trainer = Trainer(model,
                              model_name,
                              tag_to_id,
                              answer_count=args.answer_count,
                              cuda_device=args.device[0],
                              sampling_number=args.sampling_number)

            test_performance = trainer.train_supervisedLearning(
                args.num_epochs,
                labeled_train_data,
                test_data,
                args.learning_rate,
                checkpoint_path=checkpoint_path,
                batch_size=args.batch_size)

            print('.' * 50)
            print("Test performance: {}".format(test_performance))
            print('*' * 50)

            #--------------------------Send data for a visual web page------------------------------
            max_performance = config[
                "max_performance"] if "max_performance" in config else 0

            if "group_name" in config:
                updateLineChart(str(test_performance),
                                sample_method,
                                gp_name=config["group_name"],
                                max=max_performance)
            else:
                updateLineChart(str(test_performance),
                                sample_method,
                                max=max_performance)

        #     method_result.append(test_performance)
        #
        # print("acquire_method: {},sub_acquire_method: {}, warm_start_random_seed{}"
        #       .format(acquire_method, sub_acquire_method, warm_start_random_seed))
        # print(method_result)
        # allMethods_results.append(method_result)
        shutil.rmtree(checkpoint_path)
Example #12
0
class Uatu:
    """
    Uatu - he who watches
    """
    def __init__(self, config_file_path):
        """
        __init__ - self
        """
        LOGGER.info("Initializing UATU.")
        self.config_file = config_file_path
        self.cfg_organizer = ConfigOrganizer(config_file=self.config_file)
        self.cfg_organizer.read_config_data()
        self.acq_obj = Acquisition()
        self.current_max_count()
        LOGGER.info("Completed initialization.")
        self.img_processing = ImageProcessing(
            yolo_path=self.cfg_organizer.config_handler['system']['yolo_dir'])

    def __repr__(self):
        return "string"

    def current_max_count(self):
        """
        current_max_count - returns the current maximum count for the cameras
        """
        namelist = ['name', 'timestamp', 'count']

        df = pd.read_csv(
            self.cfg_organizer.config_handler['system']['csv_location'],
            index_col=False,
            names=namelist)

        df2 = df.fillna(0)
        camera_names = df['name'].unique()
        df2.sort_values(by=['name', 'count'], inplace=True)
        series = df2.groupby('name')['count'].max()
        self.stored_values = series.to_dict()

    def producer_images(self, pqueue, cqueue, lock):
        while True:
            if not pqueue.empty():
                camera_name = pqueue.get()
                image_name = '/tmp/{}-image.jpg'.format(camera_name)
                try:
                    self.acq_obj.retrieve(
                        self.cfg_organizer.config_handler[camera_name]['url'],
                        image_name)
                    with lock:
                        LOGGER.debug(f'retrieved image: {camera_name}')
                    cqueue.put((camera_name, image_name))
                except Exception as e:
                    with lock:
                        LOGGER.debug(f'exception: {e}')
        # pqueue.task_done()

    def consumer_process_image(self, cqueue, lock):
        counter = 1
        while True:
            if cqueue.empty():
                continue
            camera_name, image_name = cqueue.get()
            with lock:
                LOGGER.debug(f'processing camera: {camera_name}')
            try:
                with lock:
                    self.img_processing.load_file(
                        f'/tmp/{camera_name}-image.jpg')
            except IOError:
                with lock:
                    LOGGER.debug(f'yup - io error - skipping {camera_name}')
                return
            with lock:
                self.img_processing.preprocess_image()
            with lock:
                self.img_processing.process_bounding_boxes()
            processed_image = "/tmp/{}-{}-{}.jpg".format(
                camera_name, time.time(), self.img_processing.people_count)
            with lock:
                self.img_processing.output_adjusted_image(
                    '/tmp/what-{}.jpg'.format(counter))
            with lock:
                print("{},{},{},{}".format(camera_name, time.time(),
                                           self.img_processing.people_count,
                                           processed_image))
            if int(self.img_processing.people_count) > int(
                    self.stored_values[camera_name]):
                self.img_processing.output_adjusted_image(processed_image)

    def doit(self):
        self.current_max_count()

        pqueue = Queue(maxsize=0)
        cqueue = Queue(maxsize=0)

        lock = Lock()

        for i in range(NUM_WORKER_THREADS):
            pworker = threading.Thread(target=self.producer_images,
                                       args=(pqueue, cqueue, lock))
            pworker.daemon = True
            pworker.start()
            LOGGER.debug('initiated pworkers')
            cworker = threading.Thread(target=self.consumer_process_image,
                                       args=(cqueue, lock))
            cworker.daemon = True
            cworker.start()
            LOGGER.debug('initiated workers')

        cameras = self.cfg_organizer.find_cameras()
        for camera_name in cameras:
            LOGGER.debug(f'Loading {camera_name} into pqueue')
            pqueue.put(camera_name)

        cworker.join()

    def debug_dump(self):
        LOGGER.debug('Building camera information')
        for camera in self.cfg_organizer.find_cameras():
            LOGGER.debug(f'{camera}')
            LOGGER.debug('\t{}'.format(
                self.cfg_organizer.config_handler[camera]['url']))
Example #13
0
 def __init__(self, ws_host="", ws_port=5678):
     self.daq = Acquisition()
     self.ws_handler = WebSocketHandler(ws_host, ws_port, self.daq)
Example #14
0
def test_acquisition_retrieval():
    acquisition_obj = Acquisition()
    acquisition_obj.retrieve(
        "http://s1.twnmm.com/images/en_ca/icons/flags/all/16/ca.png", "ca.png")
    assert os.path.exists("ca.png") == True
Example #15
0
def test_acquisition_creation():
    acquisition_obj = Acquisition()
    assert acquisition_obj is not None