Esempio n. 1
0
    def generate(self,N,keep=True):
        templabels = Labels(components=self.components,weights=self.weights,N=N)

        out = np.empty(self.components[0].rvs(N).shape)
        counts = np.bincount(templabels.z,minlength=len(self.components))
        for idx,(c,count) in enumerate(zip(self.components,counts)):
            out[templabels.z == idx,...] = c.rvs(count)

        perm = np.random.permutation(N)
        out = out[perm]
        templabels.z = templabels.z[perm]

        if keep:
            templabels.data = out
            self.labels_list.append(templabels)

        return out, templabels.z
Esempio n. 2
0
    def generate(self, N, keep=True):
        templabels = Labels(components=self.components,
                            weights=self.weights,
                            N=N)

        out = np.empty(self.components[0].rvs(N).shape)
        counts = np.bincount(templabels.z, minlength=len(self.components))
        for idx, (c, count) in enumerate(zip(self.components, counts)):
            out[templabels.z == idx, ...] = c.rvs(count)

        perm = np.random.permutation(N)
        out = out[perm]
        templabels.z = templabels.z[perm]

        if keep:
            templabels.data = out
            self.labels_list.append(templabels)

        return out, templabels.z
Esempio n. 3
0
def starter(feature_model, dimension_reduction, k, label_choice):
    label, value, _ = Labels(label_choice).label
    filtered_image_ids = [
        item["image_id"]
        for item in Database().retrieve_metadata_with_labels(label, value)
    ]

    _, latent_symantics = functions.store_in_db(feature_model,
                                                dimension_reduction, k, 3,
                                                filtered_image_ids, label,
                                                value)

    print(latent_symantics)
Esempio n. 4
0
def findlabel(feature_model, dimension_reduction, k, label_choice, image_id):
    descriptor_type = DescriptorType(feature_model).descriptor_type
    symantics_type = LatentSymanticsType(dimension_reduction).symantics_type
    label, value, complementary_value = Labels(label_choice).label

    source = Database().retrieve_one(image_id, 5)
    label_targets = Database().retrieve_many(5, label, value)
    complementary_label_targets = Database().retrieve_many(
        5, label, complementary_value)

    label_similarity_info = functions.compare(source, label_targets, 1,
                                              descriptor_type)
    complementary_label_similarity_info = functions.compare(
        source, complementary_label_targets, 1, descriptor_type)

    if label_similarity_info[0][1] > complementary_label_similarity_info[0][1]:
        predicted = Labels(label_choice)._detupleize_label((label, value))
    else:
        predicted = Labels(label_choice)._detupleize_label(
            (label, complementary_value))

    print(predicted)
Esempio n. 5
0
 def login(self, email, password):
     """
     Login user to Producteev.
     """
     response = self.__request(
                     'users/login',
                     email=email,
                     password=password)
     self.token = response['login']['token']
     self.users = Users(self)
     self.tasks = Tasks(self)
     self.dashboards = Dashboards(self)
     self.labels = Labels(self)
     self.activities = Activities(self)
     self.subtasks = Subtasks(self)
Esempio n. 6
0
def starter(feature_model, dimension_reduction, k, label_choice, image_id, m):
    if not feature_model:
        target_results = Database().retrieve_many(task=3)
        source_result = Database().retrieve_one(image_id, task=3)
        descriptor_type = source_result["descriptor_type"]
    else:
        label, value, _ = Labels(label_choice).label
        filtered_image_ids = [
            item["image_id"]
            for item in Database().retrieve_metadata_with_labels(label, value)
        ]
        _, _ = functions.store_in_db(feature_model, dimension_reduction, k, 4,
                                     filtered_image_ids, label, value)

        target_results = Database().retrieve_many(task=4)
        source_result = Database().retrieve_one(image_id, task=4)
        descriptor_type = source_result["descriptor_type"]

    print(functions.compare(source_result, target_results, m, descriptor_type))
Esempio n. 7
0
    def __init__(self,
                 config,
                 image_dir=None,
                 labels_json=None,
                 mode=None,
                 group=None,
                 class_counts=None):
        self.log = logging.getLogger('root')
        self.config = config
        self.image_dir = tools.str2path(image_dir)
        self.labels_json = tools.str2path(labels_json)
        self.mode = mode
        self.group = group
        self.channels = config.get('channels', 3)
        self._batch_size = config.get('flow', {}).get('batch_size', 4)
        self._validate_input()
        self.color_mode = 'rgb' if self.channels == 3 else 'grayscale'
        self.target_imsize = (self.config['target_height'],
                              self.config['target_width'])
        self.blacklist = self.config.get('class_blacklist', [])
        self.class_counts = class_counts

        if labels_json:
            self.keras_dir = self.image_dir.parent / f'{self.image_dir.name}_{self.group}_{str(uuid.uuid1())[:8]}'
            if not self.keras_dir.exists():
                self.keras_dir.mkdir()
                self.log.info(
                    f'Creating Keras Directory Tree - {self.mode}...')
                tools.create_keras_image_directory_tree(
                    self.image_dir, self.keras_dir, self.labels_json,
                    self.group, self.blacklist, self.class_counts)
            else:
                self.log.info(
                    'Skipped Keras Directory Tree creation as it already exists'
                )

            self.labels = Labels(labels_json, self.keras_dir, group)
        else:
            self.keras_dir = image_dir

        self._data_generator = self._init_data_gen()
        self._flow_gen = self._flow_from_directory()
Esempio n. 8
0
def auto_shift(snd_path, labels_path):
    '''
    自動ずれ補正
    '''

    snd = pausewave.open(snd_path, 'rb')
    labels = Labels(labels_path)

    if labels.dist_s == NO_DISTINCTION:
        return

    d = find_dist_s(snd)
    auto_shift_s = auto_shift_diff_s(d, labels.dist_s)

    if auto_shift_s != 0:
        labels.dist_s = d
        labels.shift(auto_shift_s)
        labels.write(labels_path)
        base_name = os.path.basename(snd_path)
        print 'auto shift: %.6fs %s' % (auto_shift_s, base_name)
Esempio n. 9
0
    def LoadLabels(self, f, snd):
        '''
        ラベル情報ファイルを読み込む。なければ作成する
        '''

        if os.path.exists(f):
            labels = Labels(f)
        else:
            sil_lv = self.conf.sil_lv
            sil_dur = self.conf.sil_dur_s
            before_dur = self.conf.before_dur_s
            after_dur = self.conf.after_dur_s

            vol = Volume(snd, FIND_RATE)
            labels = find_sound(vol, sil_lv, sil_dur, before_dur, after_dur)

            labels.write(f)

        if not hasattr(labels, 'dist_s') or labels.dist_s == NO_DISTINCTION:
            labels.dist_s = find_dist_s(snd)
            labels.write(f)

        return labels
Esempio n. 10
0
    #md.count()

    #from booth.LabelFirstYear import LabelFirstYear
    #lfy = LabelFirstYear()
    #lfy.do_xlsx()


    # from booth.MQ_network_mod import MQ_network_mod
    # mq_network_mod = MQ_network_mod()
    #mq_network_mod.do_csv()
    #mq_network_mod.do_by_word_csv()
    #mq_network_mod.get_word_list()
    #mq_network_mod.update_word_space()

    from labels import Labels
    l = Labels()
    #l.fill_label_table()
    #l.doc_title_get_word_vector()
    l.doc_title_shorten_word_vector()
    l.mq_title_shorten_word_vector()
    #l.update_super_label()
    #l.update_mq_id()
    #l.update_labelid_first()
    #l.update_labelid_geo()
    #l.update_labelid_vert()
    #l.update_labelid_mkt()
    #l.check_hr()
    #l.update_labelid()
    # l.update_flag_in_mq_table()
    # l.update_numeric_docid()
    # l.remove_invalid_symbols()
Esempio n. 11
0
def find_sound(vol, sil_lv=SIL_LV, sil_dur_s=SIL_DUR_S,
               before_dur_s=BEFORE_DUR_S, after_dur_s=AFTER_DUR_S):
    '''
    音がある部分にポーズラベルをつける

    based on the Sound Finder Audacity script
    by Jeremy R. Brown
    Sound Finder based on the Silence Finder Audacity script
    by Alex S. Brown

    @param sil_lv 無音認識レベル(%)。最大音に対して何%以下を無音と判定するか
    @param sil_dur_s 無音認識時間(秒)
    @param before_dur_s ラベルの前に余裕をもたせる秒数
    @param after_dur_s ラベルの後に余裕をもたせる秒数

    @return ラベル
    '''

    dur_s = vol.dur_s

    sil_lv = max(0, min(sil_lv, 100))

    max_val = max(vol)
    thres = sil_lv * max_val / 100  # 無音判定レベル
    sil_f = sil_dur_s * vol.rate  # 無音判定フレーム数
    snd_f = SND_DUR_S * vol.rate  # 音あり判定フレーム数
    sil_c = 0  # 無音カウンタ
    snd_c = 0  # 音ありカウンタ
    sil_start = -1  # 無音開始位置
    snd_start = -1  # 音あり開始位置
    is_first_snd = True  # 音が見つかった最初の時のみTrue
    labels = Labels()

    for n, v in enumerate(vol):
        if v <= thres:  # 無音
            sil_c = sil_c + 1
            if sil_start == -1:
                sil_start = n
            elif (not is_first_snd) and (snd_start != -1) and \
                    (sil_c > sil_f) and (snd_c > snd_f):

                max_f = sil_start + int(after_dur_s * vol.rate)
                sil_f = _get_sil_f(vol, sil_start, thres, max_f)
                start_time = (float(snd_start) / vol.rate) - before_dur_s
                start_time = max(0, min(start_time, dur_s))
                end_time = float(sil_start + sil_f) / vol.rate
                end_time = max(0, min(end_time, dur_s))

                try:
                    labels.append(Label(start_time, end_time))
                except Exception as e:
                    print str(e)

                is_first_snd = True
                sil_c = 0
                snd_c = 0
                sil_start = -1
        else:  # 音あり
            snd_c = snd_c + 1
            if is_first_snd:
                is_first_snd = False
                snd_start = n
            sil_c = 0
            sil_start = -1

    if not is_first_snd:
        start_time = float(snd_start) / vol.rate - before_dur_s
        start_time = max(0, min(start_time, dur_s))
        end_time = float(sil_start) / vol.rate + after_dur_s
        end_time = max(0, min(end_time, dur_s))

        if end_time < start_time:
            end_time = dur_s

        try:
            labels.append(Label(start_time, end_time))
        except Exception as e:
            print str(e)

    labels.subtract()

    return labels
Esempio n. 12
0
def helper(feature_model, dimension_reduction, k, label_choice, image_id):
    path, pos = Config().read_path(), None
    descriptor_type = DescriptorType(feature_model).descriptor_type
    symantics_type = LatentSymanticsType(dimension_reduction).symantics_type
    label, value, complementary_value = Labels(label_choice).label

    image = cv2.imread("{}{}{}".format(Config().read_all_path(), image_id,
                                       ".jpg"))
    image_feature_vector = Descriptor(image, feature_model,
                                      dimension_reduction).feature_descriptor

    label_filtered_image_ids = [
        item["image_id"]
        for item in Database().retrieve_metadata_with_labels(label, value)
    ]
    complementary_label_filtered_image_ids = [
        item["image_id"] for item in Database().retrieve_metadata_with_labels(
            label, complementary_value)
    ]

    if DescriptorType(feature_model).check_sift():
        label_feature_vector, label_ids, label_pos = functions.process_files(
            path, feature_model, dimension_reduction, label_filtered_image_ids)
        complementary_label_feature_vector, complementary_label_ids, complementary_label_pos = functions.process_files(
            path,
            feature_model,
            dimension_reduction,
            complementary_label_filtered_image_ids,
        )
        feature_vector = np.concatenate((
            label_feature_vector,
            complementary_label_feature_vector,
            image_feature_vector,
        ))
        pos = label_pos + complementary_label_pos + [
            image_feature_vector.shape[0]
        ]
    else:
        label_feature_vector, label_ids = functions.process_files(
            path, feature_model, dimension_reduction, label_filtered_image_ids)
        complementary_label_feature_vector, complementary_label_ids = functions.process_files(
            path,
            feature_model,
            dimension_reduction,
            complementary_label_filtered_image_ids,
        )
        feature_vector = np.concatenate((
            label_feature_vector,
            complementary_label_feature_vector,
            np.array([image_feature_vector]),
        ))

    ids = label_ids + complementary_label_ids + [image_id]

    _, latent_symantics = LatentSymantics(feature_vector, k,
                                          dimension_reduction).latent_symantics

    records = functions.set_records(ids, descriptor_type, symantics_type, k,
                                    latent_symantics, pos, 5)

    for record in records:
        if record["image_id"] == image_id:
            continue
        elif record["image_id"] in label_ids:
            record[label] = value
        elif record["image_id"] in complementary_label_ids:
            record[label] = complementary_value

    Database().insert_many(records)
Esempio n. 13
0
    if "order" not in instructions[err.inst_order].attributes:
        err.exit_script(err.lexical_or_syntax)

# Sort instruction elements by order number
instructions.sort(key=lambda x: int(x.attributes["order"].value))

# Check for valid order sequence
order_check = 1
for inst in instructions:
    if order_check != int(inst.attributes["order"].value):
        err.exit_script(err.lexical_or_syntax)
    order_check += 1

### Interpretation ###
# 1st passing of all instructions to define labels
labels = Labels()
err.inst_order = 0
for inst_order in range(0, len(instructions)):
    err.inst_order += 1
    inst = Instruction(instructions[inst_order])
    if inst.opcode == "LABEL":
        labels.add(inst.arg1, inst_order)

# 2nd passing of all instructions to interprete them
inst_order = 0
inst_exectuted = 0
err.inst_order = 0
symtable = SymTable()

while inst_order < len(instructions):
    err.inst_order += 1
Esempio n. 14
0
 def html5(self):
     subprocess.Popen(['python', '/usr/share/html5.py'])
     self.label.setText(label.html5())
Esempio n. 15
0
 def printme(self):
     self.label.setText(label.printer())
Esempio n. 16
0
    def __init__(self):

        self.labels = Labels()
Esempio n. 17
0
 def loadings(self):
     self.label.setText(label.loadings())
Esempio n. 18
0
def main():
    if len(sys.argv) != 17:
        print(
            "USAGE: ./prepare_table_files.py table_var_file table_ann_file table_var2ann_file table_ann_desc_file table_back_set_file "
            +
            "table_ann2back_set_file table_var2back_set_file tables_row_counts_file MSigDB_files MSigDB_not_found_HGNC_file"
            +
            "var_annotations_file genes_annotations_file get_evidence_file back_set_not_counted_file back_sets_folder top_alleles"
        )
        print(
            "Use varanto_import.sh and varanto_import.conf for easier setting of parameters."
        )
        sys.exit(-1)

    params = Params(sys.argv)

    print("Generating input data files...")
    start = datetime.now()

    #initialize counters of annotations in background lists and load background set description file
    print("* Loading and counting variations from background sets files...")
    back_sets = BackgroundSets(params.back_set_not_counted_file,
                               params.back_sets_folder)
    print(
        "* Loading and counting variations from background sets files completed at "
        + str(datetime.now() - start))

    headers = [
        "refsnp_id", "allele", "chr_name", "chrom_start", "chrom_strand",
        "phenotype_description", "study_external_ref", "study_description",
        "consequence_type_tv", "ensembl_gene_stable_id", "associated_gene",
        "polyphen_prediction", "sift_prediction"
    ]

    #initialize labels
    print("* Loading data for annotation processing...")
    labels = Labels(headers, back_sets.get_back_sets_ids(),
                    params.table_ann_desc_file, params.genes_annotations_file,
                    params.get_evidence_file, params.top_alleles,
                    params.msigdb_files, params.msigdb_not_found_hgnc)
    print("* Loading data for annotation processing completed at " +
          str(datetime.now() - start))

    #open annotated snps file
    with open(params.var_annotations_file,
              'r', encoding='latin-1') as input_var, open(
                  params.table_var_file, 'w') as variations_output, open(
                      params.table_ann_file, 'w') as annotations_output, open(
                          params.table_var2ann_file,
                          'w') as var2ann_output, open(
                              params.table_var2back_set_file,
                              'w') as var2back_set_output:

        #initialize csv reader and writers
        reader = csv.reader(input_var, delimiter='\t')
        variations_csv = csv.writer(variations_output, delimiter='\t')
        annotations_csv = csv.writer(annotations_output,
                                     delimiter='\t',
                                     quoting=csv.QUOTE_NONE,
                                     escapechar='\\')
        var2ann_csv = csv.writer(var2ann_output, delimiter='\t')
        var2back_set_csv = csv.writer(var2back_set_output, delimiter='\t')

        #current name of variant and status of change
        current_variant = None
        #current id of variant
        variation_id_cnt = 0
        #for outputting progress
        last_time = datetime.now()
        rows_processed = 0
        #process each file
        for row in reader:
            #check if it is another snp than snp in the previous line
            if row[0] != current_variant:
                #check if it is variant in standard track (according to chromosome name)
                if not row[2] in [
                        "1", "2", "3", "4", "5", "6", "7", "8", "9", "10",
                        "11", "12", "13", "14", "15", "16", "17", "18", "19",
                        "20", "21", "22", "MT", "X", "Y"
                ]:
                    continue
                #add row in variations csv
                current_variant = row[0]
                variation_id_cnt += 1
                #write variation
                variations_csv.writerow(
                    [variation_id_cnt, row[0], row[4], row[3], row[1], row[2]])
                #list of background sets ids which contain variation
                var_is_in_back_sets = back_sets.process_variant(
                    current_variant, variation_id_cnt, var2back_set_csv)

            labels.process_variation_row(variation_id_cnt, var_is_in_back_sets,
                                         row, annotations_csv, var2ann_csv)

            rows_processed += 1

            if rows_processed % 10000 == 0:
                sys.stdout.write(
                    "\rLines processed: %d, time of processing last 10000 rows: "
                    % rows_processed)
                sys.stdout.write(str(datetime.now() - last_time))
                sys.stdout.flush()
                last_time = datetime.now()

        #process annotations dependent on the data gained by processing of all variation rows (top alleles)
        labels.process_additional_annotation(annotations_csv, var2ann_csv)

        sys.stdout.write("\n")

    #write to file descriptions of background sets with counted variations
    back_sets.write_table_back_sets_file(variation_id_cnt,
                                         params.table_back_set_file)

    #write associations between annotations and their counts in all background sets
    labels.write_annotation_counts(params.table_ann2back_set_file)

    #write counts of rows for db tables with primary keys sequences to increment them.
    with open(params.tables_row_counts_file, 'w') as out:
        writer = csv.writer(out, delimiter='\t')
        writer.writerow([
            variation_id_cnt, labels.annotation_id_cnt,
            labels.ann_descriptions.id_count(),
            len(back_sets.get_back_sets_ids())
        ])

    print("Generating input files completed in " + str(datetime.now() - start))
Esempio n. 19
0
 def add_data(self, data, **kwargs):
     self.labels_list.append(
         Labels(data=np.asarray(data),
                components=self.components,
                weights=self.weights,
                **kwargs))
Esempio n. 20
0
 def begin(self):
     self.label.setText(label.begin())
Esempio n. 21
0
 def notice_new_bookmark(self):
     self.label.setText(label.bookmarked())
Esempio n. 22
0
 def complete(self):
     self.label.setText(label.complete())
Esempio n. 23
0
class VOC2COCO:
    def __init__(self):

        self.labels = Labels()

    def generateVOC2Json(self, xmlFiles):
        attrDict = dict()

        attrDict["categories"] = [self.labels.create_attrDict()]

        images = list()
        annotations = list()

        image_id = 0

        for file_ in xmlFiles:
            image_id = image_id + 1

            annotation_path = file_
            image = dict()

            doc = xmltodict.parse(open(annotation_path).read())

            image['file_name'] = str(doc['annotation']['filename'])
            image['height'] = int(doc['annotation']['size']['height'])
            image['width'] = int(doc['annotation']['size']['width'])
            image['id'] = image_id
            #print ("File Name: {} and image_id {}".format(file_, image_id))
            images.append(image)

            id1 = 1

            if 'object' in doc['annotation']:
                for obj in doc['annotation']['object']:  #48 boxes

                    for value in attrDict["categories"]:
                        #print(value)
                        annotation = dict()
                        obj = to_dict(obj)
                        for val in value:
                            if str(obj['name']) == val["name"]:
                                #print str(obj['name'])
                                #annotation["segmentation"] = []
                                annotation["iscrowd"] = 0
                                #annotation["image_id"] = str(doc['annotation']['filename']).split('.jpg')[0] #attrDict["images"]["id"]
                                annotation["image_id"] = image_id
                                x1 = float(obj["bndbox"]["xmin"])  # - 1
                                y1 = float(obj["bndbox"]["ymin"])  # - 1
                                x2 = float(obj["bndbox"]["xmax"])  # - x1
                                y2 = float(obj["bndbox"]["ymax"])  # - y1
                                annotation["bbox"] = [x1, y1, x2, y2]
                                annotation["area"] = float(x2 * y2)
                                annotation["category_id"] = val["id"]
                                annotation["ignore"] = 0
                                annotation["id"] = id1
                                annotation["segmentation"] = [[
                                    x1, y1, x1, (y1 + y2), (x1 + x2),
                                    (y1 + y2), (x1 + x2), y1
                                ]]
                                id1 += 1

                                annotations.append(annotation)

            else:
                print("{} doesn't have any object".format(file_))
            #image_id = image_id + 1
            #print(len(annotations))

        attrDict["images"] = images
        attrDict["annotations"] = annotations
        attrDict["type"] = "instances"

        attrDict['categories'] = attrDict['categories'][0]
        classes = {}
        for o in attrDict['categories']:
            classes[o['id']] = o['name']
        #print attrDict
        jsonString = json.dumps(attrDict)
        with open("receipts_valid.json", "w") as f:
            f.write(jsonString)

    def main(self, trainXMLFiles="path/to/annotation/xml/files"):

        self.generateVOC2Json(trainXMLFiles)
Esempio n. 24
0
    def system_arguments(self):
        if len(sys.argv) == 1:
            url = self.address.text()
            if url.startswith('http://'):
                change = str(url)
                self.address.setText(change)
                load = self.address.text()
                self.label.setText(load)
                self.web.load(QUrl(load))
                del sys.argv[1:]

            elif url.startswith('https://'):
                change = str(url)
                self.address.setText(change)
                load = self.address.text()
                self.label.setText(load)
                self.web.load(QUrl(load))
                del sys.argv[1:]

            elif url.startswith('find:'):
                load = self.address.text()
                self.result0 = load.replace('find:', '')
                print "Finding:" + self.result0
                self.web.findText(self.result0)
                self.label.setText(label.finding())
                del sys.argv[1:]

            else:
                add = "https://" + url
                change = str(add)
                self.address.setText(change)
                load = self.address.text()
                self.label.setText(load)
                self.web.load(QUrl(load))
                del sys.argv[1:]

        else:
            self.location = sys.argv[1:]
            self.temp = str(self.location)
            self.result0 = self.temp.replace('[', '')
            self.result1 = self.result0.replace(']', '')
            self.result_final = self.result1.replace("'", '')

            url = self.result_final
            if url.startswith('http://'):
                change = str(url)
                self.address.setText(change)
                load = self.address.text()
                self.label.setText(load)
                self.loading1 = self.web.load(QUrl(load))
                del sys.argv[1:]

            elif url.startswith('https://'):
                change = str(url)
                self.address.setText(change)
                load = self.address.text()
                self.label.setText(load)
                self.loading1 = self.web.load(QUrl(load))
                del sys.argv[1:]

            else:
                change = str("https://" + url)
                self.address.setText(change)
                load = self.address.text()
                self.label.setText(load)
                self.web.load(QUrl(load))
                del sys.argv[1:]