def generate(self,N,keep=True):
templabels = Labels(components=self.components,weights=self.weights,N=N)
out = np.empty(self.components[0].rvs(N).shape)
counts = np.bincount(templabels.z,minlength=len(self.components))
for idx,(c,count) in enumerate(zip(self.components,counts)):
out[templabels.z == idx,...] = c.rvs(count)
perm = np.random.permutation(N)
out = out[perm]
templabels.z = templabels.z[perm]
if keep:
templabels.data = out
self.labels_list.append(templabels)
return out, templabels.z
def generate(self, N, keep=True):
templabels = Labels(components=self.components,
weights=self.weights,
N=N)
out = np.empty(self.components[0].rvs(N).shape)
counts = np.bincount(templabels.z, minlength=len(self.components))
for idx, (c, count) in enumerate(zip(self.components, counts)):
out[templabels.z == idx, ...] = c.rvs(count)
perm = np.random.permutation(N)
out = out[perm]
templabels.z = templabels.z[perm]
if keep:
templabels.data = out
self.labels_list.append(templabels)
return out, templabels.z
def starter(feature_model, dimension_reduction, k, label_choice):
label, value, _ = Labels(label_choice).label
filtered_image_ids = [
item["image_id"]
for item in Database().retrieve_metadata_with_labels(label, value)
]
_, latent_symantics = functions.store_in_db(feature_model,
dimension_reduction, k, 3,
filtered_image_ids, label,
value)
print(latent_symantics)
def findlabel(feature_model, dimension_reduction, k, label_choice, image_id):
descriptor_type = DescriptorType(feature_model).descriptor_type
symantics_type = LatentSymanticsType(dimension_reduction).symantics_type
label, value, complementary_value = Labels(label_choice).label
source = Database().retrieve_one(image_id, 5)
label_targets = Database().retrieve_many(5, label, value)
complementary_label_targets = Database().retrieve_many(
5, label, complementary_value)
label_similarity_info = functions.compare(source, label_targets, 1,
descriptor_type)
complementary_label_similarity_info = functions.compare(
source, complementary_label_targets, 1, descriptor_type)
if label_similarity_info[0][1] > complementary_label_similarity_info[0][1]:
predicted = Labels(label_choice)._detupleize_label((label, value))
else:
predicted = Labels(label_choice)._detupleize_label(
(label, complementary_value))
print(predicted)
def login(self, email, password):
"""
Login user to Producteev.
"""
response = self.__request(
'users/login',
email=email,
password=password)
self.token = response['login']['token']
self.users = Users(self)
self.tasks = Tasks(self)
self.dashboards = Dashboards(self)
self.labels = Labels(self)
self.activities = Activities(self)
self.subtasks = Subtasks(self)
def starter(feature_model, dimension_reduction, k, label_choice, image_id, m):
if not feature_model:
target_results = Database().retrieve_many(task=3)
source_result = Database().retrieve_one(image_id, task=3)
descriptor_type = source_result["descriptor_type"]
else:
label, value, _ = Labels(label_choice).label
filtered_image_ids = [
item["image_id"]
for item in Database().retrieve_metadata_with_labels(label, value)
]
_, _ = functions.store_in_db(feature_model, dimension_reduction, k, 4,
filtered_image_ids, label, value)
target_results = Database().retrieve_many(task=4)
source_result = Database().retrieve_one(image_id, task=4)
descriptor_type = source_result["descriptor_type"]
print(functions.compare(source_result, target_results, m, descriptor_type))
def __init__(self,
config,
image_dir=None,
labels_json=None,
mode=None,
group=None,
class_counts=None):
self.log = logging.getLogger('root')
self.config = config
self.image_dir = tools.str2path(image_dir)
self.labels_json = tools.str2path(labels_json)
self.mode = mode
self.group = group
self.channels = config.get('channels', 3)
self._batch_size = config.get('flow', {}).get('batch_size', 4)
self._validate_input()
self.color_mode = 'rgb' if self.channels == 3 else 'grayscale'
self.target_imsize = (self.config['target_height'],
self.config['target_width'])
self.blacklist = self.config.get('class_blacklist', [])
self.class_counts = class_counts
if labels_json:
self.keras_dir = self.image_dir.parent / f'{self.image_dir.name}_{self.group}_{str(uuid.uuid1())[:8]}'
if not self.keras_dir.exists():
self.keras_dir.mkdir()
self.log.info(
f'Creating Keras Directory Tree - {self.mode}...')
tools.create_keras_image_directory_tree(
self.image_dir, self.keras_dir, self.labels_json,
self.group, self.blacklist, self.class_counts)
else:
self.log.info(
'Skipped Keras Directory Tree creation as it already exists'
)
self.labels = Labels(labels_json, self.keras_dir, group)
else:
self.keras_dir = image_dir
self._data_generator = self._init_data_gen()
self._flow_gen = self._flow_from_directory()
def auto_shift(snd_path, labels_path):
'''
自動ずれ補正
'''
snd = pausewave.open(snd_path, 'rb')
labels = Labels(labels_path)
if labels.dist_s == NO_DISTINCTION:
return
d = find_dist_s(snd)
auto_shift_s = auto_shift_diff_s(d, labels.dist_s)
if auto_shift_s != 0:
labels.dist_s = d
labels.shift(auto_shift_s)
labels.write(labels_path)
base_name = os.path.basename(snd_path)
print 'auto shift: %.6fs %s' % (auto_shift_s, base_name)
def LoadLabels(self, f, snd):
'''
ラベル情報ファイルを読み込む。なければ作成する
'''
if os.path.exists(f):
labels = Labels(f)
else:
sil_lv = self.conf.sil_lv
sil_dur = self.conf.sil_dur_s
before_dur = self.conf.before_dur_s
after_dur = self.conf.after_dur_s
vol = Volume(snd, FIND_RATE)
labels = find_sound(vol, sil_lv, sil_dur, before_dur, after_dur)
labels.write(f)
if not hasattr(labels, 'dist_s') or labels.dist_s == NO_DISTINCTION:
labels.dist_s = find_dist_s(snd)
labels.write(f)
return labels
#md.count()
#from booth.LabelFirstYear import LabelFirstYear
#lfy = LabelFirstYear()
#lfy.do_xlsx()
# from booth.MQ_network_mod import MQ_network_mod
# mq_network_mod = MQ_network_mod()
#mq_network_mod.do_csv()
#mq_network_mod.do_by_word_csv()
#mq_network_mod.get_word_list()
#mq_network_mod.update_word_space()
from labels import Labels
l = Labels()
#l.fill_label_table()
#l.doc_title_get_word_vector()
l.doc_title_shorten_word_vector()
l.mq_title_shorten_word_vector()
#l.update_super_label()
#l.update_mq_id()
#l.update_labelid_first()
#l.update_labelid_geo()
#l.update_labelid_vert()
#l.update_labelid_mkt()
#l.check_hr()
#l.update_labelid()
# l.update_flag_in_mq_table()
# l.update_numeric_docid()
# l.remove_invalid_symbols()
def find_sound(vol, sil_lv=SIL_LV, sil_dur_s=SIL_DUR_S,
before_dur_s=BEFORE_DUR_S, after_dur_s=AFTER_DUR_S):
'''
音がある部分にポーズラベルをつける
based on the Sound Finder Audacity script
by Jeremy R. Brown
Sound Finder based on the Silence Finder Audacity script
by Alex S. Brown
@param sil_lv 無音認識レベル(%)。最大音に対して何％以下を無音と判定するか
@param sil_dur_s 無音認識時間（秒）
@param before_dur_s ラベルの前に余裕をもたせる秒数
@param after_dur_s ラベルの後に余裕をもたせる秒数
@return ラベル
'''
dur_s = vol.dur_s
sil_lv = max(0, min(sil_lv, 100))
max_val = max(vol)
thres = sil_lv * max_val / 100 # 無音判定レベル
sil_f = sil_dur_s * vol.rate # 無音判定フレーム数
snd_f = SND_DUR_S * vol.rate # 音あり判定フレーム数
sil_c = 0 # 無音カウンタ
snd_c = 0 # 音ありカウンタ
sil_start = -1 # 無音開始位置
snd_start = -1 # 音あり開始位置
is_first_snd = True # 音が見つかった最初の時のみTrue
labels = Labels()
for n, v in enumerate(vol):
if v <= thres: # 無音
sil_c = sil_c + 1
if sil_start == -1:
sil_start = n
elif (not is_first_snd) and (snd_start != -1) and \
(sil_c > sil_f) and (snd_c > snd_f):
max_f = sil_start + int(after_dur_s * vol.rate)
sil_f = _get_sil_f(vol, sil_start, thres, max_f)
start_time = (float(snd_start) / vol.rate) - before_dur_s
start_time = max(0, min(start_time, dur_s))
end_time = float(sil_start + sil_f) / vol.rate
end_time = max(0, min(end_time, dur_s))
try:
labels.append(Label(start_time, end_time))
except Exception as e:
print str(e)
is_first_snd = True
sil_c = 0
snd_c = 0
sil_start = -1
else: # 音あり
snd_c = snd_c + 1
if is_first_snd:
is_first_snd = False
snd_start = n
sil_c = 0
sil_start = -1
if not is_first_snd:
start_time = float(snd_start) / vol.rate - before_dur_s
start_time = max(0, min(start_time, dur_s))
end_time = float(sil_start) / vol.rate + after_dur_s
end_time = max(0, min(end_time, dur_s))
if end_time < start_time:
end_time = dur_s
try:
labels.append(Label(start_time, end_time))
except Exception as e:
print str(e)
labels.subtract()
return labels
def helper(feature_model, dimension_reduction, k, label_choice, image_id):
path, pos = Config().read_path(), None
descriptor_type = DescriptorType(feature_model).descriptor_type
symantics_type = LatentSymanticsType(dimension_reduction).symantics_type
label, value, complementary_value = Labels(label_choice).label
image = cv2.imread("{}{}{}".format(Config().read_all_path(), image_id,
".jpg"))
image_feature_vector = Descriptor(image, feature_model,
dimension_reduction).feature_descriptor
label_filtered_image_ids = [
item["image_id"]
for item in Database().retrieve_metadata_with_labels(label, value)
]
complementary_label_filtered_image_ids = [
item["image_id"] for item in Database().retrieve_metadata_with_labels(
label, complementary_value)
]
if DescriptorType(feature_model).check_sift():
label_feature_vector, label_ids, label_pos = functions.process_files(
path, feature_model, dimension_reduction, label_filtered_image_ids)
complementary_label_feature_vector, complementary_label_ids, complementary_label_pos = functions.process_files(
path,
feature_model,
dimension_reduction,
complementary_label_filtered_image_ids,
)
feature_vector = np.concatenate((
label_feature_vector,
complementary_label_feature_vector,
image_feature_vector,
))
pos = label_pos + complementary_label_pos + [
image_feature_vector.shape[0]
]
else:
label_feature_vector, label_ids = functions.process_files(
path, feature_model, dimension_reduction, label_filtered_image_ids)
complementary_label_feature_vector, complementary_label_ids = functions.process_files(
path,
feature_model,
dimension_reduction,
complementary_label_filtered_image_ids,
)
feature_vector = np.concatenate((
label_feature_vector,
complementary_label_feature_vector,
np.array([image_feature_vector]),
))
ids = label_ids + complementary_label_ids + [image_id]
_, latent_symantics = LatentSymantics(feature_vector, k,
dimension_reduction).latent_symantics
records = functions.set_records(ids, descriptor_type, symantics_type, k,
latent_symantics, pos, 5)
for record in records:
if record["image_id"] == image_id:
continue
elif record["image_id"] in label_ids:
record[label] = value
elif record["image_id"] in complementary_label_ids:
record[label] = complementary_value
Database().insert_many(records)
if "order" not in instructions[err.inst_order].attributes:
err.exit_script(err.lexical_or_syntax)
# Sort instruction elements by order number
instructions.sort(key=lambda x: int(x.attributes["order"].value))
# Check for valid order sequence
order_check = 1
for inst in instructions:
if order_check != int(inst.attributes["order"].value):
err.exit_script(err.lexical_or_syntax)
order_check += 1
### Interpretation ###
# 1st passing of all instructions to define labels
labels = Labels()
err.inst_order = 0
for inst_order in range(0, len(instructions)):
err.inst_order += 1
inst = Instruction(instructions[inst_order])
if inst.opcode == "LABEL":
labels.add(inst.arg1, inst_order)
# 2nd passing of all instructions to interprete them
inst_order = 0
inst_exectuted = 0
err.inst_order = 0
symtable = SymTable()
while inst_order < len(instructions):
err.inst_order += 1
def html5(self):
subprocess.Popen(['python', '/usr/share/html5.py'])
self.label.setText(label.html5())
def printme(self):
self.label.setText(label.printer())
def __init__(self):
self.labels = Labels()
def loadings(self):
self.label.setText(label.loadings())
def main():
if len(sys.argv) != 17:
print(
"USAGE: ./prepare_table_files.py table_var_file table_ann_file table_var2ann_file table_ann_desc_file table_back_set_file "
+
"table_ann2back_set_file table_var2back_set_file tables_row_counts_file MSigDB_files MSigDB_not_found_HGNC_file"
+
"var_annotations_file genes_annotations_file get_evidence_file back_set_not_counted_file back_sets_folder top_alleles"
)
print(
"Use varanto_import.sh and varanto_import.conf for easier setting of parameters."
)
sys.exit(-1)
params = Params(sys.argv)
print("Generating input data files...")
start = datetime.now()
#initialize counters of annotations in background lists and load background set description file
print("* Loading and counting variations from background sets files...")
back_sets = BackgroundSets(params.back_set_not_counted_file,
params.back_sets_folder)
print(
"* Loading and counting variations from background sets files completed at "
+ str(datetime.now() - start))
headers = [
"refsnp_id", "allele", "chr_name", "chrom_start", "chrom_strand",
"phenotype_description", "study_external_ref", "study_description",
"consequence_type_tv", "ensembl_gene_stable_id", "associated_gene",
"polyphen_prediction", "sift_prediction"
]
#initialize labels
print("* Loading data for annotation processing...")
labels = Labels(headers, back_sets.get_back_sets_ids(),
params.table_ann_desc_file, params.genes_annotations_file,
params.get_evidence_file, params.top_alleles,
params.msigdb_files, params.msigdb_not_found_hgnc)
print("* Loading data for annotation processing completed at " +
str(datetime.now() - start))
#open annotated snps file
with open(params.var_annotations_file,
'r', encoding='latin-1') as input_var, open(
params.table_var_file, 'w') as variations_output, open(
params.table_ann_file, 'w') as annotations_output, open(
params.table_var2ann_file,
'w') as var2ann_output, open(
params.table_var2back_set_file,
'w') as var2back_set_output:
#initialize csv reader and writers
reader = csv.reader(input_var, delimiter='\t')
variations_csv = csv.writer(variations_output, delimiter='\t')
annotations_csv = csv.writer(annotations_output,
delimiter='\t',
quoting=csv.QUOTE_NONE,
escapechar='\\')
var2ann_csv = csv.writer(var2ann_output, delimiter='\t')
var2back_set_csv = csv.writer(var2back_set_output, delimiter='\t')
#current name of variant and status of change
current_variant = None
#current id of variant
variation_id_cnt = 0
#for outputting progress
last_time = datetime.now()
rows_processed = 0
#process each file
for row in reader:
#check if it is another snp than snp in the previous line
if row[0] != current_variant:
#check if it is variant in standard track (according to chromosome name)
if not row[2] in [
"1", "2", "3", "4", "5", "6", "7", "8", "9", "10",
"11", "12", "13", "14", "15", "16", "17", "18", "19",
"20", "21", "22", "MT", "X", "Y"
]:
continue
#add row in variations csv
current_variant = row[0]
variation_id_cnt += 1
#write variation
variations_csv.writerow(
[variation_id_cnt, row[0], row[4], row[3], row[1], row[2]])
#list of background sets ids which contain variation
var_is_in_back_sets = back_sets.process_variant(
current_variant, variation_id_cnt, var2back_set_csv)
labels.process_variation_row(variation_id_cnt, var_is_in_back_sets,
row, annotations_csv, var2ann_csv)
rows_processed += 1
if rows_processed % 10000 == 0:
sys.stdout.write(
"\rLines processed: %d, time of processing last 10000 rows: "
% rows_processed)
sys.stdout.write(str(datetime.now() - last_time))
sys.stdout.flush()
last_time = datetime.now()
#process annotations dependent on the data gained by processing of all variation rows (top alleles)
labels.process_additional_annotation(annotations_csv, var2ann_csv)
sys.stdout.write("\n")
#write to file descriptions of background sets with counted variations
back_sets.write_table_back_sets_file(variation_id_cnt,
params.table_back_set_file)
#write associations between annotations and their counts in all background sets
labels.write_annotation_counts(params.table_ann2back_set_file)
#write counts of rows for db tables with primary keys sequences to increment them.
with open(params.tables_row_counts_file, 'w') as out:
writer = csv.writer(out, delimiter='\t')
writer.writerow([
variation_id_cnt, labels.annotation_id_cnt,
labels.ann_descriptions.id_count(),
len(back_sets.get_back_sets_ids())
])
print("Generating input files completed in " + str(datetime.now() - start))
def add_data(self, data, **kwargs):
self.labels_list.append(
Labels(data=np.asarray(data),
components=self.components,
weights=self.weights,
**kwargs))
def begin(self):
self.label.setText(label.begin())
def notice_new_bookmark(self):
self.label.setText(label.bookmarked())
def complete(self):
self.label.setText(label.complete())
class VOC2COCO:
def __init__(self):
self.labels = Labels()
def generateVOC2Json(self, xmlFiles):
attrDict = dict()
attrDict["categories"] = [self.labels.create_attrDict()]
images = list()
annotations = list()
image_id = 0
for file_ in xmlFiles:
image_id = image_id + 1
annotation_path = file_
image = dict()
doc = xmltodict.parse(open(annotation_path).read())
image['file_name'] = str(doc['annotation']['filename'])
image['height'] = int(doc['annotation']['size']['height'])
image['width'] = int(doc['annotation']['size']['width'])
image['id'] = image_id
#print ("File Name: {} and image_id {}".format(file_, image_id))
images.append(image)
id1 = 1
if 'object' in doc['annotation']:
for obj in doc['annotation']['object']: #48 boxes
for value in attrDict["categories"]:
#print(value)
annotation = dict()
obj = to_dict(obj)
for val in value:
if str(obj['name']) == val["name"]:
#print str(obj['name'])
#annotation["segmentation"] = []
annotation["iscrowd"] = 0
#annotation["image_id"] = str(doc['annotation']['filename']).split('.jpg')[0] #attrDict["images"]["id"]
annotation["image_id"] = image_id
x1 = float(obj["bndbox"]["xmin"]) # - 1
y1 = float(obj["bndbox"]["ymin"]) # - 1
x2 = float(obj["bndbox"]["xmax"]) # - x1
y2 = float(obj["bndbox"]["ymax"]) # - y1
annotation["bbox"] = [x1, y1, x2, y2]
annotation["area"] = float(x2 * y2)
annotation["category_id"] = val["id"]
annotation["ignore"] = 0
annotation["id"] = id1
annotation["segmentation"] = [[
x1, y1, x1, (y1 + y2), (x1 + x2),
(y1 + y2), (x1 + x2), y1
]]
id1 += 1
annotations.append(annotation)
else:
print("{} doesn't have any object".format(file_))
#image_id = image_id + 1
#print(len(annotations))
attrDict["images"] = images
attrDict["annotations"] = annotations
attrDict["type"] = "instances"
attrDict['categories'] = attrDict['categories'][0]
classes = {}
for o in attrDict['categories']:
classes[o['id']] = o['name']
#print attrDict
jsonString = json.dumps(attrDict)
with open("receipts_valid.json", "w") as f:
f.write(jsonString)
def main(self, trainXMLFiles="path/to/annotation/xml/files"):
self.generateVOC2Json(trainXMLFiles)
def system_arguments(self):
if len(sys.argv) == 1:
url = self.address.text()
if url.startswith('http://'):
change = str(url)
self.address.setText(change)
load = self.address.text()
self.label.setText(load)
self.web.load(QUrl(load))
del sys.argv[1:]
elif url.startswith('https://'):
change = str(url)
self.address.setText(change)
load = self.address.text()
self.label.setText(load)
self.web.load(QUrl(load))
del sys.argv[1:]
elif url.startswith('find:'):
load = self.address.text()
self.result0 = load.replace('find:', '')
print "Finding:" + self.result0
self.web.findText(self.result0)
self.label.setText(label.finding())
del sys.argv[1:]
else:
add = "https://" + url
change = str(add)
self.address.setText(change)
load = self.address.text()
self.label.setText(load)
self.web.load(QUrl(load))
del sys.argv[1:]
else:
self.location = sys.argv[1:]
self.temp = str(self.location)
self.result0 = self.temp.replace('[', '')
self.result1 = self.result0.replace(']', '')
self.result_final = self.result1.replace("'", '')
url = self.result_final
if url.startswith('http://'):
change = str(url)
self.address.setText(change)
load = self.address.text()
self.label.setText(load)
self.loading1 = self.web.load(QUrl(load))
del sys.argv[1:]
elif url.startswith('https://'):
change = str(url)
self.address.setText(change)
load = self.address.text()
self.label.setText(load)
self.loading1 = self.web.load(QUrl(load))
del sys.argv[1:]
else:
change = str("https://" + url)
self.address.setText(change)
load = self.address.text()
self.label.setText(load)
self.web.load(QUrl(load))
del sys.argv[1:]