def correlation_experiment(filename, language, embedding_function, name):
data, character_encoder, tag_encoder, embedded_chars = read_data(
filename, language)
character_decoder = {v: k for k, v in character_encoder.items()}
features = getphonfeatures()
language_features = [
np.array(features[character_decoder[f]])
if character_decoder[f] in features else None
for f in range(len(character_encoder))
]
feature_similarity = get_similarity_matrix(language_features,
embedded_chars)
embeddings = embedding_function(data, character_encoder, embedded_chars,
character_decoder)
similarities = [
get_similarity_matrix(m, embedded_chars) for m in embeddings
]
rs = [
correlation(feature_similarity, similarities[i])[0] for i in [0, 1, 2]
]
print("%s %s:" % (language, name))
print(" PEARSON R FOR EMBEDDING AND FEATURE REPR. SIMILARITIES:")
print(" %s,DIM=5" % language, rs[0])
print(" %s,DIM=15" % language, rs[1])
print(" %s,DIM=30" % language, rs[2])
random_rs = [[], [], []]
for i in range(N):
random_embeddings = [matshuf(m) for m in embeddings]
random_similarities = [
get_similarity_matrix(m, embedded_chars) for m in random_embeddings
]
random_rs[0].append(
correlation(feature_similarity, random_similarities[0])[0])
random_rs[1].append(
correlation(feature_similarity, random_similarities[1])[0])
random_rs[2].append(
correlation(feature_similarity, random_similarities[2])[0])
print((" P=%.2f CONF. INTERVALS FOR PEARSON R OF RANDOM ASSIGNMENT OF\n" %
P) + " EMBEDDINGS TO PHONEMES AND PHONETIC FEATURE DESCRIPTIONS:")
civals = [confidence_interval_value(random_rs[i]) for i in [0, 1, 2]]
print(" %s,DIM=5" % language, confidence_interval_value(random_rs[0]),
check_r(civals[0], rs[0]), rs[0])
print(" %s,DIM=15" % language, confidence_interval_value(random_rs[1]),
check_r(civals[1], rs[1]), rs[1])
print(" %s,DIM=30" % language, confidence_interval_value(random_rs[2]),
check_r(civals[2], rs[2]), rs[2])
print()
def test_4():
"""
Query : Find correlation between math score and reading score for each race/ethnicity
Dataset used - https://www.kaggle.com/spscientist/students-performance-in-exams
Columns : race/ethnicity, math score, reading score
"""
table = pandas.read_csv('data/data_for_test_correlation/students_performance.csv')
result_table, suggestions = correlation.correlation(table, 'math score', 'reading score',
dimensions=['race/ethnicity'])
print(result_table.to_string())
print(suggestions)
expected_result_table = """ race/ethnicity correlation between "math score" , "reading score"
0 group A 0.816310
1 group B 0.824536
2 group C 0.810855
3 group D 0.793180
4 group E 0.859600 """
assert(expected_result_table == result_table.to_string())
expected_suggestions = """[]"""
assert(expected_suggestions == str(suggestions))
def plot_transition(state, m_general_1, m_general_2, output_name):
prob_general_1 = []
prob_general_2 = []
for action in actions:
prob_general_1.append(m_general_1.get(action, 0) * 1.0 / m_general_1["total"])
prob_general_2.append(m_general_2.get(action, 0) * 1.0 / m_general_2["total"])
ind = np.arange(len(actions))
width = 0.35
names = ["ML", "MR", "MU", "MD", "ZI", "ZO", "TF", "TB", "REC", "REAN", "ROC", "ROAN"]
fig = plt.figure()
axis = fig.add_subplot(111)
rect1 = axis.bar(ind, prob_general_1, width, facecolor = 'none', hatch = '//', label="Real")
rect2 = axis.bar(ind+width, prob_general_2, width, facecolor='none', hatch='\\\\', label="Synthetic")
x, y, z, ax, ay, az, last_op = state
axis.axis(ymin=0, ymax=1)
axis.set_ylabel('Probability of Action')
axis.set_xlabel('Action')
axis.set_title('x=%s, y=%s, z=%s, ax=%s, ay=%s, az=%s, A_p=%s, count: %d'%(x,y,z,ax,ay,az,last_op, m_general_1["total"]))
axis.set_xticks(ind+width)
axis.set_xticklabels(names, fontsize='small')
axis.legend(loc="upper left")
axis.text(5, 0.95, "correlation: %0.3f"%correlation(prob_general_1, prob_general_2), horizontalalignment='center')
#autolabel(ax, rect1)
#autolabel(ax, rect2)
plt.savefig(output_name)
plt.close()
def is_pair_preferred(seq1, seq2):
if seq1 is None or seq2 is None:
print 'One or both of the sequences does not exist'
return False
if len(seq1) != len(seq2):
print 'Sequences are not the same length'
return False
values_set = set(cor.correlation(seq1, seq2))
if len(values_set) == 3:
L = float(len(seq1))
l = math.log(int(L+1), 2)
m = 2 if l%2 == 0 else 1
t = float(1 + 2**((l+m)/2))
acceptable_values = [float("{0:.5f}".format(-t/L)), float("{0:.5f}".format(-1/L)), float("{0:.5f}".format((t-2)/L))]
for value in values_set:
if value not in acceptable_values:
message = 'Autocorrelation of preferred pair can only have three values and these are: '
for acceptable_value in acceptable_values:
message += '\n' + str(acceptable_value)
message += '\nOne of existing values (' + str(value) + ') is not a valid one.'
print message
return False
print 'Preferred pair was found. Gold codes will be generated shortly.'
return True
else:
print 'Crosscorrelation function does not have three values: ' + str(len(values_set))
return False
def calculate_correlation(data):
cols = data.shape[1]
cm = np.zeros((cols,cols))
for i in range(cols):
for j in range(cols):
cm[i,j] = correlation.correlation(data[:, i], data[:, j])
np.savetxt('correlation.txt', cm)
return cm
def forward(self, x1, x2):
y = correlation(x1,
x2,
pad_size=4,
kernel_size=1,
max_displacement=4,
stride1=1,
stride2=1)
return y
def correlation_experiment(file, lan, embf, name):
data, cencoder, tencoder, embchars = readdata(file, lan)
cdecoder = {v: k for k, v in cencoder.items()}
tdecoder = {v: k for k, v in tencoder.items()}
features = getphonfeatures()
lanfeatures = [
np.array(features[cdecoder[f]]) if cdecoder[f] in features else None
for f in range(len(cencoder))
]
featsim = getsimmatrix(lanfeatures, len(cencoder), embchars)
embeddings = embf(data, cencoder, embchars, tencoder, cdecoder, tdecoder,
lan)
sims = [getsimmatrix(m, len(cencoder), embchars) for m in embeddings]
rs = [correlation(featsim, sims[i])[0] for i in [0, 1, 2]]
print("%s %s:" % (lan, name))
print(" PEARSON R FOR EMBEDDING AND FEATURE REPR. SIMILARITIES:")
print(" %s,DIM=5" % lan, rs[0])
print(" %s,DIM=15" % lan, rs[1])
print(" %s,DIM=30" % lan, rs[2])
randrs = [[], [], []]
for i in range(N):
ranembeddings = [matshuf(m) for m in embeddings]
ransims = [
getsimmatrix(m, len(cencoder), embchars) for m in ranembeddings
]
randrs[0].append(correlation(featsim, ransims[0])[0])
randrs[1].append(correlation(featsim, ransims[1])[0])
randrs[2].append(correlation(featsim, ransims[2])[0])
print((" P=%.2f CONF. INTERVALS FOR PEARSON R OF RANDOM ASSIGNMENT OF\n" %
P) + " EMBEDDINGS TO PHONEMES AND PHONETIC FEATURE DESCRIPTIONS:")
civals = [confidenceival(randrs[i]) for i in [0, 1, 2]]
print(" %s,DIM=5" % lan, confidenceival(randrs[0]),
checkr(civals[0], rs[0]), rs[0])
print(" %s,DIM=15" % lan, confidenceival(randrs[1]),
checkr(civals[1], rs[1]), rs[1])
print(" %s,DIM=30" % lan, confidenceival(randrs[2]),
checkr(civals[2], rs[2]), rs[2])
print()
def test_2():
"""
Query : Find correlation between NA_Sales & EU_Sales for each platform
Dataset Used - https://www.kaggle.com/gregorut/videogamesales
Columns :
Platform - Platform of the games release (i.e. PC,PS4, etc.)
NA_Sales - Sales in North America (in millions)
EU_Sales - Sales in Europe (in millions)
"""
table = pandas.read_csv('data/data_for_test_correlation/vgsales.csv')
result_table, suggestions = correlation.correlation(table, 'NA_Sales', 'EU_Sales',
dimensions=['Platform'])
print(result_table)
print(suggestions)
expected_result_table = """ Platform correlation between "NA_Sales" , "EU_Sales"
0 2600 0.996554
1 3DO NaN
2 3DS 0.945838
3 DC 0.797678
4 DS 0.871312
5 GB 0.705745
6 GBA 0.925557
7 GC 0.938478
8 GEN 0.973297
9 GG NaN
10 N64 0.919057
11 NES 0.735203
12 NG NaN
13 PC 0.404835
14 PCFX NaN
15 PS 0.811515
16 PS2 0.654672
17 PS3 0.813370
18 PS4 0.793474
19 PSP 0.701001
20 PSV 0.755251
21 SAT 0.999610
22 SCD 1.000000
23 SNES 0.988964
24 TG16 NaN
25 WS NaN
26 Wii 0.971428
27 WiiU 0.959718
28 X360 0.854582
29 XB 0.831869
30 XOne 0.779913"""
assert(expected_result_table == result_table.to_string())
expected_suggestions = """[]"""
assert(expected_suggestions == str(suggestions))
def test_correlationHardCoded(self):
with tf.device('/gpu:0'):
with tf.Session('') as sess:
batch_size = 1
width = 64 # 3
height = 48 # 3
depth = 256
stride_1 = 2
stride_2 = 1
kernel_size = 1
max_displacement = 40 # 3
padding = 40 # 3
my_shape = [batch_size, width, height, depth]
a = np.float32(np.ones(my_shape))
b = np.float32(np.ones(my_shape))
a = tf.convert_to_tensor(a, tf.float32)
b = tf.convert_to_tensor(b, tf.float32)
print("a : ", np.shape(a))
print("b : ", np.shape(b))
# ResizeMethod.NEAREST_NEIGHBOR
a_resize = tf.image.resize_images(a, [(width * 2) - 1,
(height * 2) - 1],
method=1)
# ResizeMethod.BILINEAR
b_resize = tf.image.resize_images(b, [(width * 2) - 1,
(height * 2) - 1],
method=0)
print("a_resize : ", np.shape(a_resize))
print("b_resize : ", np.shape(b_resize))
result = correlation(a_resize, b_resize, kernel_size,
max_displacement, stride_1, stride_2,
padding) #.eval() # Sub-pixel
print("result : ", np.shape(result))
grad_a = tf.gradients(result[:, :, :, 0], a)
print(grad_a)
# print(np.shape(sess.run(grad_a)))
grad_b = tf.gradients(result[:, :, :, 0], b)
print(grad_b)
def LAR(data, out, alpha=0.5):
beta = []
Y_means = np.ones(len(out)) * np.mean(out)
residual = out - Y_means
active_index = []
origin_index = [Item(i, False) for i in range(len(data[0]))]
B = None
while True:
max_correlation = -999999999
item = None
for it in origin_index:
if it.active == True:
continue
co = correlation.correlation(data[:, it.index], residual)
if co > max_correlation:
max_correlation = co
item = it
if item == None:
break
item.active = True
active_index.append(item)
active_index = sorted(active_index, key=get_index)
idx = 999999999
for i in range(len(active_index)):
if active_index[i].index == item.index:
idx = i
break
beta.insert(idx, 0)
indexes = [it.index for it in active_index]
B = np.array(beta)
d = direction(data[:, np.array(indexes)], out, B, residual)
B = B + alpha * d
for j in range(len(B)):
if B[j] < 0.001 and B[j] > -0.001:
it = active_index[j]
origin_index[it.index].active = False
del B[j]
del active_index[j]
residual = out - Y_means - data[:, np.array(indexes)].dot(B)
if len(B) == 4:
break
beta = B.tolist()
return B, Y_means, indexes
def cnnmodel(frame1_xyz, frame1_rgb, frame2_xyz, frame2_rgb):
frame1_rgb = tf.image.resize_images(frame1_rgb, [480, 640])
frame2_rgb = tf.image.resize_images(frame2_rgb, [480, 640])
frame1_feat_rgb, _ = get_network('resnet50',
frame1_rgb,
weight_decay=1e-5,
is_training=True)
frame2_feat_rgb, _ = get_network('resnet50',
frame2_rgb,
weight_decay=1e-5,
is_training=True,
reuse=True)
frame1_feat = encoder(frame1_xyz)
frame2_feat = encoder(frame2_xyz, reuse=True)
cc_o = correlation(frame2_feat_rgb, frame1_feat_rgb, 1, rad, 1, 1, rad)
return cc_o
def main():
dataset_path = "/home/zabot/Datasets/Lung-HCRP/Features/{}.csv"
comb_fem_path = "/home/zabot/Documents/Codes/Mestrado/Spectra/archives/combinations_fem.txt"
comb_dist_path = "/home/zabot/Documents/Codes/Mestrado/Spectra/archives/combinations_distances.txt"
# Read combinations FEMs and Distances in order
comb_fem = read_combinations(comb_fem_path)
comb_dist = read_combinations(comb_dist_path)
file = open("result-correlation.txt","w")
for metric1, metric2 in comb_dist:
for name_fem1, name_fem2 in comb_fem:
data_fem1 = read_archives(dataset_path.format(name_fem1))
data_fem2 = read_archives(dataset_path.format(name_fem2))
corr = correlation(data_fem1, data_fem2, percentage=1, metric1=metric1, metric2=metric2)
print(metric1, metric2, name_fem1, name_fem2, corr)
file.write("{};{};{};{};{}\n".format(metric1,metric2,name_fem1,name_fem2,corr))
file.close()
def test_3():
"""
Query : Find correlation between math score and reading score
Dataset used - https://www.kaggle.com/spscientist/students-performance-in-exams
Columns : race/ethnicity, math score, reading score
"""
table = pandas.read_csv('data/data_for_test_correlation/students_performance.csv')
result_table, suggestions = correlation.correlation(table, 'math score', 'reading score')
print(result_table.to_string())
print(suggestions)
expected_result_table = """ correlation between "math score" , "reading score"
0 0.81758 """
assert(expected_result_table == result_table.to_string())
expected_suggestions = """[]"""
assert(expected_suggestions == str(suggestions))
def test_check_output(self):
#x_shape = (1, 196, 3, 3)
np.random.seed(13)
np.set_printoptions(threshold=np.inf)
x_shape = (2, 10, 3, 3)
x_type = 'float32'
x1 = fluid.layers.data(name='x1',
shape=x_shape,
dtype=x_type,
append_batch_size=False)
x2 = fluid.layers.data(name='x2',
shape=x_shape,
dtype=x_type,
append_batch_size=False)
x1_np = np.random.randn(2, 3, 4, 5).astype(x_type)
x2_np = np.random.randn(2, 3, 4, 5).astype(x_type)
out_np = corr(x1_np,
x2_np,
pad_size=4,
kernel_size=1,
max_displacement=4,
stride1=1,
stride2=1)
out = correlation(x1,
x2,
pad_size=4,
kernel_size=1,
max_displacement=4,
stride1=1,
stride2=1)
place = fluid.CUDAPlace(0)
exe = fluid.Executor(place)
res = exe.run(feed={'x1': x1_np, 'x2': x2_np}, fetch_list=[out.name])
self.assertTrue(np.allclose(res[0], out_np))
def test_1():
"""
Query : Find correlation between NA_Sales & EU_Sales
Dataset Used - https://www.kaggle.com/gregorut/videogamesales
Columns :
Platform - Platform of the games release (i.e. PC,PS4, etc.)
NA_Sales - Sales in North America (in millions)
EU_Sales - Sales in Europe (in millions)
"""
table = pandas.read_csv('data/data_for_test_correlation/vgsales.csv')
result_table, suggestions = correlation.correlation(table, 'NA_Sales', 'EU_Sales')
print(result_table)
print(suggestions)
expected_result_table = """ correlation between "NA_Sales" , "EU_Sales"
0 0.767727"""
assert(expected_result_table == result_table.to_string())
expected_suggestions = """[]"""
assert(expected_suggestions == str(suggestions))
def main():
'''
Main function including :
- Arg parser Generation
- Graph Generation
- Thesaurus Generation
- Optionnal thesaurus test
- Writting Thesaurus
'''
parser = argparse.ArgumentParser()
parser.add_argument("data_file", help='A .outmalt data file')
parser.add_argument("-t",
"--theory",
default=None,
help='A compare file in the correct format.')
parser.add_argument("--thesaurus",
type=int,
default=1000,
help='The size of the thesaurus.')
parser.add_argument("--absolute",
action='store_true',
help='Use absolute mode.')
parser.add_argument("-v",
"--verbose",
action='store_true',
help='Activate maximum detail mode.')
parser.add_argument(
"-l",
"--limit",
type=int,
default=1000000,
help='The number of lexemes proceed before cleaning the graph.')
parser.add_argument(
"-m",
"--minimum_limit",
type=int,
default=10,
help=
'The number of occurrences needed for not being deleted when cleaning.'
)
parser.add_argument(
"-w",
"--write",
default=None,
help='The path to the file where thesaurus will be written.')
args = parser.parse_args(sys.argv[1:])
content = splitter(file_manager.read(args.data_file))
thesau = thesaurus(
tree_creator.token_list(content, args.limit, args.minimum_limit,
args.verbose))
print("Graph has been generated. It has", len(thesau.corpus),
"nodes inside.")
mode = 'r'
if args.absolute:
mode = 'a'
result = thesau.usable({'NC'}, thesau.cosine, thesau.PMI, mode,
args.thesaurus, args.verbose)
print("The thesaurus has been generated.")
if args.theory is not None:
c, p = correlation(
generate_compare(splitter(file_manager.read(args.theory))), result)
print("With a cover of", p, "%, there is a correlation score of", c)
if args.write:
with open(args.write, 'w') as file:
json.dump(result, file)
else:
print(result)
dist_coefs['b_left'])
print "Doing Stereo Calibration for camera 14"
(R14, T14) = calibration.StereoCalibration(
objpoints, imgpoints['t_left'], imgpoints['b_right'], cam_mats['t_left'],
cam_mats['b_right'], dist_coefs['t_left'], dist_coefs['b_right'])
'''Read Images'''
t_left = cv2.imread(sample + '_t_left.jpeg')
t_right = cv2.imread(sample + '_t_right.jpeg')
b_left = cv2.imread(sample + '_b_left.jpeg')
b_right = cv2.imread(sample + '_b_right.jpeg')
'''Zmap'''
print "computing Z values"
Zmap = []
correlation = correlation()
for v1 in range(window_size, 480 - window_size, pixel_step):
for u1 in range(window_size, 720 - window_size, pixel_step):
'''80mm'''
Z = range(80, 3081, 80)
(correlation12, u2max_right, v2max_right) = correlation.correlation(
R12, T12, t_left, t_right, cam_mats['t_left'], cam_mats['t_right'],
u1, v1, window_size, Z)
(correlation13, u2max_downl, v2max_downl) = correlation.correlation(
R13, T13, t_left, b_left, cam_mats['t_left'], cam_mats['b_left'],
u1, v1, window_size, Z)
(correlation14, u2max_downr, v2max_downr) = correlation.correlation(
R14, T14, t_left, b_right, cam_mats['t_left'], cam_mats['b_right'],
u1, v1, window_size, Z)
correlationAll = np.array(
def least_squares_fit(x, y):
"""given training values for x and y, find the least-squares values of
alpha and beta"""
beta = correlation(x, y) * standard_deviation(y) / standard_deviation(x)
alpha = mean(y) - beta * mean(x)
return alpha, beta
def hello_http(request):
"""HTTP Cloud Function.
Args:
request (flask.Request): The request object.
<http://flask.pocoo.org/docs/1.0/api/#flask.Request>
Returns:
The response text, or any set of values that can be turned into a
Response object using `make_response`
<http://flask.pocoo.org/docs/1.0/api/#flask.Flask.make_response>.
"""
request_json = request.get_json(silent=True)
request_args = request.args
# extracting the intent parameters from the json
intent = _get_value(request_json, 'intent')
table = _get_value(request_json, 'table')
metric = _get_value(request_json, 'metric')
dimensions = _get_value(request_json, 'dimensions')
summary_operator = _get_value(request_json, 'summaryOperator')
slices = _get_value(request_json, 'slices')
is_asc = _get_value(request_json, 'isAsc')
k = _get_value(request_json, 'topKLimit')
slices = _get_value(request_json, 'slices')
slice_comparision_arg = _get_value(request_json, 'comparisonValue')
time_comparision_arg = _get_value(request_json, 'compareDateRange')
date = _get_value(request_json, 'dateRange')
time_granularity = _get_value(request_json, 'timeGranularity')
correlation_metrics = _get_value(request_json, 'correlationMetrics')
rangeA1Notation = _get_value(request_json, 'rangeA1Notation')
# Converting the list of list into a pandas dataframe.
query_table = []
for row in range(1, len(table)):
if row != 0:
query_table.append(table[row])
query_table_dataframe = pandas.DataFrame(query_table, columns=table[0])
(all_dimensions,
all_metrics) = _list_all_dimensions_metrics(query_table_dataframe,
dimensions, metric)
# Remove empty columns
query_table_dataframe = remove_empty_columns(query_table_dataframe)
# Remove duplicate named columns
query_table_dataframe = remove_duplicate_named_columns(
query_table_dataframe)
# Converting the variables that contain denote the
# date range into the desired format.
date_column_name = None
date_range = None
day_first = None
if date != None:
date_columns = request_json['dateColumns']
date_column_name = date['dateCol']
date_range = (date['dateStart'], date['dateEnd'])
day_first = date_columns[date_column_name]['day_first']
# Converting the Slices passed in the json into a
# list of tuples (col, operator, val)
slices_list = None
if slices != None:
slices_list = []
for item in slices:
val = item['sliceVal']
col = item['sliceCol']
operator = _str_to_filter_enum(item['sliceOp'])
slices_list.append((col, operator, val))
if dimensions == 'null':
dimensions = None
if slice_comparision_arg is not None:
slice_compare_column = slice_comparision_arg['comparisonColumn']
slice1 = slice_comparision_arg['slice1']
slice2 = slice_comparision_arg['slice2']
if time_comparision_arg is not None:
time_compare_column = time_comparision_arg['dateCol']
date_range1 = (time_comparision_arg['dateStart1'],
time_comparision_arg['dateEnd1'])
date_range2 = (time_comparision_arg['dateStart2'],
time_comparision_arg['dateEnd2'])
day_first = request_json['dateColumns'][time_compare_column][
'day_first']
if metric == 'null':
metric = None
summary_operator = _str_to_summary_operator_enum(summary_operator)
time_granularity = _str_to_time_granularity_enum(time_granularity)
suggestions = []
wrong_points_suggestion = wrong_points.wrong_points(query_table_dataframe)
if intent == 'show':
query_table_dataframe = show(query_table_dataframe,
slices=slices_list,
metric=metric,
dimensions=dimensions,
summary_operator=summary_operator,
date_column_name=date_column_name,
day_first=day_first,
date_range=date_range)
if summary_operator == enums.SummaryOperators.MEAN:
suggestions.append(get_hardcoded_mean_vs_median_suggestion())
updated_suggestions = []
for suggestion in suggestions:
updated_suggestion = suggestion
if 'change_list' in suggestion.keys():
updated_suggestion['json'] = func(request_json,
suggestion['change_list'])
updated_suggestions.append(updated_suggestion)
suggestions = updated_suggestions
elif intent == 'topk':
query_result = topk.topk(query_table_dataframe,
metric,
dimensions,
is_asc,
k,
summary_operator=summary_operator,
slices=slices_list,
date_column_name=date_column_name,
day_first=day_first,
date_range=date_range)
query_table_dataframe = query_result[0]
suggestions = query_result[1]
updated_suggestions = []
for suggestion in suggestions:
updated_suggestion = suggestion
if 'change_list' in suggestion.keys():
updated_suggestion['json'] = func(request_json,
suggestion['change_list'])
updated_suggestion['oversight'] = updated_suggestion[
'oversight'].name
updated_suggestions.append(updated_suggestion)
suggestions = updated_suggestions
elif intent == 'slice_compare':
query_result = slice_compare.slice_compare(
query_table_dataframe,
metric,
all_dimensions,
all_metrics,
slice_compare_column,
slice1,
slice2,
summary_operator,
date_column_name=date_column_name,
date_range=date_range,
day_first=day_first,
slices=slices_list,
dimensions=dimensions)
query_table_dataframe = query_result[0]
suggestions = query_result[1]
updated_suggestions = []
for suggestion in suggestions:
updated_suggestion = suggestion
if 'change_list' in suggestion.keys():
updated_suggestion['json'] = func(request_json,
suggestion['change_list'])
updated_suggestion['oversight'] = updated_suggestion[
'oversight'].name
updated_suggestions.append(updated_suggestion)
suggestions = updated_suggestions
elif intent == 'time_compare':
query_result = time_compare.time_compare(query_table_dataframe,
metric,
all_dimensions,
time_compare_column,
date_range1,
date_range2,
day_first,
summary_operator,
slices=slices_list,
dimensions=dimensions)
query_table_dataframe = query_result[0]
suggestions = query_result[1]
updated_suggestions = []
for suggestion in suggestions:
updated_suggestion = suggestion
if 'change_list' in suggestion.keys():
updated_suggestion['json'] = func(request_json,
suggestion['change_list'])
updated_suggestion['oversight'] = updated_suggestion[
'oversight'].name
updated_suggestions.append(updated_suggestion)
suggestions = updated_suggestions
elif intent == 'correlation':
query_table_dataframe = correlation.correlation(
query_table_dataframe,
correlation_metrics['metric1'],
correlation_metrics['metric2'],
slices=slices_list,
date_column_name=date_column_name,
day_first=day_first,
date_range=date_range,
dimensions=dimensions)
elif intent == 'trend':
query_table_dataframe = trend.trend(query_table_dataframe,
metric,
time_granularity,
summary_operator,
date_column_name=date_column_name,
day_first=day_first,
date_range=date_range,
slices=slices_list)
else:
raise Exception("Intent name does not match")
if wrong_points_suggestion is not None:
wrong_points_suggestion['oversight'] = wrong_points_suggestion[
'oversight'].name
suggestions = [wrong_points_suggestion] + suggestions
final_table = []
# converting into a json object and returning
final_table = query_table_dataframe.values.tolist()
final_table.insert(0, list(query_table_dataframe.columns.values))
json_ret = {'outputTable': final_table, 'suggestions': suggestions}
if rangeA1Notation is not None:
all_row_labels = _get_all_row_labels(rangeA1Notation)
all_column_labels = _get_all_column_labels(rangeA1Notation)
cheader_to_clabel = _get_cheader_to_clabel(table, all_column_labels)
if slices_list is not None:
json_ret[
'slicing_passed_list'] = insert_as_column.insert_as_column_show(
table,
cheader_to_clabel,
all_row_labels[0],
all_row_labels[-1],
all_column_labels[0],
all_column_labels[-1],
slices=slices_list)
if intent == 'topk' and summary_operator is None:
filter_column_label_number = _get_number_of_column_label(
all_column_labels[-1]) + 1
filter_column_label = _get_label_from_number(
filter_column_label_number)
json_ret[
'list_topk_indices'] = insert_as_column.insert_as_column_topk_column(
table, cheader_to_clabel, all_row_labels[0],
all_row_labels[-1], all_column_labels[0],
all_column_labels[-1], filter_column_label, metric, is_asc,
k)
json_string = json.dumps(json_ret)
return json_string
HEIGHT = 2
WIDTH = 2
CHANNELS = 1
NEIGHBORHOOD_SIZE = 2
MAX_DISPLACEMENT = int(math.ceil(NEIGHBORHOOD_SIZE / 2.0))
STRIDE_2 = 1
test_forward = False
if test_forward:
with tf.Session('') as sess:
with tf.device('/gpu:0'):
fmA = tf.ones((BATCH_SIZE, HEIGHT, WIDTH, CHANNELS), dtype=tf.float32)
fmB = tf.convert_to_tensor(np.random.randint(1,5, size=(BATCH_SIZE, HEIGHT, WIDTH, CHANNELS)), dtype=tf.float32)
corr = correlation(fmA,fmB,1,1,1,1,1) # input_a, input_b, kernel_size, max_displacement, stride_1, stride_2, padding
sess.run(tf.initialize_all_variables())
corr_, fmA_, fmB_= sess.run([corr,fmA,fmB])
print(corr_[0])
print(fmA_[0][:,:,0])
print(fmB_[0][:,:,0])
else:
with tf.Session('') as sess:
fmA1 = np.ones((BATCH_SIZE, HEIGHT, WIDTH, CHANNELS)).astype('float32')
fmB1 = np.random.randn(BATCH_SIZE, HEIGHT, WIDTH, CHANNELS).astype('float32')
with tf.device('/gpu:0'):
fmA = tf.constant(fmA1)
fmB = tf.Variable(fmB1)
corr = correlation(fmA,fmB,1,1,1,1,1)
loss = tf.reduce_sum(corr)
train = tf.train.GradientDescentOptimizer(learning_rate=0.01).minimize(loss)
# Grafico com aumento da temperatura no Brasil ao longo dos anos
figBrasil = graficoBrasil(temperaturas_globais_paises)
figBrasil.show()
# Grafico com top paises com maior diferenca entre temperatura média ao longo dos anos e temperatura máxima (que tiveram maior aumento)
figDiferenca = graficoDiferencaGeral(temperaturas_globais_cidades)
figDiferenca.show()
# Verifica as features mais relacionadas ao target
featureSelection(temperaturas_globais)
regression(temperaturas_globais)
regressionMultipleParameters(temperaturas_globais)
# Regressoes para Media Anual de Temperatura Global
modelo = regressao_ano_temp_media(temperaturas_globais)
previsao_temp_media_futura(temperaturas_globais, modelo, 2101)
# Regressoes para Media Anual de Temperatura Global
modelo = regressao_ano_temp_media(temperaturas_globais)
previsao_temp_media_futura(temperaturas_globais, modelo, 2101)
modelo_brasil = regressao_ano_temp_media_BRASIL(temperaturas_globais_cidades)
previsao_temp_media_futura_BRASIL(temperaturas_globais_cidades, modelo_brasil,
2101)
# Correlação Temp Media com Temp LandAndOcean
correlation(temperaturas_globais)
predictLandAndOcean(
temperaturas_globais, 5.518
) # pode ser maneiro usar aqui em vez de hardcoded um valor que foi previsto anterior
modelo = regressionLandToOcean(temperaturas_globais)
graficoPredicao(temperaturas_globais, modelo, 20)
def net_structure(img1, img2):
with slim.arg_scope([slim.conv2d, slim.conv2d_transpose],
# He (aka MSRA) weight initialization
weights_initializer=slim.variance_scaling_initializer(),
activation_fn=LeakyReLU,
# We will do our own padding to match the original Caffe code
padding='VALID'):
weights_regularizer = slim.l2_regularizer(weight_decay)
with slim.arg_scope([slim.conv2d], weights_regularizer=weights_regularizer):
with slim.arg_scope([slim.conv2d], stride=2):
conv_a_1 = slim.conv2d(pad(img1, 3), 64, 7, scope='conv1')
conv_a_2 = slim.conv2d(pad(conv_a_1, 2), 128, 5, scope='conv2')
conv_a_3 = slim.conv2d(pad(conv_a_2, 2), 256, 5, scope='conv3')
conv_b_1 = slim.conv2d(pad(img2, 3), 64, 7, scope='conv1', reuse=True)
conv_b_2 = slim.conv2d(pad(conv_b_1, 2), 128, 5, scope='conv2', reuse=True)
conv_b_3 = slim.conv2d(pad(conv_b_2, 2), 256, 5, scope='conv3', reuse=True)
# Compute cross correlation with leaky relu activation
cc = correlation.correlation(conv_a_3, conv_b_3, 1, 20, 1, 2, 20)
cc_relu = LeakyReLU(cc)
# Combine cross correlation results with convolution of feature map A
netA_conv = slim.conv2d(conv_a_3, 32, 1, scope='conv_redir')
# Concatenate along the channels axis
net = tf.concat([netA_conv, cc_relu], axis=3)
conv3_1 = slim.conv2d(pad(net), 256, 3, scope='conv3_1')
with slim.arg_scope([slim.conv2d], num_outputs=512, kernel_size=3):
conv4 = slim.conv2d(pad(conv3_1), stride=2, scope='conv4')
conv4_1 = slim.conv2d(pad(conv4), scope='conv4_1')
conv5 = slim.conv2d(pad(conv4_1), stride=2, scope='conv5')
conv5_1 = slim.conv2d(pad(conv5), scope='conv5_1')
conv6 = slim.conv2d(pad(conv5_1), 1024, 3, stride=2, scope='conv6')
conv6_1 = slim.conv2d(pad(conv6), 1024, 3, scope='conv6_1')
""" START: Refinement Network """
with slim.arg_scope([slim.conv2d_transpose], biases_initializer=None):
predict_flow6 = slim.conv2d(pad(conv6_1), 2, 3,
scope='predict_flow6',
activation_fn=None)
deconv5 = antipad(slim.conv2d_transpose(conv6_1, 512, 4,
stride=2,
scope='deconv5'))
upsample_flow6to5 = antipad(slim.conv2d_transpose(predict_flow6, 2, 4,
stride=2,
scope='upsample_flow6to5',
activation_fn=None))
concat5 = tf.concat([conv5_1, deconv5, upsample_flow6to5], axis=3)
predict_flow5 = slim.conv2d(pad(concat5), 2, 3,
scope='predict_flow5',
activation_fn=None)
deconv4 = antipad(slim.conv2d_transpose(concat5, 256, 4,
stride=2,
scope='deconv4'))
upsample_flow5to4 = antipad(slim.conv2d_transpose(predict_flow5, 2, 4,
stride=2,
scope='upsample_flow5to4',
activation_fn=None))
concat4 = tf.concat([conv4_1, deconv4, upsample_flow5to4], axis=3)
predict_flow4 = slim.conv2d(pad(concat4), 2, 3,
scope='predict_flow4',
activation_fn=None)
deconv3 = antipad(slim.conv2d_transpose(concat4, 128, 4,
stride=2,
scope='deconv3'))
upsample_flow4to3 = antipad(slim.conv2d_transpose(predict_flow4, 2, 4,
stride=2,
scope='upsample_flow4to3',
activation_fn=None))
concat3 = tf.concat([conv3_1, deconv3, upsample_flow4to3], axis=3)
predict_flow3 = slim.conv2d(pad(concat3), 2, 3,
scope='predict_flow3',
activation_fn=None)
deconv2 = antipad(slim.conv2d_transpose(concat3, 64, 4,
stride=2,
scope='deconv2'))
upsample_flow3to2 = antipad(slim.conv2d_transpose(predict_flow3, 2, 4,
stride=2,
scope='upsample_flow3to2',
activation_fn=None))
concat2 = tf.concat([conv_a_2, deconv2, upsample_flow3to2], axis=3)
predict_flow2 = slim.conv2d(pad(concat2), 2, 3,
scope='predict_flow2',
activation_fn=None)
""" END: Refinement Network """
'''new loss'''
# target_height, target_width = int(predict_flow2.shape[1].value), int(predict_flow2.shape[2].value)
# predict_flow6 = tf.image.resize_bilinear(predict_flow6,
# tf.stack([target_height, target_width]),
# align_corners=True)
# predict_flow5 = tf.image.resize_bilinear(predict_flow5,
# tf.stack([target_height, target_width]),
# align_corners=True)
# predict_flow4 = tf.image.resize_bilinear(predict_flow4,
# tf.stack([target_height, target_width]),
# align_corners=True)
# predict_flow3 = tf.image.resize_bilinear(predict_flow3,
# tf.stack([target_height, target_width]),
# align_corners=True)
# predict = tf.concat([predict_flow5, predict_flow4, predict_flow3, predict_flow2], axis=3)
# flow = predict * 20.0
# flow_temp0 = slim.conv2d(pad(predict), num_outputs=2, kernel_size=2, stride=1, scope='flow_temp0')
# flow_temp = tf.image.resize_bilinear(flow_temp0,
# tf.stack([img_height, img_width]),
# align_corners=True)
# flow = flow_temp * 20.0
flow = predict_flow2 * 20.0
# TODO: Look at Accum (train) or Resample (deploy) to see if we need to do something different
flow = tf.image.resize_bilinear(flow,
tf.stack([img_height, img_width]),
align_corners=True)
return {
'predict_flow6': predict_flow6,
'predict_flow5': predict_flow5,
'predict_flow4': predict_flow4,
'predict_flow3': predict_flow3,
'predict_flow2': predict_flow2,
'flow': flow,
}
def test_correlationHardCoded(self):
with tf.device('/gpu:0'):
with tf.Session(''):
batch_size = 1
width = 40
height = 56
depth = 256
stride_1 = 1.0
stride_2 = 0.5
# 0.5 # 2.0;
#stride_2 = 2.0;
#stride_2 = 2.0;
#max_displacement = 5
max_displacement = 20
#max_displacement = 80 # 40
# sigma = -2.0; # test for error
sigma = 1.0
kernel_size = 3
#padding = 6
padding = 21
#padding = 81 # 41
#expected_depth = (2*int(max_displacement/stride)+1)**2;
my_shape = (batch_size, width, height, depth)
a_rand = np.random.randint(10, size=my_shape)
b_rand = np.random.randint(10, size=my_shape)
# (input_a, input_b, kernel_size, max_displacement, stride_1, stride_2, padding, sigma)
print("##########")
print("kernel_size : ", kernel_size)
print("max_displacement : ", max_displacement)
print("stride_1 : ", stride_1)
print("stride_2 : ", stride_2)
print("padding : ", padding)
print("##########\n")
print("a_rand : ", np.shape(a_rand))
print("b_rand : ", np.shape(b_rand))
#result = correlation(np.ones(my_shape), np.ones(my_shape), kernel_size, max_displacement, stride_1, stride_2, padding, sigma).eval() # Gaussian
result = correlation(np.ones(my_shape), np.ones(my_shape),
kernel_size, max_displacement, stride_1,
stride_2, padding).eval() # Sub-pixel
print("result : ", np.shape(result))
print("\n\n")
#result = correlation(np.ones(my_shape), np.ones(my_shape), stride=stride, max_displacement=max_displacement, kernel_size=kernel_size, sigma=sigma).eval()
# result = cl.corr(a_rand, b_rand, kernel_size=kernel_size, sigma=sigma).eval()
# result = cl.corr(np.ones(my_shape), np.ones(my_shape)).eval()
#self.assertEqual(result.shape[0], my_shape[0])
#self.assertEqual(result.shape[1], my_shape[1])
#self.assertEqual(result.shape[2], my_shape[2])
#self.assertEqual(result.shape[3], expected_depth)
# print(result[0, 0, 0, 220])
# self.assertEqual(result[0, 0, 0, 220], 1)
# print(result[0, 0, 0, 0])
# self.assertEqual(result[0, 0, 0, 0], 0)
print("HardCoded test")
#print(result, end="\n\n")
# print(np.ones(my_shape))
# print(result[0, :, :, 0], end="\n\n")
#print(result[0, :, :, 1], end="\n\n")
#print(result[0, :, :, 2], end="\n\n")
#print(result[0, :, :, 7], end="\n\n")
##print(result[0, :, :, 24], end="\n\n")
#print(result[0, :, :, 25], end="\n\n")
'''print(result[0, :, :, 41], end="\n\n")
def testClass(self):
correlation(rangen=10, label='da')
correlation(rangen=5, label='A')
correlation(rangen=5)
correlation(label='A')
correlation()
'''
from correlation import correlation
import numpy as np
if __name__ == '__main__':
series1 = np.loadtxt('test_series1.txt')
series2 = np.loadtxt('test_series2.txt')
print 'Call the cslick correlation function as follows:'
print 'cslick=correlation(x1,y1,x2,y2,hc)'
print '''where (x1,y1) and (x2,y2) are two differently and unevenly sampled series to be correlated,
in this implementation these are numpy arrays. Optional argument hc (default=0.4) is a coefficient to
tune how closely data from the two series must be to be included in the calculation.\n'''
print "test_series1.txt"
for row in series1[:]:
print "{:1.6f} {:1.6f}".format(row[0], row[1])
print "\ntest_series2.txt"
for row in series2[:]:
print "{:1.6f} {:1.6f}".format(row[0], row[1])
cslick = correlation(series1[:, 1],
series2[:, 1],
series1[:, 0],
series2[:, 0],
hc=0.4)
print '\nCorrelation: {}'.format(cslick)
def cnnmodel(frame1_xyz,frame1_rgb,frame2_xyz,frame2_rgb):
#frame1_input = tf.concat([frame1_xyz,frame1_rgb],3)
#frame2_input = tf.concat([frame2_xyz,frame2_rgb],3)
frame1_feat_rgb = encoder_rgb(frame1_rgb)
frame2_feat_rgb = encoder_rgb(frame2_rgb,reuse=True)
frame1_feat = encoder(frame1_xyz)
frame2_feat = encoder(frame2_xyz,reuse=True)
cc = correlation(frame2_feat_rgb,frame1_feat_rgb,1,rad,1,1,rad)
cc_relu = LeakyReLU(cc)
frame1_feat = tf.transpose(frame1_feat,[0,3,1,2])
frame1_feat_padded = tf.pad(frame1_feat,paddings=[[0,0],[0,0],[rad,rad],[rad,rad]])
frame1_list = []
for i in xrange(30):
for j in xrange(40):
tmp = frame1_feat_padded[:,:,0+i:2*rad+1+i,0+j:2*rad+1+j]
tmp = tf.reshape(tmp,[-1,64,dia * dia])
frame1_list.append(tmp)
frame1_list = tf.stack(frame1_list,axis=2)
frame1_list = tf.transpose(frame1_list,[0,2,3,1])
cc_relu = tf.reshape(cc_relu,[-1,30*40,dia * dia,1])
frame1_list = frame1_list * cc_relu
frame1_list = tf.nn.max_pool(frame1_list,ksize=[1,1,dia * dia,1],strides=[1,1,dia * dia,1],padding='VALID')
frame1_list = tf.reshape(frame1_list,(-1,30,40,64))
x = tf.concat([frame2_feat,frame1_list],3)
x=tflearn.layers.conv.conv_2d(x,64,(3,3),strides=1,activation='relu',weight_decay=1e-5,regularizer='L2')
x=tflearn.layers.conv.conv_2d(x,64,(3,3),strides=1,activation='relu',weight_decay=1e-5,regularizer='L2')
x2 = x
x=tflearn.layers.conv.conv_2d(x,128,(5,5),strides=2,activation='relu',weight_decay=1e-5,regularizer='L2')
#15, 20
x=tflearn.layers.conv.conv_2d(x,128,(3,3),strides=1,activation='relu',weight_decay=1e-5,regularizer='L2')
x=tflearn.layers.conv.conv_2d(x,128,(3,3),strides=1,activation='relu',weight_decay=1e-5,regularizer='L2')
x3 = x
x=tflearn.layers.conv.conv_2d(x,256,(5,5),strides=2,activation='relu',weight_decay=1e-5,regularizer='L2')
#8, 10
x=tflearn.layers.conv.conv_2d(x,256,(3,3),strides=1,activation='relu',weight_decay=1e-5,regularizer='L2')
x=tflearn.layers.conv.conv_2d(x,256,(3,3),strides=1,activation='relu',weight_decay=1e-5,regularizer='L2')
x4 = x
x=tflearn.layers.conv.conv_2d(x,512,(5,5),strides=2,activation='relu',weight_decay=1e-5,regularizer='L2')
#4, 5
x=tflearn.layers.conv.conv_2d(x,512,(3,3),strides=1,activation='relu',weight_decay=1e-5,regularizer='L2')
x=tflearn.layers.conv.conv_2d(x,512,(3,3),strides=1,activation='relu',weight_decay=1e-5,regularizer='L2')
x=tflearn.layers.conv.conv_2d_transpose(x,256,[5,5],[8,10],strides=2,activation='linear',weight_decay=1e-5,regularizer='L2')
#8,10
x4=tflearn.layers.conv.conv_2d(x4,256,(3,3),strides=1,activation='linear',weight_decay=1e-5,regularizer='L2')
x = tf.nn.relu(tf.add(x,x4))
x=tflearn.layers.conv.conv_2d(x,512,(3,3),strides=1,activation='relu',weight_decay=1e-5,regularizer='L2')
x=tflearn.layers.conv.conv_2d(x,256,(3,3),strides=1,activation='relu',weight_decay=1e-5,regularizer='L2')
x=tflearn.layers.conv.conv_2d_transpose(x,128,[5,5],[15,20],strides=2,activation='linear',weight_decay=1e-5,regularizer='L2')
#15,20
x3=tflearn.layers.conv.conv_2d(x3,128,(3,3),strides=1,activation='linear',weight_decay=1e-5,regularizer='L2')
x = tf.nn.relu(tf.add(x,x3))
x=tflearn.layers.conv.conv_2d(x,128,(3,3),strides=1,activation='relu',weight_decay=1e-5,regularizer='L2')
x=tflearn.layers.conv.conv_2d(x,128,(3,3),strides=1,activation='relu',weight_decay=1e-5,regularizer='L2')
x=tflearn.layers.conv.conv_2d_transpose(x,64,[5,5],[30,40],strides=2,activation='linear',weight_decay=1e-5,regularizer='L2')
#30,40
x2=tflearn.layers.conv.conv_2d(x2,64,(3,3),strides=1,activation='linear',weight_decay=1e-5,regularizer='L2')
x = tf.nn.relu(tf.add(x,x2))
x=tflearn.layers.conv.conv_2d(x,128,(3,3),strides=1,activation='relu',weight_decay=1e-5,regularizer='L2')
x=tflearn.layers.conv.conv_2d(x,64,(3,3),strides=1,activation='relu',weight_decay=1e-5,regularizer='L2')
x=tflearn.layers.conv.conv_2d_transpose(x,64,[5,5],[60,80],strides=2,activation='linear',weight_decay=1e-5,regularizer='L2')
#60,80
x=tflearn.layers.conv.conv_2d(x,64,(3,3),strides=1,activation='linear',weight_decay=1e-5,regularizer='L2')
x=tflearn.layers.conv.conv_2d(x,64,(3,3),strides=1,activation='relu',weight_decay=1e-5,regularizer='L2')
x=tflearn.layers.conv.conv_2d(x,64,(3,3),strides=1,activation='relu',weight_decay=1e-5,regularizer='L2')
x=tflearn.layers.conv.conv_2d_transpose(x,64,[5,5],[120,160],strides=2,activation='linear',weight_decay=1e-5,regularizer='L2')
#120,160
x=tflearn.layers.conv.conv_2d(x,64,(3,3),strides=1,activation='linear',weight_decay=1e-5,regularizer='L2')
x=tflearn.layers.conv.conv_2d(x,64,(3,3),strides=1,activation='relu',weight_decay=1e-5,regularizer='L2')
x=tflearn.layers.conv.conv_2d(x,64,(3,3),strides=1,activation='relu',weight_decay=1e-5,regularizer='L2')
x=tflearn.layers.conv.conv_2d_transpose(x,64,[5,5],[240,320],strides=2,activation='linear',weight_decay=1e-5,regularizer='L2')
#240,320
x=tflearn.layers.conv.conv_2d(x,64,(3,3),strides=1,activation='relu',weight_decay=1e-5,regularizer='L2')
#### success
x_s = tflearn.layers.conv.conv_2d(x,64,(3,3),strides=1,activation='relu',weight_decay=1e-5,regularizer='L2')
pred_frame2_xyz =tflearn.layers.conv.conv_2d(x_s,3,(3,3),strides=1,activation='linear',weight_decay=1e-3,regularizer='L2')
pred_frame2_r = tflearn.layers.conv.conv_2d(x_s,1,(3,3),strides=1,activation='linear',weight_decay=1e-3,regularizer='L2')
pred_frame2_mask = tflearn.layers.conv.conv_2d(x_s,2,(3,3),strides=1,activation='linear',weight_decay=1e-3,regularizer='L2')
pred_frame2_score = tflearn.layers.conv.conv_2d(x_s,2,(3,3),strides=1,activation='linear',weight_decay=1e-3,regularizer='L2')
pred_frame2_xyz = tf.add(pred_frame2_xyz,frame2_xyz)
x_transl = tflearn.layers.conv.conv_2d(x_s,3,(3,3),strides=1,activation='linear',weight_decay=1e-3,regularizer='L2')
x_rot = tflearn.layers.conv.conv_2d(x_s,3,(3,3),strides=1,activation='linear',weight_decay=1e-3,regularizer='L2')
return pred_frame2_mask, pred_frame2_r, pred_frame2_xyz, pred_frame2_score, x_transl, x_rot
def net(left, right):
with slim.arg_scope([slim.conv2d, slim.conv2d_transpose],
activation_fn=nn.relu,
padding='SAME'):
with tf.name_scope('Multi-Share'):
conv1a = slim.conv2d(left,
64, [7, 7],
stride=2,
scope='conv1',
reuse=False)
conv1b = slim.conv2d(right,
64, [7, 7],
stride=2,
scope='conv1',
reuse=True)
up_1a = slim.conv2d_transpose(conv1a,
32, [4, 4],
stride=2,
scope='up_1',
reuse=False)
up_1b = slim.conv2d_transpose(conv1b,
32, [4, 4],
stride=2,
scope='up_1',
reuse=True)
conv2a = slim.conv2d(conv1a,
128, [5, 5],
stride=2,
scope='conv2',
reuse=False)
conv2b = slim.conv2d(conv1b,
128, [5, 5],
stride=2,
scope='conv2',
reuse=True)
up_2a = slim.conv2d_transpose(conv2a,
32, [8, 8],
stride=4,
scope='up_2',
reuse=False)
up_2b = slim.conv2d_transpose(conv2b,
32, [8, 8],
stride=4,
scope='up_2',
reuse=True)
up_1a2a_concat = tf.concat([up_1a, up_2a], 3)
up_1b2b_concat = tf.concat([up_1b, up_2b], 3)
up_1a2a = slim.conv2d(up_1a2a_concat,
32, [1, 1],
scope='up_12_concat',
reuse=False)
up_1b2b = slim.conv2d(up_1b2b_concat,
32, [1, 1],
scope='up_12_concat',
reuse=True)
with tf.name_scope('DES-net'):
corr1d = correlation.correlation(conv2a, conv2b, 1, 8, 1, 2, 8)
conv_redir = slim.conv2d(conv2a, 64, [1, 1])
corr1d_redir_concat = tf.concat([corr1d, conv_redir], 3)
conv3 = slim.conv2d(corr1d_redir_concat, 256, [3, 3], stride=2)
conv3_1 = slim.conv2d(conv3, 256, [3, 3])
conv4 = slim.conv2d(conv3_1, 512, [3, 3], stride=2)
conv4_1 = slim.conv2d(conv4, 512, [3, 3])
conv5 = slim.conv2d(conv4_1, 512, [3, 3], stride=2)
conv5_1 = slim.conv2d(conv5, 512, [3, 3])
conv6 = slim.conv2d(conv5, 1024, [3, 3], stride=2)
conv6_1 = slim.conv2d(conv6, 1024, [3, 3])
disp6 = slim.conv2d(conv6_1, 1, [3, 3], activation_fn=None)
resized_6 = tf.image.resize_images(disp6, [
int(math.ceil(IMAGE_HIGHT / 32.)),
int(math.ceil(IMAGE_WITCH / 32.))
])
up_conv5 = slim.conv2d_transpose(conv6_1, 512, [4, 4], stride=2)
iconv5_concat = tf.concat([up_conv5, resized_6, conv5_1], 3)
iconv5 = slim.conv2d(iconv5_concat, 512, [3, 3])
disp5 = slim.conv2d(iconv5, 1, [3, 3], activation_fn=None)
resized_5 = tf.image.resize_images(disp5, [
int(math.ceil(IMAGE_HIGHT / 16.)),
int(math.ceil(IMAGE_WITCH / 16.))
])
up_conv4 = slim.conv2d_transpose(iconv5, 256, [4, 4], stride=2)
iconv4_concat = tf.concat([up_conv4, resized_5, conv4_1], 3)
iconv4 = slim.conv2d(iconv4_concat, 256, [3, 3])
disp4 = slim.conv2d(iconv4, 1, [3, 3], activation_fn=None)
resized_4 = tf.image.resize_images(disp4, [
int(math.ceil(IMAGE_HIGHT / 8.)),
int(math.ceil(IMAGE_WITCH / 8.))
])
up_conv3 = slim.conv2d_transpose(iconv4, 128, [4, 4], stride=2)
iconv3_concat = tf.concat([up_conv3, resized_4, conv3_1], 3)
iconv3 = slim.conv2d(iconv3_concat, 128, [3, 3])
disp3 = slim.conv2d(iconv3, 1, [3, 3], activation_fn=None)
resized_3 = tf.image.resize_images(disp3, [
int(math.ceil(IMAGE_HIGHT / 4.)),
int(math.ceil(IMAGE_WITCH / 4.))
])
up_conv2 = slim.conv2d_transpose(iconv3, 64, [4, 4], stride=2)
iconv2_concat = tf.concat([up_conv2, resized_3, conv2a], 3)
iconv2 = slim.conv2d(iconv2_concat, 64, [3, 3])
disp2 = slim.conv2d(iconv2, 1, [3, 3], activation_fn=None)
resized_2 = tf.image.resize_images(disp2, [
int(math.ceil(IMAGE_HIGHT / 2.)),
int(math.ceil(IMAGE_WITCH / 2.))
])
up_conv1 = slim.conv2d_transpose(iconv2, 32, [4, 4], stride=2)
iconv1_concat = tf.concat([up_conv1, resized_2, conv1a], 3)
iconv1 = slim.conv2d(iconv1_concat, 32, [3, 3])
disp1 = slim.conv2d(iconv1, 1, [3, 3], activation_fn=None)
resized_1 = tf.image.resize_images(disp1,
[IMAGE_HIGHT, IMAGE_WITCH])
up_conv0 = slim.conv2d_transpose(iconv1, 32, [4, 4], stride=2)
iconv0_concat = tf.concat([up_conv0, resized_1, up_1a2a], 3)
iconv0 = slim.conv2d(iconv0_concat, 32, [3, 3])
disp0 = slim.conv2d(iconv0, 1, [3, 3], activation_fn=None)
with tf.name_scope('DRS-net'):
r_conv0_concat = tf.concat(
[abs(up_1a2a - up_1b2b), disp0, up_1a2a], 3)
r_conv0 = slim.conv2d(r_conv0_concat, 32, [3, 3])
r_conv1 = slim.conv2d(r_conv0, 64, [3, 3], stride=2)
c_conv1a = slim.conv2d(conv1a,
16, [3, 3],
scope='c_conv1',
reuse=False)
c_conv1b = slim.conv2d(conv1b,
16, [3, 3],
scope='c_conv1',
reuse=True)
r_corr = correlation.correlation(c_conv1a, c_conv1b, 1, 4, 1, 2, 4)
r_conv1_1_concat = tf.concat([r_conv1, r_corr], 3)
r_conv1_1 = slim.conv2d(r_conv1_1_concat, 64, [3, 3])
r_conv2 = slim.conv2d(r_conv1_1, 128, [3, 3], stride=2)
r_conv2_1 = slim.conv2d(r_conv2, 128, [3, 3])
r_res2 = slim.conv2d(r_conv2_1, 1, [3, 3], activation_fn=None)
r_res2_resize = tf.image.resize_images(r_res2, [
int(math.ceil(IMAGE_HIGHT / 2)),
int(math.ceil(IMAGE_WITCH / 2))
])
r_upconv1 = slim.conv2d_transpose(r_conv2_1, 64, [4, 4], stride=2)
r_iconv1_concat = tf.concat([r_upconv1, r_res2_resize, r_conv1_1],
3)
r_iconv1 = slim.conv2d(r_iconv1_concat, 64, [3, 3])
r_res1 = slim.conv2d(r_iconv1, 1, [3, 3], activation_fn=None)
r_res1_resize = tf.image.resize_images(r_res1,
[IMAGE_HIGHT, IMAGE_WITCH])
r_upconv0 = slim.conv2d_transpose(r_conv1, 32, [4, 4], stride=2)
r_iconv0_concat = tf.concat([r_upconv0, r_res1_resize, r_conv0], 3)
r_iconv0 = slim.conv2d(r_iconv0_concat, 32, [3, 3])
r_res0 = slim.conv2d(r_iconv0, 1, [3, 3], activation_fn=None)
return {
'disp6': disp6,
'disp5': disp5,
'disp4': disp4,
'disp3': disp3,
'disp2': disp2,
'disp1': disp1,
'disp0': disp0 + r_res0
}
def testEq(self):
corr = correlation(rangen=10)
self.assertEqual(corr.rangen, corr.rangen)
self.assertEqual(len(corr.label), 4)
# -*- coding: utf-8 -*-
#! /usr/bin/python
import os.path
rootdir = "E:/bda-pylib/statistics/correlationModel"
os.chdir(rootdir)
from correlation import correlation
if __name__ == '__main__':
col = "V0,V1,V2,V3"
input = "data/iris.csv"
method = 'pearson' # 'pearson', 'kendall', 'spearman'
correlation(col, input, method)
def testMerge(self):
corr = correlation(rangen=13)
self.assertEqual(len(weather),
len(corr.weather_collision_merge(weather, data)))
def testinit(self):
corr = correlation(rangen=5, label='A')
self.assertEqual(corr.rangen, 5)
self.assertEqual(corr.label, 'A')
def cnnmodel(frame1_xyz,frame1_rgb,frame2_xyz,frame2_rgb):
frame1_feat_rgb,_ = get_network('resnet50',frame1_rgb,weight_decay=1e-5, is_training=True)
frame2_feat_rgb,_ = get_network('resnet50',frame2_rgb,weight_decay=1e-5, is_training=True, reuse=True)
frame1_feat = encoder(frame1_xyz)
frame2_feat = encoder(frame2_xyz,reuse=True)
cc_o = correlation(frame2_feat_rgb,frame1_feat_rgb,1,rad,1,1,rad)
cc = tf.reshape(cc_o,[-1, 30*40, dia * dia, 1])
cc_weight = tf.nn.relu(cc)
frame1_feat_o = frame1_feat
frame1_feat = tf.transpose(frame1_feat,[0,3,1,2])
frame1_feat_padded = tf.pad(frame1_feat,paddings=[[0,0],[0,0],[rad,rad],[rad,rad]])
frame1_list = []
for i in xrange(30):
for j in xrange(40):
tmp = frame1_feat_padded[:,:,0+i:2*rad+1+i,0+j:2*rad+1+j]
tmp = tf.reshape(tmp,[-1,64,dia * dia])
frame1_list.append(tmp)
frame1_list = tf.stack(frame1_list,axis=2)
frame1_list = tf.transpose(frame1_list,[0,2,3,1])
frame1_list = frame1_list * cc_weight
frame1_list = tf.nn.max_pool(frame1_list,ksize=[1,1,dia * dia,1],strides=[1,1,dia * dia,1],padding='VALID')
frame1_list = tf.reshape(frame1_list,(-1,30,40,64))
x = tf.concat([frame2_feat,frame1_feat_o,frame1_list],3)
x_s = decoder(x)
x_transl = tflearn.layers.conv.conv_2d(x_s,3,(3,3),strides=1,activation='linear',weight_decay=1e-3,regularizer='L2')
rot = tflearn.layers.conv.conv_2d(x_s,3,(3,3),strides=1,activation='linear',weight_decay=1e-3,regularizer='L2')
x_center = tflearn.layers.conv.conv_2d(x_s,3,(3,3),strides=1,activation='linear',weight_decay=1e-3,regularizer='L2')
x_score = tflearn.layers.conv.conv_2d(x_s,2,(3,3),strides=1,activation='linear',weight_decay=1e-3,regularizer='L2')
x_mask = tflearn.layers.conv.conv_2d(x_s,2,(3,3),strides=1,activation='linear',weight_decay=1e-3,regularizer='L2')
x_boundary = tflearn.layers.conv.conv_2d(x_s,1,(3,3),strides=1,activation='linear',weight_decay=1e-3,regularizer='L2')
x_center = tf.add(x_center,frame2_xyz)
frame2_xyz_ = frame2_xyz - x_center
angle = tf.norm(rot,axis=3)
angle_ = tf.expand_dims(angle,-1)
axis = rot / (angle_ + 0.000001)
c = tf.cos(angle)
v = 1 -c
s = tf.sin(angle)
r00 = axis[:,:,:,0] * axis[:,:,:,0] * v + c
r01 = axis[:,:,:,0] * axis[:,:,:,1] * v - axis[:,:,:,2] * s
r02 = axis[:,:,:,0] * axis[:,:,:,2] * v + axis[:,:,:,1] * s
r10 = axis[:,:,:,0] * axis[:,:,:,1] * v + axis[:,:,:,2] * s
r11 = axis[:,:,:,1] * axis[:,:,:,1] * v + c
r12 = axis[:,:,:,1] * axis[:,:,:,2] * v - axis[:,:,:,0] * s
r20 = axis[:,:,:,0] * axis[:,:,:,2] * v - axis[:,:,:,1] * s
r21 = axis[:,:,:,1] * axis[:,:,:,2] * v + axis[:,:,:,0] * s
r22 = axis[:,:,:,2] * axis[:,:,:,2] * v + c
x = r00 * frame2_xyz_[:,:,:,0] + r01 * frame2_xyz_[:,:,:,1] + r02 * frame2_xyz_[:,:,:,2]
y = r10 * frame2_xyz_[:,:,:,0] + r11 * frame2_xyz_[:,:,:,1] + r12 * frame2_xyz_[:,:,:,2]
z = r20 * frame2_xyz_[:,:,:,0] + r21 * frame2_xyz_[:,:,:,1] + r22 * frame2_xyz_[:,:,:,2]
x_flow = tf.stack((x,y,z),axis=-1)
x_flow = x_flow + x_transl + x_center - frame2_xyz
x_center_p = x_center + x_transl
x_traj = tf.concat([x_center,x_center_p],3)
return rot, x_transl, x_traj, x_flow, x_center, x_mask, x_score, x_boundary
ind = np.arange(len(actions))
width = 0.35
names = ["ML", "MR", "MU", "MD", "ZI", "ZO", "TF", "TB", "REC", "REAN", "ROC", "ROAN"]
fig = plt.figure()
ax = fig.add_subplot(111)
rect1 = ax.bar(ind, prob_general_1, width, facecolor = 'none', hatch = '//', label="Real")
rect2 = ax.bar(ind+width, prob_general_2, width, facecolor='none', hatch='\\\\', label="Synthetic")
ax.axis(ymin=0, ymax=0.5)
ax.set_ylabel('Probability of Action')
ax.set_xlabel('Action')
ax.set_xticks(ind+width)
ax.set_xticklabels(names, fontsize='small')
ax.legend(loc="upper right")
ax.text(5, 0.45, "correlation: %0.3f"%correlation(prob_general_1, prob_general_2), horizontalalignment='center')
#autolabel(ax, rect1)
#autolabel(ax, rect2)
plt.savefig("general_prob_comp.eps")
plt.close()
prob_continue_1 = []
prob_continue_2 = []
prob_continue_1.append(m_continue_1["continue"] * 1.0 / m_continue_1["total"])
prob_continue_2.append(m_continue_2["continue"] * 1.0 / m_continue_2["total"])
for action in actions:
prob_continue_1.append(m_continue_1[action] * 1.0 / m_general_1[action])
prob_continue_2.append(m_continue_2[action] * 1.0 / m_general_2[action])
ind = np.arange(len(actions)+1)
fig = plt.figure()
rects2 = ax.bar(ind+width, prop2_list, width, facecolor='none', hatch='\\\\', label='Synthetic')
# add some
ax.set_ylabel('Probability')
ax.set_xlabel(axis.upper())
#ax.set_title('Popularity of ' + axis)
ax.set_xticks(ind+width)
xtick_lst = []
for v in v_list:
if v % 2 == 0:
xtick_lst.append(v)
else:
xtick_lst.append("")
ax.set_xticklabels(xtick_lst)
ax.axis(ymin=0, ymax=1)
ax.text(N/2 - 1, 0.9, "correlation: %0.3f"%correlation(prop1_list, prop2_list), horizontalalignment='center')
#ax.legend( (rects1[0], rects2[0]), ('Original', 'Generated') )
ax.legend()
def autolabel(rects):
# attach some text labels
for rect in rects:
height = rect.get_height()
ax.text(rect.get_x()+rect.get_width()/2., 1.05*height, '%0.2f'%height, ha='center', va='bottom')
#autolabel(rects1)
#autolabel(rects2)
plt.savefig(axis+".eps")
def cnnmodel(frame1_xyz, frame1_rgb, frame2_xyz, frame2_rgb):
frame1_feat_rgb, _ = get_network('resnet50',
frame1_rgb,
weight_decay=1e-5,
is_training=True)
frame2_feat_rgb, _ = get_network('resnet50',
frame2_rgb,
weight_decay=1e-5,
is_training=True,
reuse=True)
frame1_feat = encoder(frame1_xyz)
frame2_feat = encoder(frame2_xyz, reuse=True)
cc_o = correlation(frame2_feat_rgb, frame1_feat_rgb, 1, rad, 1, 1, rad)
cc = tf.reshape(cc_o, [-1, 30 * 40, dia * dia, 1])
cc_weight = tf.nn.relu(cc)
frame1_feat_o = frame1_feat
frame1_feat = tf.transpose(frame1_feat, [0, 3, 1, 2])
frame1_feat_padded = tf.pad(frame1_feat,
paddings=[[0, 0], [0, 0], [rad, rad],
[rad, rad]])
frame1_list = []
for i in xrange(30):
for j in xrange(40):
tmp = frame1_feat_padded[:, :, 0 + i:2 * rad + 1 + i,
0 + j:2 * rad + 1 + j]
tmp = tf.reshape(tmp, [-1, 64, dia * dia])
frame1_list.append(tmp)
frame1_list = tf.stack(frame1_list, axis=2)
frame1_list = tf.transpose(frame1_list, [0, 2, 3, 1])
frame1_list = frame1_list * cc_weight
frame1_list = tf.nn.max_pool(frame1_list,
ksize=[1, 1, dia * dia, 1],
strides=[1, 1, dia * dia, 1],
padding='VALID')
frame1_list = tf.reshape(frame1_list, (-1, 30, 40, 64))
x = tf.concat([frame2_feat, frame1_feat_o, frame1_list], 3)
x_s = decoder(x)
x_transl = tflearn.layers.conv.conv_2d(x_s,
3, (3, 3),
strides=1,
activation='linear',
weight_decay=1e-3,
regularizer='L2')
rot_quaternion = tflearn.layers.conv.conv_2d(x_s,
4, (3, 3),
strides=1,
activation='linear',
weight_decay=1e-3,
regularizer='L2')
### quaternion normalize
quaternion_norm = tf.norm(rot_quaternion, axis=3) * tf.sign(
rot_quaternion[:, :, :, 0])
quaternion_norm = tf.expand_dims(quaternion_norm, -1)
x_quaternion = rot_quaternion / (quaternion_norm + 0.000001)
w1, x1, y1, z1 = tf.unstack(x_quaternion, axis=-1)
x2, y2, z2 = tf.unstack(frame2_xyz, axis=-1)
wm = -x1 * x2 - y1 * y2 - z1 * z2
xm = w1 * x2 + y1 * z2 - z1 * y2
ym = w1 * y2 + z1 * x2 - x1 * z2
zm = w1 * z2 + x1 * y2 - y1 * x2
x = -wm * x1 + xm * w1 - ym * z1 + zm * y1
y = -wm * y1 + ym * w1 - zm * x1 + xm * z1
z = -wm * z1 + zm * w1 - xm * y1 + ym * x1
x_flow = tf.stack((x, y, z), axis=-1)
x_flow = x_flow + x_transl - frame2_xyz
x_center = tflearn.layers.conv.conv_2d(x_s,
3, (3, 3),
strides=1,
activation='linear',
weight_decay=1e-3,
regularizer='L2')
x_score = tflearn.layers.conv.conv_2d(x_s,
2, (3, 3),
strides=1,
activation='linear',
weight_decay=1e-3,
regularizer='L2')
x_mask = tflearn.layers.conv.conv_2d(x_s,
2, (3, 3),
strides=1,
activation='linear',
weight_decay=1e-3,
regularizer='L2')
x_boundary = tflearn.layers.conv.conv_2d(x_s,
2, (3, 3),
strides=1,
activation='linear',
weight_decay=1e-3,
regularizer='L2')
x_center = tf.add(x_center, frame2_xyz)
xc, yc, zc = tf.unstack(x_center, axis=-1)
wmc = -x1 * xc - y1 * yc - z1 * zc
xmc = w1 * xc + y1 * zc - z1 * yc
ymc = w1 * yc + z1 * xc - x1 * zc
zmc = w1 * zc + x1 * yc - y1 * xc
xc = -wmc * x1 + xmc * w1 - ymc * z1 + zmc * y1
yc = -wmc * y1 + ymc * w1 - zmc * x1 + xmc * z1
zc = -wmc * z1 + zmc * w1 - xmc * y1 + ymc * x1
x_center_p = tf.stack((xc, yc, zc), axis=-1) + x_transl
x_traj = tf.concat([x_center, x_center_p], 3)
return x_quaternion, x_transl, x_traj, x_flow, x_center, x_mask, x_score, x_boundary
def call(self, left, right, training=None):
c1a = self.conv1a(left)
p1a = self.pool1a(c1a)
c3a = self.conv3a(p1a)
p3a = self.pool3a(c3a)
c17a = self.conv17a(p3a)
p8a = self.pool8a(c17a)
c1b = self.conv1b(right)
p1b = self.pool1b(c1b)
c3b = self.conv3b(p1b)
p3b = self.pool3b(c3b)
c17b = self.conv17b(p3b)
p8b = self.pool8b(c17b)
# c = tf.concat([p8a, p8b],axis = 3)
# cc = self.corr(c)
cc = correlation(p8a, p8b)
cc = tf.nn.leaky_relu(cc, 0.1)
ca = self.conva(p8a)
net = tf.concat([ca, cc], axis=3)
c4 = self.conv4(net)
c9 = self.conv9(c4)
p5 = self.pool5(c9)
c10 = self.conv10(p5)
c11 = self.conv11(c10)
p6 = self.pool6(c11)
c12 = self.conv12(p6)
c13 = self.conv13(c12)
p7 = self.pool7(c13)
c14 = self.conv14(p7)
c18 = self.conv18(c14)
u1 = self.up1(c18)
d4 = self.deconv4(c14)
b1 = self.bn1(d4)
merge_2 = tf.concat([c12, b1, u1], axis=3)
c19 = self.conv19(merge_2)
c20 = self.conv20(c19)
u2 = self.up2(c20)
d5 = self.deconv5(c19)
b2 = self.bn2(d5)
merge_3 = tf.concat([c10, b2, u2], axis=3)
c21 = self.conv21(merge_3)
d24 = self.deconv24(c21)
b3 = self.bn3(d24)
c22 = self.conv22(c21)
u3 = self.up3(c22)
merge_4 = tf.concat([c4, b3, u3], axis=3)
c23 = self.conv23(merge_4)
c24 = self.conv24(c23)
u4 = self.up4(c24)
d7 = self.deconv7(c23)
b4 = self.bn4(d7)
# print(p3a.shape,b4.shape,u4.shape)
merge_5 = layers.concatenate([p3a, b4, u4],
axis=3) #([p3b,b4,u4],axis = 3)
c25 = self.conv25(merge_5)
c26 = self.conv26(c25)
u5 = self.up5(c26)
d8 = self.deconv8(c25)
b5 = self.bn5(d8)
merge_6 = tf.concat([p1a, b5, u5], axis=3) #([p1b,b5,u5],axis = 3)
c27 = self.conv27(merge_6)
out = self.conv28(c27)
return out, c26, c24, c22, c20
n = 12
W = 2400
N = 1024
time = range(N)
x = rsg.getRandomSignal(n, W, N)
y = rsg.getRandomSignal(n, W, N)
list_N, list_T, list_avg = at.getAvgTime(50, n, W)
dict_N, dict_T, dict_avg = at.getAvgTime(50, n, W, True)
print('time for list - ' + str(list_avg) + ', time for dict - ' +
str(dict_avg))
autocorrelation = corr.selfcorrelation(x)
correlation = corr.correlation(x, y)
corrR = list(range(int(N / 2)))
fig, (ax1, ax2, ax3, ax4) = plt.subplots(4, 1)
plt.subplots_adjust(left=0.05, bottom=0.1, right=0.97, wspace=0.1)
fig.suptitle('Lab 1.2')
ax1.plot(x, color='r', label='s 1')
ax1.plot(y, color='g', label='s 2')
ax1.set_title('Generated signals')
ax1.set(xlabel='time', ylabel='generated signal')
ax1.legend()
ax2.plot(corrR, autocorrelation, color='r')
ax2.set_title('Autocorrelation (s 1)')