def main():
records = read_csv(input_fname)
records = _subset_extractor(records, INFO_HEADER + VITAL_SIGNS_HEADER \
+ LAB_HEADER)
recordss = hashing(records, 'ICUSTAY_ID')
_sorting(recordss, 'TIMESTAMP')
print('#icu_stays: %d' % len(recordss))
for key, records in recordss.items():
_imputation(records, VITAL_SIGNS_HEADER + LAB_HEADER)
avgs = _get_normal_value(recordss, VITAL_SIGNS_HEADER + LAB_HEADER)
_sorting(recordss, 'TIMESTAMP', reverse=True)
for key, records in recordss.items():
_imputation(records, VITAL_SIGNS_HEADER + LAB_HEADER)
_impute_NAs(recordss, VITAL_SIGNS_HEADER + LAB_HEADER, avgs)
_sorting(recordss, 'TIMESTAMP')
recordss_csv = []
for key in sorted(recordss.keys()):
records = recordss[key]
recordss_csv.extend(records)
write_csv(output_fname, INFO_HEADER + VITAL_SIGNS_HEADER + \
LAB_HEADER, recordss_csv)
def process_dir(d, outfile, file_filter):
""" Generate small dump records for all files in 'd' """
print("Processing directory " + d)
num = 0
for f in os.listdir(d):
fullp = os.path.join(d, f)
if os.path.isdir(fullp) and not f == ".svn":
num += process_dir(fullp, outfile, file_filter)
continue
if file_filter(f):
for pp in settings.pp_configs_to_test:
num += 1
print("DUMP " + fullp + "\n post-processing: " + pp)
outf = os.path.join(os.getcwd(), settings.database_name,
utils.hashing(fullp, pp))
cmd = [utils.assimp_bin_path,"dump",fullp,outf,"-b","-s","-l"] + pp.split()
outfile.write("assimp dump "+"-"*80+"\n")
outfile.flush()
if subprocess.call(cmd, stdout=outfile, stderr=outfile, shell=False):
print("Failure processing " + fullp)
# spit out an empty file to indicate that this failure is expected
with open(outf,'wb') as f:
pass
return num
def process_dir(d, outfile_results, zipin, result):
shellparams = {'stdout':outfile_results, 'stderr':outfile_results, 'shell':False}
print("Processing directory " + d)
for f in os.listdir(d):
fullpath = os.path.join(d, f)
if os.path.isdir(fullpath) and not f == ".svn":
process_dir(fullpath, outfile_results, zipin, result)
continue
for pppreset in settings.pp_configs_to_test:
filehash = utils.hashing(fullpath, pppreset)
failure = False
try:
input_expected = zipin.open(filehash, "r").read()
# empty dump files indicate 'expected import failure'
if not len(input_expected):
failure = True
except KeyError:
#print("Didn't find "+fullpath+" (Hash is "+filehash+") in database")
continue
print("-"*60 + "\n " + os.path.realpath(fullpath) + " pp: " + pppreset)
outfile_actual = mkoutputdir_andgetpath(fullpath, filehash, "ACTUAL")
outfile_expect = mkoutputdir_andgetpath(fullpath, filehash, "EXPECT")
outfile_results.write("assimp dump "+"-"*80+"\n")
outfile_results.flush()
command = [utils.assimp_bin_path,"dump",fullpath,outfile_actual,"-b","-s","-l",pppreset]
r = subprocess.call(command, **shellparams)
if r and not failure:
result.fail(fullpath, outfile_expect, pppreset, IMPORT_FAILURE, r)
continue
elif failure:
result.fail(fullpath, outfile_expect, pppreset, EXPECTED_FAILURE_NOT_MET)
continue
with open(outfile_expect, "wb") as s:
s.write(input_expected)
with open(outfile_actual, "rb") as s:
input_actual = s.read()
if len(input_expected) != len(input_actual):
result.fail(fullpath, outfile_expect, pppreset, DATABASE_LENGTH_MISMATCH,
len(input_expected), len(input_actual))
continue
outfile_results.write("assimp cmpdump "+"-"*80+"\n")
outfile_results.flush()
command = [utils.assimp_bin_path,'cmpdump',outfile_actual,outfile_expect]
if subprocess.call(command, **shellparams) != 0:
result.fail(fullpath, outfile_expect, pppreset, DATABASE_VALUE_MISMATCH)
continue
result.ok(fullpath, pppreset, COMPARE_SUCCESS,
len(input_expected))
def process_dir(d, outfile, file_filter):
""" Generate small dump records for all files in 'd' """
print("Processing directory " + d)
num = 0
for f in os.listdir(d):
fullp = os.path.join(d, f)
if os.path.isdir(fullp) and not f == ".svn":
num += process_dir(fullp, outfile, file_filter)
continue
if file_filter(f):
for pp in settings.pp_configs_to_test:
num += 1
print("DUMP " + fullp + "\n post-processing: " + pp)
outf = os.path.join(os.getcwd(), settings.database_name,
utils.hashing(fullp, pp))
cmd = [assimp_bin_path, "dump", fullp, outf, "-b", "-s", "-l"
] + pp.split()
outfile.write("assimp dump " + "-" * 80 + "\n")
outfile.flush()
if subprocess.call(cmd,
stdout=outfile,
stderr=outfile,
shell=False):
print("Failure processing " + fullp)
# spit out an empty file to indicate that this failure is expected
with open(outf, 'wb') as f:
pass
return num
def main():
records = read_csv(input_fname)
recordss = hashing(records, 'ICUSTAY_ID')
print('#icu_stays: %d' % len(recordss))
recordss = _excluding(recordss)
print('#icu_stays: %d (excluding stays having <= %d records)'\
%(len(recordss), 6))
_sorting(recordss, 'TIMESTAMP')
recordss_avg = {}
#for key, records in recordss.items():
for i, key in enumerate(sorted(recordss.keys())):
records = recordss[key]
records_avg = _averaging(records)
recordss_avg[key] = records_avg
print('%d/%d' % (i, len(recordss.keys())))
recordss_csv = []
for key in sorted(recordss_avg.keys()):
records = recordss_avg[key]
recordss_csv.extend(records)
write_csv(output_fname, INFO_HEADER + VITAL_SIGNS_HEADER + \
LAB_HEADER, recordss_csv)
def send_your_data():
"""
This is a route for ALL NODES. When a new node is inserted in
the RING (via the '/bootstrap/node/join' route), then the NEXT neighbor
of that newly inserted node must send part of its database to that new
node.
The data that will be sent are owned by the new node, because they
belong to its keyspace.
"""
print("The newly inserted PREVIOUS node requested its data...")
data = {}
for key in node.storage:
hash_key = hashing(key)
if(hash_key <= node.prev_id):
data[key] = node.storage[key]
print("The data that are now owned by the previous node are: ", data)
print("Our updated database now is:", node.storage)
return data, 200
def main():
records = read_csv(input_fname)
# first get statistics
neg_values = {}
for record in records:
if record['case/control'] == 'case':
continue
for name in HEADER:
val = record[name]
if val != 'NA':
neg_values[name] = neg_values.get(name, list())
neg_values[name].append(float(val))
print('negative mean/std')
for name in HEADER:
print('{:40s} | mean: {:.3f}, std: {:.3f}'.format(
name, np.mean(neg_values[name]), np.std(neg_values[name])))
# and then imputing values with carry forward in the middle & negative mean with no value
# multiproc needed
recordss = hashing(records, 'ICUSTAY_ID')
cnt = 0
for key, records in recordss.items():
print('processing {}... {} / {}'.format(key, cnt, len(recordss)))
cnt += 1
records = sorted(records, key=itemgetter('TIMESTAMP'))
# set intitial value
prev_val = copy.copy(records[0])
for key in HEADER:
if prev_val[key] == 'NA':
prev_val[key] = np.mean(neg_values[key])
for record in records:
for key in HEADER:
val = record[key]
if val == 'NA':
record[key] = prev_val[key]
else:
prev_val[key] = val
recordss_csv = []
for key in sorted(recordss.keys()):
records = recordss[key]
recordss_csv.extend(records)
write_csv(output_fname, INFO_HEADER + HEADER, recordss_csv)
def api_caller_all_orders(base_url, trade_pair):
'''
base_url:base url on which query strings and hashing signatures are appended
trade_pair: input trade_pair symbol
api_caller_all_orders: function that constructs url, signatures (using utils functions) and inokes api
'''
try:
servertimeint = requests.get("https://api.binance.us/api/v1/time")
timestamp = servertimeint.json()['serverTime']
# query_string = 'timestamp={}&symbol={}'.format(timestamp,"BTCUSDT")
query_string = 'timestamp={}&symbol={}'.format(timestamp, trade_pair)
url = base_url + '?' + query_string + '&signature=' + hashing(
query_string, secret)
r = requests.get(url, headers={"X-MBX-APIKEY": api_key})
return r
except Exception as e:
logging.debug(
"Error while retrieving response from binance all_orders api:", e)
'''returns the response from api call in List of jsons format'''
def main():
records = read_csv(input_fname)
recordss = hashing(records, 'ICUSTAY_ID')
print('#icu_stays: %d' % len(recordss))
mats = {}
ys = {}
for key, records in recordss.items():
mat = []
for record in records:
vec = [
float(record[name]) for name in VITAL_SIGNS_HEADER + LAB_HEADER
]
vec = np.asarray(np.expand_dims(vec, axis=0))
mat.append(vec)
mat = np.concatenate(mat, axis=0)
mat = mat.astype(np.float32)
mats[key] = mat
ys[key] = records[0]['case/control']
data = {'mats': mats, 'ys': ys}
with open(output_fname, 'wb') as fp:
pickle.dump(data, fp)
def main():
records = read_csv(input_fname)
recordss = hashing(records, 'ICUSTAY_ID')
for key, records in recordss.items():
headers = VITAL_SIGNS_HEADER + LAB_HEADER
xs = [[] for _ in range(len(headers))
] # relative time for start point in hours
ys = [[] for _ in range(len(headers))
] # actual value of the feature at that point
label = records[0]['case/control']
records = sorted(records, key=itemgetter('TIMESTAMP'))
for t, record in enumerate(records):
for hi, hname in enumerate(headers):
value = record[hname]
if value != 'NA':
xs[hi].append(t)
ys[hi].append(float(value))
fname = '{}_{}.png'.format(key, label)
plot(xs, ys, headers, fname)
def main():
values = {}
records = read_csv(input_fname)
records_dict = hashing(records, 'ICUSTAY_ID')
print('#icu_stays: %d' % len(records_dict))
vals = [[] for _ in range(len(STATS_HEADER))]
for key, record in records_dict.items():
# single measure
# only negative samples
if record[0]['case/control'] == 'case':
continue
for measure in record:
for head, value in measure.items():
if head in STATS_HEADER:
hidx = STATS_HEADER.index(head)
try:
vals[hidx].extend([float(value)])
except:
pass
pdb.set_trace()
pdb.set_trace()
output_csv = os.path.join(results_path, 'feature_stats_for_norm.csv')
with open(output_csv, 'wt') as f:
output_str = '{:15s},{:5s},{:5s}'.format('Feature', 'Avg', 'Std')
f.writeline(output_str + '\n')
for i, header in enumerate(STATS_HEADER):
output_str = '{:15s},{:.3f},{:.3f}'.format(
header,
np.mean(vals[i]),
np.std(vals[i]),
)
print(output_str)
f.writeline(output_str + '\n')
if __name__ == '__main__':
try:
seed_num = int(sys.argv[1])
except:
seed_num = 0
makedirs(output_root)
_05_fname = os.path.join(dir_path, '05_converter.pkl')
_icd_fname = os.path.join(input_root, 'icd_code.csv')
_meta_fname = os.path.join(dir_path, 'meta_data.pkl')
dynamic_data = read_pkl(_05_fname)
meta_data = read_pkl(_meta_fname)
icd_codes = read_csv(_icd_fname)
icd_codes = hashing(icd_codes, 'ICUSTAY_ID')
# copy file
save_pkl(meta_data, os.path.join(output_root, 'meta_data.pkl'))
mats = dynamic_data['mats']
labels = dynamic_data['ys']
for key, data in mats.items():
# only use test data
if key not in meta_data['test_icuid']:
continue
data_length = data.shape[0]
if data_length < 90:
continue
def delete():
"""
This is a route for ALL NODES. The client sends sequests to this route
so that a key-value pair is deleted.
In the case of "eventual consistency", the client gets a response to the
request after the primary replica of the key-value pair is deleted. The
other replicas are deleted by a thread, without blocking the response
to the client.
"""
# Extract the key of the 'delete' request.
key = request.form['key']
hash_key = hashing(key)
if between(hash_key, node.my_id, node.prev_id):
# We are the node that owns that hash key.
if (key in node.storage):
# Delete the key-value pair from our database.
del node.storage[key]
print("The key:", key, "was deleted from our database.")
response = "The key-value pair was successfully deleted."
status = 200
if (node.next_id != node.my_id and K_replicas > 1):
if (type_replicas == "eventual consistency"):
thread = Thread(
target=delete_replicas_in_next_nodes,
args=(key, node)
)
thread.start()
elif type_replicas == "linearizability":
response, status = delete_replicas_in_next_nodes(
key, node
)
else:
response = "There isn't such a key-value pair."
status = 404
else:
# We are NOT the node that owns that hash key.
# So we retransmit the request until we find the
# node-owner of the key.
url_next = "http://" + node.next_ip + ":" + \
str(node.next_port) + "/delete"
data = {
'key': key,
}
r = requests.post(url_next, data)
if r.status_code != 200:
print("Something went wrong with the request to the next node.")
response = r.text
status = r.status_code
return response, status
l = int(dim)
Ptp01, Ptp05 = [], []
S_x = int(l*0.7) # 610
group_member = 40 # 2,5,10,16,20,40
W = np.random.random([dim, l])
U, S, V = np.linalg.svd(W)
W = U[:, :l]
param = {'encode': 'soft', 'normalize': 3, 'n_Clus': n_Clus}
# Assign data to groups
data = np.array(dataset['data_x'])
data = zscore(np.array(dataset['data_x']), axis=0) # N*d
groups = partition_data(data, group_member, partitioning='random')
# Compute group representations
group_vec, VLAD_Codebook = vlad_group_representation(data, groups, param)
group_vec = np.array(group_vec).T
group_vec = hashing(group_vec, W, S_x)
# The embedding for H0 queries
n_q0 = len(dataset['H0_id'])
H0_data = zscore(np.array(dataset['H0_x']).T, axis=1) # LFW
# H0_data = zscore(np.array(dataset['H0_x']), axis=0) # CFP
H0_data = [VLADEncoding(np.expand_dims(H0_data[i, :], axis=0), VLAD_Codebook,
encode=param['encode'], normalize=param['normalize']) for i in range(n_q0)]
Q0 = np.array(H0_data).T
Q0 = hashing(np.array(H0_data).T, W, S_x)
H0_claimed_id = np.random.randint(0, len(groups['ind']), size=n_q0).astype(np.int)
D00 = np.linalg.norm(Q0 - group_vec[:, H0_claimed_id], axis=0)
# The embedding for H1 queries
n_q1 = len(dataset['H1_id'])
H1_group_id = np.zeros(n_q1)
H1_data = zscore(np.array(dataset['H1_x']), axis=0)
H1_data = [VLADEncoding(np.expand_dims(H1_data[i, :], axis=0), VLAD_Codebook,
def main():
icd_codes = read_csv(icd_fname)
icd_codes = hashing(icd_codes, 'ICUSTAY_ID')
with open(input_fname, 'rb') as fp:
records = pickle.load(fp)
mats = records['mats']
labels = records['ys']
xs = [] # static features
ys = [] # case/control
icustay_id = []
for key, data in mats.items():
avg = np.mean(data, axis=0)
std = np.std(data, axis=0)
max_val = np.max(data, axis=0)
min_val = np.min(data, axis=0)
icd_code = icd_codes[key]
assert len(icd_code) == 1
_icd_code = icd_code[0]
icd_code = [_icd_code[name] for name in HEADER]
icd_code += [str(data.shape[0])]
for _i in range(len(STR_HEADER) - 1):
# only for gender and race
c = STR_HEADER_CLASS[_i].index(_icd_code[STR_HEADER[_i]])
icd_code.extend([str(c)])
if _icd_code['Insurance2'] == 'Private':
icd_code.extend(['1', '0'])
elif _icd_code['Insurance2'] == 'Public':
icd_code.extend(['0', '1'])
else:
icd_code.extend(['0', '0'])
#feat = np.concatenate((avg, std, max_val, min_val, [float(icd_code[0])]))
feat = np.concatenate((avg, std, max_val, min_val, icd_code))
xs.append(np.expand_dims(feat, axis=0))
if labels[key] == 'control':
ys.append(0)
elif labels[key] == 'case':
ys.append(1)
else:
raise ValueError()
icustay_id.append(key)
cs = np.zeros(len(feat), dtype=int)
cs[-(len(icd_code) - 1):] = 1 # if its categorical (1) or not (0)
xs = np.concatenate(xs, axis=0)
ys = np.asarray(ys)
print(xs.shape, ys.shape)
data = {
'xs': xs,
'cs': cs,
'ys': ys,
'icustay_id': icustay_id,
}
print('ratio: %.4f' % (np.sum(ys == 1) / len(ys)))
print(xs[0], ys[0])
print(xs[-1], ys[1])
with open(output_fname, 'wb') as fp:
pickle.dump(data, fp)
W = U[:, :l]
# mat = W.T @ W
param = {'method': 'EoA', 'agg': 'pseudo', 'W': W, 'S_x': S_x}
# Assign data to groups
# data = np.array(dataset['data_x'])
data = zscore(np.array(dataset['data_x']), axis=0) # N*d
groups = partition_data(data, group_member, partitioning='random')
# Compute group representations
group_vec = baseline_group_representation(groups, param)
group_vec = np.array(group_vec)
# The embedding for H0 queries
n_q0 = len(dataset['H0_id'])
# H0_data = np.array(dataset['H0_x'])
H0_data = zscore(np.array(dataset['H0_x']), axis=1)
# H0_data = zscore(np.array(dataset['H0_x']).T, axis=1) # CFP
Q0 = hashing(H0_data, param['W'], param['S_x'])
H0_claimed_id = np.random.randint(0, len(groups['ind']),
size=n_q0).astype(np.int)
D00 = np.linalg.norm(Q0 - group_vec[:, H0_claimed_id], axis=0)
# The embedding for H1 queries
n_q1 = len(dataset['H1_id'])
H1_group_id = np.zeros(n_q1)
# H1_data = np.array(dataset['H1_x']).T
H1_data = zscore(np.array(dataset['H1_x']).T, axis=1)
Q1 = hashing(H1_data, param['W'], param['S_x'])
# Determine the group identity of H1 queries
for i in range(len(groups['ind'])):
group_id = [dataset['data_id'][x] for x in groups['ind'][i]]
a = [
n for n, x in enumerate(dataset['H1_id']) for y in group_id
def query():
"""
This is a route for ALL NODES. The client sends a get request
to this route, so that the value of a key-value pair is retrieved.
"""
key = request.form['key']
hash_key = hashing(key)
if (key == '*'):
# The user wants to GET all key-value pairs from the database,
# so we send a request to the BOOTSTRAP server. The BOOTSTRAP
# server "knows" each node in the ring network.
url_next = "http://" + bootstrap_ip + ":" + str(bootstrap_port) + \
"/bootstrap/queryall"
r = requests.get(url_next)
return r.text[:-1], 200
if between(hash_key, node.my_id, node.prev_id):
# If we are the node that owns the hash key.
if key not in node.storage:
# The key-value pair doesn't exist in the toy-chord database.
response = "Sorry, we don't have that song."
status = 404
return response, status
if type_replicas == "linearizability":
if key not in node.storage:
url_next = "http://" + node.next_ip + ":" + str(node.next_port) \
+ "/query"
data = {
'key' : key
}
r = requests.post(url_next, data)
response = r.text
status = r.status_code
else:
data_to_next = {
'id': node.my_id,
'key': key
}
url_next = "http://" + node.next_ip + ":" + str(node.next_port) \
+ "/query/replicas"
print("Submitting the query to the next node.")
r = requests.post(url_next, data_to_next)
if r.status_code == 418:
response = node.storage[key]
status = 200
else:
response = r.text
status = r.status_code
elif type_replicas == "eventual consistency":
if key not in node.storage:
url_next = "http://" + node.next_ip + ":" + str(node.next_port) \
+ "/query"
data = {
'key': key
}
r = requests.post(url_next, data)
response = r.text
status = r.status_code
else:
response = node.storage[key]
status = 200
return response, status
def insert():
"""
This is a route for ALL NODES.
The client sends POST requests to this route so that new key-value pairs
are inserted.
In the case of "eventual consistency", the client gets a response to the
request after the primary replica of the key-value pair is inserted. The
other replicas are inserted by a thread, without blocking the response
to the client.
"""
# Extract the key and value from the data of the 'insert' request.
key = request.form['key']
value = request.form['value']
# Hash the key of the inserted key-value pair in order to find the node
# that it's owned by.
hash_key = hashing(key)
if between(hash_key, node.my_id, node.prev_id):
# If we are the node that owns this hask key,
# we insert the key-value pair in our database.
node.storage[key] = value
print(
"A key-value pair with key: '" + key +
"' and hash:", hash_key, "was inserted/updated."
)
# If we aren't the only Node in the Ring, we forward the key-value pair
# to the next node while decreasing the replication factor by 1.
if (node.next_id != node.my_id and K_replicas > 1):
if type_replicas == "eventual consistency":
# In this case, we start a thread that handles the
# forwarding of the key-value pair to the next nodes
thread = Thread(
target=forward_replicas_to_next_node,
args=(key, value, node)
)
thread.start()
# We don't care if the forwarding of the key-value pair has
# completed. We return a response and a status code to the
# user.
message = "The key-value pair was successfully inserted."
status_code = 200
elif type_replicas == "linearizability":
# In this case, we wait until the key-value pair is inserted
# in the next nodes and then we return a response to the user.
message, status_code = forward_replicas_to_next_node(
key=key,
value=value,
node=node
)
return message, status_code
return "The key-value pair was successfully inserted.", 200
else:
# Otherwise, if we don't own the hash key, we forward the data
# to the NEXT node. The data will be forwarded by the next nodes
# until they reach the node that is responsible for the hash key.
url_next = "http://" + node.next_ip + ":" + \
str(node.next_port) + "/insert"
data = {
'key': key,
'value': value,
}
r = requests.post(url_next, data)
if r.status_code != 200:
message = "Error while retransmitting the key-value pair."
print(message)
return message, 500
return r.text
def process_dir(d, outfile_results, zipin, result):
shellparams = {
'stdout': outfile_results,
'stderr': outfile_results,
'shell': False
}
print("Processing directory " + d)
for f in os.listdir(d):
fullpath = os.path.join(d, f)
if os.path.isdir(fullpath) and not f == ".svn":
process_dir(fullpath, outfile_results, zipin, result)
continue
if f in settings.files_to_ignore:
print("Ignoring " + f)
continue
for pppreset in settings.pp_configs_to_test:
filehash = utils.hashing(fullpath, pppreset)
failure = False
try:
input_expected = zipin.open(filehash, "r").read()
# empty dump files indicate 'expected import failure'
if not len(input_expected):
failure = True
except KeyError:
#print("Didn't find "+fullpath+" (Hash is "+filehash+") in database")
continue
# Ignore extensions via settings.py configured list
# todo: Fix for multi dot extensions like .skeleton.xml
ext = os.path.splitext(fullpath)[1].lower()
if ext != "" and ext in settings.exclude_extensions:
continue
print("-" * 60 + "\n " + os.path.realpath(fullpath) + " pp: " +
pppreset)
outfile_actual = mkoutputdir_andgetpath(fullpath, filehash,
"ACTUAL")
outfile_expect = mkoutputdir_andgetpath(fullpath, filehash,
"EXPECT")
outfile_results.write("assimp dump " + "-" * 80 + "\n")
outfile_results.flush()
command = [
utils.assimp_bin_path, "dump", fullpath, outfile_actual, "-b",
"-s", "-l"
] + pppreset.split()
r = subprocess.call(command, **shellparams)
if r and not failure:
result.fail(fullpath, outfile_expect, pppreset, IMPORT_FAILURE,
r)
continue
elif failure and not r:
result.fail(fullpath, outfile_expect, pppreset,
EXPECTED_FAILURE_NOT_MET)
continue
with open(outfile_expect, "wb") as s:
s.write(input_expected)
try:
with open(outfile_actual, "rb") as s:
input_actual = s.read()
except IOError:
continue
if len(input_expected) != len(input_actual):
result.fail(fullpath, outfile_expect, pppreset,
DATABASE_LENGTH_MISMATCH, len(input_expected),
len(input_actual))
continue
outfile_results.write("assimp cmpdump " + "-" * 80 + "\n")
outfile_results.flush()
command = [
utils.assimp_bin_path, 'cmpdump', outfile_actual,
outfile_expect
]
if subprocess.call(command, **shellparams) != 0:
result.fail(fullpath, outfile_expect, pppreset,
DATABASE_VALUE_MISMATCH)
continue
result.ok(fullpath, pppreset, COMPARE_SUCCESS, len(input_expected))
def process_dir(d, outfile_results, zipin, result ):
shellparams = {'stdout':outfile_results, 'stderr':outfile_results, 'shell':False}
print("Processing directory " + d)
all = ""
for f in sorted(os.listdir(d)):
fullpath = os.path.join(d, f)
if os.path.isdir(fullpath) and not f[:1] == '.':
process_dir(fullpath, outfile_results, zipin, result)
continue
if f in settings.files_to_ignore or os.path.splitext(f)[1] in settings.exclude_extensions:
print("Ignoring " + f)
return
for pppreset in settings.pp_configs_to_test:
filehash = utils.hashing(fullpath, pppreset)
failure = False
try:
input_expected = zipin.open(filehash, "r").read()
# empty dump files indicate 'expected import failure'
if not len(input_expected):
failure = True
except KeyError:
# TODO(acgessler): Keep track of this and report as error in the end.
print("Didn't find "+fullpath+" (Hash is "+filehash+") in database. Outdated "+\
"regression database? Use gen_db.zip to re-generate.")
continue
print("-"*60 + "\n " + os.path.realpath(fullpath) + " pp: " + pppreset)
outfile_actual = prepare_output_dir(fullpath, filehash, "ACTUAL")
outfile_expect = prepare_output_dir(fullpath, filehash, "EXPECT")
outfile_results.write("assimp dump "+"-"*80+"\n")
outfile_results.flush()
assimp_bin_path = getEnvVar("assimp_path")
command = [assimp_bin_path,
"dump",
fullpath, outfile_actual, "-b", "-s", "-l" ] +\
pppreset.split()
print( "command = " + str( command ) )
r = subprocess.call(command, **shellparams)
outfile_results.flush()
if r and not failure:
result.fail(fullpath, outfile_expect, pppreset, IMPORT_FAILURE, r)
outfile_results.write("Failed to import\n")
continue
elif failure and not r:
result.fail(fullpath, outfile_expect, pppreset, EXPECTED_FAILURE_NOT_MET)
outfile_results.write("Expected import to fail\n")
continue
elif failure and r:
result.ok(fullpath, pppreset, EXPECTED_FAILURE)
outfile_results.write("Failed as expected, skipping.\n")
continue
with open(outfile_expect, "wb") as s:
s.write(input_expected)
try:
with open(outfile_actual, "rb") as s:
input_actual = s.read()
except IOError:
continue
outfile_results.write("Expected data length: {0}\n".format(len(input_expected)))
outfile_results.write("Actual data length: {0}\n".format(len(input_actual)))
failed = False
if len(input_expected) != len(input_actual):
result.fail(fullpath, outfile_expect, pppreset, DATABASE_LENGTH_MISMATCH,
len(input_expected), len(input_actual))
# Still compare the dumps to see what the difference is
failed = True
outfile_results.write("assimp cmpdump "+"-"*80+"\n")
outfile_results.flush()
command = [ assimp_bin_path, 'cmpdump', outfile_actual, outfile_expect ]
if subprocess.call(command, **shellparams) != 0:
if not failed:
result.fail(fullpath, outfile_expect, pppreset, DATABASE_VALUE_MISMATCH)
continue
result.ok(fullpath, pppreset, COMPARE_SUCCESS, len(input_expected))