def main():
    records = read_csv(input_fname)
    records = _subset_extractor(records, INFO_HEADER + VITAL_SIGNS_HEADER \
                                + LAB_HEADER)
    recordss = hashing(records, 'ICUSTAY_ID')
    _sorting(recordss, 'TIMESTAMP')
    print('#icu_stays: %d' % len(recordss))

    for key, records in recordss.items():
        _imputation(records, VITAL_SIGNS_HEADER + LAB_HEADER)

    avgs = _get_normal_value(recordss, VITAL_SIGNS_HEADER + LAB_HEADER)
    _sorting(recordss, 'TIMESTAMP', reverse=True)
    for key, records in recordss.items():
        _imputation(records, VITAL_SIGNS_HEADER + LAB_HEADER)

    _impute_NAs(recordss, VITAL_SIGNS_HEADER + LAB_HEADER, avgs)

    _sorting(recordss, 'TIMESTAMP')

    recordss_csv = []
    for key in sorted(recordss.keys()):
        records = recordss[key]
        recordss_csv.extend(records)

    write_csv(output_fname, INFO_HEADER + VITAL_SIGNS_HEADER + \
               LAB_HEADER, recordss_csv)
Ejemplo n.º 2
0
def process_dir(d, outfile, file_filter):
    """ Generate small dump records for all files in 'd' """
    print("Processing directory " + d)

    num = 0
    for f in os.listdir(d):
        fullp = os.path.join(d, f)
        if os.path.isdir(fullp) and not f == ".svn":
            num += process_dir(fullp, outfile, file_filter)
            continue

        if file_filter(f):
            for pp in settings.pp_configs_to_test:
                num += 1
                print("DUMP " + fullp + "\n post-processing: " + pp)
                outf = os.path.join(os.getcwd(), settings.database_name,
                    utils.hashing(fullp, pp))

                cmd = [utils.assimp_bin_path,"dump",fullp,outf,"-b","-s","-l"] + pp.split()
                outfile.write("assimp dump "+"-"*80+"\n")
                outfile.flush()
                if subprocess.call(cmd, stdout=outfile, stderr=outfile, shell=False):
                    print("Failure processing " + fullp)

                    # spit out an empty file to indicate that this failure is expected
                    with open(outf,'wb') as f:
                        pass
    return num
Ejemplo n.º 3
0
def process_dir(d, outfile_results, zipin, result):
    shellparams = {'stdout':outfile_results, 'stderr':outfile_results, 'shell':False}

    print("Processing directory " + d)
    for f in os.listdir(d):
        fullpath = os.path.join(d, f)
        if os.path.isdir(fullpath) and not f == ".svn":
            process_dir(fullpath, outfile_results, zipin, result)
            continue

        for pppreset in settings.pp_configs_to_test:
            filehash = utils.hashing(fullpath, pppreset)
            failure = False
            try:
                input_expected = zipin.open(filehash, "r").read()
                # empty dump files indicate 'expected import failure'
                if not len(input_expected):
                   failure = True
            except KeyError:
                #print("Didn't find "+fullpath+" (Hash is "+filehash+") in database")
                continue

            print("-"*60 + "\n  " + os.path.realpath(fullpath) + " pp: " + pppreset) 
            
            outfile_actual = mkoutputdir_andgetpath(fullpath, filehash, "ACTUAL")
            outfile_expect = mkoutputdir_andgetpath(fullpath, filehash, "EXPECT")
            outfile_results.write("assimp dump    "+"-"*80+"\n")
            outfile_results.flush()

            command = [utils.assimp_bin_path,"dump",fullpath,outfile_actual,"-b","-s","-l",pppreset]
            r = subprocess.call(command, **shellparams)

            if r and not failure:
                result.fail(fullpath, outfile_expect, pppreset, IMPORT_FAILURE, r)
                continue
            elif failure:
                result.fail(fullpath, outfile_expect, pppreset, EXPECTED_FAILURE_NOT_MET)
                continue
            
            with open(outfile_expect, "wb") as s:
                s.write(input_expected) 
                
            with open(outfile_actual, "rb") as s:
                input_actual = s.read() 
                
            if len(input_expected) != len(input_actual):
                result.fail(fullpath, outfile_expect, pppreset, DATABASE_LENGTH_MISMATCH,
                        len(input_expected), len(input_actual))
                continue

            outfile_results.write("assimp cmpdump "+"-"*80+"\n")
            outfile_results.flush()

            command = [utils.assimp_bin_path,'cmpdump',outfile_actual,outfile_expect]
            if subprocess.call(command, **shellparams) != 0:
                result.fail(fullpath, outfile_expect, pppreset, DATABASE_VALUE_MISMATCH)
                continue 
            
            result.ok(fullpath, pppreset, COMPARE_SUCCESS, 
                len(input_expected))     
Ejemplo n.º 4
0
def process_dir(d, outfile, file_filter):
    """ Generate small dump records for all files in 'd' """
    print("Processing directory " + d)

    num = 0
    for f in os.listdir(d):
        fullp = os.path.join(d, f)
        if os.path.isdir(fullp) and not f == ".svn":
            num += process_dir(fullp, outfile, file_filter)
            continue

        if file_filter(f):
            for pp in settings.pp_configs_to_test:
                num += 1
                print("DUMP " + fullp + "\n post-processing: " + pp)
                outf = os.path.join(os.getcwd(), settings.database_name,
                                    utils.hashing(fullp, pp))

                cmd = [assimp_bin_path, "dump", fullp, outf, "-b", "-s", "-l"
                       ] + pp.split()
                outfile.write("assimp dump " + "-" * 80 + "\n")
                outfile.flush()
                if subprocess.call(cmd,
                                   stdout=outfile,
                                   stderr=outfile,
                                   shell=False):
                    print("Failure processing " + fullp)

                    # spit out an empty file to indicate that this failure is expected
                    with open(outf, 'wb') as f:
                        pass
    return num
Ejemplo n.º 5
0
def main():
    records = read_csv(input_fname)
    recordss = hashing(records, 'ICUSTAY_ID')
    print('#icu_stays: %d' % len(recordss))

    recordss = _excluding(recordss)
    print('#icu_stays: %d (excluding stays having <= %d records)'\
          %(len(recordss), 6))

    _sorting(recordss, 'TIMESTAMP')

    recordss_avg = {}
    #for key, records in recordss.items():
    for i, key in enumerate(sorted(recordss.keys())):
        records = recordss[key]
        records_avg = _averaging(records)
        recordss_avg[key] = records_avg
        print('%d/%d' % (i, len(recordss.keys())))

    recordss_csv = []
    for key in sorted(recordss_avg.keys()):
        records = recordss_avg[key]
        recordss_csv.extend(records)

    write_csv(output_fname, INFO_HEADER + VITAL_SIGNS_HEADER + \
              LAB_HEADER, recordss_csv)
Ejemplo n.º 6
0
def send_your_data():
    """
    This is a route for ALL NODES. When a new node is inserted in
    the RING (via the '/bootstrap/node/join' route), then the NEXT neighbor
    of that newly inserted node must send part of its database to that new
    node.

    The data that will be sent are owned by the new node, because they
    belong to its keyspace.
    """

    print("The newly inserted PREVIOUS node requested its data...")

    data = {}

    for key in node.storage:
        hash_key = hashing(key)
        if(hash_key <= node.prev_id):
            data[key] = node.storage[key]

    print("The data that are now owned by the previous node are: ", data)

    print("Our updated database now is:", node.storage)

    return data, 200
def main():
    records = read_csv(input_fname)

    # first get statistics
    neg_values = {}
    for record in records:
        if record['case/control'] == 'case':
            continue

        for name in HEADER:
            val = record[name]
            if val != 'NA':
                neg_values[name] = neg_values.get(name, list())
                neg_values[name].append(float(val))

    print('negative mean/std')
    for name in HEADER:
        print('{:40s} | mean: {:.3f}, std: {:.3f}'.format(
            name, np.mean(neg_values[name]), np.std(neg_values[name])))

    # and then imputing values with carry forward in the middle & negative mean with no value
    # multiproc needed
    recordss = hashing(records, 'ICUSTAY_ID')
    cnt = 0
    for key, records in recordss.items():
        print('processing {}... {} / {}'.format(key, cnt, len(recordss)))
        cnt += 1

        records = sorted(records, key=itemgetter('TIMESTAMP'))
        # set intitial value
        prev_val = copy.copy(records[0])
        for key in HEADER:
            if prev_val[key] == 'NA':
                prev_val[key] = np.mean(neg_values[key])

        for record in records:
            for key in HEADER:
                val = record[key]
                if val == 'NA':
                    record[key] = prev_val[key]
                else:
                    prev_val[key] = val

    recordss_csv = []
    for key in sorted(recordss.keys()):
        records = recordss[key]
        recordss_csv.extend(records)

    write_csv(output_fname, INFO_HEADER + HEADER, recordss_csv)
def api_caller_all_orders(base_url, trade_pair):
    '''
    base_url:base url on which query strings and hashing signatures  are appended 
    trade_pair: input trade_pair symbol 
    api_caller_all_orders: function that constructs url, signatures (using utils functions) and inokes api
    '''

    try:
        servertimeint = requests.get("https://api.binance.us/api/v1/time")
        timestamp = servertimeint.json()['serverTime']
        # query_string = 'timestamp={}&symbol={}'.format(timestamp,"BTCUSDT")
        query_string = 'timestamp={}&symbol={}'.format(timestamp, trade_pair)
        url = base_url + '?' + query_string + '&signature=' + hashing(
            query_string, secret)
        r = requests.get(url, headers={"X-MBX-APIKEY": api_key})
        return r
    except Exception as e:
        logging.debug(
            "Error while retrieving response from binance all_orders api:", e)
    '''returns the response from api call in List of jsons format'''
Ejemplo n.º 9
0
def main():
    records = read_csv(input_fname)
    recordss = hashing(records, 'ICUSTAY_ID')
    print('#icu_stays: %d' % len(recordss))
    mats = {}
    ys = {}

    for key, records in recordss.items():
        mat = []
        for record in records:
            vec = [
                float(record[name]) for name in VITAL_SIGNS_HEADER + LAB_HEADER
            ]
            vec = np.asarray(np.expand_dims(vec, axis=0))
            mat.append(vec)
        mat = np.concatenate(mat, axis=0)
        mat = mat.astype(np.float32)
        mats[key] = mat
        ys[key] = records[0]['case/control']

    data = {'mats': mats, 'ys': ys}
    with open(output_fname, 'wb') as fp:
        pickle.dump(data, fp)
Ejemplo n.º 10
0
def main():
    records = read_csv(input_fname)
    recordss = hashing(records, 'ICUSTAY_ID')

    for key, records in recordss.items():

        headers = VITAL_SIGNS_HEADER + LAB_HEADER
        xs = [[] for _ in range(len(headers))
              ]  # relative time for start point in hours
        ys = [[] for _ in range(len(headers))
              ]  # actual value of the feature at that point
        label = records[0]['case/control']

        records = sorted(records, key=itemgetter('TIMESTAMP'))

        for t, record in enumerate(records):
            for hi, hname in enumerate(headers):
                value = record[hname]
                if value != 'NA':
                    xs[hi].append(t)
                    ys[hi].append(float(value))

        fname = '{}_{}.png'.format(key, label)
        plot(xs, ys, headers, fname)
def main():
    values = {}
    records = read_csv(input_fname)
    records_dict = hashing(records, 'ICUSTAY_ID')
    print('#icu_stays: %d' % len(records_dict))

    vals = [[] for _ in range(len(STATS_HEADER))]

    for key, record in records_dict.items():
        # single measure
        # only negative samples
        if record[0]['case/control'] == 'case':
            continue

        for measure in record:
            for head, value in measure.items():
                if head in STATS_HEADER:
                    hidx = STATS_HEADER.index(head)
                    try:
                        vals[hidx].extend([float(value)])
                    except:
                        pass
        pdb.set_trace()
    pdb.set_trace()
    output_csv = os.path.join(results_path, 'feature_stats_for_norm.csv')
    with open(output_csv, 'wt') as f:
        output_str = '{:15s},{:5s},{:5s}'.format('Feature', 'Avg', 'Std')
        f.writeline(output_str + '\n')
        for i, header in enumerate(STATS_HEADER):
            output_str = '{:15s},{:.3f},{:.3f}'.format(
                header,
                np.mean(vals[i]),
                np.std(vals[i]),
            )
            print(output_str)
            f.writeline(output_str + '\n')
Ejemplo n.º 12
0
if __name__ == '__main__':
    try:
        seed_num = int(sys.argv[1])
    except:
        seed_num = 0

    makedirs(output_root)
    _05_fname = os.path.join(dir_path, '05_converter.pkl')
    _icd_fname = os.path.join(input_root, 'icd_code.csv')
    _meta_fname = os.path.join(dir_path, 'meta_data.pkl')

    dynamic_data = read_pkl(_05_fname)
    meta_data = read_pkl(_meta_fname)
    icd_codes = read_csv(_icd_fname)
    icd_codes = hashing(icd_codes, 'ICUSTAY_ID')

    # copy file
    save_pkl(meta_data, os.path.join(output_root, 'meta_data.pkl'))

    mats = dynamic_data['mats']
    labels = dynamic_data['ys']

    for key, data in mats.items():
        # only use test data
        if key not in meta_data['test_icuid']:
            continue

        data_length = data.shape[0]
        if data_length < 90:
            continue
Ejemplo n.º 13
0
def delete():
    """
    This is a route for ALL NODES. The client sends sequests to this route
    so that a key-value pair is deleted.

    In the case of "eventual consistency", the client gets a response to the
    request after the primary replica of the key-value pair is deleted. The
    other replicas are deleted by a thread, without blocking the response
    to the client.
    """

    # Extract the key of the 'delete' request.
    key = request.form['key']
    hash_key = hashing(key)

    if between(hash_key, node.my_id, node.prev_id):
        # We are the node that owns that hash key.
        if (key in node.storage):
            # Delete the key-value pair from our database.
            del node.storage[key]

            print("The key:", key, "was deleted from our database.")

            response = "The key-value pair was successfully deleted."
            status = 200

            if (node.next_id != node.my_id and K_replicas > 1):

                if (type_replicas == "eventual consistency"):

                    thread = Thread(
                        target=delete_replicas_in_next_nodes,
                        args=(key, node)
                    )

                    thread.start()

                elif type_replicas == "linearizability":

                    response, status = delete_replicas_in_next_nodes(
                        key, node
                    )
        else:
            response = "There isn't such a key-value pair."
            status = 404
    else:
        # We are NOT the node that owns that hash key.
        # So we retransmit the request until we find the
        # node-owner of the key.

        url_next = "http://" + node.next_ip + ":" + \
            str(node.next_port) + "/delete"

        data = {
            'key': key,
        }

        r = requests.post(url_next, data)
        if r.status_code != 200:
            print("Something went wrong with the request to the next node.")

        response = r.text
        status = r.status_code

    return response, status
Ejemplo n.º 14
0
 l = int(dim)
 Ptp01, Ptp05 = [], []
 S_x = int(l*0.7)  # 610
 group_member = 40  # 2,5,10,16,20,40
 W = np.random.random([dim, l])
 U, S, V = np.linalg.svd(W)
 W = U[:, :l]
 param = {'encode': 'soft', 'normalize': 3, 'n_Clus': n_Clus}
 # Assign data to groups
 data = np.array(dataset['data_x'])
 data = zscore(np.array(dataset['data_x']), axis=0)  # N*d
 groups = partition_data(data, group_member, partitioning='random')
 # Compute group representations
 group_vec, VLAD_Codebook = vlad_group_representation(data, groups, param)
 group_vec = np.array(group_vec).T
 group_vec = hashing(group_vec, W, S_x)
 # The embedding for H0 queries
 n_q0 = len(dataset['H0_id'])
 H0_data = zscore(np.array(dataset['H0_x']).T, axis=1)  # LFW
 # H0_data = zscore(np.array(dataset['H0_x']), axis=0)  # CFP
 H0_data = [VLADEncoding(np.expand_dims(H0_data[i, :], axis=0), VLAD_Codebook,
                         encode=param['encode'], normalize=param['normalize']) for i in range(n_q0)]
 Q0 = np.array(H0_data).T
 Q0 = hashing(np.array(H0_data).T, W, S_x)
 H0_claimed_id = np.random.randint(0, len(groups['ind']), size=n_q0).astype(np.int)
 D00 = np.linalg.norm(Q0 - group_vec[:, H0_claimed_id], axis=0)
 # The embedding for H1 queries
 n_q1 = len(dataset['H1_id'])
 H1_group_id = np.zeros(n_q1)
 H1_data = zscore(np.array(dataset['H1_x']), axis=0)
 H1_data = [VLADEncoding(np.expand_dims(H1_data[i, :], axis=0), VLAD_Codebook,
Ejemplo n.º 15
0
def main():
    icd_codes = read_csv(icd_fname)
    icd_codes = hashing(icd_codes, 'ICUSTAY_ID')

    with open(input_fname, 'rb') as fp:
        records = pickle.load(fp)

    mats = records['mats']
    labels = records['ys']

    xs = []  # static features
    ys = []  # case/control
    icustay_id = []
    for key, data in mats.items():
        avg = np.mean(data, axis=0)
        std = np.std(data, axis=0)
        max_val = np.max(data, axis=0)
        min_val = np.min(data, axis=0)

        icd_code = icd_codes[key]
        assert len(icd_code) == 1
        _icd_code = icd_code[0]
        icd_code = [_icd_code[name] for name in HEADER]
        icd_code += [str(data.shape[0])]
        for _i in range(len(STR_HEADER) - 1):
            # only for gender and race
            c = STR_HEADER_CLASS[_i].index(_icd_code[STR_HEADER[_i]])
            icd_code.extend([str(c)])

        if _icd_code['Insurance2'] == 'Private':
            icd_code.extend(['1', '0'])
        elif _icd_code['Insurance2'] == 'Public':
            icd_code.extend(['0', '1'])
        else:
            icd_code.extend(['0', '0'])

        #feat = np.concatenate((avg, std, max_val, min_val, [float(icd_code[0])]))
        feat = np.concatenate((avg, std, max_val, min_val, icd_code))
        xs.append(np.expand_dims(feat, axis=0))

        if labels[key] == 'control':
            ys.append(0)
        elif labels[key] == 'case':
            ys.append(1)
        else:
            raise ValueError()

        icustay_id.append(key)

    cs = np.zeros(len(feat), dtype=int)
    cs[-(len(icd_code) - 1):] = 1  # if its categorical (1) or not (0)
    xs = np.concatenate(xs, axis=0)
    ys = np.asarray(ys)

    print(xs.shape, ys.shape)
    data = {
        'xs': xs,
        'cs': cs,
        'ys': ys,
        'icustay_id': icustay_id,
    }
    print('ratio: %.4f' % (np.sum(ys == 1) / len(ys)))
    print(xs[0], ys[0])
    print(xs[-1], ys[1])

    with open(output_fname, 'wb') as fp:
        pickle.dump(data, fp)
Ejemplo n.º 16
0
    W = U[:, :l]
    # mat = W.T @ W
    param = {'method': 'EoA', 'agg': 'pseudo', 'W': W, 'S_x': S_x}
    # Assign data to groups
    # data = np.array(dataset['data_x'])
    data = zscore(np.array(dataset['data_x']), axis=0)  # N*d
    groups = partition_data(data, group_member, partitioning='random')
    # Compute group representations
    group_vec = baseline_group_representation(groups, param)
    group_vec = np.array(group_vec)
    # The embedding for H0 queries
    n_q0 = len(dataset['H0_id'])
    # H0_data = np.array(dataset['H0_x'])
    H0_data = zscore(np.array(dataset['H0_x']), axis=1)
    # H0_data = zscore(np.array(dataset['H0_x']).T, axis=1)  # CFP
    Q0 = hashing(H0_data, param['W'], param['S_x'])

    H0_claimed_id = np.random.randint(0, len(groups['ind']),
                                      size=n_q0).astype(np.int)
    D00 = np.linalg.norm(Q0 - group_vec[:, H0_claimed_id], axis=0)
    # The embedding for H1 queries
    n_q1 = len(dataset['H1_id'])
    H1_group_id = np.zeros(n_q1)
    # H1_data = np.array(dataset['H1_x']).T
    H1_data = zscore(np.array(dataset['H1_x']).T, axis=1)
    Q1 = hashing(H1_data, param['W'], param['S_x'])
    # Determine the group identity of H1 queries
    for i in range(len(groups['ind'])):
        group_id = [dataset['data_id'][x] for x in groups['ind'][i]]
        a = [
            n for n, x in enumerate(dataset['H1_id']) for y in group_id
Ejemplo n.º 17
0
def query():
    """
    This is a route for ALL NODES. The client sends a get request
    to this route, so that the value of a key-value pair is retrieved.
    """
    key = request.form['key']
    hash_key = hashing(key)

    if (key == '*'):
        # The user wants to GET all key-value pairs from the database,
        # so we send a request to the BOOTSTRAP server. The BOOTSTRAP
        # server "knows" each node in the ring network.
        url_next = "http://" + bootstrap_ip + ":" + str(bootstrap_port) + \
            "/bootstrap/queryall"
        r = requests.get(url_next)
        return r.text[:-1], 200

    if between(hash_key, node.my_id, node.prev_id):
        # If we are the node that owns the hash key.
        if key not in node.storage:
            # The key-value pair doesn't exist in the toy-chord database.
            response = "Sorry, we don't have that song."
            status = 404
            return response, status

    if type_replicas == "linearizability":
        if key not in node.storage:
            url_next = "http://" + node.next_ip + ":" + str(node.next_port) \
                + "/query"
            data = {
                'key' : key
            }

            r = requests.post(url_next, data)
            response = r.text
            status = r.status_code
        else:
            data_to_next = {
                'id': node.my_id,
                'key': key
            }

            url_next = "http://" + node.next_ip + ":" + str(node.next_port) \
                + "/query/replicas"
            print("Submitting the query to the next node.")
            r = requests.post(url_next, data_to_next)
            if r.status_code == 418:
                response = node.storage[key]
                status = 200
            else:
                response = r.text
                status = r.status_code

    elif type_replicas == "eventual consistency":
        if key not in node.storage:
            url_next = "http://" + node.next_ip + ":" + str(node.next_port) \
                + "/query"
            data = {
                'key': key
            }

            r = requests.post(url_next, data)
            response = r.text
            status = r.status_code
        else:
            response = node.storage[key]
            status = 200

    return response, status
Ejemplo n.º 18
0
def insert():
    """
    This is a route for ALL NODES.

    The client sends POST requests to this route so that new key-value pairs
    are inserted.

    In the case of "eventual consistency", the client gets a response to the
    request after the primary replica of the key-value pair is inserted. The
    other replicas are inserted by a thread, without blocking the response
    to the client.
    """

    # Extract the key and value from the data of the 'insert' request.
    key = request.form['key']
    value = request.form['value']

    # Hash the key of the inserted key-value pair in order to find the node
    # that it's owned by.
    hash_key = hashing(key)

    if between(hash_key, node.my_id, node.prev_id):
        # If we are the node that owns this hask key,
        # we insert the key-value pair in our database.

        node.storage[key] = value
        print(
            "A key-value pair with key: '" + key +
            "' and hash:", hash_key, "was inserted/updated."
        )

        # If we aren't the only Node in the Ring, we forward the key-value pair
        # to the next node while decreasing the replication factor by 1.
        if (node.next_id != node.my_id and K_replicas > 1):
            if type_replicas == "eventual consistency":
                # In this case, we start a thread that handles the
                # forwarding of the key-value pair to the next nodes

                thread = Thread(
                    target=forward_replicas_to_next_node,
                    args=(key, value, node)
                )

                thread.start()

                # We don't care if the forwarding of the key-value pair has
                # completed. We return a response and a status code to the
                # user.
                message = "The key-value pair was successfully inserted."
                status_code = 200
            elif type_replicas == "linearizability":
                # In this case, we wait until the key-value pair is inserted
                # in the next nodes and then we return a response to the user.
                message, status_code = forward_replicas_to_next_node(
                    key=key,
                    value=value,
                    node=node
                )
            return message, status_code

        return "The key-value pair was successfully inserted.", 200
    else:
        # Otherwise, if we don't own the hash key, we forward the data
        # to the NEXT node. The data will be forwarded by the next nodes
        # until they reach the node that is responsible for the hash key.
        url_next = "http://" + node.next_ip + ":" + \
            str(node.next_port) + "/insert"

        data = {
            'key': key,
            'value': value,
        }

        r = requests.post(url_next, data)
        if r.status_code != 200:
            message = "Error while retransmitting the key-value pair."
            print(message)
            return message, 500

        return r.text
Ejemplo n.º 19
0
def process_dir(d, outfile_results, zipin, result):
    shellparams = {
        'stdout': outfile_results,
        'stderr': outfile_results,
        'shell': False
    }

    print("Processing directory " + d)
    for f in os.listdir(d):
        fullpath = os.path.join(d, f)
        if os.path.isdir(fullpath) and not f == ".svn":
            process_dir(fullpath, outfile_results, zipin, result)
            continue

        if f in settings.files_to_ignore:
            print("Ignoring " + f)
            continue

        for pppreset in settings.pp_configs_to_test:
            filehash = utils.hashing(fullpath, pppreset)
            failure = False
            try:
                input_expected = zipin.open(filehash, "r").read()
                # empty dump files indicate 'expected import failure'
                if not len(input_expected):
                    failure = True
            except KeyError:
                #print("Didn't find "+fullpath+" (Hash is "+filehash+") in database")
                continue

            # Ignore extensions via settings.py configured list
            # todo: Fix for multi dot extensions like .skeleton.xml
            ext = os.path.splitext(fullpath)[1].lower()
            if ext != "" and ext in settings.exclude_extensions:
                continue

            print("-" * 60 + "\n  " + os.path.realpath(fullpath) + " pp: " +
                  pppreset)

            outfile_actual = mkoutputdir_andgetpath(fullpath, filehash,
                                                    "ACTUAL")
            outfile_expect = mkoutputdir_andgetpath(fullpath, filehash,
                                                    "EXPECT")
            outfile_results.write("assimp dump    " + "-" * 80 + "\n")
            outfile_results.flush()

            command = [
                utils.assimp_bin_path, "dump", fullpath, outfile_actual, "-b",
                "-s", "-l"
            ] + pppreset.split()
            r = subprocess.call(command, **shellparams)

            if r and not failure:
                result.fail(fullpath, outfile_expect, pppreset, IMPORT_FAILURE,
                            r)
                continue
            elif failure and not r:
                result.fail(fullpath, outfile_expect, pppreset,
                            EXPECTED_FAILURE_NOT_MET)
                continue

            with open(outfile_expect, "wb") as s:
                s.write(input_expected)

            try:
                with open(outfile_actual, "rb") as s:
                    input_actual = s.read()
            except IOError:
                continue

            if len(input_expected) != len(input_actual):
                result.fail(fullpath, outfile_expect, pppreset,
                            DATABASE_LENGTH_MISMATCH, len(input_expected),
                            len(input_actual))
                continue

            outfile_results.write("assimp cmpdump " + "-" * 80 + "\n")
            outfile_results.flush()

            command = [
                utils.assimp_bin_path, 'cmpdump', outfile_actual,
                outfile_expect
            ]
            if subprocess.call(command, **shellparams) != 0:
                result.fail(fullpath, outfile_expect, pppreset,
                            DATABASE_VALUE_MISMATCH)
                continue

            result.ok(fullpath, pppreset, COMPARE_SUCCESS, len(input_expected))
Ejemplo n.º 20
0
def process_dir(d, outfile_results, zipin, result ):
    shellparams = {'stdout':outfile_results, 'stderr':outfile_results, 'shell':False}

    print("Processing directory " + d)
    all = ""
    for f in sorted(os.listdir(d)):
        fullpath = os.path.join(d, f)
        if os.path.isdir(fullpath) and not f[:1] == '.':
            process_dir(fullpath, outfile_results, zipin, result)
            continue
                
        if f in settings.files_to_ignore or os.path.splitext(f)[1] in settings.exclude_extensions:
            print("Ignoring " + f)
            return

        for pppreset in settings.pp_configs_to_test:
            filehash = utils.hashing(fullpath, pppreset)
            failure = False

            try:
                input_expected = zipin.open(filehash, "r").read()
                # empty dump files indicate 'expected import failure'
                if not len(input_expected):
                   failure = True
            except KeyError:
                # TODO(acgessler): Keep track of this and report as error in the end.
                print("Didn't find "+fullpath+" (Hash is "+filehash+") in database. Outdated "+\
                    "regression database? Use gen_db.zip to re-generate.")
                continue

            print("-"*60 + "\n  " + os.path.realpath(fullpath) + " pp: " + pppreset)

            outfile_actual = prepare_output_dir(fullpath, filehash, "ACTUAL")
            outfile_expect = prepare_output_dir(fullpath, filehash, "EXPECT")
            outfile_results.write("assimp dump    "+"-"*80+"\n")
            outfile_results.flush()
            assimp_bin_path = getEnvVar("assimp_path")
            command = [assimp_bin_path,
                "dump",
                fullpath, outfile_actual, "-b", "-s", "-l" ] +\
                pppreset.split()
            print( "command = " + str( command ) )
            r = subprocess.call(command, **shellparams)
            outfile_results.flush()

            if r and not failure:
                result.fail(fullpath, outfile_expect, pppreset, IMPORT_FAILURE, r)
                outfile_results.write("Failed to import\n")
                continue
            elif failure and not r:
                result.fail(fullpath, outfile_expect, pppreset, EXPECTED_FAILURE_NOT_MET)
                outfile_results.write("Expected import to fail\n")
                continue
            elif failure and r:
                result.ok(fullpath, pppreset, EXPECTED_FAILURE)
                outfile_results.write("Failed as expected, skipping.\n")
                continue

            with open(outfile_expect, "wb") as s:
                s.write(input_expected)

            try:
                with open(outfile_actual, "rb") as s:
                    input_actual = s.read()
            except IOError:
                continue

            outfile_results.write("Expected data length: {0}\n".format(len(input_expected)))
            outfile_results.write("Actual data length: {0}\n".format(len(input_actual)))
            failed = False
            if len(input_expected) != len(input_actual):
                result.fail(fullpath, outfile_expect, pppreset, DATABASE_LENGTH_MISMATCH,
                        len(input_expected), len(input_actual))
                # Still compare the dumps to see what the difference is
                failed = True

            outfile_results.write("assimp cmpdump "+"-"*80+"\n")
            outfile_results.flush()
            command = [ assimp_bin_path, 'cmpdump', outfile_actual, outfile_expect ]
            if subprocess.call(command, **shellparams) != 0:
                if not failed:
                    result.fail(fullpath, outfile_expect, pppreset, DATABASE_VALUE_MISMATCH)
                continue

            result.ok(fullpath, pppreset, COMPARE_SUCCESS, len(input_expected))