示例#1
0
def driver(argv):
    pathname = argv[1]
    pathname = abs(pathname) 

    regex = re.compile(".+\.py$")

    index = Indexer()

    index_time_start = time.time()
    for filename in iter_filenames(pathname, regex):
        index.construct(filename)
    index_time_end = time.time()

    index_file = "index.pkl"
    fh_output = open(index_file, 'wb')
    dump(index.block_graph, fh_output)
    fh_output.close()
    
    index_size = os.path.getsize(index_file)

    print "Total files scanned:", _TOTAL_NUM_FILES
    print "Python files matched:", _TOTAL_FILES_MATCHED
    print "Python files net size:", _TOTAL_MATCHED_SIZE
    print "Time required to index:", index_time_end - index_time_start
    print "Index size", index_size
示例#2
0
def rel(path, parent=None, nopar=False):
    """
  Like :func:`os.path.relpath`, but the *nopar* parameter can be set to return
  an absolute path if the relative path would create a path element that
  references a parent directory (`..`) or current directory (``.``).
  """

    try:
        res = os.path.relpath(path, parent)
    except ValueError:
        if nopar:
            return abs(path)
        raise
    else:
        if not issub(res):
            return abs(path)
        return res
示例#3
0
def rel(path, parent=None, nopar=False):
    """
  Like :func:`os.path.relpath`, but the *nopar* parameter can be set to return
  an absolute path if the relative path would create a path element that
  references a parent directory (`..`) or current directory (``.``).
  """

    try:
        res = os.path.relpath(path, parent)
    except ValueError:
        if nopar:
            return abs(path)
        raise
    else:
        if not issub(res):
            return abs(path)
        return res
示例#4
0
def iter_filenames(pathname, filename_matcher):
    global _TOTAL_NUM_FILES
    global _TOTAL_FILES_MATCHED
    global _TOTAL_MATCHED_SIZE
    """Yield every filename in the path that matches the pattern"""
    for root, directories, filenames in os.walk(pathname):
        for filename in filenames:
            _TOTAL_NUM_FILES += 1
            full = abs(join(root, filename))
            if filename_matcher.match(filename):
                _TOTAL_FILES_MATCHED += 1
                _TOTAL_MATCHED_SIZE += os.path.getsize(full)
                yield full
示例#5
0
            return True
        else:
            return False


PATH = os.path.dirname(__file__)
JSON_PATH = os.path.join(PATH, 'models.json')
MODELS_PATH = os.path.join(PATH, "models")

HOMOGENE = False
RANDOM = True
MIX = False
KEZEK = False

if HOMOGENE and not MIX and not KEZEK:
    VALID_PATH = abs(os.path.join(PATH, 'random'))
elif RANDOM and not MIX and not KEZEK:
    VALID_PATH = os.path.join(PATH, 'homogene')
elif MIX:
    NPY_PATH = os.path.join(PATH, 'mix_test.npy')
elif KEZEK:
    VALID_PATH = abs(os.path.join(PATH, 'fashion'))
    NPY_PATH = abs(os.path.join(VALID_PATH, 'kezek.npy'))

else:
    VALID_PATH = os.path.join(PATH, 'images')

if not MIX and not KEZEK:
    NPY_PATH = os.path.join(VALID_PATH, 'bone_spects.npy')

TRAIN = ["all", "all"]
示例#6
0
def validate(classify):
    label_path = abs(os.path.join(PATH, "labels.json"))

    imgs = Images(NPY_PATH)
    # Draw on image
    if DRAW_IMAGE:
        labels = []
        arrays = []
        for i in range(VALID_IMG_NUM):
            number = random.randint(0, CLASSES - 1)
            img_name = "image_%s.png" % i
            img_path = abs(os.path.join(VALID_PATH, img_name))
            labels.append(number)
            draw = Draw(imgs.arr[i], number, img_path)
            arrays.append(draw.array)

        arrays = np.array(arrays)
        try:
            with open(label_path, 'r') as file:
                j_son = json.load(file)
            if HOMOGENE:
                j_son['homogene'] = labels
            elif RANDOM:
                j_son['random'] = labels
            elif KEZEK:
                j_son['kezek'] = labels

            arrays = np.expand_dims(arrays, axis=3)
            np.save(NPY_PATH, arrays)
            try:
                with open(label_path, 'w+') as file:
                    json.dump(j_son, file, indent=4)

            except EOFError:
                print("There is no valid .json file.")
                sys.exit()

        except EOFError:
            print("There is no valid .json file.")
            exit()
    # Classifier
    if classify == 'True':
        images = Images(NPY_PATH)
        # part = int((images.arr.shape[0]) * 3 / 4)
        train_images = np.array([])
        part = 300
        if part != 0:
            train_images = images.arr[:part]
        test_images = images.arr[part:]
        try:
            with open(label_path, 'r') as file:
                j_son = json.load(file)
                label = []
                if HOMOGENE:
                    label = j_son['homogene']
                if RANDOM:
                    label = j_son['random']
                if MIX:
                    label = j_son['mix']
                if KEZEK:
                    label = j_son['kezek']
        except:
            print('Something went wrong.')

        train_labels = list()
        if part != 0:
            train_labels = to_categorical(label[:part])
        test_labels = to_categorical(label[part:])
        classifier = Classifier(imgs.arr.shape[1:], CLASSES)
        network_name = 'classifier.h5'
        network_path = os.path.join(PATH, network_name)
        if not find(network_name, PATH):
            classifier.build_classifier(network_path)
        if HOMOGENE:
            trained_name = 'hom_class.h5'
            trained_path = os.path.join(PATH, trained_name)
        elif RANDOM:
            trained_name = 'rand_class.h5'
            trained_path = os.path.join(PATH, trained_name)
        else:
            trained_path = None

        if find(trained_name, PATH):
            classifier.loadModel(trained_path)
            classifier.model.compile(optimizer=Adam(lr=0.001),
                                     loss=LOSS,
                                     metrics=['categorical_accuracy'])
        elif train_images.size != 0:
            classifier.loadModel(network_path)
            classifier.model.compile(optimizer=Adam(lr=0.001),
                                     loss=LOSS,
                                     metrics=['categorical_accuracy'])
            classifier.trainModel(train_images, train_labels, 2, EPOCH)

        json_model = {}

        try:
            with open(os.path.join(PATH, 'classific.json'), 'r') as file:
                j_son = json.load(file)
                if HOMOGENE:
                    json_model = j_son['homogene']
                elif RANDOM:
                    json_model = j_son['random']

        except:
            print('Something went wrong.')

        if train_images.size != 0:
            classifier.saveTrained(trained_path, json_model['train'])
        classifier.predictModel(test_images, test_labels, 1)
        print("itt a baj")
        test = []
        print(classifier.predicted)
        for i, label in enumerate(test_labels):
            test_ = dict()
            test_[str(np.argmax(label))] = str(
                np.argmax(classifier.predicted[i]))
            test.append(test_)
        json_model['test'] = test
        try:
            with open(os.path.join(PATH, 'classific.json'), 'w') as file:
                if HOMOGENE:
                    j_son['homogene'] = json_model
                elif RANDOM:
                    j_son['random'] = json_model
                elif MIX:
                    j_son['mix'] = json_model
                elif KEZEK:
                    j_son['kezek'] = json_model

                json.dump(j_son, file, indent=4)

        except EOFError:
            print("Save to .json file was unsuccessful.")
    else:
        pass