Пример #1
0
    def handle(self, *args, **options):
        primary = options['primary']
        secondary = options['secondary']

        if not primary or not os.path.exists(os.path.expanduser(primary)):
            raise Exception('{0} is not a valid CSV path'.format(primary))
        if not secondary or not os.path.exists(os.path.expanduser(secondary)):
            raise Exception('{0} is not a valid CSV path'.format(secondary))

        primary_dataset = dict()
        secondary_dataset = dict()

        function = None
        with open(primary) as file_:
            reader = csv.reader(file_)

            for row in reader:
                if 'Function' in row[0]:
                    function = Function()

                    function.name = row[1]
                    function.file = row[2]

                    if function not in primary_dataset:
                        primary_dataset[function] = int(row[3])
                    else:
                        logger.debug(
                            '{0} duplicate in {1}'.format(function, primary)
                        )

        with open(secondary) as file_:
            reader = csv.reader(file_)

            for row in reader:
                if 'Function' in row[0]:
                    function = Function()

                    function.name = row[1]
                    function.file = row[2]

                    if function not in secondary_dataset:
                        secondary_dataset[function] = int(row[3])
                    else:
                        logger.debug(
                            '{0} duplicate in {1}'.format(function, secondary)
                        )

        match = 0
        for item in secondary_dataset:
            if item in primary_dataset:
                if secondary_dataset[item] == primary_dataset[item]:
                    match += 1

        logger.info(
            '{0}/{1} having matching SLOC'.format(
                match, len(secondary_dataset)
            )
        )
Пример #2
0
    def handle(self, *args, **options):
        source = options['source']

        if not source or not os.path.exists(os.path.expanduser(source)):
            raise Exception('{0} is not a valid CSV path'.format(source))

        function = None
        duplicates = list()
        functions = set()
        with open(source) as file_:
            reader = csv.reader(file_)
            for row in reader:
                if 'Function' in row[0]:
                    name = row[1]
                    file = row[2]
                    sloc = int(row[3])

                    function = Function.objects.filter(
                        name=name, file=file
                    )

                    if not function.exists():
                        function = Function()

                        function.name = name
                        function.file = file
                        function.sloc = sloc

                        if function not in functions:
                            functions.add(function)
                        else:
                            duplicates.append(function)
                            function = [
                                f for f in functions if f == function
                            ][0]
                            duplicates.append(function)
                            functions.remove(function)

        if len(functions) > 0:
            logger.debug('Adding {0} functions.'.format(len(functions)))
            Function.objects.bulk_create(functions)

            if len(duplicates) > 0:
                for function in duplicates:
                    logger.debug(
                        'Duplicate {0} in {1} with {2} SLOC'.format(
                            function.name, function.file, function.sloc
                        )
                    )
            logger.info('Appended {0} functions.'.format(len(functions)))
Пример #3
0
    def handle(self, *args, **options):
        source = options['source']

        if not source or not os.path.exists(os.path.expanduser(source)):
            raise Exception('{0} is not a valid CSV path'.format(source))

        functions = set()
        files = set()
        with open(source) as file_:
            reader = csv.reader(file_)
            for row in reader:
                if 'Function' in row[0]:
                    name = row[1]
                    file = row[2]
                    sloc = int(row[3])

                    function = Function()

                    function.name = name
                    function.file = file
                    function.sloc = sloc

                    if function not in functions:
                        functions.add(function)
                    else:
                        functions = self._update(function, functions)
                elif 'File' in row[0]:
                    name = row[2]
                    sloc = int(row[3])

                    file_ = File()

                    file_.name = name
                    file_.sloc = sloc

                    if file_ not in files:
                        files.add(file_)
                    else:
                        files = self._update(file_, files)

        if len(functions) > 0:
            logger.debug('Adding {0} functions.'.format(len(functions)))
            Function.objects.bulk_create(functions)
            logger.info('Loaded {0} functions.'.format(len(functions)))

        if len(files) > 0:
            logger.debug('Adding {0} files.'.format(len(files)))
            File.objects.bulk_create(files)
            logger.info('Loaded {0} files.'.format(len(files)))