コード例 #1
0
ファイル: test_backend.py プロジェクト: jsmanrique/perceval
    def test_uuid(self):
        """Check whether the function returns the expected UUID"""

        result = uuid('1', '2', '3', '4')
        self.assertEqual(result, 'e7b71c81f5a0723e2237f157dba81777ce7c6c21')

        result = uuid('http://example.com/', '1234567')
        self.assertEqual(result, '47509b2f0d4ffc513ca9230838a69aa841d7f055')
コード例 #2
0
ファイル: test_backend.py プロジェクト: sduenas/perceval
    def test_uuid(self):
        """Check whether the function returns the expected UUID"""

        result = uuid('1', '2', '3', '4')
        self.assertEqual(result, 'e7b71c81f5a0723e2237f157dba81777ce7c6c21')

        result = uuid('http://example.com/', '1234567')
        self.assertEqual(result, '47509b2f0d4ffc513ca9230838a69aa841d7f055')
コード例 #3
0
ファイル: test_git.py プロジェクト: celan/perceval
    def test_fetch_since_date(self):
        """Test whether commits are fetched from a Git repository since the given date"""

        new_path = os.path.join(self.tmp_path, 'newgit')

        from_date = datetime.datetime(2014, 2, 11, 22, 7, 49)
        git = Git(self.git_path, new_path)
        commits = [commit for commit in git.fetch(from_date=from_date)]

        expected = [('ce8e0b86a1e9877f42fe9453ede418519115f367', 1392185269.0),
                    ('51a3b654f252210572297f47597b31527c475fb8', 1392185366.0),
                    ('456a68ee1407a77f3e804a30dff245bb6c6b872f', 1392185439.0)]

        self.assertEqual(len(commits), len(expected))

        for x in range(len(commits)):
            expected_uuid = uuid(self.git_path, expected[x][0])
            commit = commits[x]
            self.assertEqual(commit['data']['commit'], expected[x][0])
            self.assertEqual(commit['origin'], self.git_path)
            self.assertEqual(commit['uuid'], expected_uuid)
            self.assertEqual(commit['updated_on'], expected[x][1])
            self.assertEqual(commit['category'], 'commit')
            self.assertEqual(commit['tag'], self.git_path)

        # Test it using a datetime that includes the timezone
        from_date = datetime.datetime(2012,
                                      8,
                                      14,
                                      14,
                                      30,
                                      00,
                                      tzinfo=dateutil.tz.tzoffset(
                                          None, -36000))
        git = Git(self.git_path, new_path)
        commits = [commit for commit in git.fetch(from_date=from_date)]

        self.assertEqual(len(commits), len(expected))

        for x in range(len(commits)):
            expected_uuid = uuid(self.git_path, expected[x][0])
            commit = commits[x]
            self.assertEqual(commit['data']['commit'], expected[x][0])
            self.assertEqual(commit['origin'], self.git_path)
            self.assertEqual(commit['uuid'], expected_uuid)
            self.assertEqual(commit['updated_on'], expected[x][1])
            self.assertEqual(commit['category'], 'commit')
            self.assertEqual(commit['tag'], self.git_path)

        shutil.rmtree(new_path)
コード例 #4
0
    def test_run_fetch_cache(self):
        """Test whether the command runs when fetch from cache is set"""

        args = [
            '--cache-path', self.test_path, '--fetch-cache', '--from-date',
            '2015-01-01', '--tag', 'test', '--output', self.fout_path,
            'http://example.com/'
        ]

        cmd = MockedBackendCommand(*args)
        cmd.run()
        cmd.outfile.close()

        items = [item for item in convert_cmd_output_to_json(self.fout_path)]

        self.assertEqual(len(items), 5)

        for x in range(5):
            item = items[x]
            expected_uuid = uuid('http://example.com/', str(x))

            # MockedBackend sets 'cache' value when
            # 'fetch_from_cache' is called
            self.assertEqual(item['data']['item'], x)
            self.assertEqual(item['data']['cache'], True)
            self.assertEqual(item['origin'], 'http://example.com/')
            self.assertEqual(item['uuid'], expected_uuid)
            self.assertEqual(item['tag'], 'test')

        self.assertIsInstance(cmd.backend.cache, Cache)
コード例 #5
0
    def test_run_other_category(self):
        """Test whether when the category (different from the default one) is properly set"""

        args = [
            '-u', 'jsmith', '-p', '1234', '-t', 'abcd', '--archive-path',
            self.test_path, '--category', MockedBackend.OTHER_CATEGORY,
            '--subtype', 'mocksubtype', '--from-date', '2015-01-01', '--tag',
            'test', '--output', self.fout_path, 'http://example.com/'
        ]

        cmd = MockedBackendCommand(*args)
        cmd.run()
        cmd.outfile.close()

        items = [item for item in convert_cmd_output_to_json(self.fout_path)]

        self.assertEqual(len(items), 5)

        for x in range(5):
            item = items[x]
            expected_uuid = uuid('http://example.com/', str(x))

            self.assertEqual(item['data']['item'], x)
            self.assertEqual(item['origin'], 'http://example.com/')
            self.assertEqual(item['uuid'], expected_uuid)
            self.assertEqual(item['tag'], 'test')
            self.assertEqual(item['category'], MockedBackend.OTHER_CATEGORY)
コード例 #6
0
    def test_fetch_from_file(self):
        """Test whether a list of entries is returned from a file definition"""

        finosmeetings = FinosMeetings(
            "file://" + file_abs_path('data/finosmeetings/finosmeetings_entries.csv'))

        entries = [entry for entry in finosmeetings.fetch()]
        self.assertEqual(len(entries), 3)

        # Test metadata
        expected = [('*****@*****.**', 'Rob Underwood', 'brooklynrob', 'Data Tech', 'Data Tech PMC', '2018-09-28'),
                    ('*****@*****.**', 'Tosha Ellison', '', 'Data Tech', 'Security Reference Data', '2018-12-11'),
                    ('*****@*****.**', 'Maurizio Pillitu', 'maoo', 'FDC3', 'FDC3 PMC', '2018-10-19')]

        for x in range(len(expected)):
            entry = entries[x]['data']
            self.assertEqual(entries[x]['uuid'], uuid(finosmeetings.origin, FinosMeetings.metadata_id(entry)))
            self.assertEqual(entries[x]['updated_on'], FinosMeetings.metadata_updated_on(entry))
            self.assertEqual(entry['email'], expected[x][0])
            self.assertEqual(entry['name'], expected[x][1])
            self.assertEqual(entry['org'], 'FINOS')
            self.assertEqual(entry['githubid'], expected[x][2])
            self.assertEqual(entry['program'], expected[x][3])
            self.assertEqual(entry['activity'], expected[x][4])
            self.assertEqual(entry['date'], expected[x][5])
コード例 #7
0
    def test_fetch(self):
        """Test whether a list of entries is returned"""

        http_requests = configure_http_server()

        finosmeetings = FinosMeetings(MEETINGS_URL)

        entries = [entry for entry in finosmeetings.fetch()]
        self.assertEqual(len(entries), 3)
        self.assertEqual(len(http_requests), 1)

        # Test metadata
        expected = [('*****@*****.**', 'Rob Underwood', 'brooklynrob', 'Data Tech', 'Data Tech PMC', '2018-09-28'),
                    ('*****@*****.**', 'Tosha Ellison', '', 'Data Tech', 'Security Reference Data', '2018-12-11'),
                    ('*****@*****.**', 'Maurizio Pillitu', 'maoo', 'FDC3', 'FDC3 PMC', '2018-10-19')]

        for x in range(len(expected)):
            entry = entries[x]['data']
            self.assertEqual(entries[x]['uuid'], uuid(finosmeetings.origin, FinosMeetings.metadata_id(entry)))
            self.assertEqual(entries[x]['updated_on'], FinosMeetings.metadata_updated_on(entry))
            self.assertEqual(entry['email'], expected[x][0])
            self.assertEqual(entry['name'], expected[x][1])
            self.assertEqual(entry['org'], 'FINOS')
            self.assertEqual(entry['githubid'], expected[x][2])
            self.assertEqual(entry['program'], expected[x][3])
            self.assertEqual(entry['activity'], expected[x][4])
            self.assertEqual(entry['date'], expected[x][5])
コード例 #8
0
ファイル: graal.py プロジェクト: vchrombie/grimoirelab-graal
    def metadata(self, item, filter_classified=False):
        """Add metadata to an item.

        It adds metadata to a given item such as how and
        when it was fetched. The contents from the original item will
        be stored under the 'data' keyword.

        :param item: an item fetched by a backend
        :param filter_classified: sets if classified fields were filtered
        """
        item = {
            'backend_name': self.__class__.__name__,
            'backend_version': self.version,
            'graal_version': __version__,
            'timestamp': datetime_utcnow().timestamp(),
            'origin': self.origin,
            'uuid': uuid(self.origin, self.metadata_id(item)),
            'updated_on': self.metadata_updated_on(item),
            'classified_fields_filtered': self.classified_fields if filter_classified else None,
            'category': self.metadata_category(item),
            'search_fields': self.search_fields(item),
            'tag': self.tag,
            'data': item,
        }

        return item
コード例 #9
0
ファイル: test_git.py プロジェクト: jsmanrique/perceval
    def test_fetch_from_file(self):
        """Test whether commits are fetched from a Git log file"""

        git = Git('http://example.com.git', 'data/git_log.txt')
        commits = [commit for commit in git.fetch()]

        expected = [('456a68ee1407a77f3e804a30dff245bb6c6b872f', 1392185439.0),
                    ('51a3b654f252210572297f47597b31527c475fb8', 1392185366.0),
                    ('ce8e0b86a1e9877f42fe9453ede418519115f367', 1392185269.0),
                    ('589bb080f059834829a2a5955bebfd7c2baa110a', 1344967441.0),
                    ('c6ba8f7a1058db3e6b4bc6f1090e932b107605fb', 1344966351.0),
                    ('c0d66f92a95e31c77be08dc9d0f11a16715d1885', 1344965702.0),
                    ('7debcf8a2f57f86663809c58b5c07a398be7674c', 1344965607.0),
                    ('87783129c3f00d2c81a3a8e585eb86a47e39891a', 1344965535.0),
                    ('bc57a9209f096a130dcc5ba7089a8663f758a703', 1344965413.0)]

        self.assertEqual(len(commits), len(expected))

        for x in range(len(commits)):
            expected_uuid = uuid('http://example.com.git', expected[x][0])
            commit = commits[x]
            self.assertEqual(commit['commit'], expected[x][0])
            self.assertEqual(commit['__metadata__']['origin'], 'http://example.com.git')
            self.assertEqual(commit['__metadata__']['uuid'], expected_uuid)
            self.assertEqual(commit['__metadata__']['updated_on'], expected[x][1])
コード例 #10
0
    def test_run_json_line(self):
        """Test run method with --json-line"""

        args = [
            '-u', 'jsmith', '-p', '1234', '-t', 'abcd', '--archive-path',
            self.test_path, '--from-date', '2015-01-01', '--tag', 'test',
            '--output', self.fout_path, 'http://example.com/', '--json-line'
        ]

        cmd = MockedBackendCommand(*args)
        cmd.run()
        cmd.outfile.close()

        with open(self.fout_path) as fout:
            items = fout.readlines()

        self.assertEqual(len(items), 5)

        for x in range(5):
            item = json.loads(items[x])
            expected_uuid = uuid('http://example.com/', str(x))

            self.assertEqual(item['data']['item'], x)
            self.assertEqual(item['origin'], 'http://example.com/')
            self.assertEqual(item['uuid'], expected_uuid)
            self.assertEqual(item['tag'], 'test')
            self.assertEqual(item['category'], MockedBackend.DEFAULT_CATEGORY)
コード例 #11
0
ファイル: test_git.py プロジェクト: grimoirelab/perceval
    def test_fetch_from_file(self):
        """Test whether commits are fetched from a Git log file"""

        git = Git("http://example.com.git", "data/git_log.txt")
        commits = [commit for commit in git.fetch()]

        expected = [
            ("456a68ee1407a77f3e804a30dff245bb6c6b872f", 1392185439.0),
            ("51a3b654f252210572297f47597b31527c475fb8", 1392185366.0),
            ("ce8e0b86a1e9877f42fe9453ede418519115f367", 1392185269.0),
            ("589bb080f059834829a2a5955bebfd7c2baa110a", 1344967441.0),
            ("c6ba8f7a1058db3e6b4bc6f1090e932b107605fb", 1344966351.0),
            ("c0d66f92a95e31c77be08dc9d0f11a16715d1885", 1344965702.0),
            ("7debcf8a2f57f86663809c58b5c07a398be7674c", 1344965607.0),
            ("87783129c3f00d2c81a3a8e585eb86a47e39891a", 1344965535.0),
            ("bc57a9209f096a130dcc5ba7089a8663f758a703", 1344965413.0),
        ]

        self.assertEqual(len(commits), len(expected))

        for x in range(len(commits)):
            expected_uuid = uuid("http://example.com.git", expected[x][0])
            commit = commits[x]
            self.assertEqual(commit["data"]["commit"], expected[x][0])
            self.assertEqual(commit["origin"], "http://example.com.git")
            self.assertEqual(commit["uuid"], expected_uuid)
            self.assertEqual(commit["updated_on"], expected[x][1])
            self.assertEqual(commit["category"], "commit")
コード例 #12
0
ファイル: test_backend.py プロジェクト: sduenas/perceval
    def test_items_storing_archive(self):
        """Test whether items are stored in an archive"""

        manager = ArchiveManager(self.test_path)

        args = {
            'origin': 'http://example.com/',
            'category': 'mock_item',
            'tag': 'test',
            'subtype': 'mocksubtype',
            'from-date': str_to_datetime('2015-01-01')
        }

        items = fetch(CommandBackend, args, manager=manager)
        items = [item for item in items]

        self.assertEqual(len(items), 5)

        for x in range(5):
            item = items[x]
            expected_uuid = uuid('http://example.com/', str(x))

            self.assertEqual(item['data']['item'], x)
            self.assertEqual(item['origin'], 'http://example.com/')
            self.assertEqual(item['uuid'], expected_uuid)
            self.assertEqual(item['tag'], 'test')

        filepaths = manager.search('http://example.com/', 'CommandBackend',
                                   'mock_item', str_to_datetime('1970-01-01'))

        self.assertEqual(len(filepaths), 1)

        archive = Archive(filepaths[0])
        self.assertEqual(archive._count_table_rows('archive'), 5)
コード例 #13
0
    def test_fetch(self):
        """Test whether commits are fetched from a Git repository"""

        new_path = os.path.join(self.tmp_path, 'newgit')

        git = Git(self.git_path, new_path)
        commits = [commit for commit in git.fetch()]

        expected = [('bc57a9209f096a130dcc5ba7089a8663f758a703', 1344965413.0),
                    ('87783129c3f00d2c81a3a8e585eb86a47e39891a', 1344965535.0),
                    ('7debcf8a2f57f86663809c58b5c07a398be7674c', 1344965607.0),
                    ('c0d66f92a95e31c77be08dc9d0f11a16715d1885', 1344965702.0),
                    ('c6ba8f7a1058db3e6b4bc6f1090e932b107605fb', 1344966351.0),
                    ('589bb080f059834829a2a5955bebfd7c2baa110a', 1344967441.0),
                    ('ce8e0b86a1e9877f42fe9453ede418519115f367', 1392185269.0),
                    ('51a3b654f252210572297f47597b31527c475fb8', 1392185366.0),
                    ('456a68ee1407a77f3e804a30dff245bb6c6b872f', 1392185439.0)]

        self.assertEqual(len(commits), len(expected))

        for x in range(len(commits)):
            expected_uuid = uuid(self.git_path, expected[x][0])
            commit = commits[x]
            self.assertEqual(commit['data']['commit'], expected[x][0])
            self.assertEqual(commit['origin'], self.git_path)
            self.assertEqual(commit['uuid'], expected_uuid)
            self.assertEqual(commit['updated_on'], expected[x][1])
            self.assertEqual(commit['category'], 'commit')
            self.assertEqual(commit['tag'], self.git_path)

        shutil.rmtree(new_path)
コード例 #14
0
ファイル: test_git.py プロジェクト: albertinisg/perceval
    def test_fetch(self):
        """Test whether commits are fetched from a Git repository"""

        new_path = os.path.join(self.tmp_path, 'newgit')

        git = Git(self.git_path, new_path)
        commits = [commit for commit in git.fetch()]

        expected = [('bc57a9209f096a130dcc5ba7089a8663f758a703', 1344965413.0),
                    ('87783129c3f00d2c81a3a8e585eb86a47e39891a', 1344965535.0),
                    ('7debcf8a2f57f86663809c58b5c07a398be7674c', 1344965607.0),
                    ('c0d66f92a95e31c77be08dc9d0f11a16715d1885', 1344965702.0),
                    ('c6ba8f7a1058db3e6b4bc6f1090e932b107605fb', 1344966351.0),
                    ('589bb080f059834829a2a5955bebfd7c2baa110a', 1344967441.0),
                    ('ce8e0b86a1e9877f42fe9453ede418519115f367', 1392185269.0),
                    ('51a3b654f252210572297f47597b31527c475fb8', 1392185366.0),
                    ('456a68ee1407a77f3e804a30dff245bb6c6b872f', 1392185439.0)]

        self.assertEqual(len(commits), len(expected))

        for x in range(len(commits)):
            expected_uuid = uuid(self.git_path, expected[x][0])
            commit = commits[x]
            self.assertEqual(commit['data']['commit'], expected[x][0])
            self.assertEqual(commit['origin'], self.git_path)
            self.assertEqual(commit['uuid'], expected_uuid)
            self.assertEqual(commit['updated_on'], expected[x][1])
            self.assertEqual(commit['category'], 'commit')
            self.assertEqual(commit['tag'], self.git_path)

        shutil.rmtree(new_path)
コード例 #15
0
    def get_rich_item(self, item, file_path, smell):
        commit = item['data']
        smell_info = smell.split()
        smell_line = smell_info[1]
        smell_code = smell_info[2]
        smell_details = ' '.join(smell_info[3:])
        eitem = {
            'file_path': file_path,
            'smell_line': smell_line,
            'smell_code': smell_code,
            'smell': smell_details,
            'commit_sha': commit['commit'],
            'origin': item['origin'],
            'author_date': fix_field_date(commit['AuthorDate']),
            'commit_date': fix_field_date(commit['CommitDate'])
        }

        if self.prjs_map:
            eitem.update(self.get_item_project(eitem))

        # uuid
        uuid_sha1 = backend.uuid(eitem['file_path'], eitem['smell'])
        eitem['id'] = "{}_{}".format(eitem['commit_sha'], uuid_sha1)
        eitem.update(self.get_grimoire_fields(eitem["author_date"], "file"))
        self.add_repository_labels(eitem)
        self.add_metadata_filter_raw(eitem)

        return eitem
コード例 #16
0
    def test_run_no_archive(self):
        """Test whether the command runs when archive is not set"""

        args = [
            '--no-archive', '--from-date', '2015-01-01', '--tag', 'test',
            '--output', self.fout_path, '--category', 'mock_item',
            'http://example.com/'
        ]

        cmd = MockedBackendCommand(*args)
        cmd.run()
        cmd.outfile.close()

        items = [item for item in convert_cmd_output_to_json(self.fout_path)]

        self.assertEqual(len(items), 5)

        for x in range(5):
            item = items[x]
            expected_uuid = uuid('http://example.com/', str(x))

            self.assertEqual(item['data']['item'], x)
            self.assertEqual(item['origin'], 'http://example.com/')
            self.assertEqual(item['uuid'], expected_uuid)
            self.assertEqual(item['tag'], 'test')
コード例 #17
0
ファイル: test_backend.py プロジェクト: sduenas/perceval
    def test_run(self):
        """Test run method"""

        args = [
            '-u', 'jsmith', '-p', '1234', '-t', 'abcd', '--archive-path',
            self.test_path, '--category', 'mock_item', '--subtype',
            'mocksubtype', '--from-date', '2015-01-01', '--tag', 'test',
            '--output', self.fout_path, 'http://example.com/'
        ]

        cmd = MockedBackendCommand(*args)
        cmd.run()
        cmd.outfile.close()

        items = [item for item in convert_cmd_output_to_json(self.fout_path)]

        self.assertEqual(len(items), 5)

        for x in range(5):
            item = items[x]
            expected_uuid = uuid('http://example.com/', str(x))

            self.assertEqual(item['data']['item'], x)
            self.assertEqual(item['origin'], 'http://example.com/')
            self.assertEqual(item['uuid'], expected_uuid)
            self.assertEqual(item['tag'], 'test')
コード例 #18
0
ファイル: test_git.py プロジェクト: albertinisg/perceval
    def test_fetch_since_date(self):
        """Test whether commits are fetched from a Git repository since the given date"""

        new_path = os.path.join(self.tmp_path, 'newgit')

        from_date = datetime.datetime(2014, 2, 11, 22, 7, 49)
        git = Git(self.git_path, new_path)
        commits = [commit for commit in git.fetch(from_date=from_date)]

        expected = [('ce8e0b86a1e9877f42fe9453ede418519115f367', 1392185269.0),
                    ('51a3b654f252210572297f47597b31527c475fb8', 1392185366.0),
                    ('456a68ee1407a77f3e804a30dff245bb6c6b872f', 1392185439.0)]

        self.assertEqual(len(commits), len(expected))

        for x in range(len(commits)):
            expected_uuid = uuid(self.git_path, expected[x][0])
            commit = commits[x]
            self.assertEqual(commit['data']['commit'], expected[x][0])
            self.assertEqual(commit['origin'], self.git_path)
            self.assertEqual(commit['uuid'], expected_uuid)
            self.assertEqual(commit['updated_on'], expected[x][1])
            self.assertEqual(commit['category'], 'commit')
            self.assertEqual(commit['tag'], self.git_path)

        # Test it using a datetime that includes the timezone
        from_date = datetime.datetime(2012, 8, 14, 14, 30, 00,
                                      tzinfo=dateutil.tz.tzoffset(None, -36000))
        git = Git(self.git_path, new_path)
        commits = [commit for commit in git.fetch(from_date=from_date)]

        self.assertEqual(len(commits), len(expected))

        for x in range(len(commits)):
            expected_uuid = uuid(self.git_path, expected[x][0])
            commit = commits[x]
            self.assertEqual(commit['data']['commit'], expected[x][0])
            self.assertEqual(commit['origin'], self.git_path)
            self.assertEqual(commit['uuid'], expected_uuid)
            self.assertEqual(commit['updated_on'], expected[x][1])
            self.assertEqual(commit['category'], 'commit')
            self.assertEqual(commit['tag'], self.git_path)

        shutil.rmtree(new_path)
コード例 #19
0
ファイル: test_backend.py プロジェクト: chubbymaggie/perceval
    def test_ignore_corrupted_archive(self):
        """Check if a corrupted archive is ignored while fetching from archive"""

        def delete_rows(db, table_name):
            conn = sqlite3.connect(db)
            cursor = conn.cursor()
            cursor.execute("DELETE FROM " + table_name)
            cursor.close()
            conn.commit()

        manager = ArchiveManager(self.test_path)

        args = {
            'origin': 'http://example.com/',
            'category': 'mock_item',
            'tag': 'test',
            'subtype': 'mocksubtype',
            'from-date': str_to_datetime('2015-01-01')
        }

        # First, fetch the items twice to check if several archive
        # are used
        items = fetch(CommandBackend, args, manager=manager)
        items = [item for item in items]
        self.assertEqual(len(items), 5)

        items = fetch(CommandBackend, args, manager=manager)
        items = [item for item in items]
        self.assertEqual(len(items), 5)

        # Find archive names to delete the rows of one of them to make it
        # corrupted
        filepaths = manager.search('http://example.com/', 'CommandBackend',
                                   'mock_item', str_to_datetime('1970-01-01'))
        self.assertEqual(len(filepaths), 2)

        to_remove = filepaths[0]
        delete_rows(to_remove, 'archive')

        # Fetch items from the archive
        items = fetch_from_archive(CommandBackend, args, manager,
                                   'mock_item', str_to_datetime('1970-01-01'))
        items = [item for item in items]

        self.assertEqual(len(items), 5)

        for x in range(5):
            item = items[x]
            expected_uuid = uuid('http://example.com/', str(x))

            self.assertEqual(item['data']['item'], x)
            self.assertEqual(item['data']['archive'], True)
            self.assertEqual(item['origin'], 'http://example.com/')
            self.assertEqual(item['uuid'], expected_uuid)
            self.assertEqual(item['tag'], 'test')
コード例 #20
0
ファイル: test_backend.py プロジェクト: sduenas/perceval
    def test_run_fetch_from_archive(self):
        """Test whether the command runs when fetch from archive is set"""

        args = [
            '--archive-path', self.test_path, '--from-date', '2015-01-01',
            '--tag', 'test', '--category', 'mock_item', '--subtype',
            'mocksubtype', '--output', self.fout_path, 'http://example.com/'
        ]

        cmd = MockedBackendCommand(*args)
        cmd.run()
        cmd.outfile.close()

        items = [item for item in convert_cmd_output_to_json(self.fout_path)]

        self.assertEqual(len(items), 5)

        args = [
            '--archive-path', self.test_path, '--fetch-archive', '--from-date',
            '2015-01-01', '--tag', 'test', '--category', 'mock_item',
            '--subtype', 'mocksubtype', '--output', self.fout_path,
            'http://example.com/'
        ]

        cmd = MockedBackendCommand(*args)
        cmd.run()
        cmd.outfile.close()

        items = [item for item in convert_cmd_output_to_json(self.fout_path)]
        self.assertEqual(len(items), 5)

        for x in range(5):
            item = items[x]
            expected_uuid = uuid('http://example.com/', str(x))

            # ArchiveMockedBackend sets 'archive' value when
            # 'fetch-archive' option is set. This helps to know
            # the code is really running
            self.assertEqual(item['data']['item'], x)
            self.assertEqual(item['data']['archive'], True)
            self.assertEqual(item['origin'], 'http://example.com/')
            self.assertEqual(item['uuid'], expected_uuid)
            self.assertEqual(item['tag'], 'test')
コード例 #21
0
ファイル: test_backend.py プロジェクト: sduenas/perceval
    def test_archive(self):
        """Test whether a set of items is fetched from the archive"""

        manager = ArchiveManager(self.test_path)

        args = {
            'origin': 'http://example.com/',
            'category': 'mock_item',
            'tag': 'test',
            'subtype': 'mocksubtype',
            'from-date': str_to_datetime('2015-01-01')
        }

        # First, fetch the items twice to check if several archive
        # are used
        items = fetch(CommandBackend, args, manager=manager)
        items = [item for item in items]
        self.assertEqual(len(items), 5)

        items = fetch(CommandBackend, args, manager=manager)
        items = [item for item in items]
        self.assertEqual(len(items), 5)

        # Fetch items from the archive
        items = fetch_from_archive(CommandBackend, args, manager, 'mock_item',
                                   str_to_datetime('1970-01-01'))
        items = [item for item in items]

        self.assertEqual(len(items), 10)

        for x in range(2):
            for y in range(5):
                item = items[y + (x * 5)]
                expected_uuid = uuid('http://example.com/', str(y))

                self.assertEqual(item['data']['item'], y)
                self.assertEqual(item['data']['archive'], True)
                self.assertEqual(item['origin'], 'http://example.com/')
                self.assertEqual(item['uuid'], expected_uuid)
                self.assertEqual(item['tag'], 'test')
コード例 #22
0
    def test_decorator(self):
        backend = MockDecoratorBackend('test')
        before = datetime.datetime.now().timestamp()
        items = [item for item in backend.fetch()]
        after = datetime.datetime.now().timestamp()

        for x in range(5):
            item = items[x]

            expected_uuid = uuid('test', str(x))

            self.assertEqual(item['data']['item'], x)
            self.assertEqual(item['backend_name'], 'MockDecoratorBackend')
            self.assertEqual(item['backend_version'], '0.1.0')
            self.assertEqual(item['perceval_version'], __version__)
            self.assertEqual(item['origin'], 'test')
            self.assertEqual(item['uuid'], expected_uuid)
            self.assertEqual(item['updated_on'], '2016-01-01')
            self.assertGreater(item['timestamp'], before)
            self.assertLess(item['timestamp'], after)

            before = item['timestamp']
コード例 #23
0
ファイル: dockerdeps.py プロジェクト: valeriocos/GrimoireELK
    def get_rich_item(self, item, file_path, dep):
        commit = item['data']
        eitem = {
            'file_path': file_path,
            'dependency': dep,
            'commit_sha': commit['commit'],
            'origin': item['origin'],
            'author_date': fix_field_date(commit['AuthorDate']),
            'commit_date': fix_field_date(commit['CommitDate'])
        }

        if self.prjs_map:
            eitem.update(self.get_item_project(eitem))

        # uuid
        uuid_sha1 = backend.uuid(eitem['file_path'], eitem['dependency'])
        eitem['id'] = "{}_{}".format(eitem['commit_sha'], uuid_sha1)
        eitem.update(self.get_grimoire_fields(eitem["author_date"], "file"))
        self.add_repository_labels(eitem)
        self.add_metadata_filter_raw(eitem)

        return eitem
コード例 #24
0
ファイル: test_backend.py プロジェクト: jsmanrique/perceval
    def test_decorator(self):
        backend = MockDecoratorBackend('test')
        before = datetime.datetime.now().timestamp()
        items = [item for item in backend.fetch()]
        after = datetime.datetime.now().timestamp()

        for x in range(5):
            item = items[x]
            meta = item['__metadata__']

            expected_uuid = uuid('test', str(x))

            self.assertEqual(item['item'], x)
            self.assertEqual(meta['backend_name'], 'MockDecoratorBackend')
            self.assertEqual(meta['backend_version'], '0.1.0')
            self.assertEqual(meta['origin'], 'test')
            self.assertEqual(meta['uuid'], expected_uuid)
            self.assertEqual(meta['updated_on'], '2016-01-01')
            self.assertGreater(meta['timestamp'], before)
            self.assertLess(meta['timestamp'], after)

            before = meta['timestamp']
コード例 #25
0
ファイル: test_backend.py プロジェクト: grimoirelab/perceval
    def test_decorator(self):
        backend = MockDecoratorBackend('test')
        before = datetime.datetime.now().timestamp()
        items = [item for item in backend.fetch()]
        after = datetime.datetime.now().timestamp()

        for x in range(5):
            item = items[x]

            expected_uuid = uuid('test', str(x))

            self.assertEqual(item['data']['item'], x)
            self.assertEqual(item['backend_name'], 'MockDecoratorBackend')
            self.assertEqual(item['backend_version'], '0.2.0')
            self.assertEqual(item['perceval_version'], __version__)
            self.assertEqual(item['origin'], 'test')
            self.assertEqual(item['uuid'], expected_uuid)
            self.assertEqual(item['updated_on'], '2016-01-01')
            self.assertEqual(item['category'], 'mock_item')
            self.assertGreater(item['timestamp'], before)
            self.assertLess(item['timestamp'], after)

            before = item['timestamp']
コード例 #26
0
ファイル: test_backend.py プロジェクト: sduenas/perceval
    def test_items(self):
        """Test whether a set of items is returned"""

        args = {
            'origin': 'http://example.com/',
            'category': 'mock_item',
            'tag': 'test',
            'subtype': 'mocksubtype',
            'from-date': str_to_datetime('2015-01-01')
        }

        items = fetch(CommandBackend, args, manager=None)
        items = [item for item in items]

        self.assertEqual(len(items), 5)

        for x in range(5):
            item = items[x]
            expected_uuid = uuid('http://example.com/', str(x))

            self.assertEqual(item['data']['item'], x)
            self.assertEqual(item['origin'], 'http://example.com/')
            self.assertEqual(item['uuid'], expected_uuid)
            self.assertEqual(item['tag'], 'test')
コード例 #27
0
    def test_fetch_branch(self):
        """Test whether commits are fetched from a Git repository for a given branch"""

        new_path = os.path.join(self.tmp_path, 'newgit')

        from_date = datetime.datetime(2014, 2, 11, 22, 7, 49)
        git = Git(self.git_path, new_path)
        # Let's fetch master
        commits = [commit for commit in git.fetch(branches=['master'])]

        expected = ['bc57a9209f096a130dcc5ba7089a8663f758a703',
                    '87783129c3f00d2c81a3a8e585eb86a47e39891a',
                    '7debcf8a2f57f86663809c58b5c07a398be7674c',
                    'c0d66f92a95e31c77be08dc9d0f11a16715d1885',
                    'c6ba8f7a1058db3e6b4bc6f1090e932b107605fb',
                    '589bb080f059834829a2a5955bebfd7c2baa110a',
                    'ce8e0b86a1e9877f42fe9453ede418519115f367',
                    '51a3b654f252210572297f47597b31527c475fb8',
                    '456a68ee1407a77f3e804a30dff245bb6c6b872f']

        self.assertEqual(len(commits), len(expected))

        for x in range(len(commits)):
            expected_uuid = uuid(self.git_path, expected[x])
            commit = commits[x]
            self.assertEqual(commit['data']['commit'], expected[x])
            self.assertEqual(commit['origin'], self.git_path)
            self.assertEqual(commit['uuid'], expected_uuid)
            self.assertEqual(commit['category'], 'commit')
            self.assertEqual(commit['tag'], self.git_path)

        # Now let's fetch lzp
        commits = [commit for commit in git.fetch(branches=['lzp'])]

        expected = ['bc57a9209f096a130dcc5ba7089a8663f758a703',
                    '87783129c3f00d2c81a3a8e585eb86a47e39891a',
                    '7debcf8a2f57f86663809c58b5c07a398be7674c',
                    'c0d66f92a95e31c77be08dc9d0f11a16715d1885',
                    'c6ba8f7a1058db3e6b4bc6f1090e932b107605fb',
                    '589bb080f059834829a2a5955bebfd7c2baa110a',
                    '51a3b654f252210572297f47597b31527c475fb8']

        self.assertEqual(len(commits), len(expected))

        for x in range(len(commits)):
            expected_uuid = uuid(self.git_path, expected[x])
            commit = commits[x]
            self.assertEqual(commit['data']['commit'], expected[x])
            self.assertEqual(commit['origin'], self.git_path)
            self.assertEqual(commit['uuid'], expected_uuid)
            self.assertEqual(commit['category'], 'commit')
            self.assertEqual(commit['tag'], self.git_path)

        # Now, let's fech master and lzp
        commits = [commit for commit in git.fetch(branches=['master', 'lzp'])]

        expected = ['bc57a9209f096a130dcc5ba7089a8663f758a703',
                    '87783129c3f00d2c81a3a8e585eb86a47e39891a',
                    '7debcf8a2f57f86663809c58b5c07a398be7674c',
                    'c0d66f92a95e31c77be08dc9d0f11a16715d1885',
                    'c6ba8f7a1058db3e6b4bc6f1090e932b107605fb',
                    '589bb080f059834829a2a5955bebfd7c2baa110a',
                    'ce8e0b86a1e9877f42fe9453ede418519115f367',
                    '51a3b654f252210572297f47597b31527c475fb8',
                    '456a68ee1407a77f3e804a30dff245bb6c6b872f']

        self.assertEqual(len(commits), len(expected))

        for x in range(len(commits)):
            expected_uuid = uuid(self.git_path, expected[x])
            commit = commits[x]
            self.assertEqual(commit['data']['commit'], expected[x])
            self.assertEqual(commit['origin'], self.git_path)
            self.assertEqual(commit['uuid'], expected_uuid)
            self.assertEqual(commit['category'], 'commit')
            self.assertEqual(commit['tag'], self.git_path)

        # Now, let's fetch None, which means "all commits"
        commits = [commit for commit in git.fetch(branches=None)]

        expected = ['bc57a9209f096a130dcc5ba7089a8663f758a703',
                    '87783129c3f00d2c81a3a8e585eb86a47e39891a',
                    '7debcf8a2f57f86663809c58b5c07a398be7674c',
                    'c0d66f92a95e31c77be08dc9d0f11a16715d1885',
                    'c6ba8f7a1058db3e6b4bc6f1090e932b107605fb',
                    '589bb080f059834829a2a5955bebfd7c2baa110a',
                    'ce8e0b86a1e9877f42fe9453ede418519115f367',
                    '51a3b654f252210572297f47597b31527c475fb8',
                    '456a68ee1407a77f3e804a30dff245bb6c6b872f']

        self.assertEqual(len(commits), len(expected))

        for x in range(len(commits)):
            expected_uuid = uuid(self.git_path, expected[x])
            commit = commits[x]
            self.assertEqual(commit['data']['commit'], expected[x])
            self.assertEqual(commit['origin'], self.git_path)
            self.assertEqual(commit['uuid'], expected_uuid)
            self.assertEqual(commit['category'], 'commit')
            self.assertEqual(commit['tag'], self.git_path)

        # Now, let's fetch [], which means "no commits"
        commits = [commit for commit in git.fetch(branches=[])]

        expected = []

        self.assertEqual(len(commits), len(expected))

        shutil.rmtree(new_path)
コード例 #28
0
ファイル: test_git.py プロジェクト: albertinisg/perceval
    def test_fetch_branch(self):
        """Test whether commits are fetched from a Git repository for a given branch"""

        new_path = os.path.join(self.tmp_path, 'newgit')

        from_date = datetime.datetime(2014, 2, 11, 22, 7, 49)
        git = Git(self.git_path, new_path)
        # Let's fetch master
        commits = [commit for commit in git.fetch(branches=['master'])]

        expected = ['bc57a9209f096a130dcc5ba7089a8663f758a703',
                    '87783129c3f00d2c81a3a8e585eb86a47e39891a',
                    '7debcf8a2f57f86663809c58b5c07a398be7674c',
                    'c0d66f92a95e31c77be08dc9d0f11a16715d1885',
                    'c6ba8f7a1058db3e6b4bc6f1090e932b107605fb',
                    '589bb080f059834829a2a5955bebfd7c2baa110a',
                    'ce8e0b86a1e9877f42fe9453ede418519115f367',
                    '51a3b654f252210572297f47597b31527c475fb8',
                    '456a68ee1407a77f3e804a30dff245bb6c6b872f']

        self.assertEqual(len(commits), len(expected))

        for x in range(len(commits)):
            expected_uuid = uuid(self.git_path, expected[x])
            commit = commits[x]
            self.assertEqual(commit['data']['commit'], expected[x])
            self.assertEqual(commit['origin'], self.git_path)
            self.assertEqual(commit['uuid'], expected_uuid)
            self.assertEqual(commit['category'], 'commit')
            self.assertEqual(commit['tag'], self.git_path)

        # Now let's fetch lzp
        commits = [commit for commit in git.fetch(branches=['lzp'])]

        expected = ['bc57a9209f096a130dcc5ba7089a8663f758a703',
                    '87783129c3f00d2c81a3a8e585eb86a47e39891a',
                    '7debcf8a2f57f86663809c58b5c07a398be7674c',
                    'c0d66f92a95e31c77be08dc9d0f11a16715d1885',
                    'c6ba8f7a1058db3e6b4bc6f1090e932b107605fb',
                    '589bb080f059834829a2a5955bebfd7c2baa110a',
                    '51a3b654f252210572297f47597b31527c475fb8']

        self.assertEqual(len(commits), len(expected))

        for x in range(len(commits)):
            expected_uuid = uuid(self.git_path, expected[x])
            commit = commits[x]
            self.assertEqual(commit['data']['commit'], expected[x])
            self.assertEqual(commit['origin'], self.git_path)
            self.assertEqual(commit['uuid'], expected_uuid)
            self.assertEqual(commit['category'], 'commit')
            self.assertEqual(commit['tag'], self.git_path)

        # Now, let's fech master and lzp
        commits = [commit for commit in git.fetch(branches=['master', 'lzp'])]

        expected = ['bc57a9209f096a130dcc5ba7089a8663f758a703',
                    '87783129c3f00d2c81a3a8e585eb86a47e39891a',
                    '7debcf8a2f57f86663809c58b5c07a398be7674c',
                    'c0d66f92a95e31c77be08dc9d0f11a16715d1885',
                    'c6ba8f7a1058db3e6b4bc6f1090e932b107605fb',
                    '589bb080f059834829a2a5955bebfd7c2baa110a',
                    'ce8e0b86a1e9877f42fe9453ede418519115f367',
                    '51a3b654f252210572297f47597b31527c475fb8',
                    '456a68ee1407a77f3e804a30dff245bb6c6b872f']

        self.assertEqual(len(commits), len(expected))

        for x in range(len(commits)):
            expected_uuid = uuid(self.git_path, expected[x])
            commit = commits[x]
            self.assertEqual(commit['data']['commit'], expected[x])
            self.assertEqual(commit['origin'], self.git_path)
            self.assertEqual(commit['uuid'], expected_uuid)
            self.assertEqual(commit['category'], 'commit')
            self.assertEqual(commit['tag'], self.git_path)

        # Now, let's fetch None, which means "all commits"
        commits = [commit for commit in git.fetch(branches=None)]

        expected = ['bc57a9209f096a130dcc5ba7089a8663f758a703',
                    '87783129c3f00d2c81a3a8e585eb86a47e39891a',
                    '7debcf8a2f57f86663809c58b5c07a398be7674c',
                    'c0d66f92a95e31c77be08dc9d0f11a16715d1885',
                    'c6ba8f7a1058db3e6b4bc6f1090e932b107605fb',
                    '589bb080f059834829a2a5955bebfd7c2baa110a',
                    'ce8e0b86a1e9877f42fe9453ede418519115f367',
                    '51a3b654f252210572297f47597b31527c475fb8',
                    '456a68ee1407a77f3e804a30dff245bb6c6b872f']

        self.assertEqual(len(commits), len(expected))

        for x in range(len(commits)):
            expected_uuid = uuid(self.git_path, expected[x])
            commit = commits[x]
            self.assertEqual(commit['data']['commit'], expected[x])
            self.assertEqual(commit['origin'], self.git_path)
            self.assertEqual(commit['uuid'], expected_uuid)
            self.assertEqual(commit['category'], 'commit')
            self.assertEqual(commit['tag'], self.git_path)

        # Now, let's fetch [], which means "no commits"
        commits = [commit for commit in git.fetch(branches=[])]

        expected = []

        self.assertEqual(len(commits), len(expected))

        shutil.rmtree(new_path)