Ejemplo n.º 1
0
def run(project_id, repo_path, cursor, **options):
    print("----- METRIC: STARS -----")
    threshold = options.get('threshold', 0)
    cursor.execute(QUERY.format(project_id))
    full_url = cursor.fetchone()[0]
    git_tokens = options['tokens']
    token_avail = False
    # Making a github api request with the tokens provided 
    for token in git_tokens:
        try: 
            rresult = url_to_json(full_url,authentication=[git_tokens[token],token])["stargazers_count"]
            token_avail = True
            break
        except:
            continue
    # Making api request without token, in the case all OAuth tokens got expired or incorrect tokens provided  
    if(token_avail == False):
        try:
            print('without token - stargazers - fetch ok')
            rresult = url_to_json(full_url)["stargazers_count"]
            print('Fetch Successful')
        except Exception as ex:
            print(ex)
            rresult = None
    bresult = True if rresult is not None and rresult >= threshold else False
    print('stars: ',rresult)
    return bresult, rresult
    def _init_repository(self, project_id, repository_home):
        repository_path = repository_home  # Default

        if not os.path.exists(repository_path):
            os.mkdir(repository_path)

        items = os.listdir(repository_path)
        if items:
            for item in os.listdir(repository_path):
                itempath = os.path.join(repository_path, item)
                if os.path.isdir(itempath):
                    repository_path = itempath
                    break
        else:
            url = self.database.get('''
                    SELECT url FROM projects WHERE id={0}
                '''.format(project_id)).replace(
                "https://api.github.com/repos/", "").split("/")
            (repo_owner, repo_name) = (url[0], url[1])
            if not (repo_owner or repo_name):
                raise ValueError('Invalid project ID {0}.'.format(project_id))
            get_url = self.database.get('''
                    SELECT url FROM projects WHERE id={0}
                '''.format(project_id)) + "/commits"
            try:
                print(self.tokens)
            except:
                print('')
            token_avail = False
            for token in self.tokens:
                if (token_avail == True):
                    break
                else:
                    try:
                        last_commit_date = utilities.url_to_json(
                            get_url,
                            [self.tokens[token], token
                             ])[0]["commit"]["committer"]["date"].split("T")[0]
                        token_avail = True
                    except:
                        continue
            if (token_avail == False):
                try:
                    print(
                        "[Reg: Last_Commit_Date]Couldn't fetch data from API! Trying out without token.."
                    )
                    last_commit_date = utilities.url_to_json(get_url)[0][
                        "commit"]["committer"]["date"].split("T")[0]
                    print('Fetch Successful')
                except Exception as ex:
                    print(ex)
                    last_commit_date = None
            if last_commit_date is None:
                last_commit_date = self.today
            repository_path = utilities.clone(repo_owner, repo_name,
                                              repository_path,
                                              last_commit_date)

        return repository_path
Ejemplo n.º 3
0
def license():
    os.chdir(root_directory)
    json_response = {'license': False}
    try:
        flag = False
        for token in git_tokens:
            if(flag == True):
                break
            else:
                try:
                    json_response = url_to_json(URL, headers={
                            'Accept': 'application/vnd.github.drax-preview+json'
                        }
                    )
                    flag = True
                    break
                except:
                    continue
        if(flag == False):
            try:
                print("[Reg: License]Tokens didn't work! Trying out without token...")
                json_response = url_to_json(URL, headers={
                                'Accept': 'application/vnd.github.drax-preview+json'
                            }, authentication=[]
                        )
                print('Fetch ok')
            except:
                print("[Reg: License]Couldn't fetch data from API! Trying out search for patterns in the license files..")
        result = True if 'license' in json_response \
                         and json_response['license'] else False
        if not result:
            for pattern in LICENSE_PATTERNS:
                if utilities.search(pattern, repo_path, ignorecase=True):
                    result = True
                    break
        if(result == True):
            val = (str(int(result)),repo_id)
            QUERY = '''UPDATE `repoquester_results` SET `license`=? WHERE `repo_id`=?'''
            try:
                os.chdir(root_directory)
                conn = sqlite3.connect('repo_quester.db',timeout=180)
                conn.execute(QUERY,val)
                conn.commit()
                conn.close()
                print('LICENSE: ',result)
            except Exception as ex:
                print(ex)
                os.chdir(root_directory)
                conn = sqlite3.connect('repo_quester.db',timeout=180)
                conn.execute(QUERY,val)
                conn.commit()
                conn.close()
                print('LICENSE: ',result,' trying again.. ok')
    except Exception as ex:
        print(ex)
Ejemplo n.º 4
0
def run(project_id, repo_path, cursor, **options):
    print("----- METRIC: LICENSE -----")
    cursor.execute('''
        SELECT
            url
        FROM
            projects
        WHERE
            id = {0}
        '''.format(project_id))
    git_tokens = options['tokens']
    record = cursor.fetchone()
    full_url = record[0]
    token_avail = False

    # Making a github api request with the tokens provided
    for token in git_tokens:
        if (token_avail == True):
            break
        else:
            try:
                json_response = url_to_json(
                    full_url,
                    headers={
                        'Accept': 'application/vnd.github.drax-preview+json'
                    },
                    authentication=[git_tokens[token], token])
                token_avail = True
            except:
                continue
    # Making api request without token, in the case all OAuth tokens got expired or incorrect tokens provided
    if (token_avail == False):
        try:
            print(
                "[Reg: License]Tokens didn't work! Trying out without token..."
            )
            json_response = url_to_json(
                full_url,
                headers={'Accept': 'application/vnd.github.drax-preview+json'},
                authentication=[])
            print('Fetch Successful')
        except Exception as ex:
            print(ex)
            json_response = None
    result = True if 'license' in json_response \
        and json_response['license'] else False
    if not result:
        for pattern in LICENSE_PATTERNS:
            if utilities.search(pattern, repo_path, ignorecase=True):
                result = True
                break
    print('License: ', result)
    return result, result
Ejemplo n.º 5
0
def run(project_id, repo_path, cursor, **options):
    print("----- METRIC: LICENSE -----")
    cursor.execute('''
        SELECT
            url
        FROM
            projects
        WHERE
            id = {0}
        '''.format(project_id))
    git_tokens = options['tokens']
    record = cursor.fetchone()
    full_url = record[0]
    token_avail = False
    for user_name in git_tokens:
        if (token_avail == True):
            break
        else:
            try:
                json_response = url_to_json(
                    full_url,
                    headers={
                        'Accept': 'application/vnd.github.drax-preview+json'
                    },
                    authentication=[user_name, git_tokens[user_name]])
                token_avail = True
            except:
                continue
    if (token_avail == False):
        try:
            print(
                "[Reg: License]Tokens didn't work! Trying out without token..."
            )
            json_response = url_to_json(
                full_url,
                headers={'Accept': 'application/vnd.github.drax-preview+json'},
                authentication=[])
            print('Fetch Successful')
        except:
            print(
                "[Reg: License]Couldn't fetch data from API! Trying out search for patterns in the license files.."
            )
    result = True if 'license' in json_response \
        and json_response['license'] else False
    if not result:
        for pattern in LICENSE_PATTERNS:
            if utilities.search(pattern, repo_path, ignorecase=True):
                result = True
                break
    print('License: ', result)
    return result, result
Ejemplo n.º 6
0
def run(project_id, repo_path, cursor, **options):
    cursor.execute('''
        SELECT
            url
        FROM
            projects
        WHERE
            id = {0}
        '''.format(project_id))

    record = cursor.fetchone()

    full_url = record[0]
    json_response = url_to_json(
        full_url,
        headers={'Accept': 'application/vnd.github.drax-preview+json'})

    result = True if 'license' in json_response \
                     and json_response['license'] else False

    if not result:
        for pattern in LICENSE_PATTERNS:
            if utilities.search(pattern, repo_path, ignorecase=True):
                result = True
                break
    print("----- METRIC: LICENSE -----")
    print('result: ', result)
    return result, result
Ejemplo n.º 7
0
    def get_token(self):
        while True:
            if not self.scheduler.get_jobs() and self.available_tokens.empty():
                self.print_warning('No more valid OAuth tokens available.')
                return None

            token = self.available_tokens.get(block=True)

            rate_limit_url = (
                'https://api.github.com/rate_limit?access_token={0}'
            ).format(token)
            status = url_to_json(rate_limit_url)

            # Throw away bad OAuth keys.
            if 'resources' not in status:
                self.print_warning(
                    'Invalid OAuth token supplied. Trying again...'
                )
                continue

            if status['resources']['core']['remaining'] > 0:
                self.available_tokens.put_nowait(token)
                return token
            else:
                self.scheduler.add_job(
                    self.available_tokens.put_nowait,
                    'date',
                    args=[token],
                    run_date=datetime.datetime.fromtimestamp(
                        status['resources']['core']['reset']
                    )
                )
Ejemplo n.º 8
0
def run(project_id, repo_path, cursor, **options):
    cursor.execute('''
        SELECT
            url
        FROM
            projects
        WHERE
            id = {0}
        '''.format(project_id))

    record = cursor.fetchone()

    full_url = utilities.TOKENIZER.tokenize(record[0].rstrip())
    json_response = url_to_json(full_url, headers={
            'Accept': 'application/vnd.github.drax-preview+json'
        }
    )

    result = True if 'license' in json_response \
                     and json_response['license'] else False

    if not result:
        for pattern in LICENSE_PATTERNS:
            if utilities.search(pattern, repo_path, ignorecase=True):
                result = True
                break

    return result, result
Ejemplo n.º 9
0
    def get_token(self):
        while True:
            if not self.scheduler.get_jobs() and self.available_tokens.empty():
                self.print_warning('No more valid OAuth tokens available.')
                return None

            token = self.available_tokens.get(block=True)

            rate_limit_url = (
                'https://api.github.com/rate_limit?access_token={0}'
            ).format(token)
            status = url_to_json(rate_limit_url)

            # Throw away bad OAuth keys.
            if 'resources' not in status:
                self.print_warning(
                    'Invalid OAuth token supplied. Trying again...'
                )
                continue

            if status['resources']['core']['remaining'] > 0:
                self.available_tokens.put_nowait(token)
                return token
            else:
                self.scheduler.add_job(
                    self.available_tokens.put_nowait,
                    'date',
                    args=[token],
                    run_date=datetime.datetime.fromtimestamp(
                        status['resources']['core']['reset']
                    )
                )
Ejemplo n.º 10
0
def run(project_id, repo_path, cursor, **options):
    threshold = options.get('threshold', 0)

    cursor.execute('SELECT url FROM projects WHERE id = {}'.format(project_id))
    record = cursor.fetchone()

    full_url = utilities.TOKENIZER.tokenize(record[0].rstrip())
    json_response = utilities.url_to_json(full_url)
    rresult = (json_response['stargazers_count']
               if 'stargazers_count' in json_response else None)
    bresult = True if rresult is not None and rresult >= threshold else False

    return bresult, rresult