def run(project_id, repo_path, cursor, **options):
print("----- METRIC: STARS -----")
threshold = options.get('threshold', 0)
cursor.execute(QUERY.format(project_id))
full_url = cursor.fetchone()[0]
git_tokens = options['tokens']
token_avail = False
# Making a github api request with the tokens provided
for token in git_tokens:
try:
rresult = url_to_json(full_url,authentication=[git_tokens[token],token])["stargazers_count"]
token_avail = True
break
except:
continue
# Making api request without token, in the case all OAuth tokens got expired or incorrect tokens provided
if(token_avail == False):
try:
print('without token - stargazers - fetch ok')
rresult = url_to_json(full_url)["stargazers_count"]
print('Fetch Successful')
except Exception as ex:
print(ex)
rresult = None
bresult = True if rresult is not None and rresult >= threshold else False
print('stars: ',rresult)
return bresult, rresult
def _init_repository(self, project_id, repository_home):
repository_path = repository_home # Default
if not os.path.exists(repository_path):
os.mkdir(repository_path)
items = os.listdir(repository_path)
if items:
for item in os.listdir(repository_path):
itempath = os.path.join(repository_path, item)
if os.path.isdir(itempath):
repository_path = itempath
break
else:
url = self.database.get('''
SELECT url FROM projects WHERE id={0}
'''.format(project_id)).replace(
"https://api.github.com/repos/", "").split("/")
(repo_owner, repo_name) = (url[0], url[1])
if not (repo_owner or repo_name):
raise ValueError('Invalid project ID {0}.'.format(project_id))
get_url = self.database.get('''
SELECT url FROM projects WHERE id={0}
'''.format(project_id)) + "/commits"
try:
print(self.tokens)
except:
print('')
token_avail = False
for token in self.tokens:
if (token_avail == True):
break
else:
try:
last_commit_date = utilities.url_to_json(
get_url,
[self.tokens[token], token
])[0]["commit"]["committer"]["date"].split("T")[0]
token_avail = True
except:
continue
if (token_avail == False):
try:
print(
"[Reg: Last_Commit_Date]Couldn't fetch data from API! Trying out without token.."
)
last_commit_date = utilities.url_to_json(get_url)[0][
"commit"]["committer"]["date"].split("T")[0]
print('Fetch Successful')
except Exception as ex:
print(ex)
last_commit_date = None
if last_commit_date is None:
last_commit_date = self.today
repository_path = utilities.clone(repo_owner, repo_name,
repository_path,
last_commit_date)
return repository_path
def license():
os.chdir(root_directory)
json_response = {'license': False}
try:
flag = False
for token in git_tokens:
if(flag == True):
break
else:
try:
json_response = url_to_json(URL, headers={
'Accept': 'application/vnd.github.drax-preview+json'
}
)
flag = True
break
except:
continue
if(flag == False):
try:
print("[Reg: License]Tokens didn't work! Trying out without token...")
json_response = url_to_json(URL, headers={
'Accept': 'application/vnd.github.drax-preview+json'
}, authentication=[]
)
print('Fetch ok')
except:
print("[Reg: License]Couldn't fetch data from API! Trying out search for patterns in the license files..")
result = True if 'license' in json_response \
and json_response['license'] else False
if not result:
for pattern in LICENSE_PATTERNS:
if utilities.search(pattern, repo_path, ignorecase=True):
result = True
break
if(result == True):
val = (str(int(result)),repo_id)
QUERY = '''UPDATE `repoquester_results` SET `license`=? WHERE `repo_id`=?'''
try:
os.chdir(root_directory)
conn = sqlite3.connect('repo_quester.db',timeout=180)
conn.execute(QUERY,val)
conn.commit()
conn.close()
print('LICENSE: ',result)
except Exception as ex:
print(ex)
os.chdir(root_directory)
conn = sqlite3.connect('repo_quester.db',timeout=180)
conn.execute(QUERY,val)
conn.commit()
conn.close()
print('LICENSE: ',result,' trying again.. ok')
except Exception as ex:
print(ex)
def run(project_id, repo_path, cursor, **options):
print("----- METRIC: LICENSE -----")
cursor.execute('''
SELECT
url
FROM
projects
WHERE
id = {0}
'''.format(project_id))
git_tokens = options['tokens']
record = cursor.fetchone()
full_url = record[0]
token_avail = False
# Making a github api request with the tokens provided
for token in git_tokens:
if (token_avail == True):
break
else:
try:
json_response = url_to_json(
full_url,
headers={
'Accept': 'application/vnd.github.drax-preview+json'
},
authentication=[git_tokens[token], token])
token_avail = True
except:
continue
# Making api request without token, in the case all OAuth tokens got expired or incorrect tokens provided
if (token_avail == False):
try:
print(
"[Reg: License]Tokens didn't work! Trying out without token..."
)
json_response = url_to_json(
full_url,
headers={'Accept': 'application/vnd.github.drax-preview+json'},
authentication=[])
print('Fetch Successful')
except Exception as ex:
print(ex)
json_response = None
result = True if 'license' in json_response \
and json_response['license'] else False
if not result:
for pattern in LICENSE_PATTERNS:
if utilities.search(pattern, repo_path, ignorecase=True):
result = True
break
print('License: ', result)
return result, result
def run(project_id, repo_path, cursor, **options):
print("----- METRIC: LICENSE -----")
cursor.execute('''
SELECT
url
FROM
projects
WHERE
id = {0}
'''.format(project_id))
git_tokens = options['tokens']
record = cursor.fetchone()
full_url = record[0]
token_avail = False
for user_name in git_tokens:
if (token_avail == True):
break
else:
try:
json_response = url_to_json(
full_url,
headers={
'Accept': 'application/vnd.github.drax-preview+json'
},
authentication=[user_name, git_tokens[user_name]])
token_avail = True
except:
continue
if (token_avail == False):
try:
print(
"[Reg: License]Tokens didn't work! Trying out without token..."
)
json_response = url_to_json(
full_url,
headers={'Accept': 'application/vnd.github.drax-preview+json'},
authentication=[])
print('Fetch Successful')
except:
print(
"[Reg: License]Couldn't fetch data from API! Trying out search for patterns in the license files.."
)
result = True if 'license' in json_response \
and json_response['license'] else False
if not result:
for pattern in LICENSE_PATTERNS:
if utilities.search(pattern, repo_path, ignorecase=True):
result = True
break
print('License: ', result)
return result, result
def run(project_id, repo_path, cursor, **options):
cursor.execute('''
SELECT
url
FROM
projects
WHERE
id = {0}
'''.format(project_id))
record = cursor.fetchone()
full_url = record[0]
json_response = url_to_json(
full_url,
headers={'Accept': 'application/vnd.github.drax-preview+json'})
result = True if 'license' in json_response \
and json_response['license'] else False
if not result:
for pattern in LICENSE_PATTERNS:
if utilities.search(pattern, repo_path, ignorecase=True):
result = True
break
print("----- METRIC: LICENSE -----")
print('result: ', result)
return result, result
def get_token(self):
while True:
if not self.scheduler.get_jobs() and self.available_tokens.empty():
self.print_warning('No more valid OAuth tokens available.')
return None
token = self.available_tokens.get(block=True)
rate_limit_url = (
'https://api.github.com/rate_limit?access_token={0}'
).format(token)
status = url_to_json(rate_limit_url)
# Throw away bad OAuth keys.
if 'resources' not in status:
self.print_warning(
'Invalid OAuth token supplied. Trying again...'
)
continue
if status['resources']['core']['remaining'] > 0:
self.available_tokens.put_nowait(token)
return token
else:
self.scheduler.add_job(
self.available_tokens.put_nowait,
'date',
args=[token],
run_date=datetime.datetime.fromtimestamp(
status['resources']['core']['reset']
)
)
def run(project_id, repo_path, cursor, **options):
cursor.execute('''
SELECT
url
FROM
projects
WHERE
id = {0}
'''.format(project_id))
record = cursor.fetchone()
full_url = utilities.TOKENIZER.tokenize(record[0].rstrip())
json_response = url_to_json(full_url, headers={
'Accept': 'application/vnd.github.drax-preview+json'
}
)
result = True if 'license' in json_response \
and json_response['license'] else False
if not result:
for pattern in LICENSE_PATTERNS:
if utilities.search(pattern, repo_path, ignorecase=True):
result = True
break
return result, result
def get_token(self):
while True:
if not self.scheduler.get_jobs() and self.available_tokens.empty():
self.print_warning('No more valid OAuth tokens available.')
return None
token = self.available_tokens.get(block=True)
rate_limit_url = (
'https://api.github.com/rate_limit?access_token={0}'
).format(token)
status = url_to_json(rate_limit_url)
# Throw away bad OAuth keys.
if 'resources' not in status:
self.print_warning(
'Invalid OAuth token supplied. Trying again...'
)
continue
if status['resources']['core']['remaining'] > 0:
self.available_tokens.put_nowait(token)
return token
else:
self.scheduler.add_job(
self.available_tokens.put_nowait,
'date',
args=[token],
run_date=datetime.datetime.fromtimestamp(
status['resources']['core']['reset']
)
)
def run(project_id, repo_path, cursor, **options):
threshold = options.get('threshold', 0)
cursor.execute('SELECT url FROM projects WHERE id = {}'.format(project_id))
record = cursor.fetchone()
full_url = utilities.TOKENIZER.tokenize(record[0].rstrip())
json_response = utilities.url_to_json(full_url)
rresult = (json_response['stargazers_count']
if 'stargazers_count' in json_response else None)
bresult = True if rresult is not None and rresult >= threshold else False
return bresult, rresult