def change_status(target_repo, state):
from models import Repo
# class Repo(Document):
# url = StringField(max_length=100,default='Not set yet')
# last_used = DateTimeField(default=datetime.now())
# created_on = DateTimeField(default=datetime.now())
# monitoring = StringField(max_length=100,default='Not set yet')
# state = StringField(max_length=50,default='Ready')
# owner = StringField(max_length=50,default='no')
if not use_database:
return ''
try:
repo = Repo.objects.get(url=target_repo)
repo.last_used = datetime.today()
repo.state = state
repo.owner = parent_folder
repo.save()
except DoesNotExist:
repo = Repo()
repo.url=target_repo
repo.state = state
repo.owner = parent_folder
repo.save()
except Exception as e:
print 'database_exception: '+str(e)
def create(self, request, *args, **kwargs):
""" Generates db repo object """
user = request.data['user']
path = request.data['path']
version = request.data['version']
res = RepoManager(user).create()
if not version:
version = res
r = Repo(user=user, path=path, version=version)
r.save()
rs = RepoSerializer(r)
return Response(rs.data)
def create(self, request, *args, **kwargs):
""" Generates db repo object """
user = request.data['user']
path = request.data['path']
version = request.data['version']
res = RepoManager(user).create()
if not version:
version = res
r = Repo(user=user, path=path, version=version)
r.save()
rs = RepoSerializer(r)
return Response(rs.data)
def reconcile_db_with_gh(*args, **kwargs):
ghc = GitHubConnector()
issues = ghc.get_all_issues()
repos = ghc.get_all_repos()
for repo in repos:
r = Repo(github_id=repo.id, name=repo.name)
r.save()
for issue in issues:
i = Issue(github_id=issue.id)
i.title = issue.title
i.number = issue.number
i.repo = Repo.objects.get(name=issue.repository[1])
i.save()
print "Not only did your task run successfully, but you're damned good looking too."
def workflow(changed_files, repo_str):
global current_repo, current_user
current_repo = Repo(name=repo_str, user=current_user, status="starting", started_at=datetime.now(), progress=10)
current_repo.save()
dolog("deleting the repo if exists")
change_status("preparation", 10)
delete_forked_repo(repo_str)
time.sleep(60) # for GitHub to avoid the unknown issues
dolog("forking the repo %s" % repo_str)
change_status("forking", 20)
repo_url = fork_repo(repo_str)
dolog("cloning the repo %s" % repo_url)
change_status("cloning", 30)
clone_repo(repo_url)
# fork_cleanup()
change_status("updating the fork", 40)
update_fork(repo_str) # update from upstream as the cloned repo is an old fork due to Github limitation
dolog("getting jar configurations")
change_status("getting configurations", 50)
target_files, jar_command = get_jar_config(os.path.join(get_repo_abs_path(), 'jar.cfg'))
if target_files is None or jar_command is None:
dolog("get jar config failed")
change_status("Error getting configurations", 100)
delete_local_copy()
return "get jar config failed"
else:
change_status("configuration parsed", 60)
dolog("running if target")
change_status("running if target", 70)
is_found, msg = run_if_target(changed_files, target_files, jar_command)
dolog("after running")
if is_found:
dolog("is found")
change_status("pushing changes", 80)
push_changes()
dolog("after pushing the changes")
change_status("creating pull request", 90)
if create_pull_request(repo_str):
dolog("pull request is True")
change_status("pull request created", 100)
current_repo.completed_at = datetime.now()
current_repo.save()
msg += " And pull request is created"
dolog("deleting the forked repo attempt")
else:
dolog("pull request is False")
change_status("pull failed to be created", 100)
current_repo.completed_at = datetime.now()
current_repo.save()
msg += " And pull request failed to be created"
else:
change_status("not found", 100)
dolog("not found")
current_repo.completed_at = datetime.now()
current_repo.save()
return msg
def change_status(target_repo, state):
from models import Repo
if not use_database:
return ''
try:
repo = Repo.objects.get(url=target_repo)
repo.last_used = datetime.today()
repo.state = state
repo.owner = parent_folder
repo.save()
except DoesNotExist:
repo = Repo()
repo.url = target_repo
repo.state = state
repo.owner = parent_folder
repo.save()
except Exception as e:
print 'database_exception: ' + str(e)
def change_status(target_repo, state):
from models import Repo
if not use_database:
return ''
try:
repo = Repo.objects.get(url=target_repo)
repo.last_used = datetime.today()
repo.state = state
repo.owner = parent_folder
repo.save()
except DoesNotExist:
repo = Repo()
repo.url = target_repo
repo.state = state
repo.owner = parent_folder
repo.save()
except Exception as e:
print 'database_exception: ' + str(e)
def create_new_repo(url,branch):
repo = Repo(url = url, branch = branch)
repo.save()
return repo
class Analyzer(object):
def __init__(self, username, repo_name):
self.repo = Repo(repo_path(username, repo_name))
self.seen_files = {}
try:
print "Repo exists"
self.repo_model = RepoModel.objects.get(username = username, name = repo_name)
except RepoModel.DoesNotExist:
self.repo_model = RepoModel(username = username, name = repo_name)
self.repo_model.save()
self.cached_data = {}
def is_python_file(self, diff):
if diff.a_blob and re.search('.*\.py$', diff.a_blob.name):
return True
return False
def get_classes(self, entities):
return [e for e in entities if type(e) == ast.ClassDef]
def get_functions(self, entities):
return [e for e in entities if type(e) == ast.FunctionDef]
def get_all_funcs_from_body(self, body):
funcs = self.get_functions(body)
classes = self.get_classes(body)
for c in classes:
funcs = funcs + self.get_all_funcs_from_body(c.body)
for f in funcs:
funcs = funcs + self.get_all_funcs_from_body(f.body)
return funcs
def read_diffs(self, diffs):
new_funcs = []
files_changed = []
for diff in diffs:
if diff.a_blob and diff.b_blob:
a_blob_text = diff.a_blob.data_stream.read()
b_blob_text = diff.b_blob.data_stream.read()
try:
a_syntax_tree = ast.parse(a_blob_text)
except (ValueError, SyntaxError, TypeError) as e:
#Someone has committed some crap that's not valid python,
#carry on...
continue
a_entities = a_syntax_tree.body
a_funcs = self.get_all_funcs_from_body(a_entities)
a_func_names = [f.name for f in a_funcs]
file_name = diff.a_blob.abspath + diff.a_blob.name
files_changed.append(file_name)
if not self.seen_files.get(file_name, False):
#This is a new file, so ALL functions contained within it are new
self.seen_files[file_name] = True
new_funcs = new_funcs + [(diff.a_blob.abspath, fname) for fname in a_func_names]
print "New file!"
print new_funcs
else:
#Not a new file, get what has changed, so get the next blob, parse it, and get the
#functions from it.
#Get the syntax_tree for the second blob
try:
b_syntax_tree = ast.parse(b_blob_text)
except (ValueError, SyntaxError, TypeError) as e:
#Someone has committed some crap that's not valid python,
#carry on...
continue
b_entities = b_syntax_tree.body
b_funcs = self.get_all_funcs_from_body(b_entities)
b_func_names = [f.name for f in b_funcs]
#xor the functions
new_in_this_diff = list(set(a_func_names) ^ set(b_func_names))
new_funcs = new_funcs + [(diff.a_blob.abspath, fname) for fname in new_in_this_diff]
return new_funcs, files_changed
def store(self, commit, new_funcs, files_changed):
name = commit.author.name
date = commit.committed_date
self.cached_data[commit.hexsha] = {
'name' : name,
'date' : date,
'funcs' : new_funcs,
'files_changed' : files_changed
}
if(len(self.cached_data.keys()) > 30):
with transaction.commit_on_success():
self.do_save()
def do_save(self):
for hexsha in self.cached_data:
val = self.cached_data[hexsha]
try:
actor = Actor.objects.get(full_name = val['name'])
except Actor.DoesNotExist:
actor = Actor(full_name = val['name'])
actor.save()
#Create the actor
try:
commit = Commit.objects.get(hexsha = hexsha)
except Commit.DoesNotExist:
commit = Commit(hexsha = hexsha, repo = self.repo_model, actor = actor)
commit.save()
for path, fun in val['funcs']:
if not Function.objects.filter(name = fun, path = path).exists():
fmodel = Function(name = fun, commit = commit, path = path)
fmodel.save()
print "Saved `%s` : `%s`" % (path[-16:], fun)
for file_name in val['files_changed']:
FileChange(path = file_name, actor = actor, commit = commit).save()
self.cached_data.clear()
def walk_commits(self):
#This uses a lot of memory, but...I don't see another way to go backwards
#in git python
commits = []
for c in self.repo.iter_commits():
commits.append(c)
#pop the first commit off, so that all commits in the loop will have a parent
commits.pop()
while len(commits) > 0:
commit = commits.pop()
#Create a list of diffs based on the parent (aka commit before this commit)
try:
diffs = commit.diff(commit.parents[0])
diffs = [d for d in diffs if self.is_python_file(d)]
new_funcs, files_changed = self.read_diffs(diffs)
self.store(commit, new_funcs, files_changed)
except LookupError:
#This seems to be a bug in PyGit maybe?
#seems to throw this sometimes, not much we can do here...
continue