def create_revision(self, text, user, sentiment=0, create_time=None):
from revision import Revision
rev = Revision(self, user, text)
rev.sentiment = sentiment
if create_time is not None:
rev.create_time = create_time
meta.Session.add(rev)
self.revisions.append(rev)
meta.Session.flush()
return rev
def create_revision(self, text, user, sentiment=0,
create_time=None):
from revision import Revision
rev = Revision(self, user, text)
rev.sentiment = sentiment
if create_time is not None:
rev.create_time = create_time
meta.Session.add(rev)
self.revisions.append(rev)
meta.Session.flush()
return rev
def fetch_from_json(self, json_data):
self.entity_gid = json_data['entity_gid']
self.uri = json_data['uri']
self.type = json_data['_type']
self.last_updated = json_data['last_updated']
self.default_alias = Alias.from_json(json_data['default_alias'])
if 'aliases' in json_data:
self.aliases = aliases_from_json(json_data['aliases'])
if 'relationships' in json_data:
self.relationships = \
relationships_from_json(json_data['relationships'])
if 'identifiers' in json_data:
self.identifiers = identifiers_from_json(json_data['identifiers'])
if 'annotation' in json_data:
self.annotation = Annotation.from_json(json_data['annotation'])
if 'disambiguation' in json_data:
self.disambiguation = \
Disambiguation.from_json(json_data['disambiguation'])
self.revision = Revision.from_json(json_data['revision'])
def cb(paths, revnum, props, has_children=False):
paths = paths or {}
revision_dict[revnum] = Revision(
revnum, props.get('svn:author', ''), props.get('svn:log', ''),
datetime.strptime(props['svn:date'].split('.')[0],
"%Y-%m-%dT%H:%M:%S"), paths.keys(),
self.ra_api, log_path)
def iter_revisions(self, object, reverse=False):
"""
Get object's revision
:param object:
:return: List of Revisions
"""
d = pymongo.DESCENDING if reverse else pymongo.ASCENDING
for r in self.files.find({"object": object}).sort("ts", d):
yield Revision(r["_id"], r["ts"], r["ft"])
def cb(paths, revnum, props, has_children=False):
paths = paths or {}
self.rev_list.append(Revision(revnum,
props.get('svn:author', ''),
props.get('svn:log', ''),
datetime.strptime(
props['svn:date'].split('.')[0],
"%Y-%m-%dT%H:%M:%S"),
paths.keys(),
self._ra_api,
path))
def find_revision(self, object, revision):
"""
:param object:
:param revision: Revision id
:return:
"""
r = self.files.find_one({"object": object, "_id": ObjectId(revision)})
if r:
return Revision(r["_id"], r["ts"], r["ft"])
else:
return None
def __revision_from_json(json_rev):
"""Create a Revision object from a JSON representation"""
j = json.loads(json_rev)
j = {key : unquote(value) for key, value in j.items()}
rev = int(j['rev'])
branch = j['branch']
branch = branch if branch else 'default'
jparents = j['parents']
if not jparents:
parents = [rev-1]
else:
parents = [int(p.split(':')[0]) for p in jparents.split()]
return Revision(j['node'], rev, j['author'], branch, parents, j['date'], j['tags'], j['desc'])
def generate(self, message):
revision = self.next()
message_trunc = message[:15] if len(message) > 15 else message
message_trunc = string.replace(message_trunc, ' ', '_')
destination_name = "%s_%s.py" % (revision, message_trunc)
destination_path = "%s/%s" % (self.path, destination_name)
out_stream = io.FileIO(destination_path, "wb")
self._generate(message, out_stream)
out_stream.close()
logging.info("Created new migration `%s` " % destination_path)
revision = Revision(self.path, destination_name)
self.insert(revision)
return destination_path
def last_repo_revision(self, branch = None, fetch = True):
# Use repo's upstream branch by default
branch = branch or self.get_m_branch()
if not branch:
return None
# Fetch, if needed
if fetch:
self.fetch()
# Get repo's default remote
remote = self.get_remote()
ref_remotes_pattern = "refs/remotes/%s/" % remote
remote_branch = "%s/%s" % (remote, branch)
# Go over all remote commits - from the tip all the way back to our parent
# and see if they point to a published revision
# (we could look deeper, but it is safer to stop somewhere...)
try:
parent = self.merge_base("HEAD %s" % remote_branch)
revisions = self.rev_list("%s~..%s" % (parent, remote_branch)).split()
except GitError:
# Our parent is the only revision?
pass
if not revisions:
revisions = [parent]
for revision in revisions:
try:
output = self.for_each_ref("%sv/%s/ --points-at %s --format '%%(refname)'" % (ref_remotes_pattern, branch, revision))
if output:
# Output will look like this:
#
# refs/remotes/origin/v/master/0.0.10
# refs/remotes/origin/v/master/0.0.11
# refs/remotes/origin/v/master/0.0.9
#
# Now we just need the latest, largest number
revs = [ Revision(x.replace(ref_remotes_pattern, "")) for x in output.splitlines() ]
return revision_branch_name(max(revs))
# else - go one revision earlier
except GitError:
pass
def process_revision(revision_dict, repo):
""" Takes a dictionary that has revision ids as keys. This method sets
revisions for repo"""
for revision_id in revision_dict:
revision = revision_dict[revision_id]
predecessors = []
if REVISION_PREDECESSORS in revision:
predecessors = revision[REVISION_PREDECESSORS]
properties = None
if REVISION_PROPERTIES in revision:
properties = revision[REVISION_PROPERTIES]
diffwithparent = None
if REVISION_DIFFWITHPARENT in revision:
diffwithparent = revision[REVISION_DIFFWITHPARENT]
repo.get_revisions()[revision_id] = Revision(predecessors, properties,
diffwithparent)
def first_repo_revision(self, revision, branch = None):
try:
ref_remotes_pattern = "refs/remotes/%s/" % self.get_remote()
branch_pattern = "%s/*" % branch if branch else "*"
pattern = "%sv/%s" % (ref_remotes_pattern, branch_pattern)
# git sorts alphanumerically, it will bring 0.0.10 before 0.0.9
# so we need all matching refs for ourselves to sort
output = self.for_each_ref("%s --points-at %s --format '%%(refname)'" % (pattern, revision))
# Output will look like this:
#
# refs/remotes/origin/v/master/0.0.10
# refs/remotes/origin/v/master/0.0.11
# refs/remotes/origin/v/master/0.0.9
if output:
revs = [ Revision(x.replace(ref_remotes_pattern, "")) for x in output.splitlines() ]
return revision_branch_name(min(revs))
except GitError:
pass
def collect_tag_list(project_id):
tag_list=[]
local_tag_list=[]
redundancy_tag_list = Util.getpipeoutput(['git show-ref --tags --dereference']).split('\n')
##find the right tags and hash ids
redundancy_tag_list.reverse()
for item in redundancy_tag_list:
is_have=0
for tag in local_tag_list:
(hash, tagname) = tag.split(" ")
if tagname in item:
is_have=1
break
if is_have==1:
continue
if "^{}" in item:
item=item.replace("^{}","")
local_tag_list.append(item)
for tag_item in local_tag_list:
tag=Tag()
if len(tag_item) == 0:
continue
(hash, tagname) = tag_item.split()
tag_hash_list=Tag.get_tag_list(project_id)
##don't store repeated tags
bool_repeat_tag="false"
for item in tag_hash_list:
if hash==item.tag_hash_id:
bool_repeat_tag="true"
break
if bool_repeat_tag=="true":
continue
tag_name = tagname.replace('refs/tags/', '')
output = Util.getpipeoutput(['git log "%s" --pretty=format:"%%at %%aN" -n 1' % hash])
if len(output) > 0:
parts = output.split(' ')
stamp = 0
try:
stamp = int(parts[0])
except ValueError:
stamp = 0
#It's high priority to query old revision,new revisions mostly are not tag commit
project_revision_list=Revision.get_revision_list(project_id,"")
if len(project_revision_list)>0:
for revision in project_revision_list:
if revision.revision_hash_id.strip()==hash.strip():
tag.tag_branch_id=revision.revision_branch_id
break
if tag.tag_branch_id=="":
#sometimes we can't find out the branch,maybe it's a exception
tag.tag_branch_id=0
tag.tag_project_id=project_id
tag.tag_name=tag_name
tag.tag_creater=parts[1]
tag.tag_create_date=datetime.datetime.fromtimestamp(float(stamp))
tag.tag_hash_id=hash
tag_list.append(tag)
if len(tag_list)>0:
Tag.insert_tag_list(tag_list)
def __init__(self):
self.flask = Flask(__name__, template_folder='views')
self.flask.config.from_object(ServerConfiguration)
self.flask.jinja_env.add_extension('pyjade.ext.jinja.PyJadeExtension')
add_routes(self.flask)
Revision(os.path.join(os.getcwd(), 'server', 'public', 'app', 'map.json'), '/public/app').register(self.flask)
def collect(project):
#1.Server
server_list=Server.get_server_list()
#2.Project_Insert new projects
new_project_list=Project.get_new_project_list(server_list)
if len(new_project_list) >0:
Project.insert_new_project_list(new_project_list)
#2.Project_Get all projects
serial_number=0
#Collect data by server_name
server_name=""
if len(sys.argv)>1:
server_name=sys.argv[1]
if server_name=="":
project_list=Project.get_project_list()
else:
project_list=Project.get_project_list(server_name)
for project in project_list:
serial_number=serial_number+1
print (">>>>>>No%s.Git project url: %s " %(len(project_list)-serial_number, project.project_repository_url))
print (">>>>>>0_Collecting push records")
is_have_new_push=Push.collect_push_list(project)
if is_have_new_push==0:
print (">>>>>>There is nothing new in repository \n")
continue
# clean workspace
git_home=os.getcwd()
git_path=git_home+"/"+project.project_name
if os.path.isdir(git_path):
Util.getpipeoutput(["rm -rf %s " % git_path ])
print (">>>>>>1_Git path: %s" % git_path)
print (">>>>>>2_Clone git repository")
Util.getpipeoutput(["git clone %s " % project.project_repository_url+project.project_name ])
print (">>>>>>3_Collecting git data")
if os.path.isdir(git_path):
os.chdir(git_path)
#########Begin to collect
#Collect new branchs
Branch.collect_branch_list(project.project_id)
#Query all branchs from database
all_branch_list=Branch.get_branch_list(project.project_id)
branch_list=[]
for branch in all_branch_list:
revision_list=[]
print(" >>>>>>>>Branch Name:"+branch.branch_name)
current_branch=Util.getpipeoutput(["git rev-parse --abbrev-ref HEAD"])
if current_branch!=branch.branch_name:
Util.getpipeoutput(["git checkout %s" % branch.branch_name])
# if last_commit_id is empty ,it means that it's a new branch
latest_commit_id=Util.getpipeoutput(["git rev-parse HEAD"])
if branch.branch_last_commit_id!=latest_commit_id:
#Collect all the Revisions(all commits)
branch_total_line=Revision.collect_revision_list(branch,latest_commit_id)
#Collect all the files
local_file_change_list=File.collect_file_list(branch,latest_commit_id)
#Collect all the link
Revision_File_Link.collect_link_list(local_file_change_list,branch,latest_commit_id)
#Update branch info
branch.branch_type="update"
branch.branch_total_line=branch_total_line
branch.branch_last_commit_id=latest_commit_id
branch.branch_contributor_counts = int(Util.getpipeoutput(["git shortlog -s %s" % Util.getlogrange(), "wc -l"]))
branch.branch_file_counts=int(Util.getpipeoutput(["git ls-files | wc -l"]))
branch_list.append(branch)
Branch.update_branch_list(branch_list)
Tag.collect_tag_list(project.project_id)
# Merge Request
# Project LOC
#########End
os.chdir(git_home)
print (">>>>>>4.Delete the git repository diretory\n")
if os.path.isdir(git_path):
Util.getpipeoutput(["rm -rf %s " % git_path ])
def process_xml(dump_file=None):
rev_parent_id = None
page_dict = None
in_stream = dump_file.open_dump()
for event, elem in etree.iterparse(in_stream, recover=True,
huge_tree=True):
# Drop tag namespace
tag = elem.tag.split('}')[1]
# Load namespace info
if tag == 'namespaces':
ns_dict = {int(c.attrib.get('key')): c.text for c in elem}
ns_dict[0] = ''
ns_names = {c.text: int(c.attrib.get('key')) for c in elem}
ns_names[''] = 0
# Retrieve contributor info to be embedded in current revision
# TODO: Handle contributor information properly
if tag == 'contributor':
# Build dict {tag:text} for contributor info
contrib_dict = {x.tag.split('}')[1]: x.text for x in elem}
# yield User(data_dict=contrib_dict)
if tag == 'revision':
# First revision for current page, retrieve page info
if page_dict is None:
page = elem.getparent()
# Build dict {tag:text} for all children of page
# above first revision tag
page_dict = {x.tag.split('}')[1]: x.text for x in page}
# Build dict {tag:text} for all children of revision
rev_dict = {x.tag.split('}')[1]: x.text for x in elem}
# Embed page_id, contrib_dict and return item
rev_dict['page_id'] = page_dict['id']
# To skip pattern matching for non-articles
rev_dict['ns'] = page_dict['ns']
rev_dict['contrib_dict'] = contrib_dict
rev_dict['rev_parent_id'] = rev_parent_id
rev_dict['item_type'] = 'revision'
yield Revision(rev_dict)
# Save rev_id (rev_parent_id of the next revision item)
rev_parent_id = rev_dict['id']
# Clear up revision and contributor dictionaries
rev_dict = None
contrib_dict = None
# Clear memory
elem.clear()
while elem.getprevious() is not None:
del elem.getparent()[0]
if tag == 'page':
page_dict['item_type'] = 'page'
yield Page(page_dict)
# Clear memory
page_dict = None
rev_parent_id = None
elem.clear()
while elem.getprevious() is not None:
del elem.getparent()[0]
if tag == 'logitem':
log_dict = {x.tag.split('}')[1]: x.text for x in elem}
log_dict['contrib_dict'] = contrib_dict
# Get namespace for this log item from page title prefix
if 'logtitle' in log_dict and log_dict['logtitle']:
ns_prefix = log_dict['logtitle'].split(':')
if (len(ns_prefix) == 2 and ns_prefix[0] in ns_names):
log_dict['namespace'] = ns_names[ns_prefix[0]]
else:
log_dict['namespace'] = 0
else:
log_dict['logtitle'] = ''
log_dict['namespace'] = -1000 # Fake namespace
yield LogItem(log_dict)
# Clear memory
log_dict = None
elem.clear()
while elem.getprevious() is not None:
del elem.getparent()[0]
def _get_revision_by_number(self, revision):
history = self.retrieve_history()
revision_tag = history.get_tag_by_revision(revision)
return Revision(self, revision_tag)
def _get_revision_by_tag(self, tag_name):
history = self.retrieve_history()
revision_tag = history.get_tag_by_name(tag_name)
return Revision(self, revision_tag)
