def create_revision(self, text, user, sentiment=0, create_time=None): from revision import Revision rev = Revision(self, user, text) rev.sentiment = sentiment if create_time is not None: rev.create_time = create_time meta.Session.add(rev) self.revisions.append(rev) meta.Session.flush() return rev
def fetch_from_json(self, json_data): self.entity_gid = json_data['entity_gid'] self.uri = json_data['uri'] self.type = json_data['_type'] self.last_updated = json_data['last_updated'] self.default_alias = Alias.from_json(json_data['default_alias']) if 'aliases' in json_data: self.aliases = aliases_from_json(json_data['aliases']) if 'relationships' in json_data: self.relationships = \ relationships_from_json(json_data['relationships']) if 'identifiers' in json_data: self.identifiers = identifiers_from_json(json_data['identifiers']) if 'annotation' in json_data: self.annotation = Annotation.from_json(json_data['annotation']) if 'disambiguation' in json_data: self.disambiguation = \ Disambiguation.from_json(json_data['disambiguation']) self.revision = Revision.from_json(json_data['revision'])
def cb(paths, revnum, props, has_children=False): paths = paths or {} revision_dict[revnum] = Revision( revnum, props.get('svn:author', ''), props.get('svn:log', ''), datetime.strptime(props['svn:date'].split('.')[0], "%Y-%m-%dT%H:%M:%S"), paths.keys(), self.ra_api, log_path)
def iter_revisions(self, object, reverse=False): """ Get object's revision :param object: :return: List of Revisions """ d = pymongo.DESCENDING if reverse else pymongo.ASCENDING for r in self.files.find({"object": object}).sort("ts", d): yield Revision(r["_id"], r["ts"], r["ft"])
def cb(paths, revnum, props, has_children=False): paths = paths or {} self.rev_list.append(Revision(revnum, props.get('svn:author', ''), props.get('svn:log', ''), datetime.strptime( props['svn:date'].split('.')[0], "%Y-%m-%dT%H:%M:%S"), paths.keys(), self._ra_api, path))
def find_revision(self, object, revision): """ :param object: :param revision: Revision id :return: """ r = self.files.find_one({"object": object, "_id": ObjectId(revision)}) if r: return Revision(r["_id"], r["ts"], r["ft"]) else: return None
def __revision_from_json(json_rev): """Create a Revision object from a JSON representation""" j = json.loads(json_rev) j = {key : unquote(value) for key, value in j.items()} rev = int(j['rev']) branch = j['branch'] branch = branch if branch else 'default' jparents = j['parents'] if not jparents: parents = [rev-1] else: parents = [int(p.split(':')[0]) for p in jparents.split()] return Revision(j['node'], rev, j['author'], branch, parents, j['date'], j['tags'], j['desc'])
def generate(self, message): revision = self.next() message_trunc = message[:15] if len(message) > 15 else message message_trunc = string.replace(message_trunc, ' ', '_') destination_name = "%s_%s.py" % (revision, message_trunc) destination_path = "%s/%s" % (self.path, destination_name) out_stream = io.FileIO(destination_path, "wb") self._generate(message, out_stream) out_stream.close() logging.info("Created new migration `%s` " % destination_path) revision = Revision(self.path, destination_name) self.insert(revision) return destination_path
def last_repo_revision(self, branch = None, fetch = True): # Use repo's upstream branch by default branch = branch or self.get_m_branch() if not branch: return None # Fetch, if needed if fetch: self.fetch() # Get repo's default remote remote = self.get_remote() ref_remotes_pattern = "refs/remotes/%s/" % remote remote_branch = "%s/%s" % (remote, branch) # Go over all remote commits - from the tip all the way back to our parent # and see if they point to a published revision # (we could look deeper, but it is safer to stop somewhere...) try: parent = self.merge_base("HEAD %s" % remote_branch) revisions = self.rev_list("%s~..%s" % (parent, remote_branch)).split() except GitError: # Our parent is the only revision? pass if not revisions: revisions = [parent] for revision in revisions: try: output = self.for_each_ref("%sv/%s/ --points-at %s --format '%%(refname)'" % (ref_remotes_pattern, branch, revision)) if output: # Output will look like this: # # refs/remotes/origin/v/master/0.0.10 # refs/remotes/origin/v/master/0.0.11 # refs/remotes/origin/v/master/0.0.9 # # Now we just need the latest, largest number revs = [ Revision(x.replace(ref_remotes_pattern, "")) for x in output.splitlines() ] return revision_branch_name(max(revs)) # else - go one revision earlier except GitError: pass
def process_revision(revision_dict, repo): """ Takes a dictionary that has revision ids as keys. This method sets revisions for repo""" for revision_id in revision_dict: revision = revision_dict[revision_id] predecessors = [] if REVISION_PREDECESSORS in revision: predecessors = revision[REVISION_PREDECESSORS] properties = None if REVISION_PROPERTIES in revision: properties = revision[REVISION_PROPERTIES] diffwithparent = None if REVISION_DIFFWITHPARENT in revision: diffwithparent = revision[REVISION_DIFFWITHPARENT] repo.get_revisions()[revision_id] = Revision(predecessors, properties, diffwithparent)
def first_repo_revision(self, revision, branch = None): try: ref_remotes_pattern = "refs/remotes/%s/" % self.get_remote() branch_pattern = "%s/*" % branch if branch else "*" pattern = "%sv/%s" % (ref_remotes_pattern, branch_pattern) # git sorts alphanumerically, it will bring 0.0.10 before 0.0.9 # so we need all matching refs for ourselves to sort output = self.for_each_ref("%s --points-at %s --format '%%(refname)'" % (pattern, revision)) # Output will look like this: # # refs/remotes/origin/v/master/0.0.10 # refs/remotes/origin/v/master/0.0.11 # refs/remotes/origin/v/master/0.0.9 if output: revs = [ Revision(x.replace(ref_remotes_pattern, "")) for x in output.splitlines() ] return revision_branch_name(min(revs)) except GitError: pass
def collect_tag_list(project_id): tag_list=[] local_tag_list=[] redundancy_tag_list = Util.getpipeoutput(['git show-ref --tags --dereference']).split('\n') ##find the right tags and hash ids redundancy_tag_list.reverse() for item in redundancy_tag_list: is_have=0 for tag in local_tag_list: (hash, tagname) = tag.split(" ") if tagname in item: is_have=1 break if is_have==1: continue if "^{}" in item: item=item.replace("^{}","") local_tag_list.append(item) for tag_item in local_tag_list: tag=Tag() if len(tag_item) == 0: continue (hash, tagname) = tag_item.split() tag_hash_list=Tag.get_tag_list(project_id) ##don't store repeated tags bool_repeat_tag="false" for item in tag_hash_list: if hash==item.tag_hash_id: bool_repeat_tag="true" break if bool_repeat_tag=="true": continue tag_name = tagname.replace('refs/tags/', '') output = Util.getpipeoutput(['git log "%s" --pretty=format:"%%at %%aN" -n 1' % hash]) if len(output) > 0: parts = output.split(' ') stamp = 0 try: stamp = int(parts[0]) except ValueError: stamp = 0 #It's high priority to query old revision,new revisions mostly are not tag commit project_revision_list=Revision.get_revision_list(project_id,"") if len(project_revision_list)>0: for revision in project_revision_list: if revision.revision_hash_id.strip()==hash.strip(): tag.tag_branch_id=revision.revision_branch_id break if tag.tag_branch_id=="": #sometimes we can't find out the branch,maybe it's a exception tag.tag_branch_id=0 tag.tag_project_id=project_id tag.tag_name=tag_name tag.tag_creater=parts[1] tag.tag_create_date=datetime.datetime.fromtimestamp(float(stamp)) tag.tag_hash_id=hash tag_list.append(tag) if len(tag_list)>0: Tag.insert_tag_list(tag_list)
def __init__(self): self.flask = Flask(__name__, template_folder='views') self.flask.config.from_object(ServerConfiguration) self.flask.jinja_env.add_extension('pyjade.ext.jinja.PyJadeExtension') add_routes(self.flask) Revision(os.path.join(os.getcwd(), 'server', 'public', 'app', 'map.json'), '/public/app').register(self.flask)
def collect(project): #1.Server server_list=Server.get_server_list() #2.Project_Insert new projects new_project_list=Project.get_new_project_list(server_list) if len(new_project_list) >0: Project.insert_new_project_list(new_project_list) #2.Project_Get all projects serial_number=0 #Collect data by server_name server_name="" if len(sys.argv)>1: server_name=sys.argv[1] if server_name=="": project_list=Project.get_project_list() else: project_list=Project.get_project_list(server_name) for project in project_list: serial_number=serial_number+1 print (">>>>>>No%s.Git project url: %s " %(len(project_list)-serial_number, project.project_repository_url)) print (">>>>>>0_Collecting push records") is_have_new_push=Push.collect_push_list(project) if is_have_new_push==0: print (">>>>>>There is nothing new in repository \n") continue # clean workspace git_home=os.getcwd() git_path=git_home+"/"+project.project_name if os.path.isdir(git_path): Util.getpipeoutput(["rm -rf %s " % git_path ]) print (">>>>>>1_Git path: %s" % git_path) print (">>>>>>2_Clone git repository") Util.getpipeoutput(["git clone %s " % project.project_repository_url+project.project_name ]) print (">>>>>>3_Collecting git data") if os.path.isdir(git_path): os.chdir(git_path) #########Begin to collect #Collect new branchs Branch.collect_branch_list(project.project_id) #Query all branchs from database all_branch_list=Branch.get_branch_list(project.project_id) branch_list=[] for branch in all_branch_list: revision_list=[] print(" >>>>>>>>Branch Name:"+branch.branch_name) current_branch=Util.getpipeoutput(["git rev-parse --abbrev-ref HEAD"]) if current_branch!=branch.branch_name: Util.getpipeoutput(["git checkout %s" % branch.branch_name]) # if last_commit_id is empty ,it means that it's a new branch latest_commit_id=Util.getpipeoutput(["git rev-parse HEAD"]) if branch.branch_last_commit_id!=latest_commit_id: #Collect all the Revisions(all commits) branch_total_line=Revision.collect_revision_list(branch,latest_commit_id) #Collect all the files local_file_change_list=File.collect_file_list(branch,latest_commit_id) #Collect all the link Revision_File_Link.collect_link_list(local_file_change_list,branch,latest_commit_id) #Update branch info branch.branch_type="update" branch.branch_total_line=branch_total_line branch.branch_last_commit_id=latest_commit_id branch.branch_contributor_counts = int(Util.getpipeoutput(["git shortlog -s %s" % Util.getlogrange(), "wc -l"])) branch.branch_file_counts=int(Util.getpipeoutput(["git ls-files | wc -l"])) branch_list.append(branch) Branch.update_branch_list(branch_list) Tag.collect_tag_list(project.project_id) # Merge Request # Project LOC #########End os.chdir(git_home) print (">>>>>>4.Delete the git repository diretory\n") if os.path.isdir(git_path): Util.getpipeoutput(["rm -rf %s " % git_path ])
def process_xml(dump_file=None): rev_parent_id = None page_dict = None in_stream = dump_file.open_dump() for event, elem in etree.iterparse(in_stream, recover=True, huge_tree=True): # Drop tag namespace tag = elem.tag.split('}')[1] # Load namespace info if tag == 'namespaces': ns_dict = {int(c.attrib.get('key')): c.text for c in elem} ns_dict[0] = '' ns_names = {c.text: int(c.attrib.get('key')) for c in elem} ns_names[''] = 0 # Retrieve contributor info to be embedded in current revision # TODO: Handle contributor information properly if tag == 'contributor': # Build dict {tag:text} for contributor info contrib_dict = {x.tag.split('}')[1]: x.text for x in elem} # yield User(data_dict=contrib_dict) if tag == 'revision': # First revision for current page, retrieve page info if page_dict is None: page = elem.getparent() # Build dict {tag:text} for all children of page # above first revision tag page_dict = {x.tag.split('}')[1]: x.text for x in page} # Build dict {tag:text} for all children of revision rev_dict = {x.tag.split('}')[1]: x.text for x in elem} # Embed page_id, contrib_dict and return item rev_dict['page_id'] = page_dict['id'] # To skip pattern matching for non-articles rev_dict['ns'] = page_dict['ns'] rev_dict['contrib_dict'] = contrib_dict rev_dict['rev_parent_id'] = rev_parent_id rev_dict['item_type'] = 'revision' yield Revision(rev_dict) # Save rev_id (rev_parent_id of the next revision item) rev_parent_id = rev_dict['id'] # Clear up revision and contributor dictionaries rev_dict = None contrib_dict = None # Clear memory elem.clear() while elem.getprevious() is not None: del elem.getparent()[0] if tag == 'page': page_dict['item_type'] = 'page' yield Page(page_dict) # Clear memory page_dict = None rev_parent_id = None elem.clear() while elem.getprevious() is not None: del elem.getparent()[0] if tag == 'logitem': log_dict = {x.tag.split('}')[1]: x.text for x in elem} log_dict['contrib_dict'] = contrib_dict # Get namespace for this log item from page title prefix if 'logtitle' in log_dict and log_dict['logtitle']: ns_prefix = log_dict['logtitle'].split(':') if (len(ns_prefix) == 2 and ns_prefix[0] in ns_names): log_dict['namespace'] = ns_names[ns_prefix[0]] else: log_dict['namespace'] = 0 else: log_dict['logtitle'] = '' log_dict['namespace'] = -1000 # Fake namespace yield LogItem(log_dict) # Clear memory log_dict = None elem.clear() while elem.getprevious() is not None: del elem.getparent()[0]
def _get_revision_by_number(self, revision): history = self.retrieve_history() revision_tag = history.get_tag_by_revision(revision) return Revision(self, revision_tag)
def _get_revision_by_tag(self, tag_name): history = self.retrieve_history() revision_tag = history.get_tag_by_name(tag_name) return Revision(self, revision_tag)