Exemplo n.º 1
0
 def create_revision(self, text, user, sentiment=0, create_time=None):
     from revision import Revision
     rev = Revision(self, user, text)
     rev.sentiment = sentiment
     if create_time is not None:
         rev.create_time = create_time
     meta.Session.add(rev)
     self.revisions.append(rev)
     meta.Session.flush()
     return rev
Exemplo n.º 2
0
 def create_revision(self, text, user, sentiment=0,
                     create_time=None):
     from revision import Revision
     rev = Revision(self, user, text)
     rev.sentiment = sentiment
     if create_time is not None:
         rev.create_time = create_time
     meta.Session.add(rev)
     self.revisions.append(rev)
     meta.Session.flush()
     return rev
Exemplo n.º 3
0
    def fetch_from_json(self, json_data):
        self.entity_gid = json_data['entity_gid']
        self.uri = json_data['uri']
        self.type = json_data['_type']

        self.last_updated = json_data['last_updated']
        self.default_alias = Alias.from_json(json_data['default_alias'])

        if 'aliases' in json_data:
            self.aliases = aliases_from_json(json_data['aliases'])

        if 'relationships' in json_data:
            self.relationships = \
                relationships_from_json(json_data['relationships'])

        if 'identifiers' in json_data:
            self.identifiers = identifiers_from_json(json_data['identifiers'])

        if 'annotation' in json_data:
            self.annotation = Annotation.from_json(json_data['annotation'])

        if 'disambiguation' in json_data:
            self.disambiguation = \
                Disambiguation.from_json(json_data['disambiguation'])

        self.revision = Revision.from_json(json_data['revision'])
Exemplo n.º 4
0
 def cb(paths, revnum, props, has_children=False):
     paths = paths or {}
     revision_dict[revnum] = Revision(
         revnum, props.get('svn:author', ''), props.get('svn:log', ''),
         datetime.strptime(props['svn:date'].split('.')[0],
                           "%Y-%m-%dT%H:%M:%S"), paths.keys(),
         self.ra_api, log_path)
Exemplo n.º 5
0
 def iter_revisions(self, object, reverse=False):
     """
     Get object's revision
     :param object:
     :return: List of Revisions
     """
     d = pymongo.DESCENDING if reverse else pymongo.ASCENDING
     for r in self.files.find({"object": object}).sort("ts", d):
         yield Revision(r["_id"], r["ts"], r["ft"])
Exemplo n.º 6
0
 def cb(paths, revnum, props, has_children=False):
     paths = paths or {}
     self.rev_list.append(Revision(revnum,
                     props.get('svn:author', ''),
                     props.get('svn:log', ''),
                     datetime.strptime(
                     props['svn:date'].split('.')[0],
                          "%Y-%m-%dT%H:%M:%S"),
                     paths.keys(),
                     self._ra_api,
                     path))
Exemplo n.º 7
0
 def find_revision(self, object, revision):
     """
     :param object:
     :param revision: Revision id
     :return:
     """
     r = self.files.find_one({"object": object, "_id": ObjectId(revision)})
     if r:
         return Revision(r["_id"], r["ts"], r["ft"])
     else:
         return None
Exemplo n.º 8
0
 def __revision_from_json(json_rev):
     """Create a Revision object from a JSON representation"""
     j = json.loads(json_rev)
     j = {key : unquote(value)   for key, value in j.items()}
     rev = int(j['rev'])
     branch = j['branch']
     branch = branch   if branch   else 'default'
     jparents = j['parents']
     if not jparents:
         parents = [rev-1]
     else:
         parents = [int(p.split(':')[0])   for p in jparents.split()]
     return Revision(j['node'], rev, j['author'], branch, parents, j['date'], j['tags'], j['desc'])
Exemplo n.º 9
0
    def generate(self, message):
        revision = self.next()

        message_trunc = message[:15] if len(message) > 15 else message
        message_trunc = string.replace(message_trunc, ' ', '_')
        destination_name = "%s_%s.py" % (revision, message_trunc)
        destination_path = "%s/%s" % (self.path, destination_name)

        out_stream = io.FileIO(destination_path, "wb")
        self._generate(message, out_stream)
        out_stream.close()
        logging.info("Created new migration `%s` " % destination_path)

        revision = Revision(self.path, destination_name)
        self.insert(revision)

        return destination_path
Exemplo n.º 10
0
    def last_repo_revision(self, branch = None, fetch = True):

        # Use repo's upstream branch by default
        branch = branch or self.get_m_branch()
        if not branch:
            return None

        # Fetch, if needed
        if fetch:
            self.fetch()

        # Get repo's default remote
        remote = self.get_remote()
        ref_remotes_pattern = "refs/remotes/%s/" % remote
        remote_branch = "%s/%s" % (remote, branch)

        # Go over all remote commits - from the tip all the way back to our parent
        # and see if they point to a published revision
        # (we could look deeper, but it is safer to stop somewhere...)
        try:
            parent = self.merge_base("HEAD %s" % remote_branch)
            revisions = self.rev_list("%s~..%s" % (parent, remote_branch)).split()
        except GitError:
            # Our parent is the only revision?
            pass
        if not revisions:
            revisions = [parent]

        for revision in revisions:
            try:
                output = self.for_each_ref("%sv/%s/ --points-at %s --format '%%(refname)'" % (ref_remotes_pattern, branch, revision))
                if output:
                    # Output will look like this:
                    #
                    # refs/remotes/origin/v/master/0.0.10
                    # refs/remotes/origin/v/master/0.0.11
                    # refs/remotes/origin/v/master/0.0.9
                    #
                    # Now we just need the latest, largest number
                    revs = [ Revision(x.replace(ref_remotes_pattern, "")) for x in output.splitlines() ]
                    return revision_branch_name(max(revs))
                # else - go one revision earlier
            except GitError:
                pass
def process_revision(revision_dict, repo):
    """ Takes a dictionary that has revision ids as keys. This method sets 
    revisions for repo"""
    for revision_id in revision_dict:
        revision = revision_dict[revision_id]

        predecessors = []
        if REVISION_PREDECESSORS in revision:
            predecessors = revision[REVISION_PREDECESSORS]

        properties = None
        if REVISION_PROPERTIES in revision:
            properties = revision[REVISION_PROPERTIES]

        diffwithparent = None
        if REVISION_DIFFWITHPARENT in revision:
            diffwithparent = revision[REVISION_DIFFWITHPARENT]

        repo.get_revisions()[revision_id] = Revision(predecessors, properties,
                                                     diffwithparent)
Exemplo n.º 12
0
    def first_repo_revision(self, revision, branch = None):

        try:
            ref_remotes_pattern = "refs/remotes/%s/" % self.get_remote()
            branch_pattern = "%s/*" % branch if branch else "*"
            pattern = "%sv/%s" % (ref_remotes_pattern, branch_pattern)

            # git sorts alphanumerically, it will bring 0.0.10 before 0.0.9
            # so we need all matching refs for ourselves to sort
            output = self.for_each_ref("%s --points-at %s --format '%%(refname)'" % (pattern, revision))
            # Output will look like this:
            #
            # refs/remotes/origin/v/master/0.0.10
            # refs/remotes/origin/v/master/0.0.11
            # refs/remotes/origin/v/master/0.0.9
            if output:
                revs = [ Revision(x.replace(ref_remotes_pattern, "")) for x in output.splitlines() ]
                return revision_branch_name(min(revs))
        except GitError:
            pass
Exemplo n.º 13
0
 def collect_tag_list(project_id):
     tag_list=[]
     local_tag_list=[]
     redundancy_tag_list = Util.getpipeoutput(['git show-ref --tags --dereference']).split('\n')
     ##find the right tags and hash ids
     redundancy_tag_list.reverse()
     for item in redundancy_tag_list:
         is_have=0
         for tag in local_tag_list:
             (hash, tagname) = tag.split(" ")
             if tagname in item:
                 is_have=1
                 break
         if is_have==1:
             continue
         if "^{}" in item:
             item=item.replace("^{}","")
         local_tag_list.append(item)
     
     for tag_item in local_tag_list:
         tag=Tag()
         if len(tag_item) == 0:
             continue
         (hash, tagname) = tag_item.split()
         tag_hash_list=Tag.get_tag_list(project_id)
         ##don't store repeated tags
         bool_repeat_tag="false"
         for item in tag_hash_list:
             if hash==item.tag_hash_id:
                 bool_repeat_tag="true"
                 break
         if bool_repeat_tag=="true":
             continue
         tag_name = tagname.replace('refs/tags/', '')
         output = Util.getpipeoutput(['git log "%s" --pretty=format:"%%at %%aN" -n 1' % hash])       
         if len(output) > 0:
             parts = output.split(' ')
             stamp = 0
             try:
                 stamp = int(parts[0])
             except ValueError:
                 stamp = 0
             #It's high priority to query old revision,new revisions mostly are not tag commit
             project_revision_list=Revision.get_revision_list(project_id,"")
             if len(project_revision_list)>0:
                 for revision in project_revision_list:
                     if revision.revision_hash_id.strip()==hash.strip():
                         tag.tag_branch_id=revision.revision_branch_id
                         break
             if tag.tag_branch_id=="":
                 #sometimes we can't find out the branch,maybe it's a exception
                 tag.tag_branch_id=0
             tag.tag_project_id=project_id
             tag.tag_name=tag_name
             tag.tag_creater=parts[1]
             tag.tag_create_date=datetime.datetime.fromtimestamp(float(stamp))
             tag.tag_hash_id=hash
             tag_list.append(tag)
     
     if len(tag_list)>0:    
         Tag.insert_tag_list(tag_list)  
Exemplo n.º 14
0
 def __init__(self):
     self.flask = Flask(__name__, template_folder='views')
     self.flask.config.from_object(ServerConfiguration)
     self.flask.jinja_env.add_extension('pyjade.ext.jinja.PyJadeExtension')
     add_routes(self.flask)
     Revision(os.path.join(os.getcwd(), 'server', 'public', 'app', 'map.json'), '/public/app').register(self.flask)
Exemplo n.º 15
0
	def collect(project):
		#1.Server
		server_list=Server.get_server_list()
		#2.Project_Insert new projects
		new_project_list=Project.get_new_project_list(server_list)
		if len(new_project_list) >0:
			Project.insert_new_project_list(new_project_list)
		#2.Project_Get all projects
		serial_number=0
		#Collect data by server_name
		server_name=""
		if len(sys.argv)>1:
			server_name=sys.argv[1]
		if server_name=="":
			project_list=Project.get_project_list()
		else:
			project_list=Project.get_project_list(server_name)	   
		for project in project_list:
			serial_number=serial_number+1
			print (">>>>>>No%s.Git project url: %s " %(len(project_list)-serial_number, project.project_repository_url))
			print (">>>>>>0_Collecting push records")
			is_have_new_push=Push.collect_push_list(project)
			if is_have_new_push==0:
				print (">>>>>>There is nothing new in repository \n")
				continue
			# clean workspace
			git_home=os.getcwd()
			git_path=git_home+"/"+project.project_name
			if os.path.isdir(git_path):
				Util.getpipeoutput(["rm -rf %s " % git_path ])   
			print (">>>>>>1_Git path: %s" % git_path)
			print (">>>>>>2_Clone git repository")
			Util.getpipeoutput(["git clone %s " % project.project_repository_url+project.project_name ])
			print (">>>>>>3_Collecting git data")
			if os.path.isdir(git_path):
				os.chdir(git_path)
				#########Begin to collect
				#Collect new branchs 
				Branch.collect_branch_list(project.project_id)		
				#Query all branchs from database
				all_branch_list=Branch.get_branch_list(project.project_id)
				branch_list=[]
				for branch in all_branch_list:
					revision_list=[]
					print("   >>>>>>>>Branch Name:"+branch.branch_name)
					current_branch=Util.getpipeoutput(["git rev-parse --abbrev-ref HEAD"])
					if current_branch!=branch.branch_name:
						Util.getpipeoutput(["git checkout %s" % branch.branch_name])
					# if last_commit_id is empty ,it means that it's a new branch
					latest_commit_id=Util.getpipeoutput(["git rev-parse HEAD"])
					if branch.branch_last_commit_id!=latest_commit_id:
						#Collect all the Revisions(all commits)
						branch_total_line=Revision.collect_revision_list(branch,latest_commit_id)
						#Collect all the files
						local_file_change_list=File.collect_file_list(branch,latest_commit_id)
						#Collect all the link
						Revision_File_Link.collect_link_list(local_file_change_list,branch,latest_commit_id)
						#Update branch info
						branch.branch_type="update"
						branch.branch_total_line=branch_total_line
						branch.branch_last_commit_id=latest_commit_id
						branch.branch_contributor_counts = int(Util.getpipeoutput(["git shortlog -s %s" % Util.getlogrange(), "wc -l"]))
						branch.branch_file_counts=int(Util.getpipeoutput(["git ls-files | wc -l"]))
						branch_list.append(branch)
				Branch.update_branch_list(branch_list)
				Tag.collect_tag_list(project.project_id)
				# Merge Request
				# Project LOC
				#########End
				os.chdir(git_home)
			print (">>>>>>4.Delete the git repository diretory\n")
			if os.path.isdir(git_path):
				Util.getpipeoutput(["rm -rf %s " % git_path ])
Exemplo n.º 16
0
def process_xml(dump_file=None):
    rev_parent_id = None
    page_dict = None

    in_stream = dump_file.open_dump()
    for event, elem in etree.iterparse(in_stream, recover=True,
                                       huge_tree=True):
        # Drop tag namespace
        tag = elem.tag.split('}')[1]

        # Load namespace info
        if tag == 'namespaces':
            ns_dict = {int(c.attrib.get('key')): c.text for c in elem}
            ns_dict[0] = ''
            ns_names = {c.text: int(c.attrib.get('key')) for c in elem}
            ns_names[''] = 0

        # Retrieve contributor info to be embedded in current revision
        # TODO: Handle contributor information properly
        if tag == 'contributor':
            # Build dict {tag:text} for contributor info
            contrib_dict = {x.tag.split('}')[1]: x.text for x in elem}
#                yield User(data_dict=contrib_dict)

        if tag == 'revision':
            # First revision for current page, retrieve page info
            if page_dict is None:
                page = elem.getparent()
                # Build dict {tag:text} for all children of page
                # above first revision tag
                page_dict = {x.tag.split('}')[1]: x.text for x in page}

            # Build dict {tag:text} for all children of revision
            rev_dict = {x.tag.split('}')[1]: x.text for x in elem}
            # Embed page_id, contrib_dict and return item
            rev_dict['page_id'] = page_dict['id']
            # To skip pattern matching for non-articles
            rev_dict['ns'] = page_dict['ns']
            rev_dict['contrib_dict'] = contrib_dict
            rev_dict['rev_parent_id'] = rev_parent_id

            rev_dict['item_type'] = 'revision'
            yield Revision(rev_dict)

            # Save rev_id (rev_parent_id of the next revision item)
            rev_parent_id = rev_dict['id']
            # Clear up revision and contributor dictionaries
            rev_dict = None
            contrib_dict = None
            # Clear memory
            elem.clear()
            while elem.getprevious() is not None:
                del elem.getparent()[0]

        if tag == 'page':
            page_dict['item_type'] = 'page'
            yield Page(page_dict)
            # Clear memory
            page_dict = None
            rev_parent_id = None
            elem.clear()
            while elem.getprevious() is not None:
                del elem.getparent()[0]

        if tag == 'logitem':
            log_dict = {x.tag.split('}')[1]: x.text for x in elem}
            log_dict['contrib_dict'] = contrib_dict
            # Get namespace for this log item from page title prefix
            if 'logtitle' in log_dict and log_dict['logtitle']:
                ns_prefix = log_dict['logtitle'].split(':')
                if (len(ns_prefix) == 2 and ns_prefix[0] in ns_names):
                    log_dict['namespace'] = ns_names[ns_prefix[0]]
                else:
                    log_dict['namespace'] = 0
            else:
                log_dict['logtitle'] = ''
                log_dict['namespace'] = -1000  # Fake namespace

            yield LogItem(log_dict)
            # Clear memory
            log_dict = None
            elem.clear()
            while elem.getprevious() is not None:
                del elem.getparent()[0]
Exemplo n.º 17
0
 def _get_revision_by_number(self, revision):
     history = self.retrieve_history()
     revision_tag = history.get_tag_by_revision(revision)
     return Revision(self, revision_tag)
Exemplo n.º 18
0
 def _get_revision_by_tag(self, tag_name):
     history = self.retrieve_history()
     revision_tag = history.get_tag_by_name(tag_name)
     return Revision(self, revision_tag)