def cb(paths, revnum, props, has_children=False):
paths = paths or {}
revision_dict[revnum] = Revision(
revnum, props.get('svn:author', ''), props.get('svn:log', ''),
datetime.strptime(props['svn:date'].split('.')[0],
"%Y-%m-%dT%H:%M:%S"), paths.keys(),
self.ra_api, log_path)
def iter_revisions(self, object, reverse=False):
"""
Get object's revision
:param object:
:return: List of Revisions
"""
d = pymongo.DESCENDING if reverse else pymongo.ASCENDING
for r in self.files.find({"object": object}).sort("ts", d):
yield Revision(r["_id"], r["ts"], r["ft"])
def create_revision(self, text, user, sentiment=0, create_time=None):
from revision import Revision
rev = Revision(self, user, text)
rev.sentiment = sentiment
if create_time is not None:
rev.create_time = create_time
meta.Session.add(rev)
self.revisions.append(rev)
meta.Session.flush()
return rev
def find_revision(self, object, revision):
"""
:param object:
:param revision: Revision id
:return:
"""
r = self.files.find_one({"object": object, "_id": ObjectId(revision)})
if r:
return Revision(r["_id"], r["ts"], r["ft"])
else:
return None
def cb(paths, revnum, props, has_children=False):
paths = paths or {}
self.rev_list.append(Revision(revnum,
props.get('svn:author', ''),
props.get('svn:log', ''),
datetime.strptime(
props['svn:date'].split('.')[0],
"%Y-%m-%dT%H:%M:%S"),
paths.keys(),
self._ra_api,
path))
def __revision_from_json(json_rev):
"""Create a Revision object from a JSON representation"""
j = json.loads(json_rev)
j = {key : unquote(value) for key, value in j.items()}
rev = int(j['rev'])
branch = j['branch']
branch = branch if branch else 'default'
jparents = j['parents']
if not jparents:
parents = [rev-1]
else:
parents = [int(p.split(':')[0]) for p in jparents.split()]
return Revision(j['node'], rev, j['author'], branch, parents, j['date'], j['tags'], j['desc'])
def generate(self, message):
revision = self.next()
message_trunc = message[:15] if len(message) > 15 else message
message_trunc = string.replace(message_trunc, ' ', '_')
destination_name = "%s_%s.py" % (revision, message_trunc)
destination_path = "%s/%s" % (self.path, destination_name)
out_stream = io.FileIO(destination_path, "wb")
self._generate(message, out_stream)
out_stream.close()
logging.info("Created new migration `%s` " % destination_path)
revision = Revision(self.path, destination_name)
self.insert(revision)
return destination_path
def last_repo_revision(self, branch = None, fetch = True):
# Use repo's upstream branch by default
branch = branch or self.get_m_branch()
if not branch:
return None
# Fetch, if needed
if fetch:
self.fetch()
# Get repo's default remote
remote = self.get_remote()
ref_remotes_pattern = "refs/remotes/%s/" % remote
remote_branch = "%s/%s" % (remote, branch)
# Go over all remote commits - from the tip all the way back to our parent
# and see if they point to a published revision
# (we could look deeper, but it is safer to stop somewhere...)
try:
parent = self.merge_base("HEAD %s" % remote_branch)
revisions = self.rev_list("%s~..%s" % (parent, remote_branch)).split()
except GitError:
# Our parent is the only revision?
pass
if not revisions:
revisions = [parent]
for revision in revisions:
try:
output = self.for_each_ref("%sv/%s/ --points-at %s --format '%%(refname)'" % (ref_remotes_pattern, branch, revision))
if output:
# Output will look like this:
#
# refs/remotes/origin/v/master/0.0.10
# refs/remotes/origin/v/master/0.0.11
# refs/remotes/origin/v/master/0.0.9
#
# Now we just need the latest, largest number
revs = [ Revision(x.replace(ref_remotes_pattern, "")) for x in output.splitlines() ]
return revision_branch_name(max(revs))
# else - go one revision earlier
except GitError:
pass
def process_revision(revision_dict, repo):
""" Takes a dictionary that has revision ids as keys. This method sets
revisions for repo"""
for revision_id in revision_dict:
revision = revision_dict[revision_id]
predecessors = []
if REVISION_PREDECESSORS in revision:
predecessors = revision[REVISION_PREDECESSORS]
properties = None
if REVISION_PROPERTIES in revision:
properties = revision[REVISION_PROPERTIES]
diffwithparent = None
if REVISION_DIFFWITHPARENT in revision:
diffwithparent = revision[REVISION_DIFFWITHPARENT]
repo.get_revisions()[revision_id] = Revision(predecessors, properties,
diffwithparent)
def first_repo_revision(self, revision, branch = None):
try:
ref_remotes_pattern = "refs/remotes/%s/" % self.get_remote()
branch_pattern = "%s/*" % branch if branch else "*"
pattern = "%sv/%s" % (ref_remotes_pattern, branch_pattern)
# git sorts alphanumerically, it will bring 0.0.10 before 0.0.9
# so we need all matching refs for ourselves to sort
output = self.for_each_ref("%s --points-at %s --format '%%(refname)'" % (pattern, revision))
# Output will look like this:
#
# refs/remotes/origin/v/master/0.0.10
# refs/remotes/origin/v/master/0.0.11
# refs/remotes/origin/v/master/0.0.9
if output:
revs = [ Revision(x.replace(ref_remotes_pattern, "")) for x in output.splitlines() ]
return revision_branch_name(min(revs))
except GitError:
pass
def _get_revision_by_tag(self, tag_name):
history = self.retrieve_history()
revision_tag = history.get_tag_by_name(tag_name)
return Revision(self, revision_tag)
def _get_revision_by_number(self, revision):
history = self.retrieve_history()
revision_tag = history.get_tag_by_revision(revision)
return Revision(self, revision_tag)
def process_xml(dump_file=None):
rev_parent_id = None
page_dict = None
in_stream = dump_file.open_dump()
for event, elem in etree.iterparse(in_stream, recover=True,
huge_tree=True):
# Drop tag namespace
tag = elem.tag.split('}')[1]
# Load namespace info
if tag == 'namespaces':
ns_dict = {int(c.attrib.get('key')): c.text for c in elem}
ns_dict[0] = ''
ns_names = {c.text: int(c.attrib.get('key')) for c in elem}
ns_names[''] = 0
# Retrieve contributor info to be embedded in current revision
# TODO: Handle contributor information properly
if tag == 'contributor':
# Build dict {tag:text} for contributor info
contrib_dict = {x.tag.split('}')[1]: x.text for x in elem}
# yield User(data_dict=contrib_dict)
if tag == 'revision':
# First revision for current page, retrieve page info
if page_dict is None:
page = elem.getparent()
# Build dict {tag:text} for all children of page
# above first revision tag
page_dict = {x.tag.split('}')[1]: x.text for x in page}
# Build dict {tag:text} for all children of revision
rev_dict = {x.tag.split('}')[1]: x.text for x in elem}
# Embed page_id, contrib_dict and return item
rev_dict['page_id'] = page_dict['id']
# To skip pattern matching for non-articles
rev_dict['ns'] = page_dict['ns']
rev_dict['contrib_dict'] = contrib_dict
rev_dict['rev_parent_id'] = rev_parent_id
rev_dict['item_type'] = 'revision'
yield Revision(rev_dict)
# Save rev_id (rev_parent_id of the next revision item)
rev_parent_id = rev_dict['id']
# Clear up revision and contributor dictionaries
rev_dict = None
contrib_dict = None
# Clear memory
elem.clear()
while elem.getprevious() is not None:
del elem.getparent()[0]
if tag == 'page':
page_dict['item_type'] = 'page'
yield Page(page_dict)
# Clear memory
page_dict = None
rev_parent_id = None
elem.clear()
while elem.getprevious() is not None:
del elem.getparent()[0]
if tag == 'logitem':
log_dict = {x.tag.split('}')[1]: x.text for x in elem}
log_dict['contrib_dict'] = contrib_dict
# Get namespace for this log item from page title prefix
if 'logtitle' in log_dict and log_dict['logtitle']:
ns_prefix = log_dict['logtitle'].split(':')
if (len(ns_prefix) == 2 and ns_prefix[0] in ns_names):
log_dict['namespace'] = ns_names[ns_prefix[0]]
else:
log_dict['namespace'] = 0
else:
log_dict['logtitle'] = ''
log_dict['namespace'] = -1000 # Fake namespace
yield LogItem(log_dict)
# Clear memory
log_dict = None
elem.clear()
while elem.getprevious() is not None:
del elem.getparent()[0]
def __init__(self):
self.flask = Flask(__name__, template_folder='views')
self.flask.config.from_object(ServerConfiguration)
self.flask.jinja_env.add_extension('pyjade.ext.jinja.PyJadeExtension')
add_routes(self.flask)
Revision(os.path.join(os.getcwd(), 'server', 'public', 'app', 'map.json'), '/public/app').register(self.flask)
