def autonosy(db, cl, nodeid, newvalues):
components = newvalues.get('components', [])
current_nosy = set()
if 'nosy' in newvalues:
# the nosy list changed
# newvalues['nosy'] contains all the user ids (new and old)
nosy = newvalues.get('nosy', [])
nosy = [value for value in nosy if db.hasnode('user', value)]
current_nosy |= set(nosy)
else:
if nodeid:
# the issue already exists
# get the values that were already in the nosy
old_nosy = db.issue.get(nodeid, 'nosy')
current_nosy |= set(old_nosy)
# make a copy of the current_nosy where to add the new user ids
new_nosy = set(current_nosy)
for component in components:
users = db.component.get(component, 'add_as_nosy')
new_nosy |= set(users)
# get the new values if they changed or the already-set ones if they didn't
if 'priority' in newvalues:
priority_id = newvalues['priority']
elif nodeid is not None:
priority_id = db.issue.get(nodeid, 'priority')
else:
priority_id = None
priority = 'None'
if priority_id is not None:
priority = db.priority.get(priority_id, 'name')
versions = []
if 'versions' in newvalues:
versions = newvalues.get('versions', [])
elif nodeid is not None:
versions = db.issue.get(nodeid, 'versions')
if priority == 'release blocker':
for version in versions:
name = db.version.get(version, 'name')
if name in RELEASE_MANAGERS:
new_nosy.add(RELEASE_MANAGERS[name])
if current_nosy != new_nosy:
# some user ids have been added automatically, so update the nosy
newvalues['nosy'] = list(new_nosy)
def fetch_files(xml_file, file_dir):
""" Fetch files referenced in the xml_file into the dir file_dir. """
root = ElementTree.parse(xml_file).getroot()
to_fetch = set()
deleted = set()
for artifact in root.find('artifacts'):
for field in artifact.findall('field'):
if field.get('name') == 'artifact_id':
aid = field.text
for field in artifact.findall('field'):
if field.get('name') != 'artifact_history': continue
for event in field.findall('history'):
d = {}
for field in event.findall('field'):
d[field.get('name')] = field.text
if d['field_name'] == 'File Added':
fid = d['old_value'].split(':')[0]
to_fetch.add((aid, fid))
if d['field_name'] == 'File Deleted':
fid = d['old_value'].split(':')[0]
deleted.add((aid, fid))
to_fetch = to_fetch - deleted
got = set(os.listdir(file_dir))
to_fetch = to_fetch - got
# load cached urls (sigh)
urls = {}
if os.path.exists(os.path.join(file_dir, 'urls.txt')):
for line in open(os.path.join(file_dir, 'urls.txt')):
aid, url = line.strip().split()
urls[aid] = url
for aid, fid in support.Progress('Fetching files', list(to_fetch)):
if fid in got: continue
if not urls.has_key(aid):
urls[aid] = get_url(aid)
f = open(os.path.join(file_dir, 'urls.txt'), 'a')
f.write('%s %s\n'%(aid, urls[aid]))
f.close()
url = urls[aid] + '&file_id=' + fid
f = urllib2.urlopen(url)
data = f.read()
n = open(os.path.join(file_dir, fid), 'w')
n.write(data)
f.close()
n.close()
def __init__(self, db):
self.stopwords = set(STOPWORDS)
for word in db.config[('main', 'indexer_stopwords')]:
self.stopwords.add(word)
# Do not index anything longer than 25 characters since that'll be
# gibberish (encoded text or somesuch) or shorter than 2 characters
self.minlength = 2
self.maxlength = 25
def _set_val(self, val):
"""Check if self._val is already defined. If yes, we compute the
intersection of the old and the new value(s)
"""
if self.has_values:
v = self._val
if not isinstance(self._val, type([])):
v = [self._val]
vals = set(v)
vals.intersection_update(val)
self._val = [v for v in vals]
else:
self._val = val
self.has_values = True
def add_text(self, identifier, text, mime_type='text/plain'):
""" "identifier" is (classname, itemid, property) """
if mime_type != 'text/plain':
return
# Ensure all elements of the identifier are strings 'cos the itemid
# column is varchar even if item ids may be numbers elsewhere in the
# code. ugh.
identifier = tuple(map(str, identifier))
# first, find the id of the (classname, itemid, property)
a = self.db.arg
sql = 'select _textid from __textids where _class=%s and '\
'_itemid=%s and _prop=%s'%(a, a, a)
self.db.cursor.execute(sql, identifier)
r = self.db.cursor.fetchone()
if not r:
# not previously indexed
id = self.db.newid('__textids')
sql = 'insert into __textids (_textid, _class, _itemid, _prop)'\
' values (%s, %s, %s, %s)'%(a, a, a, a)
self.db.cursor.execute(sql, (id, ) + identifier)
else:
id = int(r[0])
# clear out any existing indexed values
sql = 'delete from __words where _textid=%s'%a
self.db.cursor.execute(sql, (id, ))
# ok, find all the unique words in the text
if not isinstance(text, unicode):
text = unicode(text, "utf-8", "replace")
text = text.upper()
wordlist = [w.encode("utf-8")
for w in re.findall(r'(?u)\b\w{%d,%d}\b'
% (self.minlength, self.maxlength), text)]
words = set()
for word in wordlist:
if self.is_stopword(word): continue
words.add(word)
# for each word, add an entry in the db
sql = 'insert into __words (_word, _textid) values (%s, %s)'%(a, a)
words = [(word, id) for word in words]
self.db.cursor.executemany(sql, words)
def updatenosy(db, cl, nodeid, newvalues):
'''Update the nosy list for changes to the assignedto
'''
# nodeid will be None if this is a new node
current_nosy = set()
if nodeid is None:
ok = ('new', 'yes')
else:
ok = ('yes',)
# old node, get the current values from the node if they haven't
# changed
if not newvalues.has_key('nosy'):
nosy = cl.get(nodeid, 'nosy')
for value in nosy:
current_nosy.add(value)
# if the nosy list changed in this transaction, init from the new value
if newvalues.has_key('nosy'):
nosy = newvalues.get('nosy', [])
for value in nosy:
if not db.hasnode('user', value):
continue
current_nosy.add(value)
new_nosy = set(current_nosy)
# add assignedto(s) to the nosy list
if newvalues.has_key('assignedto') and newvalues['assignedto'] is not None:
propdef = cl.getprops()
if isinstance(propdef['assignedto'], hyperdb.Link):
assignedto_ids = [newvalues['assignedto']]
elif isinstance(propdef['assignedto'], hyperdb.Multilink):
assignedto_ids = newvalues['assignedto']
for assignedto_id in assignedto_ids:
new_nosy.add(assignedto_id)
# see if there's any new messages - if so, possibly add the author and
# recipient to the nosy
if newvalues.has_key('messages'):
if nodeid is None:
ok = ('new', 'yes')
messages = newvalues['messages']
else:
ok = ('yes',)
# figure which of the messages now on the issue weren't
oldmessages = cl.get(nodeid, 'messages')
messages = []
for msgid in newvalues['messages']:
if msgid not in oldmessages:
messages.append(msgid)
# configs for nosy modifications
add_author = getattr(db.config, 'ADD_AUTHOR_TO_NOSY', 'new')
add_recips = getattr(db.config, 'ADD_RECIPIENTS_TO_NOSY', 'new')
# now for each new message:
msg = db.msg
for msgid in messages:
if add_author in ok:
authid = msg.get(msgid, 'author')
new_nosy.add(authid)
# add on the recipients of the message
if add_recips in ok:
for recipient in msg.get(msgid, 'recipients'):
new_nosy.add(recipient)
if current_nosy != new_nosy:
# that's it, save off the new nosy list
newvalues['nosy'] = list(new_nosy)
def import_xml(tracker_home, xml_file, file_dir):
""" Generate Roundup tracker import files based on the tracker schema,
sf.net xml export and downloaded files from sf.net. """
tracker = instance.open(tracker_home)
db = tracker.open('admin')
resolved = db.status.lookup('resolved')
unread = db.status.lookup('unread')
chatting = db.status.lookup('unread')
critical = db.priority.lookup('critical')
urgent = db.priority.lookup('urgent')
bug = db.priority.lookup('bug')
feature = db.priority.lookup('feature')
wish = db.priority.lookup('wish')
adminuid = db.user.lookup('admin')
anonuid = db.user.lookup('anonymous')
root = ElementTree.parse(xml_file).getroot()
def to_date(ts):
return date.Date(time.gmtime(float(ts)))
# parse out the XML
artifacts = []
categories = set()
users = set()
add_files = set()
remove_files = set()
for artifact in root.find('artifacts'):
d = {}
op = {}
artifacts.append(d)
for field in artifact.findall('field'):
name = field.get('name')
if name == 'artifact_messages':
for message in field.findall('message'):
l = d.setdefault('messages', [])
m = {}
l.append(m)
for field in message.findall('field'):
name = field.get('name')
if name == 'adddate':
m[name] = to_date(field.text)
else:
m[name] = field.text
if name == 'user_name': users.add(field.text)
elif name == 'artifact_history':
for event in field.findall('history'):
l = d.setdefault('history', [])
e = {}
l.append(e)
for field in event.findall('field'):
name = field.get('name')
if name == 'entrydate':
e[name] = to_date(field.text)
else:
e[name] = field.text
if name == 'mod_by': users.add(field.text)
if e['field_name'] == 'File Added':
add_files.add(e['old_value'].split(':')[0])
elif e['field_name'] == 'File Deleted':
remove_files.add(e['old_value'].split(':')[0])
elif name == 'details':
op['body'] = field.text
elif name == 'submitted_by':
op['user_name'] = field.text
d[name] = field.text
users.add(field.text)
elif name == 'open_date':
thedate = to_date(field.text)
op['adddate'] = thedate
d[name] = thedate
else:
d[name] = field.text
categories.add(d['category'])
if op.has_key('body'):
l = d.setdefault('messages', [])
l.insert(0, op)
add_files -= remove_files
# create users
userd = {'nobody': '2'}
users.remove('nobody')
data = [
{'id': '1', 'username': '******', 'password': password.Password('admin'),
'roles': 'Admin', 'address': '*****@*****.**'},
{'id': '2', 'username': '******', 'roles': 'Anonymous'},
]
for n, user in enumerate(list(users)):
userd[user] = n+3
data.append({'id': str(n+3), 'username': user, 'roles': 'User',
'address': '*****@*****.**'%user})
write_csv(db.user, data)
users=userd
# create categories
categoryd = {'None': None}
categories.remove('None')
data = []
for n, category in enumerate(list(categories)):
categoryd[category] = n
data.append({'id': str(n), 'name': category})
write_csv(db.keyword, data)
categories = categoryd
# create issues
issue_data = []
file_data = []
message_data = []
issue_journal = []
message_id = 0
for artifact in artifacts:
d = {}
d['id'] = artifact['artifact_id']
d['title'] = artifact['summary']
d['assignedto'] = users[artifact['assigned_to']]
if d['assignedto'] == '2':
d['assignedto'] = None
d['creation'] = artifact['open_date']
activity = artifact['open_date']
d['creator'] = users[artifact['submitted_by']]
actor = d['creator']
if categories[artifact['category']]:
d['keyword'] = [categories[artifact['category']]]
issue_journal.append((
d['id'], d['creation'].get_tuple(), d['creator'], "'create'", {}
))
p = int(artifact['priority'])
if artifact['artifact_type'] == 'Feature Requests':
if p > 3:
d['priority'] = feature
else:
d['priority'] = wish
else:
if p > 7:
d['priority'] = critical
elif p > 5:
d['priority'] = urgent
elif p > 3:
d['priority'] = bug
else:
d['priority'] = feature
s = artifact['status']
if s == 'Closed':
d['status'] = resolved
elif s == 'Deleted':
d['status'] = resolved
d['is retired'] = True
else:
d['status'] = unread
nosy = set()
for message in artifact.get('messages', []):
authid = users[message['user_name']]
if not message['body']: continue
body = convert_message(message['body'], message_id)
if not body: continue
m = {'content': body, 'author': authid,
'date': message['adddate'],
'creation': message['adddate'], }
message_data.append(m)
if authid not in (None, '2'):
nosy.add(authid)
activity = message['adddate']
actor = authid
if d['status'] == unread:
d['status'] = chatting
# add import message
m = {'content': 'IMPORT FROM SOURCEFORGE', 'author': '1',
'date': today, 'creation': today}
message_data.append(m)
# sort messages and assign ids
d['messages'] = []
message_data.sort(lambda a,b:cmp(a['date'],b['date']))
for message in message_data:
message_id += 1
message['id'] = str(message_id)
d['messages'].append(message_id)
d['nosy'] = list(nosy)
files = []
for event in artifact.get('history', []):
if event['field_name'] == 'File Added':
fid, name = event['old_value'].split(':', 1)
if fid in add_files:
files.append(fid)
name = name.strip()
try:
f = open(os.path.join(file_dir, fid))
content = f.read()
f.close()
except:
content = 'content missing'
file_data.append({
'id': fid,
'creation': event['entrydate'],
'creator': users[event['mod_by']],
'name': name,
'type': mimetypes.guess_type(name)[0],
'content': content,
})
continue
elif event['field_name'] == 'close_date':
action = "'set'"
info = { 'status': unread }
elif event['field_name'] == 'summary':
action = "'set'"
info = { 'title': event['old_value'] }
else:
# not an interesting / translatable event
continue
row = [ d['id'], event['entrydate'].get_tuple(),
users[event['mod_by']], action, info ]
if event['entrydate'] > activity:
activity = event['entrydate']
issue_journal.append(row)
d['files'] = files
d['activity'] = activity
d['actor'] = actor
issue_data.append(d)
write_csv(db.issue, issue_data)
write_csv(db.msg, message_data)
write_csv(db.file, file_data)
f = open('/tmp/imported/issue-journals.csv', 'w')
writer = csv.writer(f, colon_separated)
writer.writerows(issue_journal)
f.close()
def __init__(self, db):
self.stopwords = set(STOPWORDS)
for word in db.config[('main', 'indexer_stopwords')]:
self.stopwords.add(word)
def updatenosy(db, cl, nodeid, newvalues):
"""Update the nosy list for changes to the assignedto
"""
# nodeid will be None if this is a new node
current_nosy = set()
if nodeid is None:
ok = ("new", "yes")
else:
ok = ("yes",)
# old node, get the current values from the node if they haven't
# changed
if not newvalues.has_key("nosy"):
nosy = cl.get(nodeid, "nosy")
for value in nosy:
current_nosy.add(value)
# if the nosy list changed in this transaction, init from the new value
if newvalues.has_key("nosy"):
nosy = newvalues.get("nosy", [])
for value in nosy:
if not db.hasnode("user", value):
continue
current_nosy.add(value)
new_nosy = set(current_nosy)
# add assignedto(s) to the nosy list
if newvalues.has_key("assignedto") and newvalues["assignedto"] is not None:
propdef = cl.getprops()
if isinstance(propdef["assignedto"], hyperdb.Link):
assignedto_ids = [newvalues["assignedto"]]
elif isinstance(propdef["assignedto"], hyperdb.Multilink):
assignedto_ids = newvalues["assignedto"]
for assignedto_id in assignedto_ids:
new_nosy.add(assignedto_id)
# see if there's any new messages - if so, possibly add the author and
# recipient to the nosy
if newvalues.has_key("messages"):
if nodeid is None:
ok = ("new", "yes")
messages = newvalues["messages"]
else:
ok = ("yes",)
# figure which of the messages now on the issue weren't
oldmessages = cl.get(nodeid, "messages")
messages = []
for msgid in newvalues["messages"]:
if msgid not in oldmessages:
messages.append(msgid)
# configs for nosy modifications
add_author = getattr(db.config, "ADD_AUTHOR_TO_NOSY", "new")
add_recips = getattr(db.config, "ADD_RECIPIENTS_TO_NOSY", "new")
# now for each new message:
msg = db.msg
for msgid in messages:
if add_author in ok:
authid = msg.get(msgid, "author")
new_nosy.add(authid)
# add on the recipients of the message
if add_recips in ok:
for recipient in msg.get(msgid, "recipients"):
new_nosy.add(recipient)
if current_nosy != new_nosy:
# that's it, save off the new nosy list
newvalues["nosy"] = list(new_nosy)
def updatenosy(db, cl, nodeid, newvalues):
'''Update the nosy list for changes to the assignee
'''
# nodeid will be None if this is a new node
current_nosy = set()
if nodeid is None:
ok = ('new', 'yes')
else:
ok = ('yes', )
# old node, get the current values from the node if they haven't
# changed
if not newvalues.has_key('nosy'):
nosy = cl.get(nodeid, 'nosy')
for value in nosy:
current_nosy.add(value)
# if the nosy list changed in this transaction, init from the new value
if newvalues.has_key('nosy'):
nosy = newvalues.get('nosy', [])
for value in nosy:
if not db.hasnode('user', value):
continue
current_nosy.add(value)
new_nosy = set(current_nosy)
# add assignee(s) to the nosy list
if newvalues.has_key('assignee') and newvalues['assignee'] is not None:
propdef = cl.getprops()
if isinstance(propdef['assignee'], hyperdb.Link):
assignee_ids = [newvalues['assignee']]
elif isinstance(propdef['assignee'], hyperdb.Multilink):
assignee_ids = newvalues['assignee']
for assignee_id in assignee_ids:
new_nosy.add(assignee_id)
# see if there's any new messages - if so, possibly add the author and
# recipient to the nosy
if newvalues.has_key('messages'):
if nodeid is None:
ok = ('new', 'yes')
messages = newvalues['messages']
else:
ok = ('yes', )
# figure which of the messages now on the issue weren't
oldmessages = cl.get(nodeid, 'messages')
messages = []
for msgid in newvalues['messages']:
if msgid not in oldmessages:
messages.append(msgid)
# configs for nosy modifications
add_author = getattr(db.config, 'ADD_AUTHOR_TO_NOSY', 'new')
add_recips = getattr(db.config, 'ADD_RECIPIENTS_TO_NOSY', 'new')
# now for each new message:
msg = db.msg
for msgid in messages:
if add_author in ok:
authid = msg.get(msgid, 'author')
new_nosy.add(authid)
# add on the recipients of the message
if add_recips in ok:
for recipient in msg.get(msgid, 'recipients'):
new_nosy.add(recipient)
if current_nosy != new_nosy:
# that's it, save off the new nosy list
newvalues['nosy'] = list(new_nosy)
