def autonosy(db, cl, nodeid, newvalues): components = newvalues.get('components', []) current_nosy = set() if 'nosy' in newvalues: # the nosy list changed # newvalues['nosy'] contains all the user ids (new and old) nosy = newvalues.get('nosy', []) nosy = [value for value in nosy if db.hasnode('user', value)] current_nosy |= set(nosy) else: if nodeid: # the issue already exists # get the values that were already in the nosy old_nosy = db.issue.get(nodeid, 'nosy') current_nosy |= set(old_nosy) # make a copy of the current_nosy where to add the new user ids new_nosy = set(current_nosy) for component in components: users = db.component.get(component, 'add_as_nosy') new_nosy |= set(users) # get the new values if they changed or the already-set ones if they didn't if 'priority' in newvalues: priority_id = newvalues['priority'] elif nodeid is not None: priority_id = db.issue.get(nodeid, 'priority') else: priority_id = None priority = 'None' if priority_id is not None: priority = db.priority.get(priority_id, 'name') versions = [] if 'versions' in newvalues: versions = newvalues.get('versions', []) elif nodeid is not None: versions = db.issue.get(nodeid, 'versions') if priority == 'release blocker': for version in versions: name = db.version.get(version, 'name') if name in RELEASE_MANAGERS: new_nosy.add(RELEASE_MANAGERS[name]) if current_nosy != new_nosy: # some user ids have been added automatically, so update the nosy newvalues['nosy'] = list(new_nosy)
def fetch_files(xml_file, file_dir): """ Fetch files referenced in the xml_file into the dir file_dir. """ root = ElementTree.parse(xml_file).getroot() to_fetch = set() deleted = set() for artifact in root.find('artifacts'): for field in artifact.findall('field'): if field.get('name') == 'artifact_id': aid = field.text for field in artifact.findall('field'): if field.get('name') != 'artifact_history': continue for event in field.findall('history'): d = {} for field in event.findall('field'): d[field.get('name')] = field.text if d['field_name'] == 'File Added': fid = d['old_value'].split(':')[0] to_fetch.add((aid, fid)) if d['field_name'] == 'File Deleted': fid = d['old_value'].split(':')[0] deleted.add((aid, fid)) to_fetch = to_fetch - deleted got = set(os.listdir(file_dir)) to_fetch = to_fetch - got # load cached urls (sigh) urls = {} if os.path.exists(os.path.join(file_dir, 'urls.txt')): for line in open(os.path.join(file_dir, 'urls.txt')): aid, url = line.strip().split() urls[aid] = url for aid, fid in support.Progress('Fetching files', list(to_fetch)): if fid in got: continue if not urls.has_key(aid): urls[aid] = get_url(aid) f = open(os.path.join(file_dir, 'urls.txt'), 'a') f.write('%s %s\n'%(aid, urls[aid])) f.close() url = urls[aid] + '&file_id=' + fid f = urllib2.urlopen(url) data = f.read() n = open(os.path.join(file_dir, fid), 'w') n.write(data) f.close() n.close()
def __init__(self, db): self.stopwords = set(STOPWORDS) for word in db.config[('main', 'indexer_stopwords')]: self.stopwords.add(word) # Do not index anything longer than 25 characters since that'll be # gibberish (encoded text or somesuch) or shorter than 2 characters self.minlength = 2 self.maxlength = 25
def _set_val(self, val): """Check if self._val is already defined. If yes, we compute the intersection of the old and the new value(s) """ if self.has_values: v = self._val if not isinstance(self._val, type([])): v = [self._val] vals = set(v) vals.intersection_update(val) self._val = [v for v in vals] else: self._val = val self.has_values = True
def add_text(self, identifier, text, mime_type='text/plain'): """ "identifier" is (classname, itemid, property) """ if mime_type != 'text/plain': return # Ensure all elements of the identifier are strings 'cos the itemid # column is varchar even if item ids may be numbers elsewhere in the # code. ugh. identifier = tuple(map(str, identifier)) # first, find the id of the (classname, itemid, property) a = self.db.arg sql = 'select _textid from __textids where _class=%s and '\ '_itemid=%s and _prop=%s'%(a, a, a) self.db.cursor.execute(sql, identifier) r = self.db.cursor.fetchone() if not r: # not previously indexed id = self.db.newid('__textids') sql = 'insert into __textids (_textid, _class, _itemid, _prop)'\ ' values (%s, %s, %s, %s)'%(a, a, a, a) self.db.cursor.execute(sql, (id, ) + identifier) else: id = int(r[0]) # clear out any existing indexed values sql = 'delete from __words where _textid=%s'%a self.db.cursor.execute(sql, (id, )) # ok, find all the unique words in the text if not isinstance(text, unicode): text = unicode(text, "utf-8", "replace") text = text.upper() wordlist = [w.encode("utf-8") for w in re.findall(r'(?u)\b\w{%d,%d}\b' % (self.minlength, self.maxlength), text)] words = set() for word in wordlist: if self.is_stopword(word): continue words.add(word) # for each word, add an entry in the db sql = 'insert into __words (_word, _textid) values (%s, %s)'%(a, a) words = [(word, id) for word in words] self.db.cursor.executemany(sql, words)
def updatenosy(db, cl, nodeid, newvalues): '''Update the nosy list for changes to the assignedto ''' # nodeid will be None if this is a new node current_nosy = set() if nodeid is None: ok = ('new', 'yes') else: ok = ('yes',) # old node, get the current values from the node if they haven't # changed if not newvalues.has_key('nosy'): nosy = cl.get(nodeid, 'nosy') for value in nosy: current_nosy.add(value) # if the nosy list changed in this transaction, init from the new value if newvalues.has_key('nosy'): nosy = newvalues.get('nosy', []) for value in nosy: if not db.hasnode('user', value): continue current_nosy.add(value) new_nosy = set(current_nosy) # add assignedto(s) to the nosy list if newvalues.has_key('assignedto') and newvalues['assignedto'] is not None: propdef = cl.getprops() if isinstance(propdef['assignedto'], hyperdb.Link): assignedto_ids = [newvalues['assignedto']] elif isinstance(propdef['assignedto'], hyperdb.Multilink): assignedto_ids = newvalues['assignedto'] for assignedto_id in assignedto_ids: new_nosy.add(assignedto_id) # see if there's any new messages - if so, possibly add the author and # recipient to the nosy if newvalues.has_key('messages'): if nodeid is None: ok = ('new', 'yes') messages = newvalues['messages'] else: ok = ('yes',) # figure which of the messages now on the issue weren't oldmessages = cl.get(nodeid, 'messages') messages = [] for msgid in newvalues['messages']: if msgid not in oldmessages: messages.append(msgid) # configs for nosy modifications add_author = getattr(db.config, 'ADD_AUTHOR_TO_NOSY', 'new') add_recips = getattr(db.config, 'ADD_RECIPIENTS_TO_NOSY', 'new') # now for each new message: msg = db.msg for msgid in messages: if add_author in ok: authid = msg.get(msgid, 'author') new_nosy.add(authid) # add on the recipients of the message if add_recips in ok: for recipient in msg.get(msgid, 'recipients'): new_nosy.add(recipient) if current_nosy != new_nosy: # that's it, save off the new nosy list newvalues['nosy'] = list(new_nosy)
def import_xml(tracker_home, xml_file, file_dir): """ Generate Roundup tracker import files based on the tracker schema, sf.net xml export and downloaded files from sf.net. """ tracker = instance.open(tracker_home) db = tracker.open('admin') resolved = db.status.lookup('resolved') unread = db.status.lookup('unread') chatting = db.status.lookup('unread') critical = db.priority.lookup('critical') urgent = db.priority.lookup('urgent') bug = db.priority.lookup('bug') feature = db.priority.lookup('feature') wish = db.priority.lookup('wish') adminuid = db.user.lookup('admin') anonuid = db.user.lookup('anonymous') root = ElementTree.parse(xml_file).getroot() def to_date(ts): return date.Date(time.gmtime(float(ts))) # parse out the XML artifacts = [] categories = set() users = set() add_files = set() remove_files = set() for artifact in root.find('artifacts'): d = {} op = {} artifacts.append(d) for field in artifact.findall('field'): name = field.get('name') if name == 'artifact_messages': for message in field.findall('message'): l = d.setdefault('messages', []) m = {} l.append(m) for field in message.findall('field'): name = field.get('name') if name == 'adddate': m[name] = to_date(field.text) else: m[name] = field.text if name == 'user_name': users.add(field.text) elif name == 'artifact_history': for event in field.findall('history'): l = d.setdefault('history', []) e = {} l.append(e) for field in event.findall('field'): name = field.get('name') if name == 'entrydate': e[name] = to_date(field.text) else: e[name] = field.text if name == 'mod_by': users.add(field.text) if e['field_name'] == 'File Added': add_files.add(e['old_value'].split(':')[0]) elif e['field_name'] == 'File Deleted': remove_files.add(e['old_value'].split(':')[0]) elif name == 'details': op['body'] = field.text elif name == 'submitted_by': op['user_name'] = field.text d[name] = field.text users.add(field.text) elif name == 'open_date': thedate = to_date(field.text) op['adddate'] = thedate d[name] = thedate else: d[name] = field.text categories.add(d['category']) if op.has_key('body'): l = d.setdefault('messages', []) l.insert(0, op) add_files -= remove_files # create users userd = {'nobody': '2'} users.remove('nobody') data = [ {'id': '1', 'username': '******', 'password': password.Password('admin'), 'roles': 'Admin', 'address': '*****@*****.**'}, {'id': '2', 'username': '******', 'roles': 'Anonymous'}, ] for n, user in enumerate(list(users)): userd[user] = n+3 data.append({'id': str(n+3), 'username': user, 'roles': 'User', 'address': '*****@*****.**'%user}) write_csv(db.user, data) users=userd # create categories categoryd = {'None': None} categories.remove('None') data = [] for n, category in enumerate(list(categories)): categoryd[category] = n data.append({'id': str(n), 'name': category}) write_csv(db.keyword, data) categories = categoryd # create issues issue_data = [] file_data = [] message_data = [] issue_journal = [] message_id = 0 for artifact in artifacts: d = {} d['id'] = artifact['artifact_id'] d['title'] = artifact['summary'] d['assignedto'] = users[artifact['assigned_to']] if d['assignedto'] == '2': d['assignedto'] = None d['creation'] = artifact['open_date'] activity = artifact['open_date'] d['creator'] = users[artifact['submitted_by']] actor = d['creator'] if categories[artifact['category']]: d['keyword'] = [categories[artifact['category']]] issue_journal.append(( d['id'], d['creation'].get_tuple(), d['creator'], "'create'", {} )) p = int(artifact['priority']) if artifact['artifact_type'] == 'Feature Requests': if p > 3: d['priority'] = feature else: d['priority'] = wish else: if p > 7: d['priority'] = critical elif p > 5: d['priority'] = urgent elif p > 3: d['priority'] = bug else: d['priority'] = feature s = artifact['status'] if s == 'Closed': d['status'] = resolved elif s == 'Deleted': d['status'] = resolved d['is retired'] = True else: d['status'] = unread nosy = set() for message in artifact.get('messages', []): authid = users[message['user_name']] if not message['body']: continue body = convert_message(message['body'], message_id) if not body: continue m = {'content': body, 'author': authid, 'date': message['adddate'], 'creation': message['adddate'], } message_data.append(m) if authid not in (None, '2'): nosy.add(authid) activity = message['adddate'] actor = authid if d['status'] == unread: d['status'] = chatting # add import message m = {'content': 'IMPORT FROM SOURCEFORGE', 'author': '1', 'date': today, 'creation': today} message_data.append(m) # sort messages and assign ids d['messages'] = [] message_data.sort(lambda a,b:cmp(a['date'],b['date'])) for message in message_data: message_id += 1 message['id'] = str(message_id) d['messages'].append(message_id) d['nosy'] = list(nosy) files = [] for event in artifact.get('history', []): if event['field_name'] == 'File Added': fid, name = event['old_value'].split(':', 1) if fid in add_files: files.append(fid) name = name.strip() try: f = open(os.path.join(file_dir, fid)) content = f.read() f.close() except: content = 'content missing' file_data.append({ 'id': fid, 'creation': event['entrydate'], 'creator': users[event['mod_by']], 'name': name, 'type': mimetypes.guess_type(name)[0], 'content': content, }) continue elif event['field_name'] == 'close_date': action = "'set'" info = { 'status': unread } elif event['field_name'] == 'summary': action = "'set'" info = { 'title': event['old_value'] } else: # not an interesting / translatable event continue row = [ d['id'], event['entrydate'].get_tuple(), users[event['mod_by']], action, info ] if event['entrydate'] > activity: activity = event['entrydate'] issue_journal.append(row) d['files'] = files d['activity'] = activity d['actor'] = actor issue_data.append(d) write_csv(db.issue, issue_data) write_csv(db.msg, message_data) write_csv(db.file, file_data) f = open('/tmp/imported/issue-journals.csv', 'w') writer = csv.writer(f, colon_separated) writer.writerows(issue_journal) f.close()
def __init__(self, db): self.stopwords = set(STOPWORDS) for word in db.config[('main', 'indexer_stopwords')]: self.stopwords.add(word)
def updatenosy(db, cl, nodeid, newvalues): """Update the nosy list for changes to the assignedto """ # nodeid will be None if this is a new node current_nosy = set() if nodeid is None: ok = ("new", "yes") else: ok = ("yes",) # old node, get the current values from the node if they haven't # changed if not newvalues.has_key("nosy"): nosy = cl.get(nodeid, "nosy") for value in nosy: current_nosy.add(value) # if the nosy list changed in this transaction, init from the new value if newvalues.has_key("nosy"): nosy = newvalues.get("nosy", []) for value in nosy: if not db.hasnode("user", value): continue current_nosy.add(value) new_nosy = set(current_nosy) # add assignedto(s) to the nosy list if newvalues.has_key("assignedto") and newvalues["assignedto"] is not None: propdef = cl.getprops() if isinstance(propdef["assignedto"], hyperdb.Link): assignedto_ids = [newvalues["assignedto"]] elif isinstance(propdef["assignedto"], hyperdb.Multilink): assignedto_ids = newvalues["assignedto"] for assignedto_id in assignedto_ids: new_nosy.add(assignedto_id) # see if there's any new messages - if so, possibly add the author and # recipient to the nosy if newvalues.has_key("messages"): if nodeid is None: ok = ("new", "yes") messages = newvalues["messages"] else: ok = ("yes",) # figure which of the messages now on the issue weren't oldmessages = cl.get(nodeid, "messages") messages = [] for msgid in newvalues["messages"]: if msgid not in oldmessages: messages.append(msgid) # configs for nosy modifications add_author = getattr(db.config, "ADD_AUTHOR_TO_NOSY", "new") add_recips = getattr(db.config, "ADD_RECIPIENTS_TO_NOSY", "new") # now for each new message: msg = db.msg for msgid in messages: if add_author in ok: authid = msg.get(msgid, "author") new_nosy.add(authid) # add on the recipients of the message if add_recips in ok: for recipient in msg.get(msgid, "recipients"): new_nosy.add(recipient) if current_nosy != new_nosy: # that's it, save off the new nosy list newvalues["nosy"] = list(new_nosy)
def updatenosy(db, cl, nodeid, newvalues): '''Update the nosy list for changes to the assignee ''' # nodeid will be None if this is a new node current_nosy = set() if nodeid is None: ok = ('new', 'yes') else: ok = ('yes', ) # old node, get the current values from the node if they haven't # changed if not newvalues.has_key('nosy'): nosy = cl.get(nodeid, 'nosy') for value in nosy: current_nosy.add(value) # if the nosy list changed in this transaction, init from the new value if newvalues.has_key('nosy'): nosy = newvalues.get('nosy', []) for value in nosy: if not db.hasnode('user', value): continue current_nosy.add(value) new_nosy = set(current_nosy) # add assignee(s) to the nosy list if newvalues.has_key('assignee') and newvalues['assignee'] is not None: propdef = cl.getprops() if isinstance(propdef['assignee'], hyperdb.Link): assignee_ids = [newvalues['assignee']] elif isinstance(propdef['assignee'], hyperdb.Multilink): assignee_ids = newvalues['assignee'] for assignee_id in assignee_ids: new_nosy.add(assignee_id) # see if there's any new messages - if so, possibly add the author and # recipient to the nosy if newvalues.has_key('messages'): if nodeid is None: ok = ('new', 'yes') messages = newvalues['messages'] else: ok = ('yes', ) # figure which of the messages now on the issue weren't oldmessages = cl.get(nodeid, 'messages') messages = [] for msgid in newvalues['messages']: if msgid not in oldmessages: messages.append(msgid) # configs for nosy modifications add_author = getattr(db.config, 'ADD_AUTHOR_TO_NOSY', 'new') add_recips = getattr(db.config, 'ADD_RECIPIENTS_TO_NOSY', 'new') # now for each new message: msg = db.msg for msgid in messages: if add_author in ok: authid = msg.get(msgid, 'author') new_nosy.add(authid) # add on the recipients of the message if add_recips in ok: for recipient in msg.get(msgid, 'recipients'): new_nosy.add(recipient) if current_nosy != new_nosy: # that's it, save off the new nosy list newvalues['nosy'] = list(new_nosy)