예제 #1
0
def convert_wiki(source, dest, dest_project_id):
    if overwrite and (method == 'direct'):
        dest.clear_wiki_attachments(dest_project_id)

    exclude_authors = [a.strip() for a in config.get('wiki', 'exclude_authors').split(',')]
    target_directory = config.get('wiki', 'target-directory')
    server = xmlrpclib.MultiCall(source)
    for name in source.wiki.getAllPages():
        info = source.wiki.getPageInfo(name)
        if (info['author'] not in exclude_authors):
            page = source.wiki.getPage(name)
            print("Page %s:%s" % (name, info))
            if (name == 'WikiStart'):
                name = 'home'
            converted = trac2down.convert(page, os.path.dirname('/wikis/%s' % name))
            if method == 'direct':
                for attachment in source.wiki.listAttachments(name):
                    print(attachment)
                    binary_attachment = source.wiki.getAttachment(attachment).data
                    try:
                        attachment_path = dest.create_wiki_attachment(dest_project_id, users_map[info['author']], convert_xmlrpc_datetime(info['lastModified']), attachment, binary_attachment)
                    except KeyError:
                        attachment_path = dest.create_wiki_attachment(dest_project_id, default_user, convert_xmlrpc_datetime(info['lastModified']), attachment, binary_attachment)
                    attachment_name = attachment.split('/')[-1]
                    converted = converted.replace(r'](%s)' % attachment_name, r'](%s)' % os.path.relpath(attachment_path, '/namespace/project/wiki/page'))
            trac2down.save_file(converted, name, info['version'], info['lastModified'], info['author'], target_directory)
예제 #2
0
def convert_wiki(source, dest, dest_project_id):
    if overwrite and (method == 'direct'):
        dest.clear_wiki_attachments(dest_project_id)

    exclude_authors = [a.strip() for a in config.get('wiki', 'exclude_authors').split(',')]
    target_directory = config.get('wiki', 'target-directory')
    server = xmlrpclib.MultiCall(source)
    for name in source.wiki.getAllPages():
        info = source.wiki.getPageInfo(name)
        if (info['author'] not in exclude_authors):
            page = source.wiki.getPage(name)
            print "Page %s:%s" % (name, info)
            if (name == 'WikiStart'):
                name = 'home'
            converted = trac2down.convert(page, os.path.dirname('/wikis/%s' % name))
            if method == 'direct':
                for attachment in source.wiki.listAttachments(name):
                    print attachment
                    binary_attachment = source.wiki.getAttachment(attachment).data
                    try:
                        attachment_path = dest.create_wiki_attachment(dest_project_id, users_map[info['author']], convert_xmlrpc_datetime(info['lastModified']), attachment, binary_attachment)
                    except KeyError:
                        attachment_path = dest.create_wiki_attachment(dest_project_id, default_user, convert_xmlrpc_datetime(info['lastModified']), attachment, binary_attachment)
                    attachment_name = attachment.split('/')[-1]
                    converted = converted.replace(r'](%s)' % attachment_name, r'](%s)' % os.path.relpath(attachment_path, '/namespace/project/wiki/page'))
            trac2down.save_file(converted, name, info['version'], info['lastModified'], info['author'], target_directory)
예제 #3
0
def convert_issue_content(text, ticket_mapping):
    """
    Convert TracWiki text to GitHub Markdown.
    Change the ticket IDs to GitHub URLs according to the mapping.
    Ignore included images.
    """
    text = text.replace(config.TRAC_TICKET_PREFIX, '#')
    ticket_re = '#([0-9]+)'
    for match in matches(ticket_re, text):
        try:
            github_url = ticket_mapping[int(match)]
            new_ticket_id = github_url.rsplit('/', 1)[1]
            text = sub(f'#{match}', f'[#{new_ticket_id}]({github_url})', text)
        except KeyError:
            # We don't know this ticket. Leave it alone.
            print("Warning: unknown ticket: #" + str(match))
            pass

    return convert(text, base_path='')
def convert_issue_content(text, ticket_mapping):
    """
    Convert TracWiki text to GitHub Markdown.
    Change the ticket IDs to GitHub URLs according to the mapping.
    Ignore included images.
    """
    text = text.replace(config.TRAC_TICKET_PREFIX, '#')
    ticket_re = '#([0-9]+)'
    text = update_changeset(text)
    for match in matches(ticket_re, text):
        try:
            github_url = ticket_mapping[int(match)]
            new_ticket_id = github_url.rsplit('/', 1)[1]
            text = sub(f'#{match}', f'[#{new_ticket_id}]({github_url})', text)
        except KeyError:
            # We don't know this ticket. Warn about it.
            print(f"Warning: ticket #{match} not in tickets_expected_gold.tsv"
                  f" - leaving it as #{match}")

    return convert(text, base_path='', wiki_prefix=config.MIGRATED_WIKI_PREFIX)
예제 #5
0
def convert_issues(source,
                   dest,
                   dest_project_id,
                   only_issues=None,
                   blacklist_issues=None):
    if overwrite and (method == 'direct'):
        dest.clear_issues(dest_project_id)

    milestone_map_id = {}

    if migrate_milestones:
        milestone_id = 0
        for milestone_name in source.ticket.milestone.getAll():
            milestone = source.ticket.milestone.get(milestone_name)
            print(milestone)
            new_milestone = Milestones(
                description=trac2down.convert(
                    fix_wiki_syntax(milestone['description']), '/milestones/',
                    False),
                title=milestone['name'],
                state='active'
                if str(milestone['completed']) == '0' else 'closed')
            if method == 'direct':
                new_milestone.project = dest_project_id
            if milestone['due']:
                new_milestone.due_date = convert_xmlrpc_datetime(
                    milestone['due'])
            new_milestone = dest.create_milestone(dest_project_id,
                                                  new_milestone)
            if new_milestone.id:
                milestone_map_id[milestone_name] = new_milestone.id
                milestone_id = new_milestone.id + 1
            else:
                milestone_map_id[milestone_name] = milestone_id
                milestone_id = milestone_id + 1

    get_all_tickets = xmlrpclib.MultiCall(source)

    for ticket in source.ticket.query("max=0&order=id"):
        get_all_tickets.ticket.get(ticket)

    for src_ticket in get_all_tickets():
        src_ticket_id = src_ticket[0]
        if only_issues and src_ticket_id not in only_issues:
            print("SKIP unwanted ticket #%s" % src_ticket_id)
            continue
        if blacklist_issues and src_ticket_id in blacklist_issues:
            print("SKIP blacklisted ticket #%s" % src_ticket_id)
            continue

        src_ticket_data = src_ticket[3]
        src_ticket_reporter = src_ticket_data['reporter']
        src_ticket_priority = 'normal'
        if 'priority' in src_ticket_data:
            src_ticket_priority = src_ticket_data['priority']
        src_ticket_resolution = src_ticket_data['resolution']
        src_ticket_severity = src_ticket_data.get('severity')
        src_ticket_status = src_ticket_data['status']
        src_ticket_component = src_ticket_data.get('component', '')
        src_ticket_keywords = src_ticket_data['keywords']
        if (component_filter and src_ticket_component not in component_filter):
            continue

        new_labels = []
        if src_ticket_priority == 'high':
            new_labels.append('high priority')
        elif src_ticket_priority == 'medium':
            pass
        elif src_ticket_priority == 'low':
            new_labels.append('low priority')

        if src_ticket_resolution == '':
            # active ticket
            pass
        elif src_ticket_resolution == 'fixed':
            pass
        elif src_ticket_resolution == 'invalid':
            new_labels.append('invalid')
        elif src_ticket_resolution == 'wontfix':
            new_labels.append("won't fix")
        elif src_ticket_resolution == 'duplicate':
            new_labels.append('duplicate')
        elif src_ticket_resolution == 'worksforme':
            new_labels.append('works for me')

        if src_ticket_severity == 'high':
            new_labels.append('critical')
        elif src_ticket_severity == 'medium':
            pass
        elif src_ticket_severity == 'low':
            new_labels.append("minor")

        # Current ticket types are: enhancement, defect, compilation, performance, style, scientific, task, requirement
        # new_labels.append(src_ticket_type)

        if add_component_as_label and src_ticket_component != '':
            for component in src_ticket_component.split(','):
                new_labels.append(component.strip())

        if add_label:
            new_labels.append(add_label)

        if src_ticket_keywords != '' and migrate_keywords:
            for keyword in src_ticket_keywords.split(','):
                new_labels.append(keyword.strip())

        print("new labels: %s" % new_labels)

        new_state = ''
        if src_ticket_status == 'new':
            new_state = 'opened'
        elif src_ticket_status == 'assigned':
            new_state = 'opened'
        elif src_ticket_status == 'reopened':
            new_state = 'reopened'
        elif src_ticket_status == 'closed':
            new_state = 'closed'
        else:
            print("!!! unknown ticket status: %s" % src_ticket_status)

        new_description = (
            create_issue_header(author=src_ticket_reporter,
                                created=src_ticket[1],
                                updated=src_ticket[2]) +
            trac2down.convert(fix_wiki_syntax(src_ticket_data['description']),
                              '/issues/', False))

        # Minimal parameters
        new_issue = Issues(
            title=(src_ticket_data['summary'][:245] +
                   '...') if len(src_ticket_data['summary']) > 245 else
            src_ticket_data['summary'],
            description=new_description,
            state=new_state,
            labels=",".join(new_labels))

        if src_ticket_data['owner'] != '':
            try:
                new_issue.assignee = dest.get_user_id(
                    users_map[src_ticket_data['owner']])
            except KeyError:
                new_issue.assignee = dest.get_user_id(default_user)
        # Additional parameters for direct access
        if (method == 'direct'):
            new_issue.created_at = convert_xmlrpc_datetime(src_ticket[1])
            new_issue.updated_at = convert_xmlrpc_datetime(src_ticket[2])
            new_issue.project = dest_project_id
            new_issue.state = new_state
            new_issue.author = dest.get_user_id(
                users_map.get(src_ticket_reporter, default_user))
            if overwrite:
                new_issue.iid = src_ticket_id
            else:
                new_issue.iid = dest.get_issues_iid(dest_project_id)
        # Set correct issue id
        new_issue.iid = src_ticket_id
        if 'milestone' in src_ticket_data:
            milestone = src_ticket_data['milestone']
            if milestone and milestone in milestone_map_id:
                new_issue.milestone = milestone_map_id[milestone]
        new_ticket = dest.create_issue(dest_project_id, new_issue)

        changelog = source.ticket.changeLog(src_ticket_id)
        is_attachment = False
        attachment = None
        binary_attachment = None
        newowner = None
        for change in changelog:
            # New line
            change_time = str(convert_xmlrpc_datetime(change[0]))
            change_type = change[2]
            print(("  %s by %s (%s -> %s)" %
                   (change_type, change[1], change[3][:40].replace(
                       "\n", " "), change[4][:40].replace("\n", " "))).encode(
                           "ascii", "replace"))
            #assert attachment is None or change_type == "comment", "an attachment must be followed by a comment"
            author = dest.get_user_id(users_map[change[1]])
            if change_type == "attachment":
                # The attachment will be described in the next change!
                is_attachment = True
                attachment = change
            if (change_type == "comment"):
                desc = change[4]
                if (desc == '' and is_attachment == False):
                    continue
                if (desc != ''):
                    desc = fix_wiki_syntax(change[4])
                note = Notes(note=create_issue_header(
                    author=change[1], created=change[0], is_comment=True) +
                             trac2down.convert(desc, '/issues/', False))
                if attachment is not None:
                    note.attachment_name = attachment[4]  # name of attachment
                    binary_attachment = source.ticket.getAttachment(
                        src_ticket_id, attachment[4].encode('utf8')).data
                try:
                    note.author = dest.get_user_id(users_map[change[1]])
                    if note.author == None:
                        note.author = dest.get_user_id(default_user)
                except KeyError:
                    note.author = dest.get_user_id(default_user)
                if (method == 'direct'):
                    note.created_at = convert_xmlrpc_datetime(change[0])
                    note.updated_at = convert_xmlrpc_datetime(change[0])
                    try:
                        note.author = dest.get_user_id(users_map[change[1]])
                    except KeyError:
                        note.author = dest.get_user_id(default_user)
                    if (is_attachment):
                        note.attachment = attachment[4]
                        binary_attachment = source.ticket.getAttachment(
                            src_ticket_id, attachment[4].encode('utf8')).data
                dest.comment_issue(dest_project_id, new_ticket, note,
                                   binary_attachment)
                is_attachment = False
            if change_type == "status":
                if change[3] == 'vendor':
                    # remove label 'vendor'
                    new_ticket.labels.remove('vendor')
                    # workaround #3 dest.update_issue_property(dest_project_id, issue, author, change_time, 'labels')

                # we map here the various statii we have in trac to just 2 statii in gitlab (open or close), so loose some information
                if change[4] in [
                        'new', 'assigned', 'analyzed', 'vendor', 'reopened'
                ]:
                    newstate = 'open'
                elif change[4] in ['closed']:
                    newstate = 'closed'
                else:
                    raise ("  unknown ticket status: " + change[4])

                if new_ticket.state != newstate:
                    new_ticket.state = newstate

                if change[4] == 'vendor':
                    # add label 'vendor'
                    new_ticket.labels.append('vendor')
                    dest.ensure_label(dest_project_id, 'vendor',
                                      labelcolor['vendor'])

                if newstate == 'closed':
                    dest.close_issue(dest_project_id, new_ticket.iid)

                dest.comment_issue(
                    dest_project_id, new_ticket,
                    Notes(note='Changing status from ' + change[3] + ' to ' +
                          change[4] + '.',
                          created_at=change_time,
                          author=author), binary_attachment)
예제 #6
0
def convert_issues(source,
                   dest,
                   dest_project_id,
                   only_issues=None,
                   blacklist_issues=None):
    if overwrite and (method == 'direct'):
        dest.clear_issues(dest_project_id)

    milestone_map_id = {}

    if migrate_milestones:
        for milestone_name in source.ticket.milestone.getAll():
            milestone = source.ticket.milestone.get(milestone_name)
            print(milestone)
            new_milestone = Milestones(
                description=trac2down.convert(
                    fix_wiki_syntax(milestone['description']), '/milestones/',
                    False),
                title=milestone['name'],
                state='active'
                if str(milestone['completed']) == '0' else 'closed')
            if method == 'direct':
                new_milestone.project = dest_project_id
            if milestone['due']:
                new_milestone.due_date = convert_xmlrpc_datetime(
                    milestone['due'])
            new_milestone = dest.create_milestone(dest_project_id,
                                                  new_milestone)
            milestone_map_id[milestone_name] = new_milestone.id

    get_all_tickets = xmlrpclib.MultiCall(source)

    for ticket in source.ticket.query("max=0&order=id"):
        get_all_tickets.ticket.get(ticket)

    for src_ticket in get_all_tickets():
        src_ticket_id = src_ticket[0]
        if only_issues and src_ticket_id not in only_issues:
            print("SKIP unwanted ticket #%s" % src_ticket_id)
            continue
        if blacklist_issues and src_ticket_id in blacklist_issues:
            print("SKIP blacklisted ticket #%s" % src_ticket_id)
            continue

        src_ticket_data = src_ticket[3]
        src_ticket_priority = 'normal'
        if 'priority' in src_ticket_data:
            src_ticket_priority = src_ticket_data['priority']
        src_ticket_resolution = src_ticket_data['resolution']
        src_ticket_severity = src_ticket_data['severity']
        src_ticket_status = src_ticket_data['status']
        src_ticket_component = src_ticket_data.get('component', '')
        src_ticket_keywords = src_ticket_data['keywords']
        if (component_filter and src_ticket_component not in component_filter):
            continue

        new_labels = []
        if src_ticket_priority == 'high':
            new_labels.append('high priority')
        elif src_ticket_priority == 'medium':
            pass
        elif src_ticket_priority == 'low':
            new_labels.append('low priority')

        if src_ticket_resolution == '':
            # active ticket
            pass
        elif src_ticket_resolution == 'fixed':
            pass
        elif src_ticket_resolution == 'invalid':
            new_labels.append('invalid')
        elif src_ticket_resolution == 'wontfix':
            new_labels.append("won't fix")
        elif src_ticket_resolution == 'duplicate':
            new_labels.append('duplicate')
        elif src_ticket_resolution == 'worksforme':
            new_labels.append('works for me')

        if src_ticket_severity == 'high':
            new_labels.append('critical')
        elif src_ticket_severity == 'medium':
            pass
        elif src_ticket_severity == 'low':
            new_labels.append("minor")

        # Current ticket types are: enhancement, defect, compilation, performance, style, scientific, task, requirement
        # new_labels.append(src_ticket_type)

        if add_component_as_label and src_ticket_component != '':
            for component in src_ticket_component.split(','):
                new_labels.append(component.strip())

        if add_label:
            new_labels.append(add_label)

        if src_ticket_keywords != '' and migrate_keywords:
            for keyword in src_ticket_keywords.split(','):
                new_labels.append(keyword.strip())

        print("new labels: %s" % new_labels)

        new_state = ''
        if src_ticket_status == 'new':
            new_state = 'opened'
        elif src_ticket_status == 'assigned':
            new_state = 'opened'
        elif src_ticket_status == 'reopened':
            new_state = 'reopened'
        elif src_ticket_status == 'closed':
            new_state = 'closed'
        else:
            print("!!! unknown ticket status: %s" % src_ticket_status)

        # Minimal parameters
        new_issue = Issues(title=src_ticket_data['summary'],
                           description=trac2down.convert(
                               fix_wiki_syntax(src_ticket_data['description']),
                               '/issues/', False),
                           state=new_state,
                           labels=",".join(new_labels))

        if src_ticket_data['owner'] != '':
            try:
                new_issue.assignee = dest.get_user_id(
                    users_map[src_ticket_data['owner']])
            except KeyError:
                new_issue.assignee = dest.get_user_id(default_user)
        # Additional parameters for direct access
        if (method == 'direct'):
            new_issue.created_at = convert_xmlrpc_datetime(src_ticket[1])
            new_issue.updated_at = convert_xmlrpc_datetime(src_ticket[2])
            new_issue.project = dest_project_id
            new_issue.state = new_state
            try:
                new_issue.author = dest.get_user_id(
                    users_map[src_ticket_data['reporter']])
            except KeyError:
                new_issue.author = dest.get_user_id(default_user)
            if overwrite:
                new_issue.iid = src_ticket_id
            else:
                new_issue.iid = dest.get_issues_iid(dest_project_id)

        if 'milestone' in src_ticket_data:
            milestone = src_ticket_data['milestone']
            if milestone and milestone in milestone_map_id:
                new_issue.milestone = milestone_map_id[milestone]
        new_ticket = dest.create_issue(dest_project_id, new_issue)
        # new_ticket_id  = new_ticket.id

        changelog = source.ticket.changeLog(src_ticket_id)
        is_attachment = False
        for change in changelog:
            change_type = change[2]
            if change_type == "attachment":
                # The attachment will be described in the next change!
                is_attachment = True
                attachment = change
            if (change_type == "comment"):
                desc = change[4]
                if (desc == '' and is_attachment == False):
                    continue
                if (desc != ''):
                    desc = fix_wiki_syntax(change[4])
                note = Notes(note=trac2down.convert(desc, '/issues/', False))
                binary_attachment = None
                if (method == 'direct'):
                    note.created_at = convert_xmlrpc_datetime(change[0])
                    note.updated_at = convert_xmlrpc_datetime(change[0])
                    try:
                        note.author = dest.get_user_id(users_map[change[1]])
                    except KeyError:
                        note.author = dest.get_user_id(default_user)
                    if (is_attachment):
                        note.attachment = attachment[4]
                        binary_attachment = source.ticket.getAttachment(
                            src_ticket_id, attachment[4].encode('utf8')).data
                dest.comment_issue(dest_project_id, new_ticket, note,
                                   binary_attachment)
                is_attachment = False
예제 #7
0
def convert_issues(source, dest, dest_project_id, only_issues=None):
    if overwrite and (method == 'direct'):
        dest.clear_issues(dest_project_id)

    milestone_map_id={}
    for milestone_name in source.ticket.milestone.getAll():
        milestone = source.ticket.milestone.get(milestone_name)
        print milestone
        new_milestone = Milestones(
            description = milestone['description'],
            title = milestone['name'],
            state = 'active' if str(milestone['completed']) == '0'  else 'closed'
        )
        if method == 'direct':
            new_milestone.project = dest_project_id
        if milestone['due']:
            new_milestone.due_date = convert_xmlrpc_datetime(milestone['due'])
        new_milestone = dest.create_milestone(dest_project_id, new_milestone)
        milestone_map_id[milestone_name] = new_milestone.id

    get_all_tickets = xmlrpclib.MultiCall(source)

    for ticket in source.ticket.query("max=0"):
        get_all_tickets.ticket.get(ticket)

    for src_ticket in get_all_tickets():
        src_ticket_id = src_ticket[0]
        if only_issues and src_ticket_id not in only_issues:
            print "SKIP unwanted ticket #%s" % src_ticket_id
            continue

        src_ticket_data = src_ticket[3]

        src_ticket_priority = src_ticket_data['priority']
        src_ticket_resolution = src_ticket_data['resolution']
        # src_ticket_severity = src_ticket_data['severity']
        src_ticket_status = src_ticket_data['status']
        src_ticket_component = src_ticket_data['component']

        new_labels = []
        if src_ticket_priority == 'high':
            new_labels.append('high priority')
        elif src_ticket_priority == 'medium':
            pass
        elif src_ticket_priority == 'low':
            new_labels.append('low priority')

        if src_ticket_resolution == '':
            # active ticket
            pass
        elif src_ticket_resolution == 'fixed':
            pass
        elif src_ticket_resolution == 'invalid':
            new_labels.append('invalid')
        elif src_ticket_resolution == 'wontfix':
            new_labels.append("won't fix")
        elif src_ticket_resolution == 'duplicate':
            new_labels.append('duplicate')
        elif src_ticket_resolution == 'worksforme':
            new_labels.append('works for me')

        # if src_ticket_severity == 'high':
        #     new_labels.append('critical')
        # elif src_ticket_severity == 'medium':
        #     pass
        # elif src_ticket_severity == 'low':
        #     new_labels.append("minor")

        # Current ticket types are: enhancement, defect, compilation, performance, style, scientific, task, requirement
        # new_labels.append(src_ticket_type)

        if src_ticket_component != '':
            for component in src_ticket_component.split(','):
                new_labels.append(component.strip())

        print "new labels:", new_labels

        new_state = ''
        if src_ticket_status == 'new':
            new_state = 'opened'
        elif src_ticket_status == 'assigned':
            new_state = 'opened'
        elif src_ticket_status == 'reopened':
            new_state = 'reopened'
        elif src_ticket_status == 'closed':
            new_state = 'closed'
        else:
            print "!!! unknown ticket status:", src_ticket_status

        # Minimal parameters
        new_issue = Issues(
            title=src_ticket_data['summary'],
            description=trac2down.convert(fix_wiki_syntax(src_ticket_data['description']), '/issues/', False),
            state=new_state,
            labels=",".join(new_labels)
        )

        if src_ticket_data['owner'] != '':
            try:
                new_issue.assignee = dest.get_user_id(users_map[src_ticket_data['owner']])
            except KeyError:
                new_issue.assignee = dest.get_user_id(default_user)
        # Additional parameters for direct access
        if (method == 'direct'):
            new_issue.created_at = convert_xmlrpc_datetime(src_ticket[1])
            new_issue.updated_at = convert_xmlrpc_datetime(src_ticket[2])
            new_issue.project = dest_project_id
            new_issue.state = new_state
            try:
                new_issue.author = dest.get_user_id(users_map[src_ticket_data['reporter']])
            except KeyError:
                new_issue.author = dest.get_user_id(default_user)
            if overwrite:
                new_issue.iid = src_ticket_id
            else:
                new_issue.iid = dest.get_issues_iid(dest_project_id)

        if 'milestone' in src_ticket_data:
            milestone = src_ticket_data['milestone']
            if milestone and milestone_map_id[milestone]:
                new_issue.milestone = milestone_map_id[milestone]
        new_ticket = dest.create_issue(dest_project_id, new_issue)
        # new_ticket_id  = new_ticket.id

        changelog = source.ticket.changeLog(src_ticket_id)
        is_attachment = False
        for change in changelog:
            change_type = change[2]
            if change_type == "attachment":
                # The attachment will be described in the next change!
                is_attachment = True
                attachment = change
            if (change_type == "comment") and change[4] != '':
                note = Notes(
                    note=trac2down.convert(fix_wiki_syntax(change[4]), '/issues/', False)
                )
                binary_attachment = None
                if (method == 'direct'):
                    note.created_at = convert_xmlrpc_datetime(change[0])
                    note.updated_at = convert_xmlrpc_datetime(change[0])
                    try:
                        note.author = dest.get_user_id(users_map[change[1]])
                    except KeyError:
                        note.author = dest.get_user_id(default_user)
                    if (is_attachment):
                        note.attachment = attachment[4]
                        binary_attachment = source.ticket.getAttachment(src_ticket_id, attachment[4].encode('utf8')).data
                dest.comment_issue(dest_project_id, new_ticket, note, binary_attachment)
                is_attachment = False
    """Add double [[...]] around wikilinks (given as `targets`)

    Most prosaic method possible - simply loop over the explicit list
    of targets.  Tries to be careful in avoiding code blocks, but
    will still be fooled by inline code
    """
    lines = []
    is_code_block = False
    for line in text.split('\n'):
        # not blockquote?
        if not line.startswith('    '):
            if line.startswith("````"):
                is_code_block = not is_code_block
            if not is_code_block:
                for target in targets:
                    line = line.replace(target, f"[[{target}]]")
            # line = re.sub(r'\!(([A-Z][a-z0-9]+){2,})', r'[[\1]]', line)
        lines.append(line)
    return "\n".join(lines)


# Process all the files in the input folder
for p in input_path.glob("*"):
    # Open file from the dump of the trac database and convert it
    with open(p) as f:
        text = trac2down.convert(f.read(), ".")
        text = fixup_wikilinks(text, wikinames)
    # Save the converted file with a ".md" extension to output folder
    # The 3rd, 4th, 5th arguments to this function are unused
    trac2down.save_file(text, p.name, None, None, None, save_path)
예제 #9
0
def convert_wiki(source, dest):
    exclude_authors = [a.strip() for a in config.get('wiki', 'exclude_authors').split(',')]
    target_directory = config.get('wiki', 'target-directory')

    if wiki_override_page:
        pages = [wiki_override_page]
    else:
        pages = source.wiki.getAllPages()

    i = 0
    for name in pages:
        i += 1
        info = source.wiki.getPageInfo(name)
        if info == 0:
            raise Exception("No page named %s could be found" % name)

        if info['author'] in exclude_authors:
            continue

        page = source.wiki.getPage(name)
        print("[%d/%d] Page %s:%s" % (i, len(pages), name, info))
        if name == 'WikiStart':
            name = 'home'

        sanitized_name = name.replace('/', '-').lower()
        upload_prefix = 'uploads/%s' % sanitized_name
        old_attachment_prefix = '/attachment/wiki/%s' % name
        old_raw_attachment_prefix = '/raw-attachment/wiki/%s' % name
        converted = trac2down.convert(
            page,
            os.path.dirname('/wikis/%s' % name),
            wiki_upload_prefix=upload_prefix,
            old_attachment_prefix=old_attachment_prefix,
            old_raw_attachment_prefix=old_raw_attachment_prefix
        )

        if method == 'direct' and not ignore_wiki_attachments:
            files_not_linked_to = []

            for attachment_filename in source.wiki.listAttachments(name):
                binary_attachment = source.wiki.getAttachment(attachment_filename).data
                attachment_name = attachment_filename.split('/')[-1]
                sanitized_attachment_name = attachment_name \
                    .replace(' ', '_') \
                    .replace('(', '') \
                    .replace(')', '')
                attachment_directory = os.path.join(target_directory, 'uploads', sanitized_name)

                dest.save_wiki_attachment(attachment_directory, sanitized_attachment_name, binary_attachment)
                converted = converted.replace(r'%s/%s)' % (sanitized_name, attachment_filename),
                                                r'%s/%s)' % (sanitized_name, sanitized_attachment_name))
                if '%s/%s)' % (upload_prefix, sanitized_attachment_name) not in converted:
                    files_not_linked_to.append(sanitized_attachment_name)

                print('  ' + sanitized_attachment_name)

            if len(files_not_linked_to) > 0:
                print '  %d non-linked attachments detected, manually adding to generated Markdown' % len(files_not_linked_to)
                converted += '\n\n'
                converted += '##### Attached files:\n'
                for file_name in files_not_linked_to:
                    converted += '- [%s](uploads/%s/%s)\n' % (file_name, sanitized_name, file_name)

        trac2down.save_file(converted, name, info['version'], info['lastModified'], info['author'], target_directory)
예제 #10
0
def convert_issues(source, dest, dest_project_ids, convert_milestones, only_issues=None,
                   get_dest_project_id_for_issue=None, issue_mutator=None):
    if only_issues is None: only_issues = []

    if overwrite and method == 'direct':
        for project_id in dest_project_ids:
            dest.clear_issues(project_id, only_issues)

    milestone_map_id = {}
    if convert_milestones:
        for dest_project_id in dest_project_ids:
            for milestone_name in source.ticket.milestone.getAll():
                milestone = source.ticket.milestone.get(milestone_name)
                print("migrated milestone: %s" % milestone_name)
                new_milestone = Milestones(
                    description=trac2down.convert(fix_wiki_syntax(milestone['description']), '/milestones/', False),
                    title=milestone['name'],
                    state='active' if str(milestone['completed']) == '0' else 'closed'
                )
                if method == 'direct':
                    new_milestone.project = dest_project_id
                if milestone['due']:
                    new_milestone.due_date = convert_xmlrpc_datetime(milestone['due'])
                new_milestone = dest.create_milestone(dest_project_id, new_milestone)
                milestone_map_id[milestone_name] = new_milestone.id

    get_all_tickets = xmlrpclib.MultiCall(source)

    gitlab_user_cache = {}

    if only_issues:
        print("getting tickets from trac: %s" % only_issues)
        for ticket in only_issues:
            get_all_tickets.ticket.get(ticket)
    else:
        print("getting all tickets from trac")
        for ticket in source.ticket.query("max=0&order=id"):
            get_all_tickets.ticket.get(ticket)

    image_regexp = re.compile(r'\.(jpg|jpeg|png|gif)$')
    title_label_regexp = re.compile(r'(\[.+?\]|.+?:)')

    for src_ticket in get_all_tickets():
        src_ticket_id = src_ticket[0]

        if only_issues and src_ticket_id not in only_issues:
            print("SKIP unwanted ticket #%s" % src_ticket_id)
            continue

        print 'migrating ticket %d' % src_ticket_id

        src_ticket_data = src_ticket[3]

        src_ticket_billable = src_ticket_data.get('billable', '0')
        src_ticket_component = src_ticket_data['component']
        src_ticket_keywords = re.split(r'[, ]', src_ticket_data['keywords'])
        src_ticket_milestone = src_ticket_data['milestone']
        src_ticket_priority = src_ticket_data['priority']
        src_ticket_resolution = src_ticket_data['resolution']
        src_ticket_status = src_ticket_data['status']
        src_ticket_version = src_ticket_data['version']

        new_labels = CasePreservingSet()

        if src_ticket_billable == '1':
            new_labels.add('billable')

        if src_ticket_milestone:
            for label in translate_milestone(src_ticket_milestone):
                new_labels.add(label)

        if src_ticket_priority == 'high':
            new_labels.add('high priority')
        elif src_ticket_priority == 'medium':
            pass
        elif src_ticket_priority == 'low':
            new_labels.add('low priority')

        if src_ticket_resolution == '':
            # active ticket
            pass
        elif src_ticket_resolution == 'fixed':
            pass
        elif src_ticket_resolution == 'invalid':
            new_labels.add('invalid')
        elif src_ticket_resolution == 'wontfix':
            new_labels.add("won't fix")
        elif src_ticket_resolution == 'duplicate':
            new_labels.add('duplicate')
        elif src_ticket_resolution == 'worksforme':
            new_labels.add('works for me')

        if src_ticket_component != '':
            for component in src_ticket_component.split(','):
                component = component.strip()
                translated_component = translate_component(component)

                if translated_component:
                    new_labels.add(translated_component)
                else:
                    print('    WARN: Dropping component %s' % component)

        for keyword in src_ticket_keywords:
            keyword = keyword.strip()
            if not keyword:
                continue

            translated_keyword = translate_keyword(keyword)

            if translated_keyword:
                new_labels.add(translated_keyword)
            else:
                print('    WARN: Dropping keyword %s' % keyword)

        new_state = 'opened'
        if src_ticket_status == 'new':
            new_state = 'opened'
        elif src_ticket_status == 'assigned':
            new_state = 'opened'
            new_labels.add('Do')
        elif src_ticket_status == 'reopened':
            # There is no 'reopened' state in GitLab.
            new_state = 'opened'
        elif src_ticket_status == 'closed':
            new_state = 'closed'
        elif src_ticket_status == 'accepted':
            new_state = 'opened'
            new_labels.add('Do')
        elif src_ticket_status == 'reviewing' or src_ticket_status == 'testing':
            new_labels.add('Check')
        else:
            print("!!! Unknown ticket status: %s, not preserving in migrated data" % src_ticket_status)

        summary = src_ticket_data['summary']
        sanitized_summary = summary
        done = False

        while not done:
            title_result = title_label_regexp.search(sanitized_summary)

            if title_result:
                prefix = title_result.group(1)
                lowercased_prefix = prefix.lower()

                # Awkward way, but prefix.translate() works differently on str and unicode objects so
                # this is good enough for now.
                mangled_prefix = lowercased_prefix.replace('[', '').replace(']', '').replace(':', '')
                translated_prefix = label_prefix_translation_map.get(mangled_prefix, '')

                if translated_prefix != '':
                    if translated_prefix == None:
                        # None values have a special meaning, indicate: "Remove this prefix, but don't add a label".
                        print('    !!! Dropping prefix %s' % mangled_prefix)
                    else:
                        # Prefix found in whitelist.
                        new_labels.add(translated_prefix)

                    sanitized_summary = sanitized_summary[title_result.end():].strip()
                else:
                    # Prefix doesn't exist in mangling map. Leave it as-is, has to be manually handled.
                    done = True
            else:
                done = True

        # FIXME: Would like to put these in deeply nested folder structure instead of dashes, but
        # the GitLab uploads route only supports a single subfolder below uploads:
        # https://github.com/gitlabhq/gitlabhq/blob/master/config/routes/uploads.rb#L22-L25
        issue_attachment_path = os.path.join('issue-attachment-%d' % src_ticket_id)

        new_issue = Issues(
            title=sanitized_summary,
            description=trac2down.convert(
                fix_wiki_syntax(src_ticket_data['description']),
                '/issues/',
                False,
                issue_upload_prefix='/uploads/' + issue_attachment_path
            ),
            state=new_state,
            labels=new_labels
        )

        if get_dest_project_id_for_issue:
            dest_project_id = get_dest_project_id_for_issue(dest, new_issue)
        else:
            # No function defined - assume that we have been provided a single project ID.
            dest_project_id = dest_project_ids[0]

        if issue_mutator:
            issue_mutator(new_issue)

        print("    Final set of labels: %s" % ', '.join(new_issue.labels))

        if src_ticket_version:
            if src_ticket_version == 'trunk' or src_ticket_version == 'dev':
                pass
            else:
                release_milestone_name = 'release-%s' % src_ticket_version
                if release_milestone_name not in milestone_map_id:
                    print("    creating new milestone for %s" % release_milestone_name)
                    new_milestone = Milestones(
                        title=release_milestone_name,
                        description='',
                        state='closed'
                    )
                    if method == 'direct':
                        new_milestone.project = dest_project_id
                    new_milestone = dest.create_milestone(dest_project_id, new_milestone)
                    milestone_map_id[release_milestone_name] = new_milestone.id
                new_issue.milestone = milestone_map_id[release_milestone_name]

        # Additional parameters for direct access
        if method == 'direct':
            new_issue.created_at = convert_xmlrpc_datetime(src_ticket[1])
            new_issue.updated_at = convert_xmlrpc_datetime(src_ticket[2])

            new_issue.project = dest_project_id
            new_issue.state = new_state

            try:
                new_issue.author = get_cached_user_id(dest, gitlab_user_cache, users_map[src_ticket_data['reporter']])
            except KeyError:
                if default_user:
                    new_issue.author = get_cached_user_id(dest, gitlab_user_cache, default_user)
                else:
                    raise
            if overwrite:
                new_issue.iid = src_ticket_id
            else:
                new_issue.iid = dest.get_issues_iid(dest_project_id)

        if 'milestone' in src_ticket_data and not new_issue.milestone:
            milestone = src_ticket_data['milestone']
            if milestone and milestone_map_id.get(milestone):
                new_issue.milestone = milestone_map_id[milestone]

        new_ticket = dest.create_issue(dest_project_id, new_issue)

        if src_ticket_data['owner'] != '':
            try:
                mapped_user = users_map[src_ticket_data['owner']]
            except KeyError:
                if default_user:
                    mapped_user = default_user
                else:
                    raise
            assign_query = IssueAssignees.insert(
                issue=new_ticket.id,
                user=get_cached_user_id(dest, gitlab_user_cache, mapped_user)
            )
            dest.assign_issue(assign_query)

        changelog = source.ticket.changeLog(src_ticket_id)
        is_attachment = False

        for change in changelog:
            (change_datetime, change_user, change_type, _, change_text, _) = change

            if change_type == "attachment":
                # The attachment will be described in the next change!
                is_attachment = True
                attachment_file_name = change_text

            if change_type == "comment" and (change_text != '' or is_attachment):
                note = Notes(
                    note=trac2down.convert(
                        fix_wiki_syntax(change_text),
                        '/issues/',
                        False,
                        issue_upload_prefix=issue_attachment_path
                    )
                )
                binary_attachment = None

                if method == 'direct':
                    note.created_at = convert_xmlrpc_datetime(change_datetime)
                    note.updated_at = convert_xmlrpc_datetime(change_datetime)
                    try:
                        user = users_map[change_user]
                        note.author = get_cached_user_id(dest, gitlab_user_cache, user)
                    except KeyError:
                        if default_user:
                            note.author = get_cached_user_id(dest, gitlab_user_cache, default_user)
                        else:
                            raise
                    if is_attachment:
                        # Intermediate save needed to make note.id be populated with the real ID of the record.
                        note.save()

                        note.attachment = '%s/%s' % (issue_attachment_path, attachment_file_name)
                        image_prefix = ''
                        if image_regexp.search(attachment_file_name):
                            image_prefix = '!'

                        attachment_label = note.note
                        if not attachment_label:
                            attachment_label = attachment_file_name

                        note.note = '%s[%s](/uploads/%s)' % (image_prefix, attachment_label, note.attachment)

                        print("    migrating attachment for ticket id %s: %s" % (src_ticket_id, attachment_file_name))
                        binary_attachment = source.ticket.getAttachment(src_ticket_id,
                                                                        attachment_file_name.encode('utf8')).data

                dest.comment_issue(dest_project_id, new_ticket, note, binary_attachment)
                is_attachment = False