Пример #1
0
def discoverNamespaceMember(namespace, member):
    if not member.decode("utf_8") in DumpdataProvider.getMembers(namespace):
        raise ResourceNotFoundException()

    response = {
        "folders": [],
        "files": [],
        "classifier": "Namespace member",
        "name": member,
        "github": DumpdataProvider.getGithub(namespace, member),
    }

    # gather wiki data
    wikiNS = wikifyNamespace(namespace)
    setWikidata(response, wikiNS, member)

    response["namespace"] = wikiNS

    # gather member data
    dirPath = os.path.join(namespace, member.replace(" ", "_"))
    files, dirs = DumpdataProvider.getDirContent(dirPath)

    for d in dirs:
        response["folders"].append({"resource": os.path.join(base_uri, dirPath, d), "classifier": "Folder", "name": d})

    for f in files:
        response["files"].append({"resource": os.path.join(base_uri, dirPath, f), "classifier": "File", "name": f})

    # response['endpoint'] = TripledataProvider.getEndpointLink(wikiNS, member)
    # response['sesame']   = TripledataProvider.getSesameLink(wikiNS, member)

    if namespace == "modules":
        response["module"] = DumpdataProvider.getModuleDescription(member)

    return response
Пример #2
0
def discoverNamespace(namespace):
    if not wikifyNamespace(namespace) in DumpdataProvider.getMembers("") and not wikifyNamespace(namespace) == "":
        raise ResourceNotFoundException()

    response = {
        "classifier": "Namespace",
        "name": namespace,
        "members": [],
    }  # , 'github': DumpdataProvider.getGithub(namespace, '')}

    # gather wiki data
    wikiNS = wikifyNamespace(namespace)
    setWikidata(response, "Namespace", wikiNS)

    response["namespace"] = "Namespace"

    # gather member data
    members = DumpdataProvider.getMembers(namespace)
    for member in members:
        response["members"].append(
            {
                "resource": os.path.join(base_uri, namespace, member).replace(" ", "_"),
                "classifier": "Namespace member",
                "name": member,
            }
        )

    return response
Пример #3
0
def discoverNamespace(namespace):
    if not wikifyNamespace(namespace) in DumpdataProvider.getMembers('') and not wikifyNamespace(namespace) == '':
        raise ResourceNotFoundException()

    response = {'classifier': 'Namespace', 'name': namespace, 'members': []}#, 'github': DumpdataProvider.getGithub(namespace, '')}

    #gather wiki data
    wikiNS = wikifyNamespace(namespace)
    setWikidata(response,'Namespace', wikiNS)

    response['namespace'] = 'Namespace'

    #gather member data
    members = DumpdataProvider.getMembers(namespace)
    for member in members:
        response['members'].append({
            'resource': os.path.join(base_uri, namespace, member).replace(' ', '_'),
            'classifier': 'Namespace member',
            'name'    : member
        })

    #response['endpoint'] = TripledataProvider.getEndpointLink('Namespace', wikiNS)
    #response['sesame']   = TripledataProvider.getSesameLink('namespaces', wikiNS)

    return response
Пример #4
0
def discoverMemberPath(namespace, member, path):
    response = {
        "folders": [],
        "files": [],
        "classifier": "Folder",
        "name": os.path.basename(path),
        "github": DumpdataProvider.getGithub(namespace, member),
    }

    # update github data
    if response["github"]:
        response["github"] = os.path.join(response["github"], path)

    # gather wiki data
    wikiNS = wikifyNamespace(namespace)
    setWikidata(response, wikiNS, member)

    response["namespace"] = wikiNS

    # gather member data
    dirPath = os.path.join(namespace, member, path)
    files, dirs = DumpdataProvider.getDirContent(dirPath)

    for d in dirs:
        response["folders"].append({"resource": os.path.join(base_uri, dirPath, d), "classifier": "Folder", "name": d})

    for f in files:
        response["files"].append({"resource": os.path.join(base_uri, dirPath, f), "classifier": "File", "name": f})

    # response['endpoint'] = TripledataProvider.getEndpointLink(wikiNS, member)
    # response['sesame']   = TripledataProvider.getSesameLink(wikiNS, member)

    return response
Пример #5
0
def discoverNamespaceMember(namespace, member):
    if not member.decode('utf_8') in DumpdataProvider.getMembers(namespace):
        raise ResourceNotFoundException()

    response = {
        'folders'   : [],
        'files'     : [],
        'classifier': 'Namespace member',
        'name'      : member,
        'github'    : DumpdataProvider.getGithub(namespace, member)
    }

    #gather wiki data
    wikiNS = wikifyNamespace(namespace)
    setWikidata(response, wikiNS, member)

    response['namespace'] = wikiNS

    #gather member data
    dirPath = os.path.join(namespace, member.replace(' ','_'))
    files, dirs = DumpdataProvider.getDirContent(dirPath)

    for d in dirs:
        response['folders'].append({
            'resource'  : os.path.join(base_uri, dirPath, d),
            'classifier': 'Folder',
            'name'      : d
        })

    for f in files:
        response['files'].append({
            'resource'  : os.path.join(base_uri, dirPath, f),
            'classifier': 'File',
            'name'      : f
            })

    #response['endpoint'] = TripledataProvider.getEndpointLink(wikiNS, member)
    # response['sesame']   = TripledataProvider.getSesameLink(wikiNS, member)

    if namespace == 'modules':
        response['module'] = DumpdataProvider.getModuleDescription(member)

    return response
Пример #6
0
def discoverMemberPath(namespace, member, path):
    response = {
        'folders'   : [],
        'files'     : [],
        'classifier': 'Folder',
        'name'      : os.path.basename(path),
        'github'    : DumpdataProvider.getGithub(namespace, member)
    }

    #update github data
    if response['github']:
        response['github'] = os.path.join(response['github'], path)

    #gather wiki data
    wikiNS = wikifyNamespace(namespace)
    setWikidata(response, wikiNS, member)

    response['namespace'] = wikiNS

    #gather member data
    dirPath = os.path.join(namespace, member, path)
    files, dirs = DumpdataProvider.getDirContent(dirPath)

    for d in dirs:
        response['folders'].append({
            'resource': os.path.join(base_uri, dirPath, d),
            'classifier': 'Folder',
            'name'    : d
        })

    for f in files:
        response['files'].append({
            'resource': os.path.join(base_uri, dirPath, f),
            'classifier': 'File',
            'name'    : f,
        })

    #response['endpoint'] = TripledataProvider.getEndpointLink(wikiNS, member)
    # response['sesame']   = TripledataProvider.getSesameLink(wikiNS, member)

    return response
Пример #7
0
def setCommitInfos(response, filePath):
    # TODO update, just a rudimentary implementation
    initiator, contributors = DumpdataProvider.getCommitInfo(filePath)
    if initiator:
        response["people"] = []
        response["people"].append({"name": initiator["author"], "role": "Initiator?"})
    if contributors:
        contribList = []
        for contributor in contributors:
            if contributor["author"] not in contribList and not contributor["author"] == initiator["author"]:
                contribList.append(contributor["author"])

        for c in contribList:
            response["people"].append({"name": c, "role": None})
Пример #8
0
def setCommitInfos(response, filePath):
    #TODO update, just a rudimentary implementation
    initiator, contributors = DumpdataProvider.getCommitInfo(filePath)
    if initiator:
        response['people'] = []
        response['people'].append({'name' : initiator['author'], 'role' : 'Initiator?'})
    if contributors:
        contribList = []
        for contributor in contributors:
            if contributor['author'] not in contribList and not contributor['author'] == initiator['author']:
                contribList.append(contributor['author'])

        for c in contribList:
            response['people'].append({'name' : c, 'role' : None})
Пример #9
0
def discoverMemberFile(namespace, member, path, file):
    filePath = os.path.join(namespace, member, path, file)
    if not DumpdataProvider.exists(filePath):
        raise ResourceNotFoundException()

    # if no geshi code is defined, then we'll return basically "geshi : null" and nothing else
    locator, extractor, geshi, language = DumpdataProvider.getMetadata(filePath)

    response = {
        "geshi": geshi,
        "fragments": [],
        "classifier": "File",
        "name": file,
        "github": DumpdataProvider.getGithub(namespace, member),
    }

    if language:
        response["language"] = language

    # update github data
    if response["github"]:
        response["github"] = os.path.join(response["github"], path, file)

    # gather wiki data
    wikiNS = wikifyNamespace(namespace)
    setWikidata(response, wikiNS, member)

    response["namespace"] = wikiNS

    # gather member data - if there is a fact extractor, then we also want give back selectable fragments

    try:
        extractedFacts = DumpdataProvider.getFacts(filePath, extractor)
        print extractedFacts.get("fragments")
        for fragment in extractedFacts.get("fragments", []):
            print fragment
            fragmentPath = os.path.join(fragment["classifier"], fragment["name"])
            response["fragments"].append(mapFragment(filePath, fragmentPath, fragment))
    except OSError:
        pass

    # gather content - if there is a geshi code, we should be able to get content
    if geshi:
        response["content"] = DumpdataProvider.read(filePath)

    # commit infos
    setCommitInfos(response, filePath)

    # response['endpoint'] = TripledataProvider.getEndpointLink(wikiNS, member)
    # response['sesame']   = TripledataProvider.getSesameLink(wikiNS, member)

    response["derived"] = DumpdataProvider.getDerivedFiles(filePath)

    return response
Пример #10
0
def serveMemberPath(request, namespace, member, path):

    #needed for strange files, that have no . ! e.g. "Makefile"
    #This is also the reason why serveMemberPath doesn't need a check if a path exists, because non existing paths will
    #be classified as files first!!!
    if not DumpdataProvider.isDir(os.path.join(namespace, member, path)):
        path = os.path.dirname(path)
        file = os.path.basename(path)
        return serveMemberFile(request, namespace, member, path, file)

    response = discovery.discoverMemberPath(namespace, member,
                                            path)

    if request.GET.get('validate', None):
        validate(response, json.load(open(os.path.abspath('../../schemas/101repo_folder.json'), 'r')))

    return format_render(request, response, 'folder')
Пример #11
0
def discoverMemberFile(namespace, member, path, file):
    filePath = os.path.join(namespace, member, path, file)
    if not DumpdataProvider.exists(filePath):
        raise ResourceNotFoundException()

    #if no geshi code is defined, then we'll return basically "geshi : null" and nothing else
    locator, extractor, geshi, language = DumpdataProvider.getMetadata(filePath)

    response = {
        'geshi'     : geshi,
        'fragments' : [],
        'classifier': 'File',
        'name'      : file,
        'github'    : DumpdataProvider.getGithub(namespace,member)
    }

    if language:
        response['language'] = language

    #update github data
    if response['github']:
        response['github'] = os.path.join(response['github'], path, file)

    #gather wiki data
    wikiNS = wikifyNamespace(namespace)
    setWikidata(response, wikiNS, member)

    response['namespace'] = wikiNS

    #gather member data - if there is a fact extractor, then we also want give back selectable fragments

    try:
        extractedFacts = DumpdataProvider.getFacts(filePath, extractor)
        for fragment in extractedFacts.get('fragments', []):
            fragmentPath = os.path.join(fragment['classifier'], fragment['name'])
            response['fragments'].append( mapFragment(filePath, fragmentPath, fragment) )
    except OSError:
        pass


    #gather content - if there is a geshi code, we should be able to get content
    if geshi:
        response['content'] = DumpdataProvider.read(filePath)

    #commit infos
    setCommitInfos(response,filePath)

    #response['endpoint'] = TripledataProvider.getEndpointLink(wikiNS, member)
    # response['sesame']   = TripledataProvider.getSesameLink(wikiNS, member)

    response['derived'] = DumpdataProvider.getDerivedFiles(filePath)

    return response
Пример #12
0
def serveMemberPath(request, namespace, member, path):

    #needed for strange files, that have no . ! e.g. "Makefile"
    #This is also the reason why serveMemberPath doesn't need a check if a path exists, because non existing paths will
    #be classified as files first!!!
    if not DumpdataProvider.isDir(os.path.join(namespace, member, path)):
        path = os.path.dirname(path)
        file = os.path.basename(path)
        return serveMemberFile(request, namespace, member, path, file)

    response = discovery.discoverMemberPath(namespace, member, path)

    if request.GET.get('validate', None):
        validate(
            response,
            json.load(
                open(os.path.abspath('../../schemas/101repo_folder.json'),
                     'r')))

    return format_render(request, response, 'folder')
Пример #13
0
def discoverAllNamespaces():
    response = {'classifier': 'Namespace', 'name': 'Namespace', 'members': []}#, 'github': DumpdataProvider.getGithub('', '')}

    #gather wiki data
    setWikidata(response,'Namespace','Namespace')

    response['namespace'] = 'Namespace'

    #gather member data
    members = DumpdataProvider.getMembers('')
    for member in members:
        response['members'].append({
            'resource'  : os.path.join(base_uri, dewikifyNamespace(member)).replace(' ', '_'),
            'classifier': 'Namespace',
            'name'      : member
        })

    #response['endpoint'] = TripledataProvider.getEndpointLink('Namespace', 'Namespace')
    #response['sesame']   = TripledataProvider.getSesameLink('namespaces', 'Namespace')

    return response
Пример #14
0
def discoverAllNamespaces():
    response = {
        "classifier": "Namespace",
        "name": "Namespace",
        "members": [],
    }  # , 'github': DumpdataProvider.getGithub('', '')}

    # gather wiki data
    setWikidata(response, "Namespace", "Namespace")

    response["namespace"] = "Namespace"

    # gather member data
    members = DumpdataProvider.getMembers("")
    for member in members:
        response["members"].append(
            {
                "resource": os.path.join(base_uri, dewikifyNamespace(member)).replace(" ", "_"),
                "classifier": "Namespace",
                "name": member,
            }
        )

    return response
Пример #15
0
def discoverFileFragment(namespace, member, path, file, fragment):
    filePath = os.path.join(namespace, member, path, file)
    if not DumpdataProvider.exists(filePath):
        raise ResourceNotFoundException()

    # remove tailing slash, if there is one
    if fragment.endswith("/"):
        fragment = fragment[:-1]

    # if no geshi code is defined, then we'll return basically "geshi : null"
    locator, extractor, geshi, language = DumpdataProvider.getMetadata(filePath)

    # name and classifier are set later (in the extractor phase
    response = {"geshi": geshi, "fragments": [], "github": DumpdataProvider.getGithub(namespace, member)}

    if language:
        response["language"] = language

    # update github data
    if response["github"]:
        response["github"] = os.path.join(response["github"], path, file)

    # gather wiki data
    wikiNS = wikifyNamespace(namespace)
    setWikidata(response, wikiNS, member)

    response["namespace"] = wikiNS

    # gather member data
    lineNumbers = None
    print extractor
    # if extractor:
    try:
        extractedFacts = DumpdataProvider.getFacts(filePath, extractor)
        print extractedFacts
        # TODO There has to be a better way to do this
        for f1 in extractedFacts["fragments"]:
            selected, fragmentPath = find(f1, fragment)
            if selected:
                response["classifier"] = selected["classifier"]
                response["name"] = selected["name"]
                if "startLine" in selected:
                    lineNumbers = {"from": selected["startLine"], "to": selected["endLine"]}
                for f2 in selected.get("fragments", []):
                    response["fragments"].append(mapFragment(filePath, fragmentPath, f2))
                break
    except:
        pass

    # gather content
    if lineNumbers or locator:
        if not lineNumbers:
            lineNumbers = DumpdataProvider.getFragment(filePath, fragment, locator)

        fragmentText = DumpdataProvider.read(filePath, range(lineNumbers["from"] - 1, lineNumbers["to"]))
        response["content"] = fragmentText
        if response["github"]:
            response["github"] += "#L{0}-{1}".format(lineNumbers["from"], lineNumbers["to"])
    # except Exception as e:
    #    raise DiscoveryException('500 Internal Server Error', 'Fragment location failed:\n' + str(e))

    setCommitInfos(response, filePath)

    # response['endpoint'] = TripledataProvider.getEndpointLink(wikiNS, member)
    # response['sesame']   = TripledataProvider.getSesameLink(wikiNS, member)

    return response
Пример #16
0
def discoverFileFragment(namespace, member, path, file, fragment):
    filePath = os.path.join(namespace, member, path, file)
    if not DumpdataProvider.exists(filePath):
        raise ResourceNotFoundException()

    #remove tailing slash, if there is one
    if fragment.endswith('/'):
        fragment = fragment[:-1]

    #if no geshi code is defined, then we'll return basically "geshi : null"
    locator, extractor, geshi, language = DumpdataProvider.getMetadata(filePath)

    #name and classifier are set later (in the extractor phase
    response = {
        'geshi'    : geshi,
        'fragments': [],
        'github'   : DumpdataProvider.getGithub(namespace,member)
    }

    if language:
        response['language'] = language

    #update github data
    if response['github']:
        response['github'] = os.path.join(response['github'], path, file)

    #gather wiki data
    wikiNS = wikifyNamespace(namespace)
    setWikidata(response, wikiNS, member)

    response['namespace'] = wikiNS

    #gather member data
    lineNumbers = None
    print extractor
    #if extractor:
    try:
        extractedFacts = DumpdataProvider.getFacts(filePath, extractor)
        print extractedFacts
        #TODO There has to be a better way to do this
        for f1 in extractedFacts['fragments']:
            selected, fragmentPath = find(f1, fragment)
            if selected:
                response['classifier'] = selected['classifier']
                response['name'] = selected['name']
                if 'startLine' in selected:
                    lineNumbers = {'from':selected['startLine'], 'to': selected['endLine']}
                for f2 in selected.get('fragments',[]):
                    response['fragments'].append(mapFragment(filePath, fragmentPath, f2))
                break
    except:
        pass

    #gather content
    if lineNumbers or locator:
        if not lineNumbers:
            lineNumbers = DumpdataProvider.getFragment(filePath, fragment, locator)

        fragmentText = DumpdataProvider.read(filePath, range(lineNumbers['from'] - 1, lineNumbers['to']))
        response['content'] = fragmentText
        if response['github']:
            response['github'] += '#L{0}-{1}'.format(lineNumbers['from'], lineNumbers['to'])
    #except Exception as e:
        #    raise DiscoveryException('500 Internal Server Error', 'Fragment location failed:\n' + str(e))

    setCommitInfos(response, filePath)

    #response['endpoint'] = TripledataProvider.getEndpointLink(wikiNS, member)
    # response['sesame']   = TripledataProvider.getSesameLink(wikiNS, member)

    return response