コード例 #1
0
ファイル: discovery.py プロジェクト: GitHubTianPeng/101worker
def discoverNamespaceMember(namespace, member):
    if not member.decode("utf_8") in DumpdataProvider.getMembers(namespace):
        raise ResourceNotFoundException()

    response = {
        "folders": [],
        "files": [],
        "classifier": "Namespace member",
        "name": member,
        "github": DumpdataProvider.getGithub(namespace, member),
    }

    # gather wiki data
    wikiNS = wikifyNamespace(namespace)
    setWikidata(response, wikiNS, member)

    response["namespace"] = wikiNS

    # gather member data
    dirPath = os.path.join(namespace, member.replace(" ", "_"))
    files, dirs = DumpdataProvider.getDirContent(dirPath)

    for d in dirs:
        response["folders"].append({"resource": os.path.join(base_uri, dirPath, d), "classifier": "Folder", "name": d})

    for f in files:
        response["files"].append({"resource": os.path.join(base_uri, dirPath, f), "classifier": "File", "name": f})

    # response['endpoint'] = TripledataProvider.getEndpointLink(wikiNS, member)
    # response['sesame']   = TripledataProvider.getSesameLink(wikiNS, member)

    if namespace == "modules":
        response["module"] = DumpdataProvider.getModuleDescription(member)

    return response
コード例 #2
0
ファイル: discovery.py プロジェクト: GitHubTianPeng/101worker
def discoverMemberPath(namespace, member, path):
    response = {
        "folders": [],
        "files": [],
        "classifier": "Folder",
        "name": os.path.basename(path),
        "github": DumpdataProvider.getGithub(namespace, member),
    }

    # update github data
    if response["github"]:
        response["github"] = os.path.join(response["github"], path)

    # gather wiki data
    wikiNS = wikifyNamespace(namespace)
    setWikidata(response, wikiNS, member)

    response["namespace"] = wikiNS

    # gather member data
    dirPath = os.path.join(namespace, member, path)
    files, dirs = DumpdataProvider.getDirContent(dirPath)

    for d in dirs:
        response["folders"].append({"resource": os.path.join(base_uri, dirPath, d), "classifier": "Folder", "name": d})

    for f in files:
        response["files"].append({"resource": os.path.join(base_uri, dirPath, f), "classifier": "File", "name": f})

    # response['endpoint'] = TripledataProvider.getEndpointLink(wikiNS, member)
    # response['sesame']   = TripledataProvider.getSesameLink(wikiNS, member)

    return response
コード例 #3
0
ファイル: discovery.py プロジェクト: GitHubTianPeng/101worker
def discoverMemberFile(namespace, member, path, file):
    filePath = os.path.join(namespace, member, path, file)
    if not DumpdataProvider.exists(filePath):
        raise ResourceNotFoundException()

    # if no geshi code is defined, then we'll return basically "geshi : null" and nothing else
    locator, extractor, geshi, language = DumpdataProvider.getMetadata(filePath)

    response = {
        "geshi": geshi,
        "fragments": [],
        "classifier": "File",
        "name": file,
        "github": DumpdataProvider.getGithub(namespace, member),
    }

    if language:
        response["language"] = language

    # update github data
    if response["github"]:
        response["github"] = os.path.join(response["github"], path, file)

    # gather wiki data
    wikiNS = wikifyNamespace(namespace)
    setWikidata(response, wikiNS, member)

    response["namespace"] = wikiNS

    # gather member data - if there is a fact extractor, then we also want give back selectable fragments

    try:
        extractedFacts = DumpdataProvider.getFacts(filePath, extractor)
        print extractedFacts.get("fragments")
        for fragment in extractedFacts.get("fragments", []):
            print fragment
            fragmentPath = os.path.join(fragment["classifier"], fragment["name"])
            response["fragments"].append(mapFragment(filePath, fragmentPath, fragment))
    except OSError:
        pass

    # gather content - if there is a geshi code, we should be able to get content
    if geshi:
        response["content"] = DumpdataProvider.read(filePath)

    # commit infos
    setCommitInfos(response, filePath)

    # response['endpoint'] = TripledataProvider.getEndpointLink(wikiNS, member)
    # response['sesame']   = TripledataProvider.getSesameLink(wikiNS, member)

    response["derived"] = DumpdataProvider.getDerivedFiles(filePath)

    return response
コード例 #4
0
def discoverMemberFile(namespace, member, path, file):
    filePath = os.path.join(namespace, member, path, file)
    if not DumpdataProvider.exists(filePath):
        raise ResourceNotFoundException()

    #if no geshi code is defined, then we'll return basically "geshi : null" and nothing else
    locator, extractor, geshi, language = DumpdataProvider.getMetadata(filePath)

    response = {
        'geshi'     : geshi,
        'fragments' : [],
        'classifier': 'File',
        'name'      : file,
        'github'    : DumpdataProvider.getGithub(namespace,member)
    }

    if language:
        response['language'] = language

    #update github data
    if response['github']:
        response['github'] = os.path.join(response['github'], path, file)

    #gather wiki data
    wikiNS = wikifyNamespace(namespace)
    setWikidata(response, wikiNS, member)

    response['namespace'] = wikiNS

    #gather member data - if there is a fact extractor, then we also want give back selectable fragments

    try:
        extractedFacts = DumpdataProvider.getFacts(filePath, extractor)
        for fragment in extractedFacts.get('fragments', []):
            fragmentPath = os.path.join(fragment['classifier'], fragment['name'])
            response['fragments'].append( mapFragment(filePath, fragmentPath, fragment) )
    except OSError:
        pass


    #gather content - if there is a geshi code, we should be able to get content
    if geshi:
        response['content'] = DumpdataProvider.read(filePath)

    #commit infos
    setCommitInfos(response,filePath)

    #response['endpoint'] = TripledataProvider.getEndpointLink(wikiNS, member)
    # response['sesame']   = TripledataProvider.getSesameLink(wikiNS, member)

    response['derived'] = DumpdataProvider.getDerivedFiles(filePath)

    return response
コード例 #5
0
def discoverNamespaceMember(namespace, member):
    if not member.decode('utf_8') in DumpdataProvider.getMembers(namespace):
        raise ResourceNotFoundException()

    response = {
        'folders'   : [],
        'files'     : [],
        'classifier': 'Namespace member',
        'name'      : member,
        'github'    : DumpdataProvider.getGithub(namespace, member)
    }

    #gather wiki data
    wikiNS = wikifyNamespace(namespace)
    setWikidata(response, wikiNS, member)

    response['namespace'] = wikiNS

    #gather member data
    dirPath = os.path.join(namespace, member.replace(' ','_'))
    files, dirs = DumpdataProvider.getDirContent(dirPath)

    for d in dirs:
        response['folders'].append({
            'resource'  : os.path.join(base_uri, dirPath, d),
            'classifier': 'Folder',
            'name'      : d
        })

    for f in files:
        response['files'].append({
            'resource'  : os.path.join(base_uri, dirPath, f),
            'classifier': 'File',
            'name'      : f
            })

    #response['endpoint'] = TripledataProvider.getEndpointLink(wikiNS, member)
    # response['sesame']   = TripledataProvider.getSesameLink(wikiNS, member)

    if namespace == 'modules':
        response['module'] = DumpdataProvider.getModuleDescription(member)

    return response
コード例 #6
0
def discoverMemberPath(namespace, member, path):
    response = {
        'folders'   : [],
        'files'     : [],
        'classifier': 'Folder',
        'name'      : os.path.basename(path),
        'github'    : DumpdataProvider.getGithub(namespace, member)
    }

    #update github data
    if response['github']:
        response['github'] = os.path.join(response['github'], path)

    #gather wiki data
    wikiNS = wikifyNamespace(namespace)
    setWikidata(response, wikiNS, member)

    response['namespace'] = wikiNS

    #gather member data
    dirPath = os.path.join(namespace, member, path)
    files, dirs = DumpdataProvider.getDirContent(dirPath)

    for d in dirs:
        response['folders'].append({
            'resource': os.path.join(base_uri, dirPath, d),
            'classifier': 'Folder',
            'name'    : d
        })

    for f in files:
        response['files'].append({
            'resource': os.path.join(base_uri, dirPath, f),
            'classifier': 'File',
            'name'    : f,
        })

    #response['endpoint'] = TripledataProvider.getEndpointLink(wikiNS, member)
    # response['sesame']   = TripledataProvider.getSesameLink(wikiNS, member)

    return response
コード例 #7
0
ファイル: discovery.py プロジェクト: GitHubTianPeng/101worker
def discoverFileFragment(namespace, member, path, file, fragment):
    filePath = os.path.join(namespace, member, path, file)
    if not DumpdataProvider.exists(filePath):
        raise ResourceNotFoundException()

    # remove tailing slash, if there is one
    if fragment.endswith("/"):
        fragment = fragment[:-1]

    # if no geshi code is defined, then we'll return basically "geshi : null"
    locator, extractor, geshi, language = DumpdataProvider.getMetadata(filePath)

    # name and classifier are set later (in the extractor phase
    response = {"geshi": geshi, "fragments": [], "github": DumpdataProvider.getGithub(namespace, member)}

    if language:
        response["language"] = language

    # update github data
    if response["github"]:
        response["github"] = os.path.join(response["github"], path, file)

    # gather wiki data
    wikiNS = wikifyNamespace(namespace)
    setWikidata(response, wikiNS, member)

    response["namespace"] = wikiNS

    # gather member data
    lineNumbers = None
    print extractor
    # if extractor:
    try:
        extractedFacts = DumpdataProvider.getFacts(filePath, extractor)
        print extractedFacts
        # TODO There has to be a better way to do this
        for f1 in extractedFacts["fragments"]:
            selected, fragmentPath = find(f1, fragment)
            if selected:
                response["classifier"] = selected["classifier"]
                response["name"] = selected["name"]
                if "startLine" in selected:
                    lineNumbers = {"from": selected["startLine"], "to": selected["endLine"]}
                for f2 in selected.get("fragments", []):
                    response["fragments"].append(mapFragment(filePath, fragmentPath, f2))
                break
    except:
        pass

    # gather content
    if lineNumbers or locator:
        if not lineNumbers:
            lineNumbers = DumpdataProvider.getFragment(filePath, fragment, locator)

        fragmentText = DumpdataProvider.read(filePath, range(lineNumbers["from"] - 1, lineNumbers["to"]))
        response["content"] = fragmentText
        if response["github"]:
            response["github"] += "#L{0}-{1}".format(lineNumbers["from"], lineNumbers["to"])
    # except Exception as e:
    #    raise DiscoveryException('500 Internal Server Error', 'Fragment location failed:\n' + str(e))

    setCommitInfos(response, filePath)

    # response['endpoint'] = TripledataProvider.getEndpointLink(wikiNS, member)
    # response['sesame']   = TripledataProvider.getSesameLink(wikiNS, member)

    return response
コード例 #8
0
def discoverFileFragment(namespace, member, path, file, fragment):
    filePath = os.path.join(namespace, member, path, file)
    if not DumpdataProvider.exists(filePath):
        raise ResourceNotFoundException()

    #remove tailing slash, if there is one
    if fragment.endswith('/'):
        fragment = fragment[:-1]

    #if no geshi code is defined, then we'll return basically "geshi : null"
    locator, extractor, geshi, language = DumpdataProvider.getMetadata(filePath)

    #name and classifier are set later (in the extractor phase
    response = {
        'geshi'    : geshi,
        'fragments': [],
        'github'   : DumpdataProvider.getGithub(namespace,member)
    }

    if language:
        response['language'] = language

    #update github data
    if response['github']:
        response['github'] = os.path.join(response['github'], path, file)

    #gather wiki data
    wikiNS = wikifyNamespace(namespace)
    setWikidata(response, wikiNS, member)

    response['namespace'] = wikiNS

    #gather member data
    lineNumbers = None
    print extractor
    #if extractor:
    try:
        extractedFacts = DumpdataProvider.getFacts(filePath, extractor)
        print extractedFacts
        #TODO There has to be a better way to do this
        for f1 in extractedFacts['fragments']:
            selected, fragmentPath = find(f1, fragment)
            if selected:
                response['classifier'] = selected['classifier']
                response['name'] = selected['name']
                if 'startLine' in selected:
                    lineNumbers = {'from':selected['startLine'], 'to': selected['endLine']}
                for f2 in selected.get('fragments',[]):
                    response['fragments'].append(mapFragment(filePath, fragmentPath, f2))
                break
    except:
        pass

    #gather content
    if lineNumbers or locator:
        if not lineNumbers:
            lineNumbers = DumpdataProvider.getFragment(filePath, fragment, locator)

        fragmentText = DumpdataProvider.read(filePath, range(lineNumbers['from'] - 1, lineNumbers['to']))
        response['content'] = fragmentText
        if response['github']:
            response['github'] += '#L{0}-{1}'.format(lineNumbers['from'], lineNumbers['to'])
    #except Exception as e:
        #    raise DiscoveryException('500 Internal Server Error', 'Fragment location failed:\n' + str(e))

    setCommitInfos(response, filePath)

    #response['endpoint'] = TripledataProvider.getEndpointLink(wikiNS, member)
    # response['sesame']   = TripledataProvider.getSesameLink(wikiNS, member)

    return response