Python import_version_specific_module 예제들

예제 #1

파일: binary_protocol.py 프로젝트: ZEMUSHKA/pydoop

#   http://www.apache.org/licenses/LICENSE-2.0
# 
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# 
# END_COPYRIGHT

# FIXME: not finished

import sys, os, subprocess

import pydoop
pp = pydoop.import_version_specific_module('_pipes')


#--- message codes for the down protocol ---
START_MESSAGE = 0
SET_JOB_CONF = 1
SET_INPUT_TYPES = 2
RUN_MAP = 3
MAP_ITEM = 4
RUN_REDUCE = 5
REDUCE_KEY = 6
REDUCE_VALUE = 7
CLOSE = 8
ABORT = 9

#--- message codes for the up protocol ---

예제 #2

파일: fs.py 프로젝트: onlynone/pydoop

# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
# END_COPYRIGHT
"""
pydoop.hdfs.fs -- File System Handles
-------------------------------------
"""

import os, socket, urlparse, getpass, re
import operator as ops

import pydoop
hdfs_ext = pydoop.import_version_specific_module("_hdfs")
import common
from file import hdfs_file, local_file


class _FSStatus(object):
    def __init__(self, fs, host, port, user, refcount=1):
        self.fs = fs
        self.host = host
        self.port = port
        self.user = user
        self.refcount = refcount

    def __repr__(self):
        return "_FSStatus(%s, %s)" % (self.fs, self.refcount)

예제 #3

# of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
# END_COPYRIGHT

import unittest

import pydoop
pp = pydoop.import_version_specific_module('_pipes')
from pydoop.pipes import Factory, Partitioner


def partition_function(k, n):
    return len(k) % n


class test_partitioner(Partitioner):
    def __init__(self, ctx):
        Partitioner.__init__(self)
        self.ctx = ctx
        self.counter = 0

    def partition(self, key, num_of_reduces):
        return partition_function(key, num_of_reduces)

예제 #4

파일: fs.py 프로젝트: jkahn/pydoop-code

# License for the specific language governing permissions and limitations
# under the License.
#
# END_COPYRIGHT

"""
pydoop.hdfs.fs -- File System Handles
-------------------------------------
"""

import os, socket, urlparse, getpass, re
import operator as ops

import pydoop

hdfs_ext = pydoop.import_version_specific_module("_hdfs")
import common
from file import hdfs_file, local_file


class _FSStatus(object):
    def __init__(self, fs, host, port, user, refcount=1):
        self.fs = fs
        self.host = host
        self.port = port
        self.user = user
        self.refcount = refcount

    def __repr__(self):
        return "_FSStatus(%s, %s)" % (self.fs, self.refcount)