Esempio n. 1
0
def shard_names(spec):
  if '@' in spec:
    fname, shards = spec.split('@', 1)
    shards = int(shards)
    return [base.filename_for_shard(fname, shards, shard) for shard in range(shards)]
  else:
    return [spec]
Esempio n. 2
0
 def secondary_store(self, base_path, mode='r', compression=None):
   """Return a store that mimics the sharding of the input store and can be used as as secondary output/input."""
   if '@' in base_path:
     base_path, sharding = base_path.split('@')
     if int(sharding) != self.shards:
       raise StandardError('secondary_store sharding should be the same as input sharding. (%s vs %s' %
                           (int(sharding), self.shards))
   return store.SingleStore(base.filename_for_shard(base_path, self.shards, self.task_id),
                            mode=mode, compression=compression) 
Esempio n. 3
0
 def __init__(self, fname, shards=None, mode='r', DataStoreType=SingleStore, compression=None, buffering=-1):
   self._shards = []
   if shards is None:
     if '@' in fname:
       basename, shards = fname.split('@', 1)
       shards = int(shards)
     else:
       basename = fname
       if not os.path.exists(basename):
         if basename.endswith('-'):  # could be of shell expansion
           basename = basename[:-1]
         existing_file = glob.glob(basename +'-????-????.dst')
         if existing_file:
           shards = int(existing_file[0][len(basename) + 6:len(basename) + 10])
         else:
           basename = fname
   if shards is None:
     self._shards = [DataStoreType(basename, mode=mode, compression=compression, buffering=-1)]
   else:
     self._shards = [DataStoreType(base.filename_for_shard(basename, shards, shard),
                                   mode=mode,
                                   compression=compression,
                                   buffering=-1)
                     for shard in range(shards)]