def __init__(self, name):
CustomStorage.__init__(self)
self.db = mongoengine.connection._get_db(reconnect=False)
self.name = name
pid = self.db.rtreeseq.find_one({'_id' : self.name})
if not pid:
self.db.rtreeseq.insert({'_id' : self.name, "seq" : -1})
def __init__(self, name):
CustomStorage.__init__(self)
self.db = mongoengine.connection._get_db(reconnect=False)
self.name = name
pid = self.db.rtreeseq.find_one({'_id': self.name})
if not pid:
self.db.rtreeseq.insert({'_id': self.name, "seq": -1})
def __init__(self, file_prefix, loadexist=False, readonly=False):
CustomStorage.__init__(self)
if not loadexist:
if hdfs.path.exists('{0}_0'.format(file_prefix)):
file_prefix += '_0'
while hdfs.path.exists('{0}_0'.format(file_prefix)):
insert_index = file_prefix.rfind('_')
file_prefix = '{0}_{1}'.format(file_prefix[:insert_index], int(file_prefix[insert_index + 1:]) + 1)
self.file_prefix = file_prefix
self.read_only = readonly
self.clear()
logger.info('init hdfs storage from hdfs file_prefix {0}'.format(self.file_prefix))
try:
total_start = timeit.default_timer()
prefix_split = hdfs.path.splitpath(self.file_prefix)
folder_path = prefix_split[0]
real_prefix = prefix_split[1] + '_'
if not hdfs.path.exists(folder_path):
hdfs.mkdir(folder_path)
files_info = hdfs.lsl(folder_path)
# files_info = hdfs.lsl('{0}_*'.format(self.file_prefix))
logger.debug('files_info:{0}'.format(files_info))
sizecount = 0
for file_info in files_info:
start_time = timeit.default_timer()
file_name = hdfs.path.splitpath(file_info['path'])[1]
if file_name.startswith(real_prefix) and file_info['kind'] == 'file':
logger.debug('file info: {0}'.format(file_info))
page_id = file_name[len(real_prefix):]
if not page_id.isdigit():
continue
logger.debug('file {0} page id :{1}#'.format(file_info['path'],
page_id))
# if page_id.isdigit():
logger.info('load {0}# page file {1}'.format(page_id,
file_info['path']))
content = hdfs.load(file_info['path'], mode='r')
# logger.debug('{0}# page content:{1}'.format(page_id, content))
self.pagedict[int(page_id)] = content
logger.debug('{0}# page load complete'.format(page_id))
end_time = timeit.default_timer()
eval(generate_timer_log_str.format(
'load {0} {1} byte'.format(file_name, len(self.pagedict[int(page_id)])),
start_time,
end_time))
sizecount += len(self.pagedict[int(page_id)])
except IOError, ie:
logger.debug(traceback.format_exc())
def __init__(self, mapping, convertToInt = True):
CustomStorage.__init__( self )
self.mapping = mapping
self.blockWrites = False
self.convertToInt = convertToInt
def __init__(self):
CustomStorage.__init__( self )
self.clear()
self.print_stores = False
