def __init__(self, conf, logger=None):
"""
constructor for account updater
:param conf: configuration of account-updater
"""
self.logger = logger or SimpleLogger(conf).get_logger_object()
Daemon.__init__(self, conf, self.logger)
libraryUtils.OSDLoggerImpl(
"account-updater-monitoring").initialize_logger()
create_recovery_file('account-updater-server')
self.conf = conf
self.__interval = int(conf.get('interval', 1800))
self.__ll_port = int(conf.get('llport', 61014))
self.__account_updater_port = int(\
conf.get('account_updater_port', 61009))
self.__service_id = gethostname() + "_" + str(self.__ll_port) + \
"_account-updater-server"
self.__param = self.__get_param(conf)
self.msg = GlobalVariables(self.logger)
self.msg.set_service_id(self.__service_id)
self.walker_map = WalkerMap()
self.reader_map = ReaderMap()
# Start sending health to local leader
self.logger.info("Loading health monitoring library")
self.health_instance = healthMonitoring(self.__get_node_ip\
(gethostname()), self.__account_updater_port, \
self.__ll_port, self.__service_id)
self.logger.info("Loaded health monitoring library")
remove_recovery_file('account-updater-server')
# load global map
if not self.msg.load_gl_map():
sys.exit(130)
self.logger.info("Account updater started")
def __init__(self, conf, logger):
libraryUtils.OSDLoggerImpl("object-library").initialize_logger()
self.logger = logger
self.filesystems = conf.get('filesystems', '/export')
self.disk_chunk_size = int(conf.get('disk_chunk_size', 65536))
self.keep_cache_size = int(conf.get('keep_cache_size', 5242880))
self.mount_check = config_true_value(conf.get('mount_check', 'true'))
self.reclaim_age = int(conf.get('reclaim_age', ONE_WEEK))
threads_for_read = int(conf.get('threads_for_read', '8'))
threads_for_write = int(conf.get('threads_for_write', '4'))
self.threadpools_write = defaultdict(
lambda: ThreadPool(nthreads=threads_for_write))
self.object_server_id = get_service_id('object')
self.object_lib = ObjectLibrary(threads_for_write, threads_for_read)
def __init__(self,
conf,
global_map_object,
service_id,
communicator_object,
Request_handler,
logger=None,
source_service_obj=None):
self.logger = logger or get_logger(conf,
log_route='container-recovery')
libraryUtils.OSDLoggerImpl("container-library").initialize_logger()
self.logger.info("running RecoveryMgr ")
self.communicator_object = communicator_object
self.Request_handler = Request_handler
self.conf = conf
self.__trans_port = int(conf['child_port'])
self.global_map_object = global_map_object
self.source_service_obj = source_service_obj
self.service_id = service_id
self.id = self.service_id.split('_')[0]
self.local_leader_id = self.service_id.split('_')[1]
self.__cont_journal_path = ''.join('.' + self.id +"_" +self.local_leader_id \
+ "_container_journal")
self.__trans_journal_path = ''.join('.' + self.id + "_" +self.local_leader_id \
+ "_transaction_journal")
self.__cont_path = EXPORT + self.__cont_journal_path
self.__trans_path = EXPORT + self.__trans_journal_path
self.logger.debug("Initiaization entries: %s %s %s %s %s %s " % \
(self.global_map_object, \
self.service_id, self.__cont_journal_path, \
self.__trans_journal_path, self.__cont_path, self.__trans_path))
#get object gl version from global leader
self._object_gl_version = self.__get_object_gl_version()
self.logger.debug("Object version received: %s" %
self._object_gl_version)
#Map from gl
self.map_gl = self.global_map_object.dest_comp_map()
self.logger.debug("Map from Gl Received %s" % self.map_gl)
#start heart beat
self.logger.debug("Made heart beat thread")
#self.heart_beat_thread.spawn(self.start_heart_beat, self.Request_handler)
self.heart_beat_gthread = eventlet.greenthread.spawn(
self.start_heart_beat, self.Request_handler)
self.logger.debug("Started heart beat thread func")
def __init__(self,
conf,
memcache=None,
logger=None,
account_ring=None,
container_ring=None,
object_ring=None):
if conf is None:
conf = {}
if logger is None:
self.logger = get_logger(conf, log_route='proxy-server')
else:
self.logger = logger
libraryUtils.OSDLoggerImpl("proxy-monitoring").initialize_logger()
create_recovery_file('proxy-server')
self.ongoing_operation_list = []
self.stop_service_flag = False
osd_dir = conf.get('osd_dir', '/export/.osd_meta_config')
static_config_file = conf.get('static_config_file', \
'/opt/HYDRAstor/objectStorage/configFiles/static_proxy-server.conf')
#swift_dir = conf.get('swift_dir', '/etc/swift')
static_conf = self.readconf(static_config_file)
self.logger.debug("Static config parameters:%s" % (static_conf))
self.node_timeout = int(conf.get('node_timeout', 10))
self.recoverable_node_timeout = int(
conf.get('recoverable_node_timeout', self.node_timeout))
self.conn_timeout = float(conf.get('conn_timeout', 0.5))
self.client_timeout = int(conf.get('client_timeout', 60))
self.put_queue_depth = int(static_conf.get('put_queue_depth', 5))
self.retry_count = int(static_conf.get('retry_count', 3))
self.request_retry_time_service_unavailable = int(\
static_conf.get('request_retry_time_service_unavailable', 100))
self.request_retry_time_component_transfer = int(\
static_conf.get('request_retry_time_component_transfer', 50))
self.object_chunk_size = int(conf.get('object_chunk_size', 65536))
self.client_chunk_size = int(conf.get('client_chunk_size', 65536))
self.trans_id_suffix = conf.get('trans_id_suffix', '')
self.post_quorum_timeout = float(conf.get('post_quorum_timeout', 0.5))
#self.error_suppression_interval = \
# int(conf.get('error_suppression_interval', 60))
#self.error_suppression_limit = \
# int(conf.get('error_suppression_limit', 10))
self.recheck_container_existence = \
int(conf.get('recheck_container_existence', 60))
self.recheck_account_existence = \
int(conf.get('recheck_account_existence', 60))
self.allow_account_management = \
config_true_value(static_conf.get('allow_account_management', 'no'))
self.object_post_as_copy = \
config_true_value(conf.get('object_post_as_copy', 'false'))
self.object_ring = object_ring or ObjectRing(osd_dir, self.logger, \
self.node_timeout)
self.container_ring = container_ring or \
ContainerRing(osd_dir, self.logger, self.node_timeout)
self.account_ring = account_ring or AccountRing(osd_dir, self.logger, \
self.node_timeout)
self.memcache = memcache
mimetypes.init(mimetypes.knownfiles +
[os.path.join(osd_dir, 'mime.types')])
self.account_autocreate = \
config_true_value(static_conf.get('account_autocreate', 'yes'))
#self.expiring_objects_account = \
# (conf.get('auto_create_account_prefix') or '.') + \
# (conf.get('expiring_objects_account_name') or 'expiring_objects')
#self.expiring_objects_container_divisor = \
# int(conf.get('expiring_objects_container_divisor') or 86400)
self.max_containers_per_account = \
int(conf.get('max_containers_per_account') or 10000000)
self.max_containers_whitelist = [
a.strip()
for a in conf.get('max_containers_whitelist', '').split(',')
if a.strip()
]
self.deny_host_headers = [
host.strip()
for host in conf.get('deny_host_headers', '').split(',')
if host.strip()
]
#self.rate_limit_after_segment = \
# int(conf.get('rate_limit_after_segment', 10))
#self.rate_limit_segments_per_sec = \
# int(conf.get('rate_limit_segments_per_sec', 1))
#self.log_handoffs = config_true_value(conf.get('log_handoffs', 'true'))
#self.cors_allow_origin = [
# a.strip()
# for a in conf.get('cors_allow_origin', '').split(',')
# if a.strip()]
#self.strict_cors_mode = config_true_value(
# conf.get('strict_cors_mode', 't'))
self.node_timings = {}
self.timing_expiry = int(conf.get('timing_expiry', 300))
#self.sorting_method = conf.get('sorting_method', 'shuffle').lower()
#self.max_large_object_get_time = float(
# conf.get('max_large_object_get_time', '86400'))
#value = conf.get('request_node_count', '2 * replicas').lower().split()
#if len(value) == 1:
# value = int(value[0])
# self.request_node_count = lambda replicas: value
#elif len(value) == 3 and value[1] == '*' and value[2] == 'replicas':
# value = int(value[0])
# self.request_node_count = lambda replicas: value * replicas
#else:
# raise ValueError(
# 'Invalid request_node_count value: %r' % ''.join(value))
#try:
# self._read_affinity = read_affinity = conf.get('read_affinity', '')
# self.read_affinity_sort_key = affinity_key_function(read_affinity)
#except ValueError as err:
# # make the message a little more useful
# raise ValueError("Invalid read_affinity value: %r (%s)" %
# (read_affinity, err.message))
#try:
# write_affinity = conf.get('write_affinity', '')
# self.write_affinity_is_local_fn \
# = affinity_locality_predicate(write_affinity)
#except ValueError as err:
# # make the message a little more useful
# raise ValueError("Invalid write_affinity value: %r (%s)" %
# (write_affinity, err.message))
#value = conf.get('write_affinity_node_count',
# '2 * replicas').lower().split()
#if len(value) == 1:
# value = int(value[0])
# self.write_affinity_node_count = lambda replicas: value
#elif len(value) == 3 and value[1] == '*' and value[2] == 'replicas':
# value = int(value[0])
# self.write_affinity_node_count = lambda replicas: value * replicas
#else:
# raise ValueError(
# 'Invalid write_affinity_node_count value: %r' % ''.join(value))
# swift_owner_headers are stripped by the account and container
# controllers; we should extend header stripping to object controller
# when a privileged object header is implemented.
swift_owner_headers = conf.get(
'swift_owner_headers', 'x-container-read, x-container-write, '
'x-container-sync-key, x-container-sync-to, '
'x-account-meta-temp-url-key, x-account-meta-temp-url-key-2, '
'x-account-access-control')
self.swift_owner_headers = [
name.strip().title() for name in swift_owner_headers.split(',')
if name.strip()
]
# Initialization was successful, so now apply the client chunk size
# parameter as the default read / write buffer size for the network
# sockets.
#
# NOTE WELL: This is a class setting, so until we get set this on a
# per-connection basis, this affects reading and writing on ALL
# sockets, those between the proxy servers and external clients, and
# those between the proxy servers and the other internal servers.
#
# ** Because it affects the client as well, currently, we use the
# client chunk size as the govenor and not the object chunk size.
socket._fileobject.default_bufsize = self.client_chunk_size
self.expose_info = config_true_value(conf.get('expose_info', 'yes'))
self.disallowed_sections = list_from_csv(
conf.get('disallowed_sections'))
self.admin_key = conf.get('admin_key', None)
register_osd_info(
max_file_size=constraints.MAX_FILE_SIZE,
max_meta_name_length=constraints.MAX_META_NAME_LENGTH,
max_meta_value_length=constraints.MAX_META_VALUE_LENGTH,
max_meta_count=constraints.MAX_META_COUNT,
account_listing_limit=constraints.ACCOUNT_LISTING_LIMIT,
container_listing_limit=constraints.CONTAINER_LISTING_LIMIT,
max_account_name_length=constraints.MAX_ACCOUNT_NAME_LENGTH,
max_container_name_length=constraints.MAX_CONTAINER_NAME_LENGTH,
max_object_name_length=constraints.MAX_OBJECT_NAME_LENGTH,
non_allowed_headers=constraints.NON_ALLOWED_HEADERS)
self.proxy_port = int(static_conf.get('bind_port', 61005))
self.__ll_port = int(conf.get('llport', 61014))
self.max_bulk_delete_entries = int(conf.get(\
'max_bulk_delete_entries', 1000))
#unblock new requests which was blocked due to proxy service stop
self.__request_unblock()
hostname = socket.gethostname()
self.__server_id = hostname + "_" + str(
self.__ll_port) + "_proxy-server"
# Start sending health to local leader
self.logger.info("Loading health monitoring library")
self.health_instance = healthMonitoring(self.__get_node_ip(hostname), \
self.proxy_port, self.__ll_port, self.__server_id, True)
self.logger.info("Loaded health monitoring library")
remove_recovery_file('proxy-server')
from libContainerLib import ContainerStat, LibraryImpl, ObjectRecord, ListObjectWithStatus, ContainerStatWithStatus, Status, TransferObjectRecord
import libContainerLib
from osd.containerService.utils import AsyncLibraryHelper
import time
import libraryUtils
from subprocess import check_call
from osd.common.swob import Request, Response
import libTransactionLib #account lib path
import string
import shutil
import threading
from threading import Thread
from osd.common.utils import get_logger
import time
import subprocess
libraryUtils.OSDLoggerImpl("container-library").initialize_logger()
from subprocess import check_call
path = os.getcwd()
g = 0 #global value
id = 72057594037927936
if not os.path.exists("./JournalDir"):
os.mkdir("./JournalDir")
#transObj = libTransactionLib.TransactionLibrary('./JournalDir',1)
#compo = libContainerLib.list_less__component_names__greater_()
#compo = libTransactionLib.list_less__recordStructure_scope_ContainerRecord__greater_()
#compo.append(1)
#transObj.start_recovery(compo)