class MigrationManager(object):
def __init__(self, config: MigrationConfig):
self._config = config
_uuid = str(uuid.uuid4())
self._id = ''.join(_uuid.split('-'))
self._es = Elasticsearch(
hosts=self._config.hosts,
timeout=60,
max_retries=30,
retry_on_timeout=True,
verify_certs=False,
)
self._log_client = LogClient(
endpoint=self._config.get('endpoint'),
accessKeyId=self._config.access_key_id,
accessKey=self._config.access_key,
)
setup_logging(
self._id,
self._config.get('endpoint'),
self._config.get('project_name'),
self._config.access_key_id,
self._config.access_key,
)
self._logger = logging.getLogger(__name__)
self._shutdown_flag = op.join(self._config.cache_path, 'shutdown.lock')
print('#migration: {}'.format(self._id))
def migrate(self):
self._logger.info('Migration starts')
tasks = self._discover_tasks()
task_cnt = len(tasks)
pool_size = max(1, min(self._config.get('pool_size'), task_cnt))
print('#pool_size: {}'.format(pool_size))
print('#tasks: {}'.format(task_cnt))
self._prepare()
futures = []
state = {
'total': task_cnt,
Checkpoint.finished: 0,
Checkpoint.dropped: 0,
Checkpoint.failed: 0,
}
with ProcessPoolExecutor(max_workers=pool_size) as pool:
for task in tasks:
futures.append(
pool.submit(
_migration_worker,
self._config,
task,
self._shutdown_flag,
))
try:
for future in as_completed(futures):
res = future.result()
if res in state:
state[res] += 1
self._logger.info('State', extra=state)
print('>> state:', json.dumps(state))
except BaseException:
self._logger.error(
'Exception',
extra={'traceback': traceback.format_exc()},
)
for future in futures:
if not future.done():
future.cancel()
list(as_completed(futures, timeout=10))
if state[Checkpoint.finished] + state[Checkpoint.dropped] >= task_cnt:
self._logger.info('All migration tasks finished')
self._logger.info('Migration exits')
print('exit:', json.dumps(state))
return state
def _prepare(self):
if op.exists(self._shutdown_flag):
os.unlink(self._shutdown_flag)
def _handle_term_sig(signum, frame):
# Raise Ctrl+C
with open(self._shutdown_flag, 'w') as f:
f.write('')
raise KeyboardInterrupt()
signal.signal(signal.SIGINT, _handle_term_sig)
signal.signal(signal.SIGTERM, _handle_term_sig)
def _discover_tasks(self):
indexes = self._config.get('indexes')
data = self._es.search_shards(indexes)
tasks = []
for shard in data['shards']:
for item in shard:
# Ignore internal index
if not indexes and item['index'].startswith('.'):
continue
if item['state'] == 'STARTED' and item['primary']:
tasks.append(
{
'es_index': item['index'],
'es_shard': item['shard']
}, )
return self._handle_cache(tasks)
def _handle_cache(self, tasks):
file_tasks = op.join(self._config.cache_path, 'tasks.json')
if op.exists(file_tasks):
with open(file_tasks) as f:
cont = f.read()
else:
cont = '[]'
try:
old_tasks = json.loads(cont)
except json.JSONDecodeError:
self._logger.error('Invalid task cache', extra={'cache': cont})
old_tasks = []
task_map = {(task['es_index'], task['es_shard']): task['id']
for task in old_tasks}
_mappings = IndexLogstoreMappings(
list([task['es_index'] for task in tasks]),
self._config.get('logstore_index_mappings'),
)
cnt, new_tasks = len(old_tasks), []
for task in tasks:
_task = (task['es_index'], task['es_shard'])
if _task not in task_map:
task['id'] = cnt
task['logstore'] = _mappings.get_logstore(task['es_index'])
new_tasks.append(task)
cnt += 1
tasks = old_tasks + new_tasks
with open(file_tasks, 'w') as f:
f.write(json.dumps(tasks, indent=2))
if self._config.get('auto_creation'):
self._setup_aliyun_log(_mappings)
return tasks
def _setup_aliyun_log(self, index_logstore_mappings):
print('setup aliyun log service...')
self._logger.info('Setup AliyunLog start')
logstores = index_logstore_mappings.get_all_logstores()
for logstore in logstores:
self._setup_logstore(index_logstore_mappings, logstore)
self._logger.info('Setup AliyunLog wait')
time.sleep(self._config.get('wait_time_in_secs'))
self._logger.info('Setup AliyunLog finish')
def _setup_logstore(self, index_logstore_mappings, logstore):
try:
self._log_client.create_logstore(
project_name=self._config.get('project_name'),
logstore_name=logstore,
shard_count=8,
ttl=3650,
)
except LogException as exc:
if exc.get_error_code() == 'LogStoreAlreadyExist':
self._logger.info(
'Logstore already exist, skip creation.',
extra={'logstore': logstore},
)
else:
raise
self._setup_index(index_logstore_mappings, logstore)
def _setup_index(self, index_logstore_mappings, logstore):
indexes = index_logstore_mappings.get_indexes(logstore)
for index in indexes:
self._logger.info(
'Setup AliyunLog Logstore',
extra={
'logstore': logstore,
'es_index': index
},
)
try:
resp = self._es.indices.get(index=index)
except FileNotFoundError:
self._logger.error('Index not found',
extra={'es_index': index})
continue
mappings = resp[index]['mappings']
index_config = MappingIndexConverter.to_index_config(mappings)
try:
self._log_client.create_index(
self._config.get('project_name'),
logstore,
index_config,
)
except LogException as exc:
if exc.get_error_code() == 'IndexAlreadyExist':
self._log_client.update_index(
self._config.get('project_name'),
logstore,
index_config,
)
continue
raise
res.log_print()
if __name__ == '__main__':
endpoint = ''
accessKeyId = ''
accessKey = ''
project = ''
logstore = ''
token = None
client = LogClient(endpoint, accessKeyId, accessKey, token)
sample_logstore(client, project, logstore)
res = client.create_logstore(project, logstore, 1, 1)
time.sleep(0.1)
try:
sample_list_logstores(client, project)
sample_logtail_config(client, project, logstore)
sample_machine_group(client, project)
sample_apply_config(client, project)
sample_index(client, project, logstore)
sample_list_topics(client, project, logstore)
sample_put_logs(client, project, logstore)
sample_pull_logs(client, project, logstore)
sample_get_logs(client, project, logstore)
except LogException, e:
print e
res.log_print()
if __name__=='__main__':
endpoint = 'cn-hangzhou-staging-intranet.sls.aliyuncs.com'
accessKeyId = ''
accessKey = ''
project = 'ali-log-service'
logstore = 'test4'
token = ""
client = LogClient(endpoint, accessKeyId, accessKey,token)
sample_logstore(client, project, logstore)
res = client.create_logstore(project, logstore, 1, 1)
time.sleep(0.1)
try :
sample_list_logstores(client, project)
sample_logtail_config(client, project, logstore)
sample_machine_group(client, project)
sample_apply_config(client, project)
sample_index(client, project, logstore)
sample_list_topics(client, project, logstore)
sample_put_logs(client, project, logstore, i)
sample_pull_logs(client, project, logstore)
sample_get_logs(client, project, logstore)
except LogException, e:
print e
