class MigrationManager(object): def __init__(self, config: MigrationConfig): self._config = config _uuid = str(uuid.uuid4()) self._id = ''.join(_uuid.split('-')) self._es = Elasticsearch( hosts=self._config.hosts, timeout=60, max_retries=30, retry_on_timeout=True, verify_certs=False, ) self._log_client = LogClient( endpoint=self._config.get('endpoint'), accessKeyId=self._config.access_key_id, accessKey=self._config.access_key, ) setup_logging( self._id, self._config.get('endpoint'), self._config.get('project_name'), self._config.access_key_id, self._config.access_key, ) self._logger = logging.getLogger(__name__) self._shutdown_flag = op.join(self._config.cache_path, 'shutdown.lock') print('#migration: {}'.format(self._id)) def migrate(self): self._logger.info('Migration starts') tasks = self._discover_tasks() task_cnt = len(tasks) pool_size = max(1, min(self._config.get('pool_size'), task_cnt)) print('#pool_size: {}'.format(pool_size)) print('#tasks: {}'.format(task_cnt)) self._prepare() futures = [] state = { 'total': task_cnt, Checkpoint.finished: 0, Checkpoint.dropped: 0, Checkpoint.failed: 0, } with ProcessPoolExecutor(max_workers=pool_size) as pool: for task in tasks: futures.append( pool.submit( _migration_worker, self._config, task, self._shutdown_flag, )) try: for future in as_completed(futures): res = future.result() if res in state: state[res] += 1 self._logger.info('State', extra=state) print('>> state:', json.dumps(state)) except BaseException: self._logger.error( 'Exception', extra={'traceback': traceback.format_exc()}, ) for future in futures: if not future.done(): future.cancel() list(as_completed(futures, timeout=10)) if state[Checkpoint.finished] + state[Checkpoint.dropped] >= task_cnt: self._logger.info('All migration tasks finished') self._logger.info('Migration exits') print('exit:', json.dumps(state)) return state def _prepare(self): if op.exists(self._shutdown_flag): os.unlink(self._shutdown_flag) def _handle_term_sig(signum, frame): # Raise Ctrl+C with open(self._shutdown_flag, 'w') as f: f.write('') raise KeyboardInterrupt() signal.signal(signal.SIGINT, _handle_term_sig) signal.signal(signal.SIGTERM, _handle_term_sig) def _discover_tasks(self): indexes = self._config.get('indexes') data = self._es.search_shards(indexes) tasks = [] for shard in data['shards']: for item in shard: # Ignore internal index if not indexes and item['index'].startswith('.'): continue if item['state'] == 'STARTED' and item['primary']: tasks.append( { 'es_index': item['index'], 'es_shard': item['shard'] }, ) return self._handle_cache(tasks) def _handle_cache(self, tasks): file_tasks = op.join(self._config.cache_path, 'tasks.json') if op.exists(file_tasks): with open(file_tasks) as f: cont = f.read() else: cont = '[]' try: old_tasks = json.loads(cont) except json.JSONDecodeError: self._logger.error('Invalid task cache', extra={'cache': cont}) old_tasks = [] task_map = {(task['es_index'], task['es_shard']): task['id'] for task in old_tasks} _mappings = IndexLogstoreMappings( list([task['es_index'] for task in tasks]), self._config.get('logstore_index_mappings'), ) cnt, new_tasks = len(old_tasks), [] for task in tasks: _task = (task['es_index'], task['es_shard']) if _task not in task_map: task['id'] = cnt task['logstore'] = _mappings.get_logstore(task['es_index']) new_tasks.append(task) cnt += 1 tasks = old_tasks + new_tasks with open(file_tasks, 'w') as f: f.write(json.dumps(tasks, indent=2)) if self._config.get('auto_creation'): self._setup_aliyun_log(_mappings) return tasks def _setup_aliyun_log(self, index_logstore_mappings): print('setup aliyun log service...') self._logger.info('Setup AliyunLog start') logstores = index_logstore_mappings.get_all_logstores() for logstore in logstores: self._setup_logstore(index_logstore_mappings, logstore) self._logger.info('Setup AliyunLog wait') time.sleep(self._config.get('wait_time_in_secs')) self._logger.info('Setup AliyunLog finish') def _setup_logstore(self, index_logstore_mappings, logstore): try: self._log_client.create_logstore( project_name=self._config.get('project_name'), logstore_name=logstore, shard_count=8, ttl=3650, ) except LogException as exc: if exc.get_error_code() == 'LogStoreAlreadyExist': self._logger.info( 'Logstore already exist, skip creation.', extra={'logstore': logstore}, ) else: raise self._setup_index(index_logstore_mappings, logstore) def _setup_index(self, index_logstore_mappings, logstore): indexes = index_logstore_mappings.get_indexes(logstore) for index in indexes: self._logger.info( 'Setup AliyunLog Logstore', extra={ 'logstore': logstore, 'es_index': index }, ) try: resp = self._es.indices.get(index=index) except FileNotFoundError: self._logger.error('Index not found', extra={'es_index': index}) continue mappings = resp[index]['mappings'] index_config = MappingIndexConverter.to_index_config(mappings) try: self._log_client.create_index( self._config.get('project_name'), logstore, index_config, ) except LogException as exc: if exc.get_error_code() == 'IndexAlreadyExist': self._log_client.update_index( self._config.get('project_name'), logstore, index_config, ) continue raise
res.log_print() if __name__ == '__main__': endpoint = '' accessKeyId = '' accessKey = '' project = '' logstore = '' token = None client = LogClient(endpoint, accessKeyId, accessKey, token) sample_logstore(client, project, logstore) res = client.create_logstore(project, logstore, 1, 1) time.sleep(0.1) try: sample_list_logstores(client, project) sample_logtail_config(client, project, logstore) sample_machine_group(client, project) sample_apply_config(client, project) sample_index(client, project, logstore) sample_list_topics(client, project, logstore) sample_put_logs(client, project, logstore) sample_pull_logs(client, project, logstore) sample_get_logs(client, project, logstore) except LogException, e: print e
res.log_print() if __name__=='__main__': endpoint = 'cn-hangzhou-staging-intranet.sls.aliyuncs.com' accessKeyId = '' accessKey = '' project = 'ali-log-service' logstore = 'test4' token = "" client = LogClient(endpoint, accessKeyId, accessKey,token) sample_logstore(client, project, logstore) res = client.create_logstore(project, logstore, 1, 1) time.sleep(0.1) try : sample_list_logstores(client, project) sample_logtail_config(client, project, logstore) sample_machine_group(client, project) sample_apply_config(client, project) sample_index(client, project, logstore) sample_list_topics(client, project, logstore) sample_put_logs(client, project, logstore, i) sample_pull_logs(client, project, logstore) sample_get_logs(client, project, logstore) except LogException, e: print e