from tools.utility import messageFactory LOGGER = lo.get_logger('api') if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('--remove-dir', type=str, default='.', help='Set path to config file') parser.add_argument('--logs-path', type=str, default='.', help='Set path to config file') args = parser.parse_args() mf = messageFactory.MessageFactory() LOGGER.info('Removing unused files') to_remove = [] for root, dirs, files in os.walk(args.remove_dir): for fi_name in files: with open(args.logs_path, 'r') as f: remove = True for line in f: if fi_name in line: remove = False break if remove: remove_path = '{}/{}'.format(root, fi_name) st = os.stat(remove_path) c_time = st.st_ctime c_time = time.time() - c_time
def send_to_indexing(yc_api_prefix, modules_to_index, credentials, apiIp=None, sdo_type=False, delete=False, from_api=True, set_key=None, force_indexing=True): """ Sends the POST request which will activate indexing script for modules which will help to speed up process of searching. It will create a json body of all the modules containing module name and path where the module can be found if we are adding new modules. Other situation can be if we need to delete module. In this case we are sending list of modules that need to be deleted. Arguments: :param yc_api_prefix: (str) prefix for sending request to api :param modules_to_index: (json file) prepare.json file generated while parsing all the modules. This file is used to iterate through all the modules. :param credentials: (list) Basic authorization credentials - username, password respectively. :param sdo_type: (bool) Whether or not it is sdo that needs to be sent. :param delete: (bool) Whether or not we are deleting module. :param from_api: (bool) Whether or not api sent the request to index. :param set_key: (str) String containing key to confirm that it is receiver that sends data. This is is verified before indexing takes place. :param force_indexing: (bool) Whether or not we should force indexing even if module exists in cache. """ global api_ip if apiIp is not None: api_ip = apiIp LOGGER.debug('Sending data for indexing') mf = messageFactory.MessageFactory() if delete: body_to_send = json.dumps({'modules-to-delete': modules_to_index}, indent=4) mf.send_removed_yang_files(body_to_send) for mod in modules_to_index: name, revision_organization = mod.split('@') revision, organization = revision_organization.split('/') path_to_delete_local = "{}{}@{}.yang".format( save_file_dir, name, revision) data = {'input': {'dependents': [{'name': name}]}} response = requests.post(yc_api_prefix + 'search-filter', auth=(credentials[0], credentials[1]), json={'input': data}) if response.status_code == 201: modules = json.loads(response.content) for mod in modules: m_name = mod['name'] m_rev = mod['revision'] m_org = mod['organization'] url = ('{}://{}:{}/api/config/catalog/modules/module/' '{},{},{}/dependents/{}'.format( confd_protocol, confd_ip, confdPort, m_name, m_rev, m_org, name)) requests.delete(url, auth=(credentials[0], credentials[1]), headers={ 'Content-Type': 'application/vnd.yang.data+json' }) if os.path.exists(path_to_delete_local): os.remove(path_to_delete_local) else: with open(modules_to_index, 'r') as f: sdos_json = json.load(f) post_body = {} load_new_files_to_github = False if from_api: if sdo_type: prefix = 'api/sdo/' else: prefix = 'api/vendor/' for module in sdos_json['module']: response = http_request('{}search/modules/{},{},{}'.format( yc_api_prefix, module['name'], module['revision'], module['organization']), 'GET', '', credentials, 'application/vnd.yang.data+json', return_code=True) code = response.code if force_indexing or (code != 200 and code != 201 and code != 204): if module.get('schema'): path = prefix + module['schema'].split( 'githubusercontent.com/')[1] path = os.path.abspath( get_curr_dir(__file__) + '/../../' + path) else: path = 'module does not exist' post_body[module['name'] + '@' + module['revision'] + '/' + module['organization']] = path else: for module in sdos_json['module']: response = http_request('{}search/modules/{},{},{}'.format( yc_api_prefix, module['name'], module['revision'], module['organization']), 'GET', '', credentials, 'application/vnd.yang.data+json', return_code=True) code = response.code if code != 200 and code != 201 and code != 204: load_new_files_to_github = True if force_indexing or (code != 200 and code != 201 and code != 204): if module.get('schema'): path = module['schema'].split('master')[1] path = os.path.abspath( get_curr_dir(__file__) + '/../../' + path) else: path = 'module does not exist' post_body[module['name'] + '@' + module['revision'] + '/' + module['organization']] = path body_to_send = json.dumps({'modules-to-index': post_body}, indent=4) if len(post_body) > 0 and not force_indexing: mf.send_added_new_yang_files(body_to_send) if load_new_files_to_github: LOGGER.info('Starting a new process to populate github') cmd = ['python', '../ietfYangDraftPull/draftPull.py'] proc = subprocess.Popen(cmd, close_fds=True) LOGGER.info('Populating github with process {}'.format(proc)) try: set_key = key except NameError: pass LOGGER.info('Sending data for indexing with body {}'.format(body_to_send)) try: http_request('https://' + api_ip + '/yang-search/metadata-update.php', 'POST', body_to_send, credentials, 'application/json', indexing=create_signature(set_key, body_to_send)) except urllib2.HTTPError as e: LOGGER.error('could not send data for indexing. Reason: {}'.format( e.msg)) except URLError as e: LOGGER.error('could not send data for indexing. Reason: {}'.format( repr(e.message)))