from tools.utility import messageFactory
LOGGER = lo.get_logger('api')
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--remove-dir',
type=str,
default='.',
help='Set path to config file')
parser.add_argument('--logs-path',
type=str,
default='.',
help='Set path to config file')
args = parser.parse_args()
mf = messageFactory.MessageFactory()
LOGGER.info('Removing unused files')
to_remove = []
for root, dirs, files in os.walk(args.remove_dir):
for fi_name in files:
with open(args.logs_path, 'r') as f:
remove = True
for line in f:
if fi_name in line:
remove = False
break
if remove:
remove_path = '{}/{}'.format(root, fi_name)
st = os.stat(remove_path)
c_time = st.st_ctime
c_time = time.time() - c_time
def send_to_indexing(yc_api_prefix,
modules_to_index,
credentials,
apiIp=None,
sdo_type=False,
delete=False,
from_api=True,
set_key=None,
force_indexing=True):
""" Sends the POST request which will activate indexing script for modules which will
help to speed up process of searching. It will create a json body of all the modules
containing module name and path where the module can be found if we are adding new
modules. Other situation can be if we need to delete module. In this case we are sending
list of modules that need to be deleted.
Arguments:
:param yc_api_prefix: (str) prefix for sending request to api
:param modules_to_index: (json file) prepare.json file generated while parsing
all the modules. This file is used to iterate through all the modules.
:param credentials: (list) Basic authorization credentials - username, password
respectively.
:param sdo_type: (bool) Whether or not it is sdo that needs to be sent.
:param delete: (bool) Whether or not we are deleting module.
:param from_api: (bool) Whether or not api sent the request to index.
:param set_key: (str) String containing key to confirm that it is receiver that sends data. This is
is verified before indexing takes place.
:param force_indexing: (bool) Whether or not we should force indexing even if module exists in cache.
"""
global api_ip
if apiIp is not None:
api_ip = apiIp
LOGGER.debug('Sending data for indexing')
mf = messageFactory.MessageFactory()
if delete:
body_to_send = json.dumps({'modules-to-delete': modules_to_index},
indent=4)
mf.send_removed_yang_files(body_to_send)
for mod in modules_to_index:
name, revision_organization = mod.split('@')
revision, organization = revision_organization.split('/')
path_to_delete_local = "{}{}@{}.yang".format(
save_file_dir, name, revision)
data = {'input': {'dependents': [{'name': name}]}}
response = requests.post(yc_api_prefix + 'search-filter',
auth=(credentials[0], credentials[1]),
json={'input': data})
if response.status_code == 201:
modules = json.loads(response.content)
for mod in modules:
m_name = mod['name']
m_rev = mod['revision']
m_org = mod['organization']
url = ('{}://{}:{}/api/config/catalog/modules/module/'
'{},{},{}/dependents/{}'.format(
confd_protocol, confd_ip, confdPort, m_name,
m_rev, m_org, name))
requests.delete(url,
auth=(credentials[0], credentials[1]),
headers={
'Content-Type':
'application/vnd.yang.data+json'
})
if os.path.exists(path_to_delete_local):
os.remove(path_to_delete_local)
else:
with open(modules_to_index, 'r') as f:
sdos_json = json.load(f)
post_body = {}
load_new_files_to_github = False
if from_api:
if sdo_type:
prefix = 'api/sdo/'
else:
prefix = 'api/vendor/'
for module in sdos_json['module']:
response = http_request('{}search/modules/{},{},{}'.format(
yc_api_prefix, module['name'], module['revision'],
module['organization']),
'GET',
'',
credentials,
'application/vnd.yang.data+json',
return_code=True)
code = response.code
if force_indexing or (code != 200 and code != 201
and code != 204):
if module.get('schema'):
path = prefix + module['schema'].split(
'githubusercontent.com/')[1]
path = os.path.abspath(
get_curr_dir(__file__) + '/../../' + path)
else:
path = 'module does not exist'
post_body[module['name'] + '@' + module['revision'] + '/' +
module['organization']] = path
else:
for module in sdos_json['module']:
response = http_request('{}search/modules/{},{},{}'.format(
yc_api_prefix, module['name'], module['revision'],
module['organization']),
'GET',
'',
credentials,
'application/vnd.yang.data+json',
return_code=True)
code = response.code
if code != 200 and code != 201 and code != 204:
load_new_files_to_github = True
if force_indexing or (code != 200 and code != 201
and code != 204):
if module.get('schema'):
path = module['schema'].split('master')[1]
path = os.path.abspath(
get_curr_dir(__file__) + '/../../' + path)
else:
path = 'module does not exist'
post_body[module['name'] + '@' + module['revision'] + '/' +
module['organization']] = path
body_to_send = json.dumps({'modules-to-index': post_body}, indent=4)
if len(post_body) > 0 and not force_indexing:
mf.send_added_new_yang_files(body_to_send)
if load_new_files_to_github:
LOGGER.info('Starting a new process to populate github')
cmd = ['python', '../ietfYangDraftPull/draftPull.py']
proc = subprocess.Popen(cmd, close_fds=True)
LOGGER.info('Populating github with process {}'.format(proc))
try:
set_key = key
except NameError:
pass
LOGGER.info('Sending data for indexing with body {}'.format(body_to_send))
try:
http_request('https://' + api_ip + '/yang-search/metadata-update.php',
'POST',
body_to_send,
credentials,
'application/json',
indexing=create_signature(set_key, body_to_send))
except urllib2.HTTPError as e:
LOGGER.error('could not send data for indexing. Reason: {}'.format(
e.msg))
except URLError as e:
LOGGER.error('could not send data for indexing. Reason: {}'.format(
repr(e.message)))
