def get_google_data(snapshot_source):
"""
The Google source object to be fetched from database or the filesystem
The initial configuration for database is 'validator' and collection
is 'structures', whereas for the filesystem the path to fetch the
'structures' is $SOLUTIONDIR/realm/<structure>.json
"""
sub_data = {}
if json_source():
dbname = config_value(DATABASE, DBNAME)
collection = config_value(DATABASE, collectiontypes[STRUCTURE])
parts = snapshot_source.split('.')
qry = {'name': parts[0]}
sort = [sort_field('timestamp', False)]
docs = get_documents(collection, dbname=dbname, sort=sort, query=qry, limit=1)
logger.info('Number of Google structure Documents: %d', len(docs))
if docs and len(docs):
sub_data = docs[0]['json']
else:
json_test_dir = get_test_json_dir()
file_name = '%s.json' % snapshot_source if snapshot_source and not \
snapshot_source.endswith('.json') else snapshot_source
google_source = '%s/../%s' % (json_test_dir, file_name)
logger.info('Google source: %s', google_source)
if exists_file(google_source):
sub_data = json_from_file(google_source)
if not sub_data:
logger.error("Google connector file %s does not exist, or it does not contains the valid JSON.", snapshot_source)
return sub_data
def get_call_kwargs(node):
"""Get argument names and their values in kwargs"""
kwargs = {"params": {}}
logger.info("Get node's kwargs")
params_source = config_value('GOOGLE', 'params')
paramsversions = None
if json_source():
dbname = config_value(DATABASE, DBNAME)
collection = config_value(DATABASE, collectiontypes[STRUCTURE])
parts = params_source.rsplit('/')
name = parts[-1].split('.')
qry = {'name': name[0]}
sort = [sort_field('timestamp', False)]
docs = get_documents(collection,
dbname=dbname,
sort=sort,
query=qry,
limit=1)
logger.info('Number of Google Params versions: %s', len(docs))
if docs and len(docs):
paramsversions = docs[0]['json']
else:
paramsversions_file = '%s/%s' % (framework_dir(), params_source)
logger.info(paramsversions_file)
if exists_file(paramsversions_file):
paramsversions = json_from_file(paramsversions_file)
path = node['path']
if paramsversions and "queryprameters" in paramsversions:
if node['type'] in paramsversions["queryprameters"]:
for param, parameter_type in paramsversions["queryprameters"][
node['type']].items():
add_argument_parameter(path, kwargs, param, parameter_type)
return kwargs
def get_version_for_type(node):
"""Url version of the resource."""
version = None
apiversions = None
logger.info("Get type's version")
api_source = config_value('AZURE', 'api')
if json_source():
dbname = config_value(DATABASE, DBNAME)
collection = config_value(DATABASE, collectiontypes[STRUCTURE])
parts = api_source.rsplit('/')
name = parts[-1].split('.')
qry = {'name': name[0]}
sort = [sort_field('timestamp', False)]
docs = get_documents(collection, dbname=dbname, sort=sort, query=qry, limit=1)
logger.info('Number of Azure API versions: %s', len(docs))
if docs and len(docs):
apiversions = docs[0]['json']
else:
apiversions_file = '%s/%s' % (framework_dir(), api_source)
logger.info(apiversions_file)
if exists_file(apiversions_file):
apiversions = json_from_file(apiversions_file)
if apiversions:
if node and 'type' in node and node['type'] in apiversions:
version = apiversions[node['type']]['version']
return version
def get_custom_data(snapshot_source):
sub_data = {}
if json_source():
dbname = config_value(DATABASE, DBNAME)
collection = config_value(DATABASE, collectiontypes[STRUCTURE])
parts = snapshot_source.split('.')
qry = {'name': parts[0]}
sort = [sort_field('timestamp', False)]
docs = get_documents(collection,
dbname=dbname,
sort=sort,
query=qry,
limit=1)
logger.info('Number of Custom Documents: %d', len(docs))
if docs and len(docs):
sub_data = docs[0]['json']
else:
json_test_dir = get_test_json_dir()
file_name = '%s.json' % snapshot_source if snapshot_source and not \
snapshot_source.endswith('.json') else snapshot_source
custom_source = '%s/../%s' % (json_test_dir, file_name)
logger.info('Custom source: %s', custom_source)
if exists_file(custom_source):
sub_data = json_from_file(custom_source)
return sub_data
def get_service_name(node_type):
"""
Get service name for init compute function
"""
service = None
params_source = config_value('GOOGLE', 'params')
paramsversions = None
if json_source():
dbname = config_value(DATABASE, DBNAME)
collection = config_value(DATABASE, collectiontypes[STRUCTURE])
parts = params_source.rsplit('/')
name = parts[-1].split('.')
qry = {'name': name[0]}
sort = [sort_field('timestamp', False)]
docs = get_documents(collection,
dbname=dbname,
sort=sort,
query=qry,
limit=1)
logger.info('Number of Google Params versions: %s', len(docs))
if docs and len(docs):
paramsversions = docs[0]['json']
else:
paramsversions_file = '%s/%s' % (framework_dir(), params_source)
logger.info(paramsversions_file)
if exists_file(paramsversions_file):
paramsversions = json_from_file(paramsversions_file)
check_node_type = node_type
node_type_list = node_type.split(".")
if len(node_type_list) > 1:
del node_type_list[-1]
check_node_type = ".".join(node_type_list)
if paramsversions and "serviceName" in paramsversions:
for service_name, resource_list in paramsversions['serviceName'].items(
):
if check_node_type in resource_list:
service = service_name
return service
def get_kubernetes_structure_data(snapshot_source):
"""
get_kubernetes_structure_data going to get structure data from connector
file which specified in snapshot as source field.
Return structure data as json dictionary
"""
kubernetes_structure_data = {}
if json_source():
qry = {'name': snapshot_source}
dbname = config_value('MONGODB', 'dbname')
sort = [sort_field('timestamp', False)]
collection =config_value(DATABASE, collectiontypes[STRUCTURE])
structure_docs = get_documents(collection=collection , dbname=dbname, sort= sort, query=qry, limit=1)
logger.info('%s fetched %s number of documents: %s', Snapshot.LOGPREFIX, STRUCTURE, len(structure_docs))
if structure_docs and len(structure_docs):
kubernetes_structure_data = structure_docs[0]['json']
else:
kubernetes_structure_path = get_kubernetes_structure_path(snapshot_source)
kubernetes_structure_data = json_from_file(kubernetes_structure_path)
return kubernetes_structure_data
def get_api_versions():
""" get api versions dict """
global apiversions
if not apiversions:
api_source = config_value('AZURE', 'api')
if json_source():
dbname = config_value(DATABASE, DBNAME)
collection = config_value(DATABASE, collectiontypes[STRUCTURE])
parts = api_source.rsplit('/')
name = parts[-1].split('.')
qry = {'name': name[0]}
sort = [sort_field('timestamp', False)]
docs = get_documents(collection, dbname=dbname, sort=sort, query=qry, limit=1)
logger.info('Number of Azure API versions: %s', len(docs))
if docs and len(docs):
apiversions = docs[0]['json']
else:
apiversions_file = '%s/%s' % (framework_dir(), api_source)
# logger.info(apiversions_file)
if exists_file(apiversions_file):
apiversions = json_from_file(apiversions_file)
return apiversions
def get_call_kwargs_for_crawler(node, project_id):
"""Get argument names and their values in kwargs for Crawler"""
kwargs = {}
logger.info("Get node's kwargs")
params_source = config_value('GOOGLE', 'params')
paramsversions = None
if json_source():
dbname = config_value(DATABASE, DBNAME)
collection = config_value(DATABASE, collectiontypes[STRUCTURE])
parts = params_source.rsplit('/')
name = parts[-1].split('.')
qry = {'name': name[0]}
sort = [sort_field('timestamp', False)]
docs = get_documents(collection,
dbname=dbname,
sort=sort,
query=qry,
limit=1)
logger.info('Number of Google Params versions: %s', len(docs))
if docs and len(docs):
paramsversions = docs[0]['json']
else:
paramsversions_file = '%s/%s' % (framework_dir(), params_source)
logger.info(paramsversions_file)
if exists_file(paramsversions_file):
paramsversions = json_from_file(paramsversions_file)
if paramsversions:
if node and 'type' in node and "crawler_queryprameters" in paramsversions:
for prameter in paramsversions["crawler_queryprameters"]:
if node['type'] in prameter['services']:
for param in prameter['params']:
if param == "project":
kwargs['project'] = project_id
elif param == "projectId":
kwargs['projectId'] = project_id
elif param == "zone":
kwargs['zone'] = "-"
return kwargs
def get_google_parameters():
"""
Return the google parameter object read from database or the filesystem
"""
global google_parameters
if not google_parameters:
params_source = config_value('GOOGLE', 'params')
if json_source():
dbname = config_value(DATABASE, DBNAME)
collection = config_value(DATABASE, collectiontypes[STRUCTURE])
parts = params_source.rsplit('/')
name = parts[-1].split('.')
qry = {'name': name[0]}
sort = [sort_field('timestamp', False)]
docs = get_documents(collection, dbname=dbname, sort=sort, query=qry, limit=1)
logger.info('Number of Google Params versions: %s', len(docs))
if docs and len(docs):
google_parameters = docs[0]['json']
else:
params_file = '%s/%s' % (framework_dir(), params_source)
logger.info(params_file)
if exists_file(params_file):
google_parameters = json_from_file(params_file)
return google_parameters