def index_table(table, ip_address):
"""
Index MySQL table into elasticsearch instance
"""
parser_c = parser.MySQLParser(server="172.20.38.50",
user="******",
password="******",
database="sandbox",
table=table)
table_parsed = parser_c.parse()
with elastic_search(hosts=ip_address) as es_c:
try:
index_es = table
bulk_data = [data for data in table_parsed]
try:
es_c.indices.delete(index_es)
except NotFoundError:
logger.warning(
"Deleting index: {0} not found".format(index_es))
es_c.indices.create(index_es)
bulk(es_c, bulk_data, index=index_es, doc_type=table)
except IndexError:
raise IndexError
except RequestError as exc:
print exc
logger.error('%s bad index format', table)
def delete_data(index_del):
"""
@goal: delete index from into elasticsearch database
@return delete_error_code: error code deletion
"""
delete_error_code = None
es_c = Elasticsearch()
try:
delete_error_code = es_c.indices.delete(index=index_del)
logger.info('Delete index: "%s" in database', index_del)
except NotFoundError:
logger.error('No such index: "%s" in database', index_del)
return delete_error_code
def parse(self, ctp_file_path, version=None, module=None):
"""
CTP parser
@param ctp_file_path: wxCKCM file path
@return test_name: file script name
@return parsed_trace: octopylog formatted traces
"""
#Read octopylog file
test_title = "_".join(os.path.basename(ctp_file_path).split("_")[:-1])
with open(ctp_file_path) as ctp_file:
ctp_content = ctp_file.read()
# Loop on octopylog file
for line in ctp_content.split('\n'):
# Ensure first character is a digit (timestamp)
if line and line[0].isdigit():
ctp_line = log.OctopylogLog(self.pytestemb_version)
try:
ctp_line.data = line.decode('utf8')
# one line log = one data dictionary
data = {
'author': u'jenkins',
'test': u"%s" % test_title,
'text': u"%s" % ctp_line.message,
'timestamp': u"%s" % ctp_line.timestamp,
'library': u"%s" % ctp_line.message_type,
'index_time': 'u%s' % datetime.now().isoformat(),
'version': u'%s' % version,
'module': u'%s' % module
}
yield data
except UnicodeDecodeError:
logger.error(UnicodeDecodeError)
def parse(self, ckcm_file_path, version=None, module=None):
'''
wxCKCM parser
@param ckcm_file_path: wxCKCM file path
@return test_name: file script name
@return parsed_trace: ckcm formatted traces
'''
#Read ckcm file
test_title = "_".join(os.path.basename(ckcm_file_path).split("_")[:-2])
# Loop on ckcm file
with open(ckcm_file_path) as ckcm_file:
ckcm_content = ckcm_file.read()
for line in ckcm_content.split('\n'):
# Ensure first character is a '[' (timestamp)
if line.startswith("["):
ckcm_line = log.CkcmLog()
try:
ckcm_line.data = line.decode('utf8')
# one line log = one data dictionary
data = {
'author': u'jenkins',
'test': u"%s" % test_title,
'severity': u"%s" % ckcm_line.severity,
'text': u"%s" % ckcm_line.data,
'library': u'%s' % ckcm_line.library,
'ATCommand': u'%s' % ckcm_line.command,
'ATEvent': u'%s' % ckcm_line.event,
'index_time': u'%s' % datetime.now().isoformat(),
'module': u"%s" % module,
'version': u'%s' % version
}
yield data
except UnicodeDecodeError:
logger.error(UnicodeDecodeError)
def index_file(es_instance,
log_file_path,
es_index,
log_type,
version=None,
module=None,
pytestemb_version=None):
"""
@goal: index log file into elastic search database
@param es_instance: ElasticSearch instance
@param log_file_path: path to file traces directory
@param es_index: ElasticSearch index
@param version: field version in elastic search (optional)
@return error_code: boolean, False if one line is not indexed
@not_indexed_data: list of all data not indexed
"""
error_code = True
not_indexed_data = []
if log_type == 'ckcm':
parser_c = parser.CkcmParser()
elif log_type == 'octopylog':
parser_c = parser.OctopylogParser(pytestemb_version)
# Parse log file and format data to export
parsed_trace = parser_c.parse(log_file_path,
version=version,
module=module)
try:
bulk_data = [data for data in parsed_trace]
#TODO probleme d'update
bulk(es_instance,
actions=bulk_data,
index=es_index,
doc_type=log_type,
request_timeout=30)
except IndexError:
logger.error("%s\npytestemb version:%s\nindex:%s", log_file_path,
pytestemb_version, es_index)
except RequestError as exc:
error_code = False
not_indexed_data.append(data)
logger.error('%s bad index format', es_index)
logger.error(exc)
return error_code, not_indexed_data
def index_module(module_type,
config,
job_number='lastSuccessfulBuild',
log_type='ckcm',
url=None):
"""
@goal: index module ckcm traces
@param module_type : fc60x0 module
@param config: fc60x0 config
@param job_number: jenkins job number
@param log_type: ckcm or octopylog
"""
import glob
err_list = []
print module_type
# Create jenkins job object
jenkins_job = jenkins.JenkinsJob(config_hw=module_type,
config_sw=config,
job_number=job_number,
log_type=log_type,
url_results=url)
logger.info("Jenkins job: %s", jenkins_job.get_url())
# Decompressed ckcm.tgz into /tmp/
if log_type == 'ckcm':
tgz_file = jenkins_job.ckcm_tgz_file_name
elif log_type == 'octopylog':
tgz_file = jenkins_job.octopylog_tgz_file_name
try:
directory_c = decompressed_tgz(tgz_file, '/tmp')
logger.info("current_directory: %s", directory_c)
except UntarException as msg:
logger.error(msg)
return
# Get pytestemb version
pytestemb_version = None
if log_type == 'octopylog':
pytestemb_version = get_pytestemb_version(directory_c)
package_version = get_package_version(directory_c)
# Build elastic search index
es_index_current = "{0}_{1}_{2}_{3}_{4}".format(log_type,
package_version.lower(),
module_type.lower(),
config.lower(),
jenkins_job.build_number)
# Index each line from log file traces
logger.info("Version : %s, Package: %s, Config: %s", package_version,
module_type, config)
with elastic_search(hosts="172.20.22.104") as es_c:
# Create elastic search instance
try:
es_c.indices.delete(es_index_current)
except NotFoundError:
logger.info("Current index: {0}".format(es_index_current))
es_c.indices.create(es_index_current)
for file_c in os.listdir(directory_c):
logger.info(" Parsing... %s", file_c)
logger.info("Current index: {0}".format(es_index_current))
try:
index_file(es_c,
os.path.join(directory_c, file_c),
es_index_current,
log_type,
version=package_version,
module=module_type.lower(),
pytestemb_version=pytestemb_version)
except Exception as exc:
raise exc
finally:
#err_list.append(err)
logger.info("Removing {0}".format(
os.path.join(directory_c, file_c)))
os.remove(os.path.join(directory_c, file_c))
# clean tmp directory
deprecated_files = [
os.path.join("/tmp", f) for f in os.listdir("/tmp")
if re.search(r'(pytestemb|ckcm|octopylog).*', f)
if os.path.isfile(os.path.join("/tmp", f))
]
deprecated_dirs = [
os.path.join("/tmp", f) for f in os.listdir("/tmp")
if re.search(r'(pytestemb|ckcm|octopylog).*', f)
if os.path.isdir(os.path.join("/tmp", f))
]
for d_file in deprecated_files:
logger.info("Cleaning: delete file {0}".format(d_file))
os.remove(d_file)
#for d_file in deprecated_dirs:
# logger.info("Cleaning: delete file {0}".format(d_file))
# os.rmdir(d_file)
return all(err_list)