def __treatCsvToHtml(self):
if self.__message_csv_to_html_splited != None:
for csv in self.__message_csv_to_html_splited:
csv_stripped = csv.strip()
if self.__verifyIfExist(csv_stripped):
converted_csv = ConvertCsvToHtml(csv_stripped)
converted_csv_answer = converted_csv.convertCsv()
self.__message_adds += '<br>'
self.__message_adds += converted_csv_answer
def convertCsvToWarp(self):
if self.__file_to_convert_splited != None:
for csv in self.__file_to_convert_splited:
csv_stripped = csv.strip()
if self.__verifyIfExist(csv_stripped):
converted_csv_answer = self.__convertCsvToWarp(
csv_stripped)
self.__final_message += '<br>'
self.__final_message += converted_csv_answer
return self.__final_message
def convertCsvToHtml(self):
if self.__file_to_convert_splited != None:
for csv in self.__file_to_convert_splited:
csv_stripped = csv.strip()
if self.__verifyIfExist(csv_stripped):
converted_csv_answer = self.__convertCsvToHtml(
csv_stripped)
#self.__final_message += '<br>'
#self.__final_message += 'file {}'.format(os.path.basename(csv_stripped))
self.__final_message += '<br>'
self.__final_message += converted_csv_answer
return self.__final_message
def convertToXML(self):
if self.xmlreport == None:
return
logEvent("INFO: " + self.testName.upper() + ' Generating XML files')
summary_suite = []
cdtresultsdir = self.basedir + self.cfg.details.settings[self.platform][self.testName].csvresultdir
xmlresultdir = self.basedir + self.cfg.details.settings[self.platform][self.testName].xmlresultdir
csvlist = glob.glob(cdtresultsdir + '/*.csv')
for csv in csvlist:
csv = replaceSlash(csv.strip())
if csv.lower().find('summary') == -1: #parse only suite summaries not binary produced ones.
m = re.search('.*\\\(.*)\.csv$', csv)
if m == None:
m = re.search('.*/(.*)\.csv$', csv)
suite_name = m.group(1)
xml = xmlresultdir + '/' + suite_name + '.xml'
self.convertCsvToXml(csv, xml)
def __init__(self, row, csv, config, csv_account):
"""
Parameters:
row is the list of fields read from one line of the CSV file.
csv is the string of the original line from CSV file.
config is the ConfigParser instance
csv_account is the name of the Ledger account for which this is
a statement e.g. "Assets:Bank:Checking"
"""
self.addons = {}
if config.has_section(csv_account + "_addons"):
for item in config.items(csv_account + "_addons"):
if item in config.defaults().items(): continue
self.addons['addon_'+item[0]] = row[int(item[1])-1]
# Get the date and convert it into a ledger formatted date.
self.date = row[config.getint(csv_account, 'date') - 1]
if config.has_option(csv_account, 'csv_date_format'):
csv_date_format = config.get(csv_account, 'csv_date_format')
else:
csv_date_format = ""
if config.has_option(csv_account, 'ledger_date_format'):
ledger_date_format = config.get(csv_account, 'ledger_date_format')
else:
ledger_date_format = ""
if ledger_date_format != csv_date_format:
self.date = (datetime
.strptime(self.date, csv_date_format)
.strftime(ledger_date_format))
desc = []
for index in re.compile(',\s*').split(config.get(csv_account, 'desc')):
desc.append(row[int(index) - 1].strip())
self.desc = ' '.join(desc).strip()
if config.getint(csv_account, 'credit') < 0:
self.credit = ""
else:
self.credit = row[config.getint(csv_account, 'credit') - 1]
if config.getint(csv_account, 'debit') < 0:
self.debit = ""
else:
self.debit = row[config.getint(csv_account, 'debit') - 1]
self.csv_account = config.get(csv_account, 'account')
self.currency = config.get(csv_account, 'currency')
self.cleared_character = config.get(csv_account, 'cleared_character')
if config.has_option(csv_account, 'transaction_template'):
with open(config.get(csv_account, 'transaction_template'), 'r') as template_file:
self.transaction_template = template_file.read().rstrip()
else:
self.transaction_template = ""
self.csv = csv.strip()
# We also record this - in future we may use it to avoid duplication
self.md5sum = hashlib.md5(self.csv).hexdigest()
self.printed_header = False
def get_all_species(ok_read_ids):
"""Get all species that are found in the csv files.
Arguments:
ok_read_ids: A list with all read ids that passed the filtering stage.
Returns:
All species names and header needed to create a new species file.
Random line number per barcode for future subset functionality.
Raises:
IndexError: Set rank to (no rank) if no rank is found in the NCBI taxonomy database.
ValueError: Check if values are correct.
"""
taxids = []
allnameslist = []
headers = []
for csv in onlyfiles:
print()
print("Getting all available species from " + csv + "...")
print()
read_id_column = 1
classification_status = 2
barcode_column = 5
acc_column = 6
taxid_column = 4
with open(mypath + csv) as dummy:
total_lines = 0
dummy.readline()
for dummyline in dummy:
total_lines += 1
for bc in barcodes:
headers.append(csv.strip(".csv") + bc)
with open(mypath + csv) as nf:
nf.readline()
filecontent = nf.read().split('\n')
linenr = 0
for content in filecontent:
content = content.split(',')
# for line in nf:
# if ',,' not in line:
# line = line.split(',')
name = ""
if linenr > 0:
try:
if (
float(content[acc_column]) >= int(minaccuracy) and
content[read_id_column] in ok_read_ids and
content[barcode_column] != "NA" and
int(content[taxid_column]) not in taxids and
content[classification_status] == "Classification successful"
):
taxids.append(int(content[taxid_column]))
bestrankdict = ncbi.get_rank([int(content[taxid_column])])
bestrank = list(bestrankdict.values())
taxid2name = ncbi.get_taxid_translator([int(content[taxid_column])])
for dummytid, tname in taxid2name.items():
namesplit = tname.split(' ')
if len(namesplit) > 2:
splitnr = 0
for split in namesplit[:2]:
if splitnr < 1:
name += split + ' '
else:
name += split
splitnr += 1
else:
name = str(tname)
if str(name) not in allnameslist:
try:
fullname = str(name + " (" + bestrank[0] + ")")
except IndexError:
fullname = str(name + " (no rank)")
allnameslist.append(str(fullname))
except (IndexError, ValueError):
pass
linenr += 1
block = int(round(60*(linenr/total_lines)))
msg = "\r[{0}] {1}%".format("#"*block + "-"*(60-block), round(linenr/total_lines*100, 2))
sys.stdout.write(msg)
sys.stdout.flush()
print()
allnames = list(set(allnameslist))
print(len(allnames), "species found!")
print()
return allnames, headers
def read_csv():
"""Read the csv files and build lists with available taxids and occurance.
Raises:
IndexError: Set rank to (no rank) is none is found.
"""
taxnames = []
allnames, headers = get_all_species(ok_read_ids)
for csv in onlyfiles:
read_id_column = 1
barcode_column = 5
acc_column = 6
taxid_column = 4
for bc in barcodes:
print("Checking", bc, "in", csv)
with open(mypath + csv) as nf:
nf.readline()
filecontent = nf.read().split('\n')
linenr = 0
for content in filecontent:
content = content.split(',')
name = ""
try:
if content[barcode_column] == bc:
if linenr > 0:
if (
float(content[acc_column]) >= int(minaccuracy) and
content[read_id_column] in ok_read_ids
):
taxid2name = ncbi.get_taxid_translator([int(content[taxid_column])])
bestrankdict = ncbi.get_rank([int(content[taxid_column])])
bestrank = list(bestrankdict.values())
for dummytid, tname in taxid2name.items():
namesplit = tname.split(' ')
if len(namesplit) > 2:
splitnr = 0
for split in namesplit[:2]:
if splitnr < 1:
name += split + ' '
else:
name += split
splitnr += 1
else:
name = str(tname)
if content[barcode_column] == bc:
try:
fullname = str(name + " (" + bestrank[0] + ")")
except IndexError:
fullname = str(name + " (no rank)")
taxnames.append(str(fullname))
linenr += 1
except (IndexError, ValueError):
pass
namecounter = Counter(taxnames)
for k, v in namecounter.items():
values = []
values.append(v)
values.append(csv.strip(".csv") + bc)
d[k].append(values)
allnames = list(dict.fromkeys(allnames))
for key in allnames:
if key not in namecounter.keys():
values = []
values.append(0)
values.append(csv.strip(".csv") + bc)
d[key].append(values)
namecounter.clear()
taxnames = []
print(bc, "done!")
make_rank_csv(headers)
print("Finished\n")
def __init__(self, row, csv, config, csv_account):
"""
Parameters:
row is the list of fields read from one line of the CSV file.
csv is the string of the original line from CSV file.
config is the ConfigParser instance
csv_account is the name of the Ledger account for which this is
a statement e.g. "Assets:Bank:Checking"
"""
self.addons = {}
if config.has_section(csv_account + "_addons"):
for item in config.items(csv_account + "_addons"):
if item in config.defaults().items(): continue
self.addons['addon_'+item[0]] = row[int(item[1])-1]
# Get the date and convert it into a ledger formatted date.
self.date = row[config.getint(csv_account, 'date') - 1]
if config.has_option(csv_account, 'csv_date_format'):
csv_date_format = config.get(csv_account, 'csv_date_format')
else:
csv_date_format = ""
if config.has_option(csv_account, 'ledger_date_format'):
ledger_date_format = config.get(csv_account, 'ledger_date_format')
else:
ledger_date_format = ""
if ledger_date_format != csv_date_format:
self.date = (datetime
.strptime(self.date, csv_date_format)
.strftime(ledger_date_format))
self.desc = row[config.getint(csv_account, 'desc') - 1].strip()
if config.getint(csv_account, 'credit') < 0:
self.credit = ""
else:
self.credit = row[config.getint(csv_account, 'credit') - 1]
if config.getint(csv_account, 'debit') < 0:
self.debit = ""
else:
self.debit = row[config.getint(csv_account, 'debit') - 1]
self.csv_account = config.get(csv_account, 'account')
self.currency = config.get(csv_account, 'currency')
self.cleared_character = config.get(csv_account, 'cleared_character')
if config.has_option(csv_account, 'transaction_template'):
with open(config.get(csv_account, 'transaction_template'), 'r') as template_file:
self.transaction_template = template_file.read().rstrip()
else:
self.transaction_template = ""
self.csv = csv.strip()
# We also record this - in future we may use it to avoid duplication
self.md5sum = hashlib.md5(self.csv).hexdigest()
self.printed_header = False
continue
# call cloc to compute a summary of the script
print('Call cloc with %s' % file)
try:
csv = subprocess.check_output(['cloc', '--csv', '--quiet', file], stderr=ferr)
except subprocess.CalledProcessError:
print('Call to cloc produced an error with %s' % file)
continue
if '' == csv:
print('Clock did not yield a result for %s' % file)
continue
# save last line of CSV output containing data in vars
[files, language, blank, comment, code] = csv.strip().split('\n')[-1].split(',')
platforms[platform]['code'][language] = platforms[platform]['code'].get(language, 0) + 1
ferr.close()
# make data d3 friendly
def sortdict(d):
return sorted([{'name': k, 'value': v} for k, v in d.items()],
key=lambda x: x.values()[-1], reverse=True)
pfs = sorted([v for k, v in platforms.items()], key=lambda x: x['value'])
for idx, p in enumerate(pfs):
p['types'] = sortdict(p['types'])
p['code'] = sortdict(p['code'])
pfs[idx] = p
# call cloc to compute a summary of the script
print('Call cloc with %s' % file)
try:
csv = subprocess.check_output(['cloc', '--csv', '--quiet', file],
stderr=ferr)
except subprocess.CalledProcessError:
print('Call to cloc produced an error with %s' % file)
continue
if '' == csv:
print('Clock did not yield a result for %s' % file)
continue
# save last line of CSV output containing data in vars
[files, language, blank, comment,
code] = csv.strip().split('\n')[-1].split(',')
platforms[platform]['code'][
language] = platforms[platform]['code'].get(language, 0) + 1
ferr.close()
# make data d3 friendly
def sortdict(d):
return sorted([{
'name': k,
'value': v
} for k, v in d.items()],
key=lambda x: x.values()[-1],
reverse=True)
def get_vcf_list(filelist):
filelist = file(filelist).readlines()
return [csv.strip() for csv in filelist]
