def __treatCsvToHtml(self): if self.__message_csv_to_html_splited != None: for csv in self.__message_csv_to_html_splited: csv_stripped = csv.strip() if self.__verifyIfExist(csv_stripped): converted_csv = ConvertCsvToHtml(csv_stripped) converted_csv_answer = converted_csv.convertCsv() self.__message_adds += '<br>' self.__message_adds += converted_csv_answer
def convertCsvToWarp(self): if self.__file_to_convert_splited != None: for csv in self.__file_to_convert_splited: csv_stripped = csv.strip() if self.__verifyIfExist(csv_stripped): converted_csv_answer = self.__convertCsvToWarp( csv_stripped) self.__final_message += '<br>' self.__final_message += converted_csv_answer return self.__final_message
def convertCsvToHtml(self): if self.__file_to_convert_splited != None: for csv in self.__file_to_convert_splited: csv_stripped = csv.strip() if self.__verifyIfExist(csv_stripped): converted_csv_answer = self.__convertCsvToHtml( csv_stripped) #self.__final_message += '<br>' #self.__final_message += 'file {}'.format(os.path.basename(csv_stripped)) self.__final_message += '<br>' self.__final_message += converted_csv_answer return self.__final_message
def convertToXML(self): if self.xmlreport == None: return logEvent("INFO: " + self.testName.upper() + ' Generating XML files') summary_suite = [] cdtresultsdir = self.basedir + self.cfg.details.settings[self.platform][self.testName].csvresultdir xmlresultdir = self.basedir + self.cfg.details.settings[self.platform][self.testName].xmlresultdir csvlist = glob.glob(cdtresultsdir + '/*.csv') for csv in csvlist: csv = replaceSlash(csv.strip()) if csv.lower().find('summary') == -1: #parse only suite summaries not binary produced ones. m = re.search('.*\\\(.*)\.csv$', csv) if m == None: m = re.search('.*/(.*)\.csv$', csv) suite_name = m.group(1) xml = xmlresultdir + '/' + suite_name + '.xml' self.convertCsvToXml(csv, xml)
def __init__(self, row, csv, config, csv_account): """ Parameters: row is the list of fields read from one line of the CSV file. csv is the string of the original line from CSV file. config is the ConfigParser instance csv_account is the name of the Ledger account for which this is a statement e.g. "Assets:Bank:Checking" """ self.addons = {} if config.has_section(csv_account + "_addons"): for item in config.items(csv_account + "_addons"): if item in config.defaults().items(): continue self.addons['addon_'+item[0]] = row[int(item[1])-1] # Get the date and convert it into a ledger formatted date. self.date = row[config.getint(csv_account, 'date') - 1] if config.has_option(csv_account, 'csv_date_format'): csv_date_format = config.get(csv_account, 'csv_date_format') else: csv_date_format = "" if config.has_option(csv_account, 'ledger_date_format'): ledger_date_format = config.get(csv_account, 'ledger_date_format') else: ledger_date_format = "" if ledger_date_format != csv_date_format: self.date = (datetime .strptime(self.date, csv_date_format) .strftime(ledger_date_format)) desc = [] for index in re.compile(',\s*').split(config.get(csv_account, 'desc')): desc.append(row[int(index) - 1].strip()) self.desc = ' '.join(desc).strip() if config.getint(csv_account, 'credit') < 0: self.credit = "" else: self.credit = row[config.getint(csv_account, 'credit') - 1] if config.getint(csv_account, 'debit') < 0: self.debit = "" else: self.debit = row[config.getint(csv_account, 'debit') - 1] self.csv_account = config.get(csv_account, 'account') self.currency = config.get(csv_account, 'currency') self.cleared_character = config.get(csv_account, 'cleared_character') if config.has_option(csv_account, 'transaction_template'): with open(config.get(csv_account, 'transaction_template'), 'r') as template_file: self.transaction_template = template_file.read().rstrip() else: self.transaction_template = "" self.csv = csv.strip() # We also record this - in future we may use it to avoid duplication self.md5sum = hashlib.md5(self.csv).hexdigest() self.printed_header = False
def get_all_species(ok_read_ids): """Get all species that are found in the csv files. Arguments: ok_read_ids: A list with all read ids that passed the filtering stage. Returns: All species names and header needed to create a new species file. Random line number per barcode for future subset functionality. Raises: IndexError: Set rank to (no rank) if no rank is found in the NCBI taxonomy database. ValueError: Check if values are correct. """ taxids = [] allnameslist = [] headers = [] for csv in onlyfiles: print() print("Getting all available species from " + csv + "...") print() read_id_column = 1 classification_status = 2 barcode_column = 5 acc_column = 6 taxid_column = 4 with open(mypath + csv) as dummy: total_lines = 0 dummy.readline() for dummyline in dummy: total_lines += 1 for bc in barcodes: headers.append(csv.strip(".csv") + bc) with open(mypath + csv) as nf: nf.readline() filecontent = nf.read().split('\n') linenr = 0 for content in filecontent: content = content.split(',') # for line in nf: # if ',,' not in line: # line = line.split(',') name = "" if linenr > 0: try: if ( float(content[acc_column]) >= int(minaccuracy) and content[read_id_column] in ok_read_ids and content[barcode_column] != "NA" and int(content[taxid_column]) not in taxids and content[classification_status] == "Classification successful" ): taxids.append(int(content[taxid_column])) bestrankdict = ncbi.get_rank([int(content[taxid_column])]) bestrank = list(bestrankdict.values()) taxid2name = ncbi.get_taxid_translator([int(content[taxid_column])]) for dummytid, tname in taxid2name.items(): namesplit = tname.split(' ') if len(namesplit) > 2: splitnr = 0 for split in namesplit[:2]: if splitnr < 1: name += split + ' ' else: name += split splitnr += 1 else: name = str(tname) if str(name) not in allnameslist: try: fullname = str(name + " (" + bestrank[0] + ")") except IndexError: fullname = str(name + " (no rank)") allnameslist.append(str(fullname)) except (IndexError, ValueError): pass linenr += 1 block = int(round(60*(linenr/total_lines))) msg = "\r[{0}] {1}%".format("#"*block + "-"*(60-block), round(linenr/total_lines*100, 2)) sys.stdout.write(msg) sys.stdout.flush() print() allnames = list(set(allnameslist)) print(len(allnames), "species found!") print() return allnames, headers
def read_csv(): """Read the csv files and build lists with available taxids and occurance. Raises: IndexError: Set rank to (no rank) is none is found. """ taxnames = [] allnames, headers = get_all_species(ok_read_ids) for csv in onlyfiles: read_id_column = 1 barcode_column = 5 acc_column = 6 taxid_column = 4 for bc in barcodes: print("Checking", bc, "in", csv) with open(mypath + csv) as nf: nf.readline() filecontent = nf.read().split('\n') linenr = 0 for content in filecontent: content = content.split(',') name = "" try: if content[barcode_column] == bc: if linenr > 0: if ( float(content[acc_column]) >= int(minaccuracy) and content[read_id_column] in ok_read_ids ): taxid2name = ncbi.get_taxid_translator([int(content[taxid_column])]) bestrankdict = ncbi.get_rank([int(content[taxid_column])]) bestrank = list(bestrankdict.values()) for dummytid, tname in taxid2name.items(): namesplit = tname.split(' ') if len(namesplit) > 2: splitnr = 0 for split in namesplit[:2]: if splitnr < 1: name += split + ' ' else: name += split splitnr += 1 else: name = str(tname) if content[barcode_column] == bc: try: fullname = str(name + " (" + bestrank[0] + ")") except IndexError: fullname = str(name + " (no rank)") taxnames.append(str(fullname)) linenr += 1 except (IndexError, ValueError): pass namecounter = Counter(taxnames) for k, v in namecounter.items(): values = [] values.append(v) values.append(csv.strip(".csv") + bc) d[k].append(values) allnames = list(dict.fromkeys(allnames)) for key in allnames: if key not in namecounter.keys(): values = [] values.append(0) values.append(csv.strip(".csv") + bc) d[key].append(values) namecounter.clear() taxnames = [] print(bc, "done!") make_rank_csv(headers) print("Finished\n")
def __init__(self, row, csv, config, csv_account): """ Parameters: row is the list of fields read from one line of the CSV file. csv is the string of the original line from CSV file. config is the ConfigParser instance csv_account is the name of the Ledger account for which this is a statement e.g. "Assets:Bank:Checking" """ self.addons = {} if config.has_section(csv_account + "_addons"): for item in config.items(csv_account + "_addons"): if item in config.defaults().items(): continue self.addons['addon_'+item[0]] = row[int(item[1])-1] # Get the date and convert it into a ledger formatted date. self.date = row[config.getint(csv_account, 'date') - 1] if config.has_option(csv_account, 'csv_date_format'): csv_date_format = config.get(csv_account, 'csv_date_format') else: csv_date_format = "" if config.has_option(csv_account, 'ledger_date_format'): ledger_date_format = config.get(csv_account, 'ledger_date_format') else: ledger_date_format = "" if ledger_date_format != csv_date_format: self.date = (datetime .strptime(self.date, csv_date_format) .strftime(ledger_date_format)) self.desc = row[config.getint(csv_account, 'desc') - 1].strip() if config.getint(csv_account, 'credit') < 0: self.credit = "" else: self.credit = row[config.getint(csv_account, 'credit') - 1] if config.getint(csv_account, 'debit') < 0: self.debit = "" else: self.debit = row[config.getint(csv_account, 'debit') - 1] self.csv_account = config.get(csv_account, 'account') self.currency = config.get(csv_account, 'currency') self.cleared_character = config.get(csv_account, 'cleared_character') if config.has_option(csv_account, 'transaction_template'): with open(config.get(csv_account, 'transaction_template'), 'r') as template_file: self.transaction_template = template_file.read().rstrip() else: self.transaction_template = "" self.csv = csv.strip() # We also record this - in future we may use it to avoid duplication self.md5sum = hashlib.md5(self.csv).hexdigest() self.printed_header = False
continue # call cloc to compute a summary of the script print('Call cloc with %s' % file) try: csv = subprocess.check_output(['cloc', '--csv', '--quiet', file], stderr=ferr) except subprocess.CalledProcessError: print('Call to cloc produced an error with %s' % file) continue if '' == csv: print('Clock did not yield a result for %s' % file) continue # save last line of CSV output containing data in vars [files, language, blank, comment, code] = csv.strip().split('\n')[-1].split(',') platforms[platform]['code'][language] = platforms[platform]['code'].get(language, 0) + 1 ferr.close() # make data d3 friendly def sortdict(d): return sorted([{'name': k, 'value': v} for k, v in d.items()], key=lambda x: x.values()[-1], reverse=True) pfs = sorted([v for k, v in platforms.items()], key=lambda x: x['value']) for idx, p in enumerate(pfs): p['types'] = sortdict(p['types']) p['code'] = sortdict(p['code']) pfs[idx] = p
# call cloc to compute a summary of the script print('Call cloc with %s' % file) try: csv = subprocess.check_output(['cloc', '--csv', '--quiet', file], stderr=ferr) except subprocess.CalledProcessError: print('Call to cloc produced an error with %s' % file) continue if '' == csv: print('Clock did not yield a result for %s' % file) continue # save last line of CSV output containing data in vars [files, language, blank, comment, code] = csv.strip().split('\n')[-1].split(',') platforms[platform]['code'][ language] = platforms[platform]['code'].get(language, 0) + 1 ferr.close() # make data d3 friendly def sortdict(d): return sorted([{ 'name': k, 'value': v } for k, v in d.items()], key=lambda x: x.values()[-1], reverse=True)
def get_vcf_list(filelist): filelist = file(filelist).readlines() return [csv.strip() for csv in filelist]