def POST(self):
web.header('Content-Type', 'text/html')
web.header('Content-disposition', 'attachment; filename=trenchesCSV.csv')
kml = web.input()
kml = urllib.unquote(kml["jsondata"])
x = json.loads(kml)
csv = []
csv.append("Trench, Trench Area, Trench Number, Trench ID, Year, Excavator,\n")
for feature in x["features"]:
row = []
row.append(str(feature["properties"]["trench"].replace(",", "-")))
row.append(str(feature["properties"]["trencharea"].replace(",", "-")))
row.append(str(feature["properties"]["trenchnumb"].replace(",", "-")))
row.append(str(feature["properties"]["trenchid"]))
row.append(str(feature["properties"]["year"]))
row.append(str(feature["properties"]["excavator"]) + "\n")
csv.append(",".join(row))
csv = "".join(csv)
return csv
def xls_reader(filename):
workbook = xlrd.open_workbook(filename)
worksheet = workbook.sheet_by_name('potongan')
num_rows = worksheet.nrows - 1
num_cells = worksheet.ncols - 1
curr_row = -1
csv = []
while curr_row < num_rows:
curr_row += 1
row = worksheet.row(curr_row)
curr_cell = -1
txt = []
while curr_cell < num_cells:
curr_cell += 1
# Cell Types: 0=Empty, 1=Text, 2=Number, 3=Date, 4=Boolean, 5=Error, 6=Blank
cell_type = worksheet.cell_type(curr_row, curr_cell)
cell_value = worksheet.cell_value(curr_row, curr_cell)
if cell_type==1 or cell_type==2:
try:
cell_value = str(cell_value)
except:
cell_value = '0'
else:
cell_value = clean(cell_value)
if curr_cell==0 and cell_value.strip()=="Tanggal":
curr_cell=num_cells
elif curr_cell==0 and cell_value.strip()=="":
curr_cell = num_cells
curr_row = num_rows
else:
txt.append(cell_value)
if txt:
csv.append(txt)
return csv
def makeSheet( self ): csv = [ 'County,Type,All 2000,All 2008,All % Change,Dem 2000,Dem 2008,Dem % Change,GOP 2000,GOP 2008,GOP % Change,All 18-24,All 25-34,All 35-44,All 45-54,All 55-64,All 65-74,All 75+,Dem 18-24,Dem 25-34,Dem 35-44,Dem 45-54,Dem 55-64,Dem 65-74,Dem 75+,GOP 18-24,GOP 25-34,GOP 35-44,GOP 45-54,GOP 55-64,GOP 65-74,GOP 75+,White,Black,Asian,Other,Catholic,Evangelical,Mainline,Jewish,Other,None,Casey,Rendell' ] for place in self.places: pop = place['population']; popAll = pop['all']; popDem = pop['dem']; popGop = pop['gop'] ages = place['ages']; ageAll = ages['all']['counts']; ageDem = ages['dem']['counts']; ageGop = ages['gop']['counts'] ethnic = place['ethnic'] religion = place['religion']['percents'] gub2002 = place['gub2002'] csv.append( '%s,%s,%d,%d,%.2f%%,%d,%d,%.2f%%,%d,%d,%.2f%%,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%.2f%%,%.2f%%,%.2f%%,%.2f%%,%.2f%%,%.2f%%,%.2f%%,%.2f%%' %( place['name'], pop['type'], popAll['before'], popAll['after'], popAll['change'], popDem['before'], popDem['after'], popDem['change'], popGop['before'], popGop['after'], popGop['change'], ageAll[0], ageAll[1], ageAll[2], ageAll[3], ageAll[4], ageAll[5], ageAll[6], ageDem[0], ageDem[1], ageDem[2], ageDem[3], ageDem[4], ageDem[5], ageDem[6], ageGop[0], ageGop[1], ageGop[2], ageGop[3], ageGop[4], ageGop[5], ageGop[6], ethnic[0], ethnic[1], ethnic[2], ethnic[3], religion[0], religion[1], religion[2], religion[3], religion[4], religion[5], gub2002[0], gub2002[1] ) ) write( '%s/states/%s/spreadsheet.csv' %( datapath, self.state ), '\n'.join(csv) )
def makeSheet(self):
csv = [
'County,Type,All 2000,All 2008,All % Change,Dem 2000,Dem 2008,Dem % Change,GOP 2000,GOP 2008,GOP % Change,All 18-24,All 25-34,All 35-44,All 45-54,All 55-64,All 65-74,All 75+,Dem 18-24,Dem 25-34,Dem 35-44,Dem 45-54,Dem 55-64,Dem 65-74,Dem 75+,GOP 18-24,GOP 25-34,GOP 35-44,GOP 45-54,GOP 55-64,GOP 65-74,GOP 75+,White,Black,Asian,Other,Catholic,Evangelical,Mainline,Jewish,Other,None,Casey,Rendell'
]
for place in self.places:
pop = place['population']
popAll = pop['all']
popDem = pop['dem']
popGop = pop['gop']
ages = place['ages']
ageAll = ages['all']['counts']
ageDem = ages['dem']['counts']
ageGop = ages['gop']['counts']
ethnic = place['ethnic']
religion = place['religion']['percents']
gub2002 = place['gub2002']
csv.append(
'%s,%s,%d,%d,%.2f%%,%d,%d,%.2f%%,%d,%d,%.2f%%,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%.2f%%,%.2f%%,%.2f%%,%.2f%%,%.2f%%,%.2f%%,%.2f%%,%.2f%%'
%
(place['name'], pop['type'], popAll['before'], popAll['after'],
popAll['change'], popDem['before'], popDem['after'],
popDem['change'], popGop['before'], popGop['after'],
popGop['change'], ageAll[0], ageAll[1], ageAll[2], ageAll[3],
ageAll[4], ageAll[5], ageAll[6], ageDem[0], ageDem[1],
ageDem[2], ageDem[3], ageDem[4], ageDem[5], ageDem[6],
ageGop[0], ageGop[1], ageGop[2], ageGop[3], ageGop[4],
ageGop[5], ageGop[6], ethnic[0], ethnic[1], ethnic[2],
ethnic[3], religion[0], religion[1], religion[2], religion[3],
religion[4], religion[5], gub2002[0], gub2002[1]))
write('%s/states/%s/spreadsheet.csv' % (datapath, self.state),
'\n'.join(csv))
def _csv_lines(self, cr, uid, travel_orders, context=None):
csv = []
self._set_decimal_point(self._decimal())
for to in travel_orders:
self._travel_order_name = to.name
# hr.travel.order
document_csv = self._document(cr, uid, to, context)
csv.append(document_csv)
# daily.allowance.lines
if to.daily_allowance_ids:
allowances_csv = self._allowances(cr, uid, to.daily_allowance_ids, to, context)
csv.append(allowances_csv)
# hr.expense.line
if to.expense_line_ids:
expenses_csv = self._expenses(cr, uid, to.expense_line_ids, to, context)
csv.append(expenses_csv)
# append itinerrary to expenses hr.travel.opder.itinerary.lines
if to.itinerary_ids:
itinerary_csv =self._itinerary_expenses(cr, uid, to.itinerary_ids, to, context)
csv.append(itinerary_csv)
# append daily allowances to expenses daily.allowance.lines
if to.daily_allowance_ids:
allowance_csv =self._daily_allowance_expenses(cr, uid, to.daily_allowance_ids, to, context)
csv.append(allowance_csv)
self._set_decimal_point() # reset decimal point to default
return csv
def predict_on_images(model, images, config):
tw = 224
th = 224
mean = config['mean']
mean = np.array(mean) * 255
classes = config['class_names']
csv = []
image = np.zeros((len(images), 3, th, tw), dtype=np.float32)
for i, path in enumerate(images):
img = Image.open(path)
npimg = np.array(img).astype(np.float32)
h = npimg.shape[0]
w = npimg.shape[1]
assert h >= th
assert w >= tw
assert npimg.shape[2] == 3
dr = (h - th) // 2
dc = (w - tw) // 2
# RGB 2 BGR
for k in range(3):
image[i, 2 - k, :, :] = npimg[dr:dr + th, dc:dc + tw, k] - mean[k]
res = model.eval(image)
for i in range(len(images)):
index = np.argmax(res[i]).item()
csv.append([images[i], str(index), classes[index]] +
['%8.6f' % v for v in res[i].tolist()])
with open('report.csv', 'w') as f:
for row in csv:
line = ','.join(row) + '\n'
f.write(line)
sys.stdout.write(','.join(row[0:10] + ['...']) + '\n')
return res
def predict_on_images(model,images,device,config):
tw = 224
th = 224
mean = config['mean']
std = config['std']
classes = config['class_names']
csv = []
model.eval()
image = torch.zeros((len(images),3,th,tw),dtype=torch.float32)
for i,path in enumerate(images):
img = PIL.Image.open(path)
npimg = np.array(img).astype(np.float32) * (1.0 / 255)
h = npimg.shape[0]
w = npimg.shape[1]
assert h>=th
assert w>=tw
assert npimg.shape[2] == 3
fact = 1.0 / np.array(std)
off = -np.array(mean) * fact
dr = (h - th) // 2
dc = (w - tw) // 2
for k in range(3):
image[i,k,:,:] = torch.from_numpy(npimg[dr:dr+th,dc:dc+tw,k] * fact[k] + off[k])
image = image.to(device)
res = model(image)
for i in range(len(images)):
index = torch.argmax(res[i]).item()
csv.append([path,str(index),classes[index]] + ['%8.6f' % v for v in res[i].tolist()])
with open('report.csv','w') as f:
for row in csv:
line = ','.join(row) + '\n'
f.write(line)
sys.stdout.write(','.join(row[0:10] + ['...']) + '\n')
def xls_reader(filename):
workbook = xlrd.open_workbook(filename)
worksheet = workbook.sheet_by_name('potongan')
num_rows = worksheet.nrows - 1
num_cells = worksheet.ncols - 1
curr_row = -1
csv = []
while curr_row < num_rows:
curr_row += 1
row = worksheet.row(curr_row)
curr_cell = -1
txt = []
while curr_cell < num_cells:
curr_cell += 1
# Cell Types: 0=Empty, 1=Text, 2=Number, 3=Date, 4=Boolean, 5=Error, 6=Blank
cell_type = worksheet.cell_type(curr_row, curr_cell)
cell_value = worksheet.cell_value(curr_row, curr_cell)
if cell_type == 1 or cell_type == 2:
try:
cell_value = str(cell_value)
except:
cell_value = '0'
else:
cell_value = clean(cell_value)
if curr_cell == 0 and cell_value.strip() == "Tanggal":
curr_cell = num_cells
elif curr_cell == 0 and cell_value.strip() == "":
curr_cell = num_cells
curr_row = num_rows
else:
txt.append(cell_value)
if txt:
csv.append(txt)
return csv
def _csv_lines(self, cr, uid, travel_orders, context=None):
csv = []
self._set_decimal_point(self._decimal())
for to in travel_orders:
self._travel_order_name = to.name
# hr.travel.order
document_csv = self._document(cr, uid, to, context)
csv.append(document_csv)
# daily.allowance.lines
if to.daily_allowance_ids:
allowances_csv = self._allowances(cr, uid,
to.daily_allowance_ids, to,
context)
csv.append(allowances_csv)
# hr.expense.line
if to.expense_line_ids:
expenses_csv = self._expenses(cr, uid, to.expense_line_ids, to,
context)
csv.append(expenses_csv)
# append itinerrary to expenses hr.travel.opder.itinerary.lines
if to.itinerary_ids:
itinerary_csv = self._itinerary_expenses(
cr, uid, to.itinerary_ids, to, context)
csv.append(itinerary_csv)
# append daily allowances to expenses daily.allowance.lines
if to.daily_allowance_ids:
allowance_csv = self._daily_allowance_expenses(
cr, uid, to.daily_allowance_ids, to, context)
csv.append(allowance_csv)
self._set_decimal_point() # reset decimal point to default
return csv
def gen_swc_csv(root_dir=dir):
csv = []
with open("transcriptions.txt", 'r') as f:
lines = f.readlines()
i = 0
for line in lines:
i += 1
file_name = line.split(" ", 1)[0]
file_text = line.split(" ", 1)[1]
sentence = file_text.split(" ")
if len(sentence) <= 2:
continue
trans = clean_sentence(file_text)
file_path = os.path.join(root_dir, file_name + ".wav")
csv.append((file_path, trans))
print("File " + str(i) + " / " + str(len(lines)), end='\r')
print()
print("Writing CSV File:")
df = pandas.DataFrame(data=csv)
output_file = "/home/GPUAdmin1/asr/train_csvs/swc_train.csv"
df.to_csv(output_file, header=False, index=False, sep=",")
def atividade2_process():
url = 'http://www.imdb.com/chart/boxoffice'
page = download(url)
#print(page)
soup = BeautifulSoup(page, 'html5lib')
table = soup.find("tbody")
filmes = table.find_all("tr")
lista = []
csv = []
for filme in filmes:
nome = filme.find("td", class_="titleColumn").a.text
rating = filme.find("td", class_="ratingColumn").text
rating = re.sub(r'[^\d$.M]', '', rating)
rating_number = re.sub(r'[^\d.]', '', rating)
gross = filme.find_all("td", class_="ratingColumn")
gross_span = [
span.find("span", class_="secondaryInfo") for span in gross
]
gross_value = gross_span[1].text
weeks = filme.find("td", class_="weeksColumn").text
lista_csv = [nome, rating, gross_value, weeks]
csv.append(lista_csv)
valor = {
"nome": nome,
"rating": rating,
"gross_value": gross_value,
"weeks": weeks
}
lista.append(valor)
return {"lista": lista, "csv": csv}
def POST(self):
web.header('Content-Type', 'text/html')
web.header('Content-disposition', 'attachment; filename=artifactsCSV.csv')
kml = web.input()
kml = urllib.unquote(kml["jsondata"])
x = json.loads(kml)
csv = []
csv.append("Artifact ID, Trench, Description, Fabric, Chronology, X Coordinate, Y Coordinate\n")
for feature in x["features"]:
row = []
row.append(feature["properties"]["catalogid"].replace(",", "-"))
row.append(feature["properties"]["trench"].replace(",", "-"))
row.append(feature["properties"]["name"].replace(",", "-"))
row.append(feature["properties"]["fabric"].replace(",", "-"))
row.append(feature["properties"]["chronology"].replace(",", "-"))
row.append(str(feature["geometry"]["coordinates"][0]))
row.append(str(feature["geometry"]["coordinates"][1]) + "\n")
csv.append(",".join(row))
csv = "".join(csv)
return csv
def processa_titulos(session, titulos, operacao):
csv = []
for tit in titulos:
# Skipa os nao liquidados
if tit['Situacao'] != 'Realizado':
continue
url = "https://portalinvestidor.tesourodireto.com.br/Protocolo/{}/{}".format(
tit['CodigoProtocolo'], operacao)
print("Fetching protocolo={}, url={}".format(tit['CodigoProtocolo'],
url))
response = session.get(
url,
headers={
'Referer':
'https://portalinvestidor.tesourodireto.com.br/Consulta',
})
soup = BeautifulSoup(response.content, 'html.parser')
titulo = soup.find(class_='td-protocolo-info-titulo').text
quantidade = float(get_info_titulo(soup, 0))
valor_unitario = str(float(get_info_titulo(soup, 1))).replace('.', ',')
rentabilidade = str(float(get_info_titulo(soup, 2)) / 100).replace(
'.', ',')
liquido = float(get_info_titulo(soup, 3))
taxa_corretora = str(-float(get_info_titulo(soup, 4))).replace(
'.', ',')
taxa_b3 = str(-float(get_info_titulo(soup, 5))).replace('.', ',')
valor_bruto = float(get_info_titulo(soup, 6))
if operacao == OPERACAO_VENDA:
quantidade = -quantidade
liquido = -liquido
valor_bruto = -valor_bruto
quantidade = str(quantidade).replace('.', ',')
liquido = str(liquido).replace('.', ',')
valor_bruto = str(valor_bruto).replace('.', ',')
data_operacao = datetime.datetime.strptime(
tit["DataOperacao"],
"%d/%m/%Y") # uses datetime object for sorting
line = [
tit["TipoOperacao"], data_operacao, tit["CodigoProtocolo"], titulo,
rentabilidade, quantidade, valor_unitario, valor_bruto,
taxa_corretora, taxa_b3, liquido
]
if IS_DEBUG:
file = open('protocolos/{}.html'.format(tit['CodigoProtocolo']),
"w")
file.write(str(response.content))
file.close()
print(url)
print(line)
csv.append(line)
return csv
def scrape_city_and_states(url, csv, limit):
cities_and_states = []
i = 0
limit_count = 0
for name in scraper(url).find_all('td'):
z = (name.text).split('[')
if (not hasNum(z[0])):
zz = z[0].split('\n')
if (i == 0):
x = zz[0]
i = 1
else:
i = 0
y = zz[0]
limit_count += 1
csv.append([
str(x.encode('utf-8')) + " " + str(y.encode('utf-8')), 0,
0, 0, 0, 0, 0, 0, 0, 0
])
cities_and_states.append(
(str(x.encode('utf-8')), str(y.encode('utf-8'))))
if (limit_count == limit):
break
else:
continue
return cities_and_states
def files(path):
csv = []
extension = "csv"
files = os.listdir(path)
for f in files:
if f.endswith(extension):
csv.append(f)
return csv
def files(path):
csv = []
extension = "csv"
files = os.listdir(path)
for f in files:
if f.endswith(extension):
csv.append(f)
return csv
def profile(stockname="MSFT"):
url = 'https://ca.finance.yahoo.com/quote/%s/profile?p=%s&.tsrc=fin-srch' % (
stockname, stockname)
obj = requests.get(url)
souppage = soup(obj.text, "html.parser")
tableithink = souppage.findAll("span")
lists = []
for row in tableithink:
lists.append(row)
i = 0
for i in range(len(lists)):
lists[i] = str(lists[i]).split(">")
if (len(lists[i]) == 5):
lists[i] = lists[i][2].replace("<!-- /react-text --", '').replace(
"</span", '').replace("amp; ", " ")
#print(lists[i])
i = 0
done = 1
while i < (len(lists)):
if ("Mr" or "Ms") in lists[i] and done:
j = i
done = 0
i += 1
elif (lists[i] == "N/A"):
lists.pop(i + 1)
i += 1
else:
i += 1
csv = []
k = j + 25
while j < k:
csv.append(
[lists[j], lists[j + 1], lists[j + 2], lists[j + 3], lists[j + 4]])
j += 5
#print(csv)
finalcsv = ''
for x in csv:
for i in x:
finalcsv += i.replace(",", " ") + ","
finalcsv += "\n"
f = open('profile.csv', 'w')
f.write(finalcsv)
f.close()
print("")
paraithink = souppage.findAll("p")
print(paraithink[2].text)
print("")
for x in csv:
print(x)
print("")
jsonMaker4()
def get_medias_dia(file, csv):
valores = []
date = get_date(file)
dia = date[0]
table = csv_reader(file)
acumulado = get_acumulado_dia(table)
n_corridas = get_numero_corridas_dia(table)
pon_poff_values = get_pon_poff(table, 29.5, 9.5)
sucata = get_sucata(table)
potencia_mw = get_potencia_mw(table)
potencia_kwh = get_potencia_kwh(table)
kwh_t = get_acumulado_dia_kwh_t(table)
kwh_min = get_acumulado_dia_kwh_min(table)
lan_o2 = get_lan_o2(table)
carvao = get_carvao(table)
cj_o2 = get_cj_o2(table)
cj_gn = get_cj_gn(table)
index = check_dia(csv, dia)
if (index == -1):
valores.append(dia)
valores.append(acumulado)
valores.append(n_corridas)
valores.append(pon_poff_values[0])
valores.append(pon_poff_values[1])
valores.append(pon_poff_values[2])
valores.append(pon_poff_values[3])
valores.append(sucata)
valores.append(potencia_mw)
valores.append(potencia_kwh)
valores.append(kwh_t)
valores.append(kwh_min)
valores.append(lan_o2)
valores.append(carvao)
valores.append(cj_o2)
valores.append(cj_gn)
csv.append(valores)
else:
csv[index][1] = float(csv[index][1]) + float(acumulado)
csv[index][2] = float(csv[index][2]) + float(n_corridas)
csv[index][3] = float(csv[index][3]) + float(pon_poff_values[0])
csv[index][4] = float(csv[index][4]) + float(pon_poff_values[1])
csv[index][5] = float(csv[index][5]) + float(pon_poff_values[2])
csv[index][6] = float(csv[index][6]) + float(pon_poff_values[3])
csv[index][7] = float(csv[index][7]) + float(sucata)
csv[index][8] = float(csv[index][8]) + float(potencia_mw)
csv[index][9] = float(csv[index][9]) + float(potencia_kwh)
csv[index][10] = float(csv[index][10]) + float(kwh_t)
csv[index][11] = float(csv[index][11]) + float(kwh_min)
csv[index][12] = float(csv[index][12]) + float(lan_o2)
csv[index][13] = float(csv[index][13]) + float(carvao)
csv[index][14] = float(csv[index][14]) + float(cj_o2)
csv[index][15] = float(csv[index][15]) + float(cj_gn)
return csv
def load_csv(filename, sep=',', training_indices=(20, -20), runningTest=True):
csv = []
with open(filename, 'r') as file:
for line in file:
line = line.split(sep)
if training_indices != None:
if runningTest == True:
del line[training_indices[0]:training_indices[1]]
else:
line = line[training_indices[0]:training_indices[1]]
csv.append([float(j) for j in line])
return np.array(csv, dtype=np.float64)
def list_csv(input_dir):
csv = []
try:
files = os.listdir(input_dir)
except:
return csv
for i in files:
if i[0] == '.':
continue
if len(i) < 3:
continue
if i[-4:] == '.csv':
csv.append(i)
return csv
def rep_to_csv(rep):
csv = [[]]
header = []
header.append("Branch")
for key, value in rep.items():
row = []
row.append(key)
for branch_key, branch_value in value.items():
if header.count(branch_key) == 0:
header.append(branch_key)
row.append(branch_value)
else:
row.insert(header.index(branch_key), branch_value)
csv.append(row)
csv.insert(0, header)
return csv
def format_csv(data):
import csv
features = data[0]['features']
# build header
header = []
for feature in features:
feature.update(feature['properties'])
if 'Taxonomy' in feature and feature['Taxonomy'] is not None:
feature.update(feature['Taxonomy'])
del feature['Taxonomy']
if feature['geometry'] is not None:
feature.update({"Longitude": feature['geometry']['coordinates'][0], "Latitude": feature['geometry']['coordinates'][1]})
del feature['properties']
del feature['geometry']
for key in feature:
if key not in header:
header.append(key)
header.sort()
log.debug(header)
# populate rows
csv = []
csv.append(','.join(header))
with open('data.csv', 'w', newline='') as csvfile:
for feature in features:
row = []
for column in header:
if column in feature:
value = feature[column]
if type(value) == str:
value = strings.singlespace(value)
value.replace('"', "'")
value = "%s" % value
row.append(str(value).replace(",", ""))
else:
row.append("None")
csv.append(','.join(row))
return '\n'.join(csv)
# print(json.dumps(features, indent=4, default=lambda x: str(x)))
def financials(stockname="MSFT"):
url = 'https://ca.finance.yahoo.com/quote/%s/financials' % (stockname)
obj = requests.get(url)
souppage = soup(obj.text, "html.parser")
tableithink = souppage.findAll("span")
lists = []
for row in tableithink:
lists.append(row)
for i in range(len(lists)):
lists[i] = str(lists[i]).split(">")[1].replace("</span", "")
#print(lists[i])
for i in range(len(lists)):
if (lists[i] == "Revenue"):
j = i
elif lists[i] == "":
break
csv = []
#print(i)
while j < i:
if (any(k.isdigit() for k in lists[j + 1])):
csv.append([
lists[j], lists[j + 1], lists[j + 2], lists[j + 3],
lists[j + 4]
])
j += 5
else:
csv.append([lists[j], " ", " ", " ", " "])
j += 1
finalcsv = ''
for x in csv:
for i in x:
finalcsv += i.replace(",", " ") + ","
finalcsv += "\n"
f = open('financials.csv', 'w')
f.write(finalcsv)
f.close()
for x in csv:
print(x)
jsonMaker3()
def createCsv(tArr):
header = ["created_at", "views", "addViews", "signatures", "addSignatures"]
csv = []
csv.append("created_at,views,addViews,signatures,addSignatures")
delimiter = ","
for tEle in tArr:
line = ""
for i, key in enumerate(header):
line += str(tEle[key])
if i != (len(header) - 1):
line += delimiter
csv.append(line)
#print(line)
return csv
def view_att():
if request.method == "POST":
query_date = request.args.get('date_time')
query_date = str(query_date)
att_final = Update_form.query.filter_by(date_created=query_date).all()
csv = []
for i in list(att_final):
csv.append(str(i).split("-")[1].strip())
csv = ",".join(csv)
return Response(csv,
mimetype="text/csv",
headers={
"Content-disposition":
"attachment; filename=Attendance-" + query_date +
".csv"
})
return ("Hello World!{}".format(att_final))
def main(_):
parser = ArgumentParser(
description="This is a small tool for converting a CSV playlist file "
"generated by e.g. Microsoft Excel to JSON playlist files, "
"which can be used by AVTrack360. If you want to use this tool for "
"generating the randomized playlists, "
"you have to set the parameters directly in the code.")
parser.add_argument(
"-csvfile",
dest="csvfile",
help="The path of the CSV file you want to convert. Default: dummy.csv",
type=str)
arg = parser.parse_args()
hrcs = 8
srcs = 8
pvss = hrcs * srcs
extension = "mkv"
projection_scheme = "barrel360"
hmd = "vive"
subjects = 32
csv = ["label;filename;extension;projectionscheme;hmd"]
if arg.csvfile:
convert_to_avtrack360_playlist(arg.csvfile)
else:
for subject in range(1, subjects + 1):
csv_subject = []
for src in range(1, srcs + 1):
for hrc in range(1, hrcs + 1):
csv_subject.append(
"%s;SRC%s_HRC%03d;%s;%s;%s" %
(subject, src, hrc, extension, projection_scheme, hmd))
shuffle(csv_subject)
for element in csv_subject:
csv.append(element)
with open('playlists\\generated_playlist.csv', mode='w') as csv_file:
for element in csv:
csv_file.write("%s\n" % element)
convert_to_avtrack360_playlist("generated_playlist.csv")
def GET(self):
user = accounts.get_current_user()
username = user.key.split('/')[-1]
books = Bookshelves.get_users_logged_books(username, limit=10000)
csv = []
csv.append('Work Id,Edition Id,Bookshelf\n')
mapping = {
1: 'Want to Read',
2: 'Currently Reading',
3: 'Already Read'
}
for book in books:
row = [
'OL{}W'.format(book['work_id']), 'OL{}M'.format(
book['edition_id']) if book['edition_id'] else '',
'{}\n'.format(mapping[book['bookshelf_id']])
]
csv.append(','.join(row))
web.header('Content-Type', 'text/csv')
web.header('Content-disposition',
'attachment; filename=OpenLibrary_ReadingLog.csv')
csv = ''.join(csv)
return delegate.RawText(csv, content_type="text/csv")
def search(tweet):
for key in _emojis.keys():
# print key
# emojis = _emojis[key]
# yield emojis
if key in tweet:
# print "key: %s, value: %s" % (key, _emojis[key])
yield _emojis[key]
print _emojis[key]
# else: return False
# def remove_left(f):
num = 0
csv = []
while True:
data = f.readline()
if data == '':
break
if (num < 1 or num > 8):
csv.append(data)
num += 1
f.seek(14)
for row in csv:
f.write(row)
def _csv_lines(self, cr, uid, ids, context=None):
csv = []
for id in ids:
(invoice_data, invoice_number, partner_id, lines) = self._document(cr, uid, id, context)
csv.append(invoice_data)
(partner_data, partner_name) = self._partner(cr, uid, partner_id, context)
csv.append(partner_data)
csv.append(self._items(cr, uid, lines, context))
return (csv, invoice_number, partner_name)
def _csv_lines(self, cr, uid, ids, context=None):
csv = []
for id in ids:
(invoice_data, invoice_number, partner_id,
lines) = self._document(cr, uid, id, context)
csv.append(invoice_data)
(partner_data,
partner_name) = self._partner(cr, uid, partner_id, context)
csv.append(partner_data)
csv.append(self._items(cr, uid, lines, context))
return (csv, invoice_number, partner_name)
def xml_converter():
tree = ET.parse("book_catalog.xml")
root = tree.getroot()
csv = []
for child in root:
book = ["{0} id={1}".format(child.tag, child.get("id"))]
csv.append(book)
row = []
if (len(csv) == 1):
csv.append([
child[0].tag, child[1].tag, child[0].tag, child[3].tag,
child[4].tag, child[5].tag
])
for sub_child in child:
row.append(sub_child.text)
csv.append(row)
return csv
def find_csv():
csv = []
for f in os.listdir(os.getcwd()):
if f.startswith("wikipedia_contributors") and f.endswith(".csv"):
csv.append(f)
return csv
data = params['data']
d = {}
csvfile = open('test.csv', 'w', newline='')
writer = csv.writer(csvfile)
writer.writerow(['num', 'h', 'd', 'a'])
for key in data.keys():
csv = []
d['id'] = data[key]['id']
d['a'] = float(data[key]['had']['a'])
d['d'] = float(data[key]['had']['d'])
d['h'] = float(data[key]['had']['h'])
d['num'] = data[key]['num']
d['h_cn'] = data[key]['h_cn']
d['h_id'] = data[key]['h_id']
d['a_cn'] = data[key]['a_cn']
d['a_id'] = data[key]['a_id']
d['date'] = data[key]['date']
d['time'] = data[key]['time']
print(d)
#db.match.insert(d)
csv.append(d['num'])
csv.append(d['h'])
csv.append(d['d'])
csv.append(d['a'])
writer.writerow(csv)
csvfile.close()
# resp = res.decode(encoding='utf-8').split('(')[1].split(')')[0]
# params = json.loads(resp)
# print(resp)
def print_report(self, cr, uid, ids, context=None):
o = self.browse(cr, uid, ids)[0]
if context is None:
context = {}
csv = []
debit = 0.0; credit = 0.0; cash = 0.0; transfer = 0.0
obj_product = self.pool.get('product.product')
obj_invoice = self.pool.get('account.invoice')
obj_order = self.pool.get('sale.order')
obj_order_line = self.pool.get('sale.order.line')
obj_pos = self.pool.get('pos.order')
obj_pos_line = self.pool.get('pos.order.line')
obj_picking = self.pool.get('stock.picking')
obj_stock_move = self.pool.get('stock.move')
obj_move = self.pool.get('stock.order')
obj_move_line = self.pool.get('account.move.line')
if o.name == 'stock' :
out = False; inn = False
no = 0
for x in o.opname_id.inventory_line_id:
no += 1
out = obj_stock_move.search(cr, uid, [
('product_id', '=', x.product_id.id),
('location_id', '=', o.shop_id.warehouse_id.lot_stock_id.id),
('date', '>=', o.opname_id.date),
('date', '<=', o.dari),
('state', '=', 'done'),
('name', 'ilike', 'INV')
])
inn = obj_stock_move.search(cr, uid, [
('product_id', '=', x.product_id.id),
('location_dest_id', '=', o.shop_id.warehouse_id.lot_stock_id.id),
('date', '>=', o.opname_id.date),
('date', '<=', o.dari),
('state', '=', 'done'),
('name', 'ilike', 'INV')
])
masuk = sum([x.product_qty for x in obj_stock_move.browse(cr, uid, inn)])
keluar = sum([x.product_qty for x in obj_stock_move.browse(cr, uid, out)])
total = x.product_qty + masuk - keluar
csv.append([no, x.product_id.partner_ref, x.product_qty, masuk, keluar, total])
elif o.name == 'shop' or o.name == 'shopdf' :
pid = obj_pos.search(cr, uid, [('shop_id', '=', o.shop_id.id), ('date_order', '>=', o.dari), ('date_order', '<=', o.dari), ('state', 'not in', ('draft', 'cancel'))])
pad = obj_pos_line.search(cr, uid, [('order_id','in', pid)])
sid = obj_order.search(cr, uid, [('shop_id', '=', o.shop_id.id), ('date_order', '>=', o.dari), ('date_order', '<=', o.dari), ('state', 'not in', ('draft', 'cancel', 'shipping_except', 'invoice_except'))])
back = datetime.datetime.strptime(o.dari,'%Y-%m-%d') - datetime.timedelta(days=30)
bid = obj_order.search(cr, uid, [('shop_id', '=', o.shop_id.id), ('date_order', '>=', str(back)), ('date_order', '<', o.dari), ('state', 'not in', ('draft', 'cancel', 'shipping_except', 'invoice_except'))])
origin = [x.name for x in obj_order.browse(cr, uid, bid)]
yid = obj_invoice.search(cr, uid, [('origin', 'in', origin), ('state', '!=', 'draft')])
number = [x.number for x in obj_invoice.browse(cr, uid, yid)]
nid = obj_move_line.search(cr, uid, [('name', 'in', number), ('date', '>=', o.dari), ('date', '<=', o.dari)])
if sid:
amount_total = 0.0; tot_inv = 0.0; payment = 0.0
for x in obj_order.browse(cr, uid, sid):
iid = obj_invoice.search(cr, uid, [('origin', '=', x.name), ('state', '!=', 'draft')])
inv = '-'; sisa = 0.0
if iid:
iad = obj_invoice.browse(cr, uid, iid)[0]
inv = iad.number; sisa = iad.residual
else:
sisa = '-'
amount_total += x.amount_total
csv.append([
x.name,
x.shop_id.name,
[i.product_id.partner_ref for i in x.order_line],
sum([i.product_uom_qty for i in x.order_line]),
'-',
x.partner_id.name,
x.amount_total,
inv,
sisa,
'-', '-', '-', '-'
])
if iid:
for i in iad.payment_ids:
payment += i.credit
if i.journal_id.jenis == "db":
debit += i.credit
elif i.journal_id.jenis == "cr":
credit += i.credit
elif i.journal_id.jenis == "cash":
cash += i.credit
elif i.journal_id.jenis == "transfer":
transfer += i.credit
csv.append([
'-', '-', '-', '-', '-', time.strftime('%d %B %Y', time.strptime(i.date,'%Y-%m-%d')), '-', '-', '-',
i.ref, i.date, i.credit, i.setor
])
if nid :
csv.append(['-', '-', '-', '-', '-', '-', '-', '-', '-', '-', '-', '-', '-'])
for i in obj_move_line.browse(cr, uid, nid):
so = '-'
cari = obj_invoice.search(cr, uid, [('number', '=', i.name)])
dpt = obj_invoice.browse(cr, uid, cari)
if dpt:
so = dpt[0].origin
payment += i.credit
if i.journal_id.jenis == "db":
debit += i.credit
elif i.journal_id.jenis == "cr":
credit += i.credit
elif i.journal_id.jenis == "cash":
cash += i.credit
elif i.journal_id.jenis == "transfer":
transfer += i.credit
csv.append([
so, '-', 'Pelunasan Invoice ' + i.name, '-', '-', time.strftime('%d %B %Y', time.strptime(i.date,'%Y-%m-%d')), '-', '-', '-',
i.ref, i.date, i.credit, i.setor
])
csv.append(['-', '-', '-', '-', '-', '-', '-', '-', '-', '-', '-', '-', '-'])
csv.append([
'Total Sales Order',
'-',
'-',
'-',
'-',
'-',
amount_total,
'-',
0.0,
'-', '-', payment, '-'
])
csv.append(['-', '-', '-', '-', '-', '-', '-', '-', '-', '-', '-', '-', '-'])
if pad:
pos = ''; outlet = '-'; cake = ''; harga = 0
if o.name == 'shopdf':
for a in obj_pos.browse(cr, uid, pid):
for i in a.statement_ids:
if i.journal_id.jenis == "db":
debit += i.amount
elif i.journal_id.jenis == "cr":
credit += i.amount
elif i.journal_id.jenis == "cash":
cash += i.amount
elif i.journal_id.jenis == "transfer":
transfer += i.amount
for x in obj_pos_line.browse(cr, uid, pad):
pos = 'Rekapan POS'
outlet = x.order_id.shop_id.name
cake = 'Rekapan Product'
harga += x.price_subtotal
csv.append([
x.order_id.name,
outlet,
x.product_id.partner_ref,
x.qty,
'-', '-', '-', '-', '-', '-', '-', x.price_subtotal, '-'
])
csv.append([
pos,
outlet,
cake,
'-',
'-', '-', 0, '-', 0, '-', '-', harga, '-'
])
else:
for x in obj_pos_line.browse(cr, uid, pad):
pos += x.order_id.name + '/'
outlet = x.order_id.shop_id.name
cake += x.product_id.partner_ref + '/'
harga += x.price_unit
csv.append([
pos,
outlet,
cake,
'-',
harga,
'-', '-', '-', '-', '-', '-', '-', '-'
])
data = self.read(cr, uid, ids)[0]
datas = {'ids': [data['id']]}
datas['model'] = 'sale.varian.report'
datas['form'] = data
datas['csv'] = csv
datas['debit'] = debit
datas['credit'] = credit
datas['cash'] = cash
datas['transfer'] = transfer
title = 'sale.varian'
if data['name'] == 'shop':
title = 'wtc.shop.excel'
elif data['name'] == 'shopdf':
title = 'wtc.shop.pdf'
elif data['name'] == 'stock':
title = 'sale.stock.pdf'
return {
'type': 'ir.actions.report.xml',
'report_name': title,
'nodestroy': True,
'datas': datas,
}
def target2csv_exp(t):
#print "[DEBUG] Processing target {}".format(t['id'])
ttdls = {}
if 'tdl_infos' in t:
ttdls = t['tdl_infos']
p = t['components']['protein'][0]
ptdls = p['tdl_infos']
if not p['dtoid']: p['dtoid'] = ''
if not p['dtoclass']: p['dtoclass'] = ''
if t['idg']:
idg = 1
else:
idg = 0
csv = [ t['id'], p['name'], p['description'], p['sym'], p['geneid'], p['uniprot'], p['stringid'], t['tdl'], idg, p['dtoid'], p['dtoclass'] ]
if 'panther_classes' in p:
csv.append( '|'.join(["%s:%s"%(d['pcid'],d['name']) for d in p['panther_classes']]) )
else:
csv.append('')
if 'generifs' in p:
csv.append( len(p['generifs']) )
else:
csv.append(0)
if 'NCBI Gene PubMed Count' in ptdls:
csv.append( ptdls['NCBI Gene PubMed Count']['value'] )
else:
csv.append(0)
if 'JensenLab PubMed Score' in ptdls:
csv.append( ptdls['JensenLab PubMed Score']['value'] )
else:
csv.append(0)
if 'PubTator Score' in ptdls:
csv.append( ptdls['PubTator Score']['value'] )
else:
csv.append(0)
csv.append( ptdls['Ab Count']['value'] )
csv.append( ptdls['MAb Count']['value'] )
# Activities
if 'cmpd_activities' in t:
csv.append( len(t['cmpd_activities']) )
else:
csv.append(0)
#csv.append('')
# ChEMBL
if 'ChEMBL Selective Compound' in ttdls:
csv.append( ttdls['ChEMBL Selective Compound']['value'] )
else:
csv.append('')
if 'ChEMBL First Reference Year' in ttdls:
csv.append( ttdls['ChEMBL First Reference Year']['value'] )
else:
csv.append('')
# DrugCentral
if 'drug_activities' in t:
csv.append( len(t['drug_activities']) )
else:
csv.append(0)
#csv.append('')
# PDB
if 'PDB' in p['xrefs']:
pdbs = [d['value'] for d in p['xrefs']['PDB']]
csv.append( len(pdbs) )
csv.append( "|".join(pdbs) )
else:
csv.append(0)
csv.append('')
# GO
if 'goas' in p:
csv.append( len(p['goas']) )
else:
csv.append(0)
if 'Experimental MF/BP Leaf Term GOA' in ptdls:
csv.append( ptdls['Experimental MF/BP Leaf Term GOA']['value'] )
else:
csv.append(0)
# Phenotypes
if 'phenotypes' in p:
omims = [d['trait'] for d in p['phenotypes'] if d['ptype'] == 'OMIM']
if len(omims) > 0:
csv.append( len(omims) )
csv.append( "|".join(omims) )
else:
csv.append('')
csv.append('')
jaxs = ["%s:%s"%(d['term_id'],d['term_name']) for d in p['phenotypes'] if d['ptype'] == 'JAX/MGI Human Ortholog Phenotype']
if jaxs:
csv.append( len(jaxs) )
csv.append( '|'.join(jaxs) )
else:
csv.append('')
csv.append('')
else:
csv.append('')
csv.append('')
csv.append('')
csv.append('')
# IMPC phenotypes
if 'impcs' in p:
pts = ["%s:%s"%(d['term_id'],d['term_name']) for d in p['impcs']]
csv.append( len(pts) )
csv.append( '|'.join(pts) )
else:
csv.append('')
csv.append('')
# GWAS
if 'gwases' in p:
gwases = ["%s (%s):%s"%(d['disease_trait'],d['mapped_trait_uri'],d['p_value']) for d in p['gwases']]
csv.append( len(gwases) )
csv.append( '|'.join(gwases) )
else:
csv.append('')
csv.append('')
# Pathways
if 'pathways' in p:
pathways = ["%s:%s"%(d['pwtype'],d['name']) for d in p['pathways']]
csv.append( len(pathways) )
csv.append( "|".join(pathways) )
else:
csv.append('')
csv.append('')
# Diseases
if 'diseases' in p:
uniq = set( [d['name'] for d in p['diseases']] )
csv.append( len(uniq) )
# Top text-mining diseases
tmdiseases = ["%s (ZScore: %s)"%(d['name'],str(d['zscore'])) for d in p['diseases'] if d['dtype'] == 'JensenLab Text Mining']
if len(tmdiseases) > 0:
csv.append( "|".join(tmdiseases[:5]) ) # Only top 5
else:
csv.append('')
# eRAM diseases
erams = [d for d in p['diseases'] if d['dtype'] == 'eRAM']
if len(erams) > 0:
csv.append( "|".join(["%s: %s"%(d['did'],d['name']) for d in erams]) )
else:
csv.append('')
else:
csv.append('')
csv.append('')
csv.append('')
# Patent Count
if 'EBI Total Patent Count' in ptdls:
csv.append( ptdls['EBI Total Patent Count']['value'] )
else:
csv.append(0)
# Is TF
if 'Is Transcription Factor' in ptdls:
csv.append(1)
else:
csv.append(0)
if 'TMHMM Prediction' in ptdls:
m = re.search(r'PredHel=(\d)', ptdls['TMHMM Prediction']['value'])
if m:
csv.append(m.groups()[0])
else:
csv.append(0)
else:
csv.append(0)
# Tissue specificity
if 'HPA Tissue Specificity Index' in ptdls:
csv.append(ptdls['HPA Tissue Specificity Index']['value'])
else:
csv.append('')
if 'HPM Gene Tissue Specificity Index' in ptdls:
csv.append(ptdls['HPM Gene Tissue Specificity Index']['value'])
else:
csv.append('')
if 'HPM Protein Tissue Specificity Index' in ptdls:
csv.append(ptdls['HPM Protein Tissue Specificity Index']['value'])
else:
csv.append('')
# TIN-X
if 'tinx_novelty' in p:
csv.append(p['tinx_novelty'])
else:
csv.append('')
if 'tinx_importances' in p:
# these come back ordered by score DESC. Only output top 5
txis = ["%s: %s"%(d['disease'],str(d['score'])) for d in p['tinx_importances'][:5]]
csv.append( "|".join(txis) )
else:
csv.append('')
return csv
def process_qsub_attributes():
rawAttributes = self.nodePoolDesc.getAttrs()
# 'W:x' is used to specify torque management extentensions ie -W x= ...
resourceManagementExtensions = ''
if 'W:x' in rawAttributes:
resourceManagementExtensions = rawAttributes['W:x']
if qosLevel:
if len(resourceManagementExtensions) > 0:
resourceManagementExtensions += ';'
resourceManagementExtensions += 'QOS:%s' % (qosLevel)
rawAttributes['W:x'] = resourceManagementExtensions
hostname = local_fqdn()
rawAttributes['l:nodes'] = nodeSet._getNumNodes()
if walltime:
rawAttributes['l:walltime'] = walltime
#create a dict of dictionaries for
# various arguments of torque
cmds = {}
for key in rawAttributes:
value = rawAttributes[key]
if key.find(':') == -1:
raise ValueError, 'Syntax error: missing colon after %s in %s=%s' % (
key, key, value)
[option, subOption] = key.split(':', 1)
if not option in cmds:
cmds[option] = {}
cmds[option][subOption] = value
opts = []
#create a string from this
#dictionary of dictionaries createde above
for k in cmds:
csv = []
nv = cmds[k]
for n in nv:
v = nv[n]
if len(n) == 0:
csv.append(v)
else:
csv.append('%s=%s' % (n, v))
opts.append('-%s' % (k))
opts.append(','.join(csv))
for option in cmds:
commandList = []
for subOption in cmds[option]:
value = cmds[option][subOption]
if len(subOption) == 0:
commandList.append(value)
else:
commandList.append("%s=%s" % (subOption, value))
opts.append('-%s' % option)
opts.append(','.join(commandList))
return opts
line = re.sub('#\d\\^', "", line.rstrip())
line = re.sub('(^style=.+)|-|<sup>...</sup>|\\(|\\)', "", line.rstrip())
line = re.sub(' , ', " ", line.rstrip())
lines.append(line)
num_lines = len(lines)
i = 0
outputFile = open("output.csv", 'wb')
wr = csv.write(outputFile)
while (not re.match('[A-Z]{3}', lines[i])):
i += 1
csv = []
csv.append(["country", "year", "placing"])
while (i < num_lines and re.match('[A-Z]{3}', lines[i])):
country = lines[i].rstrip()
x = [1, 2, 3, 4]
for placing in x:
l2 = lines[i + placing]
if (not re.match('align=centersort dash', l2)):
for year in l2.split(" "):
if len(year) > 1:
csv.append([country, year, str(placing)])
#csv.append(country + ", " + year + ", " + str(placing))
i += 5
while (i < num_lines and not re.match('[A-Z]{3}', lines[i])):
i += 1
def PyPoll(data):
#Initialize some variables
total_votes = 0
candidate_voted_for = []
csv = []
for row in data:
csv.append(row)
for row in csv:
total_votes += 1
candidate_voted_for.append(row[2])
#candidate = csv[2]
#Pull unique elements from the list of candidates voted for
unique_candidates = set(candidate_voted_for)
#convert set back to a list
unique_candidates = list(unique_candidates)
#Initialize my list of candidate counts based on the index of unique_candidates
candidate_count = []
for row in csv:
#Start the count at 0 for each candidate
for candidate in unique_candidates:
candidate_count.append(0)
#for i in range(len(unique_candidates)):
for i in range(4):
if row[2] == unique_candidates[i]:
#What are we going to do if this is true? add to that index in the candidate_count
candidate_count[i] += 1
candidate_1 = unique_candidates[0]
candidate_2 = unique_candidates[1]
candidate_3 = unique_candidates[2]
candidate_4 = unique_candidates[3]
candidate_1_count = candidate_count[0]
candidate_2_count = candidate_count[1]
candidate_3_count = candidate_count[2]
candidate_4_count = candidate_count[3]
#percentage votes each candidate won
#candidate_percentage = []
# winner = []
# for i in range(len(unique_candidates)): # or range(len(candidate_count))
# #candidate_percentage[i].append(round((int(candidate_count[i])/int(total_votes[i]))*100,2)) #IndexError: list index out of range
# #who had the most votes?
# if i == 0:
# winner = unique_candidates[i]
# else: #i = 1,2,3
# #elif i == 1 or i == 2 or i == 3:
# if candidate_count[i] > winner:
# winner = unique_candidates[i]
#percentages of votes
candidate_1_per = round((int(candidate_1_count) / int(total_votes)) * 100,
1)
candidate_2_per = round((int(candidate_2_count) / int(total_votes)) * 100,
1)
candidate_3_per = round((int(candidate_3_count) / int(total_votes)) * 100,
1)
candidate_4_per = round((int(candidate_4_count) / int(total_votes)) * 100,
1)
#Determine the winner
winner = []
#Candidate 1 winner scenario:
if (candidate_1_count > candidate_2_count) and (
candidate_1_count > candidate_3_count) and (candidate_1_count >
candidate_4_count):
#Is there a way to write this like "if candidate_1_count > (candidate_2_count and candidate_3_count and candidate_4_count)"?
winner = candidate_1
#Candidate 2 winner scenario:
elif (candidate_2_count > candidate_1_count) and (
candidate_2_count > candidate_3_count) and (candidate_2_count >
candidate_4_count):
winner = candidate_2
#Candidate 3 winner scenario:
elif (candidate_3_count > candidate_2_count) and (
candidate_3_count > candidate_1_count) and (candidate_3_count >
candidate_4_count):
winner = candidate_3
return [
total_votes, candidate_1, candidate_1_count, candidate_1_per,
candidate_2, candidate_2_count, candidate_2_per, candidate_3,
candidate_3_count, candidate_3_per, candidate_4, candidate_4_count,
candidate_4_per, winner
]
line = re.sub('(^style=.+)|-|<sup>...</sup>|\\(|\\)', "", line.rstrip())
line = re.sub(' , ', " ", line.rstrip())
lines.append(line)
num_lines = len(lines)
i = 0
outputFile = open("output.csv", 'wb')
wr = csv.write(outputFile);
while(not re.match('[A-Z]{3}', lines[i])):
i += 1
csv = []
csv.append(["country", "year", "placing"])
while(i < num_lines and re.match('[A-Z]{3}', lines[i])):
country = lines[i].rstrip()
x = [1,2,3,4]
for placing in x:
l2 = lines[i+placing]
if(not re.match('align=centersort dash', l2)):
for year in l2.split(" "):
if len(year) > 1:
csv.append([country, year, str(placing)])
#csv.append(country + ", " + year + ", " + str(placing))
i += 5
while(i < num_lines and not re.match('[A-Z]{3}', lines[i])):
i += 1
if current_temperature == 9999:
lcd.top("Temperature")
lcd.bottom("Failed to read")
lcd.cleanup()
sys.exit(0)
probe_minute_01.append(current_temperature)
lcd.top("{:2.1f}".format(current_temperature) + chr(223) + "C " + current_time.strftime("%H:%M:%S"))
if last_minute != current_minute:
lcd.display_init()
probe_minute_15.append(current_temperature)
probes_minute_30.append(current_temperature)
probes_minute_60.append(current_temperature)
csv.append(current_time.strftime("%s") + ";" + str(current_time) + ";" + "{:2.1f}".format(
current_temperature).replace('.', ',') + "\n")
lcd.bottom("{:2.1f}".format(probes_minute_60.average) + chr(223) + " " + "{:2.1f}".format(
probes_minute_30.average) + chr(223) + " " + "{:2.1f}".format(probe_minute_15.average) + chr(223))
time.sleep(2)
last_minute = current_minute
last_time = current_time
except KeyboardInterrupt:
lcd.cleanup()
sys.exit(0)
msg = master.recv_msg()
except KeyboardInterrupt:
break
if msg is not None:
msg_type = msg.get_type()
if msg_type == "MISSION_COUNT":
mission_count = msg.count
if mission_count > 0:
master.mav.mission_request_int_send(0, 0, 0, mission_type)
else:
print("no mission")
elif msg_type == "MISSION_ITEM_INT":
if msg.seq == expect_seq:
print("recv mission item", msg.seq)
csv.append(
str(msg.seq) + "," + str(msg.current) + "," +
str(msg.frame) + "," + str(msg.command) + "," +
str(msg.param1) + "," + str(msg.param2) + "," +
str(msg.param3) + "," + str(msg.param4) + "," +
str(msg.x) + "," + str(msg.y) + "," + str(msg.z) + "," +
str(msg.autocontinue) + "\n")
expect_seq = expect_seq + 1
if expect_seq < mission_count:
master.mav.mission_request_int_send(
0, 0, expect_seq, mission_type)
else:
master.mav.mission_ack_send(0, 0, 0, mission_type)
print("done")
with open(file_name, 'w') as out_file:
out_file.writelines(csv)
break
def _process_metrics_for_csv(self, csv, metric_results_dict, batch, attn_list, layer_1_entropy, input_seq_entropy, scores):
batch = [x.cpu().detach().numpy() for x in batch]
scores = scores.cpu().detach().numpy()
users = None
if len(batch) == 4:
seqs, candidates, labels, users = batch
elif len(batch) == 3:
seqs, candidates, labels = batch
assert users.shape[0] == 1
row = 0
internal_user_id = users[row][0]
new_row = [internal_user_id if users is not None else -1]
new_row += [metric_results_dict['NDCG@%d' % k] for k in self.metric_ks]
new_row += [metric_results_dict['Recall@%d' % k] for k in self.metric_ks]
new_row += self._map_internal_movie_list_to_original([int(candidates[0][0])])
attn_layer_1 = attn_list[0][0]
attn_layer_2 = attn_list[1][0]
top_left_coord_to_keep = (attn_layer_1[0] == 0).sum()
new_row += [200 - top_left_coord_to_keep]
attn_layer_1 = attn_layer_1[top_left_coord_to_keep:, top_left_coord_to_keep:]
attn_layer_2 = attn_layer_2[top_left_coord_to_keep:, top_left_coord_to_keep:]
csv.append(new_row)
minmax = {
325: [0.002, 0.03],
639: [0.018, 0.087],
616: [0.017, 0.083],
500: [0.005, 0.044],
127: [0.015, 0.085],
115: [0.003, 0.045],
187: [0.004, 0.045],
59: [0.004, 0.045],
627: [0.008, 0.058],
1094: [0.01, 0.059],
880: [0.002, 0.029],
973: [0.005, 0.045],
1906: [0.017, 0.082],
1968: [0.001, 0.029],
226 : [0.013, 0.072],
490: [0.0175, 0.08],
1807: [0.012, 0.07],
}
minmax_inp_seq = {
325: [0.0025, 0.018],
639: [0.021, 0.065],
616: [0.025, 0.065],
500: [0.01, 0.038],
127: [0.02, 0.065],
115: [0.005, 0.035],
187: [0.0075, 0.034],
59: [0.0075, 0.037],
627: [0.0011, 0.038],
1094: [0.015, 0.045],
880: [0.003, 0.0225],
973: [0.01, 0.039],
1906: [0.02, 0.06],
1968: [0.003, 0.025],
226 : [0.015, 0.057],
490: [0.02, 0.062],
1807: [0.013, 0.054],}
# minmax = {k:[None, None] for k,v in minmax.items()}
# minmax_inp_seq = {k: [None, None] for k, v in minmax_inp_seq.items()}
input_item_attn_projection = self._project_attention_on_input(attn_layer_1, attn_layer_2)
l1_entr = np.average((-attn_layer_1*np.log2(attn_layer_1)).sum(axis=1))
layer_1_entropy.append(l1_entr)
inp_entr = (-input_item_attn_projection*np.log2(input_item_attn_projection)).sum(axis=1)[0]
input_seq_entropy.append(inp_entr)
return csv, layer_1_entropy, input_seq_entropy
# if internal_user_id not in minmax:
# return csv, layer_1_entropy, input_seq_entropy
temp_name = 'core_'
root_dump = os.path.join('Images', 'AttentionTemp', str(internal_user_id))
# root_dump = os.path.join(self.export_root, 'logs', 'attention', str(internal_user_id))
Path(root_dump).mkdir(parents=True, exist_ok=True)
rank = (-scores).argsort(axis=1)
top10 = candidates[0][rank[0][:10]]
input_target_dict = {'target': self._map_internal_movie_list_to_original([int(candidates[0][0])]),
'predicted': self._map_internal_movie_list_to_original(top10.tolist()),
'input_projected_attn': input_item_attn_projection[0].tolist(),
'input': self._map_internal_movie_list_to_original([x for x in seqs[row].tolist() if x != 0])}
with open(os.path.join(root_dump, temp_name+'input_target.json'), 'w') as f:
json.dump(input_target_dict, f, indent=4)
min = minmax_inp_seq[internal_user_id][0]
max = minmax_inp_seq[internal_user_id][1]
# min, max = None, None
fig, ax = plt.subplots()
im = ax.imshow(input_item_attn_projection, cmap='coolwarm', interpolation=None, vmin=min, vmax=max)
cbar = ax.figure.colorbar(im, ax=ax)
cbar.ax.set_ylabel('Attention Weight')
plt.xlabel('Input Positions')
plt.yticks([], None)
ax.xaxis.set_major_locator(MaxNLocator(integer=True))
fig.savefig(os.path.join(root_dump, temp_name+'input_proj.png'), bbox_inches='tight')
fig.clf()
plt.close()
min = minmax[internal_user_id][0]
max = minmax[internal_user_id][1]
# min, max = None, None
fig, ax = plt.subplots()
im = ax.imshow(attn_layer_1, cmap='coolwarm', interpolation=None, vmin=min, vmax=max)
cbar = ax.figure.colorbar(im, ax=ax)
cbar.ax.set_ylabel('Attention Weight')
plt.xlabel('Key Positions')
plt.ylabel('Query Positions')
ax.yaxis.set_major_locator(MaxNLocator(integer=True))
ax.xaxis.set_major_locator(MaxNLocator(integer=True))
fig.savefig(os.path.join(root_dump, temp_name+'layer1.png'), bbox_inches='tight')
fig.clf()
plt.close()
fig2, ax2 = plt.subplots()
im2 = ax2.imshow(attn_layer_2, cmap='coolwarm', interpolation=None)
cbar2 = ax2.figure.colorbar(im2, ax=ax2)
cbar2.ax.set_ylabel('Attention Weight')
plt.xlabel('Key Positions')
plt.ylabel('Query Positions')
ax2.yaxis.set_major_locator(MaxNLocator(integer=True))
ax2.xaxis.set_major_locator(MaxNLocator(integer=True))
fig2.savefig(os.path.join(root_dump, temp_name+'layer2.png'), bbox_inches='tight')
fig2.clf()
plt.close()
return csv, layer_1_entropy, input_seq_entropy
import scipy
import csv
import numpy as np
subdirs = [
'Sirene', 'Auto', 'Flugzeug', 'PartyBabble', 'Straße', 'Waschmaschine'
] ## noise subdirs
db = [20, 15, 10, 5, 0, -5] ## all SNRs
csv = []
NET_TYPE = "cnn_oned_60"
#read in .csv files with evaluation metrics:
for h in range(0, 6):
csv.append(
np.genfromtxt('metrics' + str(NET_TYPE) + subdirs[h] +
'aprioSNR_mean-30.csv',
delimiter=","))
#which metrics shall be plotted:
PESQANDSTOI = 1
SNR15 = 0
POSTGAIN = 0
LDSPLOT = 0
# convert all metrics into one 3D-Array
csv = np.array(csv)
if PESQANDSTOI == 1:
"""
Plots STOI and PESQ Metrics for all noise types at all SNRs
def dados_cnae(cnae):
# url = "http://www.fieb.org.br/guia/Resultado_Consulta?CodCnae=C&NomeAtividade=IND%c3%9aSTRIAS%20DE%20TRANSFORMA%c3%87%c3%83O.&operProduto=and&localizacao=&ordenacao=ind_razao_social&page=0&consulta=Consultas%20Especiais"
p = urllib.parse.urlencode({
'CodCnae': cnae[0],
'NomeAtividade': cnae[1],
'operProduto': 'and',
'localizacao': '',
'ordenacao': 'ind_razao_social',
'page': 0,
'consulta': 'Consultas Especiais'
})
p.encode('ascii')
url = "http://www.fieb.org.br/guia/Resultado_Consulta.aspx?%s" % p
urlemp = []
csv = []
i = 1
pagina = requests.get(url)
html = BeautifulSoup(pagina.content, 'lxml')
# PEGAR QUANTIDADE DE PAGINAS NO RESULTADO DA CONSULTA
NUM_PAGINA = html.find(
id="ContentPlaceHolder1_generalContent_rpt_lblLastPage").text
URL_EMPRESA = html.find_all(
id="label-consulta-3") #LINKS DO PERFIL DA EMPRESA PAGINA 1
for alink in URL_EMPRESA:
urlemp.append(alink.a.get('href'))
for lnk in urlemp:
url = "http://www.fieb.org.br/guia/" + lnk
pagina = requests.get(url)
html = BeautifulSoup(pagina.content, 'lxml')
dados_emp = limpa_dados(list(html.find(id="divDadosIndustria")))
#print('Obtendo Links das paginas')
if dados_emp != None and dados_emp != False:
csv.append(u.parse_csv(dados_emp))
if int(NUM_PAGINA) > 1:
urlemp.clear()
while int(NUM_PAGINA) > i: # EXECUTA PAGINAÇÃO
p = urllib.parse.urlencode({
'CodCnae': cnae,
'NomeAtividade': cnae[1],
'operProduto': 'and',
'localizacao': '',
'ordenacao': 'ind_razao_social',
'page': i,
'consulta': 'Consultas Especiais'
})
p.encode('ascii')
url = "http://www.fieb.org.br/guia/Resultado_Consulta.aspx?%s" % p
pagina = requests.get(url)
html = BeautifulSoup(pagina.content, 'lxml')
total_emp = html.find_all(id="label-consulta-3")
for alink in total_emp:
urlemp.append(alink.a.get('href'))
c = 1
for lnk in urlemp:
url = "http://www.fieb.org.br/guia/" + lnk
pagina = requests.get(url)
html = BeautifulSoup(pagina.content, 'lxml')
dados_emp = limpa_dados(list(
html.find(id="divDadosIndustria")))
if dados_emp == False:
print(url)
print(html)
exit()
#if parse_csv(dados_emp) != None:
csv.append(u.parse_csv(dados_emp))
c += 1
urlemp.clear()
i += 1
# PEGA OS LINKS DAS EMPRESAS DA PAGINA
print('DADOS COLETADOS =', i)
export_csv(csv, cnae)
csv.clear()
