def read_csv(file, heading=True):
if heading:
with open(file, 'r') as csv:
lines = tuple(tuple(line.strip().split(',')) for line in csv.readlines()[1:])
return lines
else:
with open(file, 'r') as csv:
lines = tuple(tuple(line.strip().split(',')) for line in csv.readlines())
return lines
def fileScore(score):
''' Add results to the csv score file '''
csv = open("score.csv","a")
numberPlayer = []
scoreMove = []
descriptionMove = []
for line in csv.readlines():
line = line.rstrip("\n")
line = line.split(",")
numberPlayer.append(line[0])
scoreMove.append(line[1:])
descriptionMove.append(line[2:])
csv.close()
def main():
#initializes variables (reads file)
players = {}
score = {}
if __name__=="__main__":
main()
def write(msg):
""" Writes the message to UART_2
Doesn't return anything
"""
# Creates file if it doesn't exist
os.makedirs(os.path.dirname(file), exist_ok=True)
if not os.path.exists(file):
with open(file, 'w'):
pass
lines = []
# Grab the current UART data
with open(file, "r") as csv:
fcntl.flock(csv.fileno(), fcntl.LOCK_EX)
lines = csv.readlines()
# If the file hasn't been initialzed yet, set the two entries to empty
length = len(lines)
if length < 2:
for i in range(length,2):
lines.append('\n')
# Write back the UART data, modifying the specified one
with open(file, "w") as csv:
for (i, line) in enumerate(lines):
if i == 1:
csv.write(msg)
csv.write('\n')
else:
csv.write(line)
fcntl.flock(csv.fileno(), fcntl.LOCK_UN)
def process_tweets(csv_file, test_file=True):
"""Returns a list of tuples of type (tweet_id, feature_vector)
or (tweet_id, sentiment, feature_vector)
Args:
csv_file (str): Name of processed csv file generated by preprocess.py
test_file (bool, optional): If processing test file
Returns:
list: Of tuples
"""
tweets = []
print('Generating feature vectors')
with open(csv_file, 'r') as csv:
lines = csv.readlines()
total = len(lines)
for i, line in enumerate(lines):
if test_file:
tweet_id, tweet = line.split(',')
else:
tweet_id, sentiment, tweet = line.split(',')
feature_vector = get_feature_vector(tweet)
if test_file:
tweets.append((tweet_id, feature_vector))
else:
tweets.append((tweet_id, int(sentiment), feature_vector))
utils.write_status(i + 1, total)
print('\n')
return tweets
def writeGPIO(ignStates):
# Creates file if it doesn't exist
os.makedirs(os.path.dirname(file), exist_ok=True)
if not os.path.exists(file):
with open(file, 'w'):
pass
lines = []
# Grab the current GPIO data
with open(file, "r") as csv:
fcntl.flock(csv.fileno(), fcntl.LOCK_EX)
lines = csv.readlines()
#If the file hasn't been initialzed yet, set the two entries to empty
length = len(lines)
if length < 4:
for i in range(length, 2):
lines.append('\n')
with open(file, "w") as csv:
for (i, line) in enumerate(lines):
if i == 0:
csv.write(str(ignStates >> 8))
csv.write('\n')
else:
csv.write(line)
fcntl.flock(csv.fileno(), fcntl.LOCK_UN)
def split_train_test(original_path: str, train_path: str, test_path: str, num_instances: int, enc: str = 'utf-8'):
"""
Manual splitting of training and test data. This method is deprecated due to the
introduction of scikit-learn's methods such as train test split and stratified split validation
Keyword arguments:
original_path -- path to full dataset to be split
train_path -- the path to the new training data to be formed
test_path -- the path to the new testing data to be formed
num_instances -- total number of data instances in the dataset
enc -- the encoding to be used for reading and writing the files
"""
train_data = []
test_data = []
with open(original_path, 'r', encoding='utf-8') as csv:
for idx, line in enumerate(csv.readlines()):
if "[deleted]" in line:
continue
if idx <= (round(num_instances * 0.8)):
train_data.append(line)
else:
test_data.append(line)
with open(train_path, 'a', encoding='utf-8') as train:
for line in train_data:
train.write(line)
with open(test_path, 'a', encoding='utf-8') as test:
for line in test_data:
test.write(line)
def body_from_csv(self, env, csvfile):
ffpath = os.path.dirname(self.srcpath)
print(self.srcpath, os.getcwd())
filename = os.path.join(env.srcdir, ffpath, csvfile)
print("\n\nPATH=", self.srcpath)
with open(filename, "r") as csv:
content = csv.readlines()
content = [line[:-1] for line in content]
return self.body_to_csv(content)
def delete_blank_deadlines():
csv = open('data.csv', encoding='utf-8')
string = csv.readlines()
csv.close()
file = open("data.csv", "w", encoding="utf-8")
for i in string:
if not i.isspace():
str = i.replace('\n', '')
print(str)
file.write(str + "\n")
file.close()
def readGPIO():
os.makedirs(os.path.dirname(file), exist_ok=True)
if not os.path.exists(file):
with open(file, 'w'):
pass
lines = []
# Grab the current GPIO data
with open(file, "r") as csv:
fcntl.flock(csv.fileno(), fcntl.LOCK_EX)
lines = csv.readlines()
return int(lines[0])
def get_status(file_bool):
if file_bool == True:
csv_file = '/usr/local/projects/wifi_connectivity_bot/Record_Keeping/up_down_eth.csv'
else:
csv_file = '/usr/local/projects/wifi_connectivity_bot/Record_Keeping/up_down_wifi.csv'
with closing(open(csv_file, "r")) as csv:
list_file = csv.readlines()
# grabs final line from input file
final_line = list_file[len(list_file) - 1]
csv_row = final_line.split(",")
# grabs the first field from file
return csv_row[0]
def display_scorelist():
with open('highscore.csv') as csv:
from_file = csv.readlines()
splitted = [line.split(' | ') for line in from_file]
ranking = [(line[0], line[1], (line[2]).strip()) for line in splitted]
sorted_ranking = sorted(ranking, key=get_points)
for item in sorted_ranking:
print(item)
# ranking = get_score_list(points)
# add_result_to_scorelist(ranking)
# display_scorelist()
def read_file(filename):
"""
Read in data from file.
Input: File name
Output: List of rows as lists
"""
data_from_file = []
with open(filename) as csv:
reader = csv.readlines()
for row in reader:
data_from_file.append(row.strip().split(";"))
return data_from_file
def get_data_list(self,file):
'''
Given a csv file, this will return a 2d list,
data_list, where data_list has an index for each
header in row one. Each of these headers in turn
is a list, which contains all the data in their
column.
'''
list = []
with open(file, 'r') as csv:
list_of_lines = csv.readlines()
for i in range(len(list_of_lines)):
list.append(list_of_lines[i].split(','))
return list
def load_buildings_from_file(buildingCsv):
firstLine = True
with open(buildingCsv, 'r') as csv:
for row in csv.readlines():
if firstLine == True:
firstLine = False
else:
tokens = row.strip().split(",")
yield {
"project_id" : tokens[0],
"task_id" : tokens[1],
"way_id" : tokens[2],
"bounding_box" : tokens[3:]
}
def getOffsetsAndSensitivity(self):
# i=1
fileName = self.O3boardNo + '.csv'
csv = open(fileName, 'r')
hdr = csv.readline()
hdr = hdr.strip().split(',')
self.gasOffsetsAndSensitivity = defaultdict(list)
lines = csv.readlines()
for line in lines:
row = line.strip().split(',')
for i in range(1, len(hdr)):
key = str(hdr[i])
value = str(row[i])
self.gasOffsetsAndSensitivity[key].append(value)
csv.close()
return self.gasOffsetsAndSensitivity
def read_csv2(path):
csv = open(path, 'r')
data = []
keys = []
# collect keys from 1st line and init data
for i in csv.readline().rstrip().split(','):
keys.append(i)
# fill data with values
for i in csv.readlines():
obj = {}
for key, val in zip(keys, i.rstrip().split(',')):
if re.match(r'\d*\.?\d+', val):
obj[key] = float(val)
else:
obj[key] = val
data.append(obj)
return data
def process_senti_data(csv_file_name, type):
out_list = []
with open(csv_file_name, 'r') as csv:
lines = csv.readlines()
step = int(len(lines) / 6)
if type == 'train':
lines = lines[0:4 * step]
elif type == 'test':
lines = lines[4 * step + 1:5 * step]
else:
lines = lines[5 * step + 1:-1]
for line in lines:
line_split = line.strip().split(',')
tweet = line_split[2]
tweet = tweet.replace('USER_MENTION', '').replace('URL', '')
if len(tweet) > 0:
out_list.append(line_split[1] + '\t' + tweet)
return out_list
def filePlayers(self,players):
''' Reads csv players file '''
csv = open("desktop/master/players.csv","r")
numberPlayer = []
namePlayer = []
professionPlayer = []
for row in csv.readlines():
row = row.rstrip("\n")
row = row.split(",")
numberPlayer.append(row[0])
namePlayer.append(row[1])
professionPlayer.append(row[2])
for x in range(len(numberPlayer)):
players[numberPlayer[x]] = namePlayer[x],professionPlayer[x]
csv.close()
def process_tweets(csv_file, test_file=True):
tweets = []
labels = []
print('Generating feature vectors')
with open(csv_file, 'r') as csv:
lines = csv.readlines()
total = len(lines)
for i, line in enumerate(lines):
if test_file:
tweet_id, tweet = line.split(',')
else:
tweet_id, sentiment, tweet = line.split(',')
feature_vector = get_feature_vector(tweet)
if test_file:
tweets.append(feature_vector)
else:
tweets.append(feature_vector)
labels.append(int(sentiment))
utils.write_status(i + 1, total)
print('\n')
return tweets, np.array(labels)
def write(id, msg):
"""Writes the msg and id to CAN1
Format: id, msg, msg_length
Doesn't return anything
"""
# Creates file if it doesn't exist
os.makedirs(os.path.dirname(file), exist_ok=True)
if not os.path.exists(file):
with open(file, "w"):
pass
lines = []
# Grab the current CAN data
try:
with open(file, "r") as csv:
fcntl.flock(csv.fileno(), fcntl.LOCK_EX)
lines = csv.readlines()
except Exception:
pass
# If the file hasn't been initialzed yet, set the two entries to empty
length = len(lines)
if length < 2:
for i in range(length, 2):
lines.append("\n")
# Write back the CAN data, modifying the specified one
CANtext = "%s,%s,%d" % (id, msg, round((len(msg) - 2) / 2.0 + 0.1))
# CANtext = (id + ', ' + msg ', ' + (str)(len(msg)))
with open(file, "w") as csv:
for (i, line) in enumerate(lines):
if i == 0:
csv.write(CANtext)
csv.write("\n")
else:
csv.write(line)
fcntl.flock(csv.fileno(), fcntl.LOCK_UN)
def read_files():
csv = open("labels.csv")
csv_read = csv.readlines()
csv.close()
results = open("results.csv")
results_read = results.readlines()
results.close()
control_group = []
value_data = []
for lines in csv_read[1:]:
lines = tuple(lines.rstrip().split(","))
control_group.append(lines[1])
for lines in results_read[1:]:
lines = lines.rstrip()
lines = tuple(lines.split(";"))
value_data.append(lines)
return value_data, control_group
def preprocess_csv(csv_file_name, processed_file_name, test_file=False):
save_to_file = open(processed_file_name, 'w')
with open(csv_file_name, 'r') as csv:
lines = csv.readlines()
total = len(lines)
for i, line in enumerate(lines):
tweet_id = line[:line.find(',')]
if not test_file:
line = line[1 + line.find(','):]
positive = int(line[:line.find(',')])
line = line[1 + line.find(','):]
tweet = line
processed_tweet = preprocess_tweet(tweet)
if not test_file:
save_to_file.write('%s,%d,%s\n' %
(tweet_id, positive, processed_tweet))
else:
save_to_file.write('%s,%s\n' % (tweet_id, processed_tweet))
write_status(i + 1, total)
save_to_file.close()
print('\nSaved processed tweets to: %s' % processed_file_name)
return processed_file_name
# data_path = 'spring-2020.csv'
# with open(data_path, 'r') as f:
# reader = csv.reader(f, delimiter=',')
# headers = next(reader)
# data = np.array(list(reader)).astype(str)
# plt.plot(data[:, 0], data[:, 1])
# plt.bar
# plt.axis('equal')
# plt.xlabel("College")
# plt.ylabel("Population")
# plt.show()
csv = open("spring-2020.csv", "r")
file = csv.readlines()
college = []
for line in file:
items = line.split(",")
college = items[0]
print(line)
ray = np.array(line)
print(ray)
csv.close()
import os
import csv
import matplotlib.pyplot as plt
import matplotlib.dates as dates
import datetime
os.system("python GetTweets.py"
) #runs code that handles twitter API (saves to csv file)
os.system("python ExtractStockPrices.py"
) #runs code that handles yahoo finance API (saves to csv file)
with open('stock.csv', 'r') as csv: #need to open csv with read capabilities
content = csv.readlines()
content = [x.strip()
for x in content] #strip and split the lines from the csv file
content = [x.split(",") for x in content
] #each line is of the form <date>,<closing stock price>
stockDay = [] #initialize lists
stockPrice = []
for y in reversed(
content
): #need to reverse the content because yahoo finance API returns is backwards
stockDay.append(y[0]) #daylist gets first element
stockPrice.append(y[1]) #pricelist gets second element
with open(
'data.csv', 'r'
) as csv2: #similar to the getting of the info from the stock prices csv
newcont = csv2.readlines()
newcont = [n.strip() for n in newcont]
newcont = [n.split(",") for n in newcont]
import csv
#dfImport = pd.read_csv('imports.csv')
csv = open('exports1.csv', 'r')
products = ['All Products','Animal','Textiles and Clothing','Wood','Minerals','Food Products','Chemicals','Plastic or Rubber','Fuels','Mach and Elec']
years = ['1989','1990','1991','1992','1993','1994','1995','1996','1997','1998','1999','2000','2001','2002','2003','2004','2005','2006','2007','2008','2009','2010','2011','2012','2013','2014','2015']
#Reporter Name,Partner Name,Trade Flow,Product Group,Measure,1989,1990,1991,1992,1993,1994,1995,1996,1997,1998,1999,2000,2001,2002,2003,2004,2005,2006,2007,2008,2009,2010,2011,2012,2013,2014,2015
filename = 'relationaltrade.csv'
lines = csv.readlines()
def main():
f=open(filename, 'w')
newline="Reporter Name,Partner Name,Product Group,1989,1990,1991,1992,1993,1994,1995,1996,1997,1998,1999,2000,2001,2002,2003,2004,2005,2006,2007,2008,2009,2010,2011,2012,2013,2014,2015,Total\n"
for line in lines:
row=line.split(";")
if (row[3] in products):
if "," in row[0]:
row[0] ="\""+row[0]+"\""
if "," in row[1]:
row[1] ="\""+row[1]+"\""
newline=row[0]+","+row[1]+","+row[3]
total=0
for i in range(5, len(row)):
if (i == len(row)-1):
row[i]=row[i][:-1] #remove \n from last element
total=total+float(row[i])
newline=newline+","+row[i]
newline=newline+","+str(total)+"\n"
f.write(newline)
main()
def read_the_bathroom_codes():
filename = "bathroom_codes.txt"
with open(filename) as csv:
bathroom_codes = csv.readlines(filename)
return bathroom_codes
import csv
class MyDatabase
def filePlayers(players):
''' ask for name '''
name = raw_input("Enter your name or id:")
''' Reads csv players file '''
csv = open("players.csv","r")
numberPlayer = []
namePlayer = []
professionPlayer = []
for line in csv.readlines():
line = line.rstrip("\n")
line = line.split(",")
numberPlayer.append(line[0])
namePlayer.append(line[1:])
professionPlayer.append(line[2:])
for x in range(len(numberPlayer)):
players[numberPlayer[x]] = namePlayer[x]
csv.close()
def fileScore(score):
''' Add results to the csv score file '''
csv = open("score.csv","a")
import pandas as pd
import sys
values=[]
col=5
df1 = pd.read_csv("awards.csv",delimiter=',')
df2 = pd.read_csv("contracts.csv",delimiter=',')
df = pd.merge(df1, df2, how='outer', on=['contractname'])
df.to_csv("merged.csv", index=False)
saved_column=df.contractname
print(saved_column)
with open('merged.csv', 'r') as csv:
cols=[4]
for line in csv.readlines():
content = list(line[i] for i in cols)
print(content)
elements = line.strip().split(',')
try:
if int(elements[col]) == Amount:
values.append(int(elements[col]))
break
except ValueError:
csum = sum(values)
print("Sum of column %d: %f" % (col, csum))
def geolocator():
table_string = ""
def extract_rows_from(input_file, delimiter=",", map_format=int, encoding="utf-8",
mode='r'):
"""Extract a table from a csv file. Return a generator of rows"""
with open(input_file, mode=mode, encoding=encoding) as csv:
for row in csv.readlines():
yield [el for el in map(map_format, row[:-1].split(delimiter))]