def write_to_table_via_csv(table, rows, connection):
with temp_csv(mode='r+', tmp_dir=TEMPORARY_DIR) as csv_file:
csv.writer(csv_file, delimiter='\t').writerows(rows)
csv_file.seek(0, 0)
connection.connection.cursor().copy_from(csv_file, sep='\t', null='',
table=table.name)
connection.connection.commit()
def main():
PROG = os.path.basename(os.path.splitext(__file__)[0])
description = """Scan claims files"""
parser = OptionParser(option_class=MultipleOption,
usage='usage: %prog claims_file, claims_file, ...',
version='%s %s' % (PROG, VERSION),
description=description)
if len(sys.argv) == 1:
parser.parse_args(['--help'])
args = parser.parse_args()
p2k = {}
k2p = {}
try:
with open('claimants.csv') as csv_file:
for line in csv.reader(csv_file, dialect="excel"):
p2k[line[0]] = line[1]
k2p[line[1]] = line[0]
except IOError:
pass
for filename in args[1]:
with open(filename+'_masked.csv', 'wb') as cf:
outfile = csv.writer(cf, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
analyze_file(filename, outfile, p2k, k2p)
print len(p2k), len(k2p)
with open('claimants.csv', 'wb') as cf:
cout = csv.writer(cf, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
for p in p2k:
cout.writerow([p, p2k[p]])
def compute_output_csvs():
payees = [Payee(rec) for rec in csv.reader(open(INPUT_CSV))]
payees.sort(key=lambda o: o.gross, reverse=True)
total_fees = sum([payee.assess_fee() for payee in payees]) # side-effective!
total_net = sum([p.net for p in payees])
total_gross = sum([p.gross for p in payees])
assert total_fees + total_net == total_gross
paypal_csv = csv.writer(open(PAYPAL_CSV, 'w+'))
gittip_csv = csv.writer(open(GITTIP_CSV, 'w+'))
print_rule()
print("{:<24}{:<32} {:^7} {:^7} {:^7}".format("username", "email", "gross", "fee", "net"))
print_rule()
for payee in payees:
paypal_csv.writerow((payee.email, payee.net, "usd"))
gittip_csv.writerow(( payee.username
, payee.email
, payee.gross
, payee.fee
, payee.net
, payee.additional_note
))
print("{username:<24}{email:<32} {gross:>7} {fee:>7} {net:>7}".format(**payee.__dict__))
print(" "*56, "-"*23)
print("{:>64} {:>7} {:>7}".format(total_gross, total_fees, total_net))
def __init__(self, ni, nh, no):
'''the init function to set up the NN topology.'''
# nodes.
self._ni = ni + 1
self._nh = nh
self._no = no
# activiations.
self._ai = [1.0] * self._ni
self._ah = [1.0] * self._nh
self._ao = [1.0] * self._no
# weights.
self._wi = self._create_matrix(self._ni, self._nh)
self._wo = self._create_matrix(self._nh, self._no)
self._wi = self._matrix_random_init(self._wi, -0.01, 0.01)
self._wo = self._matrix_random_init(self._wo, -0.01, 0.01)
# momentums.
self._ci = self._create_matrix(self._ni, self._nh)
self._co = self._create_matrix(self._nh, self._no)
# error log.
self._tr_logger = csv.writer(open('dump/tr_error_log.csv', 'wb'), delimiter=',')
self._te_logger = csv.writer(open('dump/te_error_log.csv', 'wb'), delimiter=',')
def hp_interest(tenure, interest, amount, l_no): ### Calculate HP interest given the param
term_charge = float(amount * float(tenure/12)) * float(interest)/100 #Getting term charge
###print term_charge
int_constant = float(((tenure)*(tenure+1))/2) #Calculating interest constant for use in formula
###print int_constant *skipping full formula for simplicity
instalment = tenure
###print instalment
interest_tab = [str(l_no)]
count = 0
count_tab = ['Loan Number']
while instalment > 0:
interest = float(((instalment) / (int_constant)) * (term_charge))
interest_tab.append(interest)
count += 1
instalment -= 1
count_tab.append(count)
hp_header = count_tab
with open('/Users/Michael/Documents/GitHub/Int-Projection/HP_Int.csv', 'w') as h:
writer = csv.writer(h)
writer.writerow(hp_header)
with open('/Users/Michael/Documents/GitHub/Int-Projection/HP_Int.csv', 'a') as ins:
writer = csv.writer(ins)
writer.writerow(interest_tab)
###for i in interest_tab:
### print i
###for i in count_tab:
### print i
return interest_tab
def save_results(self):
"""
Saves the bit_error_probability and corresponding variance in a csv file
Saves the codewords, their transmissions, and their resulting decoding in files separted by channel noise variance
:return:
"""
self.save_time = epoch_time = str(int(time.time()))
if not os.path.exists("./stats/advanced-run"):
os.makedirs("./stats/advanced-run")
m = "sum-prod" if self.mode == Decoder.SUM_PROD else "max-prod"
with open("stats/advanced-run/%(time)s-%(mode)s-%(num_codewords)s-codewords-variance-bit_error_probability.csv" % {
"time": epoch_time,
"mode": m,
"num_codewords": self.iterations}, 'wb') as result_csv:
writer = csv.writer(result_csv)
writer.writerow(["variance", "bit_error_probability"])
for v, error in zip(self.variance_levels, self.bit_error_probability):
writer.writerow([v, error])
for i, v in enumerate(self.variance_levels):
with open("stats/advanced-run/%(time)s-%(mode)s-%(num_codewords)s-codewords-variance-%(var)s-codewords-decoded.csv" % {
"time": epoch_time,
"mode": m,
"num_codewords": str(self.iterations),
"var": str(v)}, 'wb') as result_csv:
writer = csv.writer(result_csv)
writer.writerow(["codeword", "decoded", "transmission"])
for codeword, transmission, decoded in zip(self.codewords[i], self.transmissions[i], self.decoded[i]):
writer.writerow([''.join(str(elem) for elem in codeword),
''.join(str(elem) for elem in decoded),
' '.join(str(elem) for elem in transmission)])
def payin(self):
"""The first stage of payday where we charge credit cards and transfer
money internally between participants.
"""
with self.db.get_cursor() as cursor:
self.prepare(cursor, self.ts_start)
holds = self.create_card_holds(cursor)
self.transfer_tips(cursor)
self.transfer_takes(cursor, self.ts_start)
transfers = cursor.all("""
SELECT * FROM transfers WHERE "timestamp" > %s
""", (self.ts_start,))
try:
self.settle_card_holds(cursor, holds)
self.update_balances(cursor)
check_db(cursor)
except:
# Dump transfers for debugging
import csv
from time import time
with open('%s_transfers.csv' % time(), 'wb') as f:
csv.writer(f).writerows(transfers)
raise
self.take_over_balances()
# Clean up leftover functions
self.db.run("""
DROP FUNCTION process_take();
DROP FUNCTION process_tip();
DROP FUNCTION settle_tip_graph();
DROP FUNCTION transfer(text, text, numeric, context_type);
""")
def SSIExcelToCSV(SSIworkbook,tempfile,CorrectedFile):
""" constructed to consume a specific SSI document in which there are an unknown # of worksheets """
with open(tempfile,'wb') as writetest:
writer=csv.writer(writetest)
workbook = xlrd.open_workbook(SSIworkbook)
worksheets = workbook.sheet_names()
for worksheet_name in worksheets:
worksheet = workbook.sheet_by_name(worksheet_name)
num_rows = worksheet.nrows - 1
curr_row = -1
while curr_row < num_rows:
curr_row += 1
row = worksheet.row(curr_row)
writer.writerow(row[1:3])
with open(CorrectedFile,'wb') as writetest:
writer=csv.writer(writetest)
with open(tempfile,'rb') as csvfile:
reader=csv.reader(csvfile)
for row in reader:
# pprint(row)
if row[0].startswith('text'):
x=[str(item).replace('text:u','') for item in row]
y=[s.replace("'","") for s in x]
writer.writerow(y)
print 'your file is waiting here '+CorrectedFile
def magic_loop(x, y):
cls_d = {}
with open('models.csv', 'w', newline='') as csvfile:
w = csv.writer(csvfile, delimiter=',')
w.writerow(['MODEL', 'PARAMETERS', 'PRECISION', 'RECALL', 'AUC', 'F1', 'ACCURACY', 'Time'])
with open('best_models.csv', 'w', newline='') as csvfile:
c = csv.writer(csvfile, delimiter=',')
c.writerow(['MODEL', 'PARAMETERS', 'PRECISION', 'RECALL', 'AUC', 'F1', 'ACCURACY', 'Time'])
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2, random_state=0)
class_auc = {}
best_model = ''
best_AUC = 0
best_param = ''
best_ypred = ''
best_precision = 0
best_recall = 0
best_f1 = 0
best_accuracy = 0
best_time = 0
for index,clf in enumerate([clfs[x] for x in MODELS]):
cls_ypred = ''
current_model = MODELS[index]
class_auc[current_model] = 0
cls_ypred = ''
print (current_model)
parameter_values = grid[current_model]
start_time = t.time()
for p in ParameterGrid(parameter_values):
try:
start_time = t.time()
clf.set_params(**p)
y_pred_probs = clf.fit(x_train, y_train).predict_proba(x_test)[:,1]
precision, accuracy, recall, f1, threshold, AUC = model_evaluation(y_test, y_pred_probs,.05)
end_time = t.time()
total_time = end_time - start_time
w.writerow([current_model, p, precision, recall, AUC, f1, accuracy, total_time])
if AUC > class_auc[current_model]:
class_auc[current_model] = AUC
cls_ypred = y_pred_probs
cls_param = p
cls_precision = precision
cls_recall = recall
cls_f1 = f1
cls_accuracy = accuracy
cls_time = total_time
except IndexError as e:
continue
auc = class_auc[current_model]
c.writerow([current_model, cls_param, cls_precision, cls_recall, auc, cls_f1, cls_accuracy, cls_time])
cls_d[current_model] = [auc, cls_ypred]
def generateOutputCSVs(inputDict, outputDict, history=0):
for key in inputDict:
inputRows = inputDict[key];
outputRows = outputDict[key];
history = [];
i = 0;
for inputRow in inputRows:
generateOutputCSVsByDOW[int(inputRow[2])].append(inputRow);
if (i < len(outputRows)):
dayByDayOutput[int(inputRow[2])].append(outputRows[i]);
i += 1;
for day in generateOutputCSVsByDOW:
with open("3192_"
+ key
+ "_"
+ str(day)
+ "_input.csv", 'w', newline='') as csvFile:
writer = csv.writer(csvFile);
for inputRow in generateOutputCSVsByDOW[day]:
writer.writerow(inputRow);
with open("3192_"
+ key
+ "_"
+ str(day)
+ "_output.csv", 'w', newline='') as csvFile:
writer = csv.writer(csvFile);
for outputRow in dayByDayOutput[day]:
writer.writerow(outputRow);
def generateTrials(myDict, nTestTrials):
'''
generates a list of trials to write to csv file
supply 3 columns to be factorially combined
myDict = {'contrast': [1.0, 0.75, 0.5, 0.25], 'orientation':[225, 315],
'probeMatch':[1, 0], 'correctKey':[1]}
'''
columnHeaders = myDict.keys()
trialList = data.createFactorialTrialList(myDict)
for item in trialList:
if item['probeMatch'] == 1:
item['correctKey'] = expInfo['sameKey']
else:
item['correctKey'] =expInfo['differentKey']
# write trial list for experiment loop:
if expInfo['experimentType']=='practise':
trialList = rand.sample(trialList, nTestTrials)
with open('mainTrials.csv', 'wb') as mainTrialsFile:
writer = csv.writer(mainTrialsFile, dialect='excel')
writer.writerow(columnHeaders)
for row in trialList:
writer.writerow(row.values())
if expInfo['experimentType']=='practise':
# generate trial list for practise loop:
practiseTrialList = rand.sample(trialList, nTestTrials)
# write trial list for practise loop:
with open('practiseTrials.csv', 'wb') as practiseTrialsFile:
writer = csv.writer(practiseTrialsFile, dialect='excel')
writer.writerow(columnHeaders)
for row in practiseTrialList:
writer.writerow(row.values())
def main_web(args):
assert os.path.exists(args.input)
with open(args.input) as f:
contents = f.read().strip()
if args.hg19 is True and args.protein is True:
stop_err('--hg19 option conflicts with --protein')
if args.protein is False:
## Replace tabs/space with commas
re.sub('[\t\s]+', ',', contents)
if args.hg19:
## Append hg19 to each line
lines = contents.split('\n')
contents = ('\n').join(
map((lambda x: 'hg19,' + x),
lines))
payload = {'vars': contents, 'tableQ': 1}
request = requests.post(__url__, data=payload)
response = request.text
if request.status_code != requests.codes.ok:
stop_err("""Error retrieving response from server.
Server returned %s .
Output: %s
""" % (request.status_code, response))
r = StringIO.StringIO(response)
reader = csv.reader(r, delimiter=",")
csv.writer(open(args.output, "wb"), delimiter='\t').writerows(reader)
def generateOutputCSVsByDOW(inputDict, outputDict):
for key in inputDict:
inputRows = inputDict[key];
outputRows = outputDict[key];
dayByDayInput = {0:[], 1:[], 2:[], 3:[], 4:[], 5:[], 6:[]};
dayByDayOutput = {0:[], 1:[], 2:[], 3:[], 4:[], 5:[], 6:[]};
i = 0;
for inputRow in inputRows:
dayByDayInput[int(inputRow[2])].append(inputRow);
if (i < len(outputRows)):
dayByDayOutput[int(inputRow[2])].append(outputRows[i]);
i += 1;
for day in dayByDayInput:
with open("3192_"
+ key
+ "_"
+ str(day)
+ "_input.csv", 'w', newline='') as csvFile:
writer = csv.writer(csvFile);
for inputRow in dayByDayInput[day]:
writer.writerow(inputRow);
with open("3192_"
+ key
+ "_"
+ str(day)
+ "_output.csv", 'w', newline='') as csvFile:
writer = csv.writer(csvFile);
for outputRow in dayByDayOutput[day]:
writer.writerow(outputRow);
def main():
"""
"""
f = open("../data/adult/adult.data")
reader = csv.reader(f)
t1 = open("../data/adult/train.csv","w")
t2 = open("../data/adult/test.csv","w")
w1 = csv.writer(t1)
w2 = csv.writer(t2)
for line in reader:
if len(line) < 1:
continue
label = line[-1].strip()
if label == "<=50K":
label = '0'
else:
label = '1'
line = line[:-1]
ls = csv_format(line,label)
if keep_train(2,1):
w1.writerow(ls)
else:
w2.writerow(ls)
def _evaluate_multi(calls, truth_svtypes, work_dir, data):
base = os.path.join(work_dir, "%s-sv-validate" % (dd.get_sample_name(data)))
out_file = base + ".csv"
df_file = base + "-df.csv"
if any((not utils.file_uptodate(out_file, x["vrn_file"])
or not utils.file_uptodate(df_file, x["vrn_file"])) for x in calls):
with file_transaction(data, out_file) as tx_out_file:
with open(tx_out_file, "w") as out_handle:
with open(df_file, "w") as df_out_handle:
writer = csv.writer(out_handle)
dfwriter = csv.writer(df_out_handle)
writer.writerow(["svtype", "size", "caller", "sensitivity", "precision"])
dfwriter.writerow(["svtype", "size", "caller", "metric", "value", "label"])
for svtype, truth in truth_svtypes.items():
for size in EVENT_SIZES:
str_size = "%s-%s" % size
for call in calls:
call_bed = convert.to_bed(call, dd.get_sample_name(data), work_dir, calls, data)
if utils.file_exists(call_bed):
evalout = _evaluate_one(call["variantcaller"], svtype, size, call_bed,
truth, data)
writer.writerow([svtype, str_size, call["variantcaller"],
evalout["sensitivity"]["label"], evalout["precision"]["label"]])
for metric in ["sensitivity", "precision"]:
dfwriter.writerow([svtype, str_size, call["variantcaller"], metric,
evalout[metric]["val"], evalout[metric]["label"]])
return out_file, df_file
def partition_essays():
'''
partition the training essay sets into different csv files
'''
with open('../data/training_set.csv', 'rb') as f:
spamreader = csv.reader(f, delimiter=',')
index = 1
out_file = open('../data/training_set_' + str(index) + '.csv','w')
writer = csv.writer(out_file)
is_first=True
for row in spamreader:
# if (row[0]=="220"):
# print repr(row[2].decode('utf-8'))
# print row[2].decode('utf-8')
# row[2]=row[2].encode('utf8')
if(is_first):
is_first= not is_first
continue
if row[1] == str(index):
writer.writerow(row)
else:
out_file.close()
index += 1
out_file = open('../data/training_set_' + str(index) + '.csv','w')
writer = csv.writer(out_file)
writer.writerows([row])
def sonar_callback(sonar): ## Problem with logging raw data
try:
stringtime = time.time()-time_zero
raw = list(numpy.fromstring(sonar.rawData, dtype = numpy.uint8))
sonarData = [stringtime, sonar.transBearing, sonar.pitch, sonar.TargetRange, sonar.meanIntinsity ,raw]
with open('%s/sonarLogwithRaw.csv' %(dirname), "a") as f:
try:
Writer = csv.writer(f, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
Writer.writerow(sonarData)
except ValueError:
print 'writerow error'
sonarData = [stringtime, sonar.transBearing, sonar.pitch, sonar.TargetRange, sonar.meanIntinsity]
with open('%s/sonarLog.csv' %(dirname), "a") as f:
try:
Writer = csv.writer(f, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
Writer.writerow(sonarData)
except ValueError:
print 'writerow error'
except ValueError:
print 'Problem with sonar logger!!!'
def getextents():
print "1) Output only collection level extents to a csv"
print "2) Output only component level extents to a csv"
choice = raw_input("Enter a number: ")
path = 'Real_Masters_all'
if choice == "1":
outfile = raw_input("Enter a filename for the csv: ")
for filename in os.listdir(path):
tree = etree.parse(join(path, filename))
e = tree.xpath('//ead/archdesc/did//physdesc/extent')
for e in e:
extent = e.text or "EMPTY EXTENT"
extentpath = tree.getpath(e)
with open(outfile + '.csv', 'ab') as csvfile:
writer = csv.writer(csvfile, dialect='excel')
writer.writerow([filename, extentpath, extent])
csvfile.close()
print filename
print outfile + '.csv complete'
elif choice == "2":
outfile = raw_input("Enter a filename for the csv: ")
for filename in os.listdir(path):
tree = etree.parse(join(path, filename))
e = tree.xpath('//dsc//did//extent')
for e in e:
extent = e.text or "EMPTY EXTENT"
extentpath = tree.getpath(e)
with open(outfile + '.csv', 'ab') as csvfile:
writer = csv.writer(csvfile, dialect='excel')
writer.writerow([filename, extentpath, extent])
csvfile.close()
print filename
print outfile + '.csv complete'
def getdates():
print "1) Output all unitdates to a csv"
print "2) Output all unitdates to a csv that do not have a normal attribute or are not 'undated'"
choice = raw_input("Enter a number: ")
path = 'Real_Masters_all'
if choice == "1":
outfile = raw_input("Enter a filename for the csv: ")
for filename in os.listdir(path):
tree = etree.parse(join(path, filename))
d = tree.xpath('//unitdate')
for i in d:
with open(outfile + '.csv', 'ab') as csvfile:
writer = csv.writer(csvfile, dialect='excel')
writer.writerow([filename, tree.getpath(i), i.text])
print filename
print outfile + '.csv complete'
elif choice == "2":
outfile = raw_input("Enter a filename for the csv: ")
for filename in os.listdir(path):
tree = etree.parse(join(path, filename))
d = tree.xpath('//unitdate')
for i in d:
# yyyy = re.compile('^[\d]{4}s?$')
# yyyy_yyyy = re.compile('^[\d]{4}s?[-][\d]{4}s?$')
undated = re.compile('^[Uu]ndated$')
if not undated.match(i.text) and not 'normal' in i.attrib:
with open(outfile + '.csv', 'ab') as csvfile:
writer = csv.writer(csvfile, dialect='excel')
writer.writerow([filename, tree.getpath(i), i.text])
print filename
print outfile + '.csv complete'
def save_candlefile(candles, period, filename, replace=True):
""" Save candle data to local file. If replace=True, an existing
file will be rewritten, else new candles will be appended. """
p_value, p_unit = parse_period(period)
if '_{}{}'.format(p_value, p_unit) not in filename:
filename += '_{}{}'.format(p_value, p_unit)
if CANDLES not in filename:
filename = os.path.join(CANDLES, filename)
try:
with open(filename, 'a') as writefile: pass
except:
print 'Checking for directory at ', CANDLES
raw_input('Warning: Candle data directory not found. Create now?')
build_data_directories()
if replace:
with open(filename, 'wb') as writefile:
writer = csv.writer(writefile)
for row in candles:
writer.writerow(row)
else:
with open(filename, 'a') as appendfile:
writer = csv.writer(appendfile)
for row in candles:
writer.writerow(row)
def main():
parser = build_cli_parser()
args = parser.parse_args()
if not args.outfile:
cwriter = csv.writer(sys.stdout)
else:
outfile = open(args.outfile, 'w')
cwriter = csv.writer(outfile)
if args.interval:
lastupdate_time = (datetime.datetime.now() - datetime.timedelta(days=args.interval)).strftime("%Y-%m-%d")
args.query = 'alliance_score_nvd:* last_update:[%s TO *]' % lastupdate_time
else:
args.query = 'alliance_score_nvd:*'
#request the process' id and segment id
args.fields = ['id','segment_id']
##########
#######Need to write logic to handle large numbers of results to the processes query
##########
args.rows = 30
if not args.url or not args.token:
print "Missing required param; run with --help for usage"
sys.exit(-1)
#run the query using the details we built above
processes = run_query(args)
cwriter.writerow(['process_md5', 'process_path', 'hostname'])
# for each process in the results dict
for process in processes['results']:
#get the process events so we can pull the NVD feed hits for the process
events = get_events(args, process['id'], process['segment_id'])
monkey = [events['process']['process_md5'], events['process']['path'], events['process']['hostname']]
cwriter.writerow(monkey)
def report_view_csv(self, id):
report, results, total, show_org = self._report_view(id, None, None)
file_path = '%s/%s.csv' % (h.get_export_dir(), re.sub('[^a-zA-Z0-9_-]+', '_', report['name'].encode('ascii','ignore')))
if report['report_on'] == "activities":
with open(file_path, 'wb') as csvfile:
writer = csv.writer(csvfile, lineterminator = '\n')
record = ['Time', 'User', 'Activity Type', 'Data']
writer.writerow(record)
for row in results:
activity_data = {}
data_name = ''
if 'activity' in row and 'data' in row['activity']:
activity = row['activity']['data']
if 'package' in activity:
activity_data = activity.get('package')
elif 'dataset' in activity:
activity_data = activity.get('dataset')
elif 'group' in activity:
activity_data = activity.get('group')
data_name = activity_data.get('name', '')
record = [row['activity']['timestamp'], row['user']['fullname'] or row['user']['name'], row['activity']['activity_type'], data_name]
writer.writerow(record)
elif report['report_on'] == "details":
with open(file_path, 'wb') as csvfile:
writer = csv.writer(csvfile, lineterminator = '\n')
record = ['name', 'email', 'state', 'organization', 'role', 'system administrator']
writer.writerow(record)
for row in results:
record = [row['fullname'] or row['name'], row['email'], row['state'], row['organization'], row['capacity'], row['sysadmin']]
writer.writerow(record)
response = h.convertHtmlToCsv(file_path, tk.response)
return response
def save(self, name_of_file, description):
""" This function will save the beam object to an external file in the directory called "defined_beams" in the source directory"""
dir = os.path.dirname(__file__)
current_module = __name__
path = current_module.split(".")
rel_dir_name = "defined_" + path[1] + "s"
filename = os.path.join(dir, rel_dir_name, name_of_file)
if os.path.isfile(filename) == True:
in_keyboard = input(
"Filename specified" + name_of_file + "already exists, do you want to overwrite, yes [y] or no [n]?"
)
if in_keyboard == "n":
name_of_file = input("Enter new file name")
filename = os.path.join(dir, rel_dir_name, name_of_file)
if os.path.isfile(filename) == True:
print("Gave the same file name again, overwriting the file!!!!!")
writer = csv.writer(open(filename, "w"))
writer.writerow(["Description", description])
for key, value in self.parameters.items():
writer.writerow([key, value])
print(" Successfully written at" + filename)
else:
writer = csv.writer(open(filename, "w"))
writer.writerow(["Description", description])
for key, value in self.parameters.items():
writer.writerow([key, value])
print(" Successfully written at" + filename)
def make_complete():
my_todo = open(csv_input, "r")
lines = list(csv.reader(my_todo, delimiter = "|"))
rm = int(input('Which one is complete? '))
temp_list = list(lines)
print() #empty line
print('Are you sure? Y/N:')
print(temp_list[rm-1])
yes = input('')
if yes == 'y':
# add item to the comlete list csv
with open(csv_complete, "a") as csvfile:
lines = csv.writer(csvfile, delimiter = "|")
lines.writerow(temp_list[rm-1])
temp_list.remove(temp_list[rm-1])
with open(csv_input, "w") as csvfile:
lines = csv.writer(csvfile, delimiter = "|")
for line in temp_list:
lines.writerow(line)
print("\033c") #screen clr
print('Todo is completed')
elif yes == 'n':
print('Abort')
else:
print('')
def write_log(task, data, in_rows, question, out_rows, out_cols, solution, version, git, fun, run, time_sec, mem_gb, cache, chk, chk_time_sec):
batch = os.getenv('BATCH', "")
timestamp = time.time()
csv_file = os.getenv('CSV_TIME_FILE', "time.csv")
nodename = platform.node()
comment = "" # placeholder for updates to timing data
time_sec = round(time_sec, 3)
chk_time_sec = round(chk_time_sec, 3)
mem_gb = round(mem_gb, 3)
if math.isnan(time_sec):
time_sec = ""
if math.isnan(mem_gb):
mem_gb = ""
log_row = [nodename, batch, timestamp, task, data, in_rows, question, out_rows, out_cols, solution, version, git, fun, run, time_sec, mem_gb, cache, chk, chk_time_sec, comment]
log_header = ["nodename","batch","timestamp","task","data","in_rows","question","out_rows","out_cols","solution","version","git","fun","run","time_sec","mem_gb","cache","chk","chk_time_sec","comment"]
append = os.path.isfile(csv_file)
csv_verbose = os.getenv('CSV_VERBOSE', "true")
if csv_verbose.lower()=="true":
print('# ' + ','.join(str(x) for x in log_row))
if append:
with open(csv_file, 'a') as f:
w = csv.writer(f, lineterminator='\n')
w.writerow(log_row)
else:
with open(csv_file, 'w+') as f:
w = csv.writer(f, lineterminator='\n')
w.writerow(log_header)
w.writerow(log_row)
return True
def create_evaluation_distinctiveness(config, Kind):
model_fname = config.model_fname % Kind.__name__
try:
model = LdaModel.load(model_fname)
logger.info('Opened previously created model at file %s' % model_fname)
except:
error('Cannot evalutate LDA models not built yet!')
scores = utils.score(model, utils.kullback_leibler_divergence)
total = sum([x[1] for x in scores])
logger.info("%s model KL: %f" % (model_fname, total))
with open(config.path + 'evaluate-results.csv', 'a') as f:
w = csv.writer(f)
w.writerow([model_fname, total])
etas = list()
for topic in model.state.get_lambda():
topic_eta = list()
for p_w in topic:
topic_eta.append(p_w * numpy.log2(p_w))
etas.append(-sum(topic_eta))
entropy = sum(etas) / len(etas)
logger.info("%s model entropy mean: %f" % (model_fname, entropy))
with open(config.path + 'evaluate-entropy-results.csv', 'a') as f:
w = csv.writer(f)
w.writerow([model_fname, entropy])
def main():
f = open('train_p.csv', 'rb')
reader = csv.reader(f,MyDialect())
rows = []
for row in reader:
rows.append(row)
num_lines = len(rows)
num_train = num_lines * 6 / 10
random.shuffle(rows)
train_set = rows[:num_train]
test_set = rows[num_train:]
print len(train_set), len(test_set)
train_file = open('mytrain2.csv', 'w')
writer = csv.writer(train_file, MyDialect())
for row in train_set:
writer.writerow(row)
test_file = open('mytest2.csv', 'w')
writer = csv.writer(test_file, MyDialect())
test_ans = open('mytest2_label.csv', 'w')
for row in test_set:
writer.writerow(row[:1]+row[2:])
test_ans.write(row[1] + "\n")
def main():
venues_file_name = 'venues%s.csv' % datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
venue_team_file_name = 'venue_team%s.csv' % datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
with open(file_name, 'wb') as f, open(venue_team_file_name) as f2:
writer = csv.writer(f)
writer.writerow(['venue_id', 'name', 'country', 'address', 'zipcode',
'city', 'fax', 'email', 'website', 'phone', 'openend',
'architect', 'capacity', 'surface', 'facts'])
writer2 = csv.writer(f2)
writer2.writerow(['venue_id','team_id'])
for x in range(0,20000):
print x
if requests.head(BASE_URL % x).status_code == requests.codes.ok:
r = requests.get(BASE_URL % x)
soup = BS(r.text, 'html.parser')
venue_data = []
venue_data.append(x) #venue_id
name_node = soup.select('#subheading > h1')
venue_data.append(name_node[0].text.encode('utf-8') if name_node else '')
venue_data.append(get_detail('Founded', soup))
venue_data.append(get_detail('Address', soup))
venue_data.append(get_detail('Country', soup))
venue_data.append(get_detail('Phone', soup))
venue_data.append(get_detail('Fax', soup))
venue_data.append(get_detail('E-mail', soup))
website_node = soup.find('a', text='Official website')
venue_data.append(website_node['href'].encode('utf-8') if website_node else '')
venue_data.append(soup.find('div', 'logo').img['src'].encode('utf-8'))
venue_data.append(get_venue_id(x))
print name_node[0].text
writer.writerow(venue_data)
def process_files(yymm):
print 'handle the file; %s' % yymm
#
ap_target_file = Y09_ap_trips if yymm.startswith('09') else Y10_ap_trips
with open('%s/%s%s.csv' % (ap_ep_dir, ap_ep_prefix, yymm), 'rb') as r_csvfile:
reader = csv.reader(r_csvfile)
headers = reader.next()
if not check_path_exist(ap_target_file):
with open(ap_target_file, 'wt') as csvFile:
writer = csv.writer(csvFile)
writer.writerow(headers)
with open(ap_target_file, 'a') as csvFile:
writer = csv.writer(csvFile)
for row in reader:
writer.writerow(row)
#
ns_target_file = Y09_ns_trips if yymm.startswith('09') else Y10_ns_trips
with open('%s/%s%s.csv' % (ns_ep_dir, ns_ep_prefix, yymm), 'rb') as r_csvfile:
reader = csv.reader(r_csvfile)
headers = reader.next()
if not check_path_exist(ns_target_file):
with open(ns_target_file, 'wt') as csvFile:
writer = csv.writer(csvFile)
writer.writerow(headers)
with open(ns_target_file, 'a') as csvFile:
writer = csv.writer(csvFile)
for row in reader:
writer.writerow(row)
#
print 'end the file; %s' % yymm
def main():
input_file = "books.csv"
output_books = "books_new.csv"
output_authors = "authors.csv"
output_book_author_map = "book_authors.csv"
count = 0
authors = {}
author_count = 0
parse_firs_line = True
output_books_stream = csv.writer(open(output_books, 'wb'))
output_authors_stream = csv.writer(open(output_authors, 'wb'))
output_book_author_map_stream = csv.writer(open(output_book_author_map, 'wb'))
with open(input_file) as input_stream:
for line in input_stream:
line=line[:-1]
count += 1
input_str = line.split("\t")
book_line = []
author_line = []
book_author_map = []
if parse_firs_line:
book_line = ['isbn', 'title', 'isbn13', 'cover', 'publisher', 'pages']
author_line = ['author_id', 'fullname', 'title', 'fname', 'mname', 'lname', 'suffix']
book_author_map = ["isbn", "author_id"]
output_books_stream.writerow(book_line)
output_authors_stream.writerow(author_line)
output_book_author_map_stream.writerow(book_author_map)
parse_firs_line = False
else:
# parse and add book
book_line = [input_str[0], input_str[2], input_str[1], input_str[4], input_str[5], input_str[6]]
if len(book_line)>0:
output_books_stream.writerow(book_line)
# for author check in author map if not present then only add
book_authors = re.split("&|,", input_str[3])
tmp_set=set(book_authors)
book_authors=list(tmp_set)
for author in book_authors:
if author in authors:
pass
else:
author_count += 1
author = author.strip()
authors[author] = author_count
tokens = author.split(' ')
if len(tokens) == 1:
author_line = [author_count, author, '', '', '', tokens[0], '']
elif len(tokens) == 2:
author_line = [author_count, author, '', tokens[0], '', tokens[-1], '']
elif len(tokens) == 3:
author_line = [author_count, author, '', tokens[0], tokens[1], tokens[-1], '']
else:
author_line = [author_count, author, '', tokens[0], tokens[1]+' '+tokens[2], tokens[-1], '']
if len(author_line) > 0:
output_authors_stream.writerow(author_line)
author_id = authors.get(author)
book_author_map = [input_str[0],author_id]
if len(book_author_map) > 0:
output_book_author_map_stream.writerow(book_author_map)
print "Processed Records:",count
csv_stat_item['category'] = item_predict
#csv_stat_item['category'] = item_predict
i_index += 1
csv_stat_all_data.append(csv_stat_item)
csv_set_sorted = sorted(csv_stat_all_data, key=lambda x: (x['id']), reverse=False)
with open(str_file_name_tw, 'wt', newline='') as csvfile:
csv_writer = csv.writer(csvfile, dialect='excel')
header = ['Id', 'Category']
csv_writer.writerow(header)
for data_point in csv_set_sorted:
row = [data_point['id'], data_point['category']]
#str_to_write = "\n" + str(data_point['id']) + "," + str(data_point['category'])
#fout_tw.write(str_to_write)
csv_writer.writerow(row)
import csv
with open('/Users/kpentchev/data/ner_2019_04_06_prep.csv',
encoding='utf-8') as csv_read:
#with open('/home/kpentchev/data/floyd/ner_2019_02_25_fixed.csv', encoding='utf-8') as csv_read:
csv_reader = csv.reader(csv_read, delimiter='\t', quoting=csv.QUOTE_NONE)
line_count = 0
sentence_count = 1
with open('/Users/kpentchev/data/ner_2019_04_06_fixed.csv',
encoding='utf-8',
mode='w') as csv_write:
#with open('/home/kpentchev/data/floyd/ner_2019_02_25_fixed.csv', encoding='utf-8', mode='w') as csv_write:
csv_writer = csv.writer(csv_write, delimiter='\t')
for row in csv_reader:
if line_count == 0:
csv_writer.writerow(row)
else:
sentence = row[0].strip()
if (not "" == sentence):
row[0] = f"Sentence {sentence_count}"
sentence_count += 1
if len(row) > 4:
print('{} old {}'.format(line_count, row))
row = [
row[0], ''.join(row[1:-2]).replace('"', ''), row[-2],
row[-1]
]
print('{} new {}'.format(line_count, row))
csv_writer.writerow(row)
line_count += 1
def ACC_cal():
'''
calculate LCF and BCF using QP
publish the next speed
'''
global m
global g
global se
global F0
global F1
global F2
global gamma
global ca
global cd
global psc
global x2
global v0
global vd
global D_old
global D
global softness
global initD
global dt
global pub
global x_old
global pubSteer
global initTime
block = 3.5
if (D <= 1.0):
print "less than 1.0m"
return 0.0
pass
print "======================================"
z = D
y = x2 - vd
#y = vd-x2
#print "speed difference", x2, vd, y
#print "x2 and vd and y", x2, vd, y
Fr = F0 + F1*(y+vd)+F2*(y+vd)**2
Vv0 = -2.0*y/m*Fr + se*(y**2)
Vv1 = 2.0*y/m
#print "vv0, vv1", Vv0, Vv1
h = z-block*x2-0.3 #0.3m for parking
#h = z-block*x2
try:
B = -1.0* math.log(h/(1+h))
except:
h = 0.0001
B = -1.0* math.log(h/(1+h))
LfB = -1.0*(block*Fr + m*(v0 - x2))/(m*(1.0 - block*x2 + z)*(-block*x2 + z));
LgB = block/(m*(1.0 - block*x2 + z)*(-block*x2 + z))
try:
hf = -block*x2-0.5*((v0-x2)**2)/cd*g+z
Bf = -1.0* math.log(hf/(1.0+hf))
except:
hf = 0.001
Bf = -1.0* math.log(hf/(1.0+hf))
#print "!!!!!!??????????!!!!"
'''
LfBf = (v0-x2-block)/(m*cd*g)/(-1.0*hf+hf**2)
LgBf = (m*cd*g*(v0-x2)-Fr*(v0-x2-block))/(m*cd*g)/(-1.0*hf+hf**2)
H = matrix([[2.0/(m**2), 0.0], [0.0, 2*psc]]) #P
F = matrix([-2.0*Fr/(m**2), 0.0]) #q
A = matrix([[Vv1, -1.0], [LgB, 0.0], [1.0,0.0], [1.0, 0.0], [LgBf, 0.0]]).T
b = matrix([-1.0*Vv0, -1.0*LfB+gamma/B, vd/m*dt, ca*m*g, -1.0*LgBf+1.0/Bf])
'''
H = matrix([[2.0/(m**2), 0.0], [0.0, 2*psc]]) #P
F = matrix([-2.0*Fr/(m**2), 0.0]) #q
A = matrix([[Vv1, -1.0], [LgB, 0.0]]).T
b = matrix([-1.0*Vv0,-1.0*LfB+gamma/B])
#print A
#print b
#A = matrix([[Vv1, -1.0], [LgB, 0.0], [1.0, 0.0], [-1.0, 0.0], [LgBf, 0.0]]).T
#b = matrix([-1.0*Vv0, -1.0*LfB+gamma/B, ca*m*g, cd*m*g, -1.0*LgBf+1.0/Bf])
try:
U = cvxopt.solvers.qp(H, F, A, b)
#result = min(ca*g, max(-cd*g, (U["x"][0]/m)))
result = (U["x"][0]-Fr)/m
temp = [time.time()-initTime, x2, v0, D, result]
#time, currentV, frontV, Dis, force
with open ('/home/ubuntu/catkin_ws/src/ACC/accdata.csv', 'a') as f:
writer = csv.writer(f)
writer.writerow(temp)
print temp
#print "result", U["x"],result
#m/s
#print "result, result, result, result", result
except:
result = 0.0
pub.publish(0.0)#========================
print "oops Hit!!!XXXXXXXXXXXXXXXxxxxxxxxxxxxxXX"
return result
result = min(result/0.1109, 3.0)
vel = (result*dt)+x2/0.1109 #rpm
#print "vel",vel
if (vel<0.0):
pub.publish(0.0)#========================
print "Backup Protection"
elif(vel>(vd*1.1/0.1109)):
vel = vd*1.1/0.1109
#vel = 5.0/0.1109
pub.publish(vel)#=========================
print "Speeding Protection"
else:
toPub = vel
x_old = toPub
pubSteer.publish(10120)
if(x2 == 0 and toPub > 0):
pub.publish(3.0)
else:
pub.publish(toPub)
print "Moving Forward! Next speed:rpm", toPub
#simuSpeed(toPub)
return vel
def api_report(self,
request,
reporttype=None,
from_date=None,
to_date=None,
object_profile=None,
filter_default=None,
exclude_zero=None,
interface_profile=None,
selector=None,
administrative_domain=None,
columns=None,
o_format=None,
enable_autowidth=False,
**kwargs):
def translate_row(row, cmap):
return [row[i] for i in cmap]
map_table = {
"load_interfaces": r"/Interface\s\|\sLoad\s\|\s[In|Out]/",
"load_cpu": r"/[CPU|Memory]\s\|\sUsage/",
"errors": r"/Interface\s\|\s[Errors|Discards]\s\|\s[In|Out]/",
"ping": r"/Ping\s\|\sRTT/",
}
cols = [
"id",
"object_name",
"object_address",
"object_platform",
"object_adm_domain",
"object_segment",
"object_container",
# "object_hostname",
# "object_status",
# "profile_name",
# "object_profile",
# "object_vendor",
"iface_name",
"iface_description",
"iface_speed",
"load_in",
"load_in_p",
"load_out",
"load_out_p",
"errors_in",
"errors_out",
"slot",
"cpu_usage",
"memory_usage",
"ping_rtt",
"ping_attempts",
"interface_flap",
"interface_load_url",
]
header_row = [
"ID",
"OBJECT_NAME",
"OBJECT_ADDRESS",
"OBJECT_PLATFORM",
"OBJECT_ADM_DOMAIN",
"OBJECT_SEGMENT",
"OBJECT_CONTAINER",
"IFACE_NAME",
"IFACE_DESCRIPTION",
"IFACE_SPEED",
"LOAD_IN",
"LOAD_IN_P",
"LOAD_OUT",
"LOAD_OUT_P",
"ERRORS_IN",
"ERRORS_OUT",
"CPU_USAGE",
"MEMORY_USAGE",
"PING_RTT",
"PING_ATTEMPTS",
"INTERFACE_FLAP",
"INTERFACE_LOAD_URL",
]
if columns:
cmap = []
for c in columns.split(","):
try:
cmap += [cols.index(c)]
except ValueError:
continue
else:
cmap = list(range(len(cols)))
columns_order = columns.split(",")
columns_filter = set(columns_order)
r = [translate_row(header_row, cmap)]
object_columns = [c for c in columns_order if c.startswith("object")]
# Date Time Block
if not from_date:
from_date = datetime.datetime.now() - datetime.timedelta(days=1)
else:
from_date = datetime.datetime.strptime(from_date, "%d.%m.%Y")
if not to_date or from_date == to_date:
to_date = from_date + datetime.timedelta(days=1)
else:
to_date = datetime.datetime.strptime(
to_date, "%d.%m.%Y") + datetime.timedelta(days=1)
# interval = (to_date - from_date).days
ts_from_date = time.mktime(from_date.timetuple())
ts_to_date = time.mktime(to_date.timetuple())
# Load managed objects
mos = ManagedObject.objects.filter(is_managed=True)
if not request.user.is_superuser:
mos = mos.filter(
administrative_domain__in=UserAccess.get_domains(request.user))
if selector:
mos = mos.filter(
ManagedObjectSelector.objects.get(id=int(selector)).Q)
if administrative_domain:
mos = mos.filter(
administrative_domain__in=AdministrativeDomain.get_nested_ids(
int(administrative_domain)))
if object_profile:
mos = mos.filter(object_profile=object_profile)
# iface_dict = {}
d_url = {
"path": "/ui/grafana/dashboard/script/report.js",
"rname": map_table[reporttype],
"from": str(int(ts_from_date * 1000)),
"to": str(int(ts_to_date * 1000)),
# o.name.replace("#", "%23")
"biid": "",
"oname": "",
"iname": "",
}
report_map = {
"load_interfaces": {
"url": "%(path)s?title=interface&biid=%(biid)s"
"&obj=%(oname)s&iface=%(iname)s&from=%(from)s&to=%(to)s",
"q_group": ["interface"],
"q_select": {
(0, "managed_object", "id"): "managed_object",
(1, "path", "iface_name"): "arrayStringConcat(path)",
},
},
"errors": {
"url":
"""%(path)s?title=errors&biid=%(biid)s&obj=%(oname)s&iface=%(iname)s&from=%(from)s&to=%(to)s""",
"q_group": ["interface"],
},
"load_cpu": {
"url":
"""%(path)s?title=cpu&biid=%(biid)s&obj=%(oname)s&from=%(from)s&to=%(to)s""",
"q_select": {
(0, "managed_object", "id"): "managed_object",
(1, "path", "slot"): "arrayStringConcat(path)",
},
},
"ping": {
"url":
"""%(path)s?title=ping&biid=%(biid)s&obj=%(oname)s&from=%(from)s&to=%(to)s""",
"q_select": {
(0, "managed_object", "id"): "managed_object"
},
},
}
query_map = {
# "iface_description": ('', 'iface_description', "''"),
"iface_description": (
"",
"iface_description",
"dictGetString('interfaceattributes','description' , (managed_object, arrayStringConcat(path)))",
),
"iface_speed": (
"speed",
"iface_speed",
"if(max(speed) = 0, dictGetUInt64('interfaceattributes', 'in_speed', "
"(managed_object, arrayStringConcat(path))), max(speed))",
),
"load_in":
("load_in", "l_in", "round(quantile(0.90)(load_in), 0)"),
"load_in_p": (
"load_in",
"l_in_p",
"replaceOne(toString(round(quantile(0.90)(load_in) / "
"if(max(speed) = 0, dictGetUInt64('interfaceattributes', 'in_speed', "
"(managed_object, arrayStringConcat(path))), max(speed)), 4) * 100), '.', ',')",
),
"load_out":
("load_out", "l_out", "round(quantile(0.90)(load_out), 0)"),
"load_out_p": (
"load_out",
"l_out_p",
"replaceOne(toString(round(quantile(0.90)(load_out) / "
"if(max(speed) = 0, dictGetUInt64('interfaceattributes', 'in_speed', "
"(managed_object, arrayStringConcat(path))), max(speed)), 4) * 100), '.', ',')",
),
"errors_in": ("errors_in", "err_in", "quantile(0.90)(errors_in)"),
"errors_out":
("errors_out", "err_out", "quantile(0.90)(errors_out)"),
"cpu_usage": ("usage", "cpu_usage", "quantile(0.90)(usage)"),
"ping_rtt":
("rtt", "ping_rtt", "round(quantile(0.90)(rtt) / 1000, 2)"),
"ping_attempts": ("attempts", "ping_attempts", "avg(attempts)"),
}
query_fields = []
for c in report_map[reporttype]["q_select"]:
query_fields += [c[2]]
field_shift = len(query_fields) # deny replacing field
for c in columns.split(","):
if c not in query_map:
continue
field, alias, func = query_map[c]
report_map[reporttype]["q_select"][(columns_order.index(c) +
field_shift, field,
alias)] = func
query_fields += [c]
metrics_attrs = namedtuple("METRICSATTRs", query_fields)
mo_attrs = namedtuple("MOATTRs",
[c for c in cols if c.startswith("object")])
containers_address = {}
if "object_container" in columns_filter:
containers_address = ReportContainerData(
set(mos.values_list("id", flat=True)))
containers_address = dict(list(containers_address.extract()))
moss = {}
for row in mos.values_list("bi_id", "name", "address", "platform",
"administrative_domain__name", "segment",
"id"):
moss[row[0]] = mo_attrs(*[
row[1],
row[2],
str(Platform.get_by_id(row[3]) if row[3] else ""),
row[4],
str(NetworkSegment.get_by_id(row[5])) if row[5] else "",
containers_address.
get(row[6], "") if containers_address and row[6] else "",
])
url = report_map[reporttype].get("url", "")
report_metric = self.metric_source[reporttype](tuple(sorted(moss)),
from_date,
to_date,
columns=None)
report_metric.SELECT_QUERY_MAP = report_map[reporttype]["q_select"]
if exclude_zero and reporttype == "load_interfaces":
report_metric.CUSTOM_FILTER["having"] += [
"max(load_in) != 0 AND max(load_out) != 0"
]
if interface_profile:
interface_profile = InterfaceProfile.objects.filter(
id=interface_profile).first()
report_metric.CUSTOM_FILTER["having"] += [
"dictGetString('interfaceattributes', 'profile', "
"(managed_object, arrayStringConcat(path))) = '%s'" %
interface_profile.name
]
# OBJECT_PLATFORM, ADMIN_DOMAIN, SEGMENT, OBJECT_HOSTNAME
for row in report_metric.do_query():
mm = metrics_attrs(*row)
mo = moss[int(mm.id)]
res = []
for y in columns_order:
if y in object_columns:
res += [getattr(mo, y)]
else:
res += [getattr(mm, y)]
if "interface_load_url" in columns_filter:
d_url["biid"] = mm.id
d_url["oname"] = mo[2].replace("#", "%23")
# res += [url % d_url, interval]
res.insert(columns_order.index("interface_load_url"),
url % d_url)
r += [res]
filename = "metrics_detail_report_%s" % datetime.datetime.now(
).strftime("%Y%m%d")
if o_format == "csv":
response = HttpResponse(content_type="text/csv")
response[
"Content-Disposition"] = 'attachment; filename="%s.csv"' % filename
writer = csv.writer(response,
dialect="excel",
delimiter=",",
quoting=csv.QUOTE_MINIMAL)
writer.writerows(r)
return response
elif o_format == "xlsx":
response = StringIO()
wb = xlsxwriter.Workbook(response)
cf1 = wb.add_format({"bottom": 1, "left": 1, "right": 1, "top": 1})
ws = wb.add_worksheet("Alarms")
max_column_data_length = {}
for rn, x in enumerate(r):
for cn, c in enumerate(x):
if rn and (r[0][cn] not in max_column_data_length or
len(str(c)) > max_column_data_length[r[0][cn]]):
max_column_data_length[r[0][cn]] = len(str(c))
ws.write(rn, cn, c, cf1)
ws.autofilter(0, 0, rn, cn)
ws.freeze_panes(1, 0)
for cn, c in enumerate(r[0]):
# Set column width
width = get_column_width(c)
if enable_autowidth and width < max_column_data_length[c]:
width = max_column_data_length[c]
ws.set_column(cn, cn, width=width)
wb.close()
response.seek(0)
response = HttpResponse(response.getvalue(),
content_type="application/vnd.ms-excel")
response[
"Content-Disposition"] = 'attachment; filename="%s.xlsx"' % filename
response.close()
return response
import csv
import math
with open('fruits-responses-time.csv', 'w') as csvfile:
writer = csv.writer(
csvfile, delimiter=',') #,quotechar='"', quoting=csv.QUOTE_MINIMAL)
f = open('output-all.txt', 'r')
lines = f.readlines()
for line in lines:
words = line.split()
res = [words[3]]
# res = []
# res.append(int(math.log(int(words[6][:-1]),2))-5)
# res.append(int(words[4])+1)
# res.append(words[10][:-1])
# res.append(words[12][:-1])
# res.append(words[14][:-1])
writer.writerow(res)
current_legislature = "14"
enregistrement = sys.argv[1]
logging
logger = logging.getLogger('myapp')
hdlr = logging.FileHandler('./logs/'+current_legislature+'.log')
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
hdlr.setFormatter(formatter)
logger.addHandler(hdlr)
logger.setLevel(logging.WARNING)
nom_fichier = 'votes'+current_legislature+'.csv'
fichier = open(nom_fichier, 'a', newline='')
spamwriter = csv.writer(fichier, delimiter=',')
lst_scrutins_html = glob.glob('./scrutins'+current_legislature+'/*.html') #On liste uniquement les fichiers '.html'
lst_scrutins_html.sort()
print(lst_scrutins_html)
for s in range(len(lst_scrutins_html)) :
name_file = lst_scrutins_html[s];
try :
parcours_scrutin_html(lst_scrutins_html[s])
except Exception as e :
logger.error("THROW ERROR GRAVE parcours_scrutin_html : "+str(e))
logger.error("----"+info_fichier())
reset_currents_all()
def find_nearest(array, value):
array = np.asarray(array)
idx = (np.abs(array - value)).argmin()
return idx
for i in range(Ih.size):
x = l
y = maxV[1, 1, :, i]
dy = np.gradient(y)
d2y = np.gradient(dy)
d3y = np.gradient(d2y)
y2 = maxV[0, 1, :, i]
z = np.polyfit(x, y2, 1)
p = np.poly1d(z)
critical_length_20mv.append(l[find_nearest(y, -20)])
critical_length_d2y.append(l[np.argmax(d2y)])
gradients.append(z[0])
with open(r'outputs/data/figure_4_supplementary_3b.csv', 'w') as f:
writer = csv.writer(f)
writer.writerow(
['Ih', 'critical_length_d2y', 'critical_length_20mv', 'slope'])
for i in range(Ih.size):
fields = [
Ih[i], critical_length_d2y[i], critical_length_20mv[i],
gradients[i]
]
writer.writerow(fields)
s1 = s1 * 10000 + s2 * 100 + s3
time = s1
mode = date.split('-')
s1 = int(mode[0])
s2 = int(mode[1])
s3 = int(mode[2])
s1 = s1 * 10000 + s2 * 100 + s3
date = s1
filename = 'output' + str(time) + '_' + str(date)
filename += '.csv'
print('Name of File created :', filename)
header_name = ["Date", "Time", "lat1", "lng1", "alt1", "pitchd1", "yawd1", "yawo1", "pitcho1", "lat2", "lng2", "alt2",
"pitchd2", "yawd2", "yawo2", "pitcho2"]
with open(filename, 'w+') as csv_file:
csv_writer = csv.writer(csv_file)
csv_writer.writerow(header_name)
flag=0
v = visualize(0, 0, 0, 0, 0, 0, 0, 0, 0)
while True:
data1 = connrun()
data2 = conn1run()
data1 = data1.decode()
data2 = data2.decode()
data1 = data1.split('\n')
data1 = data1[-2]
data2 = data2.split('\n')
data2 = data2[-2]
data1 = data1.split(',')
data2 = data2.split(',')
currenttime = datetime.datetime.now()
def extract_text(input_image,
max_contour_area=1000,
max_contour_envelope_area=1500,
aspect_ratio=2.0,
gblur="True",
erode="True"):
"""
These are Gina`s default values for the params
:param input_image:
:param max_contour_area: controls rejection of contours
:param max_contour_envelope_area: controls rejection of contours
:param aspect_ratio: controls rejection of contours
:param gblur: apply gaussian blur
:param erode: apply erosion filter
:return:
"""
print("Hey remember, we`ve turned off most of the script! :)")
job_uuid = str(uuid.uuid4())
base_output_path = "/home/james/geocrud/adrc/"
print("processing %s" % input_image)
fname_timestamp = time.strftime("%Y%m%d-%H%M%S")
# Load map – change path and filename
# cv2.imread(filename) : loads an image from a file - returns an array/matrix
im = cv2.imread(input_image)
# im.copy() : copies one array to another
im2 = im.copy() #keep a copy
im3 = im.copy()
#print "input image..."
# cv2.imshow(winname, image) : display an image in the specified window
#cv2.imshow("1 Input Image", im)
# cv2.waitKey(delay in ms) : wait for a key event
#cv2.waitKey(0)
############################ PRE-PROCESSING ##############################
# Convert to greyscale
processed_image = cv2.cvtColor(im, cv2.COLOR_RGB2GRAY)
#cv2.imshow("2 Greyscaled", processed_image)
#cv2.waitKey(0)
# Apply Gaussian Blur - Increase in case of dotted background
if gblur == "True":
processed_image = cv2.GaussianBlur(processed_image, (5, 5), 0)
#cv2.imshow("3 GBlur Applied", processed_image)
#cv2.waitKey(0)
# Apply Otsu's threshold and invert the colors of the image - digits white on black background
(thresh,
processed_image) = cv2.threshold(processed_image, 0, 255,
cv2.THRESH_BINARY_INV | cv2.THRESH_OTSU)
#cv2.imshow("4 Thresholded", processed_image)
#cv2.waitKey(0)
if erode == "True":
# Define Kernel Size and apply erosion filter
element = cv2.getStructuringElement(cv2.MORPH_RECT, (3, 3))
processed_image = cv2.erode(processed_image, element)
blank = processed_image.copy()
blank2 = blank.copy()
processed_image_b = processed_image.copy()
#print "erosion filter applied..."
#cv2.imshow("5 Erosion Filter applied", processed_image)
#cv2.waitKey(0)
out_fname = "".join([
base_output_path,
# job_uuid,
# "_",
# str(max_contour_area),
# "_",
# str(max_contour_envelope_area),
# "_",
# str(aspect_ratio),
"img_passed_to_feature_extraction.tif"
])
cv2.imwrite(out_fname, processed_image)
############################ FEATURE DETECTION ##############################
# Contour tracing - Detect all contours with no hierarchy
image, contours, hierarchy = cv2.findContours(processed_image,
cv2.RETR_LIST,
cv2.CHAIN_APPROX_SIMPLE)
#################### JRCC - SHOW ALL Contours #############################
font = cv2.FONT_HERSHEY_COMPLEX
# draw mbr of every contour
id = 0
num_retained = 0
num_dropped = 0
lower_area_threshold = 0
contour_features = []
final_contours = []
contours_retain = []
# thresholds to decide if we retain the contour
thresholds = {
"area": 5000,
"h_lower": 20,
"h_upper": 100,
"aspect_ratio": 1.5,
"roundness": 0.1,
"solidity": 0.8
}
base_outfname = os.path.splitext(os.path.split(input_image)[-1])[0]
#out_fname = os.path.join("/home/james/geocrud/adrc", "".join([base_outfname, "_contour_properties.csv"]))
#with open(out_fname, "w") as outpf:
# my_writer = csv.writer(outpf, delimiter=',', quotechar='"', quoting=csv.QUOTE_NONNUMERIC)
# my_writer.writerow(["id", "x", "y", "w", "h", "area", "perimeter", "is_convex", "angle_of_rotation", "aspect_ratio", "extent", "solidity", "compactness", "roundness"])
for cnt in contours:
this_contour_properties = get_contour_properties(cnt)
x = this_contour_properties["x"]
y = this_contour_properties["y"]
w = this_contour_properties["w"]
h = this_contour_properties["h"]
# hw = h/w
# wh = w/h
# area = this_contour_properties["area"]
# perimeter = this_contour_properties["perimeter"]
# is_convex = this_contour_properties["is_convex"]
# angle_of_rotation = this_contour_properties["angle_of_rotation"]
# aspect_ratio = this_contour_properties["aspect_ratio"]
# extent = this_contour_properties["extent"]
# solidity = this_contour_properties["solidity"]
# compactness = this_contour_properties["compactness"]
# roundness = this_contour_properties["roundness"]
if retain_contour(this_contour_properties, thresholds):
cv2.rectangle(im2, (x, y), (x + w, y + h), (0, 255, 0), 1)
#cv2.putText(im2, str(id), (x, y+h), cv2.FONT_HERSHEY_PLAIN, 2, [0,0,255])
contours_retain.append(cnt)
num_retained += 1
else:
num_dropped += 1
#if area > lower_area_threshold:
#cv2.rectangle(im2, (x, y), (x + w, y+h), (0, 255, 0), 1)
#cv2.putText(im2, str(id), (x, y+h), cv2.FONT_HERSHEY_PLAIN, 2, [0,0,255])
#contour_features.append([id, x, y, w, h])
#my_writer.writerow([id, w, h, hw, wh, area, perimeter, is_convex, angle_of_rotation, aspect_ratio, extent, solidity, compactness, roundness])
#final_contours.append(cnt)
#num_retained += 1
#else:
#num_dropped += 1
id += 1
#dump_contours_to_shapefile(contour_features, input_image)
#cv2.drawContours(im2, final_contours, -1, (255, 64, 0), 1)
#cv2.drawContours(processed_image, final_contours, -1, (255, 64, 0), 1)
#cv2.imwrite(out_fname, processed_image)
#print "num contours retained", num_retained
#print "num contours dropped", num_dropped
cv2.imwrite("/home/james/geocrud/adrc/82877433_selected_contours.tif", im2)
###########################################################################
#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!#
###########################################################################
# 050416 stop this part of the script running
###########################################################################
cont = False
if cont:
contour_properties = [[
"contour_id", "h", "w", "area", "hw", "wh", "hxw", "x", "y"
]]
# display info about every contour and add a label to the displayed image
contour_id = 1
for cnt in contours:
[x, y, w, h] = cv2.boundingRect(cnt)
# cv2.contourArea(contour) : compute contour area
contour_area = cv2.contourArea(cnt)
hw = h / w
wh = w / h
hxw = h * w
text_x = x
text_y = y + h + 15
contour_properties.append(
[contour_id, h, w, contour_area, hw, wh, hxw, x, y + h])
#print input_image,contour_id, h, w, hw, wh, contour_area, hxw, x, y+h
contour_id += 1
csv_fname = "".join([
base_output_path,
# job_uuid,
# "_",
# str(max_contour_area),
# "_",
# str(max_contour_envelope_area),
# "_",
# str(aspect_ratio),
"contour_properties.csv"
])
with open(csv_fname, "w") as outpf:
c = csv.writer(outpf,
delimiter=",",
quotechar='"',
quoting=csv.QUOTE_NONNUMERIC)
c.writerows(contour_properties)
out_fname = "".join([
base_output_path,
# job_uuid,
# "_",
# str(max_contour_area),
# "_",
# str(max_contour_envelope_area),
# "_",
# str(aspect_ratio),
"all_contours.png"
])
# cv2.imwrite() : save image to a specified file
cv2.imwrite(out_fname, im2)
#print "All contours..."
#cv2.imshow("All contours", im2)
#cv2.waitKey(0)
#################### End of jrcc ##########################################
# Apply the first 4 rules for digit graphic separation
contours_reject = []
for cnt in contours:
[x, y, w, h] = cv2.boundingRect(cnt)
if cv2.contourArea(
cnt
) < max_contour_area and h / w < aspect_ratio and w / h < aspect_ratio and h * w < max_contour_envelope_area:
continue
else:
contours_reject.append(cnt)
# Erase contours from the image
cv2.drawContours(blank, contours_reject, -1, (0, 0, 0), -1)
#print "Contours Erased..."
#cv2.imshow("Contours Erased", blank)
#cv2.waitKey(0)
#blank2=blank.copy()
image, contours, hierarchy = cv2.findContours(blank, cv2.RETR_EXTERNAL,
cv2.CHAIN_APPROX_SIMPLE)
contour_id = 1
contours_retain = []
contours_reject = []
for cnt in contours:
if cv2.contourArea(cnt) > 50:
contours_retain.append(cnt)
else:
contours_reject.append(cnt)
contour_id += 1
# Erase small contours from the image
cv2.drawContours(blank2, contours_reject, -1, (0, 0, 0), -1)
#TODO draw the full contour rather than it`s MBR
#cv2.drawContours(im2, contours_retain, -1, (0,255,0), 2)
#cv2.drawContours(im2, contours_retain, -1, (0,255,0), 2)
#cv2.imwrite(out_fname, im2)
#print "Small contours erased..."
#cv2.imshow("Small Contours Erased", blank2)
#cv2.waitKey(0)
############# Dealing with Touching Digits ##########################
"""
Stavropoulou_Optical Character Recognition on Scanned Maps compressed.pdf p20
num_cols - choice depends on the size of the touching characters and
subsequently on the size of the font. After the white pixels have been
counted for each column the characters are seperated in the column that
has the minimum number of white pixels
"""
# 070416 - from running against 82877433.tif large important letters
# at the start of words are being incorrectly split by this part of
# the process due to the w>2*num_cols+1 and w>1.2*h criteria
deal_w_touching_digits = False
if not deal_w_touching_digits:
print(
"Hey, one more thing, we`ve turned off dealing with touching digits!"
)
#print '.....Dealing with Touching Digits'
bounding_list = []
digit_im_list = []
# Set the dimensions for resizing
dim = (24, 42)
# Specify the search number of columns for selecting cut column.
num_cols = 5
for cnt in contours_retain:
[x, y, w, h] = cv2.boundingRect(cnt) #finding bounding rectangle
if deal_w_touching_digits:
if w > 2 * num_cols + 1:
if w > 1.2 * h: #in case of fused characters
middle = int(round(w / 2))
fused_character_im = blank2[y - 1:y + h + 1,
x - 1:x + w + 1]
min_col = 0
col_wh_pixels = []
for i in range(middle - num_cols, middle + num_cols):
# examining all the middle columns
col = fused_character_im[:, i]
white_pixels = 0
for j in range(len(col)):
if col[j] != 0:
white_pixels = white_pixels + 1
col_wh_pixels.append(white_pixels)
min_index = col_wh_pixels.index(min(col_wh_pixels))
patch1 = blank2[y - 1:y + h + 1,
x - 1:x + middle - num_cols + min_index]
resized1 = cv2.resize(patch1,
dim,
interpolation=cv2.INTER_AREA)
ret, patch1 = cv2.threshold(resized1, 50, 255,
cv2.THRESH_BINARY)
#Repeat thresholding after interpolation
patch2 = blank2[y - 1:y + h + 1, x + middle - num_cols +
min_index:x + w + 1]
resized2 = cv2.resize(patch2,
dim,
interpolation=cv2.INTER_AREA)
ret, patch2 = cv2.threshold(resized2, 50, 255,
cv2.THRESH_BINARY)
#Repeat thresholding after interpolation
digit_im_list.append(patch1)
digit_im_list.append(patch2)
bounding_list.append([x, y, w / 2, h])
bounding_list.append([x + w / 2, y, w / 2, h])
else:
patch = blank2[y - 1:y + h + 1, x - 1:x + w + 1]
resized = cv2.resize(patch,
dim,
interpolation=cv2.INTER_AREA)
ret, patch = cv2.threshold(resized, 127, 255,
cv2.THRESH_BINARY)
#Repeat thresholding after interpolation
digit_im_list.append(patch)
bounding_list.append([x, y, w, h])
else:
# if we`re not dealing with touching digits do the default always
# as later on the script makes use of data structures updated here
patch = blank2[y - 1:y + h + 1, x - 1:x + w + 1]
resized = cv2.resize(patch, dim, interpolation=cv2.INTER_AREA)
ret, patch = cv2.threshold(resized, 127, 255, cv2.THRESH_BINARY)
#Repeat thresholding after interpolation
digit_im_list.append(patch)
bounding_list.append([x, y, w, h])
arr = np.array(bounding_list)
# TODO - the aoi`s clip too tightly to the contour so we need to add a buffer
# 250416 - what was I thinking when I implemented this?
# for m in xrange(len(arr)):
# x1 = arr[m, 0]
# y1 = arr[m, 1]
# w = arr[m, 2]
# h = arr[m, 3]
# x2 = x1 + w
# y2 = y1 + h
#
# # add a 1px buffer around the contour
# # errors will happen if the buffer extends beyond the extents of the image
# x1 = x1 - 10
# y1 = y1 - 10
# x2 = x2 + 10
# y2 = y2 + 10
#
# aoi_fname = base_output_path + str(m) + ".png"
# cv2.imwrite(aoi_fname, processed_image_b[y1:y2, x1:x2])
processed_image_c = cv2.cvtColor(processed_image_b, cv2.COLOR_GRAY2RGB)
#id = 0
#for m in xrange(len(arr)):
#print m
#cv2.rectangle(im3,(arr[m,0],arr[m,1]),(arr[m,0] + arr[m,2],arr[m,1]+arr[m,3]),(255,0,255),2)
#cv2.rectangle(im3,(int(arr[m,0]),int(arr[m,1])),(int(arr[m,0]) + int(arr[m,2]),int(arr[m,1])+int(arr[m,3])),(255,0,255),2)
#cv2.rectangle(processed_image_c,(int(arr[m,0]),int(arr[m,1])),(int(arr[m,0]) + int(arr[m,2]),int(arr[m,1])+int(arr[m,3])),(255,0,255),1)
#cv2.putText(processed_image_c, str(id), (int(arr[m,0]),int(arr[m,1])), cv2.FONT_HERSHEY_PLAIN, 2, [0,0,255])
#id += 1
#cv2.imshow("Detected Features", im3)
#cv2.waitKey(0)
#out_fname = "".join([base_output_path,
# job_uuid,
# "_",
# str(max_contour_area),
# "_",
# str(max_contour_envelope_area),
# "_",
# str(aspect_ratio),
# "_final_contours.png"])
#cv2.imwrite(out_fname, im3)
#cv2.putText(processed_image_c, "Hello World", (10, 10), cv2.FONT_HERSHEY_PLAIN, 2, [255,0,0] )
cv2.drawContours(processed_image_c, contours_retain, -1, (0, 255, 0), 2)
out_fname = "".join([
base_output_path,
# job_uuid,
# "_",
# str(max_contour_area),
# "_",
# str(max_contour_envelope_area),
# "_",
# str(aspect_ratio),
"img_passed_to_feature_extraction_w_contours.png"
])
cv2.imwrite(out_fname, processed_image_c)
# write out the numpy arrays of the extracted features to a text file
# in a seperate script we will classify these
id = 0
features = []
for m in range(len(arr)):
try:
feature = digit_im_list[m]
# ------------------------ 250416 ---------------------------------
out_fname = "/home/james/geocrud/adrc/" + str(id) + "_sample.png"
cv2.imwrite(out_fname, feature)
# ------------------------ 250416 ---------------------------------
#out_fname = "/home/james/geocrud/adrc/" + job_uuid + "_candidate_" + str(id) + ".png"
#out_fname = "/home/james/geocrud/adrc/" + str(id) + ".png"
#cv2.imwrite(out_fname, feature)
reshaped_feature = feature.reshape((1, 1008))
features.append(reshaped_feature)
id += 1
except Exception as ex:
print(ex)
###########################################################################
#TODO - so we can create a shapefile showing marked up locations we need
#to dump location of candidates
print("".join(["len(digit_im_list): ", str(len(digit_im_list))]))
print("".join(["len(bounding_list): ", str(len(bounding_list))]))
id = 0
with open("/home/james/geocrud/adrc/candidate_locations.csv",
"w") as outpf:
my_writer = csv.writer(outpf,
delimiter=",",
quotechar='"',
quoting=csv.QUOTE_NONNUMERIC)
my_writer.writerow(["id", "x", "y", "w", "h"])
for m in range(len(arr)):
x, y, w, h = bounding_list[m]
my_writer.writerow([id, x, y, w, h])
id += 1
candidates = np.empty((0, 1008))
for feature in features:
try:
candidates = np.append(candidates, feature, 0)
except Exception as ex:
print(ex)
np.savetxt(os.path.join(base_output_path[:-1], "candidates.data"),
candidates)
difference = numpy.zeros((num_of_board, num_of_taxel, num_of_axis))
tracking = numpy.zeros((num_of_board, num_of_taxel, num_of_axis))
limit_threshold = 0xA0
limit_upper = 0xFA
limit_lower = 0x0A
##############
### Create a CSV file ####
for i in range(num_of_board):
del_num = board_start_num + i
if os.path.exists('visualization/LOG%s.csv' % del_num):
os.remove('visualization/LOG%s.csv' % del_num)
csvfile = open('visualization/LOG%s.csv' % del_num,
'wb') #Create a new csv file
filewrite = csv.writer(csvfile)
filewrite.writerow(
('B1S1X', 'B1S1Y', 'B1S1Z', 'B1S2X', 'B1S2Y', 'B1S2Z', 'B1S3X',
'B1S3Y', 'B1S3Z', 'B1S4X', 'B1S4Y', 'B1S4Z', 'B2S1X', 'B2S1Y',
'B2S1Z', 'B2S2X', 'B2S2Y', 'B2S2Z', 'B2S3X', 'B2S3Y', 'B2S3Z',
'B2S4X', 'B2S4Y', 'B2S4Z', 'B3S1X', 'B3S1Y', 'B3S1Z', 'B3S2X',
'B3S2Y', 'B3S2Z', 'B3S3X', 'B3S3Y', 'B3S3Z', 'B3S4X', 'B3S4Y',
'B3S4Z', 'B4S1X', 'B4S1Y', 'B4S1Z', 'B4S2X', 'B4S2Y', 'B4S2Z',
'B4S3X', 'B4S3Y', 'B4S3Z', 'B4S4X', 'B4S4Y', 'B4S4Z'))
csvfile.close() # do we need to close it first?
# ---> cif = ntcan.CIF( net, RxQueueSize, RxTimeOut, TxQueueSize, TxTimeOut, Flags)
net = 0 # logical CAN Network [0, 255]
RxQS = 1 # RxQueueSize [0, 10000]
RxTO = 2000 # RxTimeOut in Millisconds
TxQS = 1 # TxQueueSize [0, 10000]
def output_writer(f):
write = csv.writer(open(f, 'wb'))
return write
def _create_cache(self):
# Save all data into cache file(s).
self._cache_positions = []
self._position = 0
percent = 0
if single_or_rankzero():
progress(None)
while self._position < self._data_source._size:
if single_or_rankzero():
if self._position % int(self._data_source._size/20+1) == 0:
progress('Create cache', self._position *
1.0 / self._data_source._size)
self._store_data_to_cache_buffer(self._position)
self._position += 1
if len(self._cache_positions) > 0:
self._save_cache_to_file()
if single_or_rankzero():
progress(None)
# Adjust data size into reseted position. In most case it means
# multiple of bunch(mini-batch) size.
num_of_cache_files = int(numpy.ceil(
float(self._data_source._size) / self._cache_size))
self._cache_file_order = self._cache_file_order[
0:num_of_cache_files]
self._cache_file_data_orders = self._cache_file_data_orders[
0:num_of_cache_files]
if self._data_source._size % self._cache_size != 0:
self._cache_file_data_orders[num_of_cache_files - 1] = self._cache_file_data_orders[
num_of_cache_files - 1][0:self._data_source._size % self._cache_size]
# Create Index
index_filename = os.path.join(self._cache_dir, "cache_index.csv")
with open(index_filename, 'w') as f:
writer = csv.writer(f, lineterminator='\n')
for fn, orders in zip(self._cache_file_names, self._cache_file_data_orders):
writer.writerow((os.path.basename(fn), len(orders)))
# Create Info
if self._cache_file_format == ".npy":
info_filename = os.path.join(self._cache_dir, "cache_info.csv")
with open(info_filename, 'w') as f:
writer = csv.writer(f, lineterminator='\n')
for variable in self._variables:
writer.writerow((variable, ))
# Create original.csv
if self._data_source._original_source_uri is not None:
fr = FileReader(self._data_source._original_source_uri)
with fr.open() as f:
csv_lines = [x.decode('utf-8') for x in f.readlines()]
with open(os.path.join(self._cache_dir, "original.csv"), 'w') as o:
for l in csv_lines:
o.write(l)
# Create order.csv
if self._data_source._order is not None and \
self._data_source._original_order is not None:
with open(os.path.join(self._cache_dir, "order.csv"), 'w') as o:
writer = csv.writer(o, lineterminator='\n')
for orders in zip(self._data_source._original_order, self._data_source._order):
writer.writerow(list(orders))
def main():
config = Config()
#modeler = model.VGG(config)
modeler = model.VGG16(include_top=False)
#modeler = model.ResNet50(include_top=False, input_shape=[config.img_height,config.img_width, 3], pooling='max')
inputs = Input(shape=[config.img_height,config.img_width,3])
y = modeler(inputs)
y = Flatten()(y)
y = Dense(4096, activation='relu', name='fc1')(y)
y = Dense(4096, activation='relu', name='fc2')(y)
y = Dense(config.type_size, activation='softmax', name='predictions')(y)
modeler = Model(inputs, y, name='vgg16')
'''
y = Dense(config.type_size, activation='softmax', name='fc17')(y)
modeler = Model(inputs, y, name='resnet50')
'''
modeler.load_weights(config.params_dir + config.net_name + "-weather-params-" + str(config.test_num) + ".h5")
#modeler.compile(loss='categorical_crossentropy', optimizer=SGD(lr=config.learning_rate, momentum=0.9,nesterov=True))
# read data to test("data/train")
test_reader = read_data.VGGReader(config)
pre_prob = list()
with open("./thresholds/" + config.net_name + "-weather-thresholds-" + config.test_num + ".txt", 'rb') as fr:
for line in fr:
tmp = re.split(' ', line.strip())
for i in range(config.type_size):
pre_prob.append(float(tmp[i]))
print "thresholds: ", pre_prob
test_labels = list()
pre_labels = list()
val_labels = list()
min_true = [1.0, 1.0, 1.0, 1.0]
max_false = [0.0, 0.0, 0.0, 0.0]
print "start testing..."
for step in range(config.test_size):
if step % (config.test_size // 10) == 0:
print 100 * step // config.test_size, "%"
images_test, labels_test, filesname_test = test_reader.batch()
prob = modeler.predict(images_test)
test_index = list()
for i in range(config.type_size):
val_labels.append(labels_test[0][i])
if prob[0][i] > pre_prob[i]:
test_index.append(i)
# get min_true
if labels_test[0][i] == 1.0 and prob[0][i] > 0.1 and prob[0][i] < min_true[i]:
min_true[i] = prob[0][i]
if labels_test[0][i] == 0.0 and prob[0][i] > max_false[i]:
max_false[i] = prob[0][i]
'''
if step % 10 == 0 and config.val_mode == True:
print labels_test[0]
print prob[0]
'''
s = filesname_test[0]
for n in range(config.type_size):
is_in = False
for m in range(len(test_index)):
if n == test_index[m]:
is_in = True
if is_in:
s += " 1.0"
pre_labels.append(1.0)
else:
s += " 0.0"
pre_labels.append(0.0)
test_labels.append(s)
print "scores: ", fbeta_score(val_labels, pre_labels, beta=2)
print "min_true: ", min_true
print "max_false, ", max_false
if config.val_mode == False:
with open("./labels/weather_test_results.csv", 'w') as fr:
fcsv = csv.writer(fr)
for i in range(len(test_labels)):
fcsv.writerow([test_labels[i]])
direction = agent.step()
board = game.move(direction)
direct={}
i=0
k=0
filename = 'test.csv'
if os.path.exists(filename):
head = True
else:
head = False
os.mknod(filename)
with open(filename, "a") as csvfile:
writer = csv.writer(csvfile)
if not head:
writer.writerow(["11","12","13","14",\
"21","22","23","24",\
"31","32","33","34",\
"41","42","43","44",\
"direction"])
while i < repeat_times:
game = Game(GAME_SIZE, SCORE_TO_WIN)
board = game.board
print('Repeat times:', i)
while(game.end == 0):
agent = ExpectiMaxAgent(game, Display())
def RandomWalkElf(program, outfile, mclass, max_subtraces, max_explored_subtraces, min_size):
csvwriter = csv.writer(open(outfile, "a+"), delimiter='\t')
elf = ELF(program)
# plt is inverted
inv_plt = dict()
for func, addr in elf.plt.items():
if func in specs: # external functions are discarded
inv_plt[addr] = func
elf.plt = inv_plt
cond_control_flow_ins = ["jo", "jno", "js", "jns", "je",
"jz","jnz", "jb", "jnae", "jc",
"jnb", "jae", "jnc", "jbe", "jna",
"ja", "jnbe", "jl", "jnge", "jge",
"jnl", "jle", "jng", "jg", "jnle",
"jp", "jpe", "jnp", "jpo", "jcxz", "jecxz"]
ncond_control_flow_ins = ["ret","jmp","call", "retq","jmp","callq"]
control_flow_ins = cond_control_flow_ins + ncond_control_flow_ins
raw_inss = elf.GetRawInss()
useful_inss_list = []
useful_inss_dict = dict()
libc_calls = []
labels = dict()
#print sys.argv[1]+"\t",
#rclass = str(1)
for i,ins in enumerate(raw_inss.split("\n")):
# prefix removal
ins = ins.replace("repz ","")
ins = ins.replace("rep ","")
pins = ins.split("\t")
#print pins
ins_addr = pins[0].replace(":","").replace(" ","")
#print pins,ins_addr
if len(pins) == 1 and ">" in ins: #label
#print ins
#assert(0)
x = pins[0].split(" ")
ins_addr = x[0]
y = [i,ins_addr, None, None]
useful_inss_dict[ins_addr] = y
useful_inss_list.append(y)
#print "label:",y
elif any(map( lambda x: x in ins, control_flow_ins)) and len(pins) == 3: # control flow instruction
#print pins
x = pins[2].split(" ")
ins_nme = x[0]
ins_jaddr = x[-2]
#if ("" == ins_jaddr):
# print pins
#print x
#print ins_nme, ins_jaddr
y = [i, ins_addr, ins_nme, ins_jaddr]
useful_inss_dict[ins_addr] = y
useful_inss_list.append(y)
if "call" in pins[2]:
if ins_jaddr <> '':
func_addr = int(ins_jaddr,16)
if func_addr in elf.plt:
libc_calls.append(i)
else: # all other instructions
y = [i, ins_addr, None, None]
useful_inss_dict[ins_addr] = y
useful_inss_list.append(y)
#print useful_inss_list
max_inss = len(useful_inss_list)
traces = set()
collected_traces = ""
# exploration time!
for _ in range(max_explored_subtraces):
# resuling (sub)trace
r = ""
# starting point
i = random.choice(libc_calls)
j = 0
#r = elf.path+"\t"
r = ""
while True:
# last instruction case
if (i+j) == max_inss:
break
_,ins_addr,ins_nme,ins_jaddr = useful_inss_list[i+j]
#print i+j,ins_nme, ins_jaddr
if ins_nme in ['call', 'callq']: # ordinary call
#"addr", ins_jaddr
if ins_jaddr == '':
break # parametric jmp, similar to ret for us
ins_jaddr = int(ins_jaddr,16)
if ins_jaddr in elf.plt:
r = r + " " + elf.plt[ins_jaddr]
if elf.plt[ins_jaddr] == "exit":
break
else:
if ins_jaddr in useful_inss_dict:
#assert(0)
#r = r + " " + hex(ins_jaddr)
i,_,_,_ = useful_inss_dict[ins_jaddr]
j = 0
continue
else:
pass # ignored call
elif ins_nme in ['ret','retq']:
break
else:
pass
#print i+j,ins_nme, ins_jaddr
#print j
if ins_nme == 'jmp' :
if ins_jaddr in elf.plt: # call equivalent using jmp
r = r + " " + elf.plt[jaddr]
else:
if ins_jaddr == '':
break # parametric jmp, similar to ret for us
ins_jaddr = int(ins_jaddr,16)
if ins_jaddr in useful_inss_dict:
#r = r + " " + hex(ins_jaddr)
i,_,_,_ = useful_inss_dict[ins_jaddr]
j = 0
continue
else:
pass # ignored call
if ins_nme in cond_control_flow_ins:
assert(ins_jaddr <> None)
cond = random.randint(0,1)
if cond == 1:
i,_,_,_ = useful_inss_dict[ins_jaddr]
j = 0
continue
j = j + 1
#r = r + "\t"+rclass
x = hash(r)
size = len(r.split(" "))-1
if x not in traces and size >= min_size:
#print r+" .",
collected_traces = collected_traces + r + " ."
traces.add(x)
if len(traces) >= max_subtraces:
break
row = [elf.path, collected_traces]
if mclass is not None:
row.append(mclass)
csvwriter.writerow(row)
if single_mode:
profile_paths = [args.profile]
else:
profile_paths = list(glob.glob(os.path.join(args.profile, "*")))
print("writing predictions to %s ..." % output_file)
print("\n-------\n")
print(" screen_name | class | confidence\n")
samples_per_inference = 5
samples = {}
with open(output_file, 'w+t', newline='') as fp:
w = csv.writer(fp)
w.writerow(['user_id', 'user_name', 'prediction', 'confidence'])
for profile_path in profile_paths:
prof = profile.load(profile_path, quiet=True)
if prof is None:
continue
((user, tweets, replies, retweets), vector) = prof
vector = data.nomalized_from_dict(norm, vector)
prediction = model.predict(vector)[0]
(label, confidence) = print_pred(args, user['screen_name'], prediction)
def __init__(self, f, dialect=csv.excel, encoding="utf-8", **kwds):
# Redirect output to a queue
self.queue = cStringIO.StringIO()
self.writer = csv.writer(self.queue, dialect=dialect, **kwds)
self.stream = f
self.encoder = getincrementalencoder(encoding)()
def save_prediction_unknowns(model, loader, dataset, args):
model.eval()
LOGGER.info('processing {}..'.format(args.unknown_path))
embeddings = pickle.load(open(args.embed_path, 'rb'))
#ingr2category = pickle.load(open(args.ingr2category_dir, 'rb'))
#category2rep = pickle.load(open(args.category2rep_dir, 'rb'))
csv_writer = csv.writer(open(args.checkpoint_dir + 'prediction_unknowns_' +
args.model_name + '.csv', 'w', newline=''))
csv_writer.writerow(['ingr1', 'ingr2', 'prediction'])
# csv_writer.writerow(['ingr1', 'ingr1_cate', 'ingr2', 'ingr2_cate', 'prediction'])
#for d_idx, (d1, d1_r, d1_l, d2, d2_r, d2_l, score) in enumerate(loader):
# print(d1, d1_r, d1_l, d2, d2_r, d2_l, score)
df = pd.read_csv(args.unknown_path, sep=",")
df = df.reset_index()
print(len(df))
batch = []
for row_idx, row in df.iterrows():
ingr1 = row['ingr1']
ingr2 = row['ingr2']
try:
ingr1_r = embeddings[ingr1]
ingr2_r = embeddings[ingr2]
#ignore this categorical features
ingr1_c = embeddings[ingr1]
ingr2_c = embeddings[ingr2]
except KeyError:
continue
example = [ingr1, ingr1_r, ingr1_c, len(ingr1_r),
ingr2, ingr2_r, ingr2_c, len(ingr2_r), 0]
batch.append(example)
if len(batch) == 256:
inputs = dataset.collate_fn(batch)
outputs = model(inputs[1].cuda(), inputs[2].cuda(), inputs[3],
inputs[5].cuda(), inputs[6].cuda(), inputs[7])
predictions = outputs[2].data.cpu().numpy()
for example, pred in zip(batch, predictions):
csv_writer.writerow([example[0], example[4], pred])
batch = []
# Print progress
if row_idx % 5000 == 0 or row_idx == len(df) - 1:
_progress = '{}/{} saving unknown predictions..'.format(
row_idx + 1, len(df))
LOGGER.info(_progress)
if len(batch) > 0:
inputs = dataset.collate_fn(batch)
outputs, _, _ = model(inputs[1].cuda(), inputs[2].cuda(), inputs[3],
inputs[5].cuda(), inputs[6].cuda(), inputs[7])
predictions = outputs[2].data.cpu().numpy()
for example, pred in zip(batch, predictions):
csv_writer.writerow([example[0], example[4], pred])
print(args)
input_file=args.input_lines_csv[0]
input_audio_dir=args.input_audio_dir[0]
output_dir=args.output_dir[0]
language_code=args.language_code[0]
sound_find_string=None
if args.sound_find_string is not None: sound_find_string=args.sound_find_string[0]
file_name='aeneas.csv'
output_file=os.path.join(output_dir,file_name)
write_file_handle=open(output_file,'w',encoding='utf-8')
write_file = csv.writer(write_file_handle)
# Write header as requested by user
if not(args.noheader):write_file.writerow(('fileset','book','chapter','line_number', 'verse_number','verse_content','time'))
book_chapter_list=list()
def get_chapters_index(df):
current_book_chapter=df['book'][0]+'_'+'0'+df['chapter'][0]
book_chapter_list.append(df['book'][0]+'_'+'0'+df['chapter'][0])
for i in range(1,len(df)):
if df['book'][i]!=current_book_chapter.split('_')[0] or df['chapter'][i]!=(current_book_chapter.split('_')[1]).lstrip('0'):
if int(df['chapter'][i])<10:
current_book_chapter=df['book'][i]+'_'+df['chapter'][i]
book_chapter_list.append((df['book'][i]+'_'+'0'+df['chapter'][i]).replace(' ','_'))
# import module
import csv
import math
# create file
file = open('cvs.csv', 'w')
# create csvwriter
csvwriter = csv.writer(file, delimiter=',')
# write info
csvwriter.writerow(['a', 'b', 'hypotenuse'])
for a in range(1,101):
for b in range(1, 101):
hypotenuse = math.sqrt(a ** 2 + b ** 2)
csvwriter.writerow([a, b, hypotenuse])
# close file
file.close()
def read(humfile, sonpath, cs2cs_args, c, draft, doplot, t, bedpick, flip_lr,
model, calc_bearing, filt_bearing, chunk): #cog = 1,
'''
Read a .DAT and associated set of .SON files recorded by a Humminbird(R)
instrument.
Parse the data into a set of memory mapped files that will
subsequently be used by the other functions of the PyHum module.
Export time-series data and metadata in other formats.
Create a kml file for visualising boat track
Syntax
----------
[] = PyHum.read(humfile, sonpath, cs2cs_args, c, draft, doplot, t, bedpick, flip_lr, chunksize, model, calc_bearing, filt_bearing, chunk)
Parameters
------------
humfile : str
path to the .DAT file
sonpath : str
path where the *.SON files are
cs2cs_args : int, *optional* [Default="epsg:26949"]
arguments to create coordinates in a projected coordinate system
this argument gets given to pyproj to turn wgs84 (lat/lon) coordinates
into any projection supported by the proj.4 libraries
c : float, *optional* [Default=1450.0]
speed of sound in water (m/s). Defaults to a value of freshwater
draft : float, *optional* [Default=0.3]
draft from water surface to transducer face (m)
doplot : float, *optional* [Default=1]
if 1, plots will be made
t : float, *optional* [Default=0.108]
length of transducer array (m).
Default value is that of the 998 series Humminbird(R)
bedpick : int, *optional* [Default=1]
if 1, bedpicking with be carried out automatically
if 0, user will be prompted to pick the bed location on screen
flip_lr : int, *optional* [Default=0]
if 1, port and starboard scans will be flipped
(for situations where the transducer is flipped 180 degrees)
model: int, *optional* [Default=998]
A 3 or 4 number code indicating the model number
Examples: 998, 997, 1198, 1199
calc_bearing : float, *optional* [Default=0]
if 1, bearing will be calculated from coordinates
filt_bearing : float, *optional* [Default=0]
if 1, bearing will be filtered
chunk : str, *optional* [Default='d100' (distance, 100 m)]
letter, followed by a number.
There are the following letter options:
'd' - parse chunks based on distance, then number which is distance in m
'p' - parse chunks based on number of pings, then number which is number of pings
'h' - parse chunks based on change in heading, then number which is the change in heading in degrees
'1' - process just 1 chunk
Returns
---------
sonpath+base+'_data_port.dat': memory-mapped file
contains the raw echogram from the port side
sidescan sonar (where present)
sonpath+base+'_data_port.dat': memory-mapped file
contains the raw echogram from the starboard side
sidescan sonar (where present)
sonpath+base+'_data_dwnhi.dat': memory-mapped file
contains the raw echogram from the high-frequency
echosounder (where present)
sonpath+base+'_data_dwnlow.dat': memory-mapped file
contains the raw echogram from the low-frequency
echosounder (where present)
sonpath+base+"trackline.kml": google-earth kml file
contains the trackline of the vessel during data
acquisition
sonpath+base+'rawdat.csv': comma separated value file
contains time-series data. columns corresponding to
longitude
latitude
easting (m)
northing (m)
depth to bed (m)
alongtrack cumulative distance (m)
vessel heading (deg.)
sonpath+base+'meta.mat': .mat file
matlab format file containing a dictionary object
holding metadata information. Fields are:
e : ndarray, easting (m)
n : ndarray, northing (m)
es : ndarray, low-pass filtered easting (m)
ns : ndarray, low-pass filtered northing (m)
lat : ndarray, latitude
lon : ndarray, longitude
shape_port : tuple, shape of port scans in memory mapped file
shape_star : tuple, shape of starboard scans in memory mapped file
shape_hi : tuple, shape of high-freq. scans in memory mapped file
shape_low : tuple, shape of low-freq. scans in memory mapped file
dep_m : ndarray, depth to bed (m)
dist_m : ndarray, distance along track (m)
heading : ndarray, heading of vessel (deg. N)
pix_m: float, size of 1 pixel in across-track dimension (m)
bed : ndarray, depth to bed (m)
c : float, speed of sound in water (m/s)
t : length of sidescan transducer array (m)
spd : ndarray, vessel speed (m/s)
time_s : ndarray, time elapsed (s)
caltime : ndarray, unix epoch time (s)
'''
# prompt user to supply file if no input file given
if not humfile:
print('An input file is required!!!!!!')
Tk().withdraw(
) # we don't want a full GUI, so keep the root window from appearing
humfile = askopenfilename(filetypes=[("DAT files", "*.DAT")])
# prompt user to supply directory if no input sonpath is given
if not sonpath:
print('A *.SON directory is required!!!!!!')
Tk().withdraw(
) # we don't want a full GUI, so keep the root window from appearing
sonpath = askdirectory()
# print given arguments to screen and convert data type where necessary
if humfile:
print('Input file is %s' % (humfile))
if sonpath:
print('Son files are in %s' % (sonpath))
if cs2cs_args:
print('cs2cs arguments are %s' % (cs2cs_args))
if draft:
draft = float(draft)
print('Draft: %s' % (str(draft)))
if c:
c = float(c)
print('Celerity of sound: %s m/s' % (str(c)))
if doplot:
doplot = int(doplot)
if doplot == 0:
print("Plots will not be made")
if flip_lr:
flip_lr = int(flip_lr)
if flip_lr == 1:
print("Port and starboard will be flipped")
if t:
t = np.asarray(t, float)
print('Transducer length is %s m' % (str(t)))
if bedpick:
bedpick = np.asarray(bedpick, int)
if bedpick == 1:
print('Bed picking is auto')
elif bedpick == 0:
print('Bed picking is manual')
else:
print('User will be prompted per chunk about bed picking method')
if chunk:
chunk = str(chunk)
if chunk[0] == 'd':
chunkmode = 1
chunkval = int(chunk[1:])
print('Chunks based on distance of %s m' % (str(chunkval)))
elif chunk[0] == 'p':
chunkmode = 2
chunkval = int(chunk[1:])
print('Chunks based on %s pings' % (str(chunkval)))
elif chunk[0] == 'h':
chunkmode = 3
chunkval = int(chunk[1:])
print('Chunks based on heading devation of %s degrees' %
(str(chunkval)))
elif chunk[0] == '1':
chunkmode = 4
chunkval = 1
print('Only 1 chunk will be produced')
else:
print(
"Chunk mode not understood - should be 'd', 'p', or 'h' - using defaults"
)
chunkmode = 1
chunkval = 100
print('Chunks based on distance of %s m' % (str(chunkval)))
if model:
try:
model = int(model)
print("Data is from the %s series" % (str(model)))
except:
if model == 'onix':
model = 0
print("Data is from the ONIX series")
elif model == 'helix':
model = 1
print("Data is from the HELIX series")
elif model == 'mega':
model = 2
print("Data is from the MEGA series")
# if cog:
# cog = int(cog)
# if cog==1:
# print "Heading based on course-over-ground"
if calc_bearing:
calc_bearing = int(calc_bearing)
if calc_bearing == 1:
print("Bearing will be calculated from coordinates")
if filt_bearing:
filt_bearing = int(filt_bearing)
if filt_bearing == 1:
print("Bearing will be filtered")
## for debugging
#humfile = r"test.DAT"; sonpath = "test_data"
#cs2cs_args = "epsg:26949"; doplot = 1; draft = 0
#c=1450; bedpick=1; fliplr=1; chunk = 'd100'
#model=998; cog=1; calc_bearing=0; filt_bearing=0
#if model==2:
# f = 1000
#else:
f = 455
try:
print(
"Checking the epsg code you have chosen for compatibility with Basemap ... "
)
from mpl_toolkits.basemap import Basemap
m = Basemap(projection='merc',
epsg=cs2cs_args.split(':')[1],
resolution='i',
llcrnrlon=10,
llcrnrlat=10,
urcrnrlon=30,
urcrnrlat=30)
del m
print("... epsg code compatible")
except (ValueError):
print(
"Error: the epsg code you have chosen is not compatible with Basemap"
)
print(
"please choose a different epsg code (http://spatialreference.org/)"
)
print("program will now close")
sys.exit()
# start timer
if os.name == 'posix': # true if linux/mac or cygwin on windows
start = time.time()
else: # windows
start = time.clock()
# if son path name supplied has no separator at end, put one on
if sonpath[-1] != os.sep:
sonpath = sonpath + os.sep
# get the SON files from this directory
sonfiles = glob.glob(sonpath + '*.SON')
if not sonfiles:
sonfiles = glob.glob(os.getcwd() + os.sep + sonpath + '*.SON')
base = humfile.split('.DAT') # get base of file name for output
base = base[0].split(os.sep)[-1]
# remove underscores, negatives and spaces from basename
base = humutils.strip_base(base)
print("WARNING: Because files have to be read in byte by byte,")
print("this could take a very long time ...")
#reading each sonfile in parallel should be faster ...
try:
o = Parallel(n_jobs=np.min([len(sonfiles), cpu_count()]), verbose=0)(
delayed(getscans)(sonfiles[k], humfile, c, model, cs2cs_args)
for k in range(len(sonfiles)))
X, Y, A, B = zip(*o)
for k in range(len(Y)):
if Y[k] == 'sidescan_port':
dat = A[k] #data.gethumdat()
metadat = B[k] #data.getmetadata()
if flip_lr == 0:
data_port = X[k].astype('int16')
else:
data_star = X[k].astype('int16')
elif Y[k] == 'sidescan_starboard':
if flip_lr == 0:
data_star = X[k].astype('int16')
else:
data_port = X[k].astype('int16')
elif Y[k] == 'down_lowfreq':
data_dwnlow = X[k].astype('int16')
elif Y[k] == 'down_highfreq':
data_dwnhi = X[k].astype('int16')
elif Y[k] == 'down_vhighfreq': #hopefully this only applies to mega systems
data_dwnhi = X[k].astype('int16')
del X, Y, A, B, o
old_pyread = 0
if 'data_port' not in locals():
data_port = ''
print("portside scan not available")
if 'data_star' not in locals():
data_star = ''
print("starboardside scan not available")
if 'data_dwnhi' not in locals():
data_dwnlow = ''
print("high-frq. downward scan not available")
if 'data_dwnlow' not in locals():
data_dwnlow = ''
print("low-frq. downward scan not available")
except: # revert back to older version if paralleleised version fails
print(
"something went wrong with the parallelised version of pyread ...")
try:
import pyread
except:
from . import pyread
data = pyread.pyread(sonfiles, humfile, c, model, cs2cs_args)
dat = data.gethumdat()
metadat = data.getmetadata()
old_pyread = 1
nrec = len(metadat['n'])
metadat['instr_heading'] = metadat['heading'][:nrec]
#metadat['heading'] = humutils.get_bearing(calc_bearing, filt_bearing, cog, metadat['lat'], metadat['lon'], metadat['instr_heading'])
try:
es = humutils.runningMeanFast(metadat['e'][:nrec],
len(metadat['e'][:nrec]) / 100)
ns = humutils.runningMeanFast(metadat['n'][:nrec],
len(metadat['n'][:nrec]) / 100)
except:
es = metadat['e'][:nrec]
ns = metadat['n'][:nrec]
metadat['es'] = es
metadat['ns'] = ns
try:
trans = pyproj.Proj(init=cs2cs_args)
except:
trans = pyproj.Proj(cs2cs_args.lstrip(), inverse=True)
lon, lat = trans(es, ns, inverse=True)
metadat['lon'] = lon
metadat['lat'] = lat
metadat['heading'] = humutils.get_bearing(calc_bearing, filt_bearing,
metadat['lat'], metadat['lon'],
metadat['instr_heading']) #cog
dist_m = humutils.get_dist(lat, lon)
metadat['dist_m'] = dist_m
if calc_bearing == 1: # recalculate speed, m/s
ds = np.gradient(np.squeeze(metadat['time_s']))
dx = np.gradient(np.squeeze(metadat['dist_m']))
metadat['spd'] = dx[:nrec] / ds[:nrec]
# theta at 3dB in the horizontal
theta3dB = np.arcsin(c / (t * (f * 1000)))
#resolution of 1 sidescan pixel to nadir
ft = (np.pi / 2) * (1 / theta3dB) #/ (f/455)
dep_m = humutils.get_depth(metadat['dep_m'][:nrec])
if old_pyread == 1: #older pyread version
# port scan
try:
if flip_lr == 0:
data_port = data.getportscans().astype('int16')
else:
data_port = data.getstarscans().astype('int16')
except:
data_port = ''
print("portside scan not available")
if data_port != '':
Zt, ind_port = makechunks_scan(chunkmode, chunkval, metadat, data_port,
0)
del data_port
## create memory mapped file for Z
shape_port = io.set_mmap_data(sonpath, base, '_data_port.dat', 'int16',
Zt)
##we are only going to access the portion of memory required
port_fp = io.get_mmap_data(sonpath, base, '_data_port.dat', 'int16',
shape_port)
if old_pyread == 1: #older pyread version
# starboard scan
try:
if flip_lr == 0:
data_star = data.getstarscans().astype('int16')
else:
data_star = data.getportscans().astype('int16')
except:
data_star = ''
print("starboardside scan not available")
if data_star != '':
Zt, ind_star = makechunks_scan(chunkmode, chunkval, metadat, data_star,
1)
del data_star
# create memory mapped file for Z
shape_star = io.set_mmap_data(sonpath, base, '_data_star.dat', 'int16',
Zt)
star_fp = io.get_mmap_data(sonpath, base, '_data_star.dat', 'int16',
shape_star)
if 'star_fp' in locals() and 'port_fp' in locals():
# check that port and starboard are same size
# and trim if not
if np.shape(star_fp) != np.shape(port_fp):
print(
"port and starboard scans are different sizes ... rectifying")
if np.shape(port_fp[0])[1] > np.shape(star_fp[0])[1]:
tmp = port_fp.copy()
tmp2 = np.empty_like(star_fp)
for k in range(len(tmp)):
tmp2[k] = tmp[k][:, :np.shape(star_fp[k])[1]]
del tmp
# create memory mapped file for Z
shape_port = io.set_mmap_data(sonpath, base, '_data_port2.dat',
'int16', tmp2)
#shape_star = shape_port.copy()
shape_star = tuple(np.asarray(shape_port).copy())
##we are only going to access the portion of memory required
port_fp = io.get_mmap_data(sonpath, base, '_data_port2.dat',
'int16', shape_port)
ind_port = list(ind_port)
ind_port[-1] = np.shape(star_fp[0])[1]
ind_port = tuple(ind_port)
elif np.shape(port_fp[0])[1] < np.shape(star_fp[0])[1]:
tmp = star_fp.copy()
tmp2 = np.empty_like(port_fp)
for k in range(len(tmp)):
tmp2[k] = tmp[k][:, :np.shape(port_fp[k])[1]]
del tmp
# create memory mapped file for Z
shape_port = io.set_mmap_data(sonpath, base, '_data_star2.dat',
'int16', tmp2)
#shape_star = shape_port.copy()
shape_star = tuple(np.asarray(shape_port).copy())
#we are only going to access the portion of memory required
star_fp = io.get_mmap_data(sonpath, base, '_data_star2.dat',
'int16', shape_star)
ind_star = list(ind_star)
ind_star[-1] = np.shape(port_fp[0])[1]
ind_star = tuple(ind_star)
if old_pyread == 1: #older pyread version
# low-freq. sonar
try:
data_dwnlow = data.getlowscans().astype('int16')
except:
data_dwnlow = ''
print("low-freq. scan not available")
if data_dwnlow != '':
Zt, ind_low = makechunks_scan(chunkmode, chunkval, metadat,
data_dwnlow, 2)
del data_dwnlow
# create memory mapped file for Z
shape_low = io.set_mmap_data(sonpath, base, '_data_dwnlow.dat',
'int16', Zt)
##we are only going to access the portion of memory required
dwnlow_fp = io.get_mmap_data(sonpath, base, '_data_dwnlow.dat',
'int16', shape_low)
if old_pyread == 1: #older pyread version
# hi-freq. sonar
try:
data_dwnhi = data.gethiscans().astype('int16')
except:
data_dwnhi = ''
print("high-freq. scan not available")
if data_dwnhi != '':
Zt, ind_hi = makechunks_scan(chunkmode, chunkval, metadat, data_dwnhi,
3)
del data_dwnhi
# create memory mapped file for Z
shape_hi = io.set_mmap_data(sonpath, base, '_data_dwnhi.dat', 'int16',
Zt)
dwnhi_fp = io.get_mmap_data(sonpath, base, '_data_dwnhi.dat', 'int16',
shape_hi)
if 'dwnhi_fp' in locals() and 'dwnlow_fp' in locals():
# check that low and high are same size
# and trim if not
if (np.shape(dwnhi_fp) != np.shape(dwnlow_fp)) and (chunkmode != 4):
print("dwnhi and dwnlow are different sizes ... rectifying")
if np.shape(dwnhi_fp[0])[1] > np.shape(dwnlow_fp[0])[1]:
tmp = dwnhi_fp.copy()
tmp2 = np.empty_like(dwnlow_fp)
for k in range(len(tmp)):
tmp2[k] = tmp[k][:, :np.shape(dwnlow_fp[k])[1]]
del tmp
# create memory mapped file for Z
shape_low = io.set_mmap_data(sonpath, base, '_data_dwnhi2.dat',
'int16', tmp2)
#shape_hi = shape_low.copy()
shape_hi = tuple(np.asarray(shape_low).copy())
##we are only going to access the portion of memory required
dwnhi_fp = io.get_mmap_data(sonpath, base, '_data_dwnhi2.dat',
'int16', shape_hi)
ind_hi = list(ind_hi)
ind_hi[-1] = np.shape(dwnlow_fp[0])[1]
ind_hi = tuple(ind_hi)
elif np.shape(dwnhi_fp[0])[1] < np.shape(dwnlow_fp[0])[1]:
tmp = dwnlow_fp.copy()
tmp2 = np.empty_like(dwnhi_fp)
for k in range(len(tmp)):
tmp2[k] = tmp[k][:, :np.shape(dwnhi_fp[k])[1]]
del tmp
# create memory mapped file for Z
shape_low = io.set_mmap_data(sonpath, base,
'_data_dwnlow2.dat', 'int16',
tmp2)
#shape_hi = shape_low.copy()
shape_hi = tuple(np.asarray(shape_low).copy())
##we are only going to access the portion of memory required
dwnlow_fp = io.get_mmap_data(sonpath, base,
'_data_dwnlow2.dat', 'int16',
shape_low)
ind_low = list(ind_low)
ind_low[-1] = np.shape(dwnhi_fp[0])[1]
ind_low = tuple(ind_low)
if old_pyread == 1: #older pyread version
del data
if ('shape_port' in locals()) and (chunkmode != 4):
metadat['shape_port'] = shape_port
nrec = metadat['shape_port'][0] * metadat['shape_port'][2]
elif ('shape_port' in locals()) and (chunkmode == 4):
metadat['shape_port'] = shape_port
nrec = metadat['shape_port'][1]
else:
metadat['shape_port'] = ''
if ('shape_star' in locals()) and (chunkmode != 4):
metadat['shape_star'] = shape_star
nrec = metadat['shape_star'][0] * metadat['shape_star'][2]
elif ('shape_star' in locals()) and (chunkmode == 4):
metadat['shape_star'] = shape_star
nrec = metadat['shape_star'][1]
else:
metadat['shape_star'] = ''
if ('shape_hi' in locals()) and (chunkmode != 4):
metadat['shape_hi'] = shape_hi
#nrec = metadat['shape_hi'][0] * metadat['shape_hi'][2] * 2
elif ('shape_hi' in locals()) and (chunkmode == 4):
metadat['shape_hi'] = shape_hi
else:
metadat['shape_hi'] = ''
if ('shape_low' in locals()) and (chunkmode != 4):
metadat['shape_low'] = shape_low
#nrec = metadat['shape_low'][0] * metadat['shape_low'][2] * 2
elif ('shape_low' in locals()) and (chunkmode == 4):
metadat['shape_low'] = shape_low
else:
metadat['shape_low'] = ''
#make kml boat trackline
humutils.make_trackline(lon, lat, sonpath, base)
if 'port_fp' in locals() and 'star_fp' in locals():
#if not os.path.isfile(os.path.normpath(os.path.join(sonpath,base+'meta.mat'))):
if 2 > 1:
if bedpick == 1: # auto
x, bed = humutils.auto_bedpick(ft, dep_m, chunkmode, port_fp,
c)
if len(dist_m) < len(bed):
dist_m = np.append(
dist_m, dist_m[-1] * np.ones(len(bed) - len(dist_m)))
if doplot == 1:
if chunkmode != 4:
for k in range(len(star_fp)):
plot_2bedpicks(
port_fp[k], star_fp[k],
bed[ind_port[-1] * k:ind_port[-1] * (k + 1)],
dist_m[ind_port[-1] * k:ind_port[-1] *
(k + 1)],
x[ind_port[-1] * k:ind_port[-1] * (k + 1)], ft,
shape_port, sonpath, k, chunkmode)
else:
plot_2bedpicks(port_fp, star_fp, bed, dist_m, x, ft,
shape_port, sonpath, 0, chunkmode)
# 'real' bed is estimated to be the minimum of the two
bed = np.min(np.vstack((bed[:nrec], np.squeeze(x[:nrec]))),
axis=0)
bed = humutils.runningMeanFast(bed, 3)
elif bedpick > 1: # user prompt
x, bed = humutils.auto_bedpick(ft, dep_m, chunkmode, port_fp,
c)
if len(dist_m) < len(bed):
dist_m = np.append(
dist_m, dist_m[-1] * np.ones(len(bed) - len(dist_m)))
# 'real' bed is estimated to be the minimum of the two
bed = np.min(np.vstack((bed[:nrec], np.squeeze(x[:nrec]))),
axis=0)
bed = humutils.runningMeanFast(bed, 3)
# manually intervene
fig = plt.figure()
ax = plt.gca()
if chunkmode != 4:
im = ax.imshow(np.hstack(port_fp),
cmap='gray',
origin='upper')
else:
im = ax.imshow(port_fp, cmap='gray', origin='upper')
plt.plot(bed, 'r')
plt.axis('normal')
plt.axis('tight')
pts1 = plt.ginput(
n=300,
timeout=30) # it will wait for 200 clicks or 60 seconds
x1 = map(lambda x: x[0],
pts1) # map applies the function passed as
y1 = map(lambda x: x[1],
pts1) # first parameter to each element of pts
plt.close()
del fig
if x1 != []: # if x1 is not empty
tree = KDTree(zip(np.arange(1, len(bed)), bed))
try:
dist, inds = tree.query(zip(x1, y1),
k=100,
eps=5,
n_jobs=-1)
except:
dist, inds = tree.query(zip(x1, y1), k=100, eps=5)
b = np.interp(inds, x1, y1)
bed2 = bed.copy()
bed2[inds] = b
bed = bed2
if doplot == 1:
if chunkmode != 4:
for k in range(len(star_fp)):
plot_2bedpicks(
port_fp[k], star_fp[k],
bed[ind_port[-1] * k:ind_port[-1] * (k + 1)],
dist_m[ind_port[-1] * k:ind_port[-1] *
(k + 1)],
x[ind_port[-1] * k:ind_port[-1] * (k + 1)], ft,
shape_port, sonpath, k, chunkmode)
else:
plot_2bedpicks(port_fp, star_fp, bed, dist_m, x, ft,
shape_port, sonpath, 0, chunkmode)
else: #manual
beds = []
if chunkmode != 4:
for k in range(len(port_fp)):
raw_input(
"Bed picking " + str(k + 1) + " of " +
str(len(port_fp)) +
", are you ready? 30 seconds. Press Enter to continue..."
)
bed = {}
fig = plt.figure()
ax = plt.gca()
im = ax.imshow(port_fp[k], cmap='gray', origin='upper')
pts1 = plt.ginput(
n=300, timeout=30
) # it will wait for 200 clicks or 60 seconds
x1 = map(lambda x: x[0],
pts1) # map applies the function passed as
y1 = map(
lambda x: x[1],
pts1) # first parameter to each element of pts
bed = np.interp(np.r_[:ind_port[-1]], x1, y1)
plt.close()
del fig
beds.append(bed)
extent = np.shape(port_fp[k])[0]
bed = np.asarray(np.hstack(beds), 'float')
else:
raw_input(
"Bed picking - are you ready? 30 seconds. Press Enter to continue..."
)
bed = {}
fig = plt.figure()
ax = plt.gca()
im = ax.imshow(port_fp, cmap='gray', origin='upper')
pts1 = plt.ginput(
n=300, timeout=30
) # it will wait for 200 clicks or 60 seconds
x1 = map(lambda x: x[0],
pts1) # map applies the function passed as
y1 = map(lambda x: x[1],
pts1) # first parameter to each element of pts
bed = np.interp(np.r_[:ind_port[-1]], x1, y1)
plt.close()
del fig
beds.append(bed)
extent = np.shape(port_fp)[1]
bed = np.asarray(np.hstack(beds), 'float')
# now revise the depth in metres
dep_m = (1 / ft) * bed
if doplot == 1:
if chunkmode != 4:
for k in range(len(star_fp)):
plot_bedpick(
port_fp[k], star_fp[k], (1 / ft) *
bed[ind_port[-1] * k:ind_port[-1] * (k + 1)],
dist_m[ind_port[-1] * k:ind_port[-1] * (k + 1)],
ft, shape_port, sonpath, k, chunkmode)
else:
plot_bedpick(port_fp, star_fp, (1 / ft) * bed, dist_m, ft,
shape_port, sonpath, 0, chunkmode)
metadat['bed'] = bed[:nrec]
else:
metadat['bed'] = dep_m[:nrec] * ft
metadat['heading'] = metadat['heading'][:nrec]
metadat['lon'] = lon[:nrec]
metadat['lat'] = lat[:nrec]
metadat['dist_m'] = dist_m[:nrec]
metadat['dep_m'] = dep_m[:nrec]
metadat['pix_m'] = 1 / ft
metadat['bed'] = metadat['bed'][:nrec]
metadat['c'] = c
metadat['t'] = t
if model == 2:
metadat['f'] = f * 2
else:
metadat['f'] = f
metadat['spd'] = metadat['spd'][:nrec]
metadat['time_s'] = metadat['time_s'][:nrec]
metadat['e'] = metadat['e'][:nrec]
metadat['n'] = metadat['n'][:nrec]
metadat['es'] = metadat['es'][:nrec]
metadat['ns'] = metadat['ns'][:nrec]
try:
metadat['caltime'] = metadat['caltime'][:nrec]
except:
metadat['caltime'] = metadat['caltime']
savemat(os.path.normpath(os.path.join(sonpath, base + 'meta.mat')),
metadat,
oned_as='row')
f = open(os.path.normpath(os.path.join(sonpath, base + 'rawdat.csv')),
'wt')
writer = csv.writer(f)
writer.writerow(
('longitude', 'latitude', 'easting', 'northing', 'depth (m)',
'distance (m)', 'instr. heading (deg)', 'heading (deg.)'))
for i in range(0, nrec):
writer.writerow(
(float(lon[i]), float(lat[i]), float(es[i]), float(ns[i]),
float(dep_m[i]), float(dist_m[i]),
float(metadat['instr_heading'][i]), float(metadat['heading'][i])))
f.close()
del lat, lon, dep_m #, dist_m
if doplot == 1:
plot_pos(sonpath, metadat, es, ns)
if 'dwnlow_fp' in locals():
plot_dwnlow(dwnlow_fp, chunkmode, sonpath)
if 'dwnhi_fp' in locals():
plot_dwnhi(dwnhi_fp, chunkmode, sonpath)
if os.name == 'posix': # true if linux/mac
elapsed = (time.time() - start)
else: # windows
elapsed = (time.clock() - start)
print("Processing took " + str(elapsed) + "seconds to analyse")
print("Done!")
print("===================================================")
else path+'\\GoogleDistanceCalculator\\data_files\\'
filename = 'input_origins_and_destinations.csv'
final_path = filepath + filename
input_df = pd.read_csv(final_path)
output_df = pd.DataFrame(columns=['S.No.', 'origin', 'destination', 'distance', 'raw_response'])
url_endpoint = 'https://maps.googleapis.com/maps/api/distancematrix/json'
fields = ['S. No', 'origin', 'destination', 'distance', 'raw_response' ]
error_message = ''
distance = ''
with open(filepath + 'output_distances.csv','w', newline='') as newFile:
newFileWriter = csv.writer(newFile)
newFileWriter.writerow(['S.No.', 'origin', 'destination', 'distance', 'raw_response', 'Error' ])
for index, row in input_df.iterrows():
mydict = {'units' : measurement_unit,
'origins' : row['origin_address'],
'destinations': row['destination_address'],
'key' : google_key}
error_message = 'No Error'
try:
resp = requests.get(url_endpoint, params=mydict)
if(resp.ok):
jData = json.loads(resp.text)
distance = jData['rows'][0]['elements'][0]['distance']['text']
except Exception as e:
error_message = 'Error getting distance.'
if user in user_to_uninstall_times:
is_alive = 0
else:
is_alive = 1
#if time_since_install/ 8.64e+7 <= 7:
# continue
if users_to_conditions_in_experiment_by_name[user] == 'off':
not_suggested.append([time_since_install/8.64e+7, int(is_alive)])
intervention_suggestion.append([time_since_install/ 8.64e+7, int(is_alive), 0])
else:
suggested.append([time_since_install/8.64e+7, int(is_alive), 1])
intervention_suggestion.append([time_since_install/ 8.64e+7, int(is_alive), 1])
import csv
with open('time_to_is_alive_suggestion.csv', 'w', newline='') as csvfile:
spamwriter = csv.writer(csvfile, delimiter=',', quoting=csv.QUOTE_MINIMAL)
spamwriter.writerow(["days", "isAlive"])
for i in suggested:
spamwriter.writerow(i)
with open('time_to_is_alive_no_suggestion.csv', 'w', newline='') as csvfile:
spamwriter = csv.writer(csvfile, delimiter=',', quoting=csv.QUOTE_MINIMAL)
spamwriter.writerow(["days", "isAlive"])
for i in not_suggested:
spamwriter.writerow(i)
with open('intervention_suggestion.csv', 'w', newline='') as csvfile:
spamwriter = csv.writer(csvfile, delimiter=',', quoting=csv.QUOTE_MINIMAL)
spamwriter.writerow(["days", "isAlive", "isShownSuggesion"])
for i in intervention_suggestion:
spamwriter.writerow(i)
'''
def write_to_csv_file(pred_Y, predicted_Y_file_name):
pred_Y = pred_Y.reshape(len(pred_Y), 1)
with open(predicted_Y_file_name, 'w', newline='') as csv_file:
wr = csv.writer(csv_file)
wr.writerows(pred_Y)
csv_file.close()
def get_device_list(ip_address, user_name, password, identitypool_id, outfile):
""" Authenticate with OME and enumerate devices """
try:
base_uri = 'https://%s' % (ip_address)
next_link_url = None
headers = {'content-type': 'application/json'}
auth_token = get_session(ip_address, user_name, password)
if auth_token != None:
headers['X-Auth-Token'] = auth_token['token']
# Display Identity Pools
identitypool_url = base_uri + '/api/IdentityPoolService/IdentityPools'
identitypool_response = requests.get(identitypool_url,
headers=headers,
verify=False)
if identitypool_response.status_code == 200:
identitypool_data = identitypool_response.json()
identitypool_data = identitypool_data['value']
for i in identitypool_data:
print("Id: %s, Name: %s" % (i["Id"], i["Name"]))
else:
print("Unable to retrieve device list from %s" % (ip_address))
if identitypool_id == None:
identitypool_id_prompt = input(
"Please Enter Identity Pool Id: ")
identitypool_id = identitypool_id_prompt
# Get Identity Pool Usage Sets
identitypool_usageset_url = base_uri + "/api/IdentityPoolService/IdentityPools(%s)/UsageIdentitySets" % (
identitypool_id)
identitypool_usageset_response = requests.get(
identitypool_usageset_url, headers=headers, verify=False)
if identitypool_usageset_response.status_code == 200:
identitypool_usageset_data = identitypool_usageset_response.json(
)
identitypool_usageset_data = identitypool_usageset_data[
'value']
detailentries = []
for usageset in identitypool_usageset_data:
usageset_id = usageset["IdentitySetId"]
usageset_name = usageset["Name"]
identitypool_usageset_detail_url = base_uri + "/api/IdentityPoolService/IdentityPools(%s)/UsageIdentitySets(%s)/Details" % (
identitypool_id, usageset_id)
identitypool_usageset_detail_response = requests.get(
identitypool_usageset_detail_url,
headers=headers,
verify=False)
if identitypool_usageset_detail_response.status_code == 200:
identitypool_usageset_detail_json = identitypool_usageset_detail_response.json(
)
identitypool_usageset_detail_data = identitypool_usageset_detail_json[
'value']
for detailentry in identitypool_usageset_detail_data:
entry = {
"IdentityType":
usageset_name,
"ChassisName":
detailentry["DeviceInfo"]["ChassisName"],
"ServerName":
detailentry["DeviceInfo"]["ServerName"],
"ManagementIp":
detailentry["DeviceInfo"]["ManagementIp"],
"NicIdentifier":
detailentry["NicIdentifier"],
"MacAddress":
detailentry["MacAddress"]
}
detailentries.append(entry)
if '@odata.nextLink' in identitypool_usageset_detail_json:
next_link_url = base_uri + identitypool_usageset_detail_json[
'@odata.nextLink']
while next_link_url:
next_link_response = requests.get(
next_link_url,
headers=headers,
verify=False)
if next_link_response.status_code == 200:
next_link_json = next_link_response.json()
next_link_json_data += next_link_json[
'value']
for detailentry in next_link_json_data:
entry = {
"IdentityType":
usageset_name,
"ChassisName":
detailentry["DeviceInfo"]
["ChassisName"],
"ServerName":
detailentry["DeviceInfo"]
["ServerName"],
"ManagementIp":
detailentry["DeviceInfo"]
["ManagementIp"],
"NicIdentifier":
detailentry["NicIdentifier"],
"MacAddress":
detailentry["MacAddress"]
}
detailentries.append(entry)
if '@odata.nextLink' in next_link_json:
next_link_url = base_uri + next_link_json[
'@odata.nextLink']
else:
next_link_url = None
else:
next_link_url = None
print(
"Unable to retrieve items from nextLink %s"
% (next_link_url))
# Export results to CSV
if (len(detailentries) > 0):
currentDirectory = os.path.dirname(
os.path.abspath(__file__))
if outfile == None:
outFilePath = currentDirectory + os.path.sep + "IdentityPoolUsage.csv"
else:
outFilePath = outfile
outputFile = open(outFilePath, 'w')
output = csv.writer(outputFile)
output.writerow(detailentries[0].keys())
for row in detailentries:
output.writerow(row.values())
print("Exported data to %s" % (outFilePath))
outputFile.close()
else:
print("No data to display")
else:
print("Unable to retrieve list from %s" % (ip_address))
else:
print("Unable to create a session with appliance %s" %
(ip_address))
except:
print("Unexpected error:", sys.exc_info())
finally:
delete_session(ip_address, headers, auth_token['id'])
#!/usr/bin/env python3
import csv
import sys
input_file = sys.argv[1]
output_file = sys.argv[2]
with open(input_file, 'r', newline='') as csv_in_file:
with open(output_file, 'w', newline='') as csv_out_file:
filereader = csv.reader(csv_in_file)
filewriter = csv.writer(csv_out_file)
header = next(filereader)
filewriter.writerow(header)
for row_list in filereader:
supplier = str(row_list[0]).strip()
cost = str(row_list[3]).strip('$').replace(',', '')
if supplier == 'Supplier Z' or float(cost) > 600.0:
filewriter.writerow(row_list)
import os, csv
cand_solution = {}
for i in os.listdir('.'):
if i[0] in ['1','2','3','4','5','6','7','8','9']:
try:
cand_solution[i.replace('.txt','')] = open(i).read().replace(" ", "").replace("\n", "")
except:
pass
w = csv.writer(open("solution.csv", "w"))
for key, val in cand_solution.items():
w.writerow([key, val])
])
output = [x for x in output if x[3] != 0]
return output
with open(filename) as csvfile:
allcontent = list(csv.reader(csvfile))
header = allcontent[0]
noheader = allcontent[1:len(allcontent)]
for i in range(file_length // N):
run_events_list = [header] + noheader[N * (i + 1) - (N):N * (i + 1)]
stim_presentation_onset = makeStimPresentationOnset(run_events_list)
response_onset = makeResponseOnset(run_events_list)
run_onsets = stim_presentation_onset + response_onset
run_onsets.sort()
run_onsets_header = [
"onset", "duration", "trial_type", "response", "stimulus",
"response_time", "points_earned", "iti_start_time", "iti_length"
]
with open(
'/oak/stanford/groups/russpold/data/ds000054/0.0.4/sub-' +
sub_dir_name + '/func/sub-' + sub_id + '_task-' + task_name +
'_run-00' + str(i + 1) + '_events.tsv', 'w') as tsvfile:
writer = csv.writer(tsvfile,
quoting=csv.QUOTE_NONE,
delimiter='\t',
quotechar='')
writer.writerow(run_onsets_header)
writer.writerows(run_onsets)
print("Saved events for sub-%s run-%s" % (sub_id, str(i + 1)))
companyname = data[1].find('span', attrs={'class':'company-name'}).getText()
description = company.replace(companyname, '')
# remove unwanted characters
sales = sales.strip('*').strip('†').replace(',','')
# go to link and extract company website
url = data[1].find('a').get('href')
page = urllib.request.urlopen(url)
# parse the html using beautiful soup and store in variable 'soup'
soup = BeautifulSoup(page, 'html.parser')
# find the last result in the table and get the link
try:
tableRow = soup.find('table').find_all('tr')[-1]
webpage = tableRow.find('a').get('href')
except:
webpage = None
# write each result to rows
rows.append([rank, companyname, webpage, description, location, yearend, salesrise, sales, staff, comments])
print(rows)
## Create csv and write rows to output file
with open('techtrack100.csv','w', newline='') as f_output:
csv_output = csv.writer(f_output)
csv_output.writerows(rows)
a = list(map(int, i.split('-')))
exp = sum(a)/len(a)
print ('exp ',exp)
lst_exp.insert(0, exp)
count1 += 1
if 'to' in i:
a = list(map(int, i.split('to')))
sal = sum(a)/len(a)
print ('sal ',sal)
lst_sal.insert(0, sal)
count2 += 1
# zip(lst_exp,lst_sal)
with open('text.csv', 'w') as f:
writer = csv.writer(f, delimiter='\t')
writer.writerows(zip(lst_exp,lst_sal))
# print (lst_exp)
# print (lst_sal)
# count2 += 1
# print (count1)
# print (count2)
