def write_to_table_via_csv(table, rows, connection):
    with temp_csv(mode='r+', tmp_dir=TEMPORARY_DIR) as csv_file:
        csv.writer(csv_file, delimiter='\t').writerows(rows)
        csv_file.seek(0, 0)
        connection.connection.cursor().copy_from(csv_file, sep='\t', null='',
                                                 table=table.name)
        connection.connection.commit()
def main():
    PROG = os.path.basename(os.path.splitext(__file__)[0])
    description = """Scan claims files"""
    parser = OptionParser(option_class=MultipleOption,
                          usage='usage: %prog claims_file, claims_file, ...',
                          version='%s %s' % (PROG, VERSION),
                          description=description)
    if len(sys.argv) == 1:
        parser.parse_args(['--help'])

    args = parser.parse_args()
    p2k = {}
    k2p = {}
    try:
        with open('claimants.csv') as csv_file:
            for line in csv.reader(csv_file, dialect="excel"):
                p2k[line[0]] = line[1]
                k2p[line[1]] = line[0]
    except IOError:
        pass
    for filename in args[1]:
        with open(filename+'_masked.csv', 'wb') as cf:
            outfile = csv.writer(cf, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
            analyze_file(filename, outfile, p2k, k2p)
            print len(p2k), len(k2p)
    with open('claimants.csv', 'wb') as cf:
        cout = csv.writer(cf, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
        for p in p2k:
            cout.writerow([p, p2k[p]])
示例#3
0
def compute_output_csvs():
    payees = [Payee(rec) for rec in csv.reader(open(INPUT_CSV))]
    payees.sort(key=lambda o: o.gross, reverse=True)

    total_fees = sum([payee.assess_fee() for payee in payees])  # side-effective!
    total_net = sum([p.net for p in payees])
    total_gross = sum([p.gross for p in payees])
    assert total_fees + total_net == total_gross

    paypal_csv = csv.writer(open(PAYPAL_CSV, 'w+'))
    gittip_csv = csv.writer(open(GITTIP_CSV, 'w+'))
    print_rule()
    print("{:<24}{:<32} {:^7} {:^7} {:^7}".format("username", "email", "gross", "fee", "net"))
    print_rule()
    for payee in payees:
        paypal_csv.writerow((payee.email, payee.net, "usd"))
        gittip_csv.writerow(( payee.username
                            , payee.email
                            , payee.gross
                            , payee.fee
                            , payee.net
                            , payee.additional_note
                             ))
        print("{username:<24}{email:<32} {gross:>7} {fee:>7} {net:>7}".format(**payee.__dict__))

    print(" "*56, "-"*23)
    print("{:>64} {:>7} {:>7}".format(total_gross, total_fees, total_net))
示例#4
0
	def __init__(self, ni, nh, no):
		'''the init function to set up the NN topology.'''
		# nodes.
		self._ni = ni + 1
		self._nh = nh
		self._no = no
		
		# activiations.
		self._ai = [1.0] * self._ni
		self._ah = [1.0] * self._nh
		self._ao = [1.0] * self._no

		# weights.
		self._wi = self._create_matrix(self._ni, self._nh)
		self._wo = self._create_matrix(self._nh, self._no)
		self._wi = self._matrix_random_init(self._wi, -0.01, 0.01)
		self._wo = self._matrix_random_init(self._wo, -0.01, 0.01)

		# momentums.
		self._ci = self._create_matrix(self._ni, self._nh)
		self._co = self._create_matrix(self._nh, self._no)

		# error log.
		self._tr_logger = csv.writer(open('dump/tr_error_log.csv', 'wb'), delimiter=',')
		self._te_logger = csv.writer(open('dump/te_error_log.csv', 'wb'), delimiter=',')
示例#5
0
def hp_interest(tenure, interest, amount, l_no):                                  ### Calculate HP interest given the param
    term_charge = float(amount * float(tenure/12)) * float(interest)/100 #Getting term charge
    ###print term_charge
    int_constant = float(((tenure)*(tenure+1))/2) #Calculating interest constant for use in formula 
    ###print int_constant                           *skipping full formula for simplicity
    instalment = tenure
    ###print instalment
    interest_tab = [str(l_no)]
    count = 0
    count_tab = ['Loan Number']
    while instalment > 0:
        interest = float(((instalment) / (int_constant)) * (term_charge))
        interest_tab.append(interest)
        count += 1
        instalment -= 1
        count_tab.append(count)

    hp_header = count_tab
    with open('/Users/Michael/Documents/GitHub/Int-Projection/HP_Int.csv', 'w') as h:
        writer = csv.writer(h)
        writer.writerow(hp_header)

    with open('/Users/Michael/Documents/GitHub/Int-Projection/HP_Int.csv', 'a') as ins:
        writer = csv.writer(ins)
        writer.writerow(interest_tab)
    
    

    ###for i in interest_tab:
    ###    print i
    ###for i in count_tab:
    ###    print i
    return interest_tab
 def save_results(self):
     """
     Saves the bit_error_probability and corresponding variance in a csv file
     Saves the codewords, their transmissions, and their resulting decoding in files separted by channel noise variance
     :return:
     """
     self.save_time = epoch_time = str(int(time.time()))
     if not os.path.exists("./stats/advanced-run"):
         os.makedirs("./stats/advanced-run")
     m = "sum-prod" if self.mode == Decoder.SUM_PROD else "max-prod"
     with open("stats/advanced-run/%(time)s-%(mode)s-%(num_codewords)s-codewords-variance-bit_error_probability.csv" % {
     "time": epoch_time,
     "mode": m,
     "num_codewords": self.iterations}, 'wb') as result_csv:
         writer = csv.writer(result_csv)
         writer.writerow(["variance", "bit_error_probability"])
         for v, error in zip(self.variance_levels, self.bit_error_probability):
             writer.writerow([v, error])
     for i, v in enumerate(self.variance_levels):
         with open("stats/advanced-run/%(time)s-%(mode)s-%(num_codewords)s-codewords-variance-%(var)s-codewords-decoded.csv" % {
         "time": epoch_time,
         "mode": m,
         "num_codewords": str(self.iterations),
         "var": str(v)}, 'wb') as result_csv:
             writer = csv.writer(result_csv)
             writer.writerow(["codeword", "decoded", "transmission"])
             for codeword, transmission, decoded in zip(self.codewords[i], self.transmissions[i], self.decoded[i]):
                 writer.writerow([''.join(str(elem) for elem in codeword),
                                  ''.join(str(elem) for elem in decoded),
                                  ' '.join(str(elem) for elem in transmission)])
示例#7
0
 def payin(self):
     """The first stage of payday where we charge credit cards and transfer
     money internally between participants.
     """
     with self.db.get_cursor() as cursor:
         self.prepare(cursor, self.ts_start)
         holds = self.create_card_holds(cursor)
         self.transfer_tips(cursor)
         self.transfer_takes(cursor, self.ts_start)
         transfers = cursor.all("""
             SELECT * FROM transfers WHERE "timestamp" > %s
         """, (self.ts_start,))
         try:
             self.settle_card_holds(cursor, holds)
             self.update_balances(cursor)
             check_db(cursor)
         except:
             # Dump transfers for debugging
             import csv
             from time import time
             with open('%s_transfers.csv' % time(), 'wb') as f:
                 csv.writer(f).writerows(transfers)
             raise
     self.take_over_balances()
     # Clean up leftover functions
     self.db.run("""
         DROP FUNCTION process_take();
         DROP FUNCTION process_tip();
         DROP FUNCTION settle_tip_graph();
         DROP FUNCTION transfer(text, text, numeric, context_type);
     """)
def SSIExcelToCSV(SSIworkbook,tempfile,CorrectedFile):
  """ constructed to consume a specific SSI document in which there are an unknown # of worksheets """
  with open(tempfile,'wb') as writetest:
    writer=csv.writer(writetest)
    workbook = xlrd.open_workbook(SSIworkbook)
    worksheets = workbook.sheet_names()
    for worksheet_name in worksheets:
      worksheet = workbook.sheet_by_name(worksheet_name)
      num_rows = worksheet.nrows - 1
      curr_row = -1
      while curr_row < num_rows:
          curr_row += 1
          row = worksheet.row(curr_row)
          writer.writerow(row[1:3])

  with open(CorrectedFile,'wb') as writetest:
    writer=csv.writer(writetest)
    with open(tempfile,'rb') as csvfile:
      reader=csv.reader(csvfile)
      for row in reader:
  #       pprint(row)
        if row[0].startswith('text'):
          x=[str(item).replace('text:u','') for item in row]
          y=[s.replace("'","") for s in x]
          writer.writerow(y)
  print 'your file is waiting here '+CorrectedFile
示例#9
0
def magic_loop(x, y):
    

    cls_d = {}

    with open('models.csv', 'w', newline='') as csvfile:
        w = csv.writer(csvfile, delimiter=',')
        w.writerow(['MODEL', 'PARAMETERS', 'PRECISION', 'RECALL', 'AUC', 'F1', 'ACCURACY', 'Time'])
        with open('best_models.csv', 'w', newline='') as csvfile:
            c = csv.writer(csvfile, delimiter=',')
            c.writerow(['MODEL', 'PARAMETERS', 'PRECISION', 'RECALL', 'AUC', 'F1', 'ACCURACY', 'Time'])
            x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2, random_state=0)

            class_auc = {}
            best_model = ''
            best_AUC = 0
            best_param = ''
            best_ypred = ''
            best_precision = 0
            best_recall = 0
            best_f1 = 0
            best_accuracy = 0
            best_time = 0

            for index,clf in enumerate([clfs[x] for x in MODELS]):
        
                cls_ypred = ''
                current_model = MODELS[index]
                class_auc[current_model] = 0
                cls_ypred = ''
                print (current_model)
                parameter_values = grid[current_model]
                start_time = t.time()
                for p in ParameterGrid(parameter_values):
                    try:

                        start_time = t.time()
                        clf.set_params(**p)
                        y_pred_probs = clf.fit(x_train, y_train).predict_proba(x_test)[:,1]
                        precision, accuracy, recall, f1, threshold, AUC = model_evaluation(y_test, y_pred_probs,.05)
                        end_time = t.time()
                        total_time = end_time - start_time
                        w.writerow([current_model, p, precision, recall, AUC, f1, accuracy, total_time])
               
                        if AUC > class_auc[current_model]:
                            class_auc[current_model] = AUC
                            cls_ypred = y_pred_probs
                            cls_param = p
                            cls_precision = precision
                            cls_recall = recall
                            cls_f1 = f1
                            cls_accuracy = accuracy
                            cls_time = total_time
                   
                    except IndexError as e:
                        continue

                auc = class_auc[current_model]
                c.writerow([current_model, cls_param, cls_precision, cls_recall, auc, cls_f1, cls_accuracy, cls_time])
                cls_d[current_model] = [auc, cls_ypred]
def generateOutputCSVs(inputDict, outputDict, history=0):
    for key in inputDict:
        inputRows = inputDict[key];
        outputRows = outputDict[key];

        history = [];

        i = 0;
        for inputRow in inputRows:
            generateOutputCSVsByDOW[int(inputRow[2])].append(inputRow);
            if (i < len(outputRows)):
                dayByDayOutput[int(inputRow[2])].append(outputRows[i]);
            i += 1;

        for day in generateOutputCSVsByDOW:
            with open("3192_"
                      + key
                      + "_"
                      + str(day)
                      + "_input.csv", 'w', newline='') as csvFile:
                writer = csv.writer(csvFile);
                for inputRow in generateOutputCSVsByDOW[day]:
                    writer.writerow(inputRow);
            with open("3192_"
                      + key
                      + "_"
                      + str(day)
                      + "_output.csv", 'w', newline='') as csvFile:
                writer = csv.writer(csvFile);
                for outputRow in dayByDayOutput[day]:
                    writer.writerow(outputRow);
示例#11
0
def generateTrials(myDict, nTestTrials):
    '''
    generates a list of trials to write to csv file
    supply 3 columns to be factorially combined
    myDict = {'contrast': [1.0, 0.75, 0.5, 0.25], 'orientation':[225, 315],
        'probeMatch':[1, 0], 'correctKey':[1]}
    '''
    columnHeaders = myDict.keys()
    trialList = data.createFactorialTrialList(myDict)
    for item in trialList:
        if item['probeMatch'] == 1:
            item['correctKey'] = expInfo['sameKey']
        else:
            item['correctKey'] =expInfo['differentKey']

    # write trial list for experiment loop:       
    if expInfo['experimentType']=='practise':
        trialList = rand.sample(trialList, nTestTrials)
        
    with open('mainTrials.csv', 'wb') as mainTrialsFile:
        writer = csv.writer(mainTrialsFile, dialect='excel')
        writer.writerow(columnHeaders)
        for row in trialList:
            writer.writerow(row.values())

    if expInfo['experimentType']=='practise':
        # generate trial list for practise loop:
        practiseTrialList = rand.sample(trialList, nTestTrials)
        # write trial list for practise loop:       
        with open('practiseTrials.csv', 'wb') as practiseTrialsFile:
            writer = csv.writer(practiseTrialsFile, dialect='excel')
            writer.writerow(columnHeaders)
            for row in practiseTrialList:
                writer.writerow(row.values())
示例#12
0
def main_web(args):
    assert os.path.exists(args.input)
    with open(args.input) as f:
        contents = f.read().strip()
    if args.hg19 is True and args.protein is True:
        stop_err('--hg19 option conflicts with --protein')
    if args.protein is False:
        ## Replace tabs/space with commas
        re.sub('[\t\s]+', ',', contents)
    if args.hg19:
        ## Append hg19 to each line
        lines = contents.split('\n')
        contents = ('\n').join(
            map((lambda x: 'hg19,' + x),
                lines))

    payload = {'vars': contents, 'tableQ': 1}
    request = requests.post(__url__, data=payload)
    response = request.text
    if request.status_code != requests.codes.ok:
        stop_err("""Error retrieving response from server.
                 Server returned %s .
                 Output: %s
                 """ % (request.status_code, response))
    r = StringIO.StringIO(response)
    reader = csv.reader(r, delimiter=",")
    csv.writer(open(args.output, "wb"), delimiter='\t').writerows(reader)
def generateOutputCSVsByDOW(inputDict, outputDict):
    for key in inputDict:
        inputRows = inputDict[key];
        outputRows = outputDict[key];
        dayByDayInput = {0:[], 1:[], 2:[], 3:[], 4:[], 5:[], 6:[]};
        dayByDayOutput = {0:[], 1:[], 2:[], 3:[], 4:[], 5:[], 6:[]};

        i = 0;
        for inputRow in inputRows:
            dayByDayInput[int(inputRow[2])].append(inputRow);
            if (i < len(outputRows)):
                dayByDayOutput[int(inputRow[2])].append(outputRows[i]);
            i += 1;

        for day in dayByDayInput:
            with open("3192_"
                      + key
                      + "_"
                      + str(day)
                      + "_input.csv", 'w', newline='') as csvFile:
                writer = csv.writer(csvFile);
                for inputRow in dayByDayInput[day]:
                    writer.writerow(inputRow);
            with open("3192_"
                      + key
                      + "_"
                      + str(day)
                      + "_output.csv", 'w', newline='') as csvFile:
                writer = csv.writer(csvFile);
                for outputRow in dayByDayOutput[day]:
                    writer.writerow(outputRow);
示例#14
0
def main():
    """
    """
    f = open("../data/adult/adult.data")
    reader = csv.reader(f)

    t1 = open("../data/adult/train.csv","w")
    t2 = open("../data/adult/test.csv","w")

    w1 = csv.writer(t1)
    w2 = csv.writer(t2)

    for line in reader:
        if len(line) < 1:
            continue
        label = line[-1].strip()
        if label == "<=50K":
            label = '0'
        else:
            label = '1'
            
        line = line[:-1]
        ls = csv_format(line,label)
        if keep_train(2,1):
            w1.writerow(ls)
        else:
            w2.writerow(ls)
示例#15
0
def _evaluate_multi(calls, truth_svtypes, work_dir, data):
    base = os.path.join(work_dir, "%s-sv-validate" % (dd.get_sample_name(data)))
    out_file = base + ".csv"
    df_file = base + "-df.csv"
    if any((not utils.file_uptodate(out_file, x["vrn_file"])
            or not utils.file_uptodate(df_file, x["vrn_file"])) for x in calls):
        with file_transaction(data, out_file) as tx_out_file:
            with open(tx_out_file, "w") as out_handle:
                with open(df_file, "w") as df_out_handle:
                    writer = csv.writer(out_handle)
                    dfwriter = csv.writer(df_out_handle)
                    writer.writerow(["svtype", "size", "caller", "sensitivity", "precision"])
                    dfwriter.writerow(["svtype", "size", "caller", "metric", "value", "label"])
                    for svtype, truth in truth_svtypes.items():
                        for size in EVENT_SIZES:
                            str_size = "%s-%s" % size
                            for call in calls:
                                call_bed = convert.to_bed(call, dd.get_sample_name(data), work_dir, calls, data)
                                if utils.file_exists(call_bed):
                                    evalout = _evaluate_one(call["variantcaller"], svtype, size, call_bed,
                                                            truth, data)
                                    writer.writerow([svtype, str_size, call["variantcaller"],
                                                     evalout["sensitivity"]["label"], evalout["precision"]["label"]])
                                    for metric in ["sensitivity", "precision"]:
                                        dfwriter.writerow([svtype, str_size, call["variantcaller"], metric,
                                                           evalout[metric]["val"], evalout[metric]["label"]])
    return out_file, df_file
def partition_essays():
    '''
    partition the training essay sets into different csv files
    '''
    with open('../data/training_set.csv', 'rb') as f:
        spamreader = csv.reader(f, delimiter=',')
        index = 1
        out_file = open('../data/training_set_' + str(index) + '.csv','w')
        writer = csv.writer(out_file)
        is_first=True
        for row in spamreader:
            # if (row[0]=="220"):
            #     print repr(row[2].decode('utf-8'))
            #     print row[2].decode('utf-8')
            # row[2]=row[2].encode('utf8')
            if(is_first):
                is_first= not is_first
                continue
            if row[1] == str(index):
                writer.writerow(row)
            else:
                out_file.close()
                index += 1
                out_file = open('../data/training_set_' + str(index) + '.csv','w')
                writer = csv.writer(out_file)
                writer.writerows([row])
def sonar_callback(sonar): ## Problem with logging raw data

    try:
        stringtime = time.time()-time_zero
        
        raw = list(numpy.fromstring(sonar.rawData, dtype = numpy.uint8))
                
        sonarData = [stringtime, sonar.transBearing, sonar.pitch, sonar.TargetRange, sonar.meanIntinsity ,raw]
        
        with open('%s/sonarLogwithRaw.csv' %(dirname), "a") as f:
            try:
                Writer = csv.writer(f, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
                Writer.writerow(sonarData)
            except ValueError:
                print 'writerow error'
        
        sonarData = [stringtime, sonar.transBearing, sonar.pitch, sonar.TargetRange, sonar.meanIntinsity]
        with open('%s/sonarLog.csv' %(dirname), "a") as f:
            try:
                Writer = csv.writer(f, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
                Writer.writerow(sonarData)
            except ValueError:
                print 'writerow error'
                
    except ValueError:
        print 'Problem with sonar logger!!!'
示例#18
0
def getextents():
    print "1) Output only collection level extents to a csv"
    print "2) Output only component level extents to a csv"
    choice = raw_input("Enter a number: ")
    path = 'Real_Masters_all'
    if choice == "1":
        outfile = raw_input("Enter a filename for the csv: ")
        for filename in os.listdir(path):
            tree = etree.parse(join(path, filename))
            e = tree.xpath('//ead/archdesc/did//physdesc/extent')
            for e in e:
                extent = e.text or "EMPTY EXTENT"
                extentpath = tree.getpath(e)
                with open(outfile + '.csv', 'ab') as csvfile:
                    writer = csv.writer(csvfile, dialect='excel')
                    writer.writerow([filename, extentpath, extent])
                csvfile.close()
            print filename
        print outfile + '.csv complete'
    elif choice == "2":
        outfile = raw_input("Enter a filename for the csv: ")
        for filename in os.listdir(path):
            tree = etree.parse(join(path, filename))
            e = tree.xpath('//dsc//did//extent')
            for e in e:
                extent = e.text or "EMPTY EXTENT"
                extentpath = tree.getpath(e)
                with open(outfile + '.csv', 'ab') as csvfile:
                    writer = csv.writer(csvfile, dialect='excel')
                    writer.writerow([filename, extentpath, extent])
                csvfile.close()
            print filename
        print outfile + '.csv complete'
示例#19
0
def getdates():
    print "1) Output all unitdates to a csv"
    print "2) Output all unitdates to a csv that do not have a normal attribute or are not 'undated'"
    choice = raw_input("Enter a number: ")
    path = 'Real_Masters_all'
    if choice == "1":
        outfile = raw_input("Enter a filename for the csv: ")
        for filename in os.listdir(path):
            tree = etree.parse(join(path, filename))
            d = tree.xpath('//unitdate')
            for i in d:
                with open(outfile + '.csv', 'ab') as csvfile:
                    writer = csv.writer(csvfile, dialect='excel')
                    writer.writerow([filename, tree.getpath(i), i.text])
            print filename
        print outfile + '.csv complete'
    elif choice == "2":
        outfile = raw_input("Enter a filename for the csv: ")
        for filename in os.listdir(path):
            tree = etree.parse(join(path, filename))
            d = tree.xpath('//unitdate')
            for i in d:
                # yyyy = re.compile('^[\d]{4}s?$')
                # yyyy_yyyy = re.compile('^[\d]{4}s?[-][\d]{4}s?$')
                undated = re.compile('^[Uu]ndated$')
                if not undated.match(i.text) and not 'normal' in i.attrib:
                    with open(outfile + '.csv', 'ab') as csvfile:
                        writer = csv.writer(csvfile, dialect='excel')
                        writer.writerow([filename, tree.getpath(i), i.text])
            print filename
        print outfile + '.csv complete'
示例#20
0
def save_candlefile(candles, period, filename, replace=True):
    """ Save candle data to local file. If replace=True, an existing 
    file will be rewritten, else new candles will be appended. """
    
    p_value, p_unit = parse_period(period)

    if '_{}{}'.format(p_value, p_unit) not in filename:    
        filename += '_{}{}'.format(p_value, p_unit)
    
    if CANDLES not in filename:
        filename = os.path.join(CANDLES, filename)
        
    try:
        with open(filename, 'a') as writefile: pass
    except:
        print 'Checking for directory at ', CANDLES
        raw_input('Warning: Candle data directory not found. Create now?')
        build_data_directories()        
        
    if replace:
        with open(filename, 'wb') as writefile:
            writer = csv.writer(writefile)
            for row in candles:
                writer.writerow(row)
    else:
        with open(filename, 'a') as appendfile:
            writer = csv.writer(appendfile)
            for row in candles:            
                writer.writerow(row)
示例#21
0
def main():
    parser = build_cli_parser()
    args = parser.parse_args()
    if not args.outfile:
	    cwriter = csv.writer(sys.stdout)
    else:
        outfile = open(args.outfile, 'w')
        cwriter = csv.writer(outfile)
    if args.interval:
        lastupdate_time = (datetime.datetime.now() - datetime.timedelta(days=args.interval)).strftime("%Y-%m-%d")
        args.query = 'alliance_score_nvd:* last_update:[%s TO *]' % lastupdate_time
    else:
        args.query = 'alliance_score_nvd:*'
    #request the process' id and segment id
    args.fields = ['id','segment_id']
    ##########
    #######Need to write logic to handle large numbers of results to the processes query
    ##########
    args.rows = 30
    if not args.url or not args.token:
        print "Missing required param; run with --help for usage"
        sys.exit(-1)
    #run the query using the details we built above
    processes = run_query(args)
    
    cwriter.writerow(['process_md5', 'process_path', 'hostname'])
    # for each process in the results dict
    for process in processes['results']:
        #get the process events so we can pull the NVD feed hits for the process
        events = get_events(args, process['id'], process['segment_id'])
        monkey = [events['process']['process_md5'], events['process']['path'], events['process']['hostname']]
        cwriter.writerow(monkey)
 def report_view_csv(self, id):
     report, results, total, show_org = self._report_view(id, None, None)
     file_path = '%s/%s.csv' % (h.get_export_dir(), re.sub('[^a-zA-Z0-9_-]+', '_', report['name'].encode('ascii','ignore')))
     
     if report['report_on'] == "activities": 
         with open(file_path, 'wb') as csvfile:
             writer = csv.writer(csvfile, lineterminator = '\n')
             record = ['Time', 'User', 'Activity Type', 'Data']
             writer.writerow(record)
             for row in results:
                 activity_data = {}
                 data_name = ''
                 if 'activity' in row and 'data' in row['activity']:
                     activity = row['activity']['data']
                     if 'package' in activity:
                         activity_data = activity.get('package')
                     elif 'dataset' in activity:
                         activity_data = activity.get('dataset')
                     elif 'group' in activity:
                         activity_data = activity.get('group')
                     data_name = activity_data.get('name', '')
                 record = [row['activity']['timestamp'], row['user']['fullname'] or row['user']['name'], row['activity']['activity_type'], data_name]
                 writer.writerow(record)    
     elif report['report_on'] == "details":
         with open(file_path, 'wb') as csvfile:
             writer = csv.writer(csvfile, lineterminator = '\n')
             record = ['name', 'email', 'state', 'organization', 'role', 'system administrator']
             writer.writerow(record)
             for row in results:
                 record = [row['fullname'] or row['name'], row['email'], row['state'], row['organization'], row['capacity'], row['sysadmin']]
                 writer.writerow(record)
         
     response = h.convertHtmlToCsv(file_path, tk.response)
     return response
示例#23
0
    def save(self, name_of_file, description):

        """ This function will save the beam object to an external file in the directory called "defined_beams" in the source directory"""

        dir = os.path.dirname(__file__)
        current_module = __name__
        path = current_module.split(".")
        rel_dir_name = "defined_" + path[1] + "s"
        filename = os.path.join(dir, rel_dir_name, name_of_file)
        if os.path.isfile(filename) == True:
            in_keyboard = input(
                "Filename specified" + name_of_file + "already exists, do you want to overwrite, yes [y] or no [n]?"
            )
            if in_keyboard == "n":
                name_of_file = input("Enter new file name")
            filename = os.path.join(dir, rel_dir_name, name_of_file)
            if os.path.isfile(filename) == True:
                print("Gave the same file name again, overwriting the file!!!!!")
                writer = csv.writer(open(filename, "w"))
                writer.writerow(["Description", description])
                for key, value in self.parameters.items():
                    writer.writerow([key, value])
                print(" Successfully written at" + filename)
        else:
            writer = csv.writer(open(filename, "w"))
            writer.writerow(["Description", description])
            for key, value in self.parameters.items():
                writer.writerow([key, value])
            print(" Successfully written at" + filename)
示例#24
0
def make_complete():
        my_todo = open(csv_input, "r")
        lines = list(csv.reader(my_todo, delimiter = "|"))
        rm = int(input('Which one is complete? '))
        temp_list = list(lines)

        print() #empty line
        print('Are you sure? Y/N:')
        print(temp_list[rm-1])
        yes = input('')

        if yes == 'y':
            # add item to the comlete list csv
            with open(csv_complete, "a") as csvfile:
                lines = csv.writer(csvfile, delimiter = "|")
                lines.writerow(temp_list[rm-1])

            temp_list.remove(temp_list[rm-1])
            with open(csv_input, "w") as csvfile:
                lines = csv.writer(csvfile, delimiter = "|")

                for line in temp_list:
                    lines.writerow(line)

                print("\033c") #screen clr
                print('Todo is completed')

        elif yes == 'n':
            print('Abort')
        else:
            print('')
示例#25
0
def write_log(task, data, in_rows, question, out_rows, out_cols, solution, version, git, fun, run, time_sec, mem_gb, cache, chk, chk_time_sec):
   batch = os.getenv('BATCH', "") 
   timestamp = time.time()
   csv_file = os.getenv('CSV_TIME_FILE', "time.csv")
   nodename = platform.node()
   comment = "" # placeholder for updates to timing data
   time_sec = round(time_sec, 3)
   chk_time_sec = round(chk_time_sec, 3)
   mem_gb = round(mem_gb, 3)
   if math.isnan(time_sec):
      time_sec = ""
   if math.isnan(mem_gb):
      mem_gb = ""
   log_row = [nodename, batch, timestamp, task, data, in_rows, question, out_rows, out_cols, solution, version, git, fun, run, time_sec, mem_gb, cache, chk, chk_time_sec, comment]
   log_header = ["nodename","batch","timestamp","task","data","in_rows","question","out_rows","out_cols","solution","version","git","fun","run","time_sec","mem_gb","cache","chk","chk_time_sec","comment"]
   append = os.path.isfile(csv_file)
   csv_verbose = os.getenv('CSV_VERBOSE', "true")
   if csv_verbose.lower()=="true":
      print('# ' + ','.join(str(x) for x in log_row))
   if append:
      with open(csv_file, 'a') as f:
         w = csv.writer(f, lineterminator='\n')
         w.writerow(log_row)
   else:
      with open(csv_file, 'w+') as f:
         w = csv.writer(f, lineterminator='\n')
         w.writerow(log_header)
         w.writerow(log_row)
   return True
def create_evaluation_distinctiveness(config, Kind):
    model_fname = config.model_fname % Kind.__name__

    try:
        model = LdaModel.load(model_fname)
        logger.info('Opened previously created model at file %s' % model_fname)
    except:
        error('Cannot evalutate LDA models not built yet!')

    scores = utils.score(model, utils.kullback_leibler_divergence)
    total = sum([x[1] for x in scores])

    logger.info("%s model KL: %f" % (model_fname, total))
    with open(config.path + 'evaluate-results.csv', 'a') as f:
        w = csv.writer(f)
        w.writerow([model_fname, total])

    etas = list()
    for topic in model.state.get_lambda():
        topic_eta = list()
        for p_w in topic:
            topic_eta.append(p_w * numpy.log2(p_w))
            etas.append(-sum(topic_eta))

    entropy = sum(etas) / len(etas)

    logger.info("%s model entropy mean: %f" % (model_fname, entropy))
    with open(config.path + 'evaluate-entropy-results.csv', 'a') as f:
        w = csv.writer(f)
        w.writerow([model_fname, entropy])
def main():
    f = open('train_p.csv', 'rb')
    reader = csv.reader(f,MyDialect())
    
    rows = []
    for row in reader:
        rows.append(row)
    num_lines = len(rows)
    num_train = num_lines * 6 / 10


    random.shuffle(rows)
    train_set = rows[:num_train]
    test_set = rows[num_train:]

    print len(train_set), len(test_set)

    train_file = open('mytrain2.csv', 'w')
    writer = csv.writer(train_file, MyDialect())
    for row in train_set:
        writer.writerow(row)

    test_file = open('mytest2.csv', 'w')
    writer = csv.writer(test_file, MyDialect())

    test_ans = open('mytest2_label.csv', 'w')
    for row in test_set:
        writer.writerow(row[:1]+row[2:])
        test_ans.write(row[1] + "\n")
示例#28
0
def main():
  venues_file_name = 'venues%s.csv' % datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
  venue_team_file_name = 'venue_team%s.csv' % datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
  with open(file_name, 'wb') as f, open(venue_team_file_name) as f2:
    writer = csv.writer(f)
    writer.writerow(['venue_id', 'name', 'country', 'address', 'zipcode',
                     'city', 'fax', 'email', 'website', 'phone', 'openend',
                     'architect', 'capacity', 'surface', 'facts'])

    writer2 = csv.writer(f2)
    writer2.writerow(['venue_id','team_id'])
    

    for x in range(0,20000):
      print x
      if requests.head(BASE_URL % x).status_code == requests.codes.ok:
        r = requests.get(BASE_URL % x)
        soup = BS(r.text, 'html.parser')
        venue_data = []
        venue_data.append(x) #venue_id
        name_node = soup.select('#subheading > h1')
        venue_data.append(name_node[0].text.encode('utf-8') if name_node else '')
        venue_data.append(get_detail('Founded', soup))
        venue_data.append(get_detail('Address', soup))
        venue_data.append(get_detail('Country', soup))
        venue_data.append(get_detail('Phone', soup))
        venue_data.append(get_detail('Fax', soup))
        venue_data.append(get_detail('E-mail', soup))
        website_node = soup.find('a', text='Official website')
        venue_data.append(website_node['href'].encode('utf-8') if website_node else '')
        venue_data.append(soup.find('div', 'logo').img['src'].encode('utf-8'))
        venue_data.append(get_venue_id(x))
        print name_node[0].text
        writer.writerow(venue_data)
def process_files(yymm):
    print 'handle the file; %s' % yymm
    #
    ap_target_file = Y09_ap_trips if yymm.startswith('09') else Y10_ap_trips
    with open('%s/%s%s.csv' % (ap_ep_dir, ap_ep_prefix, yymm), 'rb') as r_csvfile:
        reader = csv.reader(r_csvfile)
        headers = reader.next()
        if not check_path_exist(ap_target_file):
            with open(ap_target_file, 'wt') as csvFile:
                writer = csv.writer(csvFile)
                writer.writerow(headers)
        with open(ap_target_file, 'a') as csvFile:
            writer = csv.writer(csvFile)
            for row in reader:
                writer.writerow(row)
    #
    ns_target_file = Y09_ns_trips if yymm.startswith('09') else Y10_ns_trips
    with open('%s/%s%s.csv' % (ns_ep_dir, ns_ep_prefix, yymm), 'rb') as r_csvfile:
        reader = csv.reader(r_csvfile)
        headers = reader.next()
        if not check_path_exist(ns_target_file):
            with open(ns_target_file, 'wt') as csvFile:
                writer = csv.writer(csvFile)
                writer.writerow(headers)
        with open(ns_target_file, 'a') as csvFile:
            writer = csv.writer(csvFile)
            for row in reader:
                writer.writerow(row)
    #
    print 'end the file; %s' % yymm
def main():
    input_file = "books.csv"
    output_books = "books_new.csv"
    output_authors = "authors.csv"
    output_book_author_map = "book_authors.csv"
    count = 0
    authors = {}
    author_count = 0
    parse_firs_line = True
    output_books_stream = csv.writer(open(output_books, 'wb'))
    output_authors_stream = csv.writer(open(output_authors, 'wb'))
    output_book_author_map_stream = csv.writer(open(output_book_author_map, 'wb'))
    with open(input_file) as input_stream:
        for line in input_stream:
            line=line[:-1]
            count += 1
            input_str = line.split("\t")
            book_line = []
            author_line = []
            book_author_map = []
            if parse_firs_line:
                book_line = ['isbn', 'title', 'isbn13', 'cover', 'publisher', 'pages']
                author_line = ['author_id', 'fullname', 'title', 'fname', 'mname', 'lname', 'suffix']
                book_author_map = ["isbn", "author_id"]
                output_books_stream.writerow(book_line)
                output_authors_stream.writerow(author_line)
                output_book_author_map_stream.writerow(book_author_map)
                parse_firs_line = False
            else:
                # parse and add book
                book_line = [input_str[0], input_str[2], input_str[1], input_str[4], input_str[5], input_str[6]]
                if len(book_line)>0:
                    output_books_stream.writerow(book_line)
                # for author check in author map if not present then only add
                book_authors = re.split("&amp;|,", input_str[3])
                tmp_set=set(book_authors)
                book_authors=list(tmp_set)
                for author in book_authors:
                    if author in authors:
                        pass
                    else:
                        author_count += 1
                        author = author.strip()
                        authors[author] = author_count
                        tokens = author.split(' ')
                        if len(tokens) == 1:
                            author_line = [author_count, author, '', '', '', tokens[0], '']
                        elif len(tokens) == 2:
                            author_line = [author_count, author, '', tokens[0], '', tokens[-1], '']
                        elif len(tokens) == 3:
                            author_line = [author_count, author, '', tokens[0], tokens[1], tokens[-1], '']
                        else:
                            author_line = [author_count, author, '', tokens[0], tokens[1]+' '+tokens[2], tokens[-1], '']
                        if len(author_line) > 0:
                            output_authors_stream.writerow(author_line)
                    author_id = authors.get(author)
                    book_author_map = [input_str[0],author_id]
                    if len(book_author_map) > 0:
                        output_book_author_map_stream.writerow(book_author_map)
    print "Processed Records:",count
    csv_stat_item['category'] = item_predict
    #csv_stat_item['category'] = item_predict    
    
    i_index += 1
     
    csv_stat_all_data.append(csv_stat_item)



csv_set_sorted = sorted(csv_stat_all_data, key=lambda x: (x['id']), reverse=False)


with open(str_file_name_tw, 'wt', newline='') as csvfile:
    
    csv_writer = csv.writer(csvfile, dialect='excel')
    
    
    header = ['Id', 'Category']
    csv_writer.writerow(header)
    
    for data_point in csv_set_sorted:
        
        row = [data_point['id'], data_point['category']]
        #str_to_write = "\n" + str(data_point['id']) + "," + str(data_point['category'])
        #fout_tw.write(str_to_write)
        csv_writer.writerow(row)
        
    

示例#32
0
import csv

with open('/Users/kpentchev/data/ner_2019_04_06_prep.csv',
          encoding='utf-8') as csv_read:
    #with open('/home/kpentchev/data/floyd/ner_2019_02_25_fixed.csv', encoding='utf-8') as csv_read:
    csv_reader = csv.reader(csv_read, delimiter='\t', quoting=csv.QUOTE_NONE)
    line_count = 0
    sentence_count = 1
    with open('/Users/kpentchev/data/ner_2019_04_06_fixed.csv',
              encoding='utf-8',
              mode='w') as csv_write:
        #with open('/home/kpentchev/data/floyd/ner_2019_02_25_fixed.csv', encoding='utf-8', mode='w') as csv_write:
        csv_writer = csv.writer(csv_write, delimiter='\t')
        for row in csv_reader:
            if line_count == 0:
                csv_writer.writerow(row)
            else:
                sentence = row[0].strip()
                if (not "" == sentence):
                    row[0] = f"Sentence {sentence_count}"
                    sentence_count += 1
                if len(row) > 4:
                    print('{} old {}'.format(line_count, row))
                    row = [
                        row[0], ''.join(row[1:-2]).replace('"', ''), row[-2],
                        row[-1]
                    ]
                    print('{} new {}'.format(line_count, row))
                csv_writer.writerow(row)
            line_count += 1
def ACC_cal():


	'''
	calculate LCF and BCF using QP
	publish the next speed
	'''

	global m
	global g 
	global se 
	global F0 
	global F1 
	global F2 
	global gamma
	global ca 
	global cd 
	global psc 
	global x2
	global v0
	global vd 
	global D_old
	global D 
	global softness
	global initD 
	global dt
	global pub
	global x_old
	global pubSteer
	global initTime

	
	block = 3.5
	if (D <= 1.0):
		print "less than 1.0m"
		return 0.0
		pass
	print "======================================"
	z = D 
	y = x2 - vd
	#y = vd-x2
	#print "speed difference", x2, vd, y 
	#print "x2 and vd and y", x2, vd, y
	Fr = F0 + F1*(y+vd)+F2*(y+vd)**2
	Vv0 = -2.0*y/m*Fr + se*(y**2)
	
	Vv1 = 2.0*y/m
	
	#print "vv0, vv1", Vv0, Vv1
	h = z-block*x2-0.3					#0.3m for parking
	#h = z-block*x2
	try:		
		B = -1.0* math.log(h/(1+h))
	except:
		h = 0.0001
		B = -1.0* math.log(h/(1+h))

	LfB = -1.0*(block*Fr + m*(v0 - x2))/(m*(1.0 - block*x2 + z)*(-block*x2 + z));
	LgB = block/(m*(1.0 - block*x2 + z)*(-block*x2 + z))
	try:
		hf = -block*x2-0.5*((v0-x2)**2)/cd*g+z
		Bf = -1.0* math.log(hf/(1.0+hf))
	except:
		hf = 0.001
		Bf = -1.0* math.log(hf/(1.0+hf))
		#print "!!!!!!??????????!!!!"	
	'''	
	LfBf = (v0-x2-block)/(m*cd*g)/(-1.0*hf+hf**2)
	LgBf = (m*cd*g*(v0-x2)-Fr*(v0-x2-block))/(m*cd*g)/(-1.0*hf+hf**2)

	H = matrix([[2.0/(m**2), 0.0], [0.0, 2*psc]])		#P

	F = matrix([-2.0*Fr/(m**2), 0.0])					#q
	A = matrix([[Vv1, -1.0], [LgB, 0.0], [1.0,0.0], [1.0, 0.0], [LgBf, 0.0]]).T

	b = matrix([-1.0*Vv0, -1.0*LfB+gamma/B, vd/m*dt, ca*m*g, -1.0*LgBf+1.0/Bf])
	'''
	H = matrix([[2.0/(m**2), 0.0], [0.0, 2*psc]])		#P



	F = matrix([-2.0*Fr/(m**2), 0.0])					#q

	A = matrix([[Vv1, -1.0], [LgB, 0.0]]).T



	b = matrix([-1.0*Vv0,-1.0*LfB+gamma/B])
	#print A
	#print b

    
	
	#A = matrix([[Vv1, -1.0], [LgB, 0.0], [1.0, 0.0], [-1.0, 0.0], [LgBf, 0.0]]).T

	#b = matrix([-1.0*Vv0, -1.0*LfB+gamma/B, ca*m*g, cd*m*g, -1.0*LgBf+1.0/Bf])




	try:
		
		U = cvxopt.solvers.qp(H, F, A, b)
		#result = min(ca*g, max(-cd*g, (U["x"][0]/m)))
		result = (U["x"][0]-Fr)/m
		temp = [time.time()-initTime, x2, v0, D, result]
		#time, currentV, frontV, Dis, force
		with open ('/home/ubuntu/catkin_ws/src/ACC/accdata.csv', 'a') as f:
			writer = csv.writer(f)
			writer.writerow(temp)

		print temp
		
		#print "result", U["x"],result
        #m/s
		#print "result, result, result, result", result
	

	
	except:
		result = 0.0
		pub.publish(0.0)#========================
		print "oops Hit!!!XXXXXXXXXXXXXXXxxxxxxxxxxxxxXX"

		return result
		
	result = min(result/0.1109, 3.0)
	vel = (result*dt)+x2/0.1109 #rpm
	#print "vel",vel




	if (vel<0.0):
		pub.publish(0.0)#========================
		print "Backup Protection"

	elif(vel>(vd*1.1/0.1109)):
		vel = vd*1.1/0.1109
		#vel = 5.0/0.1109	
		pub.publish(vel)#=========================
		print "Speeding Protection"


	
	else:

		toPub = vel	
		x_old = toPub
		pubSteer.publish(10120)	
		if(x2 == 0 and toPub > 0):
		    pub.publish(3.0)
		else:
		    pub.publish(toPub)
		print "Moving Forward! Next speed:rpm", toPub
		#simuSpeed(toPub)
	
	
	return vel
示例#34
0
    def api_report(self,
                   request,
                   reporttype=None,
                   from_date=None,
                   to_date=None,
                   object_profile=None,
                   filter_default=None,
                   exclude_zero=None,
                   interface_profile=None,
                   selector=None,
                   administrative_domain=None,
                   columns=None,
                   o_format=None,
                   enable_autowidth=False,
                   **kwargs):
        def translate_row(row, cmap):
            return [row[i] for i in cmap]

        map_table = {
            "load_interfaces": r"/Interface\s\|\sLoad\s\|\s[In|Out]/",
            "load_cpu": r"/[CPU|Memory]\s\|\sUsage/",
            "errors": r"/Interface\s\|\s[Errors|Discards]\s\|\s[In|Out]/",
            "ping": r"/Ping\s\|\sRTT/",
        }
        cols = [
            "id",
            "object_name",
            "object_address",
            "object_platform",
            "object_adm_domain",
            "object_segment",
            "object_container",
            # "object_hostname",
            # "object_status",
            # "profile_name",
            # "object_profile",
            # "object_vendor",
            "iface_name",
            "iface_description",
            "iface_speed",
            "load_in",
            "load_in_p",
            "load_out",
            "load_out_p",
            "errors_in",
            "errors_out",
            "slot",
            "cpu_usage",
            "memory_usage",
            "ping_rtt",
            "ping_attempts",
            "interface_flap",
            "interface_load_url",
        ]

        header_row = [
            "ID",
            "OBJECT_NAME",
            "OBJECT_ADDRESS",
            "OBJECT_PLATFORM",
            "OBJECT_ADM_DOMAIN",
            "OBJECT_SEGMENT",
            "OBJECT_CONTAINER",
            "IFACE_NAME",
            "IFACE_DESCRIPTION",
            "IFACE_SPEED",
            "LOAD_IN",
            "LOAD_IN_P",
            "LOAD_OUT",
            "LOAD_OUT_P",
            "ERRORS_IN",
            "ERRORS_OUT",
            "CPU_USAGE",
            "MEMORY_USAGE",
            "PING_RTT",
            "PING_ATTEMPTS",
            "INTERFACE_FLAP",
            "INTERFACE_LOAD_URL",
        ]

        if columns:
            cmap = []
            for c in columns.split(","):
                try:
                    cmap += [cols.index(c)]
                except ValueError:
                    continue
        else:
            cmap = list(range(len(cols)))
        columns_order = columns.split(",")
        columns_filter = set(columns_order)
        r = [translate_row(header_row, cmap)]
        object_columns = [c for c in columns_order if c.startswith("object")]

        # Date Time Block
        if not from_date:
            from_date = datetime.datetime.now() - datetime.timedelta(days=1)
        else:
            from_date = datetime.datetime.strptime(from_date, "%d.%m.%Y")
        if not to_date or from_date == to_date:
            to_date = from_date + datetime.timedelta(days=1)
        else:
            to_date = datetime.datetime.strptime(
                to_date, "%d.%m.%Y") + datetime.timedelta(days=1)
        # interval = (to_date - from_date).days
        ts_from_date = time.mktime(from_date.timetuple())
        ts_to_date = time.mktime(to_date.timetuple())

        # Load managed objects
        mos = ManagedObject.objects.filter(is_managed=True)
        if not request.user.is_superuser:
            mos = mos.filter(
                administrative_domain__in=UserAccess.get_domains(request.user))
        if selector:
            mos = mos.filter(
                ManagedObjectSelector.objects.get(id=int(selector)).Q)
        if administrative_domain:
            mos = mos.filter(
                administrative_domain__in=AdministrativeDomain.get_nested_ids(
                    int(administrative_domain)))
        if object_profile:
            mos = mos.filter(object_profile=object_profile)
        # iface_dict = {}

        d_url = {
            "path": "/ui/grafana/dashboard/script/report.js",
            "rname": map_table[reporttype],
            "from": str(int(ts_from_date * 1000)),
            "to": str(int(ts_to_date * 1000)),
            # o.name.replace("#", "%23")
            "biid": "",
            "oname": "",
            "iname": "",
        }

        report_map = {
            "load_interfaces": {
                "url": "%(path)s?title=interface&biid=%(biid)s"
                "&obj=%(oname)s&iface=%(iname)s&from=%(from)s&to=%(to)s",
                "q_group": ["interface"],
                "q_select": {
                    (0, "managed_object", "id"): "managed_object",
                    (1, "path", "iface_name"): "arrayStringConcat(path)",
                },
            },
            "errors": {
                "url":
                """%(path)s?title=errors&biid=%(biid)s&obj=%(oname)s&iface=%(iname)s&from=%(from)s&to=%(to)s""",
                "q_group": ["interface"],
            },
            "load_cpu": {
                "url":
                """%(path)s?title=cpu&biid=%(biid)s&obj=%(oname)s&from=%(from)s&to=%(to)s""",
                "q_select": {
                    (0, "managed_object", "id"): "managed_object",
                    (1, "path", "slot"): "arrayStringConcat(path)",
                },
            },
            "ping": {
                "url":
                """%(path)s?title=ping&biid=%(biid)s&obj=%(oname)s&from=%(from)s&to=%(to)s""",
                "q_select": {
                    (0, "managed_object", "id"): "managed_object"
                },
            },
        }

        query_map = {
            # "iface_description": ('', 'iface_description', "''"),
            "iface_description": (
                "",
                "iface_description",
                "dictGetString('interfaceattributes','description' , (managed_object, arrayStringConcat(path)))",
            ),
            "iface_speed": (
                "speed",
                "iface_speed",
                "if(max(speed) = 0, dictGetUInt64('interfaceattributes', 'in_speed', "
                "(managed_object, arrayStringConcat(path))), max(speed))",
            ),
            "load_in":
            ("load_in", "l_in", "round(quantile(0.90)(load_in), 0)"),
            "load_in_p": (
                "load_in",
                "l_in_p",
                "replaceOne(toString(round(quantile(0.90)(load_in) / "
                "if(max(speed) = 0, dictGetUInt64('interfaceattributes', 'in_speed', "
                "(managed_object, arrayStringConcat(path))), max(speed)), 4) * 100), '.', ',')",
            ),
            "load_out":
            ("load_out", "l_out", "round(quantile(0.90)(load_out), 0)"),
            "load_out_p": (
                "load_out",
                "l_out_p",
                "replaceOne(toString(round(quantile(0.90)(load_out) / "
                "if(max(speed) = 0, dictGetUInt64('interfaceattributes', 'in_speed', "
                "(managed_object, arrayStringConcat(path))), max(speed)), 4) * 100), '.', ',')",
            ),
            "errors_in": ("errors_in", "err_in", "quantile(0.90)(errors_in)"),
            "errors_out":
            ("errors_out", "err_out", "quantile(0.90)(errors_out)"),
            "cpu_usage": ("usage", "cpu_usage", "quantile(0.90)(usage)"),
            "ping_rtt":
            ("rtt", "ping_rtt", "round(quantile(0.90)(rtt) / 1000, 2)"),
            "ping_attempts": ("attempts", "ping_attempts", "avg(attempts)"),
        }
        query_fields = []
        for c in report_map[reporttype]["q_select"]:
            query_fields += [c[2]]
        field_shift = len(query_fields)  # deny replacing field
        for c in columns.split(","):
            if c not in query_map:
                continue
            field, alias, func = query_map[c]
            report_map[reporttype]["q_select"][(columns_order.index(c) +
                                                field_shift, field,
                                                alias)] = func
            query_fields += [c]
        metrics_attrs = namedtuple("METRICSATTRs", query_fields)

        mo_attrs = namedtuple("MOATTRs",
                              [c for c in cols if c.startswith("object")])
        containers_address = {}
        if "object_container" in columns_filter:
            containers_address = ReportContainerData(
                set(mos.values_list("id", flat=True)))
            containers_address = dict(list(containers_address.extract()))
        moss = {}
        for row in mos.values_list("bi_id", "name", "address", "platform",
                                   "administrative_domain__name", "segment",
                                   "id"):
            moss[row[0]] = mo_attrs(*[
                row[1],
                row[2],
                str(Platform.get_by_id(row[3]) if row[3] else ""),
                row[4],
                str(NetworkSegment.get_by_id(row[5])) if row[5] else "",
                containers_address.
                get(row[6], "") if containers_address and row[6] else "",
            ])
        url = report_map[reporttype].get("url", "")
        report_metric = self.metric_source[reporttype](tuple(sorted(moss)),
                                                       from_date,
                                                       to_date,
                                                       columns=None)
        report_metric.SELECT_QUERY_MAP = report_map[reporttype]["q_select"]
        if exclude_zero and reporttype == "load_interfaces":
            report_metric.CUSTOM_FILTER["having"] += [
                "max(load_in) != 0 AND max(load_out) != 0"
            ]
        if interface_profile:
            interface_profile = InterfaceProfile.objects.filter(
                id=interface_profile).first()
            report_metric.CUSTOM_FILTER["having"] += [
                "dictGetString('interfaceattributes', 'profile', "
                "(managed_object, arrayStringConcat(path))) = '%s'" %
                interface_profile.name
            ]
        # OBJECT_PLATFORM, ADMIN_DOMAIN, SEGMENT, OBJECT_HOSTNAME
        for row in report_metric.do_query():
            mm = metrics_attrs(*row)
            mo = moss[int(mm.id)]
            res = []
            for y in columns_order:
                if y in object_columns:
                    res += [getattr(mo, y)]
                else:
                    res += [getattr(mm, y)]
            if "interface_load_url" in columns_filter:
                d_url["biid"] = mm.id
                d_url["oname"] = mo[2].replace("#", "%23")
                # res += [url % d_url, interval]
                res.insert(columns_order.index("interface_load_url"),
                           url % d_url)
            r += [res]
        filename = "metrics_detail_report_%s" % datetime.datetime.now(
        ).strftime("%Y%m%d")
        if o_format == "csv":
            response = HttpResponse(content_type="text/csv")
            response[
                "Content-Disposition"] = 'attachment; filename="%s.csv"' % filename
            writer = csv.writer(response,
                                dialect="excel",
                                delimiter=",",
                                quoting=csv.QUOTE_MINIMAL)
            writer.writerows(r)
            return response
        elif o_format == "xlsx":
            response = StringIO()
            wb = xlsxwriter.Workbook(response)
            cf1 = wb.add_format({"bottom": 1, "left": 1, "right": 1, "top": 1})
            ws = wb.add_worksheet("Alarms")
            max_column_data_length = {}
            for rn, x in enumerate(r):
                for cn, c in enumerate(x):
                    if rn and (r[0][cn] not in max_column_data_length or
                               len(str(c)) > max_column_data_length[r[0][cn]]):
                        max_column_data_length[r[0][cn]] = len(str(c))
                    ws.write(rn, cn, c, cf1)
            ws.autofilter(0, 0, rn, cn)
            ws.freeze_panes(1, 0)
            for cn, c in enumerate(r[0]):
                # Set column width
                width = get_column_width(c)
                if enable_autowidth and width < max_column_data_length[c]:
                    width = max_column_data_length[c]
                ws.set_column(cn, cn, width=width)
            wb.close()
            response.seek(0)
            response = HttpResponse(response.getvalue(),
                                    content_type="application/vnd.ms-excel")
            response[
                "Content-Disposition"] = 'attachment; filename="%s.xlsx"' % filename
            response.close()
            return response
示例#35
0
import csv
import math

with open('fruits-responses-time.csv', 'w') as csvfile:
    writer = csv.writer(
        csvfile, delimiter=',')  #,quotechar='"', quoting=csv.QUOTE_MINIMAL)
    f = open('output-all.txt', 'r')
    lines = f.readlines()
    for line in lines:
        words = line.split()
        res = [words[3]]
        # res = []
        # res.append(int(math.log(int(words[6][:-1]),2))-5)
        # res.append(int(words[4])+1)
        # res.append(words[10][:-1])
        # res.append(words[12][:-1])
        # res.append(words[14][:-1])
        writer.writerow(res)
示例#36
0
current_legislature = "14"
enregistrement = sys.argv[1]


logging
logger = logging.getLogger('myapp')
hdlr = logging.FileHandler('./logs/'+current_legislature+'.log')
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
hdlr.setFormatter(formatter)
logger.addHandler(hdlr) 
logger.setLevel(logging.WARNING)


nom_fichier = 'votes'+current_legislature+'.csv'
fichier = open(nom_fichier, 'a', newline='')
spamwriter = csv.writer(fichier, delimiter=',')


lst_scrutins_html = glob.glob('./scrutins'+current_legislature+'/*.html')    #On liste uniquement les fichiers '.html'
lst_scrutins_html.sort()
print(lst_scrutins_html)


for s in range(len(lst_scrutins_html)) :
    name_file = lst_scrutins_html[s];
    try :
        parcours_scrutin_html(lst_scrutins_html[s])
    except Exception as e :
        logger.error("THROW ERROR GRAVE parcours_scrutin_html : "+str(e))
        logger.error("----"+info_fichier())
    reset_currents_all()
def find_nearest(array, value):
    array = np.asarray(array)
    idx = (np.abs(array - value)).argmin()
    return idx


for i in range(Ih.size):
    x = l
    y = maxV[1, 1, :, i]
    dy = np.gradient(y)
    d2y = np.gradient(dy)
    d3y = np.gradient(d2y)

    y2 = maxV[0, 1, :, i]
    z = np.polyfit(x, y2, 1)
    p = np.poly1d(z)

    critical_length_20mv.append(l[find_nearest(y, -20)])
    critical_length_d2y.append(l[np.argmax(d2y)])
    gradients.append(z[0])

with open(r'outputs/data/figure_4_supplementary_3b.csv', 'w') as f:
    writer = csv.writer(f)
    writer.writerow(
        ['Ih', 'critical_length_d2y', 'critical_length_20mv', 'slope'])
    for i in range(Ih.size):
        fields = [
            Ih[i], critical_length_d2y[i], critical_length_20mv[i],
            gradients[i]
        ]
        writer.writerow(fields)
s1 = s1 * 10000 + s2 * 100 + s3
time = s1
mode = date.split('-')
s1 = int(mode[0])
s2 = int(mode[1])
s3 = int(mode[2])
s1 = s1 * 10000 + s2 * 100 + s3
date = s1
filename = 'output' + str(time) + '_' + str(date)
filename += '.csv'
print('Name of File created :', filename)
header_name = ["Date", "Time", "lat1", "lng1", "alt1", "pitchd1", "yawd1", "yawo1", "pitcho1", "lat2", "lng2", "alt2",
               "pitchd2", "yawd2", "yawo2", "pitcho2"]

with open(filename, 'w+') as csv_file:
    csv_writer = csv.writer(csv_file)
    csv_writer.writerow(header_name)
flag=0
v = visualize(0, 0, 0, 0, 0, 0, 0, 0, 0)
while True:
    data1 = connrun()
    data2 = conn1run()
    data1 = data1.decode()
    data2 = data2.decode()
    data1 = data1.split('\n')
    data1 = data1[-2]
    data2 = data2.split('\n')
    data2 = data2[-2]
    data1 = data1.split(',')
    data2 = data2.split(',')
    currenttime = datetime.datetime.now()
示例#39
0
def extract_text(input_image,
                 max_contour_area=1000,
                 max_contour_envelope_area=1500,
                 aspect_ratio=2.0,
                 gblur="True",
                 erode="True"):
    """

    These are Gina`s default values for the params

    :param input_image:
    :param max_contour_area: controls rejection of contours
    :param max_contour_envelope_area: controls rejection of contours
    :param aspect_ratio: controls rejection of contours
    :param gblur: apply gaussian blur
    :param erode: apply erosion filter
    :return:
    """
    print("Hey remember, we`ve turned off most of the script! :)")

    job_uuid = str(uuid.uuid4())
    base_output_path = "/home/james/geocrud/adrc/"

    print("processing %s" % input_image)
    fname_timestamp = time.strftime("%Y%m%d-%H%M%S")

    # Load map – change path and filename
    # cv2.imread(filename) : loads an image from a file - returns an array/matrix
    im = cv2.imread(input_image)

    # im.copy() : copies one array to another
    im2 = im.copy()  #keep a copy
    im3 = im.copy()
    #print "input image..."

    # cv2.imshow(winname, image) : display an image in the specified window
    #cv2.imshow("1 Input Image", im)

    # cv2.waitKey(delay in ms) : wait for a key event
    #cv2.waitKey(0)

    ############################ PRE-PROCESSING ##############################
    # Convert to greyscale
    processed_image = cv2.cvtColor(im, cv2.COLOR_RGB2GRAY)
    #cv2.imshow("2 Greyscaled", processed_image)
    #cv2.waitKey(0)

    # Apply Gaussian Blur - Increase in case of dotted background
    if gblur == "True":
        processed_image = cv2.GaussianBlur(processed_image, (5, 5), 0)
        #cv2.imshow("3 GBlur Applied", processed_image)
        #cv2.waitKey(0)

    # Apply Otsu's threshold and invert the colors of the image - digits white on black background
    (thresh,
     processed_image) = cv2.threshold(processed_image, 0, 255,
                                      cv2.THRESH_BINARY_INV | cv2.THRESH_OTSU)
    #cv2.imshow("4 Thresholded", processed_image)
    #cv2.waitKey(0)

    if erode == "True":
        # Define Kernel Size and apply erosion filter
        element = cv2.getStructuringElement(cv2.MORPH_RECT, (3, 3))
        processed_image = cv2.erode(processed_image, element)

    blank = processed_image.copy()
    blank2 = blank.copy()
    processed_image_b = processed_image.copy()

    #print "erosion filter applied..."
    #cv2.imshow("5 Erosion Filter applied", processed_image)
    #cv2.waitKey(0)

    out_fname = "".join([
        base_output_path,
        # job_uuid,
        # "_",
        # str(max_contour_area),
        # "_",
        # str(max_contour_envelope_area),
        # "_",
        # str(aspect_ratio),
        "img_passed_to_feature_extraction.tif"
    ])

    cv2.imwrite(out_fname, processed_image)

    ############################ FEATURE DETECTION ##############################
    # Contour tracing - Detect all contours with no hierarchy
    image, contours, hierarchy = cv2.findContours(processed_image,
                                                  cv2.RETR_LIST,
                                                  cv2.CHAIN_APPROX_SIMPLE)

    #################### JRCC - SHOW ALL Contours #############################
    font = cv2.FONT_HERSHEY_COMPLEX

    # draw mbr of every contour
    id = 0
    num_retained = 0
    num_dropped = 0
    lower_area_threshold = 0
    contour_features = []
    final_contours = []
    contours_retain = []

    # thresholds to decide if we retain the contour
    thresholds = {
        "area": 5000,
        "h_lower": 20,
        "h_upper": 100,
        "aspect_ratio": 1.5,
        "roundness": 0.1,
        "solidity": 0.8
    }

    base_outfname = os.path.splitext(os.path.split(input_image)[-1])[0]
    #out_fname = os.path.join("/home/james/geocrud/adrc", "".join([base_outfname, "_contour_properties.csv"]))
    #with open(out_fname, "w") as outpf:
    #    my_writer = csv.writer(outpf, delimiter=',', quotechar='"', quoting=csv.QUOTE_NONNUMERIC)
    #    my_writer.writerow(["id", "x", "y", "w", "h", "area", "perimeter", "is_convex", "angle_of_rotation", "aspect_ratio", "extent", "solidity", "compactness", "roundness"])
    for cnt in contours:
        this_contour_properties = get_contour_properties(cnt)

        x = this_contour_properties["x"]
        y = this_contour_properties["y"]
        w = this_contour_properties["w"]
        h = this_contour_properties["h"]
        # hw = h/w
        # wh = w/h
        # area = this_contour_properties["area"]
        # perimeter = this_contour_properties["perimeter"]
        # is_convex = this_contour_properties["is_convex"]
        # angle_of_rotation = this_contour_properties["angle_of_rotation"]
        # aspect_ratio = this_contour_properties["aspect_ratio"]
        # extent = this_contour_properties["extent"]
        # solidity = this_contour_properties["solidity"]
        # compactness = this_contour_properties["compactness"]
        # roundness = this_contour_properties["roundness"]

        if retain_contour(this_contour_properties, thresholds):
            cv2.rectangle(im2, (x, y), (x + w, y + h), (0, 255, 0), 1)
            #cv2.putText(im2, str(id), (x, y+h), cv2.FONT_HERSHEY_PLAIN, 2, [0,0,255])
            contours_retain.append(cnt)
            num_retained += 1
        else:
            num_dropped += 1

        #if area > lower_area_threshold:
        #cv2.rectangle(im2, (x, y), (x + w, y+h), (0, 255, 0), 1)
        #cv2.putText(im2, str(id), (x, y+h), cv2.FONT_HERSHEY_PLAIN, 2, [0,0,255])
        #contour_features.append([id, x, y, w, h])
        #my_writer.writerow([id, w, h, hw, wh, area, perimeter, is_convex, angle_of_rotation, aspect_ratio, extent, solidity, compactness, roundness])
        #final_contours.append(cnt)
        #num_retained += 1
        #else:
        #num_dropped += 1

        id += 1

    #dump_contours_to_shapefile(contour_features, input_image)

    #cv2.drawContours(im2, final_contours, -1, (255, 64, 0), 1)
    #cv2.drawContours(processed_image, final_contours, -1, (255, 64, 0), 1)
    #cv2.imwrite(out_fname, processed_image)

    #print "num contours retained", num_retained
    #print "num contours dropped", num_dropped

    cv2.imwrite("/home/james/geocrud/adrc/82877433_selected_contours.tif", im2)

    ###########################################################################

    #!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!#
    ###########################################################################
    # 050416 stop this part of the script running
    ###########################################################################

    cont = False

    if cont:
        contour_properties = [[
            "contour_id", "h", "w", "area", "hw", "wh", "hxw", "x", "y"
        ]]

        # display info about every contour and add a label to the displayed image
        contour_id = 1

        for cnt in contours:
            [x, y, w, h] = cv2.boundingRect(cnt)
            # cv2.contourArea(contour) : compute contour area
            contour_area = cv2.contourArea(cnt)

            hw = h / w
            wh = w / h

            hxw = h * w
            text_x = x
            text_y = y + h + 15

            contour_properties.append(
                [contour_id, h, w, contour_area, hw, wh, hxw, x, y + h])

            #print input_image,contour_id, h, w, hw, wh, contour_area, hxw, x, y+h

            contour_id += 1

        csv_fname = "".join([
            base_output_path,
            # job_uuid,
            # "_",
            # str(max_contour_area),
            # "_",
            # str(max_contour_envelope_area),
            # "_",
            # str(aspect_ratio),
            "contour_properties.csv"
        ])

        with open(csv_fname, "w") as outpf:
            c = csv.writer(outpf,
                           delimiter=",",
                           quotechar='"',
                           quoting=csv.QUOTE_NONNUMERIC)
            c.writerows(contour_properties)

        out_fname = "".join([
            base_output_path,
            # job_uuid,
            # "_",
            # str(max_contour_area),
            # "_",
            # str(max_contour_envelope_area),
            # "_",
            # str(aspect_ratio),
            "all_contours.png"
        ])

        # cv2.imwrite() : save image to a specified file
        cv2.imwrite(out_fname, im2)

        #print "All contours..."
        #cv2.imshow("All contours", im2)
        #cv2.waitKey(0)

        #################### End of jrcc ##########################################

        # Apply the first 4 rules for digit graphic separation
        contours_reject = []
        for cnt in contours:
            [x, y, w, h] = cv2.boundingRect(cnt)
            if cv2.contourArea(
                    cnt
            ) < max_contour_area and h / w < aspect_ratio and w / h < aspect_ratio and h * w < max_contour_envelope_area:
                continue
            else:
                contours_reject.append(cnt)

        # Erase contours from the image
        cv2.drawContours(blank, contours_reject, -1, (0, 0, 0), -1)

        #print "Contours Erased..."
        #cv2.imshow("Contours Erased", blank)
        #cv2.waitKey(0)

        #blank2=blank.copy()
        image, contours, hierarchy = cv2.findContours(blank, cv2.RETR_EXTERNAL,
                                                      cv2.CHAIN_APPROX_SIMPLE)

        contour_id = 1
        contours_retain = []
        contours_reject = []
        for cnt in contours:
            if cv2.contourArea(cnt) > 50:
                contours_retain.append(cnt)
            else:
                contours_reject.append(cnt)
            contour_id += 1

        # Erase small contours from the image
        cv2.drawContours(blank2, contours_reject, -1, (0, 0, 0), -1)

        #TODO draw the full contour rather than it`s MBR
        #cv2.drawContours(im2, contours_retain, -1, (0,255,0), 2)
        #cv2.drawContours(im2, contours_retain, -1, (0,255,0), 2)
        #cv2.imwrite(out_fname, im2)

        #print "Small contours erased..."
        #cv2.imshow("Small Contours Erased", blank2)
        #cv2.waitKey(0)

    ############# Dealing with Touching Digits ##########################
    """
    Stavropoulou_Optical Character Recognition on Scanned Maps compressed.pdf p20

    num_cols - choice depends on the size of the touching characters and
    subsequently on the size of the font. After the white pixels have been
    counted for each column the characters are seperated in the column that
    has the minimum number of white pixels
    """

    # 070416 - from running against 82877433.tif large important letters
    # at the start of words are being incorrectly split by this part of
    # the process due to the w>2*num_cols+1 and w>1.2*h criteria

    deal_w_touching_digits = False

    if not deal_w_touching_digits:
        print(
            "Hey, one more thing, we`ve turned off dealing with touching digits!"
        )

    #print '.....Dealing with Touching Digits'
    bounding_list = []
    digit_im_list = []
    # Set the dimensions for resizing
    dim = (24, 42)

    # Specify the search number of columns for selecting cut column.
    num_cols = 5

    for cnt in contours_retain:
        [x, y, w, h] = cv2.boundingRect(cnt)  #finding bounding rectangle

        if deal_w_touching_digits:
            if w > 2 * num_cols + 1:
                if w > 1.2 * h:  #in case of fused characters
                    middle = int(round(w / 2))
                    fused_character_im = blank2[y - 1:y + h + 1,
                                                x - 1:x + w + 1]
                    min_col = 0
                    col_wh_pixels = []
                    for i in range(middle - num_cols, middle + num_cols):
                        # examining all the middle columns
                        col = fused_character_im[:, i]
                        white_pixels = 0
                        for j in range(len(col)):
                            if col[j] != 0:
                                white_pixels = white_pixels + 1
                        col_wh_pixels.append(white_pixels)
                    min_index = col_wh_pixels.index(min(col_wh_pixels))
                    patch1 = blank2[y - 1:y + h + 1,
                                    x - 1:x + middle - num_cols + min_index]
                    resized1 = cv2.resize(patch1,
                                          dim,
                                          interpolation=cv2.INTER_AREA)
                    ret, patch1 = cv2.threshold(resized1, 50, 255,
                                                cv2.THRESH_BINARY)
                    #Repeat thresholding after interpolation
                    patch2 = blank2[y - 1:y + h + 1, x + middle - num_cols +
                                    min_index:x + w + 1]
                    resized2 = cv2.resize(patch2,
                                          dim,
                                          interpolation=cv2.INTER_AREA)
                    ret, patch2 = cv2.threshold(resized2, 50, 255,
                                                cv2.THRESH_BINARY)
                    #Repeat thresholding after interpolation
                    digit_im_list.append(patch1)
                    digit_im_list.append(patch2)
                    bounding_list.append([x, y, w / 2, h])
                    bounding_list.append([x + w / 2, y, w / 2, h])
                else:
                    patch = blank2[y - 1:y + h + 1, x - 1:x + w + 1]
                    resized = cv2.resize(patch,
                                         dim,
                                         interpolation=cv2.INTER_AREA)
                    ret, patch = cv2.threshold(resized, 127, 255,
                                               cv2.THRESH_BINARY)
                    #Repeat thresholding after interpolation
                    digit_im_list.append(patch)
                    bounding_list.append([x, y, w, h])
        else:
            # if we`re not dealing with touching digits do the default always
            # as later on the script makes use of data structures updated here
            patch = blank2[y - 1:y + h + 1, x - 1:x + w + 1]
            resized = cv2.resize(patch, dim, interpolation=cv2.INTER_AREA)
            ret, patch = cv2.threshold(resized, 127, 255, cv2.THRESH_BINARY)
            #Repeat thresholding after interpolation
            digit_im_list.append(patch)
            bounding_list.append([x, y, w, h])

    arr = np.array(bounding_list)

    # TODO - the aoi`s clip too tightly to the contour so we need to add a buffer

    # 250416 - what was I thinking when I implemented this?
    # for m in xrange(len(arr)):
    #     x1 = arr[m, 0]
    #     y1 = arr[m, 1]
    #     w  = arr[m, 2]
    #     h  = arr[m, 3]
    #     x2 = x1 + w
    #     y2 = y1 + h
    #
    #     # add a 1px buffer around the contour
    #     # errors will happen if the buffer extends beyond the extents of the image
    #     x1 = x1 - 10
    #     y1 = y1 - 10
    #     x2 = x2 + 10
    #     y2 = y2 + 10
    #
    #     aoi_fname = base_output_path + str(m) + ".png"
    #     cv2.imwrite(aoi_fname, processed_image_b[y1:y2, x1:x2])

    processed_image_c = cv2.cvtColor(processed_image_b, cv2.COLOR_GRAY2RGB)

    #id = 0

    #for m in xrange(len(arr)):
    #print m
    #cv2.rectangle(im3,(arr[m,0],arr[m,1]),(arr[m,0] + arr[m,2],arr[m,1]+arr[m,3]),(255,0,255),2)
    #cv2.rectangle(im3,(int(arr[m,0]),int(arr[m,1])),(int(arr[m,0]) + int(arr[m,2]),int(arr[m,1])+int(arr[m,3])),(255,0,255),2)
    #cv2.rectangle(processed_image_c,(int(arr[m,0]),int(arr[m,1])),(int(arr[m,0]) + int(arr[m,2]),int(arr[m,1])+int(arr[m,3])),(255,0,255),1)
    #cv2.putText(processed_image_c, str(id), (int(arr[m,0]),int(arr[m,1])), cv2.FONT_HERSHEY_PLAIN, 2, [0,0,255])
    #id += 1

    #cv2.imshow("Detected Features", im3)
    #cv2.waitKey(0)

    #out_fname = "".join([base_output_path,
    #                     job_uuid,
    #                     "_",
    #                     str(max_contour_area),
    #                     "_",
    #                     str(max_contour_envelope_area),
    #                     "_",
    #                     str(aspect_ratio),
    #                     "_final_contours.png"])

    #cv2.imwrite(out_fname, im3)

    #cv2.putText(processed_image_c, "Hello World", (10, 10), cv2.FONT_HERSHEY_PLAIN, 2, [255,0,0] )

    cv2.drawContours(processed_image_c, contours_retain, -1, (0, 255, 0), 2)

    out_fname = "".join([
        base_output_path,
        # job_uuid,
        # "_",
        # str(max_contour_area),
        # "_",
        # str(max_contour_envelope_area),
        # "_",
        # str(aspect_ratio),
        "img_passed_to_feature_extraction_w_contours.png"
    ])

    cv2.imwrite(out_fname, processed_image_c)

    # write out the numpy arrays of the extracted features to a text file
    # in a seperate script we will classify these

    id = 0

    features = []
    for m in range(len(arr)):
        try:
            feature = digit_im_list[m]

            # ------------------------ 250416 ---------------------------------
            out_fname = "/home/james/geocrud/adrc/" + str(id) + "_sample.png"
            cv2.imwrite(out_fname, feature)
            # ------------------------ 250416 ---------------------------------

            #out_fname = "/home/james/geocrud/adrc/" + job_uuid + "_candidate_" + str(id) + ".png"
            #out_fname = "/home/james/geocrud/adrc/" + str(id) + ".png"
            #cv2.imwrite(out_fname, feature)
            reshaped_feature = feature.reshape((1, 1008))
            features.append(reshaped_feature)

            id += 1
        except Exception as ex:
            print(ex)

    ###########################################################################
    #TODO - so we can create a shapefile showing marked up locations we need
    #to dump location of candidates

    print("".join(["len(digit_im_list): ", str(len(digit_im_list))]))
    print("".join(["len(bounding_list): ", str(len(bounding_list))]))

    id = 0
    with open("/home/james/geocrud/adrc/candidate_locations.csv",
              "w") as outpf:
        my_writer = csv.writer(outpf,
                               delimiter=",",
                               quotechar='"',
                               quoting=csv.QUOTE_NONNUMERIC)
        my_writer.writerow(["id", "x", "y", "w", "h"])
        for m in range(len(arr)):
            x, y, w, h = bounding_list[m]
            my_writer.writerow([id, x, y, w, h])
            id += 1

    candidates = np.empty((0, 1008))
    for feature in features:
        try:
            candidates = np.append(candidates, feature, 0)
        except Exception as ex:
            print(ex)

    np.savetxt(os.path.join(base_output_path[:-1], "candidates.data"),
               candidates)
示例#40
0
difference = numpy.zeros((num_of_board, num_of_taxel, num_of_axis))
tracking = numpy.zeros((num_of_board, num_of_taxel, num_of_axis))
limit_threshold = 0xA0
limit_upper = 0xFA
limit_lower = 0x0A

##############
### Create a CSV file ####
for i in range(num_of_board):
    del_num = board_start_num + i
    if os.path.exists('visualization/LOG%s.csv' % del_num):
        os.remove('visualization/LOG%s.csv' % del_num)

    csvfile = open('visualization/LOG%s.csv' % del_num,
                   'wb')  #Create a new csv file
    filewrite = csv.writer(csvfile)
    filewrite.writerow(
        ('B1S1X', 'B1S1Y', 'B1S1Z', 'B1S2X', 'B1S2Y', 'B1S2Z', 'B1S3X',
         'B1S3Y', 'B1S3Z', 'B1S4X', 'B1S4Y', 'B1S4Z', 'B2S1X', 'B2S1Y',
         'B2S1Z', 'B2S2X', 'B2S2Y', 'B2S2Z', 'B2S3X', 'B2S3Y', 'B2S3Z',
         'B2S4X', 'B2S4Y', 'B2S4Z', 'B3S1X', 'B3S1Y', 'B3S1Z', 'B3S2X',
         'B3S2Y', 'B3S2Z', 'B3S3X', 'B3S3Y', 'B3S3Z', 'B3S4X', 'B3S4Y',
         'B3S4Z', 'B4S1X', 'B4S1Y', 'B4S1Z', 'B4S2X', 'B4S2Y', 'B4S2Z',
         'B4S3X', 'B4S3Y', 'B4S3Z', 'B4S4X', 'B4S4Y', 'B4S4Z'))
    csvfile.close()  # do we need to close it first?

# ---> cif = ntcan.CIF( net, RxQueueSize, RxTimeOut, TxQueueSize, TxTimeOut, Flags)
net = 0  # logical CAN Network [0, 255]
RxQS = 1  # RxQueueSize [0, 10000]
RxTO = 2000  # RxTimeOut in Millisconds
TxQS = 1  # TxQueueSize [0, 10000]
示例#41
0
def output_writer(f):
    write = csv.writer(open(f, 'wb'))
    return write
示例#42
0
    def _create_cache(self):
        # Save all data into cache file(s).
        self._cache_positions = []
        self._position = 0

        percent = 0

        if single_or_rankzero():
            progress(None)

        while self._position < self._data_source._size:

            if single_or_rankzero():
                if self._position % int(self._data_source._size/20+1) == 0:
                    progress('Create cache', self._position *
                             1.0 / self._data_source._size)

            self._store_data_to_cache_buffer(self._position)
            self._position += 1
        if len(self._cache_positions) > 0:
            self._save_cache_to_file()

        if single_or_rankzero():
            progress(None)

        # Adjust data size into reseted position. In most case it means
        # multiple of bunch(mini-batch) size.
        num_of_cache_files = int(numpy.ceil(
            float(self._data_source._size) / self._cache_size))
        self._cache_file_order = self._cache_file_order[
            0:num_of_cache_files]
        self._cache_file_data_orders = self._cache_file_data_orders[
            0:num_of_cache_files]
        if self._data_source._size % self._cache_size != 0:
            self._cache_file_data_orders[num_of_cache_files - 1] = self._cache_file_data_orders[
                num_of_cache_files - 1][0:self._data_source._size % self._cache_size]

        # Create Index
        index_filename = os.path.join(self._cache_dir, "cache_index.csv")
        with open(index_filename, 'w') as f:
            writer = csv.writer(f, lineterminator='\n')
            for fn, orders in zip(self._cache_file_names, self._cache_file_data_orders):
                writer.writerow((os.path.basename(fn), len(orders)))
        # Create Info
        if self._cache_file_format == ".npy":
            info_filename = os.path.join(self._cache_dir, "cache_info.csv")
            with open(info_filename, 'w') as f:
                writer = csv.writer(f, lineterminator='\n')
                for variable in self._variables:
                    writer.writerow((variable, ))

        # Create original.csv
        if self._data_source._original_source_uri is not None:
            fr = FileReader(self._data_source._original_source_uri)
            with fr.open() as f:
                csv_lines = [x.decode('utf-8') for x in f.readlines()]
                with open(os.path.join(self._cache_dir, "original.csv"), 'w') as o:
                    for l in csv_lines:
                        o.write(l)

        # Create order.csv
        if self._data_source._order is not None and \
                self._data_source._original_order is not None:
            with open(os.path.join(self._cache_dir, "order.csv"), 'w') as o:
                writer = csv.writer(o, lineterminator='\n')
                for orders in zip(self._data_source._original_order, self._data_source._order):
                    writer.writerow(list(orders))
示例#43
0
def main():
    config = Config()
    #modeler = model.VGG(config)
    modeler = model.VGG16(include_top=False)
    #modeler = model.ResNet50(include_top=False, input_shape=[config.img_height,config.img_width, 3], pooling='max')
    inputs = Input(shape=[config.img_height,config.img_width,3])
    y = modeler(inputs)

    y = Flatten()(y)
    y = Dense(4096, activation='relu', name='fc1')(y)
    y = Dense(4096, activation='relu', name='fc2')(y)
    y = Dense(config.type_size, activation='softmax', name='predictions')(y)
    modeler = Model(inputs, y, name='vgg16')

    '''
    y = Dense(config.type_size, activation='softmax', name='fc17')(y)
    modeler = Model(inputs, y, name='resnet50')
    '''
    
    modeler.load_weights(config.params_dir + config.net_name + "-weather-params-" + str(config.test_num) + ".h5")

    #modeler.compile(loss='categorical_crossentropy', optimizer=SGD(lr=config.learning_rate, momentum=0.9,nesterov=True))

    # read data to test("data/train")
    test_reader = read_data.VGGReader(config)

    pre_prob = list()
    with open("./thresholds/" + config.net_name + "-weather-thresholds-" + config.test_num + ".txt", 'rb')  as fr:
        for line in fr:
            tmp = re.split(' ', line.strip())
            for i in range(config.type_size):
                pre_prob.append(float(tmp[i]))
    print "thresholds: ", pre_prob

    test_labels = list()
    pre_labels = list()
    val_labels = list()
    min_true = [1.0, 1.0, 1.0, 1.0]
    max_false = [0.0, 0.0, 0.0, 0.0]
    print "start testing..."
    for step in range(config.test_size):
        if step % (config.test_size // 10) == 0:
            print 100 * step // config.test_size, "%"

        images_test, labels_test, filesname_test = test_reader.batch()
        prob = modeler.predict(images_test)
        test_index = list()
        for i in range(config.type_size):
            val_labels.append(labels_test[0][i])

            if prob[0][i] > pre_prob[i]:
                test_index.append(i)

            # get min_true
            if labels_test[0][i] == 1.0 and prob[0][i] > 0.1 and prob[0][i] < min_true[i]:
                min_true[i] = prob[0][i]
            if labels_test[0][i] == 0.0 and prob[0][i] > max_false[i]:
                max_false[i] = prob[0][i]

        '''
        if step % 10 == 0 and config.val_mode == True:
            print labels_test[0]
            print prob[0]
        '''
    
        s = filesname_test[0]
        for n in range(config.type_size):
            is_in = False
            for m in range(len(test_index)):
                if n == test_index[m]:
                    is_in = True
            if is_in:
                s += " 1.0"
                pre_labels.append(1.0)
            else:
                s += " 0.0"
                pre_labels.append(0.0)

        test_labels.append(s)

    print "scores: ", fbeta_score(val_labels, pre_labels, beta=2)
    print "min_true: ", min_true
    print "max_false, ", max_false
   
    if config.val_mode == False:
        with open("./labels/weather_test_results.csv", 'w') as fr:
            fcsv = csv.writer(fr)
            for i in range(len(test_labels)):
                fcsv.writerow([test_labels[i]])
示例#44
0
direction = agent.step()
board = game.move(direction)

direct={}
i=0
k=0
filename = 'test.csv'

if os.path.exists(filename):
	head = True
else:
	head = False
	os.mknod(filename)

with open(filename, "a") as csvfile:
	writer = csv.writer(csvfile)
	if not head:
		writer.writerow(["11","12","13","14",\
			         "21","22","23","24",\
			         "31","32","33","34",\
			         "41","42","43","44",\
			         "direction"])

	while i < repeat_times:

		game = Game(GAME_SIZE, SCORE_TO_WIN)
		board = game.board
		print('Repeat times:', i)

		while(game.end == 0):
			agent = ExpectiMaxAgent(game, Display())
示例#45
0
def RandomWalkElf(program, outfile, mclass, max_subtraces, max_explored_subtraces, min_size):


  csvwriter = csv.writer(open(outfile, "a+"), delimiter='\t')
  elf = ELF(program)

  # plt is inverted
  inv_plt = dict()

  for func, addr in elf.plt.items():
    if func in specs:  # external functions are discarded
      inv_plt[addr] = func

  elf.plt = inv_plt

  cond_control_flow_ins = ["jo", "jno", "js", "jns", "je",
                           "jz","jnz", "jb", "jnae", "jc",
                           "jnb", "jae", "jnc", "jbe", "jna",
                           "ja", "jnbe", "jl", "jnge", "jge",
                           "jnl", "jle", "jng", "jg",  "jnle",
                           "jp", "jpe", "jnp", "jpo", "jcxz", "jecxz"]

  ncond_control_flow_ins = ["ret","jmp","call", "retq","jmp","callq"]

  control_flow_ins = cond_control_flow_ins + ncond_control_flow_ins

  raw_inss = elf.GetRawInss()
  useful_inss_list = []
  useful_inss_dict = dict()
  libc_calls = []
  labels = dict()

  #print sys.argv[1]+"\t",
  #rclass = str(1)

  for i,ins in enumerate(raw_inss.split("\n")):

    # prefix removal
    ins = ins.replace("repz ","")
    ins = ins.replace("rep ","")

    pins = ins.split("\t")
    #print pins
    ins_addr = pins[0].replace(":","").replace(" ","")
    #print pins,ins_addr

    if len(pins) == 1 and ">" in ins: #label
      #print ins
      #assert(0)
      x = pins[0].split(" ")

      ins_addr = x[0]

      y = [i,ins_addr, None, None]
      useful_inss_dict[ins_addr] = y
      useful_inss_list.append(y)

      #print "label:",y

    elif any(map( lambda x: x in ins, control_flow_ins)) and len(pins) == 3: # control flow instruction
      #print pins
      x = pins[2].split(" ")

      ins_nme = x[0]
      ins_jaddr = x[-2]

      #if ("" == ins_jaddr):
      #  print pins
      #print x
      #print ins_nme, ins_jaddr
      y = [i, ins_addr, ins_nme, ins_jaddr]

      useful_inss_dict[ins_addr] = y
      useful_inss_list.append(y)

      if "call" in pins[2]:
        if ins_jaddr <> '':
          func_addr = int(ins_jaddr,16)
          if func_addr in elf.plt:
            libc_calls.append(i)

    else: # all other instructions 
      y = [i, ins_addr, None, None]

      useful_inss_dict[ins_addr] = y
      useful_inss_list.append(y)

  #print useful_inss_list
  max_inss = len(useful_inss_list)
  traces = set()
  collected_traces = ""

  # exploration time!
  for _ in range(max_explored_subtraces):

    # resuling (sub)trace
    r = ""
    # starting point
    i = random.choice(libc_calls)
    j = 0

    #r = elf.path+"\t"
    r = ""

    while True:

      # last instruction case
      if (i+j) == max_inss:
        break

      _,ins_addr,ins_nme,ins_jaddr = useful_inss_list[i+j]

      #print i+j,ins_nme, ins_jaddr

      if ins_nme in ['call', 'callq']: # ordinary call
        #"addr", ins_jaddr

        if ins_jaddr == '':
          break # parametric jmp, similar to ret for us

        ins_jaddr = int(ins_jaddr,16)
        if ins_jaddr in elf.plt:
          r = r + " " + elf.plt[ins_jaddr]
          if elf.plt[ins_jaddr] == "exit":
            break
        else:

          if ins_jaddr in useful_inss_dict:
            #assert(0)
            #r = r + " " + hex(ins_jaddr)
            i,_,_,_ = useful_inss_dict[ins_jaddr]
            j = 0
            continue

          else:
            pass # ignored call

      elif ins_nme in ['ret','retq']:
        break
      else:
        pass
        #print i+j,ins_nme, ins_jaddr

      #print j
      if ins_nme == 'jmp' :

        if ins_jaddr in elf.plt: # call equivalent using jmp
          r = r + " " + elf.plt[jaddr]

        else:

          if ins_jaddr == '':
            break # parametric jmp, similar to ret for us

          ins_jaddr = int(ins_jaddr,16)
          if ins_jaddr in useful_inss_dict:
            #r = r + " " + hex(ins_jaddr)
            i,_,_,_ = useful_inss_dict[ins_jaddr]
            j = 0
            continue

          else:
            pass # ignored call


      if ins_nme in cond_control_flow_ins:

        assert(ins_jaddr <> None)

        cond = random.randint(0,1)

        if cond == 1:

          i,_,_,_ = useful_inss_dict[ins_jaddr]
          j = 0
          continue

      j = j + 1

    #r = r + "\t"+rclass
    x = hash(r)
    size = len(r.split(" "))-1

    if x not in traces and size >= min_size:
      #print r+" .",
      collected_traces = collected_traces + r + " ."
      traces.add(x)
      if len(traces) >= max_subtraces:
        break

  row = [elf.path, collected_traces]
  if mclass is not None:
    row.append(mclass)

  csvwriter.writerow(row)
示例#46
0
if single_mode:
    profile_paths = [args.profile]
else:
    profile_paths = list(glob.glob(os.path.join(args.profile, "*")))

print("writing predictions to %s ..." % output_file)

print("\n-------\n")
print("         screen_name | class | confidence\n")

samples_per_inference = 5
samples = {}

with open(output_file, 'w+t', newline='') as fp:
    w = csv.writer(fp)

    w.writerow(['user_id', 'user_name', 'prediction', 'confidence'])

    for profile_path in profile_paths:
        prof = profile.load(profile_path, quiet=True)
        if prof is None:
            continue

        ((user, tweets, replies, retweets), vector) = prof
        
        vector = data.nomalized_from_dict(norm, vector)
        prediction = model.predict(vector)[0]
        
        (label, confidence) = print_pred(args, user['screen_name'], prediction)
示例#47
0
 def __init__(self, f, dialect=csv.excel, encoding="utf-8", **kwds):
     # Redirect output to a queue
     self.queue = cStringIO.StringIO()
     self.writer = csv.writer(self.queue, dialect=dialect, **kwds)
     self.stream = f
     self.encoder = getincrementalencoder(encoding)()
示例#48
0
def save_prediction_unknowns(model, loader, dataset, args):
    model.eval()

    LOGGER.info('processing {}..'.format(args.unknown_path))
    embeddings = pickle.load(open(args.embed_path, 'rb'))

    #ingr2category = pickle.load(open(args.ingr2category_dir, 'rb'))
    #category2rep = pickle.load(open(args.category2rep_dir, 'rb'))

    csv_writer = csv.writer(open(args.checkpoint_dir + 'prediction_unknowns_' +
                                 args.model_name + '.csv', 'w', newline=''))
    csv_writer.writerow(['ingr1', 'ingr2', 'prediction'])
    # csv_writer.writerow(['ingr1', 'ingr1_cate', 'ingr2', 'ingr2_cate', 'prediction'])

    #for d_idx, (d1, d1_r, d1_l, d2, d2_r, d2_l, score) in enumerate(loader):
    #    print(d1, d1_r, d1_l, d2, d2_r, d2_l, score)

    df = pd.read_csv(args.unknown_path, sep=",")
    df = df.reset_index()
    print(len(df))

    batch = []
    for row_idx, row in df.iterrows():
        ingr1 = row['ingr1']
        ingr2 = row['ingr2']
        try:
            ingr1_r = embeddings[ingr1]
            ingr2_r = embeddings[ingr2]

            #ignore this categorical features
            ingr1_c = embeddings[ingr1]
            ingr2_c = embeddings[ingr2]
        except KeyError:
            continue

        example = [ingr1, ingr1_r, ingr1_c, len(ingr1_r),
                   ingr2, ingr2_r, ingr2_c, len(ingr2_r), 0]
        batch.append(example)

        if len(batch) == 256:
            inputs = dataset.collate_fn(batch)
            outputs = model(inputs[1].cuda(), inputs[2].cuda(), inputs[3],
                                    inputs[5].cuda(), inputs[6].cuda(), inputs[7])
            predictions = outputs[2].data.cpu().numpy()

            for example, pred in zip(batch, predictions):
                csv_writer.writerow([example[0], example[4], pred])

            batch = []

        # Print progress
        if row_idx % 5000 == 0 or row_idx == len(df) - 1:
            _progress = '{}/{} saving unknown predictions..'.format(
                row_idx + 1, len(df))
            LOGGER.info(_progress)

    if len(batch) > 0:
        inputs = dataset.collate_fn(batch)
        outputs, _, _ = model(inputs[1].cuda(), inputs[2].cuda(), inputs[3],
                                inputs[5].cuda(), inputs[6].cuda(), inputs[7])
        predictions = outputs[2].data.cpu().numpy()

        for example, pred in zip(batch, predictions):
            csv_writer.writerow([example[0], example[4], pred])
print(args)

input_file=args.input_lines_csv[0]
input_audio_dir=args.input_audio_dir[0]
output_dir=args.output_dir[0]
language_code=args.language_code[0]
sound_find_string=None
if args.sound_find_string is not None: sound_find_string=args.sound_find_string[0]




file_name='aeneas.csv'
output_file=os.path.join(output_dir,file_name)
write_file_handle=open(output_file,'w',encoding='utf-8')
write_file = csv.writer(write_file_handle)
# Write header as requested by user
if not(args.noheader):write_file.writerow(('fileset','book','chapter','line_number', 'verse_number','verse_content','time'))


book_chapter_list=list()
def get_chapters_index(df):
    current_book_chapter=df['book'][0]+'_'+'0'+df['chapter'][0]
    book_chapter_list.append(df['book'][0]+'_'+'0'+df['chapter'][0])

    for i in range(1,len(df)):
        if df['book'][i]!=current_book_chapter.split('_')[0] or df['chapter'][i]!=(current_book_chapter.split('_')[1]).lstrip('0'):
            if int(df['chapter'][i])<10:
                current_book_chapter=df['book'][i]+'_'+df['chapter'][i]

                book_chapter_list.append((df['book'][i]+'_'+'0'+df['chapter'][i]).replace(' ','_'))
# import module
import csv
import math

# create file
file = open('cvs.csv', 'w')

# create csvwriter
csvwriter = csv.writer(file, delimiter=',')

# write info
csvwriter.writerow(['a', 'b', 'hypotenuse'])
for a in range(1,101):
    for b in range(1, 101):
        hypotenuse = math.sqrt(a ** 2 + b ** 2)
        csvwriter.writerow([a, b, hypotenuse])

# close file
file.close()
示例#51
0
def read(humfile, sonpath, cs2cs_args, c, draft, doplot, t, bedpick, flip_lr,
         model, calc_bearing, filt_bearing, chunk):  #cog = 1,
    '''
    Read a .DAT and associated set of .SON files recorded by a Humminbird(R)
    instrument.

    Parse the data into a set of memory mapped files that will
    subsequently be used by the other functions of the PyHum module.

    Export time-series data and metadata in other formats.

    Create a kml file for visualising boat track

    Syntax
    ----------
    [] = PyHum.read(humfile, sonpath, cs2cs_args, c, draft, doplot, t, bedpick, flip_lr, chunksize, model, calc_bearing, filt_bearing, chunk)

    Parameters
    ------------
    humfile : str
       path to the .DAT file
    sonpath : str
       path where the *.SON files are
    cs2cs_args : int, *optional* [Default="epsg:26949"]
       arguments to create coordinates in a projected coordinate system
       this argument gets given to pyproj to turn wgs84 (lat/lon) coordinates
       into any projection supported by the proj.4 libraries
    c : float, *optional* [Default=1450.0]
       speed of sound in water (m/s). Defaults to a value of freshwater
    draft : float, *optional* [Default=0.3]
       draft from water surface to transducer face (m)
    doplot : float, *optional* [Default=1]
       if 1, plots will be made
    t : float, *optional* [Default=0.108]
       length of transducer array (m).
       Default value is that of the 998 series Humminbird(R)
    bedpick : int, *optional* [Default=1]
       if 1, bedpicking with be carried out automatically
       if 0, user will be prompted to pick the bed location on screen
    flip_lr : int, *optional* [Default=0]
       if 1, port and starboard scans will be flipped
       (for situations where the transducer is flipped 180 degrees)
    model: int, *optional* [Default=998]
       A 3 or 4 number code indicating the model number
       Examples: 998, 997, 1198, 1199
    calc_bearing : float, *optional* [Default=0]
       if 1, bearing will be calculated from coordinates
    filt_bearing : float, *optional* [Default=0]
       if 1, bearing will be filtered
    chunk : str, *optional* [Default='d100' (distance, 100 m)]
       letter, followed by a number.
       There are the following letter options:
       'd' - parse chunks based on distance, then number which is distance in m
       'p' - parse chunks based on number of pings, then number which is number of pings
       'h' - parse chunks based on change in heading, then number which is the change in heading in degrees
       '1' - process just 1 chunk

    Returns
    ---------
    sonpath+base+'_data_port.dat': memory-mapped file
        contains the raw echogram from the port side
        sidescan sonar (where present)

    sonpath+base+'_data_port.dat': memory-mapped file
        contains the raw echogram from the starboard side
        sidescan sonar (where present)

    sonpath+base+'_data_dwnhi.dat': memory-mapped file
        contains the raw echogram from the high-frequency
        echosounder (where present)

    sonpath+base+'_data_dwnlow.dat': memory-mapped file
        contains the raw echogram from the low-frequency
        echosounder (where present)

    sonpath+base+"trackline.kml": google-earth kml file
        contains the trackline of the vessel during data
        acquisition

    sonpath+base+'rawdat.csv': comma separated value file
        contains time-series data. columns corresponding to
        longitude
        latitude
        easting (m)
        northing (m)
        depth to bed (m)
        alongtrack cumulative distance (m)
        vessel heading (deg.)

    sonpath+base+'meta.mat': .mat file
        matlab format file containing a dictionary object
        holding metadata information. Fields are:
        e : ndarray, easting (m)
        n : ndarray, northing (m)
        es : ndarray, low-pass filtered easting (m)
        ns : ndarray, low-pass filtered northing (m)
        lat : ndarray, latitude
        lon : ndarray, longitude
        shape_port : tuple, shape of port scans in memory mapped file
        shape_star : tuple, shape of starboard scans in memory mapped file
        shape_hi : tuple, shape of high-freq. scans in memory mapped file
        shape_low : tuple, shape of low-freq. scans in memory mapped file
        dep_m : ndarray, depth to bed (m)
        dist_m : ndarray, distance along track (m)
        heading : ndarray, heading of vessel (deg. N)
        pix_m: float, size of 1 pixel in across-track dimension (m)
        bed : ndarray, depth to bed (m)
        c : float, speed of sound in water (m/s)
        t : length of sidescan transducer array (m)
        spd : ndarray, vessel speed (m/s)
        time_s : ndarray, time elapsed (s)
        caltime : ndarray, unix epoch time (s)
    '''

    # prompt user to supply file if no input file given
    if not humfile:
        print('An input file is required!!!!!!')
        Tk().withdraw(
        )  # we don't want a full GUI, so keep the root window from appearing
        humfile = askopenfilename(filetypes=[("DAT files", "*.DAT")])

    # prompt user to supply directory if no input sonpath is given
    if not sonpath:
        print('A *.SON directory is required!!!!!!')
        Tk().withdraw(
        )  # we don't want a full GUI, so keep the root window from appearing
        sonpath = askdirectory()

    # print given arguments to screen and convert data type where necessary
    if humfile:
        print('Input file is %s' % (humfile))

    if sonpath:
        print('Son files are in %s' % (sonpath))

    if cs2cs_args:
        print('cs2cs arguments are %s' % (cs2cs_args))

    if draft:
        draft = float(draft)
        print('Draft: %s' % (str(draft)))

    if c:
        c = float(c)
        print('Celerity of sound: %s m/s' % (str(c)))

    if doplot:
        doplot = int(doplot)
        if doplot == 0:
            print("Plots will not be made")

    if flip_lr:
        flip_lr = int(flip_lr)
        if flip_lr == 1:
            print("Port and starboard will be flipped")

    if t:
        t = np.asarray(t, float)
        print('Transducer length is %s m' % (str(t)))

    if bedpick:
        bedpick = np.asarray(bedpick, int)
        if bedpick == 1:
            print('Bed picking is auto')
        elif bedpick == 0:
            print('Bed picking is manual')
        else:
            print('User will be prompted per chunk about bed picking method')

    if chunk:
        chunk = str(chunk)
        if chunk[0] == 'd':
            chunkmode = 1
            chunkval = int(chunk[1:])
            print('Chunks based on distance of %s m' % (str(chunkval)))
        elif chunk[0] == 'p':
            chunkmode = 2
            chunkval = int(chunk[1:])
            print('Chunks based on %s pings' % (str(chunkval)))
        elif chunk[0] == 'h':
            chunkmode = 3
            chunkval = int(chunk[1:])
            print('Chunks based on heading devation of %s degrees' %
                  (str(chunkval)))
        elif chunk[0] == '1':
            chunkmode = 4
            chunkval = 1
            print('Only 1 chunk will be produced')
        else:
            print(
                "Chunk mode not understood - should be 'd', 'p', or 'h' - using defaults"
            )
            chunkmode = 1
            chunkval = 100
            print('Chunks based on distance of %s m' % (str(chunkval)))

    if model:
        try:
            model = int(model)
            print("Data is from the %s series" % (str(model)))
        except:
            if model == 'onix':
                model = 0
                print("Data is from the ONIX series")
            elif model == 'helix':
                model = 1
                print("Data is from the HELIX series")
            elif model == 'mega':
                model = 2
                print("Data is from the MEGA series")
#    if cog:
#       cog = int(cog)
#       if cog==1:
#          print "Heading based on course-over-ground"

    if calc_bearing:
        calc_bearing = int(calc_bearing)
        if calc_bearing == 1:
            print("Bearing will be calculated from coordinates")

    if filt_bearing:
        filt_bearing = int(filt_bearing)
        if filt_bearing == 1:
            print("Bearing will be filtered")

    ## for debugging
    #humfile = r"test.DAT"; sonpath = "test_data"
    #cs2cs_args = "epsg:26949"; doplot = 1; draft = 0
    #c=1450; bedpick=1; fliplr=1; chunk = 'd100'
    #model=998; cog=1; calc_bearing=0; filt_bearing=0

    #if model==2:
    #   f = 1000
    #else:
    f = 455

    try:
        print(
            "Checking the epsg code you have chosen for compatibility with Basemap ... "
        )
        from mpl_toolkits.basemap import Basemap
        m = Basemap(projection='merc',
                    epsg=cs2cs_args.split(':')[1],
                    resolution='i',
                    llcrnrlon=10,
                    llcrnrlat=10,
                    urcrnrlon=30,
                    urcrnrlat=30)
        del m
        print("... epsg code compatible")
    except (ValueError):
        print(
            "Error: the epsg code you have chosen is not compatible with Basemap"
        )
        print(
            "please choose a different epsg code (http://spatialreference.org/)"
        )
        print("program will now close")
        sys.exit()

    # start timer
    if os.name == 'posix':  # true if linux/mac or cygwin on windows
        start = time.time()
    else:  # windows
        start = time.clock()

    # if son path name supplied has no separator at end, put one on
    if sonpath[-1] != os.sep:
        sonpath = sonpath + os.sep

    # get the SON files from this directory
    sonfiles = glob.glob(sonpath + '*.SON')
    if not sonfiles:
        sonfiles = glob.glob(os.getcwd() + os.sep + sonpath + '*.SON')

    base = humfile.split('.DAT')  # get base of file name for output
    base = base[0].split(os.sep)[-1]

    # remove underscores, negatives and spaces from basename
    base = humutils.strip_base(base)

    print("WARNING: Because files have to be read in byte by byte,")
    print("this could take a very long time ...")

    #reading each sonfile in parallel should be faster ...
    try:
        o = Parallel(n_jobs=np.min([len(sonfiles), cpu_count()]), verbose=0)(
            delayed(getscans)(sonfiles[k], humfile, c, model, cs2cs_args)
            for k in range(len(sonfiles)))
        X, Y, A, B = zip(*o)

        for k in range(len(Y)):
            if Y[k] == 'sidescan_port':
                dat = A[k]  #data.gethumdat()
                metadat = B[k]  #data.getmetadata()
                if flip_lr == 0:
                    data_port = X[k].astype('int16')
                else:
                    data_star = X[k].astype('int16')

            elif Y[k] == 'sidescan_starboard':
                if flip_lr == 0:
                    data_star = X[k].astype('int16')
                else:
                    data_port = X[k].astype('int16')

            elif Y[k] == 'down_lowfreq':
                data_dwnlow = X[k].astype('int16')

            elif Y[k] == 'down_highfreq':
                data_dwnhi = X[k].astype('int16')

            elif Y[k] == 'down_vhighfreq':  #hopefully this only applies to mega systems
                data_dwnhi = X[k].astype('int16')

        del X, Y, A, B, o
        old_pyread = 0

        if 'data_port' not in locals():
            data_port = ''
            print("portside scan not available")

        if 'data_star' not in locals():
            data_star = ''
            print("starboardside scan not available")

        if 'data_dwnhi' not in locals():
            data_dwnlow = ''
            print("high-frq. downward scan not available")

        if 'data_dwnlow' not in locals():
            data_dwnlow = ''
            print("low-frq. downward scan not available")

    except:  # revert back to older version if paralleleised version fails

        print(
            "something went wrong with the parallelised version of pyread ...")

        try:
            import pyread
        except:
            from . import pyread

        data = pyread.pyread(sonfiles, humfile, c, model, cs2cs_args)

        dat = data.gethumdat()

        metadat = data.getmetadata()

        old_pyread = 1

    nrec = len(metadat['n'])

    metadat['instr_heading'] = metadat['heading'][:nrec]

    #metadat['heading'] = humutils.get_bearing(calc_bearing, filt_bearing, cog, metadat['lat'], metadat['lon'], metadat['instr_heading'])

    try:
        es = humutils.runningMeanFast(metadat['e'][:nrec],
                                      len(metadat['e'][:nrec]) / 100)
        ns = humutils.runningMeanFast(metadat['n'][:nrec],
                                      len(metadat['n'][:nrec]) / 100)
    except:
        es = metadat['e'][:nrec]
        ns = metadat['n'][:nrec]

    metadat['es'] = es
    metadat['ns'] = ns

    try:
        trans = pyproj.Proj(init=cs2cs_args)
    except:
        trans = pyproj.Proj(cs2cs_args.lstrip(), inverse=True)

    lon, lat = trans(es, ns, inverse=True)
    metadat['lon'] = lon
    metadat['lat'] = lat

    metadat['heading'] = humutils.get_bearing(calc_bearing, filt_bearing,
                                              metadat['lat'], metadat['lon'],
                                              metadat['instr_heading'])  #cog

    dist_m = humutils.get_dist(lat, lon)
    metadat['dist_m'] = dist_m

    if calc_bearing == 1:  # recalculate speed, m/s
        ds = np.gradient(np.squeeze(metadat['time_s']))
        dx = np.gradient(np.squeeze(metadat['dist_m']))
        metadat['spd'] = dx[:nrec] / ds[:nrec]

    # theta at 3dB in the horizontal
    theta3dB = np.arcsin(c / (t * (f * 1000)))
    #resolution of 1 sidescan pixel to nadir
    ft = (np.pi / 2) * (1 / theta3dB)  #/ (f/455)

    dep_m = humutils.get_depth(metadat['dep_m'][:nrec])

    if old_pyread == 1:  #older pyread version

        # port scan
        try:
            if flip_lr == 0:
                data_port = data.getportscans().astype('int16')
            else:
                data_port = data.getstarscans().astype('int16')
        except:
            data_port = ''
            print("portside scan not available")

    if data_port != '':

        Zt, ind_port = makechunks_scan(chunkmode, chunkval, metadat, data_port,
                                       0)
        del data_port

        ## create memory mapped file for Z
        shape_port = io.set_mmap_data(sonpath, base, '_data_port.dat', 'int16',
                                      Zt)

        ##we are only going to access the portion of memory required
        port_fp = io.get_mmap_data(sonpath, base, '_data_port.dat', 'int16',
                                   shape_port)

    if old_pyread == 1:  #older pyread version
        # starboard scan
        try:
            if flip_lr == 0:
                data_star = data.getstarscans().astype('int16')
            else:
                data_star = data.getportscans().astype('int16')
        except:
            data_star = ''
            print("starboardside scan not available")

    if data_star != '':

        Zt, ind_star = makechunks_scan(chunkmode, chunkval, metadat, data_star,
                                       1)
        del data_star

        # create memory mapped file for Z
        shape_star = io.set_mmap_data(sonpath, base, '_data_star.dat', 'int16',
                                      Zt)

        star_fp = io.get_mmap_data(sonpath, base, '_data_star.dat', 'int16',
                                   shape_star)

    if 'star_fp' in locals() and 'port_fp' in locals():
        # check that port and starboard are same size
        # and trim if not
        if np.shape(star_fp) != np.shape(port_fp):
            print(
                "port and starboard scans are different sizes ... rectifying")
            if np.shape(port_fp[0])[1] > np.shape(star_fp[0])[1]:
                tmp = port_fp.copy()
                tmp2 = np.empty_like(star_fp)
                for k in range(len(tmp)):
                    tmp2[k] = tmp[k][:, :np.shape(star_fp[k])[1]]
                del tmp

                # create memory mapped file for Z
                shape_port = io.set_mmap_data(sonpath, base, '_data_port2.dat',
                                              'int16', tmp2)
                #shape_star = shape_port.copy()
                shape_star = tuple(np.asarray(shape_port).copy())

                ##we are only going to access the portion of memory required
                port_fp = io.get_mmap_data(sonpath, base, '_data_port2.dat',
                                           'int16', shape_port)

                ind_port = list(ind_port)
                ind_port[-1] = np.shape(star_fp[0])[1]
                ind_port = tuple(ind_port)

            elif np.shape(port_fp[0])[1] < np.shape(star_fp[0])[1]:
                tmp = star_fp.copy()
                tmp2 = np.empty_like(port_fp)
                for k in range(len(tmp)):
                    tmp2[k] = tmp[k][:, :np.shape(port_fp[k])[1]]
                del tmp

                # create memory mapped file for Z
                shape_port = io.set_mmap_data(sonpath, base, '_data_star2.dat',
                                              'int16', tmp2)
                #shape_star = shape_port.copy()
                shape_star = tuple(np.asarray(shape_port).copy())

                #we are only going to access the portion of memory required
                star_fp = io.get_mmap_data(sonpath, base, '_data_star2.dat',
                                           'int16', shape_star)

                ind_star = list(ind_star)
                ind_star[-1] = np.shape(port_fp[0])[1]
                ind_star = tuple(ind_star)

    if old_pyread == 1:  #older pyread version
        # low-freq. sonar
        try:
            data_dwnlow = data.getlowscans().astype('int16')
        except:
            data_dwnlow = ''
            print("low-freq. scan not available")

    if data_dwnlow != '':

        Zt, ind_low = makechunks_scan(chunkmode, chunkval, metadat,
                                      data_dwnlow, 2)
        del data_dwnlow

        # create memory mapped file for Z
        shape_low = io.set_mmap_data(sonpath, base, '_data_dwnlow.dat',
                                     'int16', Zt)

        ##we are only going to access the portion of memory required
        dwnlow_fp = io.get_mmap_data(sonpath, base, '_data_dwnlow.dat',
                                     'int16', shape_low)

    if old_pyread == 1:  #older pyread version
        # hi-freq. sonar
        try:
            data_dwnhi = data.gethiscans().astype('int16')
        except:
            data_dwnhi = ''
            print("high-freq. scan not available")

    if data_dwnhi != '':

        Zt, ind_hi = makechunks_scan(chunkmode, chunkval, metadat, data_dwnhi,
                                     3)
        del data_dwnhi

        # create memory mapped file for Z
        shape_hi = io.set_mmap_data(sonpath, base, '_data_dwnhi.dat', 'int16',
                                    Zt)

        dwnhi_fp = io.get_mmap_data(sonpath, base, '_data_dwnhi.dat', 'int16',
                                    shape_hi)

    if 'dwnhi_fp' in locals() and 'dwnlow_fp' in locals():
        # check that low and high are same size
        # and trim if not
        if (np.shape(dwnhi_fp) != np.shape(dwnlow_fp)) and (chunkmode != 4):
            print("dwnhi and dwnlow are different sizes ... rectifying")
            if np.shape(dwnhi_fp[0])[1] > np.shape(dwnlow_fp[0])[1]:
                tmp = dwnhi_fp.copy()
                tmp2 = np.empty_like(dwnlow_fp)
                for k in range(len(tmp)):
                    tmp2[k] = tmp[k][:, :np.shape(dwnlow_fp[k])[1]]
                del tmp

                # create memory mapped file for Z
                shape_low = io.set_mmap_data(sonpath, base, '_data_dwnhi2.dat',
                                             'int16', tmp2)
                #shape_hi = shape_low.copy()
                shape_hi = tuple(np.asarray(shape_low).copy())

                ##we are only going to access the portion of memory required
                dwnhi_fp = io.get_mmap_data(sonpath, base, '_data_dwnhi2.dat',
                                            'int16', shape_hi)

                ind_hi = list(ind_hi)
                ind_hi[-1] = np.shape(dwnlow_fp[0])[1]
                ind_hi = tuple(ind_hi)

            elif np.shape(dwnhi_fp[0])[1] < np.shape(dwnlow_fp[0])[1]:
                tmp = dwnlow_fp.copy()
                tmp2 = np.empty_like(dwnhi_fp)
                for k in range(len(tmp)):
                    tmp2[k] = tmp[k][:, :np.shape(dwnhi_fp[k])[1]]
                del tmp

                # create memory mapped file for Z
                shape_low = io.set_mmap_data(sonpath, base,
                                             '_data_dwnlow2.dat', 'int16',
                                             tmp2)
                #shape_hi = shape_low.copy()
                shape_hi = tuple(np.asarray(shape_low).copy())

                ##we are only going to access the portion of memory required
                dwnlow_fp = io.get_mmap_data(sonpath, base,
                                             '_data_dwnlow2.dat', 'int16',
                                             shape_low)

                ind_low = list(ind_low)
                ind_low[-1] = np.shape(dwnhi_fp[0])[1]
                ind_low = tuple(ind_low)

    if old_pyread == 1:  #older pyread version
        del data

    if ('shape_port' in locals()) and (chunkmode != 4):
        metadat['shape_port'] = shape_port
        nrec = metadat['shape_port'][0] * metadat['shape_port'][2]
    elif ('shape_port' in locals()) and (chunkmode == 4):
        metadat['shape_port'] = shape_port
        nrec = metadat['shape_port'][1]
    else:
        metadat['shape_port'] = ''

    if ('shape_star' in locals()) and (chunkmode != 4):
        metadat['shape_star'] = shape_star
        nrec = metadat['shape_star'][0] * metadat['shape_star'][2]
    elif ('shape_star' in locals()) and (chunkmode == 4):
        metadat['shape_star'] = shape_star
        nrec = metadat['shape_star'][1]
    else:
        metadat['shape_star'] = ''

    if ('shape_hi' in locals()) and (chunkmode != 4):
        metadat['shape_hi'] = shape_hi
        #nrec = metadat['shape_hi'][0] * metadat['shape_hi'][2] * 2
    elif ('shape_hi' in locals()) and (chunkmode == 4):
        metadat['shape_hi'] = shape_hi
    else:
        metadat['shape_hi'] = ''

    if ('shape_low' in locals()) and (chunkmode != 4):
        metadat['shape_low'] = shape_low
        #nrec = metadat['shape_low'][0] * metadat['shape_low'][2] * 2
    elif ('shape_low' in locals()) and (chunkmode == 4):
        metadat['shape_low'] = shape_low
    else:
        metadat['shape_low'] = ''

    #make kml boat trackline
    humutils.make_trackline(lon, lat, sonpath, base)

    if 'port_fp' in locals() and 'star_fp' in locals():

        #if not os.path.isfile(os.path.normpath(os.path.join(sonpath,base+'meta.mat'))):
        if 2 > 1:
            if bedpick == 1:  # auto

                x, bed = humutils.auto_bedpick(ft, dep_m, chunkmode, port_fp,
                                               c)

                if len(dist_m) < len(bed):
                    dist_m = np.append(
                        dist_m, dist_m[-1] * np.ones(len(bed) - len(dist_m)))

                if doplot == 1:
                    if chunkmode != 4:
                        for k in range(len(star_fp)):
                            plot_2bedpicks(
                                port_fp[k], star_fp[k],
                                bed[ind_port[-1] * k:ind_port[-1] * (k + 1)],
                                dist_m[ind_port[-1] * k:ind_port[-1] *
                                       (k + 1)],
                                x[ind_port[-1] * k:ind_port[-1] * (k + 1)], ft,
                                shape_port, sonpath, k, chunkmode)
                    else:
                        plot_2bedpicks(port_fp, star_fp, bed, dist_m, x, ft,
                                       shape_port, sonpath, 0, chunkmode)

                # 'real' bed is estimated to be the minimum of the two
                bed = np.min(np.vstack((bed[:nrec], np.squeeze(x[:nrec]))),
                             axis=0)
                bed = humutils.runningMeanFast(bed, 3)

            elif bedpick > 1:  # user prompt

                x, bed = humutils.auto_bedpick(ft, dep_m, chunkmode, port_fp,
                                               c)

                if len(dist_m) < len(bed):
                    dist_m = np.append(
                        dist_m, dist_m[-1] * np.ones(len(bed) - len(dist_m)))

                # 'real' bed is estimated to be the minimum of the two
                bed = np.min(np.vstack((bed[:nrec], np.squeeze(x[:nrec]))),
                             axis=0)
                bed = humutils.runningMeanFast(bed, 3)

                # manually intervene
                fig = plt.figure()
                ax = plt.gca()
                if chunkmode != 4:
                    im = ax.imshow(np.hstack(port_fp),
                                   cmap='gray',
                                   origin='upper')
                else:
                    im = ax.imshow(port_fp, cmap='gray', origin='upper')
                plt.plot(bed, 'r')
                plt.axis('normal')
                plt.axis('tight')

                pts1 = plt.ginput(
                    n=300,
                    timeout=30)  # it will wait for 200 clicks or 60 seconds
                x1 = map(lambda x: x[0],
                         pts1)  # map applies the function passed as
                y1 = map(lambda x: x[1],
                         pts1)  # first parameter to each element of pts
                plt.close()
                del fig

                if x1 != []:  # if x1 is not empty
                    tree = KDTree(zip(np.arange(1, len(bed)), bed))
                    try:
                        dist, inds = tree.query(zip(x1, y1),
                                                k=100,
                                                eps=5,
                                                n_jobs=-1)
                    except:
                        dist, inds = tree.query(zip(x1, y1), k=100, eps=5)

                    b = np.interp(inds, x1, y1)
                    bed2 = bed.copy()
                    bed2[inds] = b
                    bed = bed2

                if doplot == 1:
                    if chunkmode != 4:
                        for k in range(len(star_fp)):
                            plot_2bedpicks(
                                port_fp[k], star_fp[k],
                                bed[ind_port[-1] * k:ind_port[-1] * (k + 1)],
                                dist_m[ind_port[-1] * k:ind_port[-1] *
                                       (k + 1)],
                                x[ind_port[-1] * k:ind_port[-1] * (k + 1)], ft,
                                shape_port, sonpath, k, chunkmode)
                    else:
                        plot_2bedpicks(port_fp, star_fp, bed, dist_m, x, ft,
                                       shape_port, sonpath, 0, chunkmode)

            else:  #manual

                beds = []

                if chunkmode != 4:
                    for k in range(len(port_fp)):
                        raw_input(
                            "Bed picking " + str(k + 1) + " of " +
                            str(len(port_fp)) +
                            ", are you ready? 30 seconds. Press Enter to continue..."
                        )
                        bed = {}
                        fig = plt.figure()
                        ax = plt.gca()
                        im = ax.imshow(port_fp[k], cmap='gray', origin='upper')
                        pts1 = plt.ginput(
                            n=300, timeout=30
                        )  # it will wait for 200 clicks or 60 seconds
                        x1 = map(lambda x: x[0],
                                 pts1)  # map applies the function passed as
                        y1 = map(
                            lambda x: x[1],
                            pts1)  # first parameter to each element of pts
                        bed = np.interp(np.r_[:ind_port[-1]], x1, y1)
                        plt.close()
                        del fig
                        beds.append(bed)
                        extent = np.shape(port_fp[k])[0]
                    bed = np.asarray(np.hstack(beds), 'float')
                else:
                    raw_input(
                        "Bed picking - are you ready? 30 seconds. Press Enter to continue..."
                    )
                    bed = {}
                    fig = plt.figure()
                    ax = plt.gca()
                    im = ax.imshow(port_fp, cmap='gray', origin='upper')
                    pts1 = plt.ginput(
                        n=300, timeout=30
                    )  # it will wait for 200 clicks or 60 seconds
                    x1 = map(lambda x: x[0],
                             pts1)  # map applies the function passed as
                    y1 = map(lambda x: x[1],
                             pts1)  # first parameter to each element of pts
                    bed = np.interp(np.r_[:ind_port[-1]], x1, y1)
                    plt.close()
                    del fig
                    beds.append(bed)
                    extent = np.shape(port_fp)[1]
                    bed = np.asarray(np.hstack(beds), 'float')

            # now revise the depth in metres
            dep_m = (1 / ft) * bed

            if doplot == 1:
                if chunkmode != 4:
                    for k in range(len(star_fp)):
                        plot_bedpick(
                            port_fp[k], star_fp[k], (1 / ft) *
                            bed[ind_port[-1] * k:ind_port[-1] * (k + 1)],
                            dist_m[ind_port[-1] * k:ind_port[-1] * (k + 1)],
                            ft, shape_port, sonpath, k, chunkmode)
                else:
                    plot_bedpick(port_fp, star_fp, (1 / ft) * bed, dist_m, ft,
                                 shape_port, sonpath, 0, chunkmode)

            metadat['bed'] = bed[:nrec]

    else:
        metadat['bed'] = dep_m[:nrec] * ft

    metadat['heading'] = metadat['heading'][:nrec]
    metadat['lon'] = lon[:nrec]
    metadat['lat'] = lat[:nrec]
    metadat['dist_m'] = dist_m[:nrec]
    metadat['dep_m'] = dep_m[:nrec]
    metadat['pix_m'] = 1 / ft
    metadat['bed'] = metadat['bed'][:nrec]
    metadat['c'] = c
    metadat['t'] = t
    if model == 2:
        metadat['f'] = f * 2
    else:
        metadat['f'] = f

    metadat['spd'] = metadat['spd'][:nrec]
    metadat['time_s'] = metadat['time_s'][:nrec]
    metadat['e'] = metadat['e'][:nrec]
    metadat['n'] = metadat['n'][:nrec]
    metadat['es'] = metadat['es'][:nrec]
    metadat['ns'] = metadat['ns'][:nrec]
    try:
        metadat['caltime'] = metadat['caltime'][:nrec]
    except:
        metadat['caltime'] = metadat['caltime']

    savemat(os.path.normpath(os.path.join(sonpath, base + 'meta.mat')),
            metadat,
            oned_as='row')

    f = open(os.path.normpath(os.path.join(sonpath, base + 'rawdat.csv')),
             'wt')
    writer = csv.writer(f)
    writer.writerow(
        ('longitude', 'latitude', 'easting', 'northing', 'depth (m)',
         'distance (m)', 'instr. heading (deg)', 'heading (deg.)'))
    for i in range(0, nrec):
        writer.writerow(
            (float(lon[i]), float(lat[i]), float(es[i]), float(ns[i]),
             float(dep_m[i]), float(dist_m[i]),
             float(metadat['instr_heading'][i]), float(metadat['heading'][i])))
    f.close()

    del lat, lon, dep_m  #, dist_m

    if doplot == 1:

        plot_pos(sonpath, metadat, es, ns)

        if 'dwnlow_fp' in locals():

            plot_dwnlow(dwnlow_fp, chunkmode, sonpath)

        if 'dwnhi_fp' in locals():

            plot_dwnhi(dwnhi_fp, chunkmode, sonpath)

    if os.name == 'posix':  # true if linux/mac
        elapsed = (time.time() - start)
    else:  # windows
        elapsed = (time.clock() - start)
    print("Processing took " + str(elapsed) + "seconds to analyse")

    print("Done!")
    print("===================================================")
           else path+'\\GoogleDistanceCalculator\\data_files\\'

filename = 'input_origins_and_destinations.csv'

final_path = filepath + filename
input_df = pd.read_csv(final_path)

output_df = pd.DataFrame(columns=['S.No.', 'origin', 'destination', 'distance', 'raw_response'])

url_endpoint = 'https://maps.googleapis.com/maps/api/distancematrix/json'
fields = ['S. No', 'origin', 'destination', 'distance', 'raw_response' ]
error_message = ''
distance = ''

with open(filepath + 'output_distances.csv','w', newline='') as newFile:
    newFileWriter = csv.writer(newFile)
    newFileWriter.writerow(['S.No.', 'origin', 'destination', 'distance', 'raw_response', 'Error' ])

    for index, row in input_df.iterrows():
        mydict = {'units'       : measurement_unit,
                  'origins'     : row['origin_address'],
                  'destinations': row['destination_address'],
                  'key'         : google_key}
        error_message = 'No Error'
        try:
            resp = requests.get(url_endpoint, params=mydict)
            if(resp.ok):
                jData = json.loads(resp.text)
                distance = jData['rows'][0]['elements'][0]['distance']['text']
        except Exception as e:
                error_message = 'Error getting distance.'
    if user in user_to_uninstall_times:
        is_alive = 0
    else:
        is_alive = 1
    #if time_since_install/ 8.64e+7 <= 7:
    #    continue
    if users_to_conditions_in_experiment_by_name[user] == 'off':
        not_suggested.append([time_since_install/8.64e+7, int(is_alive)])
        intervention_suggestion.append([time_since_install/ 8.64e+7, int(is_alive), 0])
    else:
        suggested.append([time_since_install/8.64e+7, int(is_alive), 1])
        intervention_suggestion.append([time_since_install/ 8.64e+7, int(is_alive), 1])

import csv
with open('time_to_is_alive_suggestion.csv', 'w', newline='') as csvfile:
    spamwriter = csv.writer(csvfile, delimiter=',', quoting=csv.QUOTE_MINIMAL)
    spamwriter.writerow(["days", "isAlive"])
    for i in suggested:
        spamwriter.writerow(i)
with open('time_to_is_alive_no_suggestion.csv', 'w', newline='') as csvfile:
    spamwriter = csv.writer(csvfile, delimiter=',', quoting=csv.QUOTE_MINIMAL)
    spamwriter.writerow(["days", "isAlive"])
    for i in not_suggested:
        spamwriter.writerow(i)
with open('intervention_suggestion.csv', 'w', newline='') as csvfile:
    spamwriter = csv.writer(csvfile, delimiter=',', quoting=csv.QUOTE_MINIMAL)
    spamwriter.writerow(["days", "isAlive", "isShownSuggesion"])
    for i in intervention_suggestion:
        spamwriter.writerow(i)

'''
def write_to_csv_file(pred_Y, predicted_Y_file_name):
    pred_Y = pred_Y.reshape(len(pred_Y), 1)
    with open(predicted_Y_file_name, 'w', newline='') as csv_file:
        wr = csv.writer(csv_file)
        wr.writerows(pred_Y)
        csv_file.close()
def get_device_list(ip_address, user_name, password, identitypool_id, outfile):
    """ Authenticate with OME and enumerate devices """
    try:
        base_uri = 'https://%s' % (ip_address)
        next_link_url = None
        headers = {'content-type': 'application/json'}

        auth_token = get_session(ip_address, user_name, password)
        if auth_token != None:
            headers['X-Auth-Token'] = auth_token['token']

            # Display Identity Pools
            identitypool_url = base_uri + '/api/IdentityPoolService/IdentityPools'
            identitypool_response = requests.get(identitypool_url,
                                                 headers=headers,
                                                 verify=False)
            if identitypool_response.status_code == 200:
                identitypool_data = identitypool_response.json()
                identitypool_data = identitypool_data['value']
                for i in identitypool_data:
                    print("Id: %s, Name: %s" % (i["Id"], i["Name"]))
            else:
                print("Unable to retrieve device list from %s" % (ip_address))

            if identitypool_id == None:
                identitypool_id_prompt = input(
                    "Please Enter Identity Pool Id: ")
                identitypool_id = identitypool_id_prompt

            # Get Identity Pool Usage Sets
            identitypool_usageset_url = base_uri + "/api/IdentityPoolService/IdentityPools(%s)/UsageIdentitySets" % (
                identitypool_id)
            identitypool_usageset_response = requests.get(
                identitypool_usageset_url, headers=headers, verify=False)
            if identitypool_usageset_response.status_code == 200:
                identitypool_usageset_data = identitypool_usageset_response.json(
                )
                identitypool_usageset_data = identitypool_usageset_data[
                    'value']

                detailentries = []
                for usageset in identitypool_usageset_data:
                    usageset_id = usageset["IdentitySetId"]
                    usageset_name = usageset["Name"]

                    identitypool_usageset_detail_url = base_uri + "/api/IdentityPoolService/IdentityPools(%s)/UsageIdentitySets(%s)/Details" % (
                        identitypool_id, usageset_id)
                    identitypool_usageset_detail_response = requests.get(
                        identitypool_usageset_detail_url,
                        headers=headers,
                        verify=False)
                    if identitypool_usageset_detail_response.status_code == 200:
                        identitypool_usageset_detail_json = identitypool_usageset_detail_response.json(
                        )
                        identitypool_usageset_detail_data = identitypool_usageset_detail_json[
                            'value']
                        for detailentry in identitypool_usageset_detail_data:
                            entry = {
                                "IdentityType":
                                usageset_name,
                                "ChassisName":
                                detailentry["DeviceInfo"]["ChassisName"],
                                "ServerName":
                                detailentry["DeviceInfo"]["ServerName"],
                                "ManagementIp":
                                detailentry["DeviceInfo"]["ManagementIp"],
                                "NicIdentifier":
                                detailentry["NicIdentifier"],
                                "MacAddress":
                                detailentry["MacAddress"]
                            }
                            detailentries.append(entry)

                        if '@odata.nextLink' in identitypool_usageset_detail_json:
                            next_link_url = base_uri + identitypool_usageset_detail_json[
                                '@odata.nextLink']
                            while next_link_url:
                                next_link_response = requests.get(
                                    next_link_url,
                                    headers=headers,
                                    verify=False)
                                if next_link_response.status_code == 200:
                                    next_link_json = next_link_response.json()
                                    next_link_json_data += next_link_json[
                                        'value']
                                    for detailentry in next_link_json_data:
                                        entry = {
                                            "IdentityType":
                                            usageset_name,
                                            "ChassisName":
                                            detailentry["DeviceInfo"]
                                            ["ChassisName"],
                                            "ServerName":
                                            detailentry["DeviceInfo"]
                                            ["ServerName"],
                                            "ManagementIp":
                                            detailentry["DeviceInfo"]
                                            ["ManagementIp"],
                                            "NicIdentifier":
                                            detailentry["NicIdentifier"],
                                            "MacAddress":
                                            detailentry["MacAddress"]
                                        }
                                        detailentries.append(entry)

                                    if '@odata.nextLink' in next_link_json:
                                        next_link_url = base_uri + next_link_json[
                                            '@odata.nextLink']
                                    else:
                                        next_link_url = None
                                else:
                                    next_link_url = None
                                    print(
                                        "Unable to retrieve items from nextLink %s"
                                        % (next_link_url))

                # Export results to CSV
                if (len(detailentries) > 0):
                    currentDirectory = os.path.dirname(
                        os.path.abspath(__file__))
                    if outfile == None:
                        outFilePath = currentDirectory + os.path.sep + "IdentityPoolUsage.csv"
                    else:
                        outFilePath = outfile
                    outputFile = open(outFilePath, 'w')
                    output = csv.writer(outputFile)
                    output.writerow(detailentries[0].keys())
                    for row in detailentries:
                        output.writerow(row.values())
                    print("Exported data to %s" % (outFilePath))
                    outputFile.close()
                else:
                    print("No data to display")
            else:
                print("Unable to retrieve list from %s" % (ip_address))
        else:
            print("Unable to create a session with appliance %s" %
                  (ip_address))
    except:
        print("Unexpected error:", sys.exc_info())
    finally:
        delete_session(ip_address, headers, auth_token['id'])
#!/usr/bin/env python3

import csv
import sys

input_file = sys.argv[1]
output_file = sys.argv[2]

with open(input_file, 'r', newline='') as csv_in_file:
    with open(output_file, 'w', newline='') as csv_out_file:
        filereader = csv.reader(csv_in_file)
        filewriter = csv.writer(csv_out_file)
        header = next(filereader)
        filewriter.writerow(header)
        for row_list in filereader:
            supplier = str(row_list[0]).strip()
            cost = str(row_list[3]).strip('$').replace(',', '')
            if supplier == 'Supplier Z' or float(cost) > 600.0:
                filewriter.writerow(row_list)
示例#57
0
import os, csv
cand_solution = {}
for i in os.listdir('.'):
 if i[0] in ['1','2','3','4','5','6','7','8','9']:
  try:
   cand_solution[i.replace('.txt','')] = open(i).read().replace(" ", "").replace("\n", "")
  except:
   pass
w = csv.writer(open("solution.csv", "w"))
for key, val in cand_solution.items():
    w.writerow([key, val])
示例#58
0
        ])
    output = [x for x in output if x[3] != 0]
    return output


with open(filename) as csvfile:
    allcontent = list(csv.reader(csvfile))
    header = allcontent[0]
    noheader = allcontent[1:len(allcontent)]
    for i in range(file_length // N):
        run_events_list = [header] + noheader[N * (i + 1) - (N):N * (i + 1)]
        stim_presentation_onset = makeStimPresentationOnset(run_events_list)
        response_onset = makeResponseOnset(run_events_list)
        run_onsets = stim_presentation_onset + response_onset
        run_onsets.sort()
        run_onsets_header = [
            "onset", "duration", "trial_type", "response", "stimulus",
            "response_time", "points_earned", "iti_start_time", "iti_length"
        ]
        with open(
                '/oak/stanford/groups/russpold/data/ds000054/0.0.4/sub-' +
                sub_dir_name + '/func/sub-' + sub_id + '_task-' + task_name +
                '_run-00' + str(i + 1) + '_events.tsv', 'w') as tsvfile:
            writer = csv.writer(tsvfile,
                                quoting=csv.QUOTE_NONE,
                                delimiter='\t',
                                quotechar='')
            writer.writerow(run_onsets_header)
            writer.writerows(run_onsets)
            print("Saved events for sub-%s run-%s" % (sub_id, str(i + 1)))
    companyname = data[1].find('span', attrs={'class':'company-name'}).getText()    
    description = company.replace(companyname, '')
    
    # remove unwanted characters
    sales = sales.strip('*').strip('†').replace(',','')
    
    # go to link and extract company website
    url = data[1].find('a').get('href')
    page = urllib.request.urlopen(url)
    # parse the html using beautiful soup and store in variable 'soup'
    soup = BeautifulSoup(page, 'html.parser')
    # find the last result in the table and get the link
    try:
        tableRow = soup.find('table').find_all('tr')[-1]
        webpage = tableRow.find('a').get('href')
    except:
        webpage = None
    
    # write each result to rows
    rows.append([rank, companyname, webpage, description, location, yearend, salesrise, sales, staff, comments])


print(rows)

    
## Create csv and write rows to output file
with open('techtrack100.csv','w', newline='') as f_output:
    csv_output = csv.writer(f_output)
    csv_output.writerows(rows)

            a = list(map(int, i.split('-')))
            exp = sum(a)/len(a)
            print ('exp ',exp)
            lst_exp.insert(0, exp)
            count1 += 1
        if 'to' in i:
            a = list(map(int, i.split('to')))
            sal = sum(a)/len(a)
            print ('sal ',sal)
            lst_sal.insert(0, sal)
            count2 += 1
        
# zip(lst_exp,lst_sal)   

with open('text.csv', 'w') as f:
    writer = csv.writer(f, delimiter='\t')
    writer.writerows(zip(lst_exp,lst_sal))     
        
        
    

# print (lst_exp)
# print (lst_sal)
        
        
            
#            count2 += 1
            
# print (count1)
# print (count2)