Exemple #1
0
                def showmapItemNewyork(self):

                    predictions = []

                    with open('venv/include/predictionTokyoUser.csv', 'r') as read_obj:
                        csv_reader = reader(read_obj)
                        for row in csv_reader:
                            predictions.append(row)

                    map_newyork = folium.Map(location=[35.70154378,139.7433847, ], zoom_start=11)
                    max=-5.0
                    max2=-5.0
                    max3=-5.0
                    for j in predictions:
                        if (searchedUser == j[0] ):
                            if (float(j[3]) > float(max)):
                                max3=max2
                                max2=max
                                max = j[3]
                            elif (float(j[3]) > float(max2)):
                                max3 = max2
                                max2 = j[3]
                            elif (float(j[3]) > float(max3)):
                                max3 = j[3]

                    for j in predictions:

                        if (searchedUser == j[0]):
                            if(j[3]==max or j[3]==max2 or j[3]==max3):

                                color='green'

                            elif(float(j[3])<0):
                                color='red'
                            else:
                                color='blue'

                            latitude=0
                            longitude=0
                            with open('venv/include/TokyoCategory.csv', 'r') as read_obj:
                                predictionCsv_reader = reader(read_obj)
                                for row in predictionCsv_reader:
                                    if (j[1] == row[0]):
                                        latitude =  row[2]
                                        longitude =  row[3]


                                        folium.CircleMarker([latitude, longitude],popup=str("Location name:"+row[1])+"\nprediction:"+str(round(float(j[3]),3)),radius=5,color=color,fill = True).add_to(map_newyork)
                                        break
                    map_newyork.save('tokyoUser.html')
Exemple #2
0
def fill_degree_table():
    with open("degree/data/degree-info.csv") as degree_names:
        d = _csv.reader(degree_names, delimiter=";")
        degree_names = []
        for r in d:
            degree_names.append(r)
        with open('degree/data/program-reqs-table.csv') as degree_info:
            degrees = _csv.reader(degree_info)
            global current_code, current_name, requirements
            current_code = ""
            current_name = ""
            requirements = {}
            first_pass = True
            global year
            year = 0
            row_no = 0
            for row in degrees:

                code = row[0].lstrip()
                data = row[1].lstrip()
                if not (code.strip() == "" or code.strip() == "Code"):
                    if first_pass:
                        current_name = degree_names[row_no]
                        current_code = code
                        first_pass = False
                        continue
                    else:
                        row_no += 1
                    final_reqs = prepare_reqs(requirements)
                    name = lookup_name(current_code, degree_names)
                    d = Degree(code=current_code,
                               name=name,
                               requirements=final_reqs)
                    d.save()
                    year = 0
                    current_code = code
                    requirements = {}
                    continue

                if (year > 0 and not (data.strip() == "")
                        and not "units" in data
                        and not ("year" in data or "Year" in data)):
                    requirements[year].append(data)

                if ("Year" in data[0:10]) or ("year" in data[0:10]):
                    l = [int(data) for data in data.split() if data.isdigit()]

                    if not (l == []):
                        year += 1
                    requirements[year] = []
Exemple #3
0
 def __init__(self, f, fieldnames = None, restkey = None, restval = None, dialect = 'excel', *args, **kwds):
     self._fieldnames = fieldnames
     self.restkey = restkey
     self.restval = restval
     self.reader = reader(f, dialect, *args, **kwds)
     self.dialect = dialect
     self.line_num = 0
Exemple #4
0
def single_csv_read(read_start, csv_file, single_round_number):
    print 'begining new round'
    data = [['0' for i in range(0, 4)] for j in range(0, single_round_number)]
    index = 0
    temp_index = 0
    with open(csv_file, 'rb') as csvfile:
        csvreader = _csv.reader(csvfile, delimiter=' ', quotechar='|')
        for row in csvreader:
            if row[0] == '"SERIAL_NUMBER","EXEC_TIME","TRADE_TYPE_CODE"':
                continue
            temp_index = temp_index + 1
            if temp_index < read_start:
                continue
            if temp_index > read_start + single_round_number:
                break
            #data[][0]  user_id,day,hour,type
            type = row[2][3:-1]
            if index < single_round_number:
                data[index][0] = str(row[0].split(',')[0][1:-1])  #UserId
                data[index][1] = str(
                    row[0].split(',')[1][1:])  # 2015-02-26  Day
                data[index][2] = row[1]  # '11:16:03' hour
                data[index][3] = row[2][3:-1]  # 7220 type
                index = index + 1
    sum_gap(data, single_round_number)
 def __init__(self, f, fieldnames=None, restkey=None, restval=None, dialect='excel', *args, **kwds):
     self._fieldnames = fieldnames
     self.restkey = restkey
     self.restval = restval
     self.reader = reader(f, dialect, *args, **kwds)
     self.dialect = dialect
     self.line_num = 0
Exemple #6
0
 def test_escaped_char_quotes(self):
     import _csv
     from io import StringIO
     r = _csv.reader(StringIO('a\\\nb,c\n'),
                     quoting=_csv.QUOTE_NONE,
                     escapechar='\\')
     assert next(r) == ['a\nb', 'c']
Exemple #7
0
def read_comma_separated_file(_file_name, val=None):
    """Read a named csv file and return the values

    This is used within the sort_columns() and sort_rows() functions for reading
    from a fixed.txt file for fixed position rows/columns within the sorting.
    
    """

    from logger import logger

    import _csv
    global _f
    logger("_file_name: ", str(_file_name))
    try:
        _f = _csv.reader(open(_file_name, "r"))
        for row in _f:
            if val is not None:
                if (row[0] == val):
                    return row[1:]
            else:
                return row

    except:
        logger("_file_name was not found: " + str(_file_name))
        _file_name = None
        return
Exemple #8
0
def personas_carga(request):

    prompt = {
        'Orden':
        'El csv debe seguir el siguiente orden: RUT, nombre y telefono',
    }
    if request.method == 'GET':
        return render(request, "uploadfiles/personas_cargar.html", prompt)

    csv_file = request.FILES['file']

    if not csv_file.name.endswith('.csv'):
        messages.error(request, 'Solo se aceptan archivos .CSV')

    data_set = csv_file.read().decode('UTF-8')
    io_string = io.StringIO(data_set)
    next(io_string)
    for column in _csv.reader(io_string, delimiter=',', quotechar='|'):
        created = Persona.objects.create(nombre=column[0],
                                         paterno=column[1],
                                         materno=column[2])
        print(column[0])
    context = {}

    return render(request, "uploadfiles/personas_cargar.html", context)
Exemple #9
0
def read_transformed_log_file(log_file_path):

    log_list = []

    with open(log_file_path) as input:

        line_list = []

        for line in input.readlines():
            if line.startswith("#"):
                continue

            line_list.append(line.strip())


        for csv_line in reader(line_list, delimiter=" "):

            #Fields:  date time s-sitename s-computername s-ip cs-method cs-uri-stem cs-uri-query s-port cs-username c-ip cs-version cs(User-Agent) cs(Cookie) cs(Referer) cs(Host) sc-status sc-substatus sc-win32-status sc-bytes cs-bytes TimeTakenMS

            log = Log(date=csv_line[0], time=csv_line[1], s_sitename=csv_line[2], s_computername=csv_line[3], s_ip=csv_line[4], cs_method=csv_line[5], cs_uri=csv_line[6], s_port=csv_line[8],
                                cs_username=csv_line[9], c_ip=csv_line[10], cs_version=csv_line[11], cs_user_agent=csv_line[12], cs_cookie=csv_line[13], cs_referer=csv_line[14], cs_host=csv_line[15], sc_status=csv_line[16], sc_substatus=csv_line[17],
                                sc_win32_status=csv_line[18], sc_bytes=csv_line[19], cs_bytes=csv_line[20], time_taken_ms=csv_line[21])

            log_list.append(log)

    print len(log_list)

    return log_list
Exemple #10
0
def create_posts_from_file(user, posts_file: str) -> (bool, str):
    """ Метод создания постов из файла

    :param user: Пользователь автор поста
    :param posts_file: Файл со списком постов.Разделитель - значение POSTS_FILE_DELIMITER из настроек.
     Формат файла <Заголовок поста><Содержание><Дата публикации>.Дата публикации в формате hh:mi:ss dd.mm.yyyy'
    :return: Флаг успешности, Сообщение.
    """
    try:
        posts_file_str = posts_file.read().decode('utf-8').split('\n')
        csv_reader = reader(posts_file_str,
                            delimiter=POSTS_FILE_DELIMITER,
                            quotechar='"')
        posts = []
        post_counter = 0
        for row in csv_reader:
            check_post_title(row[0])
            check_post_content(row[1])
            post = Post(post_author=user,
                        post_title=row[0],
                        post_content=row[1],
                        publication_date=datetime.strptime(
                            row[2], DATETIME_FORMAT_FOR_DATETIME))
            posts.append(post)
            post_counter += 1
        save_posts_in_post_list(posts)
        return True, _('The file was processed successfully.'
                       ' Posted by %(post_counter)s posts') % {
                           'post_counter': post_counter
                       }
    except UnicodeDecodeError:
        return False, _(
            'Posts not created. Error reading file. The file must be text,'
            ' the delimiter of values is %(delimiter)s') % {
                'delimiter': POSTS_FILE_DELIMITER
            }
    except IndexError:
        return False, _(
            'No posts have been created. Not all values are specified '
            'in line %(line)s') % {
                'line': post_counter + 1
            }
    except ValueError:
        return False, _('No posts have been created. Incorrect date value '
                        'in line %(line)s') % {
                            'line': post_counter + 1
                        }
    except PostTitleNullException:
        return False, _('No posts have been created. Empty post title value'
                        ' in line %(line)s') % {
                            'line': post_counter + 1
                        }
    except PostContentNullException:
        return False, _('No posts have been created. Empty post content value'
                        ' in line %(line)s') % {
                            'line': post_counter + 1
                        }
    except Exception:
        return False, _('An unexpected error has occurred')
Exemple #11
0
	def __init__(self, data_filename):
		""" load csv data into a dict of lists """		
		with open(data_filename) as f:
			rows = reader(f)
			#	headers will be the dict keys
			headers = next(rows)[1:] #	omit the first column, which is names
			#	the dict values will be lists of the column values
			super().__init__(zip(headers, zip(*(self.process_row(row, headers) for row in rows))))
def GetCsvVal2(name):
    global f
    f = _csv.reader(open("c:/temp/Spend.txt", "r"))
    for row in f:
        if(row[0] == name):
            return row[1]
        
    return "0"
Exemple #13
0
def parse_csv(csvfile, dlmtr):
    spamreader = None
    try:
        spamreader = csv.reader(csvfile, delimiter=dlmtr, quotechar='"')
        return spamreader
    except csv.Error as e:
        print_err('file {}, line {}: {}'.format(filename, reader.line_num, e))
        sys.exit(4)
Exemple #14
0
 def __init__(self, f, fieldnames=None, restkey=None, restval=None,
              dialect="excel", *args, **kwds):
     self._fieldnames = fieldnames   # list of keys for the dict
     self.restkey = restkey          # key to catch long rows
     self.restval = restval          # default value for short rows
     self.reader = reader(f, dialect, *args, **kwds)
     self.dialect = dialect
     self.line_num = 0
def GetCsvVal(file, name):
    global f
    f = _csv.reader(open(file, "r"))
    for row in f:
        if(row[0] == name):
            return row[1:]
        
    return "0"
 def __init__(self, f, fieldnames=None, restkey=None, restval=None,
              dialect="excel", *args, **kwds):
     self._fieldnames = fieldnames   # list of keys for the dict
     self.restkey = restkey          # key to catch long rows
     self.restval = restval          # default value for short rows
     self.reader = reader(f, dialect, *args, **kwds)
     self.dialect = dialect
     self.line_num = 0
Exemple #17
0
def lanzarBots(target):
    with open('../resources/botInstagramInfo.csv', 'r') as read_obj:
        csv_reader = reader(read_obj)
        header = next(csv_reader)

        if header is not None:
            for row in csv_reader:
                followAccount(row[0], row[1], target)
Exemple #18
0
def load_csv(filename):
    dataset = list()
    with open(filename, 'r') as file:
        csv_reader = reader(file)
        for row in csv_reader:
            if not row:
                continue
            dataset.append(row)
    return dataset
Exemple #19
0
def load_dataset(filename):
    dataset = list()
    with open(filename, 'r') as file:
        readerDataset = reader(file)
        for row in readerDataset:
            if not row:
                continue
            dataset.append(row)
    return np.array(dataset)
def nxpg(names, edges):
    """
    calculating pagerank using built-in function from networkx
    """
    with open(names, "r") as filein:
        csv_reader = reader(filein)
        names = list(map(tuple, csv_reader))[1:]
    names = dict(names)
    nodes = [str(node) for node in range(0, len(names))]

    with open(edges, "r") as filein:
        csv_reader = reader(filein)
        edges = list(map(tuple, csv_reader))[1:]
    G = nx.Graph()
    G.add_nodes_from(nodes)
    for el in edges:
        G.add_edge(el[0], el[1])

    return nx.pagerank(G, alpha=0.85)
Exemple #21
0
    def __init__(self, filename=None):
        """
        Constructor
        """
        self.__filename = filename
        self.__file = sys.stdin if self.__filename is None else open(
            self.__filename, "r")

        self.__reader = _csv.reader(self.__file)
        self.__header = next(self.__reader)
Exemple #22
0
 def load_csv(fileN):
     dataset = list()
     with open(r'D:\papers\Project Material\new_try\CQA\forum\revised.csv',
               'r') as file:
         csv_reader = reader(file)
         for row in csv_reader:
             if not row:
                 continue
             dataset.append(row)
     return dataset
Exemple #23
0
 def responses_to_csv(responses_string):
     """
     :param responses_string:
     :return:
     """
     response_reader = reader(StringIO(responses_string), delimiter=',')
     responses_list = []
     for row in response_reader:
         responses_list.append(row)
     return responses_list
Exemple #24
0
def dataExport():
    with open('C://Users/Alec/Desktop/Video_Data_Export.csv', 'rb') as columns:
        read = csv.reader(columns)
        position_list = []
        prev_row = []
        current_row = []
        vehicle_number = 1

        # Add code for first frame of data
        first_row = filter(None, read.next())
        num_cars = (len(first_row) - 2) / 2
        for k in range(0, num_cars):
            position_list.append([
                vehicle_number, first_row[1], first_row[vehicle_number * 2],
                first_row[vehicle_number * 2 + 1]
            ])
            vehicle_number += 1
        prev_row = first_row

        # CSV files force uniform length
        # If second row has 4 columns and the first row only has 2
        # Then the file will automatically add two empty strings to the first row
        # Filter function used to prevent this

        # Code for subsequent frames
        for row in read:
            current_row = filter(None, row)
            if len(current_row) > len(
                    prev_row
            ):  # If the length of the row has increased a new car has been added
                iterations = (
                    len(current_row) - len(prev_row)
                ) / 2  # Find out how many new cars have been added
                for j in range(0, iterations):
                    # Add vehicle number, timestamp, x coord, and y coord to a list
                    position_list.append([
                        vehicle_number, row[1], row[vehicle_number * 2],
                        row[vehicle_number * 2 + 1]
                    ])
                    vehicle_number += 1
            # Check if there are any new dashes in the row
            k = 0
            while k <= len(current_row) / 2 - 2:
                if (current_row[2 +
                                k * 2]) == '-' and prev_row[2 + k * 2] != '-':
                    # For each new dash, append the corresponding time stamp to the position list
                    position_list[k].append(prev_row[1])
                    position_list[k].append(prev_row[2 + k * 2])
                    position_list[k].append(prev_row[3 + k * 2])
                k += 1
            prev_row = current_row

    return position_list
Exemple #25
0
    def has_header(self, sample):
        rdr = reader(StringIO(sample), self.sniff(sample))
        header = rdr.next()
        columns = len(header)
        columnTypes = {}
        for i in range(columns):
            columnTypes[i] = None

        checked = 0
        for row in rdr:
            if checked > 20:
                break
            checked += 1
            if len(row) != columns:
                continue
            for col in columnTypes.keys():
                for thisType in [int,
                 long,
                 float,
                 complex]:
                    try:
                        thisType(row[col])
                        break
                    except (ValueError, OverflowError):
                        pass

                else:
                    thisType = len(row[col])

                if thisType == long:
                    thisType = int
                if thisType != columnTypes[col]:
                    if columnTypes[col] is None:
                        columnTypes[col] = thisType
                    else:
                        del columnTypes[col]

        hasHeader = 0
        for col, colType in columnTypes.items():
            if type(colType) == type(0):
                if len(header[col]) != colType:
                    hasHeader += 1
                else:
                    hasHeader -= 1
            else:
                try:
                    colType(header[col])
                except (ValueError, TypeError):
                    hasHeader += 1
                else:
                    hasHeader -= 1

        return hasHeader > 0
Exemple #26
0
 def test_read_linenum(self):
     import _csv as csv
     r = csv.reader(['line,1', 'line,2', 'line,3'])
     assert r.line_num == 0
     next(r)
     assert r.line_num == 1
     next(r)
     assert r.line_num == 2
     next(r)
     assert r.line_num == 3
     raises(StopIteration, "next(r)")
     assert r.line_num == 3
Exemple #27
0
def getBots():
    index = 0

    with open('../resources/botInstagramInfo.csv', 'r') as read_obj:
        csv_reader = reader(read_obj)
        header = next(csv_reader)

        if header is not None:
            for row in csv_reader:
                index = index + 1

    return index
Exemple #28
0
def refresh_users():
    global USERS, GROUPS, USER_GROUPS, GROUP_USERS
    with open('/etc/passwd') as _in:
        USERS = {}
        for row in reader(_in, delimiter=':'):
            name = row[4].split(',')
            l_name = len(name)
            USERS[row[0]] = {'home': row[5], 'terminal': row[6], 'username': row[0], 'name': name[0], 'room': name[1] if l_name > 1 else '', 'office phone': name[2] if l_name > 2 else '', 'home phone': name[3] if l_name > 3 else '', 'other': name[4:]}
        USER_GROUPS = {u: set() for u in USERS}
    with open('/etc/group') as _in:
        GROUPS = {}
        for row in reader(_in, delimiter=':'):
            GROUPS[row[0]] = {'group': row[0]}
            for u in row[3].split(','):
                if not u:
                    continue
                try:
                    GROUP_USERS[row[0]][u] = USERS[u]
                except KeyError:
                    GROUP_USERS[row[0]] = {u: USERS[u]}
                USER_GROUPS[u].add(row[0])
Exemple #29
0
 def test_read_linenum(self):
     import _csv as csv
     r = csv.reader(['line,1', 'line,2', 'line,3'])
     assert r.line_num == 0
     r.next()
     assert r.line_num == 1
     r.next()
     assert r.line_num == 2
     r.next()
     assert r.line_num == 3
     raises(StopIteration, r.next)
     assert r.line_num == 3
Exemple #30
0
    def has_header(self, sample):
        rdr = reader(StringIO(sample), self.sniff(sample))
        header = rdr.next()
        columns = len(header)
        columnTypes = {}
        for i in range(columns):
            columnTypes[i] = None

        checked = 0
        for row in rdr:
            if checked > 20:
                break
            checked += 1
            if len(row) != columns:
                continue
            for col in columnTypes.keys():
                for thisType in [int,
                 long,
                 float,
                 complex]:
                    try:
                        thisType(row[col])
                        break
                    except (ValueError, OverflowError):
                        pass

                else:
                    thisType = len(row[col])

                if thisType == long:
                    thisType = int
                if thisType != columnTypes[col]:
                    if columnTypes[col] is None:
                        columnTypes[col] = thisType
                    else:
                        del columnTypes[col]

        hasHeader = 0
        for col, colType in columnTypes.items():
            if type(colType) == type(0):
                if len(header[col]) != colType:
                    hasHeader += 1
                else:
                    hasHeader -= 1
            else:
                try:
                    colType(header[col])
                except (ValueError, TypeError):
                    hasHeader += 1
                else:
                    hasHeader -= 1

        return hasHeader > 0
Exemple #31
0
 def test_read_linenum(self):
     import _csv as csv
     r = csv.reader(['line,1', 'line,2', 'line,3'])
     assert r.line_num == 0
     r.next()
     assert r.line_num == 1
     r.next()
     assert r.line_num == 2
     r.next()
     assert r.line_num == 3
     raises(StopIteration, r.next)
     assert r.line_num == 3
def readCSV(fileName, header=True):
    output = []

    try:
        with open(fileName, newline='', encoding='iso-8859-1') as csvFile:
            openCSV = reader(csvFile, delimiter=',')

            for row in openCSV:
                output.append(row)
    except:
        with open(fileName, newline='', encoding='utf-8') as csvFile:
            openCSV = reader(csvFile, delimiter=',')

            for row in openCSV:
                output.append(row)
    if header:
        header = output[0]
        output = pd.DataFrame(output[1:], columns=header)
    else:
        output = pd.DataFrame(output)

    return output
Exemple #33
0
    def read(self):
        """Reads CSV file and returns list of contents"""
        # Validate file path
        assert os.path.isfile(
            self.file_path), 'No such file exists: ' + str(self.file_path)

        # Open CSV file and read contents
        with open(self.file_path, 'r') as f:
            reader = csv_builtin.reader(f)
            loaded_data = list(reader)

        # Force digits to become integers
        return juggle_types(loaded_data)
        def _get_csv_value(label):
            """Return the RGB value for the label from csv txt file."""

            import _csv
            global _f
            _f = _csv.reader(open(fileName, "r"))
            for _row in _f:
                if (label == _row[0]):
                    _r = int(_row[1])
                    _g = int(_row[2])
                    _b = int(_row[3])
                    return str(RGB(_r, _g, _b))
            return
Exemple #35
0
def cartExplorerInit(StudioTitle, cartFiles=None, refresh=False, carts=None):
	global _cartEditTimestamps
	debugOutput("refreshing Cart Explorer" if refresh else "preparing cart Explorer")
	# Use cart files in SPL's data folder to build carts dictionary.
	# use a combination of SPL user name and static cart location to locate cart bank files.
	# Once the cart banks are located, use the routines in the populate method above to assign carts.
	# Since sstandard edition does not support number row carts, skip them if told to do so.
	if carts is None: carts = {"standardLicense":StudioTitle.startswith("StationPlaylist Studio Standard")}
	if refresh: carts["modifiedBanks"] = []
	# Obtain the "real" path for SPL via environment variables and open the cart data folder.
	cartsDataPath = os.path.join(os.environ["PROGRAMFILES"],"StationPlaylist","Data") # Provided that Studio was installed using default path.
	if cartFiles is None:
		# See if multiple users are using SPl Studio.
		userNameIndex = StudioTitle.find("-")
		# Read *.cart files and process the cart entries within (be careful when these cart file names change between SPL releases).
		# Until NVDA core moves to Python 3, assume that file names aren't unicode.
		cartFiles = [u"main carts.cart", u"shift carts.cart", u"ctrl carts.cart", u"alt carts.cart"]
		if userNameIndex >= 0:
			cartFiles = [StudioTitle[userNameIndex+2:]+" "+cartFile for cartFile in cartFiles]
	faultyCarts = False
	if not refresh:
		_cartEditTimestamps = []
	for f in cartFiles:
		# Only do this if told to build cart banks from scratch, as refresh flag is set if cart explorer is active in the first place.
		try:
			mod = f.split()[-2] # Checking for modifier string such as ctrl.
			# Todo: Check just in case some SPL flavors doesn't ship with a particular cart file.
		except IndexError:
			faultyCarts = True # In a rare event that the broadcaster has saved the cart bank with the name like "carts.cart".
			continue
		cartFile = os.path.join(cartsDataPath,f)
		# Cart explorer can safely assume that the cart bank exists if refresh flag is set.
		if not refresh and not os.path.isfile(cartFile): # Cart explorer will fail if whitespaces are in the beginning or at the end of a user name.
			faultyCarts = True
			continue
		debugOutput("examining carts from file %s"%cartFile)
		cartTimestamp = os.path.getmtime(cartFile)
		if refresh and _cartEditTimestamps[cartFiles.index(f)] == cartTimestamp:
			debugOutput("no changes to cart bank, skipping")
			continue
		_cartEditTimestamps.append(cartTimestamp)
		with open(cartFile) as cartInfo:
			cl = [row for row in reader(cartInfo)]
		# 17.04 (optimization): let empty string represent main cart bank to avoid this being partially consulted up to 24 times.
		# The below method will just check for string length, which is faster than looking for specific substring.
		_populateCarts(carts, cl[1], mod if mod != "main" else "", standardEdition=carts["standardLicense"], refresh=refresh) # See the comment for _populate method above.
		if not refresh:
			debugOutput("carts processed so far: %s"%(len(carts)-1))
	carts["faultyCarts"] = faultyCarts
	debugOutput("total carts processed: %s"%(len(carts)-2))
	return carts
Exemple #36
0
    def identify_fields(self, quiet=False):
        """
        Identify self.csv.fields got in __init__
        Sets them possible types (sorted, higher score mean bigger probability that the field is of that type)
        :type quiet: bool If True, we do not raise exception when sample cannot be processed.
                            Ex: We attempt consider user input "1,2,3" as single field which is not, we silently return False
        """
        samples = [[] for _ in self.parser.fields]
        if len(self.parser.sample
               ) == 1:  # we have too few values, we have to use them
            s = self.parser.sample[:1]
        else:  # we have many values and the first one could be header, let's omit it
            s = self.parser.sample[1:]

        for row in reader(
                s,
                dialect=self.parser.dialect) if self.parser.dialect else [s]:
            for i, val in enumerate(row):
                try:
                    samples[i].append(val)
                except IndexError:
                    if not quiet:
                        print(
                            "It seems rows have different lengths. Cannot help you with column identifying."
                        )
                        print("Fields row: " +
                              str([(i, str(f))
                                   for i, f in enumerate(self.parser.fields)]))
                        print("Current row: " + str(list(enumerate(row))))
                        if not Config.error_caught():
                            input("\n... Press any key to continue.")
                    return False

        for i, field in enumerate(self.parser.fields):
            possible_types = {}
            for type_ in Types.get_guessable_types():
                score = type_.check_conformity(samples[i],
                                               self.parser.has_header, field)
                if score:
                    possible_types[type_] = score
                # print("hits", hits)

            if possible_types:  # sort by biggest score - biggest probability the column is of this type
                field.possible_types = {
                    k: v
                    for k, v in sorted(possible_types.items(),
                                       key=lambda k: k[1],
                                       reverse=True)
                }
        return True
Exemple #37
0
    def post(self, request, *args, **kwargs):
        Deal.objects.all().delete()
        response = ''
        file = request.FILES['file']

        if file.name != 'deals.csv':
            response = 'Status: Error, Desc: Wrong file name. Should be: "deals.csv" Try again.'
            print(response)
            return Response({'response': response}, status=406)

        handle_uploaded_file(file)

        flag = True
        with open('api/data/deals.csv', encoding='utf-8') as data:
            deals = reader(data, delimiter=',')

            for deal in deals:
                if flag:
                    flag = not flag
                    continue

                try:
                    Deal.objects.create(username=deal[0],
                                        gem_name=deal[1],
                                        spent_money=int(deal[2]),
                                        gems=int(deal[3]),
                                        date=deal[4])
                except ValueError:
                    response = 'Status: Error, Desc: Wrong data format.' \
                               ' Should be: 1st column: str,' \
                               ' 2nd column: str,' \
                               ' 3rd column: int,' \
                               ' 4th column: int,' \
                               ' 5th column: datetime. Try again.'
                    print(response)
                    return Response({'response': response}, status=406)
                except ValidationError:
                    response = 'Status: Error, Desc: Wrong data format.' \
                               ' Should be: 1st column: str,' \
                               ' 2nd column: str,' \
                               ' 3rd column: int,' \
                               ' 4th column: int,' \
                               ' 5th column: datetime. Try again.'
                    print(response)
                    return Response({'response': response}, status=406)

        print('Status: OK')
        return Response({'response': 'File uploaded, go to /deal-view/'},
                        status=200)
Exemple #38
0
    def has_header(self, file_data):
        """Creates a dictionary of types of data in each column. If any
        column is of a single type (say, integers), *except* for the first
        row, then the first row is presumed to be labels. If the type
        can't be determined, it is assumed to be a string in which case
        the length of the string is the determining factor: if all of the
        rows except for the first are the same length, it's a header.
        Finally, a 'vote' is taken at the end for each column, adding or
        subtracting from the likelihood of the first row being a header.

        :param file_data: sample csv file data
        """

        reader_instance = reader(StringIO(file_data), self.sniff(file_data))
        header = next(reader_instance)  # rows starts with the header
        columns_num = len(header)
        column_types_dict = dict.fromkeys(range(0, columns_num), None)

        for row in reader_instance[:20]:
            if len(row) != columns_num:
                continue  # skip rows that have irregular number of columns

            for col in column_types_dict:
                for try_type in [int, float, complex]:
                    try:
                        try_type(row[col])
                        break
                    except (ValueError, OverflowError):
                        pass
                else:
                    # fallback to length of string
                    try_type = len(row[col])

                if try_type != column_types_dict[col]:
                    if column_types_dict[col] is None:  # add new column type
                        column_types_dict[col] = try_type
                    else:
                        # type is inconsistent, remove column from
                        # consideration
                        del column_types_dict[col]

        # Get results against first row and "vote" on whether it's a header.
        votes = self._count_header_votes(header, column_types_dict)

        # our assume about header exist is
        return votes > 0
Exemple #39
0
def load_train_data(file_name):
    # Function to read the input file and store it in a list
    try:
        train = open(file_name)
        data = csv.reader(train)
        for rec in data:
            temp_list = []
            for cols in rec:
                if rec.index(cols) != len(rec)-1:
                    temp_list.append(float(cols))
                else:
                    temp_list.append(cols)

            training_data.append(temp_list)

        print ("training data loaded")
    except IOError:
        print ("File not available. Please check the filename")
        exit()
Exemple #40
0
import sys
import _csv
import math

ifile = open(sys.argv[1] + "RunTimes.csv", "rt")
reader = _csv.reader(ifile)

rownum = 0
timelist = []
for row in reader:
    # Save header row.
    if rownum == 0:
        header = row
    else:
        # colnum = 0
        # for col in row:
        # print ('%s: %s' % (header[colnum], col))
        # colnum += 1
        time = [float(t) for t in row[5:]]
        timelist.append(time[0] * 3600 + time[1] * 60 + time[2])
        # print ('{0}: Total simulation time = {1:.2f}s'.format(row[1], time[0]*3600+time[1]*60+time[2]))
    rownum += 1
ifile.close()
n = len(timelist)
mean = sum(timelist) / n
sd = math.sqrt(sum((x - mean) ** 2 for x in timelist) / n)
print("{0:10d} jobs done, mean simulation time = {1:.2f}s, stdev = {2:.2f}s".format(n, mean, sd))
    def has_header(self, sample):
        # Creates a dictionary of types of data in each column. If any
        # column is of a single type (say, integers), *except* for the first
        # row, then the first row is presumed to be labels. If the type
        # can't be determined, it is assumed to be a string in which case
        # the length of the string is the determining factor: if all of the
        # rows except for the first are the same length, it's a header.
        # Finally, a 'vote' is taken at the end for each column, adding or
        # subtracting from the likelihood of the first row being a header.

        rdr = reader(StringIO(sample), self.sniff(sample))

        header = rdr.next() # assume first row is header

        columns = len(header)
        columnTypes = {}
        for i in range(columns): columnTypes[i] = None

        checked = 0
        for row in rdr:
            # arbitrary number of rows to check, to keep it sane
            if checked > 20:
                break
            checked += 1

            if len(row) != columns:
                continue # skip rows that have irregular number of columns

            for col in columnTypes.keys():

                for thisType in [int, long, float, complex]:
                    try:
                        thisType(row[col])
                        break
                    except (ValueError, OverflowError):
                        pass
                else:
                    # fallback to length of string
                    thisType = len(row[col])

                # treat longs as ints
                if thisType == long:
                    thisType = int

                if thisType != columnTypes[col]:
                    if columnTypes[col] is None: # add new column type
                        columnTypes[col] = thisType
                    else:
                        # type is inconsistent, remove column from
                        # consideration
                        del columnTypes[col]

        # finally, compare results against first row and "vote"
        # on whether it's a header
        hasHeader = 0
        for col, colType in columnTypes.items():
            if type(colType) == type(0): # it's a length
                if len(header[col]) != colType:
                    hasHeader += 1
                else:
                    hasHeader -= 1
            else: # attempt typecast
                try:
                    colType(header[col])
                except (ValueError, TypeError):
                    hasHeader += 1
                else:
                    hasHeader -= 1

        return hasHeader > 0
    def load_matrix_data(self, file):
        def is_comment(row):
            return row[0].startswith("#")

        return [map(int, row) for row in reader(open(file, 'rb')) if not is_comment(row)]
Exemple #43
0
            str_ += str(list_[i])
            str_ += " "
            i += 1
        return str_


biggest_id = 0
freed_id = []
is_up_to_date = 1
try:
    file = open('database.csv', 'r+')
except FileNotFoundError:
    file = open('database.csv', 'w+')

writer = _csv.writer(file, delimiter=';')
reader = _csv.reader(file, delimiter=';')

id_to_person = {}
last_name_to_id = {}
first_name_to_id = {}
patronymic_to_id = {}
phone_number_to_id = {}
date_of_birth_to_id = {}


def print_line():
    print("---------------------------------------------------------------------")


def write_header():
    writer.writerow(["Id", "Last Name", "First Name", "Patronymic", "Date of Birth", "Phone Number"])
import os
import re
import _csv as csv

fin = open('entity.csv')
cw = csv.reader(fin)
lst = list()
for row in cw:
    lst.append(row)
fin.close()
first_line = lst[0]
req_edit = list()

def mproc(a):
    if a == 'TRUE': return 'true'
    if a == 'FALSE': return 'false'
    return a

for nm in range(1, len(lst)):
    idx = dict()
    for i in range(0, len(lst[nm])):
        idx[first_line[i]] = mproc(lst[nm][i])
    nstr = """
{
    "Properties": {
        "Name": "%s",
        "Type": "%s",
        "Description": "%s",
        "ShowInCreative": %s
    },
Exemple #45
0
from __future__ import absolute_import, unicode_literals

import _csv
import io


reader = type(_csv.reader(io.StringIO('')))
writer = type(_csv.writer(io.StringIO()))