def testRead(self):
occurrence = CSVReader("data/occurrence.txt",
delimiter="\t",
quoteChar="\"",
indexFields=["id", "eventID"])
indexes = occurrence.indexes()
self.assertTrue("id" in indexes)
self.assertTrue("eventID" in indexes)
self.assertTrue(
"Cruise68:Station593:EventSorbeSledge9887:Subsample16687" in
indexes["eventID"])
self.assertTrue(
len(indexes["eventID"]
["Cruise68:Station593:EventSorbeSledge9887:Subsample16687"]) ==
39)
records = occurrence.getLines(
"id", "Cruise68:Station565:EventSorbeSledge9781:Subsample17409")
self.assertTrue(len(records) > 0)
self.assertTrue(
records[0]["eventID"] ==
"Cruise68:Station565:EventSorbeSledge9781:Subsample17409")
self.assertTrue(records[0]["scientificNameID"] ==
"urn:lsid:marinespecies.org:taxname:131495")
records = occurrence.getLines(
"eventID",
"Cruise68:Station622:EventSorbeSledge10018:Subsample15224")
self.assertTrue(
records[21]["eventID"] ==
"Cruise68:Station622:EventSorbeSledge10018:Subsample15224")
self.assertTrue(records[21]["scientificNameID"] ==
"urn:lsid:marinespecies.org:taxname:120144")
def test_init(self):
"""Testing __init__"""
file_object = StringIO()
for class_name in entity.keys():
reader = CSVReader(file_object, class_name)
with self.assertRaises(DataReaderException):
reader = CSVReader(file_object, 'Foo')
def test_get_line(self):
occurrence = CSVReader("data/occurrence.txt",
delimiter="\t",
quote_char="\"")
record = occurrence.get_line(0)
self.assertTrue(record["occurrenceID"] == "Ugenthyperbenthos45979")
record = occurrence.get_line(5)
self.assertTrue(record["occurrenceID"] == "Ugenthyperbenthos95454")
def test_not_indexed(self):
occurrence = CSVReader("data/occurrence.txt",
delimiter="\t",
quote_char="\"")
with self.assertRaises(RuntimeError):
records = list(
occurrence.get_lines(
"eventID",
"Cruise68:Station593:EventSorbeSledge9887:Subsample16687"))
def testDev(self):
occurrence = CSVReader("data/occurrence.txt",
delimiter="\t",
quoteChar="\"",
indexFields=["scientificName"])
print occurrence.indexes()
records = occurrence.getLines("scientificName", "Neomysis integer")
print records
def _index_files(self):
"""Index the appropriate columns of the core and extension files."""
# index core
i_fields = self._which_field_to_index(self.core)
self.core["reader"] = CSVReader(self._temp_dir + "/" + self.core.file, delimiter=self.core.delimiter, quote_char=self.core.quote_char, field_names=self.core.fields, index_fields=i_fields)
# index extensions
for e in self.extensions:
e["reader"] = CSVReader(self._temp_dir + "/" + e.file, delimiter=e.delimiter, quote_char=e.quote_char, field_names=e.fields, index_fields=self._which_field_to_index(e))
def test_field_names_dict(self):
names = {"scientificName": 7}
occurrence = CSVReader("data/occurrence.txt",
delimiter="\t",
quote_char="\"",
index_fields=["scientificName"],
field_names=names)
records = list(
occurrence.get_lines("scientificName", "Neomysis integer"))
self.assertTrue(records[0]["scientificName"] == "Neomysis integer")
self.assertTrue(
records[0]["col_6"] == "urn:lsid:marinespecies.org:taxname:120136")
def open_button_callback(self):
try:
path = self.view.fileNameLineEdit.text()
reader = CSVReader(path)
content = reader.read()
delimiter = reader.delimiter()
self.view.delimiterLabel.setText(delimiter)
self.view.outputTextEdit.setText(json.dumps(content, indent=4))
except Exception as e:
self.view.outputTextEdit.setText(e.strerror)
def test_str(self):
self.assertTrue(
isinstance(
str(
CSVReader("data/occurrence.txt",
delimiter="\t",
quote_char="\"",
index_fields=["scientificName"])), str))
self.assertTrue(
isinstance(
str(
CSVReader("data/occurrence.txt",
delimiter="\t",
quote_char="\"")), str))
def testStr(self):
self.assertTrue(
isinstance(
str(
CSVReader("data/occurrence.txt",
delimiter="\t",
quoteChar="\"",
indexFields=["scientificName"])), basestring))
self.assertTrue(
isinstance(
str(
CSVReader("data/occurrence.txt",
delimiter="\t",
quoteChar="\"")), basestring))
def test_iterator(self):
occurrence = CSVReader("data/occurrence.txt",
delimiter="\t",
quote_char="\"")
for line in occurrence:
self.assertTrue(line["occurrenceID"] == "Ugenthyperbenthos45979")
break
class Analyzer(object):
csvReader = None
def __init__(self, directory):
self.csvReader = CSVReader(directory)
def analyze(self):
self.csvReader.read()
analyzed = defaultdict()
for data in self.csvReader.getData():
analyzed[data.name] = data.getAnalyzed()
i = dt.datetime.now()
analyzed["datetime"] = i.isoformat()
self.csvReader.clear()
return analyzed
class Analyzer(object):
csvReader = None
def __init__(self, directory):
self.csvReader = CSVReader(directory)
def analyze(self):
self.csvReader.read()
analyzed = defaultdict()
for data in self.csvReader.getData():
analyzed[data.name] = data.getAnalyzed()
i = dt.datetime.now()
analyzed["datetime"] = i.isoformat()
self.csvReader.clear()
return analyzed
def test_field_names_list(self):
names = [
"id", "basisOfRecord", "occurrenceID", "sex", "lifeStage",
"eventID"
]
occurrence = CSVReader("data/occurrence.txt",
delimiter="\t",
quote_char="\"",
index_fields=["eventID"],
field_names=names)
records = list(
occurrence.get_lines(
"eventID",
"Cruise68:Station593:EventSorbeSledge9887:Subsample16687"))
self.assertTrue(
records[0]["eventID"] ==
"Cruise68:Station593:EventSorbeSledge9887:Subsample16687")
self.assertTrue(records[0]["occurrenceID"] == "Ugenthyperbenthos51168")
self.assertTrue(records[0]["col_6"] == "Sagitta elegans")
def test_skip_blank(self):
occurrence = CSVReader("data/occurrence_blanklines.txt",
delimiter="\t",
quote_char="\"",
index_fields=["institutionCode"],
skip_blank=False)
self.assertTrue("" in occurrence.indexes()["institutionCode"])
occurrence = CSVReader("data/occurrence_blanklines.txt",
delimiter="\t",
quote_char="\"",
index_fields=["institutionCode"],
skip_blank=True)
self.assertFalse("" in occurrence.indexes()["institutionCode"])
def test_read_fourth_line(self):
lines = CSVReader.read(TEST_FILENAME)
self.assertEqual("Peter", lines[3]["Name"])
self.assertEqual(12, int(lines[3]["Anzahl"]))
from csvreader import CSVReader
#names = {"scientificName": 7}
#occurrence = CSVReader("data/occurrence.txt", delimiter="\t", quoteChar="\"", indexFields=["scientificName"], fieldNames=names)
#print occurrence.indexes()
#records = occurrence.getLines("scientificName", "Neomysis integer")
#print records
#names = ["id", "basisOfRecord", "occurrenceID", "sex", "lifeStage", "eventID", "scientificNameID", "scientificName"]
#occurrence = CSVReader("data/occurrence.txt", delimiter="\t", quoteChar="\"", indexFields=["scientificName"], fieldNames=names)
#print occurrence.indexes()
#records = occurrence.getLines("scientificName", "Neomysis integer")
#print records
names = ["id", "basisOfRecord", "occurrenceID", "sex", "lifeStage", "eventID"]
occurrence = CSVReader("data/occurrence.txt",
delimiter="\t",
quoteChar="\"",
indexFields=["eventID"],
fieldNames=names)
print occurrence.indexes()
records = occurrence.getLines(
"eventID", "Cruise68:Station593:EventSorbeSledge9887:Subsample16687")
for r in records:
print r
#print len(occurrence)
#for line in occurrence:
# print line
# break
def test_read_first_line(self):
lines = CSVReader.read(TEST_FILENAME)
self.assertEqual("Mathias", lines[0]["Name"])
self.assertEqual(10, int(lines[0]["Anzahl"]))
def test_len(self):
occurrence = CSVReader("data/occurrence.txt",
delimiter="\t",
quote_char="\"")
self.assertTrue(len(occurrence) == 8434)
def test_read_second_line(self):
lines = CSVReader.read(TEST_FILENAME)
self.assertEqual("Stefan", lines[1]["Name"])
self.assertEqual(7, int(lines[1]["Anzahl"]))
def __init__(self, directory):
self.csvReader = CSVReader(directory)
def create_csvreader_workers(csv_file: str) -> List[object]:
"""Create CSV Reader task"""
CSV_Worker = CSVReader(csv_file, Domains[0])
return [CSV_Worker]
def __init__(self, directory):
self.csvReader = CSVReader(directory)
def test_read_third_line(self):
lines = CSVReader.read(TEST_FILENAME)
self.assertEqual("Markus", lines[2]["Name"])
self.assertEqual(5, int(lines[2]["Anzahl"]))
def test_delimiter_none(self):
reader = CSVReader("data.csv")
self.assertIsNone(reader.delimiter())
def test_success(self):
reader = CSVReader("data.csv")
data = reader.read()
self.assertTrue(len(data) == 3)
'level': 'INFO',
'handlers': ['wsgi']
}
})
app = Flask(__name__)
# Debug mode (development environment)
app.debug = False
debug_mode = False
# Init modules
app.config.from_object(Config)
configfile = ConfigFile(app, 'main.cfg')
csvreader = CSVReader(configfile.laser_etch_QC['PNFile'])
db = SQLAlchemy(app)
migrate = Migrate(app, db)
login = LoginManager(app)
login.login_view = 'login'
socketio = SocketIO(app, manage_session=False)
Session(app)
# Serial setup
# PLC serial
plc_ser = None
from flask_serial import Serial
import sys
from csvreader import CSVReader
from process import start_process
if __name__ == "__main__":
file_name = sys.argv[1]
stock_dict = CSVReader(file_name).read_csv()
start_process(stock_dict)
# locust --slave &
# locust --slave &
# You should have the Web UI running at http://localhost:8089/
# IMPORTANT: to start locust without the Web UI and generate the results in csv output: env API_TOKEN=<apiKey> locust --master --csv=results --no-web -c 200 -r 50 --host
# for this project use
# env API_TOKEN=<apiKey> locust --master --csv=results --no-web -c 200 -r 50 --host http://api.openweathermap.org
import os
from locust import TaskSet, task, constant
from locust.contrib.fasthttp import FastHttpLocust # for making an HTTP request
from csvreader import CSVReader
reader = CSVReader("file_name.csv") # or path to file
class MyLocust(FastHttpLocust):
task_set = # TaskSet class
wait_time = constant(0)
# Есть несколько встроенных функций для определения времени ожидания, таких как between , constant и constant_pacing .
class MyTaskSet(TaskSet):
api_path = "/data/2.5/weather?q={},{}&appid={}"
@task
def api_call(self):
data = next(reader)
response = self.client.get(self.api_path.format(data[0], data[1], os.environ.get("API_KEY")))
def test_next_entity_end(self):
"""next_entity should return None when ending"""
file_object = StringIO()
reader = CSVReader(file_object, 'Testing')
self.assertIsNone(reader.next_entity())
def test_delimiter(self):
reader = CSVReader("data.csv")
reader.read()
self.assertEqual(reader.delimiter(), ',')
def reader_once(self, string, filetype):
"""Helper function for only read once testcases"""
file_object = StringIO(string.strip())
reader = CSVReader(file_object, filetype)
instance = reader.next_entity()
return instance
def load_dictionary(self, dict_f):
self._words = CSVReader(dict_f).get_content()
#occurrence = CSVReader("data/occurrence.txt", delimiter="\t", quote_char="\"", index_fields=["scientificName"], field_names=names)
#print(occurrence.indexes())
#records = occurrence.get_lines("scientificName", "Neomysis integer")
#print(records)
names = {
'eventID': 2,
'decimalLatitude': 10,
'minimumDepthInMeters': 8,
'habitat': 5,
'locality': 7,
'parentEventID': 3,
'modified': 1,
'footprintWKT': 13,
'decimalLongitude': 11,
'maximumDepthInMeters': 9,
'locationID': 6,
'id': 0,
'coordinateUncertaintyInMeters': 12,
'eventDate': 4
}
event = CSVReader("data/event2.txt",
delimiter="\t",
quote_char=None,
index_fields=["eventID", "parentEventID", "id"],
field_names=names)
print(json.dumps(event.indexes(), indent=4))
#event = CSVReader("data/occurrence_blanklines.txt", delimiter="\t", quote_char=None, index_fields=["institutionCode"], skip_blank=True)
#print(json.dumps(event.indexes(), indent=4))