def __init__(self, input_file, output_file):
super().__init__()
self.output_file = output_file
self.input_file = input_file
self.signals = WorkerSignal()
self.parser = Sps21Parser()
self.commit_every = 100000
def __init__(self, sess, file_name):
super().__init__()
self.sess = sess
self.file_name = file_name
self.signals = WorkerSignal()
self.parser = Sps21Parser()
self.commit_every = 10000
def run(self):
self.signals.start.emit(False)
session = sessionmaker(bind=self.conn)
session = session()
line_numbers = check.iter_count(self.data_file)
self.signals.process_max.emit(line_numbers)
parser = Sps21Parser()
counter = 0
try:
with open(self.data_file) as fr:
for line in fr:
parsed = parser.parse_relation(line)
if parsed:
sx = session.query(X).filter(
and_(X.line == parsed[5], X.point == parsed[6]))
if sx == None:
counter += 1
xx = X(id=counter,
line=parsed[5],
point=parsed[6],
rl=parsed[11],
fr=parsed[12],
tr=parsed[13])
session.add(xx)
if counter % 100000 == 0:
print(counter)
session.commit()
session.commit()
except Exception as e:
print(str(e))
self.signals.process.emit(counter)
self.signals.finished.emit(True)
def run(self):
self.signals.start.emit(False)
line_numbers = check.iter_count(self.data_file)
self.signals.process_max.emit(line_numbers)
table_name = choose_table(self.data_file)
parser = Sps21Parser()
# print(line_numbers)
counter = 0
with self.engine.connect() as con:
#con.execute("PRAGMA synchronous=OFF") # 关闭同步
#con.execute("BEGIN TRANSACTION") # 显式开启事务
try:
with open(self.data_file) as fr:
for line in fr:
parsed = parser.parse_point(line)
if parsed:
counter += 1
db_point_update(con, table_name, parsed)
if counter % 100000 == 0:
self.signals.process.emit(counter)
print(counter)
#con.commit()
#con.commit()
except Exception as e:
print(str(e))
self.signals.process.emit(counter)
self.signals.finished.emit(True)
def __init__(self, sess, from_file_name, to_file_name):
super().__init__()
self.sess = sess
self.from_file_name = from_file_name
self.to_file_name = to_file_name
self.signals = WorkerSignal()
self.parser = Sps21Parser()
self.commit_every = 5000
def select_from_sps(line, session, table):
parser = Sps21Parser()
parsed = parser.parse_point(line)
tc = config.table_class[table]
vcounter = 0
ncounter = 0
data = []
if parsed:
query = session.query(tc).filter(
and_(tc.line == parsed[1], tc.point == parsed[2],
tc.idx == parsed[3]))
exsit_obj = session.query(query.exists()).scalar()
if exsit_obj:
d = session.query(tc).filter(
and_(tc.line == parsed[1], tc.point == parsed[2],
tc.idx == parsed[3])).first()
data.append(d)
vcounter += 1
else:
ncounter += 1
return data, vcounter, ncounter