def test_LogicalFile(tmpdir):
prefix = os.path.join(tmpdir, 'log')
L = util.LogicalFile(prefix, 2, 6)
with pytest.raises(FileNotFoundError):
L.open_file(0, create=False)
# Check L.open creates a file
with L.open_file(8, create=True) as f:
pass
with util.open_file(prefix + '01') as f:
pass
L.write(0, b'987')
assert L.read(0, -1) == b'987'
assert L.read(0, 4) == b'987'
assert L.read(1, 1) == b'8'
L.write(0, b'01234567890')
assert L.read(0, -1) == b'01234567890'
assert L.read(5, -1) == b'567890'
with util.open_file(prefix + '01') as f:
assert f.read(-1) == b'67890'
# Test file boundary
L.write(0, b'957' * 6)
assert L.read(0, -1) == b'957' * 6
def xlsx_report(header='1', sort='1', report_path=None,
launch='1'):
if report_path is None:
report_path = tempfile.mktemp('.xlsx')
workbook = xlsxwriter.Workbook(report_path)
worksheet = workbook.add_worksheet()
col = 0
row = 0
bold = workbook.add_format({'bold': True})
if sort == '1':
report.sort()
if header == '1':
for column_title in report.header:
worksheet.write(row, col, column_title, bold)
col += 1
col = 0
row += 1
for report_row in report:
for report_cell in report_row:
worksheet.write(row, col, report_cell)
col += 1
col = 0
row += 1
workbook.close()
if (launch == '1'):
util.open_file(report_path)
else:
print("%s generated" % report_path)
serverinfo.server_info_cache.save()
def save_report(header='1', sort='1', separator='\t', vseparator='\n',
report_path=None, launch='0'):
if report_path is None:
report_path = tempfile.mktemp('.tsv')
codecs.open(report_path,
mode='w',
encoding='utf-8').write(gen_report(header,
sort,
separator,
vseparator))
if (launch == '1'):
util.open_file(report_path)
else:
print("%s generated" % report_path)
serverinfo.server_info_cache.save()
def test_open_fns(tmpdir):
tmpfile = os.path.join(tmpdir, 'file1')
with pytest.raises(FileNotFoundError):
util.open_file(tmpfile)
with util.open_file(tmpfile, create=True) as f:
f.write(b'56')
with util.open_file(tmpfile) as f:
assert f.read(3) == b'56'
# Test open_truncate truncates and creates
with util.open_truncate(tmpfile) as f:
assert f.read(3) == b''
tmpfile = os.path.join(tmpdir, 'file2')
with util.open_truncate(tmpfile) as f:
assert f.read(3) == b''
def open_dbs(self):
'''Open the databases. If already open they are closed and re-opened.
When syncing we want to reserve a lot of open files for the
synchtonization. When serving clients we want the open files for
serving network connections.
'''
def log_reason(message, is_for_sync):
reason = 'sync' if is_for_sync else 'serving'
self.logger.info('{} for {}'.format(message, reason))
# Assume we're serving until we find out otherwise
for for_sync in [False, True]:
if self.utxo_db:
if self.utxo_db.for_sync == for_sync:
return
log_reason('closing DB to re-open', for_sync)
self.utxo_db.close()
self.hist_db.close()
self.eventlog_db.close()
self.hashY_db.close()
# Open DB and metadata files. Record some of its state.
self.utxo_db = self.db_class('utxo', for_sync)
self.hist_db = self.db_class('hist', for_sync)
self.eventlog_db = self.db_class('eventlog', for_sync)
self.hashY_db = self.db_class('hashY', for_sync)
if self.utxo_db.is_new:
self.logger.info('created new database')
self.logger.info('creating metadata directory')
os.mkdir('meta')
with util.open_file('COIN', create=True) as f:
f.write(
'ElectrumX databases and metadata for {} {}'.format(
self.coin.NAME, self.coin.NET).encode())
else:
log_reason('opened DB', self.utxo_db.for_sync)
self.read_utxo_state()
if self.first_sync == self.utxo_db.for_sync:
break
self.read_history_state()
self.read_eventlog_state()
self.logger.info('DB version: {:d}'.format(self.db_version))
self.logger.info('coin: {}'.format(self.coin.NAME))
self.logger.info('network: {}'.format(self.coin.NET))
self.logger.info('height: {:,d}'.format(self.db_height))
self.logger.info('tip: {}'.format(hash_to_str(self.db_tip)))
self.logger.info('tx count: {:,d}'.format(self.db_tx_count))
self.logger.info('flush count: {:,d}'.format(self.flush_count))
self.logger.info('eventlog flush count: {:,d}'.format(
self.eventlog_flush_count))
if self.first_sync:
self.logger.info('sync time so far: {}'.format(
util.formatted_time(self.wall_time)))
def read_peers_file(self):
try:
with util.open_file(PEERS_FILE, create=True) as f:
data = f.read(-1).decode()
except Exception as e:
self.logger.error('error reading peers file {}'.format(e))
else:
if data:
version, items = ast.literal_eval(data)
if version == 1:
peers = [Peer.deserialize(item) for item in items]
self.add_peers(peers, source='peers file', limit=None)
def read_peers_file(self):
try:
with util.open_file(PEERS_FILE, create=True) as f:
data = f.read(-1).decode()
except Exception as e:
self.logger.error('error reading peers file {}'.format(e))
else:
if data:
version, items = ast.literal_eval(data)
if version == 1:
peers = []
for item in items:
if 'last_connect' in item:
item['last_good'] = item.pop('last_connect')
try:
peers.append(Peer.deserialize(item))
except Exception:
pass
self.add_peers(peers, source='peers file', limit=None)
def load_one_file(args, cxn, file_name, ends, seq_end_clamp=''):
"""Load sequences from a fasta/fastq file into the atram database."""
log.info('Loading "{}" into sqlite database'.format(file_name))
parser = get_parser(args, file_name)
with util.open_file(args, file_name) as sra_file:
batch = []
for rec in parser(sra_file):
title = rec[0].strip()
seq = rec[1]
seq_name, seq_end = blast.parse_fasta_title(
title, ends, seq_end_clamp)
batch.append((seq_name, seq_end, seq))
if len(batch) >= db.BATCH_SIZE:
db.insert_sequences_batch(cxn, batch)
batch = []
db.insert_sequences_batch(cxn, batch)
def load_one_file(args, cxn, file_name, ends, seq_end_clamp=''):
"""Load sequences from a fasta/fastq file into the atram database."""
log.info('Loading "{}" into sqlite database'.format(file_name))
parser = get_parser(args, file_name)
with util.open_file(args, file_name) as sra_file:
batch = []
for rec in parser(sra_file):
title = rec[0].strip()
seq = rec[1]
seq_name, seq_end = blast.parse_fasta_title(
title, ends, seq_end_clamp)
batch.append((seq_name, seq_end, seq))
if len(batch) >= db.BATCH_SIZE:
db.insert_sequences_batch(cxn, batch)
batch = []
db.insert_sequences_batch(cxn, batch)
def OnOpenFile(self, event):
obj = self.myOlv.GetSelectedObject()
open_file(obj.get_media_path())
def OnOpenFile(self, event):
obj = self.myOlv.GetSelectedObject()
open_file(obj.get_media_path())
