def stamp_command(fd, args):
# Create initial commitment ops for all files
merkle_roots = []
try:
file_timestamp = DetachedTimestampFile.from_fd(OpSHA256(), fd)
except OSError as exp:
logging.error("Could not read %r: %s" % (fd.name, exp))
return
# Add nonce
nonce_appended_stamp = file_timestamp.timestamp.ops.add(
OpAppend(os.urandom(16)))
merkle_root = nonce_appended_stamp.ops.add(OpSHA256())
merkle_roots.append(merkle_root)
merkle_tip = make_merkle_tree(merkle_roots)
create_timestamp(merkle_tip, CALENDAR_URLS, parse_ots_args(args))
try:
with open("%s.ots" % fd.name, "wb") as timestamp_fd:
ctx = StreamSerializationContext(timestamp_fd)
file_timestamp.serialize(ctx)
except IOError as exp:
logger.error("Failed to create timestamp: %s" % exp)
return
def from_file(self, path):
self.path = path
self.status = "tracked"
self.agt = None
self.txid = None
self.block = None
self.date = None
with open(self.path, "rb") as fo:
self.detached_timestamp = DetachedTimestampFile.from_fd(OpSHA256(), fo)
def upgrade_timestamps_tx(self, tx):
for category, script, amount in tx.outputs():
if category == 2: # agt -> txid
agt = script[4:] # drop "6a20" op_return and op_pushdata(32)
tx_raw = tx.serialize(witness=False)
if len(x(tx_raw)) <= Op.MAX_MSG_LENGTH:
i = tx_raw.find(agt)
prepend = x(tx_raw[:i])
append = x(tx_raw[i + len(agt):])
t_agt = Timestamp(x(agt))
t = t_agt.ops.add(OpPrepend(prepend))
t = t.ops.add(OpAppend(append))
t = t.ops.add(OpSHA256())
t = t.ops.add(OpSHA256()) # txid in little endian
for f in self.proofs_storage_file.incomplete_proofs:
tf = roll_timestamp(f.detached_timestamp.timestamp)
if tf.msg == x(agt):
tf.merge(t_agt)
f.status = "pending"
f.txid = t.msg[::-1].hex()
self.update_storage()
def generate(msg_bytes):
'''Generates certificate'''
hashed_bytes = hashlib.new('sha256', msg_bytes).digest()
file_timestamp = DetachedTimestampFile(OpSHA256(), Timestamp(hashed_bytes))
nonce_appended_stamp = file_timestamp.timestamp.ops.add(OpAppend(os.urandom(16)))
timestamp = nonce_appended_stamp.ops.add(OpSHA256())
remote_calendar = RemoteCalendar(CALENDAR_URL)
result = remote_calendar.submit(timestamp.msg, timeout=None)
try:
if isinstance(result, Timestamp):
timestamp.merge(result)
else:
logging.debug(str(result))
except Exception as error:
logging.debug(str(error))
return file_timestamp
def __pending_to_merkle_tree(self, n):
# Update the most recent timestamp transaction with new commitments
commitment_timestamps = [Timestamp(commitment) for commitment in tuple(self.pending_commitments)[0:n]]
# Remember that commitment_timestamps contains raw commitments,
# which are longer than necessary, so we sha256 them before passing
# them to make_merkle_tree, which concatenates whatever it gets (or
# for the matter, returns what it gets if there's only one item for
# the tree!)
commitment_digest_timestamps = [stamp.ops.add(OpSHA256()) for stamp in commitment_timestamps]
logging.debug("Making merkle tree")
tip_timestamp = make_merkle_tree(commitment_digest_timestamps)
logging.debug("Done making merkle tree")
return (tip_timestamp, commitment_timestamps)
def __do_ethereum(self):
if self.pending_commitments and time.time() > self.last_timestamp_tx + self.wait_time_between_txs:
logging.info("we have commitments and enough time has passed")
# logging.info(self.pending_commitments)
# Update the most recent timestamp transaction with new commitments
commitment_timestamps = [Timestamp(commitment) for commitment in self.pending_commitments]
# Remember that commitment_timestamps contains raw commitments,
# which are longer than necessary, so we sha256 them before passing
# them to make_merkle_tree, which concatenates whatever it gets (or
# for the matter, returns what it gets if there's only one item for
# the tree!)
commitment_digest_timestamps = [stamp.ops.add(OpSHA256()) for stamp in commitment_timestamps]
tip_timestamp = make_merkle_tree(commitment_digest_timestamps)
eth_tx = {'from': self.account, 'to': self.account, 'value': 0, 'data': '0x' + bytes.hex(tip_timestamp.msg)}
logging.info(eth_tx)
tx_hash = self.web3.eth.sendTransaction(eth_tx)
logging.info("tx_hash " + str(tx_hash))
self.last_timestamp_tx = time.time()
self.pending_commitments = set()
self.unconfirmed_txs.append(TimestampTx(tx_hash, tip_timestamp, commitment_timestamps))
def __init__(self,
tree,
db=None,
file_hash_op=OpSHA256(),
tree_hash_op=OpSHA256()):
self.tree = tree
if db is None:
os.makedirs(tree.repo.git_dir + '/ots', exist_ok=True)
# WARNING: change the version number if any of the following is
# changed; __init__() is consensus-critical!
db = dbm.open(tree.repo.git_dir + '/ots/tree-hash-cache-v3', 'c')
self.db = db
self.file_hash_op = file_hash_op
self.tree_hash_op = tree_hash_op
def do_item(item):
try:
return (item, Timestamp(db[item.hexsha]))
except KeyError:
timestamp = None
if isinstance(item, git.Blob):
timestamp = Timestamp(
file_hash_op.hash_fd(item.data_stream[3]))
elif isinstance(item, git.Tree):
stamper = GitTreeTimestamper(item,
db=db,
file_hash_op=file_hash_op,
tree_hash_op=tree_hash_op)
timestamp = stamper.timestamp
elif isinstance(item, git.Submodule):
# A submodule is just a git commit hash.
#
# Unfortunately we're not guaranteed to have the repo
# behind it, so all we can do is timestamp that SHA1 hash.
#
# We do run it through the tree_hash_op to make it
# indistinguishable from other things; consider the
# degenerate case where the only thing in a git repo was a
# submodule.
timestamp = Timestamp(tree_hash_op(item.binsha))
else:
raise NotImplementedError("Don't know what to do with %r" %
item)
db[item.hexsha] = timestamp.msg
return (item, timestamp)
self.contents = tuple(do_item(item) for item in self.tree)
if len(self.contents) > 1:
# Deterministically nonce contents in an all-or-nothing transform. As
# mentioned in the class docstring, we want to ensure that the the
# siblings of any leaf in the merkle tree don't give the attacker any
# information about what else is in the tree, unless the attacker
# already knows (or can brute-force) the entire contents of the tree.
#
# While not perfect - a user-provided persistant key would prevent the
# attacker from being able to brute-force the contents - this option
# has the advantage of being possible to calculate deterministically
# using only the tree itself, removing the need to keep secret keys
# that can easily be lost.
#
# First, calculate a nonce_key that depends on the entire contents of
# the tree. The 8-byte tag ensures the key calculated is unique for
# this purpose.
contents_sum = b''.join(stamp.msg for item, stamp in self.contents
) + b'\x01\x89\x08\x0c\xfb\xd0\xe8\x08'
nonce_key = tree_hash_op.hash_fd(io.BytesIO(contents_sum))
# Second, calculate per-item nonces deterministically from that key,
# and add those nonces to the timestamps of every item in the tree.
#
# While we usually use 128-bit nonces, here we're using full-length
# nonces. Additionally, we pick append/prepend pseudo-randomly. This
# helps obscure the directory structure, as a commitment for a git tree
# is indistinguishable from a inner node in the per-git-tree merkle
# tree.
def deterministically_nonce_stamp(private_stamp):
nonce1 = tree_hash_op(private_stamp.msg + nonce_key)
nonce2 = tree_hash_op(nonce1)
side = OpPrepend if nonce1[0] & 0b1 else OpAppend
nonce_added = private_stamp.ops.add(side(nonce2))
return nonce_added.ops.add(tree_hash_op)
nonced_contents = (deterministically_nonce_stamp(stamp)
for item, stamp in self.contents)
# Note how the current algorithm, if asked to timestamp a tree
# with a single thing in it, will return the hash of that thing
# directly. From the point of view of just commiting to the data that's
# perfectly fine, and probably (slightly) better as it reveals a little
# less information about directory structure.
self.timestamp = make_merkle_tree(
nonced_stamp for nonced_stamp in nonced_contents)
elif len(self.contents) == 1:
# If there's only one item in the tree, the fancy all-or-nothing
# transform above is just a waste of ops, so use the tree contents
# directly instead.
self.timestamp = tuple(self.contents)[0][1]
else:
raise AssertionError("Empty git tree")
def cat_sha256d(left, right):
sha256_timestamp = cat_sha256(left, right)
return sha256_timestamp.ops.add(OpSHA256())
def str_tree(self, indent=0, verbosity=0):
"""Convert to tree (for debugging)"""
class bcolors:
HEADER = '\033[95m'
OKBLUE = '\033[94m'
OKGREEN = '\033[92m'
WARNING = '\033[93m'
FAIL = '\033[91m'
ENDC = '\033[0m'
BOLD = '\033[1m'
UNDERLINE = '\033[4m'
def str_result(verb, parameter, result):
rr = ""
if verb > 0 and result is not None:
rr += " == "
result_hex = b2x(result)
if parameter is not None:
parameter_hex = b2x(parameter)
try:
index = result_hex.index(parameter_hex)
parameter_hex_highlight = bcolors.BOLD + parameter_hex + bcolors.ENDC
if index == 0:
rr += parameter_hex_highlight + result_hex[
index + len(parameter_hex):]
else:
rr += result_hex[0:index] + parameter_hex_highlight
except ValueError:
rr += result_hex
else:
rr += result_hex
return rr
r = ""
if len(self.attestations) > 0:
for attestation in sorted(self.attestations):
r += " " * indent + "verify %s" % str(
attestation) + str_result(verbosity, self.msg, None) + "\n"
if attestation.__class__ == BitcoinBlockHeaderAttestation:
r += " " * indent + "# Bitcoin block merkle root " + b2lx(
self.msg) + "\n"
if len(self.ops) > 1:
for op, timestamp in sorted(self.ops.items()):
try:
CTransaction.deserialize(self.msg)
r += " " * indent + "* Bitcoin transaction id " + b2lx(
OpSHA256()(OpSHA256()(self.msg))) + "\n"
except SerializationError:
pass
cur_res = op(self.msg)
cur_par = op[0]
r += " " * indent + " -> " + "%s" % str(op) + str_result(
verbosity, cur_par, cur_res) + "\n"
r += timestamp.str_tree(indent + 4, verbosity=verbosity)
elif len(self.ops) > 0:
try:
CTransaction.deserialize(self.msg)
r += " " * indent + "# Bitcoin transaction id " + \
b2lx(OpSHA256()(OpSHA256()(self.msg))) + "\n"
except SerializationError:
pass
op = tuple(self.ops.keys())[0]
cur_res = op(self.msg)
cur_par = op[0] if len(op) > 0 else None
r += " " * indent + "%s" % str(op) + str_result(
verbosity, cur_par, cur_res) + "\n"
r += tuple(self.ops.values())[0].str_tree(indent,
verbosity=verbosity)
return r
def hash_signed_commit(git_commit, gpg_sig):
return OpSHA256()(OpSHA256()(git_commit) + OpSHA256()(gpg_sig))
def main():
parser = otsclient.args.make_common_options_arg_parser()
parser.add_argument("-g",
"--gpg-program",
action="store",
default="/usr/bin/gpg",
help="Path to the GnuPG binary (default %(default)s)")
parser.add_argument(
'-c',
'--calendar',
metavar='URL',
dest='calendar_urls',
action='append',
type=str,
default=[
"https://calendar.bitmark.one",
"https://a.pool.opentimestamps.org",
"https://b.pool.opentimestamps.org",
"https://a.pool.eternitywall.com", "https://ots.btc.catallaxy.com"
],
help=
'Create timestamp with the aid of a remote calendar. May be specified multiple times. Default: %(default)r'
)
parser.add_argument(
'-b',
'--btc-wallet',
dest='use_btc_wallet',
action='store_true',
help='Create timestamp locally with the local Bitcoin wallet.')
parser.add_argument("gpgargs",
nargs=argparse.REMAINDER,
help='Arguments passed to GnuPG binary')
parser.add_argument("--timeout",
type=int,
default=5,
help="Timeout before giving up on a calendar. "
"Default: %(default)d")
parser.add_argument("-m",
type=int,
default="2",
help="Commitments are sent to remote calendars,"
"in the event of timeout the timestamp is considered "
"done if at least M calendars replied. "
"Default: %(default)s")
parser.add_argument('--rehash-trees',
action='store_true',
help=argparse.SUPPRESS)
args = otsclient.args.handle_common_options(parser.parse_args(), parser)
logging.basicConfig(format='ots: %(message)s')
args.verbosity = args.verbose - args.quiet
if args.verbosity == 0:
logging.root.setLevel(logging.INFO)
elif args.verbosity > 0:
logging.root.setLevel(logging.DEBUG)
elif args.verbosity == -1:
logging.root.setLevel(logging.WARNING)
elif args.verbosity < -1:
logging.root.setLevel(logging.ERROR)
if len(args.gpgargs) == 0 or args.gpgargs[0] != '--':
parser.error("You need to have '--' as the last argument; see docs")
args.gpgargs = args.gpgargs[1:]
parser = argparse.ArgumentParser()
parser.add_argument("-bsau", action="store")
parser.add_argument("--verify", action="store")
gpgargs = parser.parse_known_args(args.gpgargs)[0]
if gpgargs.bsau:
with subprocess.Popen([args.gpg_program] + args.gpgargs,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE) as gpg_proc:
logging.debug("Reading Git commit")
git_commit = sys.stdin.buffer.read()
logging.debug("Git commit: %r" % git_commit)
# FIXME: can this fail to write all bytes?
n = gpg_proc.stdin.write(git_commit)
logging.debug("Wrote %d bytes to GnuPG out of %d" %
(n, len(git_commit)))
gpg_proc.stdin.close()
gpg_sig = gpg_proc.stdout.read()
# GnuPG produces no output on failure
if not gpg_sig:
sys.exit(1)
logging.debug("PGP sig: %r" % gpg_sig)
# Timestamp the commit and tag together
signed_commit_timestamp = Timestamp(
hash_signed_commit(git_commit, gpg_sig))
final_timestamp = signed_commit_timestamp
# with git tree rehashing
minor_version = 1
# CWD will be the git repo, so this should get us the right one
repo = git.Repo()
hextree_start = None
if git_commit.startswith(b'tree '):
hextree_start = 5
elif git_commit.startswith(b'object '):
# I believe this is always a git tag
hextree_start = 7
else:
raise AssertionError("Don't know what to do with %r" %
git_commit)
hextree = git_commit[hextree_start:hextree_start + 20 * 2].decode()
tree = repo.tree(hextree)
tree.path = ''
tree_stamper = GitTreeTimestamper(tree)
final_timestamp = signed_commit_timestamp.ops.add(
OpAppend(tree_stamper.timestamp.msg)).ops.add(OpSHA256())
otsclient.cmds.create_timestamp(final_timestamp,
args.calendar_urls, args)
if args.wait:
# Interpreted as override by the upgrade command
# FIXME: need to clean this bad abstraction up!
args.calendar_urls = []
otsclient.cmds.upgrade_timestamp(signed_commit_timestamp, args)
sys.stdout.buffer.write(gpg_sig)
write_ascii_armored(signed_commit_timestamp, sys.stdout.buffer,
minor_version)
elif gpgargs.verify:
# Verify
with open(gpgargs.verify, 'rb') as gpg_sig_fd:
gpg_sig = gpg_sig_fd.read()
git_commit = sys.stdin.buffer.read()
(major_version, minor_version,
timestamp) = deserialize_ascii_armored_timestamp(
git_commit, gpg_sig)
if timestamp is None:
print("OpenTimestamps: No timestamp found", file=sys.stderr)
else:
good = otsclient.cmds.verify_timestamp(timestamp, args)
if good:
logging.info("Good timestamp")
else:
logging.warning("Could not verify timestamp!")
sys.stderr.flush()
logging.debug("Running GnuPG binary: %r" %
([args.gpg_program] + args.gpgargs))
with subprocess.Popen([args.gpg_program] + args.gpgargs,
stdin=subprocess.PIPE) as gpg_proc:
gpg_proc.stdin.write(git_commit)
gpg_proc.stdin.close()
def __do_bitcoin(self):
"""Do Bitcoin-related maintenance"""
# FIXME: we shouldn't have to create a new proxy each time, but with
# current python-bitcoinlib and the RPC implementation it seems that
# the proxy connection can timeout w/o recovering properly.
proxy = bitcoin.rpc.Proxy()
new_blocks = self.known_blocks.update_from_proxy(proxy)
for (block_height, block_hash) in new_blocks:
logging.info("New block %s at height %d" % (b2lx(block_hash), block_height))
# Save commitments to disk that have reached min_confirmations
confirmed_tx = self.txs_waiting_for_confirmation.pop(block_height - self.min_confirmations + 1, None)
if confirmed_tx is not None:
self.__save_confirmed_timestamp_tx(confirmed_tx)
# If there already are txs waiting for confirmation at this
# block_height, there was a reorg and those pending commitments now
# need to be added back to the pool
reorged_tx = self.txs_waiting_for_confirmation.pop(block_height, None)
if reorged_tx is not None:
# FIXME: the reorged transaction might get mined in another
# block, so just adding the commitments for it back to the pool
# isn't ideal, but it is safe
logging.info('tx %s at height %d removed by reorg, adding %d commitments back to pending' % (b2lx(reorged_tx.tx.GetHash()), block_height, len(reorged_tx.commitment_timestamps)))
for reorged_commitment_timestamp in reorged_tx.commitment_timestamps:
self.pending_commitments.add(reorged_commitment_timestamp.msg)
# Check if this block contains any of the pending transactions
try:
block = proxy.getblock(block_hash)
except KeyError:
# Must have been a reorg or something, return
logging.error("Failed to get block")
return
# Check all potential pending txs against this block.
for tx in self.unconfirmed_txs:
block_timestamp = make_timestamp_from_block(tx.tip_timestamp.msg, block, block_height)
if block_timestamp is None:
continue
# Success!
tx.tip_timestamp.merge(block_timestamp)
for commitment_timestamp in tx.commitment_timestamps:
self.pending_commitments.remove(commitment_timestamp.msg)
logging.debug("Removed commitment %s from pending" % b2x(commitment_timestamp.msg))
assert self.min_confirmations > 1
logging.info("Success! %d commitments timestamped, now waiting for %d more confirmations" %
(len(tx.commitment_timestamps), self.min_confirmations - 1))
# Add pending_tx to the list of timestamp transactions that
# have been mined, and are waiting for confirmations.
self.txs_waiting_for_confirmation[block_height] = tx
# Since all unconfirmed txs conflict with each other, we can clear the entire lot
self.unconfirmed_txs.clear()
# And finally, we can reset the last time a timestamp
# transaction was mined to right now.
self.last_timestamp_tx = time.time()
time_to_next_tx = int(self.last_timestamp_tx + self.min_tx_interval - time.time())
if time_to_next_tx > 0:
# Minimum interval between transactions hasn't been reached, so do nothing
logging.debug("Waiting %ds before next tx" % time_to_next_tx)
return
prev_tx = None
if self.pending_commitments and not self.unconfirmed_txs:
# Find the biggest unspent output that's confirmed
unspent = find_unspent(proxy)
if not len(unspent):
logging.error("Can't timestamp; no spendable outputs")
return
# For the change scriptPubKey, we can save a few bytes by using
# a pay-to-pubkey rather than the usual pay-to-pubkeyhash
change_addr = proxy.getnewaddress()
change_pubkey = proxy.validateaddress(change_addr)['pubkey']
change_scriptPubKey = CScript([change_pubkey, OP_CHECKSIG])
prev_tx = self.__create_new_timestamp_tx_template(unspent[-1]['outpoint'], unspent[-1]['amount'], change_scriptPubKey)
logging.debug('New timestamp tx, spending output %r, value %s' % (unspent[-1]['outpoint'], str_money_value(unspent[-1]['amount'])))
elif self.unconfirmed_txs:
(prev_tx, prev_tip_timestamp, prev_commitment_timestamps) = self.unconfirmed_txs[-1]
# Send the first transaction even if we don't have a new block
if prev_tx and (new_blocks or not self.unconfirmed_txs):
# Update the most recent timestamp transaction with new commitments
commitment_timestamps = [Timestamp(commitment) for commitment in self.pending_commitments]
# Remember that commitment_timestamps contains raw commitments,
# which are longer than necessary, so we sha256 them before passing
# them to make_merkle_tree, which concatenates whatever it gets (or
# for the matter, returns what it gets if there's only one item for
# the tree!)
commitment_digest_timestamps = [stamp.ops.add(OpSHA256()) for stamp in commitment_timestamps]
tip_timestamp = make_merkle_tree(commitment_digest_timestamps)
sent_tx = None
relay_feerate = self.relay_feerate
while sent_tx is None:
unsigned_tx = self.__update_timestamp_tx(prev_tx, tip_timestamp.msg,
proxy.getblockcount(), relay_feerate)
fee = _get_tx_fee(unsigned_tx, proxy)
if fee is None:
logging.debug("Can't determine txfee of transaction; skipping")
return
if fee > self.max_fee:
logging.error("Maximum txfee reached!")
return
r = proxy.signrawtransaction(unsigned_tx)
if not r['complete']:
logging.error("Failed to sign transaction! r = %r" % r)
return
signed_tx = r['tx']
try:
txid = proxy.sendrawtransaction(signed_tx)
except bitcoin.rpc.JSONRPCError as err:
if err.error['code'] == -26:
logging.debug("Err: %r" % err.error)
# Insufficient priority - basically means we didn't
# pay enough, so try again with a higher feerate
relay_feerate *= 2
continue
else:
raise err # something else, fail!
sent_tx = signed_tx
if self.unconfirmed_txs:
logging.info("Sent timestamp tx %s, replacing %s; %d total commitments" % (b2lx(sent_tx.GetHash()), b2lx(prev_tx.GetHash()), len(commitment_timestamps)))
else:
logging.info("Sent timestamp tx %s; %d total commitments" % (b2lx(sent_tx.GetHash()), len(commitment_timestamps)))
self.unconfirmed_txs.append(TimestampTx(sent_tx, tip_timestamp, commitment_timestamps))
def nonce_timestamp(private_timestamp, crypt_op=OpSHA256(), length=16):
"""Create a nonced version of a timestamp for privacy"""
stamp2 = private_timestamp.ops.add(OpAppend(os.urandom(length)))
return stamp2.ops.add(crypt_op)
