def filecoin_wallets():
"""
Retrieve all wallets from all FFSes and save them in a list for
presentation on the UI template
"""
powergate = PowerGateClient(app.config["POWERGATE_ADDRESS"])
try:
ffs = Ffs.query.filter_by(user_id=current_user.id).one()
except:
flash("No wallets created yet.", "alert-danger")
return render_template("filecoin/filecoin-wallets.html",
wallets=None,
breadcrumb="Filecoin / Wallets")
wallets = []
addresses = powergate.ffs.addrs_list(ffs.token)
for address in addresses.addrs:
balance = powergate.wallet.balance(address.addr)
wallets.append({
"ffs": ffs.ffs_id,
"name": address.name,
"address": address.addr,
"type": address.type,
"balance": str(balance.balance),
})
return render_template("filecoin/filecoin-wallets.html",
wallets=wallets,
breadcrumb="Filecoin / Wallets")
def push(cid):
try:
abs_path = os.getcwd()
split = os.path.split(abs_path)
workflow_db_path = os.path.join(
split[0], "pipeline/deplatformr_open_images_workflow.sqlite")
workflow_db = sqlite3.connect(workflow_db_path)
api = os.getenv('POWERGATE_API')
ffs = os.getenv('POWERGATE_FFS')
token = os.getenv('POWERGATE_TOKEN')
powergate = PowerGateClient(api, is_secure=False)
utctime = datetime.utcnow()
job = powergate.ffs.push(cid, token, override=False)
print("Repushed CID " + cid + " to Filecoin.")
print("Job ID: " + job.job_id)
utctime = datetime.utcnow()
cursor = workflow_db.cursor()
cursor.execute("INSERT INTO jobs (job_id, cid, ffs, timestamp, status) VALUES (?,?,?,?,?)",
(job.job_id, cid, ffs, utctime, "JOB_STATUS_EXECUTING",),)
workflow_db.commit()
workflow_db.close()
except Exception as e:
print("Repush of CID " + cid + " failed.")
print(e)
return()
def job_checker():
sqlite_conn = sqlite3.connect('auditprotocol_1.db')
sqlite_cursor = sqlite_conn.cursor()
pow_client = PowerGateClient(fast_settings.config.powergate_url, False)
while True:
done_deal_jids = list()
deals_lock.acquire()
for deal_jid, deal in deals.items():
j_stat = pow_client.ffs.get_storage_job(jid=deal['jid'], token=deal['token'])
# print(j_stat.job)
if j_stat.job.status == 5: # 'JOB_STATUS_SUCCESS':
deal_watcher_logger.debug('Hurrah. Removing from deals to be watched...')
deal_watcher_logger.debug(j_stat.job)
done_deal_jids.append(deal_jid)
# update status
sqlite_cursor.execute("""
UPDATE accounting_records SET confirmed=1 WHERE cid=?
""", (deal['cid'], ))
sqlite_cursor.connection.commit()
elif j_stat.job.status == 3:
deal_watcher_logger.error('Job failed. Removing from deals to be watched...')
deal_watcher_logger.error(j_stat.job)
done_deal_jids.append(deal_jid)
# update status
sqlite_cursor.execute("""
UPDATE accounting_records SET confirmed=2 WHERE cid=?
""", (deal['cid'],))
sqlite_cursor.connection.commit()
for each_done_jid in done_deal_jids:
del deals[each_done_jid]
deals_lock.release()
time.sleep(5)
def create_ffs():
"""
Create a new Powergate Filecoin Filesystem (FFS)
"""
powergate = PowerGateClient(app.config["POWERGATE_ADDRESS"])
ffs = powergate.ffs.create()
creation_date = datetime.now().replace(microsecond=0)
# TODO salt token id
filecoin_file_system = Ffs(
ffs_id=ffs.id,
token=ffs.token,
creation_date=creation_date,
user_id=current_user.id,
)
db.session.add(filecoin_file_system)
# Create new FFS wallet and add entry in log table
address = powergate.ffs.addrs_list(ffs.token)
obj = MessageToDict(address)
wallet = obj["addrs"][0]["addr"]
wallet = Wallets(
created=creation_date,
address=wallet,
ffs=ffs.id,
user_id=current_user.id,
)
db.session.add(wallet)
db.session.commit()
new_ffs = Ffs.query.filter_by(ffs_id=ffs.id).first()
return new_ffs
async def create_filecoin_filesystem(request: Request):
if settings.file_upload_method == 'sia':
api_key = str(uuid4())
token = str(uuid4())
api_keys_table.add_row({'token': token, 'api_key': api_key})
return {'apiKey': api_key}
elif settings.file_upload_method == 'filecoin':
req_json = await request.json()
hot_enabled = req_json.get('hotEnabled', True)
pow_client = PowerGateClient(settings.powergate_url, False)
new_ffs = pow_client.ffs.create()
rest_logger.info('Created new FFS')
rest_logger.info(new_ffs)
if not hot_enabled:
default_config = pow_client.ffs.default_config(new_ffs.token)
rest_logger.debug(default_config)
new_storage_config = STORAGE_CONFIG
new_storage_config['cold']['filecoin'][
'addr'] = default_config.default_storage_config.cold.filecoin.addr
new_storage_config['hot']['enabled'] = False
new_storage_config['hot']['allowUnfreeze'] = False
pow_client.ffs.set_default_config(json.dumps(new_storage_config),
new_ffs.token)
rest_logger.debug('Set hot storage to False')
rest_logger.debug(new_storage_config)
# rest_logger.debug(type(default_config))
api_key = str(uuid4())
api_keys_table.add_row({'token': new_ffs.token, 'api_key': api_key})
# Add row to skydb
api_keys_table.add_row({'api_key': api_key, 'token': new_ffs.token})
rest_logger.debug("Added a row to api_keys_table")
return {'apiKey': api_key}
async def create_filecoin_filesystem(request: Request):
req_json = await request.json()
hot_enabled = req_json.get('hotEnabled', True)
pow_client = PowerGateClient(fast_settings.config.powergate_url, False)
new_ffs = pow_client.ffs.create()
rest_logger.info('Created new FFS')
rest_logger.info(new_ffs)
if not hot_enabled:
default_config = pow_client.ffs.default_config(new_ffs.token)
rest_logger.debug(default_config)
new_storage_config = STORAGE_CONFIG
new_storage_config['cold']['filecoin'][
'addr'] = default_config.default_storage_config.cold.filecoin.addr
new_storage_config['hot']['enabled'] = False
new_storage_config['hot']['allowUnfreeze'] = False
pow_client.ffs.set_default_config(json.dumps(new_storage_config),
new_ffs.token)
rest_logger.debug('Set hot storage to False')
rest_logger.debug(new_storage_config)
# rest_logger.debug(type(default_config))
api_key = str(uuid4())
request.app.sqlite_cursor.execute(
"""
INSERT INTO api_keys VALUES (?, ?)
""", (new_ffs.token, api_key))
request.app.sqlite_cursor.connection.commit()
return {'apiKey': api_key}
def push_to_filecoin(upload_path, file_name, platform):
# Push file to Filecoin via Powergate
powergate = PowerGateClient(app.config["POWERGATE_ADDRESS"])
# Retrieve information for default Filecoin FileSystem (FFS)
ffs = Ffs.query.filter_by(user_id=current_user.id).first()
if ffs is None:
# No FFS exists yet so create one
ffs = create_ffs()
try:
# Create an iterator of the uploaded file using the helper function
file_iterator = get_file_bytes(os.path.join(upload_path, file_name))
# Convert the iterator into request and then add to the hot set (IPFS)
file_hash = powergate.ffs.stage(bytes_to_chunks(file_iterator),
ffs.token)
# Push the file to Filecoin
powergate.ffs.push(file_hash.cid, ffs.token)
# Note the upload date and file size
upload_date = datetime.now().replace(microsecond=0)
file_size = os.path.getsize(os.path.join(upload_path, file_name))
# Save file information to database
file_upload = Files(
file_path=upload_path,
file_name=file_name,
upload_date=upload_date,
file_size=file_size,
CID=file_hash.cid,
platform=platform,
user_id=current_user.id,
ffs_id=ffs.id,
)
db.session.add(file_upload)
db.session.commit()
print(file_name + " added to Filecoin.")
except Exception as e:
# Output error message if pushing to Filecoin fails
flash("'{}' failed to upload to Filecoin. {}.".format(file_name, e),
"alert-danger")
# Update log table with error
event = Logs(
timestamp=datetime.now().replace(microsecond=0),
event="Upload ERROR: " + file_name + " " + str(e),
user_id=current_user.id,
)
db.session.add(event)
db.session.commit()
# TODO: RESPOND TO SPECIFIC STATUS CODE DETAILS
# (how to isolate these? e.g. 'status_code.details = ...')
return ()
def wallets():
"""
Retrieve all wallets from all FFSes and save them in a list for
presentation on the UI template
"""
powergate = PowerGateClient(app.config["POWERGATE_ADDRESS"])
ffses = Ffs.query.all()
wallets = []
for filecoin_filesystem in ffses:
addresses = powergate.ffs.addrs_list(filecoin_filesystem.token)
for address in addresses.addrs:
balance = powergate.wallet.balance(address.addr)
wallets.append({
"ffs": filecoin_filesystem.ffs_id,
"name": address.name,
"address": address.addr,
"type": address.type,
"balance": str(balance.balance),
})
return render_template("wallets.html", wallets=wallets)
def filecoin_upload(package):
try:
abs_path = os.getcwd()
package_dir = os.path.join(abs_path, "source_data/packages")
upload_file = os.path.join(package_dir, package)
api = os.getenv('POWERGATE_API')
token = os.getenv('POWERGATE_TOKEN')
powergate = PowerGateClient(api, is_secure=False)
db_path = os.path.join(abs_path,
"deplatformr_open_images_workflow.sqlite")
workflow_db = sqlite3.connect(db_path)
staged_file = powergate.data.stage_file(upload_file, token)
job = powergate.config.apply(staged_file.cid,
override=False,
token=token)
print("Uploaded package " + package + " to Filecoin.")
print("CID: " + staged_file.cid)
print("Job ID: " + job.jobId)
utctime = datetime.utcnow()
cursor = workflow_db.cursor()
cursor.execute(
"UPDATE packages SET cid = ?, cid_timestamp = ? WHERE name = ?",
(
staged_file.cid,
utctime,
package,
),
)
cursor.execute(
"INSERT INTO jobs (job_id, cid, timestamp, status) VALUES (?,?,?,?)",
(
job.jobId,
staged_file.cid,
utctime,
"JOB_STATUS_EXECUTING",
),
)
workflow_db.commit()
workflow_db.close()
return ("Success")
except Exception as e:
print("Unable to push package " + package + " to Filecoin.")
print(e)
return ("Failure")
def deal_status(cid):
api = os.getenv('POWERGATE_API')
token = os.getenv('POWERGATE_TOKEN')
powergate = PowerGateClient(api, is_secure=False)
status = powergate.data.cid_info(cids=[cid], token=token)
print("CID: " + status[0][0])
print("Job ID: " + status[0][2]["id"])
print("Status: " + status[0][2]["status"])
table = []
for deal in status[0][2]["dealInfo"]:
if deal["stateName"]=="StorageDealError":
message = deal["message"]
else:
message = ""
table+=[(deal["stateName"], deal["miner"], deal["pricePerEpoch"], message)]
print(tabulate(table))
async def root(request: Request,
response: Response,
api_key_extraction=Depends(load_user_from_auth)):
if not api_key_extraction:
response.status_code = status.HTTP_403_FORBIDDEN
return {'error': 'Forbidden'}
if not api_key_extraction['token']:
response.status_code = status.HTTP_403_FORBIDDEN
return {'error': 'Forbidden'}
pow_client = PowerGateClient(fast_settings.config.powergate_url, False)
# if request.method == 'POST':
req_args = await request.json()
payload = req_args['payload']
token = api_key_extraction['token']
payload_bytes = BytesIO(payload.encode('utf-8'))
payload_iter = bytes_to_chunks(payload_bytes)
# adds to hot tier, IPFS
stage_res = pow_client.ffs.stage(payload_iter, token=token)
rest_logger.debug('Staging level results:')
rest_logger.debug(stage_res)
# uploads to filecoin
push_res = pow_client.ffs.push(stage_res.cid, token=token)
rest_logger.debug('Cold tier finalization results:')
rest_logger.debug(push_res)
await request.app.redis_pool.publish_json('new_deals', {
'cid': stage_res.cid,
'jid': push_res.job_id,
'token': token
})
payload_hash = '0x' + keccak(text=payload).hex()
token_hash = '0x' + keccak(text=token).hex()
tx_hash_obj = contract.commitRecordHash(
**dict(payloadHash=payload_hash, apiKeyHash=token_hash))
tx_hash = tx_hash_obj[0]['txHash']
rest_logger.debug('Committed record append to contract..')
rest_logger.debug(tx_hash_obj)
local_id = str(uuid4())
request.app.sqlite_cursor.execute(
'INSERT INTO accounting_records VALUES '
'(?, ?, ?, ?, ?, ?)',
(token, stage_res.cid, local_id, tx_hash, 0, int(time.time())))
request.app.sqlite_cursor.connection.commit()
return {'commitTx': tx_hash, 'recordCid': local_id}
def create_ffs(default=False):
"""
Create a new Powergate Filecoin Filesystem (FFS)
"""
powergate = PowerGateClient(app.config["POWERGATE_ADDRESS"])
if default == True:
default_ffs = Ffs.query.filter_by(default=True).first()
if default_ffs is not None:
default_ffs.default = False
db.session.commit()
ffs = powergate.ffs.create()
creation_date = datetime.now().replace(microsecond=0)
filecoin_file_system = Ffs(
ffs_id=ffs.id, token=ffs.token, creation_date=creation_date, default=default,
)
db.session.add(filecoin_file_system)
# Record new FFS creation in log table
event = Logs(
timestamp=creation_date,
event="Created new Filecoin FileSystem (FFS): " + ffs.id,
)
db.session.add(event)
db.session.commit()
# Record creation of new FFS wallet in log table
address = powergate.ffs.addrs_list(ffs.token)
obj = MessageToDict(address)
wallet = obj["addrs"][0]["addr"]
event = Logs(timestamp=creation_date, event="Created new Wallet: " + wallet,)
db.session.add(event)
db.session.commit()
new_ffs = Ffs.query.filter_by(ffs_id=ffs.id).first()
return new_ffs
from pygate_grpc.client import PowerGateClient
from pygate_grpc.ffs import get_file_bytes, bytes_to_chunks
if __name__ == "__main__":
hostName = "127.0.0.1:5002"
# Create client
c = PowerGateClient(hostName, False)
# Create FFS
ffs = c.ffs.create()
print("FFS created:")
print(ffs)
# Create an iterator of the given file using the helper function
iter = get_file_bytes("README.md")
print("Grabbing pygate-grpc 'README.md' file...")
print("Adding file to IPFS (hot storage)...")
# Convert the iterator into request and then add to hot set
res = c.ffs.stage(bytes_to_chunks(iter), ffs.token)
print(res)
print("Pushing file to FFS...")
# Push the given file
c.ffs.push(res.cid, override=False, token=ffs.token)
# Override push with another config
addresses = c.ffs.addrs_list(ffs.token)
wallet = addresses.addrs[0].addr
new_config = (
async def root(request: Request,
response: Response,
api_key_extraction=Depends(load_user_from_auth)):
if not api_key_extraction:
response.status_code = status.HTTP_403_FORBIDDEN
return {'error': 'Forbidden'}
if not api_key_extraction['token']:
response.status_code = status.HTTP_403_FORBIDDEN
return {'error': 'Forbidden'}
req_args = await request.json()
payload = req_args['payload']
token = api_key_extraction['token']
if settings.file_upload_method == "sia":
h = hashlib.sha256()
h.update(payload.encode())
sha_payload_hash = h.hexdigest()
upload_to_sia(sha_payload_hash, payload)
payload_hash = '0x' + keccak(text=payload).hex()
token_hash = '0x' + keccak(text=token).hex()
tx_hash_obj = contract.commitRecordHash(
**dict(payloadHash=payload_hash, apiKeyHash=token_hash))
tx_hash = tx_hash_obj[0]['txHash']
rest_logger.debug('Committed record append to contract..')
rest_logger.debug(tx_hash_obj)
local_id = str(uuid4())
timestamp = int(time.time())
rest_logger.debug("Adding row to accounting_records_table")
accounting_records_table.add_row({
'token': token,
'cid': sha_payload_hash,
'localCID': local_id,
'txHash': tx_hash,
'timestamp': timestamp,
'confirmed': -1,
})
return {'commitTx': tx_hash, 'recordCid': local_id}
elif settings.file_upload_method == "filecoin":
pow_client = PowerGateClient(settings.powergate_url, False)
# if request.method == 'POST':
payload_bytes = BytesIO(payload.encode('utf-8'))
payload_iter = bytes_to_chunks(payload_bytes)
# adds to hot tier, IPFS
stage_res = pow_client.ffs.stage(payload_iter, token=token)
rest_logger.debug('Staging level results:')
rest_logger.debug(stage_res)
# uploads to filecoin
push_res = pow_client.ffs.push(stage_res.cid, token=token)
rest_logger.debug('Cold tier finalization results:')
rest_logger.debug(push_res)
await request.app.redis_pool.publish_json('new_deals', {
'cid': stage_res.cid,
'jid': push_res.job_id,
'token': token
})
payload_hash = '0x' + keccak(text=payload).hex()
token_hash = '0x' + keccak(text=token).hex()
tx_hash_obj = contract.commitRecordHash(
**dict(payloadHash=payload_hash, apiKeyHash=token_hash))
tx_hash = tx_hash_obj[0]['txHash']
rest_logger.debug('Committed record append to contract..')
rest_logger.debug(tx_hash_obj)
local_id = str(uuid4())
timestamp = int(time.time())
rest_logger.debug("Adding row to accounting_records_table")
# Add row to skydb
print(f"Adding cid: {stage_res.cid}")
accounting_records_table.add_row({
'token': token,
'cid': stage_res.cid,
'localCID': local_id,
'txHash': tx_hash,
'confirmed': 0,
'timestamp': timestamp
})
return {'commitTx': tx_hash, 'recordCid': local_id}
from io import BytesIO
from pygate_grpc.client import PowerGateClient
from pygate_grpc.ffs import bytes_to_chunks
from pygate_grpc.exceptions import GRPCTimeoutException
client = PowerGateClient("127.0.0.1:5002", False)
print("Creating a new FFS:")
ffs = client.ffs.create()
print(ffs)
test_file = BytesIO(b"These are the contents of a test file")
stage_requests_iter = bytes_to_chunks(test_file)
stage_res = client.ffs.stage(stage_requests_iter, ffs.token)
push_res = client.ffs.push(stage_res.cid, token=ffs.token)
logs_res = client.ffs.logs(stage_res.cid, ffs.token, history=True, timeout=5)
logs = []
# iterating through the logs is a blocking operation, by using a timeout and
# the following exception handling we can make sure that the operation exits
# after the specified timeout.
try:
for f in logs_res:
logs.append(f)
except GRPCTimeoutException:
pass
print(logs)
def pygate_client():
return PowerGateClient("127.0.0.1:5002", False)
def main():
r = redis.StrictRedis(host=settings['REDIS']['HOST'],
port=settings['REDIS']['PORT'],
db=settings['REDIS']['DB'],
password=settings['REDIS']['PASSWORD'])
pow_client = PowerGateClient(fast_settings.config.powergate_url, False)
awaited_deals = list()
while True:
retrieval_worker_logger.debug("Looping...")
while True:
retrieval_worker_logger.debug('Waiting for Lock')
v = r.incr('my_lock')
if v == 1:
retreivals_single_table.calibrate_index()
time.sleep(0.1)
rows = retreivals_single_table.fetch(
condition={'completed': '0'},
start_index=retreivals_single_table.index - 1,
n_rows=retreivals_single_table.index - 1)
v = r.decr('my_lock')
break
v = r.decr('my_lock')
time.sleep(0.1)
#single_retrieval_requests = c.fetchall()
single_retrieval_requests = rows.keys()
for retrieval_request in single_retrieval_requests:
retrieval_worker_logger.debug(retrieval_request)
request_id = rows[retrieval_request]['requestID']
ffs_cid = rows[retrieval_request]['cid']
local_cid = rows[retrieval_request]['localCID']
while True:
retrieval_worker_logger.debug('Waiting for Lock')
v = r.incr('my_lock')
if v == 1:
accounting_records_table.calibrate_index()
time.sleep(0.1)
acc_row = accounting_records_table.fetch(
condition={'localCID': local_cid},
start_index=accounting_records_table.index - 1,
n_rows=1,
)
v = r.decr('my_lock')
break
v = r.decr('my_lock')
time.sleep(0.1)
assert len(
acc_row) == 1, f"No row found for localCID: {local_cid}"
acc_row = acc_row[next(iter(acc_row.keys()))]
token = acc_row['token']
retrieval_worker_logger.debug("Retrieving file " + ffs_cid +
" from FFS.")
file_ = pow_client.ffs.get(ffs_cid, token)
file_name = f'static/{request_id}'
retrieval_worker_logger.debug('Saving to ' + file_name)
try:
with open(file_name, 'wb') as f_:
for _ in file_:
f_.write(_)
except Exception:
retrieval_worker_logger.debug('File has alreadby been saved')
retreivals_single_table.update_row(row_index=retrieval_request,
data={
'retreived_file':
'/' + file_name,
'completed': 1
})
# bulk retrievals
while True:
retrieval_worker_logger.debug('Waiting for Lock')
v = r.incr('my_lock')
if v == 1:
retreivals_bulk_table.calibrate_index()
time.sleep(0.1)
bulk_rows = retreivals_bulk_table.fetch(
condition={'completed': '0'},
start_index=retreivals_bulk_table.index - 1,
n_rows=retreivals_bulk_table.index - 1)
v = r.decr('my_lock')
break
v = r.decr('my_lock')
time.sleep(0.1)
bulk_retrieval_requests = bulk_rows.keys()
for bulk_retrieval_request in bulk_retrieval_requests:
retrieval_worker_logger.debug(bulk_rows[bulk_retrieval_request])
request_id = bulk_rows[bulk_retrieval_request]['requestID']
api_key = bulk_rows[bulk_retrieval_request]['api_key']
ffs_token = bulk_rows[bulk_retrieval_request]['token']
retrieval_worker_logger.debug(
f"{request_id}, {api_key}, {ffs_token}")
file_name = f'static/{request_id}'
fp = open(file_name, 'wb')
while True:
retrieval_worker_logger.debug('Waiting for Lock')
v = r.incr('my_lock')
if v == 1:
accounting_records_table.calibrate_index()
time.sleep(0.1)
records_rows = accounting_records_table.fetch(
condition={'token': ffs_token},
n_rows=accounting_records_table.index - 1,
start_index=accounting_records_table.index - 1)
v = r.decr('my_lock')
break
v = r.decr('my_lock')
time.sleep(0.1)
records_c = records_rows.keys()
for each_record in records_c:
ffs_cid = records_rows[each_record]['cid']
retrieval_worker_logger.debug("Bulk mode: Retrieving CID " +
ffs_cid + " from FFS.")
file_ = pow_client.ffs.get(ffs_cid, ffs_token)
try:
for _ in file_:
fp.write(_)
except Exception as e:
print(e)
fp.close()
retreivals_bulk_table.update_row(row_index=bulk_retrieval_request,
data={
'retreived_file':
'/' + file_name,
'completed': 1
})
retrieval_worker_logger.debug('Bulk mode: Marking request ID ' +
request_id + ' as completed')
time.sleep(5)
def config(ffs_id=None):
"""
Create and edit FFS config settings
"""
NewFFSForm = NewFfsForm()
powergate = PowerGateClient(app.config["POWERGATE_ADDRESS"])
if ffs_id == None:
active_ffs = Ffs.query.filter_by(default=True).first()
else:
active_ffs = Ffs.query.filter_by(ffs_id=ffs_id).first()
if active_ffs == None:
active_ffs = create_ffs(default=True)
default_config = powergate.ffs.default_config(active_ffs.token)
"""TODO: Move MessageToDict helper to pygate-grpc client and return dict"""
msg_dict = MessageToDict(default_config)
# populate form values from config dictionary
try:
hot_enabled = msg_dict["defaultStorageConfig"]["hot"]["enabled"]
except KeyError:
hot_enabled = False
try:
allow_unfreeze = msg_dict["defaultStorageConfig"]["hot"][
"allowUnfreeze"]
except KeyError:
allow_unfreeze = False
try:
add_timeout = msg_dict["defaultStorageConfig"]["hot"]["ipfs"][
"addTimeout"]
except KeyError:
add_timeout = 0
try:
cold_enabled = msg_dict["defaultStorageConfig"]["cold"]["enabled"]
except KeyError:
cold_enabled = False
try:
rep_factor = msg_dict["defaultStorageConfig"]["cold"]["filecoin"][
"repFactor"]
except KeyError:
rep_factor = 0
try:
deal_min_duration = msg_dict["defaultStorageConfig"]["cold"][
"filecoin"]["dealMinDuration"]
except KeyError:
deal_min_duration = 0
try:
excluded_miners_list = msg_dict["defaultStorageConfig"]["cold"][
"filecoin"]["excludedMiners"]
# Break out list into comma separate values
excluded_miners = ""
for miner in excluded_miners_list:
excluded_miners += miner
# Don't add a comma if this is only or last item in list
if (len(excluded_miners_list)) > 1:
if (excluded_miners_list.index(miner) +
1) != len(excluded_miners_list):
excluded_miners += ","
except KeyError:
excluded_miners = None
try:
trusted_miners_list = msg_dict["defaultStorageConfig"]["cold"][
"filecoin"]["trustedMiners"]
# Break out list into comma separate values
trusted_miners = ""
for miner in trusted_miners_list:
trusted_miners += miner
# Don't add a comma if this is only or last item in list
if (len(trusted_miners_list)) > 1:
if (trusted_miners_list.index(miner) +
1) != len(trusted_miners_list):
trusted_miners += ","
except KeyError:
trusted_miners = None
try:
country_codes_list = msg_dict["defaultStorageConfig"]["cold"][
"filecoin"]["countryCodes"]
# Break out list into comma separate values
country_codes = ""
for country in country_codes_list:
country_codes += country
# Don't add a comma if this is only or last item in list
if (len(country_codes_list)) > 1:
if (country_codes_list.index(country) +
1) != len(country_codes_list):
country_codes += ","
except KeyError:
country_codes = None
try:
renew_enabled = msg_dict["defaultStorageConfig"]["cold"]["filecoin"][
"renew"]["enabled"]
except KeyError:
renew_enabled = False
try:
renew_threshold = msg_dict["defaultStorageConfig"]["cold"]["filecoin"][
"renew"]["threshold"]
except KeyError:
renew_threshold = 0
wallet_address = msg_dict["defaultStorageConfig"]["cold"]["filecoin"][
"addr"]
try:
max_price = msg_dict["defaultStorageConfig"]["cold"]["filecoin"][
"maxPrice"]
except KeyError:
max_price = 0
try:
repairable = msg_dict["defaultStorageConfig"]["repairable"]
except KeyError:
repairable = True
# Instantiate config form
ConfigForm = FfsConfigForm()
ConfigForm.make_default.data = active_ffs.default
ConfigForm.hot_enabled.data = hot_enabled
ConfigForm.allow_unfreeze.data = allow_unfreeze
ConfigForm.add_timeout.data = add_timeout
ConfigForm.cold_enabled.data = cold_enabled
ConfigForm.rep_factor.data = rep_factor
ConfigForm.deal_min_duration.data = deal_min_duration
ConfigForm.excluded_miners.data = excluded_miners
ConfigForm.trusted_miners.data = trusted_miners
ConfigForm.country_codes.data = country_codes
ConfigForm.renew_enabled.data = renew_enabled
ConfigForm.renew_threshold.data = renew_threshold
ConfigForm.max_price.data = max_price
ConfigForm.repairable.data = repairable
all_ffses = Ffs.query.order_by((Ffs.default).desc()).all()
return render_template(
"config.html",
NewFfsForm=NewFFSForm,
FfsConfigForm=ConfigForm,
wallet_address=wallet_address,
active_ffs=active_ffs,
all_ffses=all_ffses,
)
def change_config(ffs_id, wallet):
"""
Change the default configuration for a FFS, triggering a change to all files
that were uploaded using that config.
"""
ffs = Ffs.query.filter_by(ffs_id=ffs_id).first()
form = FfsConfigForm(request.form)
# Change default FFS
if form.make_default.data == True:
current_default_ffs = Ffs.query.filter_by(default=True).first()
if current_default_ffs is not None:
current_default_ffs.default = False
ffs.default = True
db.session.commit()
exlcude_miners = []
trust_miners = []
countries = []
exclude_miners = form.excluded_miners.data.split(",")
trust_miners = form.trusted_miners.data.split(",")
countries = form.country_codes.data.split(",")
new_config = {
"hot": {
"enabled": form.hot_enabled.data,
"allowUnfreeze": form.allow_unfreeze.data,
"ipfs": {
"addTimeout": form.add_timeout.data
},
},
"cold": {
"enabled": form.cold_enabled.data,
"filecoin": {
"repFactor": form.rep_factor.data,
"dealMinDuration": form.deal_min_duration.data,
"excludedMiners": exclude_miners,
"trustedMiners": trust_miners,
"countryCodes": countries,
"renew": {
"enabled": form.renew_enabled.data,
"threshold": form.renew_threshold.data,
},
"addr": wallet,
"maxPrice": form.max_price.data,
},
},
"repairable": True,
}
new_config_json = json.dumps(new_config)
powergate = PowerGateClient(app.config["POWERGATE_ADDRESS"])
try:
powergate.ffs.set_default_config(new_config_json, ffs.token)
event = "Changed default configuration for FFS " + ffs.ffs_id
except Exception as e:
# Output error message if download from Filecoin fails
flash("failed to change configuration. {}".format(e))
event = str(e)
# Log the configuration change or error
event = Logs(
timestamp=datetime.now().replace(microsecond=0),
event=event,
)
db.session.add(event)
db.session.commit()
return redirect(url_for("config", ffs_id=ffs_id))
def filecoin_download(cid):
"""
Retrieve a file from Filecoin via IPFS using Powergate and offer the user
the option to save it to their machine.
"""
# Retrieve File and FFS info using the CID
file = Files.query.filter_by(CID=cid, user_id=current_user.id).first()
ffs = Ffs.query.get(file.ffs_id)
try:
# Retrieve data from Filecoin
powergate = PowerGateClient(app.config["POWERGATE_ADDRESS"])
data_ = powergate.ffs.get(file.CID, ffs.token)
# Save the downloaded data as a file
# Use the user data directory configured for the app
user_data = app.config["USER_DATA_DIR"]
if not os.path.exists(user_data):
os.makedirs(user_data)
print(user_data)
# Create a subdirectory per username. Usernames are unique.
user_dir = os.path.join(
user_data,
str(current_user.id) + "-" + current_user.username)
if not os.path.exists(user_dir):
os.makedirs(user_dir)
print(user_dir)
# Create a Filecoin downloads subdirectory.
filecoin_dir = os.path.join(user_dir, "filecoin/downloads")
if not os.path.exists(filecoin_dir):
os.makedirs(filecoin_dir)
print(filecoin_dir)
with open(os.path.join(filecoin_dir, file.file_name),
"wb") as out_file:
# Iterate over the data byte chunks and save them to an output file
for data in data_:
out_file.write(data)
# Create path to download file
safe_path = safe_join("../" + filecoin_dir, file.file_name)
print(safe_path)
# Offer the file for download to local machine
return send_file(safe_path, as_attachment=True)
# TODO: CLEAR CACHED FILES IN DOWNLOAD DIRECTORY
except Exception as e:
# Output error message if download from Filecoin fails
flash(
"failed to download '{}' from Filecoin. {}".format(
file.file_name, e), "alert-danger")
# Update log table with error
event = Logs(
timestamp=datetime.now().replace(microsecond=0),
event="Download ERROR: " + file.file_name + " CID: " + file.CID +
" " + str(e),
user_id=current_user.id,
)
db.session.add(event)
db.session.commit()
files = Files.query.filter_by(user_id=current_user.id).all()
return render_template("filecoin/filecoin-files.html",
files=files,
breadcrumb="Filecoin / Files")
def test_get_user_id(pygate_client: PowerGateClient):
user = pygate_client.admin.users.create()
user_id = pygate_client.user_id(user.token)
assert user_id == user.id
def test_build_info(pygate_client: PowerGateClient):
build_info = pygate_client.build_info()
assert build_info is not None
assert type(build_info) == BuildInfo
import time
from pygate_grpc.client import PowerGateClient
if __name__ == "__main__":
hostName = "127.0.0.1:5002"
# Create client
client = PowerGateClient(hostName)
# Create user
user = client.admin.users.create()
print("User created:")
print(user)
print("Applying storage config...")
stage_res = client.data.stage_bytes(
b"These are the contents of a test file", token=user.token)
apply_res = client.config.apply(stage_res.cid, token=user.token)
# Check that cid is in the process of being stored by Powegate
check = client.data.cid_info([stage_res.cid], user.token)
print("Checking cid storage...")
print(check)
# Wait some time so that we can get some deals
time.sleep(60)
# Check information about the storage deal
storage_deals = client.deals.storage_deal_records(include_pending=True,
import os
import csv
from datetime import datetime
from pygate_grpc.client import PowerGateClient
api = os.getenv('POWERGATE_API')
token = os.getenv('POWERGATE_TOKEN')
powergate = PowerGateClient(api, False)
# get final storage deals info
storage_deals = powergate.deals.storage_deal_records(include_pending=True,
include_final=False,
token=token)
total_deals = len(storage_deals)
print(str(total_deals) + " pending storage deals found.")
if total_deals > 0:
today = datetime.today()
date_string = today.strftime("%b-%d-%Y")
wallet_alias = storage_deals[1]["address"][-6:]
filename = "pending-deals-" + wallet_alias + "-" + date_string + ".csv"
print("Writing to " + filename)
with open(filename, 'w') as csvfile:
csvwriter = csv.writer(csvfile)
# write column headers
csvwriter.writerow([
'Deal ID', 'CID', 'Miner ID', 'Filename', 'Size (bytes)',
import os
import sqlite3
import csv
from datetime import datetime
from pygate_grpc.client import PowerGateClient
from google.protobuf.json_format import MessageToDict
api = os.getenv('POWERGATE_API')
ffs = os.getenv('POWERGATE_FFS')
token = os.getenv('POWERGATE_TOKEN')
powergate = PowerGateClient(api, is_secure=False)
# get final storage deals info
cids = powergate.ffs.show_all(token=token)
from pygate_grpc.client import PowerGateClient
client = PowerGateClient("127.0.0.1:5002", False)
build_info = client.build_info()
print(build_info)
user = client.admin.users.create()
user_id = client.user_id(user.token)
print(user_id)
def main():
sqlite_conn = sqlite3.connect('auditprotocol_1.db')
sqlite_cursor = sqlite_conn.cursor()
r = redis.StrictRedis(host=settings['REDIS']['HOST'],
port=settings['REDIS']['PORT'],
db=settings['REDIS']['DB'],
password=settings['REDIS']['PASSWORD'])
pow_client = PowerGateClient(fast_settings.config.powergate_url, False)
awaited_deals = list()
while True:
c = sqlite_cursor.execute("""
SELECT requestID, cid, localCID, retrievedFile FROM retrievals_single WHERE completed=0
""")
single_retrieval_requests = c.fetchall()
for retrieval_request in single_retrieval_requests:
retrieval_worker_logger.debug(retrieval_request)
request_id = retrieval_request[0]
ffs_cid = retrieval_request[1]
local_cid = retrieval_request[2]
s = sqlite_cursor.execute(
"""
SELECT token FROM accounting_records WHERE localCID=?
""", (local_cid, ))
res = s.fetchone()
token = res[0]
retrieval_worker_logger.debug("Retrieving file " + ffs_cid +
" from FFS.")
file_ = pow_client.ffs.get(ffs_cid, token)
file_name = f'static/{request_id}'
retrieval_worker_logger.debug('Saving to ' + file_name)
with open(file_name, 'wb') as f_:
for _ in file_:
f_.write(_)
sqlite_cursor.execute(
"""
UPDATE retrievals_single SET retrievedFile=?, completed=1 WHERE requestID=?
""", ('/' + file_name, request_id))
sqlite_conn.commit()
# bulk retrievals
c = sqlite_cursor.execute("""
SELECT requestID, api_key, token FROM retrievals_bulk WHERE completed=0
""")
bulk_retrieval_requests = c.fetchall()
for bulk_retrieval_request in bulk_retrieval_requests:
retrieval_worker_logger.debug(bulk_retrieval_request)
request_id = bulk_retrieval_request[0]
api_key = bulk_retrieval_request[1]
ffs_token = bulk_retrieval_request[2]
records_c = sqlite_cursor.execute(
'''
SELECT cid, localCID, txHash, confirmed FROM accounting_records WHERE token=?
''', (ffs_token, ))
file_name = f'static/{request_id}'
fp = open(file_name, 'wb')
for each_record in records_c:
ffs_cid = each_record[0]
retrieval_worker_logger.debug("Bulk mode: Retrieving CID " +
ffs_cid + " from FFS.")
file_ = pow_client.ffs.get(ffs_cid, ffs_token)
for _ in file_:
fp.write(_)
fp.close()
sqlite_cursor.execute(
"""
UPDATE retrievals_bulk SET retrievedFile=?, completed=1 WHERE requestID=?
""", ('/' + file_name, request_id))
sqlite_conn.commit()
retrieval_worker_logger.debug('Bulk mode: Marking request ID ' +
request_id + ' as completed')
time.sleep(5)
def job_checker():
pow_client = PowerGateClient(fast_settings.config.powergate_url, False)
while True:
deal_watcher_logger.debug("Job checker Looping....")
done_deal_jids = list()
deals_lock.acquire()
for deal_jid, deal in deals.items():
deal_watcher_logger.debug(f"Getting data for deal {deal['cid']} from SkyDB")
# Get the status of the storage job from powergate
j_stat = pow_client.ffs.get_storage_job(jid=deal['jid'], token=deal['token'])
# print(j_stat.job)
if j_stat.job.status == 5: # 'JOB_STATUS_SUCCESS':
deal_watcher_logger.debug('Hurrah. Removing from deals to be watched...')
deal_watcher_logger.debug(j_stat.job)
done_deal_jids.append(deal_jid)
# update status
required_row = None
while True:
while True:
deal_watcher_logger.debug('Waiting for Lock')
v = redis_lock.incr('my_lock')
required_row = None
if v == 1:
accounting_records_table.calibrate_index()
required_row = accounting_records_table.fetch(
condition = {'c1':['cid',deal['cid']]},
start_index=accounting_records_table.index-1,
n_rows=1,
condition_func=_value_in,
)
v = redis_lock.decr('my_lock')
print(required_row)
break
v = redis_lock.decr('my_lock')
time.sleep(0.1)
try:
assert len(required_row.keys()) >= 1, "No rows found that match the condition"
break
except AssertionError:
deal_watcher_logger.debug("Refetching Data")
update_row_index = list(required_row.keys())[0]
# Update status on SkyDB and also log it to console
update_data = required_row[update_row_index]['c1'].split(';')
update_data[accounting_records_table.column_split.index('confirmed')] = 1
update_data = ';'.join(update_data)
accounting_records_table.update_row(row_index=update_row_index, data={'c1':update_data})
deal_watcher_logger.debug(f"Accounting records table updated.!")
elif j_stat.job.status == 3:
deal_watcher_logger.error('Job failed. Removing from deals to be watched...')
deal_watcher_logger.error(j_stat.job)
done_deal_jids.append(deal_jid)
# update status
required_row == None
while True:
deal_watcher_logger.debug('Waiting for Lock')
v = redis_lock.incr('my_lock')
if v == 1:
required_row = accounting_records_table.fetch({'cid':deal['cid']},
start_index=accounting_records_table.index-1,
n_rows=1,
num_workers=1)
v = redis_lock.decr('my_lock')
break
v = redis_lock.decr('my_lock')
time.sleep(0.1)
assert len(required_row.keys()) >= 1, "No rows found that match the condition"
update_row_index = list(required_row.keys())[0]
sqlite_cursor.execute("""
UPDATE accounting_records SET confirmed=2 WHERE cid=?
""", (deal['cid'],))
# Update status on SkyDB and also log it to console
update_data = required_row[update_row_index]['c1'].split(';')
update_data[accounting_records_table.column_split.index('confirmed')] = 2
update_data = ';'.join(update_data)
accounting_records_table.update_row(row_index=update_row_index, data={'c1':update_data})
deal_watcher_logger.debug(f"Accounting records table updated.!")
for each_done_jid in done_deal_jids:
del deals[each_done_jid]
deals_lock.release()
time.sleep(5)
def download(cid):
"""
Retrieve a file from Filecoin via IPFS using Powergate and offer the user
the option to save it to their machine.
"""
# Retrieve File and FFS info using the CID
file = Files.query.filter_by(CID=cid).first()
ffs = Ffs.query.get(file.ffs_id)
try:
# Retrieve data from Filecoin
powergate = PowerGateClient(app.config["POWERGATE_ADDRESS"])
data_ = powergate.ffs.get(file.CID, ffs.token)
# Save the downloaded data as a file
# Use the default download directory configured for the app
download_path = app.config["DOWNLOADDIR"]
if not os.path.exists(download_path):
os.makedirs(download_path)
with open(os.path.join(download_path, file.file_name),
"wb") as out_file:
# Iterate over the data byte chunks and save them to an output file
for data in data_:
out_file.write(data)
# Create path to download file
safe_path = safe_join("../" + app.config["DOWNLOADDIR"],
file.file_name)
# Update log table with download information
event = Logs(
timestamp=datetime.now().replace(microsecond=0),
event="Downloaded " + file.file_name + " (CID: " + file.CID +
") from Filecoin.",
)
db.session.add(event)
db.session.commit()
# Offer the file for download to local machine
return send_file(safe_path, as_attachment=True)
"""TODO: CLEAR CACHED FILES IN DOWNLOAD DIRECTORY"""
except Exception as e:
# Output error message if download from Filecoin fails
flash("failed to download '{}' from Filecoin. {}".format(
file.file_name, e))
# Update log table with error
event = Logs(
timestamp=datetime.now().replace(microsecond=0),
event="Download ERROR: " + file.file_name + " CID: " + file.CID +
" " + str(e),
)
db.session.add(event)
db.session.commit()
stored_files = Files.query.all()
return render_template("files.html", stored_files=stored_files)
import os
import sqlite3
from flask import Flask, render_template, redirect, flash, url_for, safe_join, send_file, jsonify, request
from datetime import datetime
from pygate_grpc.client import PowerGateClient
from map import app
import requests
images_db = "map/deplatformr-open-images.sqlite"
api = os.getenv('POWERGATE_API')
ffs = os.getenv('POWERGATE_FFS')
token = os.getenv('POWERGATE_TOKEN')
powergate = PowerGateClient(api, is_secure=True)
@app.route('/')
def index():
return (redirect(url_for('label', name='Swimming')))
@app.route('/about')
def about():
return (render_template("about.html"))
@app.route('/image/<id>')
def image(id):
