def main():
log.info("datanode start")
loop = asyncio.get_event_loop()
metadata_mem_cache_size = int(config.get("metadata_mem_cache_size"))
log.info("Using metadata memory cache size of: {}".format(
metadata_mem_cache_size))
chunk_mem_cache_size = int(config.get("chunk_mem_cache_size"))
log.info(
"Using chunk memory cache size of: {}".format(chunk_mem_cache_size))
#create the app object
app = loop.run_until_complete(init(loop))
app['meta_cache'] = LruCache(mem_target=metadata_mem_cache_size,
chunk_cache=False)
app['chunk_cache'] = LruCache(mem_target=chunk_mem_cache_size,
chunk_cache=True)
app['deleted_ids'] = set()
app['dirty_ids'] = {
} # map of objids to timestamp and bucket of which they were last updated
app['deflate_map'] = {
} # map of dataset ids to deflate levels (if compressed)
app["shuffle_map"] = {
} # map of dataset ids to shuffle items size (if shuffle filter is applied)
app["pending_s3_read"] = {
} # map of s3key to timestamp for in-flight read requests
app["pending_s3_write"] = {
} # map of s3key to timestamp for in-flight write requests
app["pending_s3_write_tasks"] = {
} # map of objid to asyncio Task objects for writes
app["root_notify_ids"] = {
} # map of root_id to bucket name used for notify root of changes in domain
app["root_scan_ids"] = {
} # map of root_id to bucket name for pending root scans
app["gc_ids"] = set() # set of root or dataset ids for deletion
app["objDelete_prefix"] = None # used by async_lib removeKeys
# TODO - there's nothing to prevent the deflate_map from getting ever larger
# (though it is only one int per dataset id)
# add a timestamp and remove at a certain time?
# delete entire map whenver the synch queue is empty?
# run background tasks
asyncio.ensure_future(healthCheck(app), loop=loop)
# run data sync tasks
asyncio.ensure_future(s3syncCheck(app), loop=loop)
# run root scan
asyncio.ensure_future(bucketScan(app), loop=loop)
# run root/dataset GC
asyncio.ensure_future(bucketGC(app), loop=loop)
# run the app
port = int(config.get("dn_port"))
log.info(f"run_app on port: {port}")
run_app(app, port=port)
metadata_mem_cache_size = int(config.get("metadata_mem_cache_size"))
log.info("Using metadata memory cache size of: {}".format(metadata_mem_cache_size))
chunk_mem_cache_size = int(config.get("chunk_mem_cache_size"))
log.info("Using chunk memory cache size of: {}".format(chunk_mem_cache_size))
#create the app object
app = loop.run_until_complete(init(loop))
app['meta_cache'] = LruCache(mem_target=metadata_mem_cache_size, chunk_cache=False)
app['chunk_cache'] = LruCache(mem_target=chunk_mem_cache_size, chunk_cache=True)
app['deleted_ids'] = set()
app['dirty_ids'] = {} # map of objids to timestamp of which they were last updated
app['deflate_map'] = {} # map of dataset ids to deflate levels (if compressed)
app["pending_s3_read"] = {} # map of s3key to timestamp for in-flight read requests
app["pending_s3_write"] = {} # map of s3key to timestamp for in-flight write requests
app["pending_s3_write_tasks"] = {} # map of objid to asyncio Task objects for writes
app["an_notify_objs"] = set() # set of objids to tell the AN about
# TODO - there's nothing to prevent the deflate_map from getting ever larger
# (though it is only one int per dataset id)
# add a timestamp and remove at a certain time?
# delete entire map whenver the synch queue is empty?
# run background tasks
asyncio.ensure_future(healthCheck(app), loop=loop)
# run data sync tasks
asyncio.ensure_future(s3syncCheck(app), loop=loop)
# run the app
port = int(config.get("dn_port"))
run_app(app, port=port)