class Migration:
def __init__(self):
self.m = MinioWrapper()
self.clean()
return
def run(self):
self.m.createBucket(conf.new_bucket_name)
packet_names = getJsonFile(packetListPath)
myPrint("Total " + str(len(packet_names)) + " packets found", 12)
if conf.records is not None:
packet_names = packet_names[0:conf.records]
packet_names_chunks = chunkIt(packet_names, conf.threads)
i = 0
for packet_names_chunk in packet_names_chunks:
myPrint("Chunk " + str(i + 1) +
": total packet to be migrated are " +
str(len(packet_names_chunk)))
pool = Pool(conf.threads)
pool.map(runner, packet_names_chunks)
def migrate(self, packet_name):
myPrint("Migrating " + packet_name, 3)
objects = self.m.listObjects(packet_name, recursive=True)
for obj in objects:
new_obj = packet_name + "/" + obj
myPrint("Copying from " + packet_name + " -> " + obj)
myPrint("Copying to " + conf.new_bucket_name + " -> " + new_obj)
self.m.copyObject(conf.new_bucket_name, new_obj, packet_name, obj)
def clean(self):
myPrint("Cleaning stat folder ", 3)
for item in os.listdir(statPath):
if item.endswith(".log"):
os.remove(os.path.join(statPath, item))
def checkHash(self, packet_name):
myPrint("Migrating " + packet_name, 3)
bucketObjects = self.m.listObjects(packet_name, True)
newBucketObjects = self.m.listObjects(conf.new_bucket_name, True,
"/" + packet_name)
for obj in bucketObjects:
for new_obj in newBucketObjects:
if getLastPath(obj) == getLastPath(new_obj):
o1 = self.m.getObject(packet_name, obj)
o2 = self.m.getObject(conf.new_bucket_name, new_obj)
h1 = getHash(o1)
myPrint("Hash of " + obj + ": " + h1)
h2 = getHash(o2)
myPrint("Hash of " + new_obj + ": " + h2)
if h1 == h2:
myPrint("Hashes match")
else:
myPrint("Hashes not match")
raise RuntimeError("Hashes not match")
class GetBuckets:
def __init__(self):
self.m = MinioWrapper()
return
def run(self):
bucket_names = self.m.listBucketNames()
writeJsonFile(bucketListPath, bucket_names)
myPrint("Total " + str(len(bucket_names)) + " buckets found")
def runner(packet_names_chunk):
uid = str(uuid.uuid4())
file_path = os.path.join(statPath, uid + ".log")
m = MinioWrapper()
coui = 1
for packet_name in packet_names_chunk:
migrate(m, packet_name)
writeFile(file_path,
str(coui) + " out of " + str(len(packet_names_chunk)) + "\n")
coui = coui + 1
def main():
args, parser = args_parse()
initLogger(logPath)
start_time = getTimeInSec()
myPrint(conf.minio_endpoint)
try:
prev_time = start_time
if args.action == 'get_buckets' or args.action == 'all':
myPrint("Action: get_buckets", 1)
GetBuckets().run()
prev_time, prstr = timeDiff(prev_time)
myPrint("Time taken by Action get_buckets: " + prstr, 11)
if args.action == 'find_packets' or args.action == 'all':
myPrint("Action: find_packets", 1)
FindPackets().run()
prev_time, prstr = timeDiff(prev_time)
myPrint("Time taken by Action find_packets: " + prstr, 11)
if args.action == 'migrate' or args.action == 'all':
myPrint("Action: migrate", 1)
Migration().run()
prev_time, prstr = timeDiff(prev_time)
myPrint("Time taken by Action migrate: " + prstr, 11)
if args.action == 'get_records' or args.action == 'all':
myPrint("Action: get_records", 1)
m = MinioWrapper()
objs = m.listObjects(conf.new_bucket_name, False)
new_objs = []
for ob in objs:
new_objs.append(ob.replace("/", ""))
writeJsonFile(migratedPackets, new_objs)
myPrint("Total objects level 1 " + str(len(new_objs)))
prev_time, prstr = timeDiff(prev_time)
myPrint("Time taken by Action get_records: " + prstr, 11)
except:
prev_time, prstr = timeDiff(start_time)
myPrint("Total time taken by the script: " + prstr, 11)
formatted_lines = traceback.format_exc()
myPrint(formatted_lines, 13)
sys.exit(1)
prev_time, prstr = timeDiff(start_time)
myPrint("Total time taken by the script: " + prstr, 11)
return sys.exit(0)
def main():
args, parser = args_parse()
initLogger(logPath)
start_time = getTimeInSec()
myPrint(conf.minio_endpoint)
try:
prev_time = start_time
if args.action == 'check_conn' or args.action == 'all':
myPrint("Action: check minio connection", 1)
m = MinioWrapper()
myPrint(m.bucketExists("my-test-bucket"))
prev_time, prstr = timeDiff(prev_time)
myPrint("Time taken by Action check_conn: " + prstr, 11)
if args.action == 'remove_bucket' or args.action == 'all':
myPrint("Action: remove_bucket test", 1)
m = MinioWrapper()
myPrint(m.deleteBucket())
prev_time, prstr = timeDiff(prev_time)
myPrint("Time taken by Action remove_bucket: " + prstr, 11)
if args.action == 'check_hash' or args.action == 'all':
myPrint("Action: check_hash test", 1)
packet_names = getJsonFile(hashCheckPacketsPacket)
for packet in packet_names:
myPrint("Packet name: " + packet, 2)
Migration().checkHash(packet)
prev_time, prstr = timeDiff(prev_time)
myPrint("Time taken by Action check_hash: " + prstr, 11)
if args.action == 'check_records' or args.action == 'all':
myPrint("Action: check_records", 1)
total_buckets = getJsonFile(bucketListPath)
total_packets = getJsonFile(packetListPath)
total_ignored = getJsonFile(ignoredBucketListPath)
total_migrated = getJsonFile(migratedPackets)
myPrint("total_buckets: " + str(len(total_buckets)))
myPrint("total_packets: " + str(len(total_packets)))
myPrint("total_ignored: " + str(len(total_ignored)))
myPrint("total_migrated: " + str(len(total_migrated)))
myPrint(list(set(total_packets) - set(total_migrated)))
prev_time, prstr = timeDiff(prev_time)
myPrint("Time taken by Action check_records: " + prstr, 11)
except:
prev_time, prstr = timeDiff(start_time)
myPrint("Total time taken by the script: " + prstr, 11)
formatted_lines = traceback.format_exc()
myPrint(formatted_lines, 13)
sys.exit(1)
prev_time, prstr = timeDiff(start_time)
myPrint("Total time taken by the script: " + prstr, 11)
return sys.exit(0)
def test_getObject(self):
m = MinioWrapper()
objects = [
'RESIDENT/RES_UPDATE/10001100010002420210223073024_evidence',
'RESIDENT/RES_UPDATE/10001100010002420210223073024_id',
'RESIDENT/RES_UPDATE/10001100010002420210223073024_optional'
]
myPrint(m.bucketExists("10001100010002420210223073024"))
myPrint(m.listObjects("10001100010002420210223073024", recursive=True))
myPrint(m.listObjects("10001100010002420210223073024",
recursive=False))
myPrint(
m.copyObject(
"my-test-bucket",
'RESIDENT/RES_PDATE/10001100010002420210223073024_evidence',
"10001100010002420210223073024",
"RESIDENT/RES_UPDATE/10001100010002420210223073024_evidence"))
def __init__(self):
self.m = MinioWrapper()
self.clean()
return
def test_deleteBucket(self):
m = MinioWrapper()
m.deleteBucket()
def test_bucketExists(self):
m = MinioWrapper()
myPrint(m.bucketExists("10001100010002420210223073024"))
def test_listBuckets(self):
print("OKay")
m = MinioWrapper()
myPrint(m.listBuckets())