def main():
""" Main Control Function dude
"""
global LOGGER
global DB
log_style = '%(asctime)s %(levelname)s %(message)s'
logging.basicConfig(format=log_style, filename=LOG_FILE)
LOGGER = logging.getLogger('mega_sync')
LOGGER.info("Mega Sync started...")
user, passwd = load_config('config')
mega_login(user, passwd)
DB = bucket('cache.db')
build_mega_store()
# Do our add and delete operations
add_operation()
delete_operation()
# Update our caches so we have a difference for next run
update_local_cache()
update_mega_cache()
# Check for modifications to files
check_modifications()
DB.close()
def unpack(self, binary_string, bytes=None):
"""
Unpack a list of buckets
Unpack buckets from a binary string, creating an array
of objects of the appropriate type
@param binary_string The string to be unpacked
@param bytes The total length of the instruction list in bytes.
Ignored if decode is True. If bytes is None and decode is false, the
list is assumed to extend through the entire string.
@return The remainder of binary_string that was not parsed
"""
if bytes == None:
bytes = len(binary_string)
bytes_done = 0
cur_string = binary_string
while bytes_done < bytes:
b = bucket()
cur_string = b.unpack(cur_string)
self.buckets.append(b)
bytes_done += len(b)
return cur_string
def unpack(self, binary_string, bytes=None):
"""
Unpack a list of buckets
Unpack buckets from a binary string, creating an array
of objects of the appropriate type
@param binary_string The string to be unpacked
@param bytes The total length of the instruction list in bytes.
Ignored if decode is True. If bytes is None and decode is false, the
list is assumed to extend through the entire string.
@return The remainder of binary_string that was not parsed
"""
if bytes == None:
bytes = len(binary_string)
bytes_done = 0
cur_string = binary_string
while bytes_done < bytes:
b = bucket()
cur_string = b.unpack(cur_string)
self.buckets.append(b)
bytes_done += len(b)
return cur_string
def main():
""" Main Control Function dude
"""
global LOGGER
global DB
log_style = '%(asctime)s %(levelname)s %(message)s'
logging.basicConfig(format=log_style, filename=LOG_FILE)
LOGGER = logging.getLogger('mega_sync')
LOGGER.info("Mega Sync started...")
user,passwd = load_config('config')
mega_login(user, passwd)
DB = bucket('cache.db')
build_mega_store()
# Do our add and delete operations
add_operation()
delete_operation()
# Update our caches so we have a difference for next run
update_local_cache()
update_mega_cache()
# Check for modifications to files
check_modifications()
DB.close()
def simulate(max_records):
for i in range(0, tot_buckets):
b = bucket.bucket(max_records)
buckets.append(b)
buckets[tot_buckets - 1].next_bucket = 0
#print(len(buckets))
return (buckets)
def bucket_paths(path='~/train_32x32', outfolder='~/train_32x32/', n=1):
path = join(os.path.dirname(os.path.abspath(__file__)),path)
outfolder = join(os.path.dirname(os.path.abspath(__file__)),outfolder)
assert isinstance(path, str), "Expected a string input for the path"
assert os.path.exists(path), "Input path doesn't exist"
# make list of files
files = [join(path,f) for f in listdir(path) if isfile(join(path, f))]
print('Number of valid images is:', len(files))
images = [_process_path(path) for path in files]
print("pre-processing paths complete")
kmeans = bucket([x for x in images if x is not None], n)
files = [files[i] for i in range(len(files)) if images[i] is not None]
groups = {}
print(kmeans.labels_)
for i in range(len(kmeans.labels_)):
if kmeans.labels_[i] not in groups.keys():
groups[kmeans.labels_[i]] = []
groups[kmeans.labels_[i]].append(files[i])
else:
groups[kmeans.labels_[i]].append(files[i])
print("clustering complete")
# check all images for correct shapes etc. and dump them into
for cluster in groups.keys():
clust_files = groups[cluster]
imgs = []
for i in tqdm(range(len(clust_files))):
img = cv2.imread(join(path, clust_files[i]))
img = img.astype('uint8')
assert img.shape == (32, 32, 3)
assert np.max(img) <= 255
assert np.min(img) >= 0
assert img.dtype == 'uint8'
assert isinstance(img, np.ndarray)
imgs.append(img)
resolution_x, resolution_y = img.shape[0], img.shape[1]
imgs = np.asarray(imgs).astype('uint8')
assert imgs.shape[1:] == (resolution_x, resolution_y, 3)
assert np.max(imgs) <= 255
assert np.min(imgs) >= 0
print('Total number of images is:', imgs.shape[0])
print('All assertions done, dumping into npy file')
outfile = join(outfolder, "bucket"+str(cluster)+".npy")
if not os.path.isdir(os.path.dirname(os.path.abspath(outfile))):
os.makedirs(os.path.dirname(os.path.abspath(outfile)))
np.save(outfile, imgs)
print("quicksort incorrect.")
except:
print("quicksort function errored or is incomplete.")
try:
from heap import heap
if(heap(list(nums)) == sortedNums):
print("Heap Sort success!")
else:
print("Heap Sort incorrect.")
except:
print("Heapsort function errored or is incomplete.")
try:
from bucket import bucket
if(bucket(list(nums)) == sortedNums):
print("Bucket Sort success.")
else:
print("Bucket Sort incorrect.")
except:
print("Bucketsort function errored or is incomplete.")
try:
from combsort import combsort
if(combsort(list(nums)) == sortedNums):
print("Comb Sort success biatch!")
else:
print("Comb Sort incorrect.")
except:
print("Comb Sort function errored or is incomplete.")
def dispatch_buckets():
bucket.bucket()
