def start(self, *argc):
"""
@param argc:
@type argc:
"""
if self.verbose:
console("task_start", self.object_id, color="magenta")
if not self.m_crypto_user_object_id:
raise TaskException("start: no crypto_user_object_id set")
self.save_callable(argc)
rs = RedisServer("crypto_taskworker", verbose=self.verbose)
def taskdone(taskid):
"""
@type taskid: str
"""
if taskid is None:
return
if self.verbose:
console("taskdone", taskid, source_code_link(), color="magenta")
if taskid == self.object_id:
self.load()
self.tasksubscription = rs.event_subscribe("taskdone:" + str(self.object_id), taskdone)
rs.list_push("tasks", self.object_id)
rs.event_emit("runtasks", self.get_serverconfig().get_namespace())
def encrypt_file_chunked(cb, fp, pwd, serverconfig):
"""
@type cb: str, unicode
@type fp: str, unicode
@type pwd: str, unicode
@type serverconfig: ServerConfig
@return: None
"""
chunksize = 1048576 * 5
cl, chunk_size = make_chunklist(fp, chunksize)
num_chunks = len(cl)
chunk_num = 0
dec_crc = str(crc32_blob(fp=open(fp)))
commit = []
for cs in cl:
chunk = read_chunk(fp, cs[0], cs[1])
tf = get_named_temporary_file(False)
tf.write(chunk)
tf.close()
seek_pos = cs[0]
commit.append((serverconfig, cb.object_id, (pwd, tf.name, chunk_num, seek_pos, dec_crc, num_chunks, chunksize)))
chunk_num += 1
console("chunk_num", chunk_num)
def taskdone(taskid):
"""
@type taskid: str
"""
if taskid is None:
return
if self.verbose:
console("taskdone", taskid, source_code_link(), color="magenta")
if taskid == self.object_id:
self.load()
def main2():
"""
main
"""
with Events() as e:
e.event("walk")
mylist = []
# os.path.walk("/Volumes/bu/Dropbox (Active8)", walker, mylist)
os.path.walk("/Users/rabshakeh/Dropbox (Active8)", walker, mylist)
e.event("test files")
items = {}
for fp in mylist:
console(fp, once=True)
crc = crc32_blob(open(fp))
if crc in items:
items[crc].append(fp)
else:
items[crc] = [fp]
for crc in items:
if len(items[crc]) > 1:
console(crc, len(items[crc]))
for i in items[crc]:
console(i)
console(len(items), len(mylist), len(mylist) - len(items))
def walker(mylist, tdir, files):
"""
@type mylist: list
@type tdir: str, unicode
@type files: str, unicode
@return:
@raise:
"""
console(tdir, once=True)
for f in files:
fp = os.path.join(tdir, f)
if os.path.isfile(fp):
mylist.append(fp)
def generate_testdata():
"""
generate_testdata
"""
if not os.path.exists("testdata"):
os.mkdir("testdata")
console("1kb")
c = b"*"
make_test_file("testdata/1b", str(c))
c *= 1024
make_test_file("testdata/1kb", str(c))
make_test_file("testdata/10kb", str(c * 10))
make_test_file("testdata/100kb", str(c * 100))
make_test_file("testdata/500kb", str(c * 500))
make_test_file("testdata/1MB", make_testdata(1))
make_test_file("testdata/2MB", make_testdata(2))
make_test_file("testdata/3MB", make_testdata(3))
make_test_file("testdata/3_4MB", make_testdata(3) + str(c * 400))
make_test_file("testdata/4MB", make_testdata(4))
make_test_file("testdata/5MB", make_testdata(5))
make_test_file("testdata/10MB", make_testdata(10))
make_test_file("testdata/12MB", make_testdata(12))
make_test_file("testdata/12_7MB", make_testdata(12) + str(c * 700))
make_test_file("testdata/27MB", make_testdata(27))
make_test_file("testdata/33MB", make_testdata(33))
make_test_file("testdata/100MB", make_testdata(100))
make_test_file("testdata/200MB", make_testdata(200))
make_test_file("testdata/300MB", make_testdata(300))
make_test_file("testdata/400MB", make_testdata(400))
make_test_file("testdata/500MB", make_testdata(500))
make_test_file("testdata/750MB", make_testdata(750))
make_test_file("testdata/1000MB", make_testdata(1000))
make_test_file("testdata/1.23GB", make_testdata(1230))
os.system("cp empty.txt ./testdata")
os.system("cp unicode_testdata.txt ./testdata")