def cmd_create_job(self, args): vault_name = parse_vault_name(args.vault) archive_id = args.archive_id desc = args.desc start = self._parse_size(args.start) size = self._parse_size(args.size) tier = args.tier jtype = 'archive-retrieval' if args.archive_id else 'inventory-retrieval' if jtype == 'inventory-retrieval': if start or size: print 'Tip: Inventory-retrieval does NOT support range, ignored' start = size = None byte_range = None if start is not None and size is not None: byte_range = '%s-%s' % (start, start + size - 1) elif start is not None and size is None: byte_range = '%s-' % start elif start is None and size is not None: byte_range = '0-%s' % (size - 1) if byte_range: print 'Archive retrieval range: %s' % byte_range res = self.api.initiate_job(vault_name, jtype, archive_id, desc, byte_range, tier) check_response(res) job_id = res.getheader('x-cas-job-id') print '%s job created, job ID: %s' % (jtype, job_id) print 'Use\n\n cascmd.py fetch %s %s <local_file>\n\nto check job ' \ 'progress and download the data when job finished' % \ (args.vault, job_id) print 'NOTICE: Jobs usually take about 4 HOURS to complete.' return job_id
def cmd_list_job(self, args): vault_name = parse_vault_name(args.vault) marker = args.marker limit = args.limit res = self.api.list_jobs(vault_name=vault_name, marker=marker, limit=limit) check_response(res) rjson = json.loads(res.read(), 'UTF8') marker = rjson['Marker'] job_list = rjson['JobList'] print 'Marker: %s' % marker print 'Job count: %s' % len(job_list) if not job_list: return print title = ('JobId', 'Action', 'StatusCode', 'StatusMessage', 'ArchiveId', 'ArchiveSizeInBytes', 'SHA256TreeHash', 'ArchiveSHA256TreeHash', 'RetrievalByteRange', 'Completed', 'CompletionDate', 'CreationDate', 'InventorySizeInBytes', 'JobDescription', 'VaultQCS') for job in job_list: print '================ JobId: %s ================' % job['JobId'] self.kv_print(job, title) print
def cmd_list_multipart_upload(self, args): vault_name = parse_vault_name(args.vault) marker = args.marker limit = args.limit res = self.api.list_multipart_uploads(vault_name, marker, limit) check_response(res) rjson = json.loads(res.read(), 'UTF8') marker = rjson['Marker'] upload_list = rjson['UploadsList'] print 'Marker: %s' % marker print 'Multipart upload count: %s' % len(upload_list) if upload_list: print maxdesc = max( max([len(m['ArchiveDescription']) for m in upload_list]), len('ArchiveDescription')) + 1 fmt = u'{:<66} {:<26} {:<10} {:<{ldesc}}' title = ('MultipartUploadId', 'CreationDate', 'PartSize', 'ArchiveDescription') print fmt.format(*title, ldesc=maxdesc) print '-' * (105 + maxdesc) for upload in upload_list: print fmt.format(upload['MultipartUploadId'], upload['CreationDate'], self._byte_humanize( upload['PartSizeInBytes']), upload['ArchiveDescription'], ldesc=maxdesc) print
def cmd_desc_vault(self, args): vault_name = parse_vault_name(args.vault) res = self.api.describe_vault(vault_name) check_response(res) rjson = json.loads(res.read(), 'UTF8') title = ('VaultName', 'VaultQCS', 'CreationDate', 'NumberOfArchives', 'SizeInBytes', 'LastInventoryDate') self.kv_print(rjson, title)
def cmd_init_multipart_upload(self, args): vault_name = parse_vault_name(args.vault) part_size = self._parse_size(args.part_size) desc = args.desc res = self.api.initiate_multipart_upload(vault_name, part_size, desc) check_response(res) upload_id = res.getheader('x-cas-multipart-upload-id') print 'MultiPartUpload ID: %s' % upload_id return upload_id
def cmd_complete_multipart_upload(self, args): vault_name = parse_vault_name(args.vault) upload_id = args.upload_id file_size = self._parse_size(args.size) tree_etag = args.tree_etag if args.tree_etag else '' res = self.api.complete_multipart_upload(vault_name, upload_id, file_size, tree_etag) check_response(res) archive_id = res.getheader('x-cas-archive-id') print 'Archive ID: %s' % archive_id
def cmd_desc_job(self, args): vault_name = parse_vault_name(args.vault) job_id = args.jobid res = self.api.describe_job(vault_name, job_id) check_response(res) rjson = json.loads(res.read(), 'UTF8') title = ('JobId', 'Action', 'StatusCode', 'StatusMessage', 'ArchiveId', 'ArchiveSizeInBytes', 'SHA256TreeHash', 'ArchiveSHA256TreeHash', 'RetrievalByteRange', 'Completed', 'CompletionDate', 'CreationDate', 'InventorySizeInBytes', 'JobDescription', 'Tier', 'VaultQCS') self.kv_print(rjson, title)
def cmd_post_archive(self, args): vault_name = parse_vault_name(args.vault) filepath = args.local_file desc = args.desc or args.local_file[:128] if not os.path.isfile(filepath): sys.stderr.write("Error: file '%s' not existed!\n" % filepath) sys.exit(1) file_size = os.path.getsize(filepath) etag, tree_etag = compute_hash_from_file(args.local_file) with open(filepath, 'rb') as fp: res = self.api.post_archive_from_reader(vault_name, fp, file_size, etag, tree_etag, desc) check_response(res) archive_id = res.getheader('x-cas-archive-id') print 'Archive ID: %s' % archive_id
def cmd_upload_part(self, args): vault_name = parse_vault_name(args.vault) upload_id = args.upload_id file_path = args.local_file start = self._parse_size(args.start) if not file_path or not os.path.isfile(file_path): sys.stderr.write("Error: file '%s' not existed!\n" % file_path) sys.exit(1) total_size = os.path.getsize(file_path) if total_size == 0: sys.stderr.write("empty file can not be uploaded!") sys.exit(1) end = self._parse_size(args.end) size = long(end) - long(start) + 1 if start < 0 or start >= total_size: sys.stderr.write("start is invalid, legal value [0,%d] !\n" % (total_size - 1)) sys.exit(1) if end < 0 or end >= total_size: sys.stderr.write( "end is invalid, legal value [0, %d] !\n" (total_size - 1)) sys.exit(1) if start >= end: sys.stderr.write("start must less than end") sys.exit(1) etag = args.etag.upper() if args.etag else None tree_etag = args.tree_etag.upper() if args.tree_etag else None if not (etag and tree_etag): tmpsum, tmptree = compute_hash_from_file(args.local_file, start, size) etag = etag or tmpsum tree_etag = tree_etag or tmptree with open(file_path, 'rb') as file_reader: prange = '%s-%s' % (start, end) file_reader.seek(long(start)) res = self.api.post_multipart_from_reader(vault_name, upload_id, file_reader, size, prange, etag, tree_etag) check_response(res) print 'Upload success'
def cmd_list_part(self, args): vault_name = parse_vault_name(args.vault) upload_id = args.upload_id marker = args.marker limit = args.limit res = self.api.list_parts(vault_name, upload_id, marker, limit) check_response(res) rjson = json.loads(res.read(), 'UTF8') rjson['CreationDate'] = rjson['CreationDate'] title = ('MultipartUploadId', 'CreationDate', 'PartSizeInBytes', 'ArchiveDescription', 'Marker', 'VaultQCS') self.kv_print(rjson, title) part_list = rjson['Parts'] print '-' * 88 print 'Part count: %s' % len(part_list) if len(part_list) > 0: print 'Part Ranges: ' maxlen = max([len(p['RangeInBytes']) for p in part_list]) fmt = u'{:>{lrange}}: {}' for part in part_list: print fmt.format(part['RangeInBytes'], part['SHA256TreeHash'], lrange=maxlen)
def cmd_upload(self, args): if not args.local_file or not os.path.isfile(args.local_file): sys.stderr.write("Error: file '%s' not existed\n" % args.local_file) sys.exit(1) size = os.path.getsize(args.local_file) desc = args.desc or args.local_file[:128] if (not args.part_size and size >= DEFAULT_NORMAL_UPLOAD_THRESHOLD) or \ (args.part_size and size > RECOMMEND_MIN_PART_SIZE): resume = 0 if not args.upload_id: partsize = self._parse_size(args.part_size) if partsize: if partsize % (RECOMMEND_MIN_PART_SIZE) != 0: sys.stderr.write( 'Error: partsize must be divided by 16MB!\n') sys.exit(1) if partsize * MAX_PART_NUM < size: print 'specified partsize too small, will be adjusted' if partsize > size: print 'specified partsize too large, will be adjusted' while partsize > size: partsize /= 2 else: print 'test_utils larger than 100MB, multipart upload will be used' partsize = 16 * 1024 * 1024 while partsize * MAX_PART_NUM < size: partsize *= 2 nparts = size // partsize if size % partsize: nparts += 1 print 'Use %s parts with partsize %s to upload' % \ (nparts, self._byte_humanize(partsize)) Create = namedtuple('Namespace', ['vault', 'part_size', 'desc']) cargs = Create(args.vault, partsize, desc) upload_id = self.cmd_init_multipart_upload(cargs) else: upload_id = args.upload_id vault_name = parse_vault_name(args.vault) res = self.api.list_parts(vault_name, upload_id, None, None) check_response(res) rjson = json.loads(res.read(), 'UTF8') plist = rjson['Parts'] partsize = rjson['PartSizeInBytes'] nparts = size // partsize if size % partsize: nparts += 1 # 这里有个问题, 如果已上传的不是连续的, 那么就会漏掉中间的分块 if plist: resumebytes = max([int(x['RangeInBytes'].split('-')[1]) \ for x in plist]) + 1 if resumebytes == size: resume = nparts else: resume = resumebytes // partsize print 'Resume last upload with partsize %s' % \ self._byte_humanize(partsize) Part = namedtuple('Namespace', [ 'vault', 'upload_id', 'local_file', 'start', 'end', 'etag', 'tree_etag' ]) start = 0 etaglist = [] for i in range(nparts): end = min(size, start + partsize) - 1 etag, tree_etag = compute_hash_from_file( args.local_file, start, end - start + 1) pargs = Part(args.vault, upload_id, args.local_file, start, end, etag, tree_etag) start += partsize etaglist.append(binascii.unhexlify(tree_etag)) if i < resume: continue print 'Uploading part %s...' % (i + 1) self.cmd_upload_part(pargs) Complete = namedtuple('Namespace', ['vault', 'upload_id', 'size', 'tree_etag']) etree = compute_combine_tree_etag_from_list(etaglist) cargs = Complete(args.vault, upload_id, size, etree) self.cmd_complete_multipart_upload(cargs) return if size <= RECOMMEND_MIN_PART_SIZE and args.part_size: print 'test_utils smaller than 16MB, part-size will be ignored.' Post = namedtuple('Namespace', ['vault', 'local_file', 'etag', 'tree_etag', 'desc']) etag, tree_etag = compute_hash_from_file(args.local_file) pargs = Post(args.vault, args.local_file, etag, tree_etag, desc) return self.cmd_post_archive(pargs)
def cmd_fetch_job_output(self, args): vault_name = parse_vault_name(args.vault) job_id = args.jobid dst = args.local_file start = self._parse_size(args.start) size = self._parse_size(args.size) res = self.api.describe_job(vault_name, args.jobid) check_response(res) job = json.loads(res.read(), 'UTF8') status = job['StatusCode'].lower() jtype = 'inventory-retrieval' if job['Action'] == 'InventoryRetrieval' \ else 'archive-retrieval' if status == 'inprogress': print '%s job still in progress. Repeat this ' \ 'command later' % jtype sys.exit(0) elif status == 'failed': print '%s job failed.' % jtype sys.exit(0) brange = None if jtype == 'archive-retrieval': brange = [int(x) for x in job['RetrievalByteRange'].split('-')] else: brange = [0, int(job['InventorySizeInBytes']) - 1] output_range = None if start is not None and size is None: size = brange[1] - brange[0] - start + 1 elif start is None and size is not None: start = 0 if start is not None and size is not None: output_range = 'bytes=%s-%s' % (start, start + size - 1) if not args.force and os.path.exists(dst): ans = raw_input('Output file %s existed. Do you wish to ' \ 'overwrite it? (y/n): ' % dst) if ans.strip().lower() != 'y': print 'Answer is no. Quit now.' sys.exit(0) res = self.api.get_job_output(vault_name, job_id, output_range) check_response(res) with open(dst, 'wb') as f: total_read = 0 while True: data = res.read(1024 * 1024) if len(data) > 0: f.write(data) total_read += len(data) if jtype == 'inventory-retrieval': continue total_size = size or brange[1] - brange[0] + 1 percent = total_read * 100 // total_size nbar = percent // 2 sys.stdout.write('\r') msgbar = '[%s] %s%%' % ('=' * nbar + '>' + ' ' * (50 - nbar), percent) sys.stdout.write(msgbar) sys.stdout.flush() else: break sys.stdout.write('\n') print 'Download job output success'
def cmd_abort_multipart_upload(self, args): vault_name = parse_vault_name(args.vault) res = self.api.abort_multipart_upload(vault_name, args.upload_id) check_response(res) print 'Delete success'
def cmd_delete_archive(self, args): vault_name = parse_vault_name(args.vault) res = self.api.delete_archive(vault_name, args.archive_id) check_response(res) print 'Delete success'
def cmd_delete_vault(self, args): vault_name = parse_vault_name(args.vault) res = self.api.delete_vault(vault_name) check_response(res) print 'Delete success'
def cmd_create_vault(self, args): vault_name = parse_vault_name(args.vault) res = self.api.create_vault(vault_name) check_response(res) vault_location = res.getheader('Location') print 'Vault Location: %s' % vault_location