def remove(self): if not isdir(self.meta_dir): print "Error: Can't find meta data for:", self.cname return self.read_meta() n = 0 getLogger('progress.start').info(dict( amount = len(self.files), # number of files disp_amount = human_bytes(self.installed_size), filename = self.fn, action = 'removing')) self.install_app(remove=True) self.run('pre_egguninst.py') for p in self.files: n += 1 getLogger('progress.update').info(n) if self.hook and not p.startswith(self.pkgs_dir): continue rm_rf(p) if p.endswith('.py'): rm_rf(p + 'c') self.rm_dirs() rm_rf(self.meta_dir) if self.hook: rm_empty_dir(self.pkg_dir) else: rm_empty_dir(self.egginfo_dir) getLogger('progress.stop').info(None)
def extract(self): if self.evt_mgr: from encore.events.api import ProgressManager else: from console import ProgressManager n = 0 size = sum(self.z.getinfo(name).file_size for name in self.arcnames) self.installed_size = size progress = ProgressManager( self.evt_mgr, source=self, operation_id=uuid4(), message="installing egg", steps=size, # --- progress_type="installing", filename=self.fn, disp_amount=human_bytes(self.installed_size), super_id=getattr(self, 'super_id', None)) with progress: for name in self.arcnames: n += self.z.getinfo(name).file_size self.write_arcname(name) progress(step=n)
def __unicode__(self): account_info = self.swift_client.head_account() l1 = u"| # Objects : {0:^10} | # containers : {1:^4} | Quota : {2:>6}/{3:<6} ({4:^6}%)|".format( account_info['x-account-object-count'], account_info['x-account-container-count'], human_bytes(int(account_info['x-account-bytes-used'])), human_bytes(int(account_info['x-account-meta-quota'])), round(int(account_info['x-account-bytes-used']) * 100.0/ int(account_info['x-account-meta-quota']),2) ) out = u"-"*len(l1) out += "\r\n" out += l1 out += "\r\n" out += u"-"*len(l1) out += "\r\n" return out
def remove(self): if not isdir(self.meta_dir): print "Error: Can't find meta data for:", self.cname return if self.evt_mgr: from encore.events.api import ProgressManager else: from console import ProgressManager self.read_meta() n = 0 progress = ProgressManager( self.evt_mgr, source=self, operation_id=uuid4(), message="removing egg", steps=len(self.files), # --- progress_type="removing", filename=self.fn, disp_amount=human_bytes(self.installed_size), super_id=getattr(self, 'super_id', None)) self.install_app(remove=True) self.run('pre_egguninst.py') with progress: for p in self.files: n += 1 progress(step=n) rm_rf(p) if p.endswith('.py'): rm_rf(p + 'c') self.rm_dirs() rm_rf(self.meta_dir) rm_empty_dir(self.egginfo_dir)
def print_dists(dists, index=None): fmt = " %-27s|%17s" print fmt % ('package', 'build') print fmt % ('-' * 27, '-' * 17) for dist in dists: line = fmt % tuple(dist.rsplit('-', 1)) fn = dist + '.tar.bz2' if index and fn in index: line += '%15s' % human_bytes(index[fn]['size']) print line
def extract(self): n = 0 size = sum(self.z.getinfo(name).file_size for name in self.arcnames) self.installed_size = size getLogger('progress.start').info(dict( amount = size, disp_amount = human_bytes(size), filename = self.fn, action = 'installing')) for name in self.arcnames: n += self.z.getinfo(name).file_size getLogger('progress.update').info(n) self.write_arcname(name)
def upload(self, to_upload): uploaded = 1 dirs_to_create = [e for e in to_upload if e['is_dir']] files_to_upload = [e for e in to_upload if not e['is_dir']] """ In swift objects are stored with an uri based name, there are no directories, BUT hubic only shows content if pseudo directories are created (empty file with content type directory) Even if objects are not shown in hubic due to a lack of pseudo dir, objects still exists """ for dir_ in dirs_to_create: if dir_["name"][0] == "/": # Swift client (or hubic?) lib does not handle properly name starting with / (-> test.com/container//object) dir_["name"] = dir_["name"][1:] if len(dir_["name"])>1024: logger.error("dir %s name is too long (>1024)"%(dir_["name"])) #Swift object names are limited to 1024 self.createDirectory(dir_['name']) for file_to_upload in files_to_upload: if file_to_upload["name"][0] == "/": # Swift client (or hubic?) lib does not handle properly name starting with / (-> test.com/container//object) file_to_upload["name"] = file_to_upload["name"][1:] if len(file_to_upload["name"])>1024: logger.error("filename %s is too long (>1024)"%(file_to_upload["name"])) #Swift object names are limited to 1024 logger.info("Uploading %s (%s)- (%s/%s)"%( file_to_upload["name"], human_bytes(file_to_upload['bytes']), uploaded, len(files_to_upload) ) ) with open(file_to_upload['local_path']) as f: response_dict = {} try: self.swift_client.put_object( container = self.container, obj = file_to_upload['name'], contents=f, content_length=file_to_upload['bytes'], response_dict = response_dict, etag=file_to_upload['hash'] ) except Exception as e: logger.error("While storing %s : %s"%(file_to_upload,e)) try: stored_obj_attributes = self.swift_client.head_object(self.container, file_to_upload['name']) if (response_dict['headers']['etag'] != file_to_upload['hash']) or (stored_obj_attributes['etag'] != file_to_upload['hash']): logger.critical("File %s got corrupted during upload"%(file_to_upload)) except swiftclient.ClientException as e: logger.error("While storing %s : %s"%(file_to_upload, e)) if e.http_status == 404: logger.error("File was not stored properly on hubic (404)") logger.debug("File %s stored sucessfully on hubic"%(file_to_upload)) uploaded += 1
def __unicode__(self): out = u"" out += u"|{0:^50}|{1:^25}|{2:^25}|\r\n".format( "Name", "# Objects", "Size" ) for container in self.containers: out += u"|{0:^50}|{1:^25}|{2:^25}|\r\n".format( container['name'], container['count'], human_bytes(int(container['bytes'])) ) return out
def extract(self): if self.evt_mgr: from encore.events.api import ProgressManager else: from console import ProgressManager is_custom_egg = eggmeta.is_custom_egg(self.path) n = 0 size = sum(self.z.getinfo(name).file_size for name in self.arcnames) self.installed_size = size progress = ProgressManager( self.evt_mgr, source=self, operation_id=uuid4(), message="installing egg", steps=size, # --- progress_type="installing", filename=self.fn, disp_amount=human_bytes(self.installed_size), super_id=getattr(self, 'super_id', None)) use_legacy_egg_info_format = has_legacy_egg_info_format(self.arcnames, is_custom_egg) if use_legacy_egg_info_format: with progress: for name in self.arcnames: zip_info = self.z.getinfo(name) n += zip_info.file_size if is_in_legacy_egg_info(name, is_custom_egg): self._write_legacy_egg_info_metadata(zip_info) else: self.write_arcname(name) progress(step=n) else: with progress: for name in self.arcnames: zip_info = self.z.getinfo(name) n += zip_info.file_size self.write_arcname(name) if should_copy_in_egg_info(name, is_custom_egg): self._write_standard_egg_info_metadata(zip_info) progress(step=n)
def remove(self): if not isdir(self.meta_dir): print "Error: Can't find meta data for:", self.cname return if self.evt_mgr: from encore.events.api import ProgressManager else: from console import ProgressManager self.read_meta() n = 0 progress = ProgressManager( self.evt_mgr, source=self, operation_id=uuid4(), message="removing egg", steps=len(self.files), # --- progress_type="removing", filename=self.fn, disp_amount=human_bytes(self.installed_size), super_id=getattr(self, 'super_id', None)) self.install_app(remove=True) self.run('pre_egguninst.py') with progress: for p in self.files: n += 1 progress(step=n) if self.hook and not p.startswith(self.pkgs_dir): continue rm_rf(p) if p.endswith('.py'): rm_rf(p + 'c') self.rm_dirs() rm_rf(self.meta_dir) if self.hook: rm_empty_dir(self.pkg_dir) else: rm_empty_dir(self.egginfo_dir)
def __unicode__(self): if self.objects is None: self.refresh() out = u"" out += u"|{0:^200}|{4:^10}|{1:^34}|{2:^28}|{3:^40}|\r\n".format( "Name", "Hash", "Last modified", "Content_type", "Size", ) for object in self.objects: out += u"|{0:<200}|{4:^10}|{1:^34}|{2:<28}|{3:^40}|\r\n".format( object['name'], object['hash'], object['last_modified'], object['content_type'], human_bytes(int(object['bytes'])) ) return out
def upload(self, files): size = sum([f['bytes'] for f in files]) logger.warning("%s will be uploaded"%(human_bytes(size))) hubic_files_container = self.containers['default'] hubic_files_container.upload(files) logger.warning("Upload done")
def print_info(self): # print all socks info. # os.system("clear") s_id_audited = [] for k in self.config_sockets: s_id_audited.append(k) print("# Extension Connections: %d" % (len(self.ws_helper_dict))) s_id_audited.append(id(self.listen_data_socket)) s_id_audited.append(id(self.listen_config_socket)) print("##### Hosts: ##############") if len(self.socks5_parser.target_hosts) > 0: if len(self.socks5_parser.target_hosts) <= 10: print(self.socks5_parser.target_hosts) else: print("# Target hosts: %d" % len(self.socks5_parser.target_hosts)) print("##### Routes: #############") current_time = time.time() for k in self.socks5_parser.route_info: route_rec = self.socks5_parser.route_info[k] proxy_ip = route_rec["proxy_ip"] data_avail = human_bytes( route_rec["data_remain"] ) if route_rec["data_remain"] is not None else "INF" if route_rec["valid_until"] is None: valid_in = "forever" elif route_rec["valid_until"] == "INF": valid_in = "forever" else: valid_in = "%d s" % int(route_rec["valid_until"] - current_time) print("%s Via %s, valid in %s with data remaining %s" % (k, proxy_ip, valid_in, data_avail)) if len(self.socks5_parser.not_up_sockets_info): print("########## Warming Up sockets:") for s_id in self.socks5_parser.not_up_sockets_info: # {"state": state, "side": side, # "route": None, # "dst_host": None, # "dst_port": None} s_info = "[%d] %s state: %s" % ( s_id, self.socks5_parser.not_up_sockets_info[s_id]["side"], self.socks5_parser.not_up_sockets_info[s_id]["state"]) print(s_info) if len(self.socks5_parser.record): print("##### Data Sockets:") for k in self.socks5_parser.record: s_id_audited.append(k) rec = self.socks5_parser.record[k] try: r_s_id = id(self.lr_dict[k]) s_id_audited.append(r_s_id) except KeyError: r_s_id = -1 data_usage = "S/R %s/%s" % (human_bytes( rec['up']), human_bytes(rec['down'])) dst = rec['dst'] if rec['dst'] is not None else "None" proxy = rec['proxy'] if rec['proxy'] is not None else "None" print("%d/(%d), %s, %s via %s" % (k, r_s_id, data_usage, dst, proxy)) for k in self.ws_helper_dict: s_id_audited.append(k) s_id_audited.append(id(self.listen_ws_socket)) for k in self.connection_list: if id(k) not in s_id_audited: print("##### Orphan Socket:", id(k)) if len(self.socks5_parser.route_waiting_queue): print("##### Waiting queue:") for i in self.socks5_parser.route_waiting_queue: print(i) print("###########################")