def remove(self):
if not isdir(self.meta_dir):
print "Error: Can't find meta data for:", self.cname
return
self.read_meta()
n = 0
getLogger('progress.start').info(dict(
amount = len(self.files), # number of files
disp_amount = human_bytes(self.installed_size),
filename = self.fn,
action = 'removing'))
self.install_app(remove=True)
self.run('pre_egguninst.py')
for p in self.files:
n += 1
getLogger('progress.update').info(n)
if self.hook and not p.startswith(self.pkgs_dir):
continue
rm_rf(p)
if p.endswith('.py'):
rm_rf(p + 'c')
self.rm_dirs()
rm_rf(self.meta_dir)
if self.hook:
rm_empty_dir(self.pkg_dir)
else:
rm_empty_dir(self.egginfo_dir)
getLogger('progress.stop').info(None)
def extract(self):
if self.evt_mgr:
from encore.events.api import ProgressManager
else:
from console import ProgressManager
n = 0
size = sum(self.z.getinfo(name).file_size for name in self.arcnames)
self.installed_size = size
progress = ProgressManager(
self.evt_mgr,
source=self,
operation_id=uuid4(),
message="installing egg",
steps=size,
# ---
progress_type="installing",
filename=self.fn,
disp_amount=human_bytes(self.installed_size),
super_id=getattr(self, 'super_id', None))
with progress:
for name in self.arcnames:
n += self.z.getinfo(name).file_size
self.write_arcname(name)
progress(step=n)
def __unicode__(self):
account_info = self.swift_client.head_account()
l1 = u"| # Objects : {0:^10} | # containers : {1:^4} | Quota : {2:>6}/{3:<6} ({4:^6}%)|".format(
account_info['x-account-object-count'],
account_info['x-account-container-count'],
human_bytes(int(account_info['x-account-bytes-used'])),
human_bytes(int(account_info['x-account-meta-quota'])),
round(int(account_info['x-account-bytes-used']) * 100.0/ int(account_info['x-account-meta-quota']),2)
)
out = u"-"*len(l1)
out += "\r\n"
out += l1
out += "\r\n"
out += u"-"*len(l1)
out += "\r\n"
return out
def remove(self):
if not isdir(self.meta_dir):
print "Error: Can't find meta data for:", self.cname
return
if self.evt_mgr:
from encore.events.api import ProgressManager
else:
from console import ProgressManager
self.read_meta()
n = 0
progress = ProgressManager(
self.evt_mgr, source=self,
operation_id=uuid4(),
message="removing egg",
steps=len(self.files),
# ---
progress_type="removing", filename=self.fn,
disp_amount=human_bytes(self.installed_size),
super_id=getattr(self, 'super_id', None))
self.install_app(remove=True)
self.run('pre_egguninst.py')
with progress:
for p in self.files:
n += 1
progress(step=n)
rm_rf(p)
if p.endswith('.py'):
rm_rf(p + 'c')
self.rm_dirs()
rm_rf(self.meta_dir)
rm_empty_dir(self.egginfo_dir)
def print_dists(dists, index=None):
fmt = " %-27s|%17s"
print fmt % ('package', 'build')
print fmt % ('-' * 27, '-' * 17)
for dist in dists:
line = fmt % tuple(dist.rsplit('-', 1))
fn = dist + '.tar.bz2'
if index and fn in index:
line += '%15s' % human_bytes(index[fn]['size'])
print line
def extract(self):
n = 0
size = sum(self.z.getinfo(name).file_size for name in self.arcnames)
self.installed_size = size
getLogger('progress.start').info(dict(
amount = size,
disp_amount = human_bytes(size),
filename = self.fn,
action = 'installing'))
for name in self.arcnames:
n += self.z.getinfo(name).file_size
getLogger('progress.update').info(n)
self.write_arcname(name)
def upload(self, to_upload):
uploaded = 1
dirs_to_create = [e for e in to_upload if e['is_dir']]
files_to_upload = [e for e in to_upload if not e['is_dir']]
"""
In swift objects are stored with an uri based name, there are no directories, BUT hubic only shows content if pseudo directories are created (empty file with content type directory)
Even if objects are not shown in hubic due to a lack of pseudo dir, objects still exists
"""
for dir_ in dirs_to_create:
if dir_["name"][0] == "/": # Swift client (or hubic?) lib does not handle properly name starting with / (-> test.com/container//object)
dir_["name"] = dir_["name"][1:]
if len(dir_["name"])>1024:
logger.error("dir %s name is too long (>1024)"%(dir_["name"])) #Swift object names are limited to 1024
self.createDirectory(dir_['name'])
for file_to_upload in files_to_upload:
if file_to_upload["name"][0] == "/": # Swift client (or hubic?) lib does not handle properly name starting with / (-> test.com/container//object)
file_to_upload["name"] = file_to_upload["name"][1:]
if len(file_to_upload["name"])>1024:
logger.error("filename %s is too long (>1024)"%(file_to_upload["name"])) #Swift object names are limited to 1024
logger.info("Uploading %s (%s)- (%s/%s)"%(
file_to_upload["name"],
human_bytes(file_to_upload['bytes']),
uploaded,
len(files_to_upload)
)
)
with open(file_to_upload['local_path']) as f:
response_dict = {}
try:
self.swift_client.put_object(
container = self.container,
obj = file_to_upload['name'],
contents=f,
content_length=file_to_upload['bytes'],
response_dict = response_dict,
etag=file_to_upload['hash']
)
except Exception as e:
logger.error("While storing %s : %s"%(file_to_upload,e))
try:
stored_obj_attributes = self.swift_client.head_object(self.container, file_to_upload['name'])
if (response_dict['headers']['etag'] != file_to_upload['hash']) or (stored_obj_attributes['etag'] != file_to_upload['hash']):
logger.critical("File %s got corrupted during upload"%(file_to_upload))
except swiftclient.ClientException as e:
logger.error("While storing %s : %s"%(file_to_upload, e))
if e.http_status == 404:
logger.error("File was not stored properly on hubic (404)")
logger.debug("File %s stored sucessfully on hubic"%(file_to_upload))
uploaded += 1
def __unicode__(self):
out = u""
out += u"|{0:^50}|{1:^25}|{2:^25}|\r\n".format(
"Name",
"# Objects",
"Size"
)
for container in self.containers:
out += u"|{0:^50}|{1:^25}|{2:^25}|\r\n".format(
container['name'],
container['count'],
human_bytes(int(container['bytes']))
)
return out
def extract(self):
if self.evt_mgr:
from encore.events.api import ProgressManager
else:
from console import ProgressManager
is_custom_egg = eggmeta.is_custom_egg(self.path)
n = 0
size = sum(self.z.getinfo(name).file_size for name in self.arcnames)
self.installed_size = size
progress = ProgressManager(
self.evt_mgr, source=self,
operation_id=uuid4(),
message="installing egg",
steps=size,
# ---
progress_type="installing", filename=self.fn,
disp_amount=human_bytes(self.installed_size),
super_id=getattr(self, 'super_id', None))
use_legacy_egg_info_format = has_legacy_egg_info_format(self.arcnames,
is_custom_egg)
if use_legacy_egg_info_format:
with progress:
for name in self.arcnames:
zip_info = self.z.getinfo(name)
n += zip_info.file_size
if is_in_legacy_egg_info(name, is_custom_egg):
self._write_legacy_egg_info_metadata(zip_info)
else:
self.write_arcname(name)
progress(step=n)
else:
with progress:
for name in self.arcnames:
zip_info = self.z.getinfo(name)
n += zip_info.file_size
self.write_arcname(name)
if should_copy_in_egg_info(name, is_custom_egg):
self._write_standard_egg_info_metadata(zip_info)
progress(step=n)
def remove(self):
if not isdir(self.meta_dir):
print "Error: Can't find meta data for:", self.cname
return
if self.evt_mgr:
from encore.events.api import ProgressManager
else:
from console import ProgressManager
self.read_meta()
n = 0
progress = ProgressManager(
self.evt_mgr,
source=self,
operation_id=uuid4(),
message="removing egg",
steps=len(self.files),
# ---
progress_type="removing",
filename=self.fn,
disp_amount=human_bytes(self.installed_size),
super_id=getattr(self, 'super_id', None))
self.install_app(remove=True)
self.run('pre_egguninst.py')
with progress:
for p in self.files:
n += 1
progress(step=n)
if self.hook and not p.startswith(self.pkgs_dir):
continue
rm_rf(p)
if p.endswith('.py'):
rm_rf(p + 'c')
self.rm_dirs()
rm_rf(self.meta_dir)
if self.hook:
rm_empty_dir(self.pkg_dir)
else:
rm_empty_dir(self.egginfo_dir)
def __unicode__(self):
if self.objects is None:
self.refresh()
out = u""
out += u"|{0:^200}|{4:^10}|{1:^34}|{2:^28}|{3:^40}|\r\n".format(
"Name",
"Hash",
"Last modified",
"Content_type",
"Size",
)
for object in self.objects:
out += u"|{0:<200}|{4:^10}|{1:^34}|{2:<28}|{3:^40}|\r\n".format(
object['name'],
object['hash'],
object['last_modified'],
object['content_type'],
human_bytes(int(object['bytes']))
)
return out
def extract(self):
if self.evt_mgr:
from encore.events.api import ProgressManager
else:
from console import ProgressManager
n = 0
size = sum(self.z.getinfo(name).file_size for name in self.arcnames)
self.installed_size = size
progress = ProgressManager(
self.evt_mgr, source=self,
operation_id=uuid4(),
message="installing egg",
steps=size,
# ---
progress_type="installing", filename=self.fn,
disp_amount=human_bytes(self.installed_size),
super_id=getattr(self, 'super_id', None))
with progress:
for name in self.arcnames:
n += self.z.getinfo(name).file_size
self.write_arcname(name)
progress(step=n)
def upload(self, files):
size = sum([f['bytes'] for f in files])
logger.warning("%s will be uploaded"%(human_bytes(size)))
hubic_files_container = self.containers['default']
hubic_files_container.upload(files)
logger.warning("Upload done")
def print_info(self):
# print all socks info.
# os.system("clear")
s_id_audited = []
for k in self.config_sockets:
s_id_audited.append(k)
print("# Extension Connections: %d" % (len(self.ws_helper_dict)))
s_id_audited.append(id(self.listen_data_socket))
s_id_audited.append(id(self.listen_config_socket))
print("##### Hosts: ##############")
if len(self.socks5_parser.target_hosts) > 0:
if len(self.socks5_parser.target_hosts) <= 10:
print(self.socks5_parser.target_hosts)
else:
print("# Target hosts: %d" %
len(self.socks5_parser.target_hosts))
print("##### Routes: #############")
current_time = time.time()
for k in self.socks5_parser.route_info:
route_rec = self.socks5_parser.route_info[k]
proxy_ip = route_rec["proxy_ip"]
data_avail = human_bytes(
route_rec["data_remain"]
) if route_rec["data_remain"] is not None else "INF"
if route_rec["valid_until"] is None:
valid_in = "forever"
elif route_rec["valid_until"] == "INF":
valid_in = "forever"
else:
valid_in = "%d s" % int(route_rec["valid_until"] -
current_time)
print("%s Via %s, valid in %s with data remaining %s" %
(k, proxy_ip, valid_in, data_avail))
if len(self.socks5_parser.not_up_sockets_info):
print("########## Warming Up sockets:")
for s_id in self.socks5_parser.not_up_sockets_info:
# {"state": state, "side": side,
# "route": None,
# "dst_host": None,
# "dst_port": None}
s_info = "[%d] %s state: %s" % (
s_id, self.socks5_parser.not_up_sockets_info[s_id]["side"],
self.socks5_parser.not_up_sockets_info[s_id]["state"])
print(s_info)
if len(self.socks5_parser.record):
print("##### Data Sockets:")
for k in self.socks5_parser.record:
s_id_audited.append(k)
rec = self.socks5_parser.record[k]
try:
r_s_id = id(self.lr_dict[k])
s_id_audited.append(r_s_id)
except KeyError:
r_s_id = -1
data_usage = "S/R %s/%s" % (human_bytes(
rec['up']), human_bytes(rec['down']))
dst = rec['dst'] if rec['dst'] is not None else "None"
proxy = rec['proxy'] if rec['proxy'] is not None else "None"
print("%d/(%d), %s, %s via %s" %
(k, r_s_id, data_usage, dst, proxy))
for k in self.ws_helper_dict:
s_id_audited.append(k)
s_id_audited.append(id(self.listen_ws_socket))
for k in self.connection_list:
if id(k) not in s_id_audited:
print("##### Orphan Socket:", id(k))
if len(self.socks5_parser.route_waiting_queue):
print("##### Waiting queue:")
for i in self.socks5_parser.route_waiting_queue:
print(i)
print("###########################")
