def make_app_container(args):
# We add some empty values here so it's easier to modify this template later
app_container = {
'name': 'app',
'image': args.image,
'imagePullPolicy': args.image_pull_policy,
'resources': {
'limits': {},
'requests': {},
},
'volumeMounts': [],
'env': [],
'envFrom': [],
}
# Resource limits
if args.cpu_limit:
app_container['resources']['limits']['cpu'] = args.cpu_limit
if args.cpu_request:
app_container['resources']['requests']['cpu'] = args.cpu_request
if args.memory_limit != 'none':
app_container['resources']['limits']['memory'] = \
humanfriendly.parse_size(args.memory_limit, binary=True)
if args.memory_request != 'none':
app_container['resources']['requests']['memory'] = \
humanfriendly.parse_size(args.memory_request, binary=True)
return app_container
def check_raw_config(raw_config):
C = int(humanfriendly.parse_size(raw_config['C'], binary=True))
L = int(humanfriendly.parse_size(raw_config['L'], binary=True))
K = int(raw_config['K'])
N = int(raw_config['N'])
return C == L * K * N
def container_ram_swap() -> Tuple[int, int]:
"""Return ram and swap available to a container.
See https://docs.docker.com/config/containers/resource_constraints/#--memory-swap-details
for mere details.
"""
ram = settings.DOCKER_PARAMETERS.get("mem_limit", -1)
if ram == "-1":
ram = -1
if ram != -1:
ram = humanfriendly.parse_size(ram)
swap = settings.DOCKER_PARAMETERS.get("memswap_limit", 0)
if swap == "-1":
swap = -1
if swap != -1:
swap = humanfriendly.parse_size(swap)
if ram == -1 and swap == -1:
return -1, -1
if ram == swap:
return ram, 0
if ram >= 0 and swap == 0:
return ram, ram
return ram, swap - ram
def update_project_quota(apiversion, project):
headers = {
'X-Auth-Token': request.headers['X-Auth-Token'],
'X-Subject-Token': request.headers['X-Auth-Token']
}
ks_url = config['swift']['keystone_url'] + '/auth/tokens'
r = requests.get(ks_url, headers=headers)
if not r.status_code == 200:
abort(r.status_code)
ks_token = r.json()
user = ks_token['token']['user']['name']
if user not in config['admin']:
abort(403)
data = request.get_json()
quota = db_quota.find_one({'id': project})
if quota:
db_quota.update({'id': project}, {
'$set': {
'quota': humanfriendly.parse_size(data['quota'], binary=True)
}
})
else:
db_quota.insert({
'id':
project,
'quota':
humanfriendly.parse_size(data['quota'], binary=True)
})
__set_quotas(project)
return jsonify({'project': project, 'quota': data['quota']})
def calc_score(metrics, baseline):
if not isinstance(baseline, numbers.Number):
baseline = humanfriendly.parse_size(baseline)
return mean([
m if isinstance(m, numbers.Number) else humanfriendly.parse_size(m)
for m in metrics
]) / baseline
def info(self) -> SwarmStats:
info = self.cli.info()
pl_status = SwarmStats()
pl_status.container_count = info["Containers"]
pl_status.image_count = info["Images"]
pl_status.memory_total = info["MemTotal"]
pl_status.cores_total = info["NCPU"]
# DriverStatus is a list...
idx = 1
assert 'Strategy' in info["DriverStatus"][idx][0]
pl_status.placement_strategy = info["DriverStatus"][idx][1]
idx = 2
assert 'Filters' in info["DriverStatus"][idx][0]
pl_status.active_filters = [
x.strip() for x in info["DriverStatus"][idx][1].split(", ")
]
idx = 3
assert 'Nodes' in info["DriverStatus"][idx][0]
node_count = int(info["DriverStatus"][idx][1])
idx = 4
for node in range(node_count):
idx2 = 0
ns = SwarmNodeStats(info["DriverStatus"][idx + node][0])
ns.docker_endpoint = info["DriverStatus"][idx + node][1]
idx2 += 1
if 'Status' in info["DriverStatus"][idx + node +
idx2][0]: # new docker version
ns.status = info["DriverStatus"][idx + node + idx2][1]
idx2 += 1
ns.container_count = int(info["DriverStatus"][idx + node +
idx2][1])
idx2 += 1
ns.cores_reserved = int(
info["DriverStatus"][idx + node + idx2][1].split(' / ')[0])
ns.cores_total = int(info["DriverStatus"][idx + node +
idx2][1].split(' / ')[1])
idx2 += 1
ns.memory_reserved = info["DriverStatus"][idx + node +
idx2][1].split(' / ')[0]
ns.memory_total = info["DriverStatus"][idx + node +
idx2][1].split(' / ')[1]
idx2 += 1
ns.labels = info["DriverStatus"][idx + node + idx2][1:]
idx2 += 1
ns.error = info["DriverStatus"][idx + node + idx2][1]
idx2 += 1
ns.last_update = info["DriverStatus"][idx + node + idx2][1]
idx2 += 1
ns.server_version = info["DriverStatus"][idx + node + idx2][1]
ns.memory_reserved = humanfriendly.parse_size(ns.memory_reserved)
ns.memory_total = humanfriendly.parse_size(ns.memory_total)
pl_status.nodes.append(ns)
idx += idx2
pl_status.timestamp = time.time()
return pl_status
def get_stats(self):
self.logger.info("Getting stats")
now = datetime.now(timezone.utc).astimezone().isoformat()
influx_payload = []
with Session() as s:
try:
cookies = {}
for key, morsel in self.cookies.items():
cookies[key] = morsel.value
response = s.get(self.base_url + "/userdetails.php",
cookies=cookies)
soup = BeautifulSoup(response.content, "lxml")
usertable = soup.find('div', class_='p sh').find('table')
for tablerow in usertable.find_all('tr'):
key = tablerow.find("th").text
value = tablerow.find("td").text
if key == "Opplastet":
upload = tablerow.find("td").text
upload = upload[upload.find("[") +
1:upload.find("]")].replace(',', '')
upload = humanfriendly.parse_size(upload)
elif key == "Nedlastet":
download = tablerow.find("td").text
download = download[download.find("[") +
1:download.find("]")].replace(
',', '')
download = humanfriendly.parse_size(download)
elif key == "Poeng":
bonus_points = tablerow.find("td").text
bonus_points = bonus_points.split('.')[0]
bonus_points = int(bonus_points)
elif key == "Ratio":
ratio = tablerow.find("td").find("span").text
ratio = float(ratio)
influx_payload.append(
{
"measurement": "stats",
"tags": {
"tracker": self.name,
},
"time": now,
"fields": {
"ratio": ratio,
"download": download,
"upload": upload,
"bonus_points": bonus_points
},
}, )
self.dbmanager.write_points(influx_payload)
self.logger.info(f"Updated {self.name}")
except:
self.logger.error(f"Failed to update {self.name}")
def get_dir_size(self):
"""Will go one level deeper and sum the sizes of all objects. If the objects are directories, you'll
have to do the iteration yourself!"""
total = humanfriendly.parse_size(self.size)
for f in self.list_dir():
total += humanfriendly.parse_size(f.size)
print(total)
t = humanfriendly.parse_size(str(total))
return humanfriendly.format_size(t)
def memory_expand( tree, subnode, memory_start = 0xbeef, prop_name = 'memory', verbose = 0 ):
# /*
# * 1:1 map, it should match the memory regions
# * specified under access below.
# *
# * It is in the form:
# * memory = <address size address size ...>
# */
try:
mem = [subnode.props( prop_name )[0][0]]
mem_list = []
for m in mem:
try:
start = str(m['start'])
except:
start = str(int(memory_start))
try:
size = str(m['size'])
except:
size = str(int(0xbeef))
if 'flags' in m.keys():
flags = str(m['flags'])
flag_node = LopperNode(-1, subnode.abs_path+'/flags')
tree.add(flag_node)
flag_reference_name = flag_node.abs_path+"/"+str(flags)
flag_reference_node = LopperNode(-1, flag_reference_name )
tree.add(flag_reference_node)
flags_cells = LopperProp(name='#flags-cells',value=0x1)
flags_prop = LopperProp(name='flags',value=0x0)
flags_names = LopperProp('flags-names',value = str(flags))
subnode + flags_cells
subnode + flags_prop
subnode + flags_names
#print( "memory expand: start/size as read: %s/%s" % (start,size))
start = humanfriendly.parse_size( start, True )
size = humanfriendly.parse_size( size, True )
#print( "memory expand: start/size as converted: %s/%s" % (start,size))
mem_list.append(int(start))
mem_list.append(int(size))
except Exception as e:
mem_list = [0xdead, 0xffff ]
if verbose:
# dump the memory as hex
print( '[DBG] memory: [{}]'.format(', '.join(hex(x) for x in mem_list)) )
property_set( prop_name, mem_list, subnode )
def save_mp3s(id_list,
name='unknown',
start_enum=1,
size_threshold='100MB',
overwrite=False):
# First result has enum value of 1, instead of 0
enum = start_enum
print("Query: {}".format(name))
pathlib.Path('../_data/mp3s/{}'.format(name)).mkdir(parents=True,
exist_ok=True)
for file_id in id_list:
file_path = "../_data/mp3s/{0}/{2:04}_{0}_{1}.mp3".format(
name, file_id, enum)
print("======== Result no: {} | File id: {} ========".format(
enum, file_id))
print("Saving to {}".format(file_path))
if overwrite:
mode = "wb+"
else:
mode = "xb"
try:
with open(file_path, mode) as handle:
res = requests.get(
'https://freesound.org/apiv2/sounds/{}/'.format(file_id),
params={
'token': token,
})
json_res = json.loads(res.content.decode('utf8'))
filesize = json_res['filesize']
print("File size is {}".format(format_size(filesize)))
if filesize > parse_size(size_threshold):
raise Exception("size_threshold")
url = json_res['previews']['preview-hq-mp3']
response = requests.get(url, stream=True)
for data in tqdm(response.iter_content()):
handle.write(data)
except FileExistsError as e:
print("File already exists, skipping.")
except Exception as e:
if str(e) != "size_threshold":
raise
else:
print("File size greater than {}, skipping.".format(
format_size(parse_size(size_threshold))))
os.remove(file_path)
enum += 1
return
def __init__(self, max_size: str, usual_object_size: str, margin: str,
index: Index, storage: Storage):
self.max_size = humanfriendly.parse_size(max_size, binary=True)
self.usual_object_size = humanfriendly.parse_size(usual_object_size,
binary=True)
self.margin = humanfriendly.parse_size(margin, binary=True)
self._index = index
self._storage = storage
def parse_log_line(self, line) -> None:
groups = self.regex.match(line).groups()
self.date_time = datetime.datetime.strptime(groups[0], '%Y-%m-%d %H:%M:%S.%f')
self.bytes_copied = humanfriendly.parse_size(groups[1].replace(',', '.'))
self.bytes_to_copy = humanfriendly.parse_size(groups[2].replace(',', '.'))
self.items_copied = int(groups[3])
self.items_to_copy = int(groups[4])
self.bytes_per_second = humanfriendly.parse_size("0" if "-" in groups[5] else groups[5].replace(',', '.'))
self.items_per_second = float("0" if "-" in groups[6] else groups[6].replace(',', '.'))
self.last_path_seen = groups[7]
class Config:
web_core_api_url = os.getenv("LEANDA_WEB_CORE_API_URL")
web_blob_api_url = os.getenv("LEANDA_WEB_BLOB_API_URL")
web_socket_url = os.getenv("LEANDA_WEB_SOCKET_URL")
identity_server_url = os.getenv("LEANDA_IDENTITY_SERVER_URL")
file_upload_limit = os.getenv("LEANDA_FILE_UPLOAD_LIMIT")
file_upload_limit_int = humanfriendly.parse_size(
os.getenv("LEANDA_FILE_UPLOAD_LIMIT") or '50MB', binary=True)
file_download_limit = os.getenv("LEANDA_FILE_DOWNLOAD_LIMIT")
file_download_limit_int = humanfriendly.parse_size(
os.getenv("LEANDA_FILE_DOWNLOAD_LIMIT") or '50MB', binary=True)
def chunksize(self):
if not hasattr(self, '_chunksize'):
try:
transferchunk = self.annex.getconfig('transferchunk')
self._chunksize = humanfriendly.parse_size(transferchunk)
self.annex.debug("Using chunksize: {}".format(transferchunk))
except humanfriendly.InvalidSize:
self.annex.debug(
"No valid chunksize specified. Using default value: {}".
format(self.DEFAULT_CHUNKSIZE))
self._chunksize = humanfriendly.parse_size(
self.DEFAULT_CHUNKSIZE)
return self._chunksize
def is_expired(status, since, size=0):
now = datetime.datetime.now()
min_bps = int(humanfriendly.parse_size("4MiB") / 8)
if status == Tasks.building:
return since < now - datetime.timedelta(hours=12)
if status == Tasks.wiping_sdcard:
return since < now - datetime.timedelta(minutes=30)
if status in (Tasks.uploading, Tasks.downloading, Tasks.writing):
size = humanfriendly.parse_size(size)
return since < now - datetime.timedelta(seconds=int(size / min_bps))
def print_humanfriendly(fs_stats: Dict) -> None:
print("Top PyPI Disk Users:")
total_pypi_size = fs_stats["total_packages_size"]
top_total_bytes = 0
for pkg_name, pkg_data in fs_stats["top_packages"].items():
top_total_bytes += pkg_data["size"]
hfs = humanfriendly.format_size(humanfriendly.parse_size(str(pkg_data["size"])))
print(f"{pkg_name}: {hfs}")
pct = int((top_total_bytes / total_pypi_size) * 100)
ttb = humanfriendly.format_size(humanfriendly.parse_size(str(top_total_bytes)))
print(f"\nTop Packages consume {ttb}\n- This is {pct}% of PyPI")
def memory_expand(tree,
subnode,
memory_start=0xbeef,
prop_name='memory',
verbose=0):
# /*
# * 1:1 map, it should match the memory regions
# * specified under access below.
# *
# * It is in the form:
# * memory = <address size address size ...>
# */
try:
mem = []
prop = subnode[prop_name]
for i in range(0, len(prop)):
mem.append(prop[i])
mem_list = []
for m in mem:
try:
start = str(m['start'])
except:
start = str(int(memory_start))
try:
size = str(m['size'])
except:
size = str(int(0xbeef))
if 'flags' in m.keys():
flags = str(m['flags'])
flags_names = LopperProp(prop_name + '-flags-names',
value=str(flags))
subnode + flags_names
#print( "memory expand: start/size as read: %s/%s" % (start,size))
start = humanfriendly.parse_size(start, True)
size = humanfriendly.parse_size(size, True)
#print( "memory expand: start/size as converted: %s/%s" % (start,size))
mem_list.append(int(start))
mem_list.append(int(size))
except Exception as e:
mem_list = [0xdead, 0xffff]
if verbose:
# dump the memory as hex
print('[DBG] memory: [{}]'.format(', '.join(hex(x) for x in mem_list)))
property_set(prop_name, mem_list, subnode)
def list_dups(filepaths, file_suffix, min_size, hash_type, block_size, format, recursive, debug):
logger.debug(f'hash-type: {hash_type}')
hash_constructor = available_hashes[hash_type]
min_sz = parse_size(min_size)
blk_sz = parse_size(block_size, binary=True)
dups = defaultdict(list)
hardlinked = defaultdict(list)
logger.debug(f'min-size: {min_sz}')
logger.debug(f'block-size: {blk_sz}')
glob_prefix = '**/' if recursive else ''
glob_pattern = f'{glob_prefix}{file_suffix}'
p1 = chain(*(Path(fp).expanduser().glob(glob_pattern) for fp in filepaths))
for f in tqdm(p1):
if f.is_file() and not f.is_symlink():
s = f.stat()
if (s.st_size > min_sz):
#logger.debug(f'processing: {f}')
if s.st_nlink > 1:
logger.debug(f'hardlinks [{s.st_nlink}]: file: {f}')
# Same Inodes:
hardlinked[(s.st_ino, s.st_dev)].append(f)
# FIXME: adaptive block_size??
hash_res = hash_file(f, block_size=blk_sz, hex_result=True, hash_constructor=hash_constructor)
#click.echo(f'hash[{hash_type}]: {hash_res}')
dups[hash_res].append(f)
records = [[(filehash,
path.resolve().as_posix(),
path.stat().st_size,
format_size(path.stat().st_size),
str(path.stat().st_ino) if (path.stat().st_ino, path.stat().st_dev) in hardlinked else None,
str(path.stat().st_dev) if (path.stat().st_ino, path.stat().st_dev) in hardlinked else None)
for path in paths] for (filehash, paths) in dups.items() if len(paths) > 1]
if len(records) > 0:
cols = ['hash', 'file_path', 'size', 'human_size', 'inode', 'device']
df = (pd.concat(pd.DataFrame.from_records(r, columns=cols) for r in records)
.sort_values(by=['size', 'inode', 'device'], ascending=False))
if debug:
from IPython import embed
embed()
print(tabulate(df, headers=df.columns, tablefmt=format, showindex=False))
def info(self) -> ClusterStats: # pylint: disable=too-many-locals
"""Retrieve Kubernetes cluster statistics."""
pl_status = ClusterStats()
node_list = pykube.Node.objects(
self.api).filter(namespace=pykube.all).iterator()
node_dict = {}
# Get basic information from nodes
for node in node_list:
nss = NodeStats(node.name)
nss.cores_total = float(node.obj['status']['allocatable']['cpu']
[1]) ## Bug found on GKE
nss.memory_total = humanfriendly.parse_size(
node.obj['status']['allocatable']['memory'])
nss.labels = node.obj['metadata']['labels']
nss.status = 'online'
node_dict[str(socket.gethostbyname(node.name))] = nss
# Get information from all running pods, then accumulate to nodes
pod_list = pykube.Pod.objects(
self.api).filter(namespace=pykube.all).iterator()
for pod in pod_list:
try:
host_ip = pod.obj['status']['hostIP']
except KeyError:
continue
nss = node_dict[host_ip]
nss.container_count += 1
spec_cont = pod.obj['spec']['containers'][0]
if 'resources' in spec_cont:
if 'requests' in spec_cont['resources']:
if 'memory' in spec_cont['resources']['requests']:
memory = spec_cont['resources']['requests']['memory']
nss.memory_reserved = nss.memory_reserved + humanfriendly.parse_size(
memory)
if 'cpu' in spec_cont['resources']['requests']:
cpu = spec_cont['resources']['requests']['cpu']
# ex: cpu could be 100m or 0.1
cpu_splitted = cpu.split('m')
if len(cpu_splitted) > 1:
cpu_float = int(cpu_splitted[0]) / 1000
else:
cpu_float = int(cpu_splitted[0])
nss.cores_reserved = round(
nss.cores_reserved + cpu_float, 3)
for node_ip in node_dict:
pl_status.nodes.append(node_dict[node_ip])
return pl_status
def info(self) -> SwarmStats:
"""Retrieve Swarm statistics. The Docker API returns a mess difficult to parse."""
info = self.cli.info()
pl_status = SwarmStats()
pl_status.container_count = info["Containers"]
pl_status.image_count = info["Images"]
pl_status.memory_total = info["MemTotal"]
pl_status.cores_total = info["NCPU"]
# DriverStatus is a list...
idx = 1
assert 'Strategy' in info["DriverStatus"][idx][0]
pl_status.placement_strategy = info["DriverStatus"][idx][1]
idx = 2
assert 'Filters' in info["DriverStatus"][idx][0]
pl_status.active_filters = [x.strip() for x in info["DriverStatus"][idx][1].split(", ")]
idx = 3
assert 'Nodes' in info["DriverStatus"][idx][0]
node_count = int(info["DriverStatus"][idx][1])
idx = 4
for node in range(node_count):
idx2 = 0
node_stats = SwarmNodeStats(info["DriverStatus"][idx + node][0])
node_stats.docker_endpoint = info["DriverStatus"][idx + node][1]
idx2 += 1
if 'Status' in info["DriverStatus"][idx + node + idx2][0]: # new docker version
node_stats.status = info["DriverStatus"][idx + node + idx2][1]
idx2 += 1
node_stats.container_count = int(info["DriverStatus"][idx + node + idx2][1])
idx2 += 1
node_stats.cores_reserved = int(info["DriverStatus"][idx + node + idx2][1].split(' / ')[0])
node_stats.cores_total = int(info["DriverStatus"][idx + node + idx2][1].split(' / ')[1])
idx2 += 1
node_stats.memory_reserved = info["DriverStatus"][idx + node + idx2][1].split(' / ')[0]
node_stats.memory_total = info["DriverStatus"][idx + node + idx2][1].split(' / ')[1]
idx2 += 1
node_stats.labels = info["DriverStatus"][idx + node + idx2][1:]
idx2 += 1
node_stats.error = info["DriverStatus"][idx + node + idx2][1]
idx2 += 1
node_stats.last_update = info["DriverStatus"][idx + node + idx2][1]
idx2 += 1
node_stats.server_version = info["DriverStatus"][idx + node + idx2][1]
node_stats.memory_reserved = humanfriendly.parse_size(node_stats.memory_reserved)
node_stats.memory_total = humanfriendly.parse_size(node_stats.memory_total)
pl_status.nodes.append(node_stats)
idx += idx2
pl_status.timestamp = time.time()
return pl_status
def constructResources(resources, config):
# add CPU
cpu_resource = resources.add()
cpu_resource.name = "cpus"
cpu_resource.type = messages_pb2.Value.SCALAR
cpu_list = psutil.cpu_percent(interval=1, percpu=True)
cpu_value = 0
for cpu in cpu_list:
cpu_value += (100 - cpu) / 100
if 'maxCPUs' in config:
if config['maxCPUs'] < cpu_value:
cpu_value = config['maxCPUs']
cpu_resource.scalar.value = cpu_value
# add MEMORY
mem_resource = resources.add()
mem_resource.name = "mem"
mem_resource.type = messages_pb2.Value.SCALAR
mem = psutil.virtual_memory().available
if 'maxMem' in config:
maxmem = humanfriendly.parse_size(config['maxMem'])
if maxmem < mem:
mem = maxmem
mem_resource.scalar.value = mem
# add DISK
disk_resource = resources.add()
disk_resource.name = "disk"
disk_resource.type = messages_pb2.Value.SCALAR
disk = os.statvfs('./').f_frsize * os.statvfs('./').f_bavail
if 'maxDisk' in config:
maxdisk = humanfriendly.parse_size(config['maxDisk'])
if maxdisk < disk:
disk = maxdisk
disk_resource.scalar.value = disk
# add devices
if config and 'devices' in config:
for device in config['devices']:
device_resource = resources.add()
device_resource.name = device['name']
device_resource.type = messages_pb2.Value.DEVICE
device_resource.text.value = device['id']
if 'shared' in device:
device_resource.shared = device['shared']
else:
device_resource.shared = False
def info(self) -> ClusterStats:
"""Retrieve Swarm statistics. The Docker API returns a mess difficult to parse."""
info = self.cli.info()
pl_status = ClusterStats()
# SystemStatus is a list...
idx = 0 # Role, skip
idx += 1
assert 'Strategy' in info["SystemStatus"][idx][0]
pl_status.placement_strategy = info["SystemStatus"][idx][1]
idx += 1
assert 'Filters' in info["SystemStatus"][idx][0]
pl_status.active_filters = [x.strip() for x in info["SystemStatus"][idx][1].split(", ")]
idx += 1
assert 'Nodes' in info["SystemStatus"][idx][0]
node_count = int(info["SystemStatus"][idx][1])
idx += 1 # At index 4 the nodes begin
for node in range(node_count):
idx2 = 0
node_stats = NodeStats(info["SystemStatus"][idx + node][0].strip())
node_stats.docker_endpoint = info["SystemStatus"][idx + node][1]
idx2 += 1 # ID, skip
idx2 += 1 # Status
if info["SystemStatus"][idx + node + idx2][1] == 'Healthy':
node_stats.status = 'online'
else:
node_stats.status = 'offline'
idx2 += 1 # Containers
node_stats.container_count = int(info["SystemStatus"][idx + node + idx2][1].split(' ')[0])
idx2 += 1 # CPUs
node_stats.cores_reserved = int(info["SystemStatus"][idx + node + idx2][1].split(' / ')[0])
node_stats.cores_total = int(info["SystemStatus"][idx + node + idx2][1].split(' / ')[1])
idx2 += 1 # Memory
node_stats.memory_reserved = info["SystemStatus"][idx + node + idx2][1].split(' / ')[0]
node_stats.memory_total = info["SystemStatus"][idx + node + idx2][1].split(' / ')[1]
idx2 += 1 # Labels
node_stats.labels = info["SystemStatus"][idx + node + idx2][1].split(', ')
idx2 += 1 # Last update
node_stats.last_update = info["SystemStatus"][idx + node + idx2][1]
idx2 += 1 # Docker version
node_stats.server_version = info["SystemStatus"][idx + node + idx2][1]
node_stats.memory_reserved = humanfriendly.parse_size(node_stats.memory_reserved)
node_stats.memory_total = humanfriendly.parse_size(node_stats.memory_total)
pl_status.nodes.append(node_stats)
idx += idx2
pl_status.timestamp = time.time()
return pl_status
def __init__(self, C, L, K, N, BIT_SIZE, input_label):
self.C = int(humanfriendly.parse_size(C, binary=True))
self.L = int(humanfriendly.parse_size(L, binary=True))
self.K = int(K)
self.N = int(N)
if self.C != self.L * self.K * self.N:
raise RuntimeError('Invalid matching L, K, N, C parameters')
self.input_label = input_label
self.BIT_SIZE = BIT_SIZE
self.BYTE_SELECT = int(math.log(self.L, 2))
self.CACHE_INDEX = int(math.log(self.N, 2))
self.CACHE_TAG = self.BIT_SIZE - self.BYTE_SELECT - self.CACHE_INDEX
def extract_nand_info(self):
fmt = "mnand_read_id: CS{:d} NAND {} {} {} size({:d}) writesize({:d}) oobsize({:d}) erasesize({:d})"
chips = []
_id, lines = self.terminal.wait("mnand_read_id: flash id {:x}")
if _id[0] > 0:
params, lines = self.terminal.wait(fmt)
chips.append(params)
_id, lines = self.terminal.wait("mnand_read_id: flash id {:x}")
if _id[0] > 0:
params, lines = self.terminal.wait(fmt)
chips.append(params)
size = 0
part = ""
for c in chips:
size += humanfriendly.parse_size(c[1])
if len(chips) > 1:
l = "(x2)"
else:
l = "(x1)"
attrs = {
"size": size,
"part": f"NAND {chips[0][1]} {chips[0][2]} {chips[0][3]} {l}",
"read_size": chips[0][4],
"write_size": chips[0][5],
"oob_size": chips[0][6],
"erase_size": chips[0][7],
}
self.nand_chips = chips
return attrs
def __init__(self, input_fasta, database,
num_parts = None, # How many fasta pieces should we make
part_size = None, # What size in MB should a fasta piece be
seqs_per_part = None, # How many sequences in one fasta piece
slurm_params = None, # Additional parameters for possible SLURM jobs
parts_dir = None, # If you want a special direcotry for the fasta pieces
**kwargs):
# Determine number of parts #
self.num_parts = None
# Three possible options #
if num_parts:
self.num_parts = num_parts
if part_size:
self.bytes_target = humanfriendly.parse_size(part_size)
self.num_parts = int(math.ceil(input_fasta.count_bytes / self.bytes_target))
if seqs_per_part:
self.num_parts = int(math.ceil(input_fasta.count / seqs_per_part))
# Default case #
if self.num_parts is None:
self.num_parts = kwargs.get('num_threads', min(multiprocessing.cpu_count(), 32))
# In case the user has some special slurm params #
self.slurm_params = slurm_params
# In case the user wants a special parts directory #
self.parts_dir = parts_dir
# Super #
SeqSearch.__init__(self, input_fasta, database, **kwargs)
def build(
self,
bloomfilters: hug.types.multiple,
samples: hug.types.multiple = [],
config: hug.types.text = None,
):
config = get_config_from_file(config)
if samples:
assert len(samples) == len(bloomfilters)
else:
samples = bloomfilters
if config.get("max_build_mem_bytes"):
max_memory_bytes = humanfriendly.parse_size(
config["max_build_mem_bytes"])
else:
max_memory_bytes = None
return build(
config=config,
bloomfilter_filepaths=bloomfilters,
samples=samples,
max_memory=max_memory_bytes,
)
def __init__(
self,
fs: Union[int, str] = 16000,
frontend_conf: Optional[dict] = get_default_kwargs(Frontend),
download_dir: str = None,
multilayer_feature: bool = False,
):
assert check_argument_types()
super().__init__()
if isinstance(fs, str):
fs = humanfriendly.parse_size(fs)
if download_dir is not None:
torch.hub.set_dir(download_dir)
self.multilayer_feature = multilayer_feature
self.upstream, self.featurizer = self._get_upstream(frontend_conf)
if getattr(
self.upstream, "model", None
) is not None and self.upstream.model.__class__.__name__ in [
"Wav2Vec2Model",
"HuberModel",
]:
self.upstream.model.encoder.layerdrop = 0.0
self.pretrained_params = copy.deepcopy(self.upstream.state_dict())
self.output_dim = self.featurizer.output_dim
def __init__(
self,
name,
modules=None,
cpus=None,
memory=None,
use_temporary_directory=None,
temporary_directory_path=None,
hard_resources=None,
soft_resources=None,
inputs=None,
outputs=None,
parameters=None,
script=None):
self.name = name
self.modules = modules or []
self.cpus = cpus or 1
self.memory = memory or humanfriendly.parse_size('1GB')
self.use_temporary_directory = use_temporary_directory or False
self.temporary_directory_path = temporary_directory_path
self.hard_resources = hard_resources or []
self.soft_resources = soft_resources or []
self.inputs = inputs or collections.OrderedDict()
self.outputs = outputs or {}
self.parameters = parameters or {}
self.script = script or ''
def _parse_field(self, field_metric, field_key, field_value):
try:
# sizes
if (field_metric, field_key) in self._size_fields:
return humanfriendly.parse_size(field_value)
# time spans
elif (field_metric, field_key) in self._time_span_fields:
return sum(
humanfriendly.parse_timespan(v) if v else 0
for v in re.findall(r'([\d,\.]+\s*\D+)', field_value))
# percentages
elif (field_metric, field_key) in self._percentage_fields:
return int(field_value.replace('%', '').strip())
# floats
elif (field_metric, field_key) in self._float_fields:
return float(decimal.Decimal(field_value))
# integers
elif (field_metric, field_key) in self._integer_fields:
return int(decimal.Decimal(field_value))
except ValueError:
pass
def parse_row(row):
details = row.select('font.detDesc')[0].text
# Parse category
try:
typ = self.parse_category(row.select('td')[0].text)
except _CategoryUnknowError as e:
typ = 'other'
msg = "Unknow category: '{category}'"
msg = msg.format(category=e.args[0])
print(msg, file=sys.stderr)
# Parse size
size = re.findall(r'([0-9\.]+\s*[GMK]i?B)', details,
re.IGNORECASE)[0]
size = humanfriendly.parse_size(size)
# Parse created
try:
created = self.parse_timestamp(row.select('.detDesc')[0].text)
except IndexError:
created = 'other'
return {
'name': row.findAll('a')[2].text,
'uri': row.findAll('a')[3]['href'],
'type': typ,
'size': size,
'created': created,
'seeds': int(row.findAll('td')[-2].text),
'leechers': int(row.findAll('td')[-1].text)
}
def get_memory_info(self, arc_size):
with open("/proc/meminfo") as f:
meminfo = {
s[0]: humanfriendly.parse_size(s[1], binary=True)
for s in [line.split(":", 1) for line in f.readlines()]
}
classes = {}
classes["page_tables"] = meminfo["PageTables"]
classes["swap_cache"] = meminfo["SwapCached"]
classes["slab_cache"] = meminfo["Slab"]
classes["cache"] = meminfo["Cached"]
classes["buffers"] = meminfo["Buffers"]
classes["unused"] = meminfo["MemFree"]
classes["arc"] = arc_size
classes["apps"] = meminfo["MemTotal"] - sum(classes.values())
extra = {
"inactive": meminfo["Inactive"],
"committed": meminfo["Committed_AS"],
"active": meminfo["Active"],
"vmalloc_used": meminfo["VmallocUsed"],
"mapped": meminfo["Mapped"],
}
swap = {
"used": meminfo["SwapTotal"] - meminfo["SwapFree"],
"total": meminfo["SwapTotal"],
}
return {
"classes": classes,
"extra": extra,
"swap": swap,
}
def _parse_field(self, field_metric, field_key, field_value):
try:
# sizes
if (field_metric, field_key) in self._size_fields:
return humanfriendly.parse_size(field_value)
# time spans
elif (field_metric, field_key) in self._time_span_fields:
return sum(humanfriendly.parse_timespan(v)
if v else 0
for v in re.findall(r'([\d,\.]+\s*\D+)', field_value))
# percentages
elif (field_metric, field_key) in self._percentage_fields:
return int(field_value.replace('%', '').strip())
# floats
elif (field_metric, field_key) in self._float_fields:
return float(decimal.Decimal(field_value))
# integers
elif (field_metric, field_key) in self._integer_fields:
return int(decimal.Decimal(field_value))
except ValueError:
pass
def download_member_project_data(cls,
member_data,
target_member_dir,
max_size=MAX_SIZE_DEFAULT,
id_filename=False):
"""
Download files to sync a local dir to match OH member project data.
:param member_data: This field is data related to member in a project.
:param target_member_dir: This field is the target directory where data
will be downloaded.
:param max_size: This field is the maximum file size. It's default
value is 128m.
"""
logging.debug('Download member project data...')
sources_shared = member_data['sources_shared']
file_data = cls._get_member_file_data(member_data,
id_filename=id_filename)
for basename in file_data:
# This is using a trick to identify a project's own data in an API
# response, without knowing the project's identifier: if the data
# isn't a shared data source, it must be the project's own data.
if file_data[basename]['source'] in sources_shared:
continue
target_filepath = os.path.join(target_member_dir, basename)
download_file(download_url=file_data[basename]['download_url'],
target_filepath=target_filepath,
max_bytes=parse_size(max_size))
def _convert_memory_size(value):
if isinstance(value, str):
return humanfriendly.parse_size(value)
elif callable(value):
return value(_workflow_builder_stack.peek().execution_count)
else:
raise Exception('Unsupported')
def _get_disk_size(self):
"""Returns data used by Docker in bytes."""
# docker.info() returns an awful data structure...
with self.allocate() as docker:
for field in docker.info()['DriverStatus']:
if field[0]=='Data Space Used':
return parse_size(field[1]) # Value in bytes
logging.error('"Data Space Used" field was not found in the data returned by Docker.')
def from_du(cls, line):
self = cls()
if '\t' not in line:
return
a, b = line.split("\t", 1)
self.size = humanfriendly.parse_size(a.strip())
self.path = b.strip()
return self
def runTests(self):
"""
Run the tests prescribed in the configuration
"""
sites = self.get_sites()
templates_dir = os.path.join(sys.prefix, "etc/stashcache-tester/templates")
# Parse the size of the test in bytes
raw_testsize = humanfriendly.parse_size(get_option("testsize"))
md5sum = self.createTestFile(raw_testsize, get_option("stashdir"))
# Create the site specific tests
env = Environment(loader=FileSystemLoader(templates_dir))
env.globals = {
"config_location": self.config_location,
"stash_test_location": os.path.abspath(sys.argv[0]),
"pythonpath": ":".join(sys.path),
"testurl": get_option("testurl"),
"localpath": get_option("stashdir"),
"testsize": raw_testsize,
"humantestsize": humanfriendly.format_size(raw_testsize)
}
test_dirs = []
testingdir = get_option("testingdir")
for site in sites:
tmp_site = Site(site)
test_dir = tmp_site.createTest(testingdir, env)
test_dirs.append(test_dir)
# Create the DAG from the template
dag_template = env.get_template("dag.tmpl")
test_dag = os.path.join(testingdir, "submit.dag")
with open(test_dag, 'w') as f:
f.write(dag_template.render(sites=sites, md5sum=md5sum))
reduce_template = env.get_template("test_reduce.tmpl")
reduce_submit = os.path.join(testingdir, "reduce.submit")
with open(reduce_submit, 'w') as f:
f.write(reduce_template.render())
shutil.copyfile(os.path.join(templates_dir, "site_post.py"), os.path.join(get_option("testingdir"), "site_post.py"))
os.chmod(os.path.join(get_option("testingdir"), "site_post.py"), 0755)
# Start the DAG
(stdout, stderr) = RunExternal("cd %s; condor_submit_dag submit.dag" % testingdir)
logging.debug("output from condor_submit_dag: %s" % stdout)
if stderr is not None or stderr is not "":
logging.error("Error from condor_submit_dag: %s" % stderr)
def test_parse_size(self):
self.assertEqual(0, humanfriendly.parse_size('0B'))
self.assertEqual(42, humanfriendly.parse_size('42'))
self.assertEqual(42, humanfriendly.parse_size('42B'))
self.assertEqual(1024, humanfriendly.parse_size('1k'))
self.assertEqual(1024, humanfriendly.parse_size('1 KB'))
self.assertEqual(1024, humanfriendly.parse_size('1 kilobyte'))
self.assertEqual(1024 ** 3, humanfriendly.parse_size('1 GB'))
self.assertEqual(1024 ** 3 * 1.5, humanfriendly.parse_size('1.5 GB'))
self.assertRaises(humanfriendly.InvalidSize, humanfriendly.parse_size, '1z')
self.assertRaises(humanfriendly.InvalidSize, humanfriendly.parse_size, 'a')
def main():
args = parser.parse_args()
size_kb = humanfriendly.parse_size(args.size) / 1024
read_units_per_second = humanfriendly.parse_timespan(args.time_span)
# Scan operation is eventually consistent, which is half the cost (hence /2 )
# Each 4kb consumes one capacity unit.
read_capacity_units_needed = size_kb / 2 / 4
time_needed = read_capacity_units_needed / read_units_per_second
print 'It will take (roughly): {0}'.format(humanfriendly.format_timespan(time_needed))
def __init__(self, configFiles):
# First, read in the configuration
set_config_file(configFiles)
self.config_location = os.path.abspath(configFiles)
loglevel = get_option("loglevel", default="warning", section="logging")
logdirectory = get_option("logdirectory", default="log", section="logging")
self._setLogging(loglevel, logdirectory)
raw_testsize = humanfriendly.parse_size(get_option("testsize"))
set_option("raw_testsize", raw_testsize)
def abstract_test_chargeback_cost(
rate_key, obj_type, interval, chargeback_report_data, compute_rate, soft_assert):
"""This is an abstract test function for testing rate costs.
It's comparing the expected value that calculated by the rate
to the value in the chargeback report
Args:
:py:type:`str` rate_key: The rate key as it appear in the CHARGEBACK_HEADER_NAMES keys.
:py:type:`str` obj_type: Object being tested; only 'Project' and 'Image' are supported
:py:type:`str` interval: The rate interval, (Hourly/Daily/Weekly/Monthly)
:py:class:`Report` chargeback_report_data: The chargeback report data.
:py:class:`ComputeRate` compute_rate: The compute rate object.
:var soft_assert: soft_assert fixture.
"""
report_headers = CHARGEBACK_HEADER_NAMES[rate_key]
found_something_to_test = False
for row in chargeback_report_data.rows:
if row['Chargeback Rates'].lower() != compute_rate.description.lower():
continue
found_something_to_test = True
fixed_rate = float(compute_rate.fields[report_headers.rate_name]['fixed_rate'])
variable_rate = float(compute_rate.fields[report_headers.rate_name].get('variable_rate', 0))
# Calculate numerical metric
if rate_key == 'Memory':
size_, unit_ = tokenize(row[report_headers.metric_name].upper())
metric = round(parse_size(str(size_) + unit_, binary=True) / 1048576.0, 2)
else:
metric = parse_number(row[report_headers.metric_name])
# Calculate fixed product and cost
num_hours = parse_number(row[CHARGEBACK_HEADER_NAMES['Fixed1'].metric_name])
num_intervals = num_hours / hours_count_lut[interval]
fixed_cost = num_intervals * fixed_rate
variable_cost = num_intervals * metric * variable_rate
# Calculate expected cost
expected_cost = round(variable_cost + fixed_cost, 2)
found_cost = round(parse_number(row[report_headers.cost_name]), 2)
match_threshold = TEST_MATCH_ACCURACY * expected_cost
soft_assert(
abs(found_cost - expected_cost) <= match_threshold,
'Unexpected Chargeback: {}'.format(dump_args(
charge_for=obj_type, rate_key=rate_key, metric=metric, num_hours=num_hours,
num_intervals=num_intervals, fixed_rate=fixed_rate, variable_rate=variable_rate,
fixed_cost=fixed_cost, variable_cost=variable_cost,
expected_full_cost=expected_cost, found_full_cost=found_cost
))
)
assert found_something_to_test, \
'Could not find {} with the assigned rate: {}'.format(obj_type, compute_rate.description)
def __init__(self, path, part_size, base_dir=None):
# Basic #
self.path = path
# Directory #
if base_dir is None: self.base_dir = path + '.parts/'
else: self.base_dir = base_dir
if not os.path.exists(self.base_dir): os.makedirs(self.base_dir)
# Evaluate size #
self.bytes_target = humanfriendly.parse_size(part_size)
# Chose number of parts #
self.num_parts = int(math.ceil(self.count_bytes / self.bytes_target))
# Make parts #
self.parts = [FASTA(self.make_part_paths(i)) for i in range(self.num_parts)]
def max_size(directory):
"""Test directory for maximum size
returns true if directory size is larger than spec property "max_size"
"""
try:
max_size = humanfriendly.parse_size(directory.spec['max_size'])
except KeyError:
raise UnregulatableError('No "max_size" configured in specification!')
size = directory.size
logger.debug("Directory size is %s", humanfriendly.format_size(size))
logger.debug("max_size set to: %s", humanfriendly.format_size(max_size))
return size > max_size
def test_parse_size(self):
self.assertEqual(42, humanfriendly.parse_size('42'))
self.assertEqual(1024, humanfriendly.parse_size('1k'))
self.assertEqual(1024, humanfriendly.parse_size('1 KB'))
self.assertEqual(1024, humanfriendly.parse_size('1 kilobyte'))
self.assertEqual(1024 ** 3, humanfriendly.parse_size('1 GB'))
try:
humanfriendly.parse_size('1z')
self.assertTrue(False)
except Exception, e:
self.assertTrue(isinstance(e, humanfriendly.InvalidSize))
def clean_fs_size():
try:
cron_cfg = config.frontend_config['cron_clean_file_size']
max_size = cron_cfg['clean_fs_max_size']
# 0 means disabled
if max_size == '0':
log.debug("disabled by config")
return 0
max_size_bytes = humanfriendly.parse_size(max_size, binary=True)
nb_files = file_ctrl.remove_files_size(max_size_bytes)
log.info("removed %d files", nb_files)
return nb_files
except (IrmaDatabaseError, IrmaFileSystemError) as e:
log.exception(e)
def create(ctx, name, size, description, storage_profile, iops):
try:
restore_session(ctx, vdc_required=True)
client = ctx.obj['client']
vdc_href = ctx.obj['profiles'].get('vdc_href')
vdc = VDC(client, href=vdc_href)
disk_resource = vdc.create_disk(
name=name,
size=humanfriendly.parse_size(size),
description=description,
storage_profile_name=storage_profile,
iops=iops)
stdout(disk_resource.Tasks.Task[0], ctx)
except Exception as e:
stderr(e, ctx)
def __init__(self, root=None, settings=None):
self.fs_root = root
self.fs_path = join(root, 'drops')
self.fs_submission_queue = join(root, 'submissions')
self.fs_scratch = join(root, 'scratch')
# initialise settings from disk and parameters
# settings provided as init parameter take precedence over values on-disk
# which in turn take precedence over default values
self.settings = dict(
attachment_size_threshold=u'2Mb',
)
self.settings.update(**self.parse_settings())
if settings is not None:
self.settings.update(**settings)
# set archive paths
self.fs_archive_cleansed = self.settings.get('dropbox_cleansed_archive_path', join(root, 'archive_cleansed'))
self.fs_archive_dirty = self.settings.get('dropbox_dirty_archive_path', join(root, 'archive_dirty'))
self.fs_archive = dict(
clean=self.fs_archive_cleansed,
dirty=self.fs_archive_dirty,
)
# set smtp instance defensively, to not overwrite mocked version from test settings:
if 'smtp' not in self.settings:
self.settings['smtp'] = setup_smtp_factory(**self.settings)
# setup GPG
self.gpg_context = gnupg.GPG(
gnupghome=self.settings['fs_pgp_pubkeys'],
gpgbinary=self.settings.get('fs_gpg_path', 'gpg'),
)
# convert human readable size to bytes
self.settings['attachment_size_threshold'] = parse_size(self.settings['attachment_size_threshold'])
# ensure directories exist
for directory in [
self.fs_root,
self.fs_path,
self.fs_submission_queue,
self.fs_archive_cleansed,
self.fs_archive_dirty,
self.fs_scratch]:
if not exists(directory):
makedirs(directory)
def process_module_programming(module, lines, specific, source_path):
log("Processing programming module")
# Hledame vzorovy kod v zadani
line = 0
while (line < len(lines)) and (not re.match(r"^```~python", lines[line])): line += 1
if line == len(lines): return lines
# Hledame konec kodu
end = line+1
while (end < len(lines)) and (not re.match(r"^```", lines[end])): end += 1
code = ''.join(lines[line+1:end])
# Pridame vzorovy kod do \module.data
data = {}
#old_data = json.loads(module.data) if module.data else None
#data['programming'] = old_data['programming'] if (old_data) and ('programming' in old_data) else {}
data['programming'] = {}
data['programming']['default_code'] = code
# Zkopirujeme skripty do prislusnych adresaru
target_path = "data/modules/" + str(module.id) + "/"
files = [ "eval.py", "merge.py", "post.py", "stdin.txt" ]
if 'available' in specific: files.extend(specific['available'])
if not os.path.isdir(target_path): os.makedirs(target_path)
for f in files:
if os.path.isfile(source_path+"/"+f):
shutil.copy2(source_path+"/"+f, target_path+f)
data['programming']['merge_script'] = target_path + "merge.py"
data['programming']['stdin'] = target_path + "stdin.txt"
if not os.path.isfile(source_path+"/stdin.txt"):
open(target_path + "stdin.txt", "a").close() # create empty stdin
if os.path.isfile(source_path+"/post.py"): data['programming']['post_trigger_script'] = target_path + "post.py"
data['programming']['check_script'] = target_path + "eval.py"
# direktivy z module.json
if 'timeout' in specific: data['programming']['timeout'] = specific['timeout']
if 'args' in specific: data['programming']['args'] = specific['args']
if 'heaplimit' in specific:
data['programming']['heaplimit'] = parse_size(specific['heaplimit'])
log("Heaplimit parsed as " + str(data['programming']['heaplimit']) + " bytes")
module.data = json.dumps(data, indent=2)
return lines[:line]
def __init__(self, path, num_parts=None, part_size=None, base_dir=None):
# Basic #
self.path = path
# Directory #
if base_dir is None: self.base_dir = path + '.parts/'
else: self.base_dir = base_dir
# Num parts #
if num_parts is not None: self.num_parts = num_parts
# Evaluate size #
if part_size is not None:
self.bytes_target = humanfriendly.parse_size(part_size)
self.num_parts = int(math.ceil(self.count_bytes / self.bytes_target))
# Make parts #
self.make_name = lambda i: self.base_dir + "%03d/part.fasta" % i
self.parts = [FASTA(self.make_name(i)) for i in range(1, self.num_parts+1)]
# Give a number to each part #
for i, part in enumerate(self.parts): part.num = i
def download(source, username, directory, max_size):
"""
Download data from Open Humans.
"""
signal.signal(signal.SIGINT, signal_handler_cb)
max_bytes = parse_size(max_size)
options = {}
if source:
options['source'] = source
if username:
options['username'] = username
page = '{}?{}'.format(BASE_URL_API, urlencode(options))
results = []
counter = 1
print 'Retrieving metadata'
while True:
print 'Retrieving page {}'.format(counter)
response = get_page(page)
results = results + response['results']
if response['next']:
page = response['next']
else:
break
counter += 1
print 'Downloading {} files'.format(len(results))
download_url_partial = partial(download_url, directory=directory,
max_bytes=max_bytes)
with concurrent.futures.ProcessPoolExecutor(max_workers=4) as executor:
for value in executor.map(download_url_partial, results):
if value:
print value
def _scrape_movie(url):
request = YtsRequest(url)
response = ScraperBase._try_open_stream(request)
soup = BeautifulSoup(response, YtsScraper.HtmlLibrary)
movie_wrapper = soup.find("div", {'id': YtsScraper.MovieWrapperClass})
image_url = movie_wrapper.find("div", YtsScraper.MovieImageClass).find("img")["src"]
movie_info_section = movie_wrapper.find("div", YtsScraper.MovieInfoClass)
full_title = movie_info_section.find("h1").getText()
title_re = re.search('^(.+)\s+\((\d+)\).*$', full_title)
title = str(title_re.group(1))
year = int(title_re.group(2))
info_dict = {str(info_rex.group(1)).lower().replace(' ', ''): str(info_rex.group(2)) for info_rex in
[re.search('^(.+?):\s*(.+)$', info.getText().replace('\n', '').strip()) for info in
movie_info_section.findAll("p")] if info_rex is not None}
genres = [genre.strip() for genre in info_dict["genre"].split('|')]
size = humanfriendly.parse_size(info_dict["size"])
rating = float(info_dict["imdbrating"].split("/")[0])
quality = MovieQuality.parse_p_value(info_dict["quality"])
tex = re.search('^(?:(\d+)(?:hr\s*))?(\d+)\s+min$', info_dict["runtime"])
hours = int(tex.group(1)) if tex.group(1) is not None else 0
minutes = int(tex.group(2))
runtime = hours * 60 + minutes
description_section = movie_wrapper.find("div", YtsScraper.MovieDescriptionClass)
synopsis_section = description_section.find("div", YtsScraper.MovieDescriptionInnerClass)
synopsis = synopsis_section.find('p').getText().replace('\n', '')
page_links = [urllib.parse.urlparse(anchor['href']) for anchor in movie_info_section.findAll('a')]
imdb = youtube_id = magnet = ""
for page_link in page_links:
if page_link.netloc == "www.imdb.com":
imdb = re.search('.+?/(.+)/$', page_link.path).group(1)
elif page_link.netloc == "www.youtube.com":
youtube_id = page_link.params['v']
elif page_link.scheme == "magnet":
magnet = page_link.geturl()
return Movie(MediaSource.Yts, title, year, url, magnet, imdb, rating, genres, quality, size, runtime,
synopsis, image_url, youtube_id)
def metrics_series(self):
'''
Aggregate check results metrics series attached to this state.
'''
results = CheckResult.objects.get_state_log(self, max_results=1000)
metrics = defaultdict(list)
for result in reversed(results):
for name, value in result.metrics.items():
if isinstance(value, six.string_types):
try:
value = humanfriendly.parse_size(value)
except humanfriendly.InvalidSize:
continue
metrics[name].append((result.timestamp, value))
# Disable default_factory in defaultdict so it can be used in django
# templates
metrics.default_factory = None
return metrics
def render_container_spec(app_name, proc):
c = ContainerSpec()
c.Image = proc.image
c.Env = copy.deepcopy(proc.env)
c.set_env("TZ", 'Asia/Shanghai')
c.User = '' if not hasattr(proc, 'user') else proc.user
c.WorkingDir = '' if not hasattr(proc, 'working_dir') else proc.working_dir
c.DnsSearch = [] if not hasattr(proc, 'dns_search') else copy.deepcopy(proc.dns_search)
c.Volumes = copy.deepcopy(proc.volumes)
c.SystemVolumes = copy.deepcopy(proc.system_volumes) + get_system_volumes_from_etcd(app_name)
c.CloudVolumes = render_cloud_volumes(proc.cloud_volumes)
c.Command = proc.cmd
c.Entrypoint = proc.entrypoint
c.CpuLimit = proc.cpu
c.MemoryLimit = humanfriendly.parse_size(proc.memory)
c.Expose = 0 if not proc.port else proc.port.keys()[0]
c.LogConfig = None
return c
def main():
set_stdout_encoding()
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
description=''
)
parser.add_argument('fetch_source')
parser.add_argument('--host', default='localhost',
help='Host interface to listen on')
parser.add_argument('-p', '--port', type=int, default=8080)
parser.add_argument('-l', '--loglevel', '--log-level',
help='Logging verbosity level threshold (to stderr)',
default='info')
parser.add_argument('--static-path', default='.',
help='Path where HTML files are stored')
parser.add_argument('--store-path', default='.',
help='Path where Communications should be stored')
parser.add_argument('--max-file-size', type=str, default='1GiB',
help="Maximum size of (non-ZIP) files that can be read into memory "
"(e.g. '2G', '300MB')")
args = parser.parse_args()
logging.basicConfig(format='%(asctime)-15s %(levelname)s: %(message)s',
level=args.loglevel.upper())
comm_container = {}
if os.path.isdir(args.fetch_source):
comm_container = DirectoryBackedCommunicationContainer(args.fetch_source)
elif zipfile.is_zipfile(args.fetch_source):
comm_container = ZipFileBackedCommunicationContainer(args.fetch_source)
else:
max_file_size = humanfriendly.parse_size(args.max_file_size, binary=True)
comm_container = MemoryBackedCommunicationContainer(args.fetch_source,
max_file_size=max_file_size)
fetch_handler = CommunicationContainerFetchHandler(comm_container)
store_handler = DirectoryBackedStoreHandler(args.store_path)
logging.info('Fetch endpoint: http://{}:{}/fetch_http_endpoint/'.format(args.host, args.port))
logging.info('Store endpoint: http://{}:{}/store_http_endpoint/'.format(args.host, args.port))
ahs = AccessHTTPServer(args.host, args.port, args.static_path, fetch_handler, store_handler)
ahs.serve()
def scan(self):
with self.getdbcursor() as (db, cursor):
selected = cursor.execute(select1).fetchall()
for pkg, path in selected:
print Fore.BLUE + 'Uploading %s for scan ...' % pkg,
path = os.path.abspath(os.path.join(self.basedir, path))
if os.path.getsize(path) >= hf.parse_size('32M'):
cursor.execute(insert, (pkg, "file-too-big", "", -100))
print Fore.RED + 'File too big for VirusTotal'
continue
try:
id = self.api.scan(path)
cursor.execute(insert, (pkg, id, time(), -1))
print Fore.GREEN + 'OK'
except:
cursor.execute(insert, (pkg, "api-scan-error", "", -100))
print Fore.RED + 'API Error'
db.commit()
return len(selected)
def update(ctx, name, new_name, size, description, storage_profile, iops):
try:
restore_session(ctx, vdc_required=True)
client = ctx.obj['client']
vdc_href = ctx.obj['profiles'].get('vdc_href')
vdc = VDC(client, href=vdc_href)
new_size = None
if size is not None:
new_size = humanfriendly.parse_size(size)
name, id = extract_name_and_id(name)
task = vdc.update_disk(
name=name,
disk_id=id,
new_name=new_name,
new_size=new_size,
new_description=description,
new_storage_profile_name=storage_profile,
new_iops=iops)
stdout(task, ctx)
except Exception as e:
stderr(e, ctx)
def create(filename, size):
size_in_bytes = humanfriendly.parse_size(size)
with open(filename, "wb") as out:
out.truncate(size_in_bytes)
cmd = "cryptsetup -y -v luksFormat %s" % filename
subprocess.call(cmd.split())
mapping_name = os.path.basename(filename)
cmd = "cryptsetup luksOpen %s %s" % (filename, mapping_name)
subprocess.call(cmd.split())
cmd = "dd if=/dev/zero of=/dev/mapper/%s" % mapping_name
subprocess.call(cmd.split())
cmd = "mkfs.ext4 /dev/mapper/%s" % mapping_name
subprocess.call(cmd.split())
cmd = "cryptsetup luksClose %s" % mapping_name
subprocess.call(cmd.split())
def parse_result(self, tree):
rows = tree.xpath(CN_XPATH_RESULTS_ROWS)
if len(rows) >= 1:
logging.debug('found %d rows' % len(rows))
results = []
for row in rows:
result = {}
tds = row.getchildren()
a = tds[1].find('.//a[@href]')
ep_found = 1
for regex in EP_PATTERN_REGEX:
r = re.match(regex, a.text_content())
if r:
result['season'] = r.group('season')
result['ep_start'] = r.group('start')
result['ep_end'] = result['ep_start']
if 'end' in regex:
result['ep_end'] = r.group('end')
ep_found = (int(result['ep_end']) -
int(result['ep_start']))
if ep_found == 0:
ep_found = 1
break
result['text'] = a.text_content()
result['link'] = a.attrib['href']
size = humanfriendly.parse_size(tds[2].text_content())
result['size'] = size / ep_found
result['data'] = time.strptime(tds[4].text_content(),
"%d.%m.%y")
result['seeds'] = int(tds[5].text_content())
result['leech'] = int(tds[6].text_content())
item_tree = html.parse(urllib2.urlopen(result['link']))
links = item_tree.xpath('//a[@href]')
for link in links:
if link.attrib['href'].startswith('magnet:?'):
result['torrent'] = link.attrib['href']
results.append(result)
return(results)
else:
return([])
def test_cli(self):
"""Test the command line interface."""
# Test that the usage message is printed by default.
returncode, output = main()
assert 'Usage:' in output
# Test that the usage message can be requested explicitly.
returncode, output = main('--help')
assert 'Usage:' in output
# Test handling of invalid command line options.
returncode, output = main('--unsupported-option')
assert returncode != 0
# Test `humanfriendly --format-number'.
returncode, output = main('--format-number=1234567')
assert output.strip() == '1,234,567'
# Test `humanfriendly --format-size'.
random_byte_count = random.randint(1024, 1024 * 1024)
returncode, output = main('--format-size=%i' % random_byte_count)
assert output.strip() == humanfriendly.format_size(random_byte_count)
# Test `humanfriendly --format-table'.
returncode, output = main('--format-table', '--delimiter=\t', input='1\t2\t3\n4\t5\t6\n7\t8\t9')
assert output.strip() == dedent('''
-------------
| 1 | 2 | 3 |
| 4 | 5 | 6 |
| 7 | 8 | 9 |
-------------
''').strip()
# Test `humanfriendly --format-timespan'.
random_timespan = random.randint(5, 600)
returncode, output = main('--format-timespan=%i' % random_timespan)
assert output.strip() == humanfriendly.format_timespan(random_timespan)
# Test `humanfriendly --parse-size'.
returncode, output = main('--parse-size=5 KB')
assert int(output) == humanfriendly.parse_size('5 KB')
# Test `humanfriendly --run-command'.
returncode, output = main('--run-command', 'bash', '-c', 'sleep 2 && exit 42')
assert returncode == 42
def validate_parameters(cpu, memory, numinstances):
if all([cpu is None, memory is None, numinstances is None]):
warn("please input at least one param in cpu/memory/numinstances")
exit(1)
if numinstances is not None:
try:
numinstances = int(numinstances)
except ValueError:
warn('invalid parameter: num_instances (%s) should be integer'%numinstances)
exit(1)
if numinstances <= 0:
warn('invalid parameter: num_instances (%s) should > 0'%numinstances)
exit(1)
if cpu is not None:
try:
cpu = int(cpu)
except ValueError:
warn('invalid parameter: cpu (%s) should be integer'%cpu)
exit(1)
if cpu < 0:
warn('invalid parameter: cpu (%s) should >= 0'%cpu)
exit(1)
if memory is not None:
memory = str(memory)
try:
if humanfriendly.parse_size(memory) < 4194304:
error('invalid parameter: memory (%s) should >= 4M'%memory)
exit(1)
except humanfriendly.InvalidSize:
error('invalid parameter: memory (%s) humanfriendly.parse_size(memory) failed'%memory)
exit(1)
return cpu, memory, numinstances
