def testDontMatchAcrossContainers(self):
container_a = models.Container('A', metadata={}, section_sizes={})
container_b = models.Container('B', metadata={}, section_sizes={})
containers = [container_a, container_b]
size_info1 = _CreateSizeInfo(containers=containers)
size_info1.raw_symbols[0].container = container_b
size_info2 = _CreateSizeInfo(containers=containers)
d = diff.Diff(size_info1, size_info2)
# Should show as one add and one remove rather than a change.
self.assertEqual((0, 1, 1), d.raw_symbols.CountsByDiffStatus()[1:])
def _DescribeSizeInfo(self, size_info):
desc_list = []
# Describe |build_config| and each container. If there is only one container
# then support legacy output by reporting |build_config| as part of the
# first container's metadata.
if len(size_info.containers) > 1:
desc_list.append(('Build Configs:', ))
desc_list.append(' %s' % line
for line in DescribeDict(size_info.build_config))
containers = size_info.containers
else:
containers = [
models.Container(
name='',
metadata=size_info.metadata_legacy,
section_sizes=size_info.containers[0].section_sizes)
]
for c in containers:
if c.name:
desc_list.append(('', ))
desc_list.append(('Container <%s>' % c.name, ))
desc_list.append(('Metadata:', ))
desc_list.append(' %s' % line
for line in DescribeDict(c.metadata))
unsummed_sections, summed_sections = c.ClassifySections()
desc_list.append(
self._DescribeSectionSizes(unsummed_sections, summed_sections,
c.section_sizes))
if self.verbose:
desc_list.append(('', ))
desc_list.append(DescribeSizeInfoCoverage(size_info))
desc_list.append(('', ))
desc_list.append(self.GenerateLines(size_info.symbols))
return itertools.chain.from_iterable(desc_list)
def _CreateSizeInfo(aliases=None, containers=None):
build_config = {}
metadata = {}
section_sizes = {'.text': 100, '.bss': 40}
if not containers:
containers = [
models.Container('',
metadata=metadata,
section_sizes=section_sizes)
]
models.BaseContainer.AssignShortNames(containers)
TEXT = models.SECTION_TEXT
symbols = [
_MakeSym(models.SECTION_DEX_METHOD, 10, 'a', 'com.Foo#bar()'),
_MakeSym(TEXT, 20, 'a', '.Lfoo'),
_MakeSym(TEXT, 30, 'b'),
_MakeSym(TEXT, 40, 'b'),
_MakeSym(TEXT, 50, 'b'),
_MakeSym(TEXT, 60, ''),
]
for s in symbols:
s.container = containers[0]
if aliases:
for tup in aliases:
syms = symbols[tup[0]:tup[1]]
for sym in syms:
sym.aliases = syms
return models.SizeInfo(build_config, containers, symbols)
def _CreateSizeInfo(aliases=None):
build_config = {}
metadata = {}
section_sizes = {'.text': 100, '.bss': 40}
containers = [
models.Container(name='',
metadata=metadata,
section_sizes=section_sizes)
]
TEXT = models.SECTION_TEXT
symbols = [
_MakeSym(models.SECTION_DEX_METHOD, 10, 'a', 'com.Foo#bar()'),
_MakeSym(TEXT, 20, 'a', '.Lfoo'),
_MakeSym(TEXT, 30, 'b'),
_MakeSym(TEXT, 40, 'b'),
_MakeSym(TEXT, 50, 'b'),
_MakeSym(TEXT, 60, ''),
]
# For simplicity, not associating |symbols| with |containers|.
if aliases:
for tup in aliases:
syms = symbols[tup[0]:tup[1]]
for sym in syms:
sym.aliases = syms
return models.SizeInfo(build_config, containers, symbols)
def add_container(values, project):
#session = get_session()
session = inspect(project).session
with session.begin():
container_ref = models.Container()
container_ref.update(values)
container_ref.project = project
container_ref.save(session=session)
def _load(self, force):
if self._model == None or force:
try:
self._model = models.Container.objects.get(name=self._name)
# print "container %s loaded" % self._name
except:
self._model = models.Container(name=self._name)
try: #<-- initial syncdb bug fix
self._model.save()
except:
pass
def _DiffContainerLists(before_containers, after_containers):
"""Computes diffs between two lists of Containers."""
# TODO(huangs): Add support for multiple containers (needs name matching).
assert len(before_containers) == 1
assert len(after_containers) == 1
ret = []
for (before_c, after_c) in zip(before_containers, after_containers):
name = after_c.name
assert before_c.name == name
ret.append(
models.Container(name=name,
metadata=_DiffObj(before_c.metadata, after_c.metadata),
section_sizes=_DiffObj(before_c.section_sizes,
after_c.section_sizes)))
return ret
def add_container():
username = None
if 'username' in session:
username = session['username']
flag = int(request.form['flag'])
bay = int(request.form['bay']) if request.form['bay'] else None
cycle_bay = int(
request.form['cycle_bay']) if request.form['cycle_bay'] else None
yard = int(request.form['yard']) if request.form['yard'] else None
cycle_yard = int(
request.form['cycle_yard']) if request.form['cycle_yard'] else None
container = models.Container(user=username,
flag=flag,
bay=bay,
cycle_bay=cycle_bay,
yard=yard,
cycle_yard=cycle_yard)
db.session.add(container)
db.session.commit()
return '{"status": "ok", "container_id": "%s"}' % container.id
def _DiffContainerLists(before_containers, after_containers):
"""Computes diff of Containers lists, matching names."""
# Find ordered unique names, preferring order of |container_after|.
pairs = collections.OrderedDict()
for c in after_containers:
pairs[c.name] = [models.Container.Empty(), c]
for c in before_containers:
if c.name in pairs:
pairs[c.name][0] = c
else:
pairs[c.name] = [c, models.Container.Empty()]
ret = []
for name, [before_c, after_c] in pairs.items():
ret.append(
models.Container(name=name,
metadata=_DiffObj(before_c.metadata,
after_c.metadata),
section_sizes=_DiffObj(before_c.section_sizes,
after_c.section_sizes)))
# This update newly created diff Containers, not existing ones or EMPTY.
models.Container.AssignShortNames(ret)
return ret
def _LoadSizeInfoFromFile(file_obj, size_path):
"""Loads a size_info from the given file.
See _SaveSizeInfoToFile() for details on the .size file format.
Args:
file_obj: File to read, should be a GzipFile
"""
# Split lines on '\n', since '\r' can appear in some lines!
lines = io.TextIOWrapper(file_obj, newline='\n')
header_line = _ReadLine(lines).encode('ascii')
assert header_line == _COMMON_HEADER[:-1], 'was ' + str(header_line)
header_line = _ReadLine(lines).encode('ascii')
if header_line == _SIZE_HEADER_SINGLE_CONTAINER[:-1]:
has_multi_containers = False
elif header_line == _SIZE_HEADER_MULTI_CONTAINER[:-1]:
has_multi_containers = True
else:
raise ValueError('Version mismatch. Need to write some upgrade code.')
# JSON header fields
json_len = int(_ReadLine(lines))
json_str = lines.read(json_len)
fields = json.loads(json_str)
assert ('containers' in fields) == has_multi_containers
assert ('build_config' in fields) == has_multi_containers
assert ('containers' in fields) == has_multi_containers
assert ('metadata' not in fields) == has_multi_containers
assert ('section_sizes' not in fields) == has_multi_containers
containers = []
if has_multi_containers: # New format.
build_config = fields['build_config']
for cfield in fields['containers']:
c = models.Container(name=cfield['name'],
metadata=cfield['metadata'],
section_sizes=cfield['section_sizes'])
containers.append(c)
else: # Old format.
build_config = {}
metadata = fields.get('metadata')
if metadata:
for key in models.BUILD_CONFIG_KEYS:
if key in metadata:
build_config[key] = metadata[key]
del metadata[key]
section_sizes = fields['section_sizes']
containers.append(
models.Container(name='',
metadata=metadata,
section_sizes=section_sizes))
models.Container.AssignShortNames(containers)
has_components = fields.get('has_components', False)
has_padding = fields.get('has_padding', False)
# Eat empty line.
_ReadLine(lines)
# Path list.
num_path_tuples = int(_ReadLine(lines)) # Number of paths in list.
# Read the path list values and store for later.
path_tuples = [
_ReadValuesFromLine(lines, split='\t') for _ in range(num_path_tuples)
]
if num_path_tuples == 0:
logging.warning('File contains no symbols: %s', size_path)
return models.SizeInfo(build_config,
containers, [],
size_path=size_path)
# Component list.
if has_components:
num_components = int(_ReadLine(lines)) # Number of components in list.
components = [_ReadLine(lines) for _ in range(num_components)]
# Symbol counts by "segments", defined as (container, section) tuples.
segment_names = _ReadValuesFromLine(lines, split='\t')
symbol_counts = [int(c) for c in _ReadValuesFromLine(lines, split='\t')]
# Addresses, sizes, paddings, path indices, component indices.
def read_numeric(delta=False):
"""Read numeric values, where each line corresponds to a symbol group.
The values in each line are space separated.
If |delta| is True, the numbers are read as a value to add to the sum of the
prior values in the line, or as the amount to change by.
"""
ret = []
delta_multiplier = int(delta)
for _ in symbol_counts:
value = 0
fields = []
for f in _ReadValuesFromLine(lines, split=' '):
value = value * delta_multiplier + int(f)
fields.append(value)
ret.append(fields)
return ret
addresses = read_numeric(delta=True)
sizes = read_numeric(delta=False)
if has_padding:
paddings = read_numeric(delta=False)
else:
paddings = [None] * len(segment_names)
path_indices = read_numeric(delta=True)
if has_components:
component_indices = read_numeric(delta=True)
else:
component_indices = [None] * len(segment_names)
raw_symbols = [None] * sum(symbol_counts)
symbol_idx = 0
for (cur_segment_name, cur_symbol_count, cur_addresses, cur_sizes,
cur_paddings, cur_path_indices,
cur_component_indices) in zip(segment_names, symbol_counts, addresses,
sizes, paddings, path_indices,
component_indices):
if has_multi_containers:
# Extract '<cur_container_idx_str>cur_section_name'.
assert cur_segment_name.startswith('<')
cur_container_idx_str, cur_section_name = (
cur_segment_name[1:].split('>', 1))
cur_container = containers[int(cur_container_idx_str)]
else:
cur_section_name = cur_segment_name
cur_container = containers[0]
alias_counter = 0
for i in range(cur_symbol_count):
parts = _ReadValuesFromLine(lines, split='\t')
full_name = parts[0]
flags_part = None
aliases_part = None
# aliases_part or flags_part may have been omitted.
if len(parts) == 3:
# full_name aliases_part flags_part
aliases_part = parts[1]
flags_part = parts[2]
elif len(parts) == 2:
if parts[1][0] == '0':
# full_name aliases_part
aliases_part = parts[1]
else:
# full_name flags_part
flags_part = parts[1]
# Use a bit less RAM by using the same instance for this common string.
if full_name == models.STRING_LITERAL_NAME:
full_name = models.STRING_LITERAL_NAME
flags = int(flags_part, 16) if flags_part else 0
num_aliases = int(aliases_part, 16) if aliases_part else 0
# Skip the constructor to avoid default value checks.
new_sym = models.Symbol.__new__(models.Symbol)
new_sym.container = cur_container
new_sym.section_name = cur_section_name
new_sym.full_name = full_name
new_sym.address = cur_addresses[i]
new_sym.size = cur_sizes[i]
paths = path_tuples[cur_path_indices[i]]
new_sym.object_path, new_sym.source_path = paths
component = components[
cur_component_indices[i]] if has_components else ''
new_sym.component = component
new_sym.flags = flags
# Derived.
if cur_paddings:
new_sym.padding = cur_paddings[i]
if not new_sym.IsOverhead():
new_sym.size += new_sym.padding
else:
new_sym.padding = 0 # Computed below.
new_sym.template_name = ''
new_sym.name = ''
if num_aliases:
assert alias_counter == 0
new_sym.aliases = [new_sym]
alias_counter = num_aliases - 1
elif alias_counter > 0:
new_sym.aliases = raw_symbols[symbol_idx - 1].aliases
new_sym.aliases.append(new_sym)
alias_counter -= 1
else:
new_sym.aliases = None
raw_symbols[symbol_idx] = new_sym
symbol_idx += 1
if not has_padding:
CalculatePadding(raw_symbols)
return models.SizeInfo(build_config,
containers,
raw_symbols,
size_path=size_path)
def create_container(body):
""" return info about container to create
get data from body and return information that will used to create
container
Author: Frazy Lee
Author Email: [email protected]
Params:
body: DICT # the data receive from mq
Return:( # 返回值
status: INT, # execute status 0/Success, -1/other-Fault
msgs: STRING, # the error message, when excecute fault
results: DICT, # excecute result
)
Results Format:{
'status': u'', # the container status, when the container create
successful,this field will update
'user_id': '2', # the user id who create this container
'name': u'', # the name of the container, when the container create
successful,this field will update
'created': u'', # the created of the new container, when the container
create successful,this field will update
'hostname': u'', # the hostname of the new container, when the
container create successful,this field will update
'image_name': u'ubuntu:latest', # the name of image that used to create
the new container
'id': 10L, # the new container db note, but the status is False
'host': u'127.0.0.1', # the docker server ip
'cid': u'', # the new container id, when the container create
successful,this field will update
'command': u'', # the command of the new container
'hostport': u'2375', # the docker server port
'flavor':{
'name': 'tiny',
'mem': 128,
'volume': 0,
'bandwidth': 512,
'sys_disk': 5120,
'cpu': 1
} # the detail info about flavor
}
"""
flavor = body.get('flavor_id')
image = body.get('image')
hostdic = scheduler_host(
flavor=flavor,
image=image,
)
if hostdic[0] == 0:
hostdic = hostdic[2]
else:
return hostdic
containerobject = models.Container(
flavor_id=flavor,
image=hostdic.get("image"),
user_id=body.get('user_id'),
host=hostdic.get('hostobject'),
)
containerobject.save()
body.pop('image')
body.pop('host')
body.pop('flavor_id')
hostdic.pop('image')
hostdic.pop('hostobject')
hostdic['id'] = containerobject.id
resultdic = dict(body.items() + hostdic.items())
return (0, '', resultdic)
def _LoadSizeInfoFromFile(file_obj, size_path):
"""Loads a size_info from the given file.
See _SaveSizeInfoToFile() for details on the .size file format.
Args:
file_obj: File to read, should be a GzipFile
"""
# Split lines on '\n', since '\r' can appear in some lines!
lines = io.TextIOWrapper(file_obj, newline='\n')
_ReadLine(lines) # Line 0: Created by supersize header
actual_version = _ReadLine(lines)
assert actual_version == _SERIALIZATION_VERSION, (
'Version mismatch. Need to write some upgrade code.')
# JSON header fields
json_len = int(_ReadLine(lines))
json_str = lines.read(json_len)
fields = json.loads(json_str)
has_multi_containers = False
containers = []
if has_multi_containers: # New format.
raise ValueError('Multiple container not yet supported.')
else:
# Parse old format, but separate data into build_config and metadata.
build_config = {}
metadata = fields.get('metadata')
if metadata:
for key in models.BUILD_CONFIG_KEYS:
if key in metadata:
build_config[key] = metadata[key]
del metadata[key]
section_sizes = fields['section_sizes']
containers.append(
models.Container(name='',
metadata=metadata,
section_sizes=section_sizes))
has_components = fields.get('has_components', False)
has_padding = fields.get('has_padding', False)
# Eat empty line.
_ReadLine(lines)
# Path list
num_path_tuples = int(_ReadLine(lines)) # Number of paths in list
# Read the path list values and store for later
path_tuples = [
_ReadValuesFromLine(lines, split='\t') for _ in range(num_path_tuples)
]
# Component list
if has_components:
num_components = int(_ReadLine(lines)) # number of components in list
components = [_ReadLine(lines) for _ in range(num_components)]
# Symbol counts by section.
section_names = _ReadValuesFromLine(lines, split='\t')
symbol_counts = [int(c) for c in _ReadValuesFromLine(lines, split='\t')]
# Addresses, sizes, paddings, path indices, component indices
def read_numeric(delta=False):
"""Read numeric values, where each line corresponds to a symbol group.
The values in each line are space separated.
If |delta| is True, the numbers are read as a value to add to the sum of the
prior values in the line, or as the amount to change by.
"""
ret = []
delta_multiplier = int(delta)
for _ in symbol_counts:
value = 0
fields = []
for f in _ReadValuesFromLine(lines, split=' '):
value = value * delta_multiplier + int(f)
fields.append(value)
ret.append(fields)
return ret
addresses = read_numeric(delta=True)
sizes = read_numeric(delta=False)
if has_padding:
paddings = read_numeric(delta=False)
else:
paddings = [None] * len(section_names)
path_indices = read_numeric(delta=True)
if has_components:
component_indices = read_numeric(delta=True)
else:
component_indices = [None] * len(section_names)
raw_symbols = [None] * sum(symbol_counts)
symbol_idx = 0
for (cur_section_name, cur_symbol_count, cur_addresses, cur_sizes,
cur_paddings, cur_path_indices,
cur_component_indices) in zip(section_names, symbol_counts, addresses,
sizes, paddings, path_indices,
component_indices):
if has_multi_containers:
raise ValueError('Multiple container not yet supported.')
else:
cur_container = containers[0]
alias_counter = 0
for i in range(cur_symbol_count):
parts = _ReadValuesFromLine(lines, split='\t')
full_name = parts[0]
flags_part = None
aliases_part = None
# aliases_part or flags_part may have been omitted.
if len(parts) == 3:
# full_name aliases_part flags_part
aliases_part = parts[1]
flags_part = parts[2]
elif len(parts) == 2:
if parts[1][0] == '0':
# full_name aliases_part
aliases_part = parts[1]
else:
# full_name flags_part
flags_part = parts[1]
# Use a bit less RAM by using the same instance for this common string.
if full_name == models.STRING_LITERAL_NAME:
full_name = models.STRING_LITERAL_NAME
flags = int(flags_part, 16) if flags_part else 0
num_aliases = int(aliases_part, 16) if aliases_part else 0
# Skip the constructor to avoid default value checks
new_sym = models.Symbol.__new__(models.Symbol)
new_sym.container = cur_container
new_sym.section_name = cur_section_name
new_sym.full_name = full_name
new_sym.address = cur_addresses[i]
new_sym.size = cur_sizes[i]
paths = path_tuples[cur_path_indices[i]]
new_sym.object_path, new_sym.source_path = paths
component = components[
cur_component_indices[i]] if has_components else ''
new_sym.component = component
new_sym.flags = flags
# Derived
if cur_paddings:
new_sym.padding = cur_paddings[i]
new_sym.size += new_sym.padding
else:
# This will be computed during CreateSizeInfo()
new_sym.padding = 0
new_sym.template_name = ''
new_sym.name = ''
if num_aliases:
assert alias_counter == 0
new_sym.aliases = [new_sym]
alias_counter = num_aliases - 1
elif alias_counter > 0:
new_sym.aliases = raw_symbols[symbol_idx - 1].aliases
new_sym.aliases.append(new_sym)
alias_counter -= 1
else:
new_sym.aliases = None
raw_symbols[symbol_idx] = new_sym
symbol_idx += 1
if not has_padding:
CalculatePadding(raw_symbols)
return models.SizeInfo(build_config,
containers,
raw_symbols,
size_path=size_path)