def print_query(query, families=False, pad=1):
if type(query) in (sqlite3.Cursor, list):
longest = ""
results = []
for q in query:
results.append(q)
longest = max((q[1], longest), key=len) # to ensure proper padding
for r in results:
print_query(r, families=families, pad=len(longest))
return
n, p_str, sol = query
family = ""
if families:
_p = Partition(*map(int, p_str.split(",")))
if _p.is_one_dimensional():
family = "[1D]"
if _p.is_hook():
family = "[HOOK]"
elif _p.is_self_conjugate():
family = "[SELF-CONJ]"
print u"{:{pad}} \u22a2 {:2}: solution {} {}".format("(" + p_str + ")",
n,
sol,
family,
pad=pad +
2).encode("utf-8")
def generate_refined_intermediate_sonata_queries(self):
qid_2_queries_refined = {}
refined_sonata_queries = {}
filter_mappings = {}
# First update the Sonata queries for different levels
for (qid, sonata_query) in self.qid_2_query.iteritems():
if qid in self.qid_2_query:
refined_sonata_queries[qid] = {}
refinement_key = self.per_query_refinement_key[qid]
ref_levels = self.ref_levels
for ref_level in ref_levels[1:]:
refined_sonata_queries[qid][ref_level] = {}
refined_query_id = get_refined_query_id(sonata_query, ref_level)
refined_sonata_query = apply_refinement_plan(sonata_query, refinement_key, refined_query_id,
ref_level)
qid_2_queries_refined[refined_query_id] = refined_sonata_query
# Create target-specific partition object for this refined query
partition_object = Partition(refined_sonata_query, self.target, ref_level)
# generate intermediate queries for learning
partition_object.generate_partitioned_queries_learning()
# update intermediate queries and filter mappings
sonata_intermediate_queries = partition_object.intermediate_learning_queries
filter_mappings_tmp = partition_object.filter_mappings
filter_mappings.update(filter_mappings_tmp)
# Update refined sonata queries
for part_qid in sonata_intermediate_queries:
refined_sonata_queries[qid][ref_level][part_qid] = sonata_intermediate_queries[part_qid]
self.refined_sonata_queries = refined_sonata_queries
self.filter_mappings = filter_mappings
self.qid_2_refined_queries = qid_2_queries_refined
def filter(self, f):
r = RDD(self.__new_rdd_id())
r.partitions = []
for p in self.partitions:
rp = Partition(r.id, p.id, filter(lambda x: f(x), p.get_content()))
if rp.get_content():
r.partitions.append(rp)
r.num_partitions = len(r.partitions)
return r
def predict(image_path, sess, sr):
print image_path
seg = Segmentation(image_path)
d = seg.get_labels()
mst = MinimumSpanningTree(d).get_mst()
pa = Partition(mst, seg, sess, sr)
l = pa.getList()
c = Classify()
result = c.classify(l)
img_prediction = ImgPred(basename(image_path), l, result[1])
return img_prediction
def draw_str(self):
"""
Returns the string holding the human readable drawing of the
partitions in the memory map.
"""
table = prettytable.PrettyTable([
"Name", "Start (cyl)", "Size* (cyl)", "Start (b)", "Size (b)",
"Size (mb)", "Bootable", "Type", "Filesystem"
])
table.sortby = "Start (cyl)"
table.align["Name"] = "l" # Left align names
for part in self._partitions:
# Start info - bytes
start_b = part.start * geometry.CYLINDER_BYTE_SIZE
start_b = hexutils.hex_format(start_b)
# Size info - bytes and mbytes
size_b = geometry.FULL_SIZE
size_mb = geometry.FULL_SIZE
if part.size != geometry.FULL_SIZE:
size_b = int(part.size * geometry.CYLINDER_BYTE_SIZE)
size_mb = int(size_b) >> 20
table.add_row([
part.name, part.start, part.size, start_b, size_b, size_mb,
'*' if part.is_bootable else '',
Partition.decode_partition_type(part.type), part.filesystem
])
return table.get_string()
def _getartition(self, part, ready=True):
geom = part.geometry
size = part.getSize()
#print "part.path :%s part.type:%s" % (part.path ,part.type)
#print "###partition.geom:%s partition.size:%s" % (geom.start, size )
if part.number >= 1:
#except free parts number are greater than 0
#print "TYPEEEE:%s" % part
filesystem = ""
#print part.getFlagsAsString()
if part.fileSystem and (part.fileSystem.type is not None):
filesystem = part.fileSystem.type
#FIXME:Check LVM, RAID, Partition assignment
if part.getFlagsAsString()=="lvm":
#print "part.path :%s part.getFlagAsString:%s" % (part.path ,part.getFlagsAsString())
if not filesystem:
filesystem = "physicalVolume"
return PhysicalVolume(self, part, part.number, size, geom.start, geom.end, filesystem, ready)
elif part.getFlagsAsString()=="raid":
#print "part.path :%s part.getFlagAsString:%s" % (part.path ,part.getFlagsAsString())
if not filesystem:
filesystem = "raidMember"
return RaidMember(self, part, part.number, size, geom.start, geom.end, filesystem, ready)
else:
if part.type & parted.PARTITION_EXTENDED:
filesystem = "extended"
#print " partition.name:%s" % part.path
return Partition(self, part, part.number, size, geom.start, geom.end, filesystem, ready)
elif part.type & parted.PARTITION_FREESPACE and size >= 10:
return FreeSpace(self, part, size, geom.start, geom.end)
def _generate_boot_partition(self):
boot = Partition('boot')
if self._config.has_option(
'CONFIG_BSP_ARCH_SD_CARD_INSTALLER_BOOTLOADER_OUT_OF_FS'):
boot.start = 1 # Leave room for the MBR at cylinder 0
else:
boot.start = 0
boot.size = geometry.FULL_SIZE
boot.bootable = True
boot.type = Partition.TYPE_FAT32_LBA
boot.filesystem = Partition.FILESYSTEM_VFAT
boot.components = [boot.COMPONENT_BOOTLOADER]
self._partitions.append(boot)
def __partitioned(self, x, num_partitions=1):
x_len_iter, x = itertools.tee(x, 2)
len_x = sum(1 for _ in x_len_iter)
for i in range(num_partitions):
start = int(i * len_x / num_partitions)
end = int((i + 1) * len_x / num_partitions)
if i + 1 == num_partitions:
end += 1
yield Partition(self.id, i, itertools.islice(x, end - start))
def predict(image_path, sess, sr):
"""
Add your code here
"""
"""
# Don't forget to store your prediction into ImgPred
img_prediction = ImgPred(...)
"""
print image_path
seg = Segmentation(image_path)
d = seg.get_labels()
mst = MinimumSpanningTree(d).get_mst()
pa = Partition(mst, seg, sess, sr)
l = pa.getList()
c = Classify()
result = c.classify(l)
img_prediction = ImgPred(basename(image_path), l, result[1])
return img_prediction
def mapValues(self, f):
r = RDD(self.__new_rdd_id())
r.partitions = []
for p in self.partitions:
rp = Partition(r.id, p.id,
map(lambda x: [x[0], f(x[1])], p.get_content()))
r.partitions.append(rp)
r.num_partitions = len(r.partitions)
return r
def __init__(self, args):
# create logger
self.logger = logging.getLogger(__name__)
self.logger.setLevel(logging.DEBUG)
# create console handler and set level to debug
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
# create formatter
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
# add formatter to ch
ch.setFormatter(formatter)
# add ch to logger
self.logger.addHandler(ch)
self.logger.debug("Starting Collector process in %s" % os.getcwd())
self.logger.debug("Gevent Version %s" % gevent.__version__)
#TODO: move output file name to config
#fname = "./NetFlow.%s.bin"%str(time.time()*100000)
#WARN: might want to remove this after testing
#self.out = open(fname,"wb")
#create tool instances
self.interface = Interface()
self.parse = Parse()
self.describe = Describe()
self.standardize = Standardize()
self.transform = Transform()
self.partition = Partition()
self.q = Queue()
self.inWindow = False
self.score = Score()
#TODO: move csv name to config
self.csv = CSV("output.csv")
return super(Collector, self).__init__(args)
def put_partition_vector(self, partition_vector, name):
"""
Given a partition vector (i.e. a tuple containing the class-
membership for each gene alignment), inserts the relevant data
structures into the SequenceCollection object.
NEXT: run concatenate_records(), put_cluster_trees()
"""
self.clusters_to_partitions[name] = partition_vector
self.partitions[partition_vector] = Partition(partition_vector)
def executeOperation(self, conn, commandClient, commandServer):
partition = Partition()
userFileName = commandServer.processDownload()
hexValue = partition.findHash(userFileName)
print(str(hexValue))
remainder = partition.findShiftedValue(hexValue, userFileName,
partition)
disk1 = Partition.hashDiskMap[remainder]
disk2 = Partition.hashDiskBackupMap[remainder]
print("disk1:" + str(disk1) + " disk2:" + str(disk2))
ipMachine = Partition.ipList[disk1]
print("IpMachine: " + ipMachine)
print("Backup Disk: " + str(disk2))
backupMachine = Partition.ipList[disk2]
print("Backup Machine: " + backupMachine)
ipMachine = ipMachine + " " + backupMachine
conn.send(ipMachine.encode('ascii'))
data = conn.recv(2048)
msg = str(data.decode('ascii'))
print(str(msg))
def list(self):
"""
List the partitions available on the cluster.
:rtype: ``list`` of :class:`.Partition`
:return: A list of ``Partition`` objects.
"""
current_partitions = self._partitions
partitions = []
for key in current_partitions.iterkeys():
log.debug("Adding partition {0} to .list".format(key))
partitions.append(Partition(current_partitions[key]))
return partitions
def _generate_boot_partition(self):
"""
Generates the start and size (in cylinders) of the boot partition.
"""
boot = Partition('boot')
# Config variables
boot_partition_size_mb = \
self._config.get_clean('CONFIG_INSTALLER_UBOOT_PARTITION_SIZE')
if self._config.has_option(
'CONFIG_BSP_ARCH_SD_CARD_INSTALLER_BOOTLOADER_OUT_OF_FS'):
boot.start = 1 # Leave room for the MBR at cylinder 0
else:
boot.start = 0
if boot_partition_size_mb == geometry.FULL_SIZE:
boot.size = geometry.FULL_SIZE
else:
boot_partition_size_b = float(int(boot_partition_size_mb) << 20)
boot_partition_size = math.ceil(boot_partition_size_b /
geometry.CYLINDER_BYTE_SIZE)
boot.size = int(boot_partition_size)
# Boot partition is bootable
boot.bootable = True
# Boot partition is FAT32/VFAT
boot.type = Partition.TYPE_FAT32_LBA
boot.filesystem = Partition.FILESYSTEM_VFAT
# Boot components are: bootloader and kernel.
boot.components = [boot.COMPONENT_BOOTLOADER, boot.COMPONENT_KERNEL]
self._partitions.append(boot)
def free_partitions(self):
"""
Returns a list of the current free space allocations as Partition
instances.
.. note::
If the disk is initialized (no partition table) it
will return None.
"""
partitions = []
part = disk_next_partition(self._ped_disk, None)
while part:
if part.contents.type != 4:
part = disk_next_partition(self._ped_disk, part)
continue
p = Partition(disk=self, part=part)
partitions.append(p)
part = disk_next_partition(self._ped_disk, part)
return partitions
def partitions(self):
"""
Returns a list of the current disk partitions.
.. note::
If the disk is initialized (no partition table) it
will return None, if the disk has a partition table
but no partitions it will return an empty list.
"""
partitions = []
part = disk_next_partition(self._ped_disk, None)
while part:
if part.contents.type > 2:
part = disk_next_partition(self._ped_disk, part)
continue
p = Partition(disk=self, part=part)
partitions.append(p)
part = disk_next_partition(self._ped_disk, part)
return partitions
def get_partition(self, part_num):
"""
Returns a Partition instance.
*Args:*
* part_num (int): A partition number.
*Raises:*
* PartitionError
.. note::
If the disk is initialized (no partition table) it
will raise DiskError.
"""
partition = Partition(disk=self,
part=self._get_ped_partition(part_num))
return partition