def differential_branch_number(self): r""" Return differential branch number of this S-Box. The differential branch number of an S-Box `S` is defined as .. MATH:: \min_{v, w \neq v} \{ \mathrm{wt}(v \oplus w) + \mathrm{wt}(S(v) \oplus S(w)) \} where `\mathrm{wt}(x)` denotes the Hamming weight of vector `x`. EXAMPLES:: sage: S = mq.SBox([12,5,6,11,9,0,10,13,3,14,15,8,4,7,1,2]) sage: S.differential_branch_number() 3 """ m = self.m n = self.n ret = (1<<m) + (1<<n) for a in range(1<<m): for b in range(1<<n): if (a != b): x = a ^ b y = self(a) ^ self(b) w = ZZ(x).popcount() + ZZ(y).popcount() if w < ret: ret = w return ret
def __init__(self, host='127.0.0.1', port=9042, authenticator=None, ssl_options=None, sockopts=None, compression=True, cql_version=None, protocol_version=MAX_SUPPORTED_VERSION, is_control_connection=False, user_type_map=None, connect_timeout=None): self.host = host self.port = port self.authenticator = authenticator self.ssl_options = ssl_options self.sockopts = sockopts self.compression = compression self.cql_version = cql_version self.protocol_version = protocol_version self.is_control_connection = is_control_connection self.user_type_map = user_type_map self.connect_timeout = connect_timeout self._push_watchers = defaultdict(set) self._requests = {} self._iobuf = io.BytesIO() if protocol_version >= 3: self.max_request_id = (2 ** 15) - 1 # Don't fill the deque with 2**15 items right away. Start with 300 and add # more if needed. self.request_ids = deque(range(300)) self.highest_request_id = 299 else: self.max_request_id = (2 ** 7) - 1 self.request_ids = deque(range(self.max_request_id + 1)) self.highest_request_id = self.max_request_id self.lock = RLock() self.connected_event = Event()
def test_task_list(self): INIT = consts.TaskStatus.INIT task_init = sorted(self._create_task()["uuid"] for i in moves.range(3)) FINISHED = consts.TaskStatus.FINISHED task_finished = sorted(self._create_task( {"status": FINISHED, "deployment_uuid": self.deploy["uuid"]} )["uuid"] for i in moves.range(3)) task_all = sorted(task_init + task_finished) def get_uuids(status=None, deployment=None): tasks = db.task_list(status=status, deployment=deployment) return sorted(task["uuid"] for task in tasks) self.assertEqual(task_all, get_uuids(None)) self.assertEqual(task_init, get_uuids(status=INIT)) self.assertEqual(task_finished, get_uuids(status=FINISHED)) self.assertRaises(exceptions.DeploymentNotFound, get_uuids, deployment="non-existing-deployment") deleted_task_uuid = task_finished.pop() db.task_delete(deleted_task_uuid) self.assertEqual(task_init, get_uuids(INIT)) self.assertEqual(sorted(task_finished), get_uuids(FINISHED))
def linear_branch_number(self): r""" Return linear branch number of this S-Box. The linear branch number of an S-Box `S` is defined as .. MATH:: \min_{\substack{\alpha \neq 0, \beta \\ \mathrm{LAM}(\alpha, \beta) \neq 0}} \{ \mathrm{wt}(\alpha) + \mathrm{wt}(\beta) \} where `\mathrm{LAM}(\alpha, \beta)` is the entry at row `\alpha` and column `\beta` of linear approximation matrix correspond to this S-Box. The `\mathrm{wt}(x)` denotes the Hamming weight of `x`. EXAMPLES:: sage: S = mq.SBox([12,5,6,11,9,0,10,13,3,14,15,8,4,7,1,2]) sage: S.linear_branch_number() 2 """ m = self.m n = self.n ret = (1<<m) + (1<<n) lat = self.linear_approximation_matrix() for a in range(1, 1<<m): for b in range(1<<n): if lat[a,b] != 0: w = ZZ(a).popcount() + ZZ(b).popcount() if w < ret: ret = w return ret
def make_domains(lists): """ Given a list of lists, return a list of domain for each list to produce all combinations of possibles values. :rtype: list Example: >>> make_domains(['a', 'b'], ['c','d', 'e']) [['a', 'b', 'a', 'b', 'a', 'b'], ['c', 'c', 'd', 'd', 'e', 'e']] """ from six.moves import range domains = [] for iterable in lists: new_domain = iterable[:] for i in range(len(domains)): domains[i] = domains[i] * len(iterable) if domains: missing = (len(domains[0]) - len(iterable)) / len(iterable) i = 0 for j in range(len(iterable)): value = iterable[j] for dummy in range(missing): new_domain.insert(i, value) i += 1 i += 1 domains.append(new_domain) return domains
def test_evaluate_dofs_manifolds_affine(): "Testing evaluate_dofs vs tabulated coordinates." n = 4 mesh = BoundaryMesh(UnitSquareMesh(n, n), "exterior") mesh2 = BoundaryMesh(UnitCubeMesh(n, n, n), "exterior") DG0 = FunctionSpace(mesh, "DG", 0) DG1 = FunctionSpace(mesh, "DG", 1) CG1 = FunctionSpace(mesh, "CG", 1) CG2 = FunctionSpace(mesh, "CG", 2) DG20 = FunctionSpace(mesh2, "DG", 0) DG21 = FunctionSpace(mesh2, "DG", 1) CG21 = FunctionSpace(mesh2, "CG", 1) CG22 = FunctionSpace(mesh2, "CG", 2) elements = [DG0, DG1, CG1, CG2, DG20, DG21, CG21, CG22] f = Expression("x[0]+x[1]") for V in elements: sdim = V.element().space_dimension() gdim = V.mesh().geometry().dim() coords = numpy.zeros((sdim, gdim), dtype="d") coord = numpy.zeros(gdim, dtype="d") values0 = numpy.zeros(sdim, dtype="d") values1 = numpy.zeros(sdim, dtype="d") for cell in cells(V.mesh()): vx = cell.get_vertex_coordinates() orientation = cell.orientation() V.dofmap().tabulate_coordinates(cell, coords) for i in range(coords.shape[0]): coord[:] = coords[i,:] values0[i] = f(*coord) V.element().evaluate_dofs(values1, f, vx, orientation, cell) for i in range(sdim): assert round(values0[i] - values1[i], 7) == 0
def test_deep_merge_lists_delete_no_conflict(): # local removes an entry b = [[1, 3, 5], [2, 4, 6]] for i in range(len(b)): for j in range(len(b[i])): l = copy.deepcopy(b) r = copy.deepcopy(b) l[i].pop(j) m, lc, rc = merge(b, l, r) assert m == l assert lc == [] assert rc == [] # remote removes an entry b = [[1, 3, 5], [2, 4, 6]] for i in range(len(b)): for j in range(len(b[i])): l = copy.deepcopy(b) r = copy.deepcopy(b) r[i].pop(j) m, lc, rc = merge(b, l, r) assert m == r assert lc == [] assert rc == [] # both remove the same entry and one each b = [[1, 3, 5], [2, 4, 6]] l = [[1, 5], [2, 4]] # deletes 3 and 6 r = [[1, 5], [4, 6]] # deletes 3 and 2 m, lc, rc = merge(b, l, r) assert m == [[1, 5], [2, 4], [1, 5], [4, 6]] # This is expected behaviour today: clear b, add l, add r #assert m == [[1, 5], [4]] # 2,3,6 should be gone. TODO: This is the behaviour we want. assert lc == [] assert rc == []
def filtering_join(*args, **kwargs): left = args[0] right = args[1] if 'by' in kwargs: left_cols, right_cols = get_join_cols(kwargs['by']) cols = lambda right, left: right_cols else: left_cols, right_cols = None, None cols = lambda right, left: [x for x in left.columns.values.tolist() if x in right.columns.values.tolist()] if left._grouped_on: outDf = DplyFrame((left >> ungroup()) .merge(right[cols(left, right)].drop_duplicates(), how=kwargs['how'], left_on=left_cols, right_on=right_cols, indicator=True, suffixes=('', '_y')) .query(kwargs['query']) .regroup(left._grouped_on) .iloc[:, range(0, len(left.columns))]) else: outDf = DplyFrame(left.merge(right[cols(left, right)] .drop_duplicates(), how=kwargs['how'], left_on=left_cols, right_on=right_cols, indicator=True, suffixes=('', '_y')) .query(kwargs['query']) .iloc[:, range(0, len(left.columns))]) return outDf
def fit(self, X, augment=False, rounds=1, seed=None): '''Required for featurewise_center, featurewise_std_normalization and zca_whitening. # Arguments X: Numpy array, the data to fit on. augment: whether to fit on randomly augmented samples rounds: if `augment`, how many augmentation passes to do over the data seed: random seed. ''' X = np.copy(X) if augment: aX = np.zeros(tuple([rounds * X.shape[0]] + list(X.shape)[1:])) for r in range(rounds): for i in range(X.shape[0]): aX[i + r * X.shape[0]] = self.random_transform(X[i]) X = aX if self.featurewise_center: self.mean = np.mean(X, axis=0) X -= self.mean if self.featurewise_std_normalization: self.std = np.std(X, axis=0) X /= (self.std + 1e-7) if self.zca_whitening: flatX = np.reshape(X, (X.shape[0], X.shape[1] * X.shape[2] * X.shape[3])) sigma = np.dot(flatX.T, flatX) / flatX.shape[1] U, S, V = linalg.svd(sigma) self.principal_components = np.dot(np.dot(U, np.diag(1. / np.sqrt(S + 10e-7))), U.T)
def test_randint(self): random = RandomGenerator() lower = 0 upper = 10 for _ in range(10000): assert random.randint(lower, upper) >= lower assert random.randint(lower, upper) <= upper lower = -10 upper = 100 for _ in range(10000): assert random.randint(lower, upper) >= lower assert random.randint(lower, upper) <= upper lower = 5 upper = 21 for _ in range(10000): assert random.randint(lower, upper) >= lower assert random.randint(lower, upper) <= upper lower = -5 upper = 5 for _ in range(10000): assert random.randint(lower, upper) >= lower assert random.randint(lower, upper) <= upper
def test_add_variable(self, core_model): cache = ProblemCache(core_model) def add_var(model, var_id): return model.solver.interface.Variable(var_id, ub=0) def update_var(model, var): return setattr(var, "ub", 1000) for i in range(10): cache.add_variable("%i" % i, add_var, update_var) for i in range(10): assert cache.variables["%i" % i] in core_model.solver.variables assert cache.variables["%i" % i].ub == 0 assert core_model.solver.variables["%i" % i].ub == 0 for i in range(10): cache.add_variable("%i" % i, add_var, update_var) assert cache.variables["%i" % i].ub == 1000 assert core_model.solver.variables["%i" % i].ub == 1000 cache.reset() for i in range(10): with pytest.raises(KeyError): core_model.solver.variables.__getitem__("%i" % i)
def eval_epoch(Xs, Ys, y, sess, stream, cw): """ Evaluate the model against a dataset, and return the PSNR. Args: Xs: example placeholders list Ys: label placeholders list y: model output tensor sess: session stream: DataStream for the dataset cw: crop border Returns: psnr: PSNR of model's inference on dataset """ se = 0. for X_c, y_c in stream.get_epoch_iterator(): y_c = y_c[:, cw:-cw, cw:-cw] chunk_size = X_c.shape[0] gpu_chunk = chunk_size // FLAGS.num_gpus dict_input1 = [(Xs[i], X_c[i*gpu_chunk : \ ((i + 1)*gpu_chunk) \ if (i != FLAGS.num_gpus - 1) \ else chunk_size]) \ for i in range(FLAGS.num_gpus)] dict_input2 = [(Ys[i], y_c[i*gpu_chunk : \ ((i + 1)*gpu_chunk) \ if (i != FLAGS.num_gpus - 1) \ else chunk_size]) \ for i in range(FLAGS.num_gpus)] feed = dict(dict_input1 + dict_input2) y_eval = sess.run(y, feed_dict=feed) se += np.sum((y_eval - y_c) ** 2.0) rmse = np.sqrt(se / (stream.dataset.num_examples * y_c.shape[1] * y_c.shape[2])) psnr = 20 * np.log10(1.0 / rmse) return psnr
def test_add_constraint(self, core_model): cache = ProblemCache(core_model) def add_var(model, var_id): return model.solver.interface.Variable(var_id, ub=0) def add_constraint(m, const_id, var): return m.solver.interface.Constraint(var, lb=-10, ub=10, name=const_id) def update_constraint(model, const, var): return setattr(const, "ub", 1000) for i in range(10): cache.add_variable("%i" % i, add_var, None) cache.add_constraint("c%i" % i, add_constraint, update_constraint, cache.variables["%i" % i]) for i in range(10): assert cache.constraints["c%i" % i] in core_model.solver.constraints assert cache.constraints["c%i" % i].ub == 10 assert cache.constraints["c%i" % i].lb == -10 assert core_model.solver.constraints["c%i" % i].ub == 10 assert core_model.solver.constraints["c%i" % i].lb == -10 for i in range(10): cache.add_constraint("c%i" % i, add_constraint, update_constraint, cache.variables["%i" % i]) assert core_model.solver.constraints["c%i" % i].ub == 1000 cache.reset() for i in range(10): with pytest.raises(KeyError): core_model.solver.variables.__getitem__("%i" % i) with pytest.raises(KeyError): core_model.solver.constraints.__getitem__("c%i" % i)
def setUp(self): """Setup for Identity Limit Test Cases.""" super(IdentityTestListLimitCase, self).setUp() # Create 10 entries for each of the entities we are going to test self.ENTITY_TYPES = ['user', 'group', 'project'] self.entity_lists = {} for entity in self.ENTITY_TYPES: self.entity_lists[entity] = self._create_test_data(entity, 10) # Make sure we clean up when finished self.addCleanup(self.clean_up_entity, entity) self.service_list = [] self.addCleanup(self.clean_up_service) for _ in range(10): new_entity = unit.new_service_ref() service = self.catalog_api.create_service(new_entity['id'], new_entity) self.service_list.append(service) self.policy_list = [] self.addCleanup(self.clean_up_policy) for _ in range(10): new_entity = unit.new_policy_ref() policy = self.policy_api.create_policy(new_entity['id'], new_entity) self.policy_list.append(policy)
def test_cloud_cover(self): iplt.symbols(list(range(10)), [0] * 10, [iris.symbols.CLOUD_COVER[i] for i in range(10)], 0.375) iplt.plt.axis('off') self.check_graphic()
def get_put_kernel(context, dtype, idx_dtype, vec_count=1): ctx = { "idx_tp": dtype_to_ctype(idx_dtype), "tp": dtype_to_ctype(dtype), } args = [ VectorArg(dtype, "dest%d" % i, with_offset=True) for i in range(vec_count) ] + [ VectorArg(idx_dtype, "gmem_dest_idx", with_offset=True), ] + [ VectorArg(dtype, "src%d" % i, with_offset=True) for i in range(vec_count) ] + [ VectorArg(np.uint8, "use_fill", with_offset=True) ] + [ VectorArg(np.int64, "val_ary_lengths", with_offset=True) ] body = ( "%(idx_tp)s dest_idx = gmem_dest_idx[i];\n" % ctx + "\n".join( "dest{i}[dest_idx] = (use_fill[{i}] ? src{i}[0] : " "src{i}[i % val_ary_lengths[{i}]]);".format(i=i) for i in range(vec_count) ) ) return get_elwise_kernel(context, args, body, preamble=dtype_to_c_struct(context.devices[0], dtype), name="put")
def stl_to_plot3d_filename(stl_filename, p3d_filename, log=None, ascii=True): model = STL(log=log) model.read_stl(stl_filename) # nodal_normals = model.get_normals_at_nodes(model.elements) with open(p3d_filename, "wb") as p3d: nblocks = len(model.elements) # nblocks = 10 p3d.write("%i\n" % nblocks) for iblock in range(nblocks): p3d.write("2 2 1\n") nodes = model.nodes elements = model.elements if 0: for i in [0, 1, 2]: for iblock in range(nblocks): (n1, n2, n3) = elements[iblock] p1 = nodes[n1, :] p2 = nodes[n2, :] p3 = nodes[n3, :] p4 = p3 xi = [[p1[i], p2[i], p3[i], p4[i]]] savetxt(p3d, xi, fmt="%f") else: for iblock in range(nblocks): for i in [0, 1, 2]: (n1, n2, n3) = elements[iblock] p1 = nodes[n1, :] p2 = nodes[n2, :] p3 = nodes[n3, :] p4 = p3 xi = [[p1[i], p2[i], p3[i], p4[i]]] savetxt(p3d, xi, fmt="%f")
def write_as_plot3d(self, f): X = [] Y = [] Z = [] for i in range(self.nRows): #points2 = [] for j in range(self.nCols): (x, y, z) = self.Points[i][j] X.append(x) Y.append(y) Z.append(z) msg = '' for x in X: msg += '%s ' % (x) f.write(msg + '\n') msg = '' for y in Y: msg += '%s ' % (y) f.write(msg + '\n') msg = '' for z in Z: msg += '%s ' % (z) f.write(msg + '\n')
def evaluate_territory(board): """Map a board into territory and dame. Any points that are completely surrounded by a single color are counted as territory; it makes no attempt to identify even trivially dead groups. """ status = {} for r, c in itertools.product(list(range(board.board_size)), list(range(board.board_size))): if (r, c) in status: # Already visited this as part of a different group. continue if (r, c) in board.board: # It's a stone. status[r, c] = board.board[r, c] else: group, neighbors = _collect_region((r, c), board) if len(neighbors) == 1: # Completely surrounded by black or white. fill_with = 'territory_' + neighbors.pop() else: # Dame. fill_with = 'dame' for pos in group: status[pos] = fill_with return Territory(status)
def coproduct_on_basis(self, A): r""" Return the coproduct of a `\mathbf{w}` basis element. The coproduct on the basis element `\mathbf{w}_A` is the sum over tensor product terms `\mathbf{w}_B \otimes \mathbf{w}_C` where `B` is the restriction of `A` to `\{1,2,\ldots,k\}` and `C` is the restriction of `A` to `\{k+1, k+2, \ldots, n\}`. INPUT: - ``A`` -- a set partition OUTPUT: - The coproduct applied to the `\mathbf{w}` dual symmetric function in non-commuting variables indexed by ``A`` expressed in the `\mathbf{w}` basis. EXAMPLES:: sage: w = SymmetricFunctionsNonCommutingVariables(QQ).dual().w() sage: w[[1], [2,3]].coproduct() w{} # w{{1}, {2, 3}} + w{{1}} # w{{1, 2}} + w{{1}, {2}} # w{{1}} + w{{1}, {2, 3}} # w{} sage: w.coproduct_on_basis(SetPartition([])) w{} # w{} """ n = A.size() return self.tensor_square().sum_of_terms([ (( A.restriction(range(1, i+1)).standardization(), A.restriction(range(i+1, n+1)).standardization() ), 1) for i in range(n+1)], distinct=True)
def get_points(self): Points = [] for i in range(self.nRows): #points2 = [] for j in range(self.nCols): Points.append(self.Points[i][j]) return Points, len(Points)
def left_table(self): """ Return the list of matrices for left multiplication by the basis elements. EXAMPLES:: sage: B = FiniteDimensionalAlgebra(QQ, [Matrix([[1,0], [0,1]]), Matrix([[0,1],[-1,0]])]) sage: T = B.left_table(); T ( [1 0] [ 0 1] [0 1], [-1 0] ) We check immutability:: sage: T[0] = "vandalized by h4xx0r" Traceback (most recent call last): ... TypeError: 'tuple' object does not support item assignment sage: T[1][0] = [13, 37] Traceback (most recent call last): ... ValueError: matrix is immutable; please change a copy instead (i.e., use copy(M) to change a copy of M). """ B = self.table() n = self.degree() table = [Matrix([B[j][i] for j in range(n)]) for i in range(n)] for b in table: b.set_immutable() return tuple(table)
def fixed_ips_fake(*args, **kwargs): global fixed_ips ips = [next_fixed_ip(i, floating_ips_per_fixed_ip) for i in range(1, num_networks + 1) for j in range(ips_per_vif)] fixed_ips = ips return ips
def parsesection_mapper(self, numlines, mapper): """Parses FORTRAN formatted section, and returns a list of all entries in each line Parameters ---------- numlines : int The number of lines to be parsed in this section mapper : lambda operator Operator to format entries in current section Returns ------- section : list A list of all entries in a given parm7 section """ section = [] y = next(self.topfile).strip("%FORMAT(") y.strip(")") x = FORTRANReader(y) for i in range(numlines): l = next(self.topfile) for j in range(len(x.entries)): val = l[x.entries[j].start:x.entries[j].stop].strip() if val: section.append(mapper(val)) return section
def test_auto_cohorting_randomization(self): """ Make sure cohorts.get_cohort() randomizes properly. """ course = modulestore().get_course(self.toy_course_key) self.assertFalse(cohorts.is_course_cohorted(course.id)) groups = ["group_{0}".format(n) for n in range(5)] config_course_cohorts( course, is_cohorted=True, auto_cohorts=groups ) # Assign 100 users to cohorts for i in range(100): user = UserFactory( username="******".format(i), email="a@b{0}.com".format(i) ) cohorts.get_cohort(user, course.id) # Now make sure that the assignment was at least vaguely random: # each cohort should have at least 1, and fewer than 50 students. # (with 5 groups, probability of 0 users in any group is about # .8**100= 2.0e-10) for cohort_name in groups: cohort = cohorts.get_cohort_by_name(course.id, cohort_name) num_users = cohort.users.count() self.assertGreater(num_users, 1) self.assertLess(num_users, 50)
def test_multiple_connections(self): """ Test multiple connections with pipelined requests. """ conns = [self.get_connection() for i in range(5)] events = [Event() for i in range(5)] query = "SELECT keyspace_name FROM system.schema_keyspaces LIMIT 1" def cb(event, conn, count, *args, **kwargs): count += 1 if count >= 10: conn.close() event.set() else: conn.send_msg( QueryMessage(query=query, consistency_level=ConsistencyLevel.ONE), request_id=count, cb=partial(cb, event, conn, count)) for event, conn in zip(events, conns): conn.send_msg( QueryMessage(query=query, consistency_level=ConsistencyLevel.ONE), request_id=0, cb=partial(cb, event, conn, 0)) for event in events: event.wait()
def __init__(self, card=None, data=None, comment=''): Constraint.__init__(self, card, data) if comment: self._comment = comment self.IDs = [] ## TODO: IDs reference nodes??? self.Cs = [] if card: # TODO: remove fields... fields = card.fields(1) nfields = len(card) assert len(card) > 1, card nterms = int(nfields / 2.) n = 1 for i in range(nterms): nstart = 1 + 2 * i ID = integer(card, nstart, 'ID%s' % n) C = components_or_blank(card, nstart + 1, 'component%s' % n, '0') self.IDs.append(ID) self.Cs.append(C) n += 1 else: fields = data for i in range(0, len(fields), 2): self.IDs.append(fields[i]) self.Cs.append(fields[i + 1]) assert len(self.IDs) > 0 assert len(self.IDs) == len(self.Cs)
def get_refined_face(a, b): if a > b: a, b = b, a flipped = True else: flipped = False try: face_points = face_point_dict[a, b] except KeyError: a_pt, b_pt = [points[idx] for idx in [a, b]] dx = (b_pt - a_pt)/factor # build subdivided facet face_points = [a] for i in range(1, points_per_edge-1): face_points.append(len(new_points)) new_points.append(a_pt + dx*i) face_points.append(b) face_point_dict[a, b] = face_points # build old_face_to_new_faces old_face_to_new_faces[frozenset([a, b])] = [ (face_points[i], face_points[i+1]) for i in range(factor)] if flipped: return face_points[::-1] else: return face_points
def linear_structures(self): r""" Return a list of 3-valued tuple `(b, \alpha, c)` such that `\alpha` is a `c`-linear structure of the component function `b \cdot S(x)`. A Boolean function `f : \GF{2}^m \mapsto \GF{2}` is said to have a `c`-linear structure if there exists a nonzero `\alpha` such that `f(x) \oplus f(x \oplus \alpha)` is a constant function `c`. An `m \times n` S-Box `S` has a linear structure if there exists a component function `b \cdot S(x)` that has a linear structure. The three valued tuple `(b, \alpha, c)` shows that `\alpha` is a `c`-linear structure of the component function `b \cdot S(x)`. This implies that for all output differences `\beta` of the S-Box correspond to input difference `\alpha`, we have `b \cdot \beta = c`. EXAMPLES:: sage: S = mq.SBox([0,1,3,6,7,4,5,2]) sage: S.linear_structures() [(1, 1, 1), (2, 2, 1), (3, 3, 1), (4, 4, 1), (5, 5, 1), (6, 6, 1), (7, 7, 1)] """ n = self.n m = self.m act = self.autocorrelation_matrix() ret = [] for j in range(1, 1<<n): for i in range(1, 1<<m): if (abs(act[i,j]) == (1<<m)): c = ((1 - (act[i][j] >> self.m)) >> 1) ret.append((j, i, c)) return ret
def is_associative(self): """ Return ``True`` if ``self`` is associative. EXAMPLES:: sage: A = FiniteDimensionalAlgebra(QQ, [Matrix([[1,0], [0,1]]), Matrix([[0,1],[-1,0]])]) sage: A.is_associative() True sage: B = FiniteDimensionalAlgebra(QQ, [Matrix([[1,0,0], [0,1,0], [0,0,1]]), Matrix([[0,1,0], [0,0,0], [0,0,0]]), Matrix([[0,0,1], [0,0,0], [1,0,0]])]) sage: B.is_associative() False sage: e = B.basis() sage: (e[1]*e[2])*e[2]==e[1]*(e[2]*e[2]) False """ B = self.table() n = self.degree() for i in range(n): for j in range(n): eiej = B[j][i] if B[i]*B[j] != sum(eiej[k] * B[k] for k in range(n)): return False return True
def sample(self, split_dir, num_samples=25, draw_bbox=False): from PIL import Image, ImageDraw, ImageFont import cPickle as pickle import torchvision import torchvision.utils as vutils if cfg.TRAIN.NET_G == '': print('Error: the path for model NET_G is not found!') else: if split_dir == 'test': split_dir = 'valid' # Build and load the generator text_encoder = RNN_ENCODER(self.n_words, nhidden=cfg.TEXT.EMBEDDING_DIM) state_dict = \ torch.load(cfg.TRAIN.NET_E, map_location=lambda storage, loc: storage) text_encoder.load_state_dict(state_dict) print('Load text encoder from:', cfg.TRAIN.NET_E) text_encoder = text_encoder.cuda() text_encoder.eval() batch_size = cfg.TRAIN.BATCH_SIZE nz = cfg.GAN.Z_DIM model_dir = cfg.TRAIN.NET_G state_dict = torch.load(model_dir, map_location=lambda storage, loc: storage) # state_dict = torch.load(cfg.TRAIN.NET_G) netG = G_NET() print('Load G from: ', model_dir) netG.apply(weights_init) netG.load_state_dict(state_dict["netG"]) netG.cuda() netG.eval() # the path to save generated images s_tmp = model_dir[:model_dir.rfind('.pth')] save_dir = '%s_%s' % (s_tmp, split_dir) mkdir_p(save_dir) ####################################### noise = Variable(torch.FloatTensor(9, nz)) imsize = 256 for step, data in enumerate(self.data_loader, 0): if step >= num_samples: break imgs, captions, cap_lens, class_ids, keys, transformation_matrices, label_one_hot, bbox = \ self.prepare_data(data, eval=True) transf_matrices_inv = transformation_matrices[1][0].unsqueeze( 0) label_one_hot = label_one_hot[0].unsqueeze(0) img = imgs[-1][0] val_image = img.view(1, 3, imsize, imsize) hidden = text_encoder.init_hidden(batch_size) # words_embs: batch_size x nef x seq_len # sent_emb: batch_size x nef words_embs, sent_emb = text_encoder(captions, cap_lens, hidden) words_embs, sent_emb = words_embs[0].unsqueeze( 0).detach(), sent_emb[0].unsqueeze(0).detach() words_embs = words_embs.repeat(9, 1, 1) sent_emb = sent_emb.repeat(9, 1) mask = (captions == 0) mask = mask[0].unsqueeze(0) num_words = words_embs.size(2) if mask.size(1) > num_words: mask = mask[:, :num_words] mask = mask.repeat(9, 1) transf_matrices_inv = transf_matrices_inv.repeat(9, 1, 1, 1) label_one_hot = label_one_hot.repeat(9, 1, 1) ####################################################### # (2) Generate fake images ###################################################### noise.data.normal_(0, 1) inputs = (noise, sent_emb, words_embs, mask, transf_matrices_inv, label_one_hot) with torch.no_grad(): fake_imgs, _, mu, logvar = nn.parallel.data_parallel( netG, inputs, self.gpus) data_img = torch.FloatTensor(10, 3, imsize, imsize).fill_(0) data_img[0] = val_image data_img[1:10] = fake_imgs[-1] if draw_bbox: for idx in range(3): x, y, w, h = tuple( [int(imsize * x) for x in bbox[0, idx]]) w = imsize - 1 if w > imsize - 1 else w h = imsize - 1 if h > imsize - 1 else h if x <= -1: break data_img[:10, :, y, x:x + w] = 1 data_img[:10, :, y:y + h, x] = 1 data_img[:10, :, y + h, x:x + w] = 1 data_img[:10, :, y:y + h, x + w] = 1 # get caption cap = captions[0].data.cpu().numpy() sentence = "" for j in range(len(cap)): if cap[j] == 0: break word = self.ixtoword[cap[j]].encode( 'ascii', 'ignore').decode('ascii') sentence += word + " " sentence = sentence[:-1] vutils.save_image(data_img, '{}/{}_{}.png'.format( save_dir, sentence, step), normalize=True, nrow=10) print("Saved {} files to {}".format(step, save_dir))
0.5, staircase=True) optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize( loss, global_step=global_step) # Predictions for the training, validation, and test data. train_prediction = tf.nn.softmax(logits) valid_prediction = tf.nn.softmax(model(tf_valid_dataset)) test_prediction = tf.nn.softmax(model(tf_test_dataset)) num_steps = 3001 with tf.Session(graph=graph) as session: tf.initialize_all_variables().run() print("Initialized") for step in range(num_steps): # Pick an offset within the training data, which has been randomized. # Note: we could use better randomization across epochs. offset = (step * batch_size) % (train_labels.shape[0] - batch_size) # Generate a minibatch. batch_data = train_dataset[offset:(offset + batch_size), :] batch_labels = train_labels[offset:(offset + batch_size), :] # Prepare a dictionary telling the session where to feed the minibatch. # The key of the dictionary is the placeholder node of the graph to be fed, # and the value is the numpy array to feed to it. feed_dict = { tf_train_dataset: batch_data, tf_train_labels: batch_labels } _, l, predictions = session.run([optimizer, loss, train_prediction], feed_dict=feed_dict)
def populate_job_func(): self._populate_job_queue.get() for _ in range(self.update_frequency): self._populate_exp()
def track_periodic_data(): """ Sync data that is neither event or page based with hubspot/Kissmetrics :return: """ # Start by getting a list of web users mapped to their domains three_months_ago = date.today() - timedelta(days=90) user_query = (UserES() .web_users() .last_logged_in(gte=three_months_ago) .sort('date_joined', desc=True) .source(['domains', 'email', 'date_joined']) .analytics_enabled()) total_users = user_query.count() chunk_size = 100 num_chunks = int(math.ceil(float(total_users) / float(chunk_size))) # Track no of users and domains with max_forms greater than HUBSPOT_THRESHOLD hubspot_number_of_users = 0 hubspot_number_of_domains_with_forms_gt_threshold = 0 for chunk in range(num_chunks): users_to_domains = (user_query .size(chunk_size) .start(chunk * chunk_size) .run() .hits) # users_to_domains is a list of dicts domains_to_forms = (FormES() .terms_aggregation('domain', 'domain') .size(0) .run() .aggregations.domain.counts_by_bucket()) domains_to_mobile_users = (UserES() .mobile_users() .terms_aggregation('domain', 'domain') .size(0) .run() .aggregations .domain .counts_by_bucket()) # Keep track of india and www data seperately env = get_instance_string() for num_forms in domains_to_forms.values(): if num_forms > HUBSPOT_THRESHOLD: hubspot_number_of_domains_with_forms_gt_threshold += 1 # For each web user, iterate through their domains and select the max number of form submissions and # max number of mobile workers submit = [] for user in users_to_domains: email = user.get('email') if not _email_is_valid(email): continue hubspot_number_of_users += 1 date_created = user.get('date_joined') max_forms = 0 max_workers = 0 max_export = 0 max_report = 0 for domain in user['domains']: if domain in domains_to_forms and domains_to_forms[domain] > max_forms: max_forms = domains_to_forms[domain] if domain in domains_to_mobile_users and domains_to_mobile_users[domain] > max_workers: max_workers = domains_to_mobile_users[domain] if _get_export_count(domain) > max_export: max_export = _get_export_count(domain) if _get_report_count(domain) > max_report: max_report = _get_report_count(domain) project_spaces_created = ", ".join(get_domains_created_by_user(email)) user_json = { 'email': email, 'properties': [ { 'property': '{}max_form_submissions_in_a_domain'.format(env), 'value': max_forms }, { 'property': '{}max_mobile_workers_in_a_domain'.format(env), 'value': max_workers }, { 'property': '{}project_spaces_created_by_user'.format(env), 'value': project_spaces_created, }, { 'property': '{}over_300_form_submissions'.format(env), 'value': max_forms > HUBSPOT_THRESHOLD }, { 'property': '{}date_created'.format(env), 'value': date_created }, { 'property': '{}max_exports_in_a_domain'.format(env), 'value': max_export }, { 'property': '{}max_custom_reports_in_a_domain'.format(env), 'value': max_report } ] } submit.append(user_json) submit_json = json.dumps(submit) submit_data_to_hub_and_kiss(submit_json) update_datadog_metrics({ DATADOG_WEB_USERS_GAUGE: hubspot_number_of_users, DATADOG_DOMAINS_EXCEEDING_FORMS_GAUGE: hubspot_number_of_domains_with_forms_gt_threshold })
def refine_expanding(params, merged_scope, combine_phil): assert params.start_at_hierarchy_level == 0 if params.rmsd_filter.enable: input_name = "filtered" command = "cctbx.xfel.filter_experiments_by_rmsd %s %s output.filtered_experiments=%s output.filtered_reflections=%s" command = command % ( "%s_combined.expt" % params.tag, "%s_combined.refl" % params.tag, "%s_filtered.expt" % params.tag, "%s_filtered.refl" % params.tag) command += " iqr_multiplier=%f" % params.rmsd_filter.iqr_multiplier print(command) result = easy_run.fully_buffered(command=command).raise_if_errors() result.show_stdout() else: input_name = "combined" # -------------------------- if params.panel_filter is not None: from libtbx import easy_pickle print("Filtering out all reflections except those on panels %s" % (", ".join(["%d" % p for p in params.panel_filter]))) combined_path = "%s_combined.refl" % params.tag data = easy_pickle.load(combined_path) sel = None for panel_id in params.panel_filter: if sel is None: sel = data['panel'] == panel_id else: sel |= data['panel'] == panel_id print("Retaining", len(data.select(sel)), "out of", len(data), "reflections") easy_pickle.dump(combined_path, data.select(sel)) # ---------------------------------- # this is the order to refine the CSPAD in steps = {} steps[0] = [2, 3] steps[1] = steps[0] + [0, 1] steps[2] = steps[1] + [14, 15] steps[3] = steps[2] + [6, 7] steps[4] = steps[3] + [4, 5] steps[5] = steps[4] + [12, 13] steps[6] = steps[5] + [8, 9] steps[7] = steps[6] + [10, 11] for s, panels in six.iteritems(steps): rest = [] for p in panels: rest.append(p + 16) rest.append(p + 32) rest.append(p + 48) panels.extend(rest) levels = {0: (0, 1)} # levels 0 and 1 for i in range(7): levels[i + 1] = (2, ) # level 2 previous_step_and_level = None for j in range(8): from libtbx import easy_pickle print("Filtering out all reflections except those on panels %s" % (", ".join(["%d" % p for p in steps[j]]))) combined_path = "%s_%s.refl" % (params.tag, input_name) output_path = "%s_step%d.refl" % (params.tag, j) data = easy_pickle.load(combined_path) sel = None for panel_id in steps[j]: if sel is None: sel = data['panel'] == panel_id else: sel |= data['panel'] == panel_id print("Retaining", len(data.select(sel)), "out of", len(data), "reflections") easy_pickle.dump(output_path, data.select(sel)) for i in levels[j]: print("Step", j, "refining at hierarchy level", i) refine_phil_file = "%s_refine_step%d_level%d.phil" % (params.tag, j, i) if i == 0: if params.refine_distance: diff_phil = "refinement.parameterisation.detector.fix_list=Tau1" # fix detector rotz else: diff_phil = "refinement.parameterisation.detector.fix_list=Dist,Tau1" # fix detector rotz, distance if params.flat_refinement: diff_phil += ",Tau2,Tau3" # Also fix x and y rotations diff_phil += "\n" if params.refine_energy: diff_phil += "refinement.parameterisation.beam.fix=in_spindle_plane+out_spindle_plane\n" # allow energy to refine else: # Note, always need to fix something, so pick a panel group and fix its Tau1 (rotation around Z) always if params.flat_refinement and params.flat_refinement_with_distance: diff_phil = "refinement.parameterisation.detector.fix_list=Group1Tau1,Tau2,Tau3\n" # refine distance, rotz and xy translation diff_phil += "refinement.parameterisation.detector.constraints.parameter=Dist\n" # constrain distance to be refined identically for all panels at this hierarchy level elif params.flat_refinement: diff_phil = "refinement.parameterisation.detector.fix_list=Dist,Group1Tau1,Tau2,Tau3\n" # refine only rotz and xy translation else: diff_phil = "refinement.parameterisation.detector.fix_list=Group1Tau1\n" # refine almost everything if previous_step_and_level is None: command = "dials.refine %s %s_%s.expt %s_step%d.refl"%( \ refine_phil_file, params.tag, input_name, params.tag, j) else: p_step, p_level = previous_step_and_level if p_step == j: command = "dials.refine %s %s_refined_step%d_level%d.expt %s_refined_step%d_level%d.refl"%( \ refine_phil_file, params.tag, p_step, p_level, params.tag, p_step, p_level) else: command = "dials.refine %s %s_refined_step%d_level%d.expt %s_step%d.refl"%( \ refine_phil_file, params.tag, p_step, p_level, params.tag, j) diff_phil += "refinement.parameterisation.detector.hierarchy_level=%d\n" % i output_experiments = "%s_refined_step%d_level%d.expt" % ( params.tag, j, i) command += " output.experiments=%s output.reflections=%s_refined_step%d_level%d.refl"%( \ output_experiments, params.tag, j, i) scope = merged_scope.fetch(parse(diff_phil)) f = open(refine_phil_file, 'w') f.write(refine_scope.fetch_diff(scope).as_str()) f.close() print(command) result = easy_run.fully_buffered(command=command).raise_if_errors() result.show_stdout() # In expanding mode, if using flat refinement with distance, after having refined this step as a block, unrefined # panels will have been left behind. Read back the new metrology, compute the shift applied to the panels refined # in this step,and apply that shift to the unrefined panels in this step if params.flat_refinement and params.flat_refinement_with_distance and i > 0: from dxtbx.model.experiment_list import ExperimentListFactory, ExperimentListDumper from xfel.command_line.cspad_detector_congruence import iterate_detector_at_level, iterate_panels from scitbx.array_family import flex from scitbx.matrix import col from libtbx.test_utils import approx_equal experiments = ExperimentListFactory.from_json_file( output_experiments, check_format=False) assert len(experiments.detectors()) == 1 detector = experiments.detectors()[0] # Displacements: deltas along the vector normal to the detector displacements = flex.double() # Iterate through the panel groups at this level for panel_group in iterate_detector_at_level( detector.hierarchy(), 0, i): # Were there panels refined in this step in this panel group? if params.panel_filter: test = [ list(detector).index(panel) in steps[j] for panel in iterate_panels(panel_group) if list( detector).index(panel) in params.panel_filter ] else: test = [ list(detector).index(panel) in steps[j] for panel in iterate_panels(panel_group) ] if not any(test): continue # Compute the translation along the normal of this panel group. This is defined as distance in dials.refine displacements.append( col(panel_group.get_local_fast_axis()).cross( col(panel_group.get_local_slow_axis())).dot( col(panel_group.get_local_origin()))) # Even though the panels are constrained to move the same amount, there is a bit a variation. stats = flex.mean_and_variance(displacements) displacement = stats.mean() print("Average displacement along normals: %f +/- %f" % (stats.mean(), stats.unweighted_sample_standard_deviation())) # Verify the variation isn't significant for k in range(1, len(displacements)): assert approx_equal(displacements[0], displacements[k]) # If all of the panel groups in this level moved, no need to do anything. if len(displacements) != len( list( iterate_detector_at_level(detector.hierarchy(), 0, i))): for panel_group in iterate_detector_at_level( detector.hierarchy(), 0, i): if params.panel_filter: test = [ list(detector).index(panel) in steps[j] and list(detector).index(panel) in params.panel_filter for panel in iterate_panels(panel_group) ] else: test = [ list(detector).index(panel) in steps[j] for panel in iterate_panels(panel_group) ] # If any of the panels in this panel group moved, no need to do anything if any(test): continue # None of the panels in this panel group moved in this step, so need to apply displacement from other panel # groups at this level fast = col(panel_group.get_local_fast_axis()) slow = col(panel_group.get_local_slow_axis()) ori = col(panel_group.get_local_origin()) normal = fast.cross(slow) panel_group.set_local_frame( fast, slow, (ori.dot(fast) * fast) + (ori.dot(slow) * slow) + (normal * displacement)) # Check the new displacements. Should be the same across all panels. displacements = [] for panel_group in iterate_detector_at_level( detector.hierarchy(), 0, i): displacements.append( col(panel_group.get_local_fast_axis()).cross( col(panel_group.get_local_slow_axis())).dot( col(panel_group.get_local_origin()))) for k in range(1, len(displacements)): assert approx_equal(displacements[0], displacements[k]) dump = ExperimentListDumper(experiments) dump.as_json(output_experiments) previous_step_and_level = j, i output_geometry(params)
def refine_hierarchical(params, merged_scope, combine_phil): if params.panel_filter is not None: from libtbx import easy_pickle print("Filtering out all reflections except those on panels %s" % (", ".join(["%d" % p for p in params.panel_filter]))) combined_path = "%s_combined.refl" % params.tag data = easy_pickle.load(combined_path) sel = None for panel_id in params.panel_filter: if sel is None: sel = data['panel'] == panel_id else: sel |= data['panel'] == panel_id print("Retaining", len(data.select(sel)), "out of", len(data), "reflections") easy_pickle.dump(combined_path, data.select(sel)) for i in range(params.start_at_hierarchy_level, params.refine_to_hierarchy_level + 1): if params.rmsd_filter.enable: input_name = "filtered" else: if i == params.start_at_hierarchy_level: input_name = "combined" else: input_name = "refined" if params.rmsd_filter.enable: command = "cctbx.xfel.filter_experiments_by_rmsd %s %s output.filtered_experiments=%s output.filtered_reflections=%s" if i == params.start_at_hierarchy_level: command = command % ("%s_combined.expt" % params.tag, "%s_combined.refl" % params.tag, "%s_filtered.expt" % params.tag, "%s_filtered.refl" % params.tag) else: command = command % ( "%s_refined_level%d.expt" % (params.tag, i - 1), "%s_refined_level%d.refl" % (params.tag, i - 1), "%s_filtered_level%d.expt" % (params.tag, i - 1), "%s_filtered_level%d.refl" % (params.tag, i - 1)) command += " iqr_multiplier=%f" % params.rmsd_filter.iqr_multiplier print(command) result = easy_run.fully_buffered(command=command).raise_if_errors() result.show_stdout() print("Refining at hierarchy level", i) refine_phil_file = "%s_refine_level%d.phil" % (params.tag, i) if i == 0: fix_list = ['Tau1'] # fix detector rotz if not params.refine_distance: fix_list.append('Dist') if params.flat_refinement: fix_list.extend(['Tau2', 'Tau3']) diff_phil = "refinement.parameterisation.detector.fix_list=%s\n" % ",".join( fix_list) if params.refine_energy: diff_phil += " refinement.parameterisation.beam.fix=in_spindle_plane+out_spindle_plane\n" # allow energy to refine else: # Note, always need to fix something, so pick a panel group and fix its Tau1 (rotation around Z) always if params.flat_refinement and params.flat_refinement_with_distance: diff_phil = "refinement.parameterisation.detector.fix_list=Group1Tau1,Tau2,Tau3\n" # refine distance, rotz and xy translation diff_phil += "refinement.parameterisation.detector.constraints.parameter=Dist\n" # constrain distance to be refined identically for all panels at this hierarchy level elif params.flat_refinement: diff_phil = "refinement.parameterisation.detector.fix_list=Dist,Group1Tau1,Tau2,Tau3\n" # refine only rotz and xy translation else: diff_phil = "refinement.parameterisation.detector.fix_list=Group1Tau1\n" # refine almost everything if i == params.start_at_hierarchy_level: command = "dials.refine %s %s_%s.expt %s_%s.refl" % ( refine_phil_file, params.tag, input_name, params.tag, input_name) else: command = "dials.refine %s %s_%slevel%d.expt %s_%s_level%d.refl" % ( refine_phil_file, params.tag, input_name, i - 1, params.tag, input_name, i - 1) diff_phil += "refinement.parameterisation.detector.hierarchy_level=%d\n" % i command += " output.experiments=%s_refined_level%d.expt output.reflections=%s_refined_level%d.refl"%( \ params.tag, i, params.tag, i) scope = merged_scope.fetch(parse(diff_phil)) f = open(refine_phil_file, 'w') f.write(refine_scope.fetch_diff(scope).as_str()) f.close() print(command) result = easy_run.fully_buffered(command=command).raise_if_errors() result.show_stdout() output_geometry(params)
def gen_example(self, data_dic): if cfg.TRAIN.NET_G == '': print('Error: the path for models is not found!') else: # Build and load the generator text_encoder = \ RNN_ENCODER(self.n_words, nhidden=cfg.TEXT.EMBEDDING_DIM) state_dict = \ torch.load(cfg.TRAIN.NET_E, map_location=lambda storage, loc: storage) text_encoder.load_state_dict(state_dict) print('Load text encoder from:', cfg.TRAIN.NET_E) text_encoder = text_encoder.cuda() text_encoder.eval() # the path to save generated images if cfg.GAN.B_DCGAN: netG = G_DCGAN() else: netG = G_NET() s_tmp = cfg.TRAIN.NET_G[:cfg.TRAIN.NET_G.rfind('.pth')] model_dir = cfg.TRAIN.NET_G state_dict = \ torch.load(model_dir, map_location=lambda storage, loc: storage) netG.load_state_dict(state_dict["netG"]) print('Load G from: ', model_dir) netG.cuda() netG.eval() for key in data_dic: save_dir = '%s/%s' % (s_tmp, key) mkdir_p(save_dir) captions, cap_lens, sorted_indices = data_dic[key] batch_size = captions.shape[0] nz = cfg.GAN.Z_DIM captions = Variable(torch.from_numpy(captions), volatile=True) cap_lens = Variable(torch.from_numpy(cap_lens), volatile=True) captions = captions.cuda() cap_lens = cap_lens.cuda() for i in range(1): # 16 noise = Variable(torch.FloatTensor(batch_size, nz), volatile=True) noise = noise.cuda() ####################################################### # (1) Extract text embeddings ###################################################### hidden = text_encoder.init_hidden(batch_size) # words_embs: batch_size x nef x seq_len # sent_emb: batch_size x nef words_embs, sent_emb = text_encoder( captions, cap_lens, hidden) mask = (captions == 0) ####################################################### # (2) Generate fake images ###################################################### noise.data.normal_(0, 1) with torch.no_grad(): fake_imgs, attention_maps, _, _ = netG( noise, sent_emb, words_embs, mask) # G attention cap_lens_np = cap_lens.cpu().data.numpy() for j in range(batch_size): save_name = '%s/%d_s_%d' % (save_dir, i, sorted_indices[j]) for k in range(len(fake_imgs)): im = fake_imgs[k][j].data.cpu().numpy() im = (im + 1.0) * 127.5 im = im.astype(np.uint8) # print('im', im.shape) im = np.transpose(im, (1, 2, 0)) # print('im', im.shape) im = Image.fromarray(im) fullpath = '%s_g%d.png' % (save_name, k) im.save(fullpath) for k in range(len(attention_maps)): if len(fake_imgs) > 1: im = fake_imgs[k + 1].detach().cpu() else: im = fake_imgs[0].detach().cpu() attn_maps = attention_maps[k] att_sze = attn_maps.size(2) img_set, sentences = \ build_super_images2(im[j].unsqueeze(0), captions[j].unsqueeze(0), [cap_lens_np[j]], self.ixtoword, [attn_maps[j]], att_sze) if img_set is not None: im = Image.fromarray(img_set) fullpath = '%s_a%d.png' % (save_name, k) im.save(fullpath)
def build_models(self): # ###################encoders######################################## # if cfg.TRAIN.NET_E == '': print('Error: no pretrained text-image encoders') return image_encoder = CNN_ENCODER(cfg.TEXT.EMBEDDING_DIM) img_encoder_path = cfg.TRAIN.NET_E.replace('text_encoder', 'image_encoder') state_dict = \ torch.load(img_encoder_path, map_location=lambda storage, loc: storage) image_encoder.load_state_dict(state_dict) for p in image_encoder.parameters(): p.requires_grad = False print('Load image encoder from:', img_encoder_path) image_encoder.eval() text_encoder = HigherLevelRNN(ninput=cfg.TEXT.EMBEDDING_DIM, nhidden=cfg.TEXT.EMBEDDING_DIM) state_dict = torch.load(cfg.TRAIN.NET_E, map_location=lambda s, l: s) text_encoder.load_state_dict(state_dict) text_encoder_L = RNN_ENCODER(self.n_words, nhidden=cfg.TEXT.EMBEDDING_DIM) L_path = cfg.TRAIN.NET_E.replace('text_encoder', 'text_encoder_L') state_dict = torch.load(L_path, map_location=lambda s, l: s) text_encoder_L.load_state_dict(state_dict) for p in itertools.chain(text_encoder.parameters(),\ text_encoder_L.parameters()): p.requires_grad = False print('Loaded text encoder: %s' % cfg.TRAIN.NET_E) print('Loaded low level text encoder: %s' % L_path) text_encoder.eval() text_encoder_L.eval() # #######################generator and discriminators############## # netsD = [] from model import D_NET64, D_NET128, D_NET256 netG = G_NET() if cfg.TREE.BRANCH_NUM > 0: netsD.append(D_NET64()) if cfg.TREE.BRANCH_NUM > 1: netsD.append(D_NET128()) if cfg.TREE.BRANCH_NUM > 2: netsD.append(D_NET256()) netG.apply(weights_init) # print(netG) for i in range(len(netsD)): netsD[i].apply(weights_init) # print(netsD[i]) print('# of netsD', len(netsD)) epoch = 0 if self.resume: checkpoint_list = sorted( [ckpt for ckpt in glob.glob(self.model_dir + "/" + '*.pth')]) latest_checkpoint = checkpoint_list[-1] state_dict = torch.load(latest_checkpoint, map_location=lambda storage, loc: storage) netG.load_state_dict(state_dict["netG"]) for i in range(len(netsD)): netsD[i].load_state_dict(state_dict["netD"][i]) epoch = int(latest_checkpoint[-8:-4]) + 1 print("Resuming training from checkpoint {} at epoch {}.".format( latest_checkpoint, epoch)) # if cfg.TRAIN.NET_G != '': state_dict = \ torch.load(cfg.TRAIN.NET_G, map_location=lambda storage, loc: storage) epoch = state_dict['epoch'] + 1 netG.load_state_dict(state_dict['netG']) for i in range(len(netsD)): netsD[i].load_state_dict(state_dict['netD'][i]) # netG.load_state_dict(state_dict) # print('Load G from: ', cfg.TRAIN.NET_G) # istart = cfg.TRAIN.NET_G.rfind('_') + 1 # iend = cfg.TRAIN.NET_G.rfind('.') # epoch = cfg.TRAIN.NET_G[istart:iend] # epoch = int(epoch) + 1 # if cfg.TRAIN.B_NET_D: # Gname = cfg.TRAIN.NET_G # for i in range(len(netsD)): # s_tmp = Gname[:Gname.rfind('/')] # Dname = '%s/netD%d.pth' % (s_tmp, i) # print('Load D from: ', Dname) # state_dict = \ # torch.load(Dname, map_location=lambda storage, loc: storage) # netsD[i].load_state_dict(state_dict) # ########################################################### # if cfg.CUDA: text_encoder = text_encoder.cuda() text_encoder_L = text_encoder_L.cuda() image_encoder = image_encoder.cuda() netG.cuda() for i in range(len(netsD)): netsD[i].cuda() return [(text_encoder, text_encoder_L), image_encoder, netG, netsD, epoch]
def save_img_results(self, real_img, netG, noise, sent_emb, words_embs, mask, image_encoder, captions, cap_lens, gen_iterations, transf_matrices_inv, label_one_hot, name='current', num_visualize=8): qa_nums = (cap_lens > 0).sum(1) real_captions = captions captions, _ = make_fake_captions(qa_nums) # fake caption. # Save images # fake_imgs, attention_maps, _, _ = netG(noise, sent_emb, words_embs, mask) inputs = (noise, sent_emb, words_embs, mask, transf_matrices_inv, label_one_hot) fake_imgs, attention_maps, _, _ = nn.parallel.data_parallel( netG, inputs, self.gpus) for i in range(len(attention_maps)): if len(fake_imgs) > 1: img = fake_imgs[i + 1].detach().cpu() lr_img = fake_imgs[i].detach().cpu() else: img = fake_imgs[0].detach().cpu() lr_img = None attn_maps = attention_maps[i] att_sze = attn_maps.size(2) img_set, _ = \ build_super_images(img, captions, self.ixtoword, attn_maps, att_sze, lr_imgs=lr_img, nvis = num_visualize) if img_set is not None: im = Image.fromarray(img_set) fullpath = '%s/G_%s_%d_%d.png'\ % (self.image_dir, name, gen_iterations, i) im.save(fullpath) for i in range(cfg.TREE.BRANCH_NUM): save_pure_img_results(real_img[i].detach().cpu(), fake_imgs[i].detach().cpu(), gen_iterations, self.image_dir, token='level%d' % i) i = -1 img = fake_imgs[i].detach() region_features, _ = image_encoder(img) att_sze = region_features.size(2) _, _, att_maps = words_loss(region_features.detach(), words_embs.detach(), None, qa_nums, None, self.batch_size) img_set, _ = build_super_images(fake_imgs[i].detach().cpu(), captions, self.ixtoword, att_maps, att_sze, nvis=num_visualize) # FIXME currently the `render_attn_to_html` supports only the last level. # please implement multiple level rendering. html_doc = render_attn_to_html([ real_img[i].detach().cpu(), fake_imgs[i].detach().cpu(), ], real_captions, self.ixtoword, att_maps, att_sze, None, info=['Real Images', 'Fake Images']) with open('%s/damsm_attn_%d.html' % (self.image_dir, gen_iterations), 'w') as html_f: html_f.write(str(html_doc)) if img_set is not None: im = Image.fromarray(img_set) fullpath = '%s/D_%s_%d.png'\ % (self.image_dir, name, gen_iterations) im.save(fullpath)
def sampling(self, split_dir, num_samples=30000): if cfg.TRAIN.NET_G == '': print('Error: the path for models is not found!') else: if split_dir == 'test': split_dir = 'valid' # Build and load the generator if cfg.GAN.B_DCGAN: netG = G_DCGAN() else: netG = G_NET() netG.apply(weights_init) netG.cuda() netG.eval() # text_encoder = RNN_ENCODER(self.n_words, nhidden=cfg.TEXT.EMBEDDING_DIM) state_dict = \ torch.load(cfg.TRAIN.NET_E, map_location=lambda storage, loc: storage) text_encoder.load_state_dict(state_dict) print('Load text encoder from:', cfg.TRAIN.NET_E) text_encoder = text_encoder.cuda() text_encoder.eval() batch_size = self.batch_size nz = cfg.GAN.Z_DIM noise = Variable(torch.FloatTensor(batch_size, nz)) noise = noise.cuda() model_dir = cfg.TRAIN.NET_G state_dict = \ torch.load(model_dir, map_location=lambda storage, loc: storage) # state_dict = torch.load(cfg.TRAIN.NET_G) netG.load_state_dict(state_dict["netG"]) print('Load G from: ', model_dir) # the path to save generated images s_tmp = model_dir[:model_dir.rfind('.pth')] save_dir = '%s/%s' % (s_tmp, split_dir) mkdir_p(save_dir) cnt = 0 for _ in range(1): # (cfg.TEXT.CAPTIONS_PER_IMAGE): for step, data in enumerate(self.data_loader, 0): cnt += batch_size if step % 10000 == 0: print('step: ', step) if step >= num_samples: break imgs, captions, cap_lens, class_ids, keys, transformation_matrices, label_one_hot = self.prepare_data( data) transf_matrices_inv = transformation_matrices[1] hidden = text_encoder.init_hidden(batch_size) # words_embs: batch_size x nef x seq_len # sent_emb: batch_size x nef words_embs, sent_emb = text_encoder( captions, cap_lens, hidden) words_embs, sent_emb = words_embs.detach( ), sent_emb.detach() mask = (captions == 0) num_words = words_embs.size(2) if mask.size(1) > num_words: mask = mask[:, :num_words] ####################################################### # (2) Generate fake images ###################################################### noise.data.normal_(0, 1) inputs = (noise, sent_emb, words_embs, mask, transf_matrices_inv, label_one_hot) with torch.no_grad(): fake_imgs, _, mu, logvar = nn.parallel.data_parallel( netG, inputs, self.gpus) for j in range(batch_size): s_tmp = '%s/single/%s' % (save_dir, keys[j]) folder = s_tmp[:s_tmp.rfind('/')] if not os.path.isdir(folder): print('Make a new folder: ', folder) mkdir_p(folder) k = -1 # for k in range(len(fake_imgs)): im = fake_imgs[k][j].data.cpu().numpy() # [-1, 1] --> [0, 255] im = (im + 1.0) * 127.5 im = im.astype(np.uint8) im = np.transpose(im, (1, 2, 0)) im = Image.fromarray(im) fullpath = '%s_s%d.png' % (s_tmp, k) im.save(fullpath)
def __init__(self, params, f_obs, hl_coeffs_start, ncs_object=None, map_coeffs=None, log=None, as_gui_program=False): if log is None: log = sys.stdout adopt_init_args(self, locals()) if self.params.solvent_mask.averaging_radius.final is None: if self.params.initial_d_min is not None: self.params.solvent_mask.averaging_radius.final = self.params.initial_d_min elif self.params.d_min is not None: self.params.solvent_mask.averaging_radius.final = self.params.d_min else: self.params.solvent_mask.averaging_radius.final = self.f_obs.d_min( ) if self.params.solvent_mask.averaging_radius.initial is None: self.params.solvent_mask.averaging_radius.initial = \ self.params.solvent_mask.averaging_radius.final + 1 self.matthews_analysis() self.anisotropic_correction() self.change_of_basis_op = None if self.params.change_basis_to_niggli_cell: self.change_of_basis_op = self.f_obs.change_of_basis_op_to_niggli_cell( ) if self.change_of_basis_op.is_identity_op(): self.change_of_basis_op = None if self.change_of_basis_op is not None: self.f_obs = self.f_obs.change_basis( self.change_of_basis_op).map_to_asu() self.hl_coeffs_start = self.hl_coeffs_start.change_basis( self.change_of_basis_op).map_to_asu() if self.map_coeffs is not None: self.map_coeffs = self.map_coeffs.change_basis( self.change_of_basis_op).map_to_asu() self.mean_solvent_density = 0 self.phase_source_initial = None self.phase_source = None if self.params.d_min is None: if self.params.phase_extension: self.params.d_min = self.f_obs.d_min() else: self.params.d_min = self.hl_coeffs_start.d_min() if self.params.initial_d_min is None: self.params.initial_d_min = self.params.d_min assert self.params.initial_d_min >= self.params.d_min self.max_iterations = sum( (self.params.initial_steps, self.params.shrink_steps, self.params.final_steps)) self.i_cycle = 0 if self.params.shrink_steps is not None and self.params.shrink_steps > 0: self.radius_delta = (self.params.solvent_mask.averaging_radius.initial - self.params.solvent_mask.averaging_radius.final) \ / self.params.shrink_steps if self.params.phase_extension: self.phase_extend_step = ( self.params.initial_d_min - self.params.d_min) / self.params.shrink_steps else: self.phase_extend_step = 0 self.params.initial_d_min = self.params.d_min self.complete_set = self.f_obs.complete_set() if (self.f_obs.sigmas() is not None): ref_active = (self.f_obs.sigmas() > 0) \ & (self.f_obs.d_spacings().data() >= self.params.d_min) else: ref_active = (self.f_obs.d_spacings().data() >= self.params.d_min) sigma_cutoff = 0 obs_rms = 1e4 obs_high = rms(self.f_obs.select(ref_active).data()) * obs_rms obs_low = flex.min(self.f_obs.select(ref_active).data()) if (self.f_obs.sigmas() is not None): self.ref_flags_array = self.f_obs.array( data=((self.f_obs.data() > sigma_cutoff * self.f_obs.sigmas()) & (self.f_obs.data() >= obs_low) & (self.f_obs.data() <= obs_high) & (self.f_obs.d_spacings().data() > self.params.d_min))) else: self.ref_flags_array = self.f_obs.array( data=((self.f_obs.data() >= obs_low) & (self.f_obs.data() <= obs_high) & (self.f_obs.d_spacings().data() > self.params.d_min))) # now setup for complete arrays self.ref_flags_array = self.ref_flags_array.complete_array( new_data_value=False, d_min=self.params.d_min) self.ref_flags = self.ref_flags_array.data() self.f_obs_complete = self.f_obs.complete_array( new_data_value=0, new_sigmas_value=0, d_min=self.params.d_min) self.hl_coeffs_start = self.hl_coeffs_start.complete_array( new_data_value=(0, 0, 0, 0), d_min=self.params.d_min) self.hl_coeffs = self.hl_coeffs_start.select_indices( self.active_indices) hl_coeffs = self.hl_coeffs self.compute_phase_source(hl_coeffs) fom = flex.abs(self.phase_source.data()) fom.set_selected(hl_coeffs.data() == (0, 0, 0, 0), 0) self.fom = fom if self.map_coeffs is None: self.map_coeffs = self.f_obs_active.customized_copy( data=self.f_obs_active.data() * fom, sigmas=None).phase_transfer(phase_source=self.hl_coeffs) self.map_coeffs.data().set_selected(fom <= 0, 0) self.map = self.map_coeffs.select(fom > 0).fft_map( symmetry_flags=maptbx.use_space_group_symmetry, resolution_factor=self.params.grid_resolution_factor ).apply_volume_scaling().real_map_unpadded() self.map_coeffs = self.map_coeffs.select(fom > 0) else: assert self.map_coeffs.is_complex_array() self.map = self.map_coeffs.fft_map( symmetry_flags=maptbx.use_space_group_symmetry, resolution_factor=self.params.grid_resolution_factor ).apply_volume_scaling().real_map_unpadded() self.map_coeffs_start = self.map_coeffs self.calculate_solvent_mask() n_phased = (fom > 0).count(True) if params.verbose: summary = "n phased: %d\n" % n_phased summary += "Mean solvent density: %.4f\n" % self.mean_solvent_density summary += "Mean protein density: %.4f\n" % self.mean_protein_density summary += "RMS solvent density: %.4f\n" % self.rms_solvent_density summary += "RMS protein density: %.4f\n" % self.rms_protein_density summary += "RMS solvent/protein density ratio: %.4f\n" % ( self.rms_solvent_density / self.rms_protein_density) summary += "F000/V: %.4f\n" % self.f000_over_v summary += "Mean FOM: %.4f\n" % flex.mean(fom.select(fom > 0)) print(summary, file=self.log) libtbx.call_back(message="summary", data=summary) # XXX initialize printable statistics self.truncate_min = None self.truncate_min_percent = None self.truncate_max = None self.truncate_max_percent = None self.k_flip = None self.solvent_add = None self.truncate_density = \ (self.params.density_truncation.fraction_max is not None or self.params.density_truncation.fraction_min is not None) self._stats = dm_stats() self._stats.add_cycle(cycle=0, mean_solvent_density=self.mean_solvent_density, mean_protein_density=self.mean_protein_density, f000_over_v=self.f000_over_v, rms_solvent_density=self.rms_solvent_density, rms_protein_density=self.rms_protein_density, fom=flex.mean(fom.select(fom > 0))) libtbx.call_back("start_progress_bar", data=group_args(label="Running %d cycles..." % self.max_iterations, size=self.max_iterations)) for self.i_cycle in range(self.max_iterations): self.next_cycle() libtbx.call_back(message="increment_progress_bar", data=group_args(chunk=1), cached=False) libtbx.call_back("end_progress_bar", data=None)
def train(self): text_encoder, image_encoder, netG, netsD, start_epoch = self.build_models( ) H_rnn_model, L_rnn_model = text_encoder avg_param_G = copy_G_params(netG) optimizerG, optimizersD = self.define_optimizers(netG, netsD) if cfg.TRAIN.EVQAL.B_EVQAL: netVQA_E = load_resnet_image_encoder(model_stage=2) netVQA = load_vqa_net(cfg.TRAIN.EVQAL.NET, load_program_vocab( cfg.TRAIN.EVQAL.PROGRAM_VOCAB_FILE), feature_dim=(512, 28, 28)) else: netVQA_E = netVQA = None real_labels, fake_labels, match_labels = self.prepare_labels() batch_size = self.batch_size nz = cfg.GAN.Z_DIM noise = Variable(torch.FloatTensor(batch_size, nz)) fixed_noise = Variable(torch.FloatTensor(batch_size, nz).normal_(0, 1)) if cfg.CUDA: noise, fixed_noise = noise.cuda(), fixed_noise.cuda() gen_iterations = 0 # gen_iterations = start_epoch * self.num_batches for epoch in range(start_epoch, self.max_epoch): start_t = time.time() am_vqa_loss = AverageMeter('VQA Loss') data_iter = iter(self.data_loader) step = 0 while step < self.num_batches: # reset requires_grad to be trainable for all Ds # self.set_requires_grad_value(netsD, True) ###################################################### # (1) Prepare training data and Compute text embeddings ###################################################### data = data_iter.next() imgs, captions, cap_lens, class_ids, bbox, label_one_hot, transformation_matrices, keys, prog = self.prepare_data( data) class_ids = None batch_size = captions.size(0) transf_matrices = transformation_matrices[0].detach() transf_matrices_inv = transformation_matrices[1].detach() per_qa_embs, avg_qa_embs, qa_nums =\ Level2RNNEncodeMagic(captions, cap_lens, L_rnn_model, H_rnn_model) per_qa_embs, avg_qa_embs = (per_qa_embs.detach(), avg_qa_embs.detach()) _nmaxqa = cfg.TEXT.MAX_QA_NUM mask = torch.ones(batch_size, _nmaxqa, dtype=torch.uint8).cuda() _ref = torch.arange(0, _nmaxqa).view(1, -1).repeat(batch_size, 1).cuda() _targ = qa_nums.view(-1, 1).repeat(1, _nmaxqa) mask[_ref < _targ] = 0 num_words = per_qa_embs.size(2) if mask.size(1) > num_words: mask = mask[:, :num_words] ####################################################### # (2) Generate fake images ###################################################### noise.data.normal_(0, 1) inputs = (noise, avg_qa_embs, per_qa_embs, mask, transf_matrices_inv, label_one_hot) fake_imgs, _, mu, logvar = nn.parallel.data_parallel( netG, inputs, self.gpus) ####################################################### # (3) Update D network ###################################################### errD_total = 0 D_logs = '' for i in range(len(netsD)): netsD[i].zero_grad() if i == 0: # NOTE only the first level Discriminator is modified. errD = discriminator_loss( netsD[i], imgs[i], fake_imgs[i], avg_qa_embs, real_labels, fake_labels, self.gpus, local_labels=label_one_hot, transf_matrices=transf_matrices, transf_matrices_inv=transf_matrices_inv) else: errD = discriminator_loss(netsD[i], imgs[i], fake_imgs[i], avg_qa_embs, real_labels, fake_labels, self.gpus) # backward and update parameters errD.backward() optimizersD[i].step() errD_total += errD D_logs += 'errD%d: %.2f ' % (i, errD.item()) ####################################################### # (4) Update G network: maximize log(D(G(z))) ###################################################### # compute total loss for training G step += 1 gen_iterations += 1 # do not need to compute gradient for Ds # self.set_requires_grad_value(netsD, False) netG.zero_grad() errG_total, G_logs = \ generator_loss(netsD, image_encoder, fake_imgs, real_labels, per_qa_embs, avg_qa_embs, match_labels, qa_nums, class_ids, self.gpus, local_labels=label_one_hot, transf_matrices=transf_matrices, transf_matrices_inv=transf_matrices_inv) if cfg.GAN.B_CA_NET: kl_loss = KL_loss(mu, logvar) else: kl_loss = torch.FloatTensor([0.]).squeeze().cuda() errG_total += kl_loss G_logs += 'kl_loss: %.2f ' % kl_loss.item() if cfg.TRAIN.EVQAL.B_EVQAL: fake_img_fvqa = extract_image_feats( fake_imgs[-1], netVQA_E, self.gpus) errVQA = VQA_loss(netVQA, fake_img_fvqa, prog['programs'], prog['answers'], self.gpus) else: errVQA = torch.FloatTensor([0.]).squeeze().cuda() G_logs += 'VQA_loss: %.2f ' % errVQA.data.item() beta = cfg.TRAIN.EVQAL.BETA errG_total += (errVQA * beta) # backward and update parameters errG_total.backward() optimizerG.step() am_vqa_loss.update(errVQA.cpu().item()) for p, avg_p in zip(netG.parameters(), avg_param_G): avg_p.mul_(0.999).add_(0.001, p.data) # save images if gen_iterations % 100 == 0: print(D_logs + '\n' + G_logs) if gen_iterations % 500 == 0: # FIXME original: 1000 backup_para = copy_G_params(netG) load_params(netG, avg_param_G) self.save_img_results(imgs, netG, fixed_noise, avg_qa_embs, per_qa_embs, mask, image_encoder, captions, cap_lens, epoch, transf_matrices_inv, label_one_hot, name='average') load_params(netG, backup_para) end_t = time.time() print('''[%d/%d][%d] Loss_D: %.2f Loss_G: %.2f Time: %.2fs''' % (epoch, self.max_epoch, self.num_batches, errD_total.item(), errG_total.item(), end_t - start_t)) if cfg.TRAIN.EVQAL.B_EVQAL: print('Avg. VQA Loss of this epoch: %s' % str(am_vqa_loss)) if epoch % cfg.TRAIN.SNAPSHOT_INTERVAL == 0: # and epoch != 0: self.save_model(netG, avg_param_G, netsD, optimizerG, optimizersD, epoch) self.save_model(netG, avg_param_G, netsD, optimizerG, optimizersD, epoch)
def change(x, num): y = np.zeros((x.shape[0], num)) for i in range(len(x)): y[i] = to_categorical(x[i], num) return y
def set_requires_grad_value(self, models_list, brequires): for i in range(len(models_list)): for p in models_list[i].parameters(): p.requires_grad = brequires
def emb(emb, x): y = np.zeros((x.shape[0], x.shape[1], 100)) for k in range(len(x)): for i in range(len(x[k])): y[k][i] = emb[x[k][i]] return y
def train_gan(key): ac = ACGAN() # get data ac.load_data(key) tr_x0, tr_x1, tr_x2, tr_x3, tr_x4, tr_x5, tr_x6, tr_x7, tr_x8, tr_y0, tr_y1, tr_y_one = ac.train_set va_x0, va_x1, va_x2, va_x3, va_x4, va_x5, va_x6, va_x7, va_x8, va_y0, va_y1, va_y_one = ac.valid_set te_x0, te_x1, te_x2, te_x3, te_x4, te_x5, te_x6, te_x7, te_x8, te_y0, te_y1, te_y_one = ac.test_set # build discriminator discriminator = ac.build_discriminator() discriminator.compile(optimizer=Adam(lr=ac.lr, beta_1=0.5), loss=[ 'binary_crossentropy', 'sparse_categorical_crossentropy', 'sparse_categorical_crossentropy' ]) # build generator generator = ac.build_generator() generator.compile(optimizer=Adam(lr=ac.lr, beta_1=0.5), loss='binary_crossentropy') j = 0 f = 0 for epoch in range(ac.epochs): print('Epoch {} of {}'.format(epoch + 1, ac.epochs)) batch = int(len(ac.train_set[0]) / ac.batch) progress_bar = Progbar(target=batch) epoch_disc_loss = [] for i in range(batch): # progress_bar.update(i) # generate a new batch of noise begin = i * ac.batch end = (i + 1) * ac.batch if (i + 1) * ac.batch < len( ac.train_set[0]) else len(ac.train_set[0]) lenth = end - begin noise0 = np.random.random((lenth, ac.maxlen[0])) noise1 = np.random.random((lenth, ac.maxlen[1])) noise2 = np.random.random((lenth, ac.maxlen[2])) # get a batch of real daata t = np.array(tr_x4) temp_sip = np.array(tr_x4[begin:end]) temp_rs = np.array(tr_x5[begin:end]) temp_cue = np.array(tr_x6[begin:end]) real_sip = emb(ac.embdding, temp_sip) real_rs = emb(ac.embdding, temp_rs) real_cue = emb(ac.embdding, temp_cue) idx = np.array(tr_x7[begin:end]) ty = np.array(tr_x8[begin:end]) index = emby(ac.embdding, idx) tp = emby(ac.embdding, ty) tr_yu = np.array(tr_y0[begin:end]).reshape((lenth, 1)) tr_ycue = np.array(tr_y1[begin:end]).reshape((lenth, 1)) tr_yf = np.array(tr_y_one[begin:end]) fake_sip, fake_rs, fake_cue = generator.predict( [noise0, noise1, noise2, tr_yu, tr_ycue]) sips = np.concatenate([real_sip, fake_sip], axis=0) rss = np.concatenate([real_rs, fake_rs], axis=0) cues = np.concatenate([real_cue, fake_cue], axis=0) indexs = np.concatenate([index, index], axis=0) types = np.concatenate([tp, tp], axis=0) tr_f = np.concatenate([tr_yf, tr_yf]).reshape((-1, 1)) tr_u = np.concatenate([tr_yu, tr_yu]).reshape((-1, 1)) tr_cue = np.concatenate([tr_ycue, tr_ycue]).reshape((-1, 1)) disc_loss = discriminator.train_on_batch( [sips, rss, cues, indexs, types], [tr_f, tr_u, tr_cue]) epoch_disc_loss.append(disc_loss) print('\nTesting for epoch {}:'.format(epoch + 1)) # get valid data temp_sip = np.array(va_x4) temp_rs = np.array(va_x5) temp_cue = np.array(va_x6) real_sip = emb(ac.embdding, temp_sip) real_rs = emb(ac.embdding, temp_rs) real_cue = emb(ac.embdding, temp_cue) idx = np.array(va_x7) ty = np.array(va_x8) index = emby(ac.embdding, idx) tp = emby(ac.embdding, ty) va_yu = np.array(va_y0).reshape((-1, 1)) va_ycue = np.array(va_y1).reshape((-1, 1)) va_yf = np.array(va_y_one).reshape((-1, 1)) size = va_ycue.shape[0] discriminator.train_on_batch([real_sip, real_rs, real_cue, index, tp], [va_yf, va_yu, va_ycue]) # get test data temp_sip = np.array(te_x4) temp_rs = np.array(te_x5) temp_cue = np.array(te_x6) real_sip = emb(ac.embdding, temp_sip) real_rs = emb(ac.embdding, temp_rs) real_cue = emb(ac.embdding, temp_cue) idx = np.array(te_x7) ty = np.array(te_x8) index = emby(ac.embdding, idx) tp = emby(ac.embdding, ty) te_yu = np.array(te_y0).reshape((-1, 1)) te_ycue = np.array(te_y1).reshape((-1, 1)) te_yf = np.array(te_y_one).reshape((-1, 1)) size = te_ycue.shape[0] yf, yu, ycue = discriminator.predict( [real_sip, real_rs, real_cue, index, tp]) goldu = te_yu.reshape((1, size))[0] preu = yu.argmax(axis=-1).reshape((1, size))[0] goldcue = te_yu.reshape((1, size))[0] precue = yu.argmax(axis=-1).reshape((1, size))[0] cuevalue = ycue.max(axis=-1).reshape((1, size))[0] # f1_resultu, f_sumu = cal_F1_measure(goldu, preu) # f1_resultcue, f_sumcue = cal_F1_measure(goldcue, precue) # f_sum = f_sumu +f_sumcue gold, pre, au_gold, au_pre, in_gold, in_pre = get_pre( preu, precue, cuevalue, key) P, R, F = score(gold, pre, False) # f1_result = cal_F1_measure(gold, pre) # micro_F1, macro_F1 = cal_macro_micro_F1(f1_result) # print('micro-F1: {0:.4f} - macro-F1: {1:.4f}'.format(micro_F1, macro_F1)) if F > f: f = F tpreu = yu.argmax(axis=-1).reshape((1, size))[0] tprecue = ycue.argmax(axis=-1).reshape((1, size))[0] tcuevalue = ycue.max(axis=-1).reshape((1, size))[0] gold, pre, au_gold, au_pre, in_gold, in_pre = get_pre( tpreu, tprecue, tcuevalue, key) print('best F1:') all_result = scoreF(gold, pre, False) # f1_result = cal_F1_measure(gold, pre) # micro_F1, macro_F1 = cal_macro_micro_F1(f1_result) # print('micro-F1: {0:.4f} - macro-F1: {1:.4f}'.format(micro_F1, macro_F1)) print('author:') au_result = scoreF(au_gold, au_pre, False) # f1_result = cal_F1_measure(au_gold,au_pre) # micro_F1, macro_F1 = cal_macro_micro_F1(f1_result) # print('micro-F1: {0:.4f} - macro-F1: {1:.4f}'.format(micro_F1, macro_F1)) print('in source:') in_result = scoreF(in_gold, in_pre, False) # f1_result = cal_F1_measure(in_gold, in_pre) # micro_F1, macro_F1 = cal_macro_micro_F1(f1_result) # print('micro-F1: {0:.4f} - macro-F1: {1:.4f}'.format(micro_F1, macro_F1)) return all_result, au_result, in_result
for window_size in [1,2]: data_index = 0 batch, labels = generate_batch_skip_gram(batch_size=8,window_size=window_size) print('\nwith window_size = %d:' % window_size) print(' batch:', [reverse_dictionary[bi] for bi in batch]) print(' labels:', [reverse_dictionary[li] for li in labels.shape(8)]) batch_size = 128 embedding_size = 128 window_size = 4 valid_size = 16 valid_window = 50 valid_examples = np.array(random.sample(range(valid_window),valid_size)) valid_examples = np.append(valid_examples,random.sample(range(1000, 1000 + valid_window), valid_size),axis=0) num_sampled = 32 tf.reset_default_graph() train_dataset = tf.placeholder(tf.int32,shape=[batch_size]) train_labels = tf.placeholder(tf.int32,shape=[batch_size,1]) valid_dataset = tf.constant(valid_examples, dtype=tf.int32) embeddings = tf.Variable(tf.random_uniform([vocabulary_size,embedding_size],-1.0,1.0)) softmax_weights = tf.Variable(tf.truncated_normal([vocabulary_size,embedding_size], stddev=0.5 / math.sqrt(embedding_size))) softmax_biases = tf.Variable(tf.random_uniform([vocabulary_size],0.0,0.01))
def emby(emb, x): y = np.zeros((x.shape[0], 100)) for k in range(len(x)): y[k] = emb[x[k]] return y.reshape((x.shape[0], 1, 100))
def backward_gpu(self, inputs, grad_outputs): x, W = inputs[:2] b = inputs[2] if len(inputs) == 3 else None gy = grad_outputs[0] _, out_c, out_h, out_w = gy.shape n, c, h, w = x.shape kh, kw = W.shape[2:] gW = cuda.cupy.empty_like(W) if (not self.cover_all and cuda.cudnn_enabled and self.use_cudnn and _check_cudnn_acceptable_type(x.dtype, W.dtype)): x = cuda.cupy.ascontiguousarray(x) W = cuda.cupy.ascontiguousarray(W) gy = cuda.cupy.ascontiguousarray(gy) handle = cudnn.get_handle() x_desc = cudnn.create_tensor_descriptor(x) gy_desc = cudnn.create_tensor_descriptor(gy) oz_dtype = 'd' if x.dtype == 'd' else 'f' one = numpy.array(1, dtype=oz_dtype).ctypes zero = numpy.array(0, dtype=oz_dtype).ctypes gx = cuda.cupy.empty_like(x) if _cudnn_version >= 4000: workspace_size = cuda.get_max_workspace_size() workspace = cuda.cupy.empty((workspace_size,), dtype='b') algo = libcudnn.getConvolutionBackwardFilterAlgorithm( handle, x_desc.value, gy_desc.value, self.conv_desc.value, self.filter_desc.value, _bwd_filter_pref, workspace_size) libcudnn.convolutionBackwardFilter_v3( handle, one.data, x_desc.value, x.data.ptr, gy_desc.value, gy.data.ptr, self.conv_desc.value, algo, workspace.data.ptr, workspace_size, zero.data, self.filter_desc.value, gW.data.ptr) algo = libcudnn.getConvolutionBackwardDataAlgorithm( handle, self.filter_desc.value, gy_desc.value, self.conv_desc.value, x_desc.value, _bwd_data_pref, workspace_size) libcudnn.convolutionBackwardData_v3( handle, one.data, self.filter_desc.value, W.data.ptr, gy_desc.value, gy.data.ptr, self.conv_desc.value, algo, workspace.data.ptr, workspace_size, zero.data, x_desc.value, gx.data.ptr) else: libcudnn.convolutionBackwardFilter_v2( handle, one.data, x_desc.value, x.data.ptr, gy_desc.value, gy.data.ptr, self.conv_desc.value, zero.data, self.filter_desc.value, gW.data.ptr) libcudnn.convolutionBackwardData_v2( handle, one.data, self.filter_desc.value, W.data.ptr, gy_desc.value, gy.data.ptr, self.conv_desc.value, zero.data, x_desc.value, gx.data.ptr) if b is not None: gb = cuda.cupy.empty_like(b) libcudnn.convolutionBackwardBias( handle, one.data, gy_desc.value, gy.data.ptr, zero.data, self.bias_desc.value, gb.data.ptr) else: gW_mat = gW.reshape(out_c, c * kh * kw) col_mats = self.col.reshape(n, c * kh * kw, out_h * out_w) gy_mats = gy.reshape(n, out_c, out_h * out_w) # TODO(beam2d): Use streams or batch gemm gW_mat[...] = 0 for i in moves.range(n): gW_mat += cuda.cupy.dot(gy_mats[i], col_mats[i].T) W_mat = W.reshape(out_c, -1) gcol = cuda.cupy.empty_like(self.col) gcol_mats = gcol.reshape(n, c * kh * kw, out_h * out_w) for i in moves.range(n): gcol_mats[i] = cuda.cupy.dot(W_mat.T, gy_mats[i]) gx = conv.col2im_gpu( gcol, self.sy, self.sx, self.ph, self.pw, h, w) if b is not None: gb = gy.sum(axis=(0, 2, 3)) if b is None: return gx, gW else: return gx, gW, gb
'CT+': 0, 'micro': 0, 'macro': 0 } resin = { 'Uu': 0, 'PR+': 0, 'PS+': 0, 'CT-': 0, 'PS-': 0, 'PR-': 0, 'CT+': 0, 'micro': 0, 'macro': 0 } for i in range(10): all, author, inr = train_gan(i) for key in resall.keys(): if key in all.keys(): resall[key] += all[key] if key in author.keys(): resau[key] += author[key] if key in inr.keys(): resin[key] += inr[key] print('Uu:' + str(resall['Uu'] / 10) + '\tPR+:' + str(resall['PR+'] / 10) + '\tPS+' + str(resall['PS+'] / 10) + '\tCT-' + str(resall['CT-'] / 10) + '\tPS-' + str(resall['PS-'] / 10) + '\tPR-' + str(resall['PR-'] / 10) + '\tCT+' + str(resall['CT+'] / 10) + '\tmicro' + str(resall['micro'] / 10) + '\tmacro' + str(resall['macro'] / 10)) print('Uu:' + str(resau['Uu'] / 10) + '\tPR+:' + str(resau['PR+'] / 10) +
def exercise_1(): pdb_raw = """\ CRYST1 23.000 6.666 25.000 90.00 107.08 90.00 P 1 21 1 2 ATOM 1 N GLY A 1 -9.009 4.612 6.102 1.00 16.77 N ATOM 2 CA GLY A 1 -9.052 4.207 4.651 1.00 16.57 C ATOM 3 C GLY A 1 -8.015 3.140 4.419 1.00 16.16 C ATOM 4 O GLY A 1 -7.523 2.521 5.381 1.00 16.78 O ATOM 5 N ASN A 2 -7.656 2.923 3.155 1.00 15.02 N ATOM 6 CA ASN A 2 -6.522 2.038 2.831 1.00 14.10 C ATOM 7 C ASN A 2 -5.241 2.537 3.427 1.00 13.13 C ATOM 8 O ASN A 2 -4.978 3.742 3.426 1.00 11.91 O ATOM 9 CB ASN A 2 -6.346 1.881 1.341 1.00 15.38 C ATOM 10 CG ASN A 2 -7.584 1.342 0.692 1.00 14.08 C ATOM 11 OD1 ASN A 2 -8.025 0.227 1.016 1.00 17.46 O ATOM 12 ND2 ASN A 2 -8.204 2.155 -0.169 1.00 11.72 N ATOM 13 N ASN A 3 -4.438 1.590 3.905 1.00 12.26 N ATOM 14 CA ASN A 3 -3.193 1.904 4.589 1.00 11.74 C ATOM 15 C ASN A 3 -1.955 1.332 3.895 1.00 11.10 C ATOM 16 O ASN A 3 -1.872 0.119 3.648 1.00 10.42 O ATOM 17 CB ASN A 3 -3.259 1.378 6.042 1.00 12.15 C ATOM 18 CG ASN A 3 -2.006 1.739 6.861 1.00 12.82 C ATOM 19 OD1 ASN A 3 -1.702 2.925 7.072 1.00 15.05 O ATOM 20 ND2 ASN A 3 -1.271 0.715 7.306 1.00 13.48 N ATOM 21 N MET A 4 -1.005 2.228 3.598 1.00 10.29 N ATOM 22 CA MET A 4 0.384 1.888 3.199 1.00 10.53 C ATOM 23 C MET A 4 1.435 2.606 4.088 1.00 10.24 C ATOM 24 O MET A 4 1.547 3.843 4.115 1.00 8.86 O ATOM 25 CB MET A 4 0.616 2.241 1.729 1.00 20.00 C ATOM 26 CG MET A 4 -0.207 1.416 0.754 1.00 20.00 C ATOM 27 SD MET A 4 0.132 -0.349 0.876 1.00 20.00 S ATOM 28 CE MET A 4 1.822 -0.411 0.285 1.00 20.00 C ATOM 29 N GLN A 5 2.154 1.821 4.871 1.00 10.38 N ATOM 30 CA GLN A 5 3.270 2.361 5.640 1.00 11.39 C ATOM 31 C GLN A 5 4.594 1.768 5.172 1.00 11.52 C ATOM 32 O GLN A 5 4.768 0.546 5.054 1.00 12.05 O ATOM 33 CB GLN A 5 3.056 2.183 7.147 1.00 11.96 C ATOM 34 CG GLN A 5 1.829 2.950 7.647 1.00 10.81 C ATOM 35 CD GLN A 5 1.344 2.414 8.954 1.00 13.10 C ATOM 36 OE1 GLN A 5 0.774 1.325 9.002 1.00 10.65 O ATOM 37 NE2 GLN A 5 1.549 3.187 10.039 1.00 12.30 N ATOM 38 N ASN A 6 5.514 2.664 4.856 1.00 11.99 N ATOM 39 CA ASN A 6 6.831 2.310 4.318 1.00 12.30 C ATOM 40 C ASN A 6 7.854 2.761 5.324 1.00 13.40 C ATOM 41 O ASN A 6 8.219 3.943 5.374 1.00 13.92 O ATOM 42 CB ASN A 6 7.065 3.016 2.993 1.00 12.13 C ATOM 43 CG ASN A 6 5.961 2.735 2.003 1.00 12.77 C ATOM 44 OD1 ASN A 6 5.798 1.604 1.551 1.00 14.27 O ATOM 45 ND2 ASN A 6 5.195 3.747 1.679 1.00 10.07 N ATOM 46 N TYR A 7 8.292 1.817 6.147 1.00 14.70 N ATOM 47 CA TYR A 7 9.159 2.144 7.299 1.00 15.18 C ATOM 48 C TYR A 7 10.603 2.331 6.885 1.00 15.91 C ATOM 49 O TYR A 7 11.041 1.811 5.855 1.00 15.76 O ATOM 50 CB TYR A 7 9.061 1.065 8.369 1.00 15.35 C ATOM 51 CG TYR A 7 7.665 0.929 8.902 1.00 14.45 C ATOM 52 CD1 TYR A 7 6.771 0.021 8.327 1.00 15.68 C ATOM 53 CD2 TYR A 7 7.210 1.756 9.920 1.00 14.80 C ATOM 54 CE1 TYR A 7 5.480 -0.094 8.796 1.00 13.46 C ATOM 55 CE2 TYR A 7 5.904 1.649 10.416 1.00 14.33 C ATOM 56 CZ TYR A 7 5.047 0.729 9.831 1.00 15.09 C ATOM 57 OH TYR A 7 3.766 0.589 10.291 1.00 14.39 O ATOM 58 OXT TYR A 7 11.358 2.999 7.612 1.00 17.49 O TER 59 TYR A 7 HETATM 1 CA CA A 8 10.431 1.858 3.216 1.00 30.00 CA HETATM 60 O HOH A 9 -6.471 5.227 7.124 1.00 22.62 O HETATM 62 O HOH A 10 -11.286 1.756 -1.468 1.00 17.08 O HETATM 63 O HOH A 11 11.808 4.179 9.970 1.00 23.99 O HETATM 64 O HOH A 12 13.605 1.327 9.198 1.00 26.17 O HETATM 65 O HOH A 13 -2.749 3.429 10.024 1.00 39.15 O HETATM 66 O HOH A 14 -1.500 0.682 10.967 1.00 43.49 O END """ pdb_file = "tst_xtriage_in.pdb" with open(pdb_file, "w") as f: f.write(pdb_raw) fmodel_args = [ pdb_file, "high_resolution=1.5", "k_sol=0.35", "b_sol=20", "wavelength=1.54", "add_random_error_to_amplitudes_percent=3", "random_seed=12345", "output.type=real", "output.label=F", "output.file_name=tst_xtriage_fmodel.mtz", ] # read it instead so python3 will be the same # fmodel.run(args=fmodel_args, log=null_out()) hkl_file = libtbx.env.find_in_repositories( relative_path="mmtbx/regression/mtz/tst_xtriage_fmodel.mtz", test=os.path.isfile) mtz_in = file_reader.any_file(hkl_file).assert_file_type("hkl") f_obs = mtz_in.file_server.miller_arrays[0].remove_cone(0.1) data = f_obs.data() # add some outliers #data[17] = 20 #data[334] = 26 #data[1908] = 13 # and sigmas sigf = flex.double(f_obs.size(), 0.1) + (f_obs.data() * 0.03) f_obs = f_obs.customized_copy(sigmas=sigf) mtz_file = "tst_xtriage_in.mtz" f_obs.as_mtz_dataset(column_root_label="F").mtz_object().write(mtz_file) seq_file = "tst_xtriage_in.fa" with open(seq_file, "w") as f: f.write("> tst_xtriage\nGNNMQNY") # check with completeness_as_non_anomalous=True xtriage_args = [ mtz_file, pdb_file, seq_file, "log=tst_xtriage_1.log", "l_test_dhkl=2,2,2", "completeness_as_non_anomalous=True", ] result = xtriage.run(args=xtriage_args, out=null_out()) test_pickle_consistency_and_size(result) assert (result.matthews.n_copies == 1) assert (str(result.matthews.table) == """\ Solvent content analysis Copies Solvent content Matthews coeff. P(solvent content) 1 0.472 2.33 1.000 """) data_strength = result.data_strength_and_completeness assert approx_equal(data_strength.data_strength.resolution_cut, 1.5351, eps=0.001) out1 = data_strength.low_resolution_completeness.format() assert (out1 == """\ --------------------------------------------------------- | Resolution range | N(obs)/N(possible) | Completeness | --------------------------------------------------------- | 21.9858 - 10.4368 | [6/7] | 0.857 | | 10.4368 - 8.4369 | [3/3] | 1.000 | | 8.4369 - 7.4172 | [3/4] | 0.750 | | 7.4172 - 6.7606 | [4/4] | 1.000 | | 6.7606 - 6.2882 | [5/5] | 1.000 | | 6.2882 - 5.9252 | [3/4] | 0.750 | | 5.9252 - 5.6337 | [7/7] | 1.000 | | 5.6337 - 5.3922 | [5/5] | 1.000 | | 5.3922 - 5.1874 | [4/4] | 1.000 | | 5.1874 - 5.0106 | [4/4] | 1.000 | ---------------------------------------------------------"""), out1 # ANOMALOUS SIGNAL a_meas = result.anomalous_info.measurability #assert approx_equal(a_meas.high_d_cut, 4.7636, eps=0.0001) # Why it's None? assert approx_equal(a_meas.low_d_cut, 2.3566, eps=0.0001) # ABSOLUTE SCALING ws = result.wilson_scaling assert ("%.2f" % ws.iso_p_scale) == "0.65", ws.iso_p_scale assert ("%.2f" % ws.iso_b_wilson) == "14.42", ws.iso_b_wilson # FIXME these may need to be adjusted for different hardware/OS assert approx_equal(ws.aniso_p_scale, 0.64723, eps=0.001) assert approx_equal( ws.aniso_u_star, [0.00034229, 0.00475982, 0.000285989, -0.0, 8.95386085999e-05, 0.0]) assert approx_equal(ws.aniso_b_cart, (13.218423, 16.840142, 12.948426, 1.0354e-15, -0.0685311, -7.92862e-16), 0.3) # convenience methods for GUI assert approx_equal(result.aniso_b_min, 12.895580) assert approx_equal(result.aniso_range_of_b, 3.804215) # assert approx_equal( ws.outlier_shell_table.data[0], # d_spacing [9.865131, 8.369653, 4.648634]) assert approx_equal( ws.outlier_shell_table.data[1], # z_score [5.306713, 18.068284, 5.319230]) assert (len(ws.outliers.acentric_outliers_table.data[0]) == 2) assert (ws.outliers.acentric_outliers_table.data[1] == [(0, -1, -1), (0, 1, 1)]) assert approx_equal(ws.outliers.acentric_outliers_table.data[2], [3.507247, 3.315550]) assert (ws.outliers.centric_outliers_table.data is None) assert (len(ws.ice_rings.table._rows) == 10) assert (ws.ice_rings.table._rows[0] == [' 3.897', ' 1.000', ' 0.76', ' 1.00']), \ ws.ice_rings.table._rows[0] tw = result.twin_results wm = tw.wilson_moments out = StringIO() wm.show(out) assert not show_diff( out.getvalue(), """ ----------Wilson ratio and moments---------- Acentric reflections: <I^2>/<I>^2 :2.063 (untwinned: 2.000; perfect twin 1.500) <F>^2/<F^2> :0.778 (untwinned: 0.785; perfect twin 0.885) <|E^2 - 1|> :0.745 (untwinned: 0.736; perfect twin 0.541) Centric reflections: <I^2>/<I>^2 :3.076 (untwinned: 3.000; perfect twin 2.000) <F>^2/<F^2> :0.628 (untwinned: 0.637; perfect twin 0.785) <|E^2 - 1|> :0.999 (untwinned: 0.968; perfect twin 0.736) """) # XXX PDB validation server assert approx_equal(result.iso_b_wilson, 14.51, eps=0.1) assert approx_equal(result.aniso_b_ratio, 0.271, eps=0.1) assert (result.number_of_wilson_outliers == 2) assert approx_equal(result.l_test_mean_l, 0.481, eps=0.1) assert approx_equal(result.l_test_mean_l_squared, 0.322, eps=0.1) assert approx_equal(result.i_over_sigma_outer_shell, 10.71, eps=0.01) assert ("indicating pseudo-translationa" in result.patterson_verdict) # check relative Wilson # FIXME #result.relative_wilson.show() #assert (result.relative_wilson.n_outliers() == 0) #show_pickled_object_sizes(result) # # check with completeness_as_non_anomalous=False xtriage_args = [ mtz_file, pdb_file, seq_file, "log=tst_xtriage_1.log", "l_test_dhkl=2,2,2", "completeness_as_non_anomalous=False", ] result = xtriage.run(args=xtriage_args, out=null_out()) test_pickle_consistency_and_size(result) assert (result.matthews.n_copies == 1) assert (str(result.matthews.table) == """\ Solvent content analysis Copies Solvent content Matthews coeff. P(solvent content) 1 0.472 2.33 1.000 """) data_strength = result.data_strength_and_completeness assert approx_equal(data_strength.data_strength.resolution_cut, 1.5351, eps=0.001) out1 = data_strength.low_resolution_completeness.format() assert (out1 == """\ --------------------------------------------------------- | Resolution range | N(obs)/N(possible) | Completeness | --------------------------------------------------------- | 21.9858 - 10.4368 | [ 6/7 ] | 0.857 | | 10.4368 - 8.4369 | [ 3/3 ] | 1.000 | | 8.4369 - 7.4172 | [ 3/4 ] | 0.750 | | 7.4172 - 6.7606 | [ 4/4 ] | 1.000 | | 6.7606 - 6.2882 | [ 8/8 ] | 1.000 | | 6.2882 - 5.9252 | [ 4/5 ] | 0.800 | | 5.9252 - 5.6337 | [11/11] | 1.000 | | 5.6337 - 5.3922 | [ 7/7 ] | 1.000 | | 5.3922 - 5.1874 | [ 6/6 ] | 1.000 | | 5.1874 - 5.0106 | [ 7/7 ] | 1.000 | ---------------------------------------------------------"""), out1 # ANOMALOUS SIGNAL a_meas = result.anomalous_info.measurability #assert approx_equal(a_meas.high_d_cut, 4.7636, eps=0.0001) # Why? assert approx_equal(a_meas.low_d_cut, 2.3565, eps=0.0001) # ABSOLUTE SCALING ws = result.wilson_scaling assert ("%.2f" % ws.iso_p_scale) == "0.65", ws.iso_p_scale assert ("%.2f" % ws.iso_b_wilson) == "14.42", ws.iso_b_wilson # FIXME these may need to be adjusted for different hardware/OS assert approx_equal(ws.aniso_p_scale, 0.64723, eps=0.001) assert approx_equal( ws.aniso_u_star, [0.00034473, 0.00479983, 0.000287162, -0.0, 9.00962e-05, 0.0], 6.e-5) assert approx_equal(ws.aniso_b_cart, [13.12, 16.69, 12.89, 0, -0.08, 0], 0.01) # convenience methods for GUI assert approx_equal(result.aniso_b_min, 12.9, 0.1) assert approx_equal(result.aniso_range_of_b, 3.8, 0.1) # assert approx_equal( ws.outlier_shell_table.data[0], # d_spacing [9.86, 8.36, 4.64], 0.02) assert approx_equal( ws.outlier_shell_table.data[1], # z_score [5.30, 18.06, 5.31], 0.01) assert (len(ws.outliers.acentric_outliers_table.data[0]) == 2) assert (ws.outliers.acentric_outliers_table.data[1] == [(0, -1, -1), (0, 1, 1)]) assert approx_equal(ws.outliers.acentric_outliers_table.data[2], [3.5, 3.3], 0.1) assert (ws.outliers.centric_outliers_table.data is None) assert (len(ws.ice_rings.table._rows) == 10) assert (ws.ice_rings.table._rows[0] == [' 3.897', ' 1.000', ' 0.76', ' 1.00']), \ ws.ice_rings.table._rows[0] tw = result.twin_results wm = tw.wilson_moments out = StringIO() wm.show(out) assert not show_diff( out.getvalue(), """ ----------Wilson ratio and moments---------- Acentric reflections: <I^2>/<I>^2 :2.063 (untwinned: 2.000; perfect twin 1.500) <F>^2/<F^2> :0.778 (untwinned: 0.785; perfect twin 0.885) <|E^2 - 1|> :0.745 (untwinned: 0.736; perfect twin 0.541) Centric reflections: <I^2>/<I>^2 :3.076 (untwinned: 3.000; perfect twin 2.000) <F>^2/<F^2> :0.628 (untwinned: 0.637; perfect twin 0.785) <|E^2 - 1|> :0.999 (untwinned: 0.968; perfect twin 0.736) """) # XXX PDB validation server assert approx_equal(result.iso_b_wilson, 14.51, eps=0.1) assert approx_equal(result.aniso_b_ratio, 0.271, eps=0.1) assert (result.number_of_wilson_outliers == 2) assert approx_equal(result.l_test_mean_l, 0.481, eps=0.1) assert approx_equal(result.l_test_mean_l_squared, 0.322, eps=0.1) assert approx_equal(result.i_over_sigma_outer_shell, 10.71, eps=0.01) assert ("indicating pseudo-translationa" in result.patterson_verdict) # check relative Wilson # FIXME #result.relative_wilson.show() #assert (result.relative_wilson.n_outliers() == 0) #show_pickled_object_sizes(result) # # test without sigmas f_obs_2 = f_obs.customized_copy(sigmas=None) mtz_file = "tst_xtriage_in_2.mtz" f_obs_2.as_mtz_dataset(column_root_label="F").mtz_object().write(mtz_file) xtriage_args = [ mtz_file, pdb_file, seq_file, "log=tst_xtriage_1.log", ] result = xtriage.run(args=xtriage_args, out=null_out()) result.summarize_issues() # test in lower symmetry f_obs_3 = f_obs.expand_to_p1() mtz_file = "tst_xtriage_in_3.mtz" f_obs_3.as_mtz_dataset(column_root_label="F").mtz_object().write(mtz_file) xtriage_args = [ mtz_file, seq_file, "log=tst_xtriage_2.log", ] result = xtriage.run(args=xtriage_args, out=null_out()) assert (( 1, 'One or more symmetry operators suggest that the data has a higher crystallographic symmetry (P 2 1 1).', 'Point group and R-factor analysis') in result.summarize_issues()._issues) # test with elliptical truncation f_obs_3 = f_obs.customized_copy( crystal_symmetry=crystal.symmetry((23, 5, 20, 90, 107.8, 90), "P 21")) f_obs_3 = f_obs_3.resolution_filter(d_min=1.5) f_obs_3 = f_obs_3.customized_copy( crystal_symmetry=f_obs.crystal_symmetry()) reso = ds.analyze_resolution_limits(f_obs_3) out = StringIO() reso.show(out=out) assert ("max. difference between axes = 0.652" in out.getvalue()), \ out.getvalue() assert ("elliptically truncated" in out.getvalue()) # make sure the elliptical truncation detection still works in higher space # groups - we only need a miller.set for this miller_set = miller.build_set(crystal_symmetry=crystal.symmetry( (20, 20, 20, 90, 90, 90), "P422"), d_min=1.5, anomalous_flag=False) reso = ds.analyze_resolution_limits(miller_set) out = StringIO() reso.show(out=out) assert ("Resolution limits are within expected tolerances" in out.getvalue()) # log binning out = StringIO() log_binned = ds.log_binned_completeness(f_obs_3) log_binned.show(out=out) assert ("""| 1.9724 - 1.5094 | 368/1230 | 29.9% |""" in out.getvalue()), out.getvalue() # test with no acentrics cf = f_obs.centric_flags().data() centrics = f_obs.select(cf) acentrics = f_obs.select(~cf) mtz_file = "tst_xtriage_in_3.mtz" centrics.as_mtz_dataset(column_root_label="F").mtz_object().write(mtz_file) args = [ mtz_file, pdb_file, seq_file, "log=tst_xtriage_3.log", ] try: xtriage.run(args=args, out=null_out()) except Sorry: pass else: raise Exception_expected # with only a handful of acentrics sel = flex.bool(acentrics.size(), False) for i in range(10): sel[i] = True f_obs_4 = centrics.concatenate(acentrics.select(sel)) f_obs_4.as_mtz_dataset(column_root_label="F").mtz_object().write(mtz_file) try: xtriage.run(args=args, out=null_out()) except Sorry: pass else: raise Exception_expected
def forward_gpu(self, inputs): x, W = inputs[:2] b = inputs[2] if len(inputs) == 3 else None out_c, _, kh, kw = W.shape n, c, h, w = x.shape out_h = conv.get_conv_outsize(h, kh, self.sy, self.ph, cover_all=self.cover_all) out_w = conv.get_conv_outsize(w, kw, self.sx, self.pw, cover_all=self.cover_all) y = cuda.cupy.empty((n, out_c, out_h, out_w), dtype=x.dtype) if (not self.cover_all and cuda.cudnn_enabled and self.use_cudnn and _check_cudnn_acceptable_type(x.dtype, W.dtype)): x = cuda.cupy.ascontiguousarray(x) W = cuda.cupy.ascontiguousarray(W) if b is not None: b = cuda.cupy.ascontiguousarray(b) handle = cudnn.get_handle() x_desc = cudnn.create_tensor_descriptor(x) y_desc = cudnn.create_tensor_descriptor(y) self.filter_desc = cudnn.create_filter_descriptor(W) self.conv_desc = cudnn.create_convolution_descriptor( (self.ph, self.pw), (self.sy, self.sx)) if b is not None: self.bias_desc = cudnn.create_tensor_descriptor( b[None, :, None, None]) workspace_size = cuda.get_max_workspace_size() workspace = cuda.cupy.empty((workspace_size,), dtype='b') algo = libcudnn.getConvolutionForwardAlgorithm( handle, x_desc.value, self.filter_desc.value, self.conv_desc.value, y_desc.value, _fwd_pref, workspace_size) oz_dtype = 'd' if x.dtype == 'd' else 'f' one = numpy.array(1, dtype=oz_dtype).ctypes zero = numpy.array(0, dtype=oz_dtype).ctypes libcudnn.convolutionForward( handle, one.data, x_desc.value, x.data.ptr, self.filter_desc.value, W.data.ptr, self.conv_desc.value, algo, workspace.data.ptr, workspace_size, zero.data, y_desc.value, y.data.ptr) # TODO(beam2d): Support unshared bias if b is not None: cudnn.add_tensor( handle, one.data, self.bias_desc.value, b.data.ptr, one.data, y_desc.value, y.data.ptr) else: # Implementation using im2col self.col = conv.im2col_gpu( x, kh, kw, self.sy, self.sx, self.ph, self.pw, cover_all=self.cover_all) W_mat = W.reshape(out_c, -1) col_mats = self.col.reshape(n, -1, out_h * out_w) y_mats = y.reshape(n, out_c, -1) # TODO(beam2d): Use streams or batch gemm for i in moves.range(n): y_mats[i] = W_mat.dot(col_mats[i]) # TODO(beam2d): Support unshared bias if b is not None: y += b[:, None, None] return y,
# get our mnist data, and force it to be of shape (..., 1, 28, 28) with # range [-1, 1] (X_train, y_train), (X_test, y_test) = mnist.load_data() X_train = (X_train.astype(np.float32) - 127.5) / 127.5 X_train = np.expand_dims(X_train, axis=1) X_test = (X_test.astype(np.float32) - 127.5) / 127.5 X_test = np.expand_dims(X_test, axis=1) nb_train, nb_test = X_train.shape[0], X_test.shape[0] train_history = defaultdict(list) test_history = defaultdict(list) for epoch in range(nb_epochs): # print 'Epoch {} of {}'.format(epoch + 1, nb_epochs) nb_batches = int(X_train.shape[0] / batch_size) progress_bar = Progbar(target=nb_batches) epoch_gen_loss = [] epoch_disc_loss = [] for index in range(nb_batches): progress_bar.update(index) # generate a new batch of noise noise = np.random.uniform(-1, 1, (batch_size, latent_size)) # get a batch of real images image_batch = X_train[index * batch_size:(index + 1) * batch_size]
def _random_strings(self, count): return [self._random_string() for i in range(count)]
from .Widget import Widget from .widgets import importWidgets, importSingleWidget from six.moves import range from six import PY2 SIBbase__init__ = None SIB_StartOnlyOneTime = False SIB_TOGGLE_SHOW = InfoBar.toggleShow SIB_SWOFF = InfoBar.hide SIB_STATE = -1 config.plugins.Widgets = ConfigSubsection() config.plugins.Widgets.show_empty_positions = ConfigBoolean(default=True, descriptions={False: _("hide"), True: _("show")}) config.plugins.Widgets.active_widgets = ConfigSubDict() for x in range(0, 16): for y in range(0, 16): config.plugins.Widgets.active_widgets["w%i_%i" % (x, y)] = ConfigText("") def Plugins(**kwargs): return [PluginDescriptor(where=PluginDescriptor.WHERE_SESSIONSTART, fnc=SIBautostart)] class ReplaceInfoBar(): def __init__(self): pass @cached def Replace(self): return True
import six.moves as sm from gem import matrix from gem import vector # Some vector operations examples vectorA = vector.Vector(3, data=[1, 2, 3]) vectorB = vector.Vector(3, data=[4, 5, 6]) vectorC = vectorA + vectorB print("Vector C output:" , vectorC.vector) vectorD = vectorA * 2 print("Vector A * 2:", vectorD.vector) print("Vector A magnitude:", vectorA.magnitude()) # Some matrix operations examples matrixA = matrix.Matrix(4, data=[[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12], [13, 14, 15, 16]]) matrixB = matrix.Matrix(4, data=[[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12], [13, 14, 15, 16]]) matrixC = matrixA * matrixB print("Matrix C output:") for i in sm.range(matrixC.size): print(matrixC.matrix[i]) print("End of Matrix C output")
def test_list_revoked_tokens_for_multiple_tokens(self): self.check_list_revoked_tokens([self.delete_token() for x in range(2)])
def selectionHide(self): if self.selection_x in range(1, self.num_widgets_x + 1) and self.selection_y in range(1, self.num_widgets_y + 1): self["w%i_%i_h" % (self.selection_x, self.selection_y)].hide()
class refinement(refinement_base): grid = range(-20, 20) def __init__(OO, self, use_inverse_beam=False, mosaic_refinement_target="LSQ", pvr_fix=True): OO.mosaic_refinement_target = mosaic_refinement_target # least squares or max-likelihood OO.pvr_fix = pvr_fix refinement_base.__init__(OO, self, use_inverse_beam) def contour_plot_DEPRECATED_DOES_NOT_APPLY_SUBPIXEL_METROLOGY(OO): self = OO.parent # see if I can reproduce the predicted positions pxlsz = self.pixel_size # mm/pixel SIGN = -1. # get a simple rmsd obs vs predicted sumsq = 0. nspot = 0 for pair in self.indexed_pairs: deltax = self.spots[pair["spot"]].ctr_mass_x( ) - self.predicted[pair["pred"]][0] / pxlsz deltay = self.spots[pair["spot"]].ctr_mass_y( ) - self.predicted[pair["pred"]][1] / pxlsz sumsq += deltax * deltax + deltay * deltay nspot += 1 print("RMSD obs vs pred in pixels: %7.2f" % (math.sqrt(sumsq / nspot))) excursi = flex.double() rmsdpos = flex.double() for irotx in OO.grid: rotx = (0.02 * irotx) * math.pi / 180. for iroty in OO.grid: roty = (0.02 * iroty) * math.pi / 180. #Not necessary to apply the 3 offset rotations; they have apparently # been applied already. rotz (0,0,1) is the direct beam effective_orientation = OO.input_orientation.rotate_thru( (1, 0, 0), rotx).rotate_thru((0, 1, 0), roty).rotate_thru( (0, 0, 1), 0.0) OO.ucbp3.set_orientation(effective_orientation) OO.ucbp3.gaussian_fast_slow() mean_position = OO.ucbp3.mean_position print(mean_position) sumsq = 0. nspot = 0 for pair in self.indexed_pairs: deltax = mean_position[nspot][1] - self.predicted[ pair["pred"]][0] / pxlsz deltay = mean_position[nspot][0] - self.predicted[ pair["pred"]][1] / pxlsz sumsq += deltax * deltax + deltay * deltay nspot += 1 print("RMSD markmodel vs rossmanpred in pixels: %7.2f" % (math.sqrt(sumsq / nspot))) #from matplotlib import pyplot as plt #plt.plot([mpos[0] for mpos in mean_position],[mpos[1] for mpos in mean_position],"r+") #plt.plot([self.predicted[pair["pred"]][1]/pxlsz for pair in indexed_pairs], # [self.predicted[pair["pred"]][0]/pxlsz for pair in indexed_pairs], "b.") #plt.show() sumsq = 0. nspot = 0 for pair in self.indexed_pairs: deltax = self.spots[ pair["spot"]].ctr_mass_x() - mean_position[nspot][1] deltay = self.spots[ pair["spot"]].ctr_mass_y() - mean_position[nspot][0] sumsq += deltax * deltax + deltay * deltay nspot += 1 rmsdposition = math.sqrt(sumsq / nspot) rmsdpos.append(rmsdposition) print("RMSD obs vs markmodel in pixels: %8.4f" % (rmsdposition)) excursions = flex.double([ OO.ucbp3.simple_forward_calculation_spot_position( wavelength=OO.central_wavelength_ang, observation_no=obsno).rotax_excursion_rad * 180. / math.pi for obsno in range(len(self.indexed_pairs)) ]) rmsdexc = math.sqrt(flex.mean(excursions * excursions)) excursi.append(rmsdexc) print( "rotx %7.2f roty %7.2f degrees, RMSD excursion %7.3f degrees" % ((0.02 * irotx), (0.02 * iroty), rmsdexc)) return excursi, rmsdpos def per_frame_helper_factory(OO): class per_frame_helper(normal_eqns.non_linear_ls, normal_eqns.non_linear_ls_mixin): def __init__(pfh): super(per_frame_helper, pfh).__init__(n_parameters=2) pfh.x_0 = flex.double((0., 0.)) pfh.restart() def restart(pfh): pfh.x = pfh.x_0.deep_copy() pfh.old_x = None def step_forward(pfh): pfh.old_x = pfh.x.deep_copy() pfh.x += pfh.step() def step_backward(pfh): assert pfh.old_x is not None pfh.x, pfh.old_x = pfh.old_x, None def parameter_vector_norm(pfh): return pfh.x.norm() def build_up(pfh, objective_only=False): if OO.pvr_fix: residuals = pfh.fvec_callable_pvr(pfh.x) else: residuals = pfh.fvec_callable_NOT_USED_AFTER_BUGFIX(pfh.x) pfh.reset() if objective_only: pfh.add_residuals(residuals, weights=None) else: grad_r = pfh.jacobian_callable(pfh.x) jacobian = flex.double( flex.grid(len(OO.parent.indexed_pairs), pfh.n_parameters)) for j, der_r in enumerate(grad_r): jacobian.matrix_paste_column_in_place(der_r, j) pfh.add_equations(residuals, jacobian, weights=None) def fvec_callable_pvr(pfh, current_values): rotx = current_values[0] roty = current_values[1] effective_orientation = OO.input_orientation.rotate_thru( (1, 0, 0), rotx).rotate_thru((0, 1, 0), roty).rotate_thru( (0, 0, 1), 0.0) OO.ucbp3.set_orientation(effective_orientation) pfh.last_set_orientation = effective_orientation OO.ucbp3.gaussian_fast_slow() excursions = flex.double([ OO.ucbp3.simple_forward_calculation_spot_position( wavelength=OO.central_wavelength_ang, observation_no=obsno).rotax_excursion_rad_pvr / (2. * math.pi) for obsno in range(len(OO.parent.indexed_pairs)) ]) degrees = 360. * excursions rmsdexc = math.sqrt(flex.mean(degrees * degrees)) #print "rotx %7.3f roty %7.3f degrees, -PVR excursion %7.3f degrees"%( #(rotx * 180./math.pi),(roty * 180./math.pi), rmsdexc) # Note. Luc Bourhis wants scale to be from 0 to 1. So instead of # returning on scale of degrees, use radians/(2*pi) # The parameters rotx roty are still expressed in radians return excursions def jacobian_callable(pfh, current_values): rotx = current_values[0] roty = current_values[1] from scitbx.matrix import sqr Ai = sqr(OO.input_orientation.reciprocal_matrix()) Rx = col((1, 0, 0)).axis_and_angle_as_r3_rotation_matrix(rotx) Ry = col((0, 1, 0)).axis_and_angle_as_r3_rotation_matrix(roty) Rz = col((0, 0, 1)).axis_and_angle_as_r3_rotation_matrix(0.0) dRx_drotx = col( (1, 0, 0)).axis_and_angle_as_r3_derivative_wrt_angle(rotx) dRy_droty = col( (0, 1, 0)).axis_and_angle_as_r3_derivative_wrt_angle(roty) dA_drotx = Rz * Ry * dRx_drotx * Ai dA_droty = Rz * dRy_droty * Rx * Ai dexc_drotx = [ OO.ucbp3.simple_part_excursion_part_rotxy( wavelength=OO.central_wavelength_ang, observation_no=obsno, dA_drotxy=dA_drotx) for obsno in range(len(OO.parent.indexed_pairs)) ] dexc_droty = [ OO.ucbp3.simple_part_excursion_part_rotxy( wavelength=OO.central_wavelength_ang, observation_no=obsno, dA_drotxy=dA_droty) for obsno in range(len(OO.parent.indexed_pairs)) ] return flex.double(dexc_drotx) / ( 2. * math.pi), flex.double(dexc_droty) / (2. * math.pi) value = per_frame_helper() return value def refine_rotx_roty2(OO, enable_rotational_target=True): helper = OO.per_frame_helper_factory() helper.restart() if enable_rotational_target: print("Trying least squares minimization of excursions", end=' ') from scitbx.lstbx import normal_eqns_solving iterations = normal_eqns_solving.naive_iterations( non_linear_ls=helper, gradient_threshold=1.E-10) results = helper.x print("with %d reflections" % len(OO.parent.indexed_pairs), end=' ') print("result %6.2f degrees" % (results[1] * 180. / math.pi), end=' ') print("result %6.2f degrees" % (results[0] * 180. / math.pi)) if False: # Excursion histogram print("The input mosaicity is %7.3f deg full width" % OO.parent.inputai.getMosaicity()) # final histogram if OO.pvr_fix: final = 360. * helper.fvec_callable_pvr(results) else: final = 360. * helper.fvec_callable_NOT_USED_AFTER_BUGFIX( results) rmsdexc = math.sqrt(flex.mean(final * final)) from matplotlib import pyplot as plt nbins = len(final) // 20 n, bins, patches = plt.hist(final, nbins, normed=0, facecolor="orange", alpha=0.75) plt.xlabel("Rotation on e1 axis, rmsd %7.3f deg" % rmsdexc) plt.title("Histogram of cctbx.xfel misorientation") plt.axis([-0.5, 0.5, 0, 100]) plt.plot([rmsdexc], [18], "b|") plt.show() # Determine optimal mosaicity and domain size model (monochromatic) if OO.pvr_fix: final = 360. * helper.fvec_callable_pvr(results) else: final = 360. * helper.fvec_callable_NOT_USED_AFTER_BUGFIX(results) #Guard against misindexing -- seen in simulated data, with zone nearly perfectly aligned guard_stats = flex.max(final), flex.min(final) if False and REMOVETEST_KILLING_LEGITIMATE_EXCURSIONS( guard_stats[0] > 2.0 or guard_stats[1] < -2.0): raise Exception( "Misindexing diagnosed by meaningless excursion angle (bandpass_gaussian model)" ) print("The mean excursion is %7.3f degrees" % (flex.mean(final))) two_thetas = helper.last_set_orientation.unit_cell().two_theta( OO.reserve_indices, OO.central_wavelength_ang, deg=True) dspacings = helper.last_set_orientation.unit_cell().d( OO.reserve_indices) dspace_sq = dspacings * dspacings excursion_rad = final * math.pi / 180. # First -- try to get a reasonable envelope for the observed excursions. ## minimum of three regions; maximum of 50 measurements in each bin print("fitting parameters on %d spots" % len(excursion_rad)) n_bins = min(max(3, len(excursion_rad) // 25), 50) bin_sz = len(excursion_rad) // n_bins print("nbins", n_bins, "bin_sz", bin_sz) order = flex.sort_permutation(two_thetas) two_thetas_env = flex.double() dspacings_env = flex.double() excursion_rads_env = flex.double() for x in range(0, n_bins): subset = order[x * bin_sz:(x + 1) * bin_sz] two_thetas_env.append(flex.mean(two_thetas.select(subset))) dspacings_env.append(flex.mean(dspacings.select(subset))) excursion_rads_env.append( flex.max(flex.abs(excursion_rad.select(subset)))) # Second -- parameter fit ## solve the normal equations sum_inv_u_sq = flex.sum(dspacings_env * dspacings_env) sum_inv_u = flex.sum(dspacings_env) sum_te_u = flex.sum(dspacings_env * excursion_rads_env) sum_te = flex.sum(excursion_rads_env) Normal_Mat = sqr( (sum_inv_u_sq, sum_inv_u, sum_inv_u, len(dspacings_env))) Vector = col((sum_te_u, sum_te)) solution = Normal_Mat.inverse() * Vector s_ang = 1. / (2 * solution[0]) print("Best LSQ fit Scheerer domain size is %9.2f ang" % (s_ang)) tan_phi_rad = helper.last_set_orientation.unit_cell().d( OO.reserve_indices) / (2. * s_ang) tan_phi_deg = tan_phi_rad * 180. / math.pi k_degrees = solution[1] * 180. / math.pi print("The LSQ full mosaicity is %8.5f deg; half-mosaicity %9.5f" % (2 * k_degrees, k_degrees)) tan_outer_deg = tan_phi_deg + k_degrees if OO.mosaic_refinement_target == "ML": from xfel.mono_simulation.max_like import minimizer print("input", s_ang, 2. * solution[1] * 180 / math.pi) # coerce the estimates to be positive for max-likelihood lower_limit_domain_size = math.pow( helper.last_set_orientation.unit_cell().volume(), 1. / 3.) * 20 # 10-unit cell block size minimum reasonable domain d_estimate = max(s_ang, lower_limit_domain_size) M = minimizer(d_i=dspacings, psi_i=excursion_rad, eta_rad=abs(2. * solution[1]), Deff=d_estimate) print("output", 1. / M.x[0], M.x[1] * 180. / math.pi) tan_phi_rad_ML = helper.last_set_orientation.unit_cell().d( OO.reserve_indices) / (2. / M.x[0]) tan_phi_deg_ML = tan_phi_rad_ML * 180. / math.pi # bugfix: Need factor of 0.5 because the plot shows half mosaicity (displacement from the center point defined as zero) tan_outer_deg_ML = tan_phi_deg_ML + 0.5 * M.x[1] * 180. / math.pi if OO.parent.horizons_phil.integration.mosaic.enable_polychromatic: # add code here to perform polychromatic modeling. """ get miller indices DONE get model-predicted mono-wavelength centroid S1 vectors back-convert S1vec, with mono-wavelength, to detector-plane position, factoring in subpixel correction compare with spot centroid measured position compare with locus of bodypixels """ print(list(OO.reserve_indices)) print(len(OO.reserve_indices), len(two_thetas)) positions = [ OO.ucbp3.simple_forward_calculation_spot_position( wavelength=OO.central_wavelength_ang, observation_no=obsno).position for obsno in range(len(OO.parent.indexed_pairs)) ] print(len(positions)) print(positions) # model-predicted positions print(len(OO.parent.spots)) print(OO.parent.indexed_pairs) print(OO.parent.spots) print(len(OO.parent.spots)) meas_spots = [ OO.parent.spots[pair["spot"]] for pair in OO.parent.indexed_pairs ] # for xspot in meas_spots: # xspot.ctr_mass_x(),xspot.ctr_mass_y() # xspot.max_pxl_x() # xspot.bodypixels # xspot.ctr_mass_x() # Do some work to calculate an rmsd diff_vecs = flex.vec3_double() for p, xspot in zip(positions, meas_spots): diff_vecs.append((p[0] - xspot.ctr_mass_y(), p[1] - xspot.ctr_mass_x(), 0.0)) # could use diff_vecs.rms_length() diff_vecs_sq = diff_vecs.dot(diff_vecs) mean_diff_vec_sq = flex.mean(diff_vecs_sq) rmsd = math.sqrt(mean_diff_vec_sq) print("mean obs-pred diff vec on %d spots is %6.2f pixels" % (len(positions), rmsd)) positions_to_fictitious = [ OO.ucbp3.simple_forward_calculation_spot_position( wavelength=OO.central_wavelength_ang, observation_no=obsno).position_to_fictitious for obsno in range(len(OO.parent.indexed_pairs)) ] # Do some work to calculate an rmsd diff_vecs = flex.vec3_double() for p, xspot in zip(positions_to_fictitious, meas_spots): diff_vecs.append((p[0] - xspot.ctr_mass_y(), p[1] - xspot.ctr_mass_x(), 0.0)) rmsd = diff_vecs.rms_length() print( "mean obs-pred_to_fictitious diff vec on %d spots is %6.2f pixels" % (len(positions), rmsd)) """ actually, it might be better if the entire set of experimental observations is transformed into the ideal detector plane, for the purposes of poly_treatment. start here. Now it would be good to actually implement probability of observing a body pixel given the model. We have everything needed right here. """ if OO.parent.horizons_phil.integration.mosaic.enable_AD14F7B: # Image plot: obs and predicted positions + bodypixels from matplotlib import pyplot as plt plt.plot([p[0] for p in positions_to_fictitious], [p[1] for p in positions_to_fictitious], "r.") plt.plot([xspot.ctr_mass_y() for xspot in meas_spots], [xspot.ctr_mass_x() for xspot in meas_spots], "g.") bodypx = [] for xspot in meas_spots: for body in xspot.bodypixels: bodypx.append(body) plt.plot([b.y for b in bodypx], [b.x for b in bodypx], "b.") plt.axes().set_aspect("equal") plt.show() print("MEAN excursion", flex.mean(final), end=' ') if OO.mosaic_refinement_target == "ML": print("mosaicity deg FW=", M.x[1] * 180. / math.pi) else: print() if OO.parent.horizons_phil.integration.mosaic.enable_AD14F7B: # Excursion vs resolution fit AD1TF7B_MAX2T = 30. AD1TF7B_MAXDP = 1. from matplotlib import pyplot as plt fig = plt.figure() plt.plot(two_thetas, final, "bo") mean = flex.mean(final) minplot = flex.min(two_thetas) plt.plot([0, minplot], [mean, mean], "k-") LR = flex.linear_regression(two_thetas, final) #LR.show_summary() model_y = LR.slope() * two_thetas + LR.y_intercept() plt.plot(two_thetas, model_y, "k-") print(helper.last_set_orientation.unit_cell()) #for sdp,tw in zip (dspacings,two_thetas): #print sdp,tw if OO.mosaic_refinement_target == "ML": plt.title( "ML: mosaicity FW=%4.2f deg, Dsize=%5.0fA on %d spots" % (M.x[1] * 180. / math.pi, 2. / M.x[0], len(two_thetas))) plt.plot(two_thetas, tan_phi_deg_ML, "r.") plt.plot(two_thetas, -tan_phi_deg_ML, "r.") plt.plot(two_thetas, tan_outer_deg_ML, "g.") plt.plot(two_thetas, -tan_outer_deg_ML, "g.") else: plt.plot(two_thetas_env, excursion_rads_env * 180. / math.pi, "r|") plt.plot(two_thetas_env, -excursion_rads_env * 180. / math.pi, "r|") plt.plot(two_thetas_env, excursion_rads_env * 180. / math.pi, "r-") plt.plot(two_thetas_env, -excursion_rads_env * 180. / math.pi, "r-") plt.plot(two_thetas, tan_phi_deg, "r.") plt.plot(two_thetas, -tan_phi_deg, "r.") plt.plot(two_thetas, tan_outer_deg, "g.") plt.plot(two_thetas, -tan_outer_deg, "g.") plt.xlim([0, AD1TF7B_MAX2T]) plt.ylim([-AD1TF7B_MAXDP, AD1TF7B_MAXDP]) OO.parent.show_figure(plt, fig, "psi") plt.close() from xfel.mono_simulation.util import green_curve_area, ewald_proximal_volume if OO.mosaic_refinement_target == "ML": OO.parent.green_curve_area = green_curve_area( two_thetas, tan_outer_deg_ML) OO.parent.inputai.setMosaicity(M.x[1] * 180. / math.pi) # full width, degrees OO.parent.ML_half_mosaicity_deg = M.x[1] * 180. / (2. * math.pi) OO.parent.ML_domain_size_ang = 1. / M.x[0] OO.parent.ewald_proximal_volume = ewald_proximal_volume( wavelength_ang=OO.central_wavelength_ang, resolution_cutoff_ang=OO.parent.horizons_phil.integration. mosaic.ewald_proximal_volume_resolution_cutoff, domain_size_ang=1. / M.x[0], full_mosaicity_rad=M.x[1]) return results, helper.last_set_orientation, 1. / M.x[ 0] # full width domain size, angstroms else: assert OO.mosaic_refinement_target == "LSQ" OO.parent.green_curve_area = green_curve_area( two_thetas, tan_outer_deg) OO.parent.inputai.setMosaicity(2 * k_degrees) # full width OO.parent.ML_half_mosaicity_deg = k_degrees OO.parent.ML_domain_size_ang = s_ang OO.parent.ewald_proximal_volume = ewald_proximal_volume( wavelength_ang=OO.central_wavelength_ang, resolution_cutoff_ang=OO.parent.horizons_phil.integration. mosaic.ewald_proximal_volume_resolution_cutoff, domain_size_ang=s_ang, full_mosaicity_rad=2 * k_degrees * math.pi / 180.) return results, helper.last_set_orientation, s_ang # full width domain size, angstroms def show_plot(OO, excursi, rmsdpos, minimum): excursi.reshape(flex.grid(len(OO.grid), len(OO.grid))) rmsdpos.reshape(flex.grid(len(OO.grid), len(OO.grid))) from matplotlib import pyplot as plt plt.figure() CS = plt.contour([i * 0.02 for i in OO.grid], [i * 0.02 for i in OO.grid], excursi.as_numpy_array()) plt.clabel(CS, inline=1, fontsize=10, fmt="%6.3f" + chr(176)) plt.plot([minimum[1] * 180. / math.pi], [minimum[0] * 180. / math.pi], "r+") plt.title("Rms rotational excursion to reflection condition, degrees") plt.axes().set_aspect("equal") plt.figure() CS = plt.contour([i * 0.02 for i in OO.grid], [i * 0.02 for i in OO.grid], rmsdpos.as_numpy_array()) plt.clabel(CS, inline=1, fontsize=10, fmt="%7.4f px") plt.title("Rms position shift, obs vs. model, pixels") plt.axes().set_aspect("equal") plt.show()
from deeptictactoe.game import Game, Randy from deeptictactoe.q_approximator import NeuralNetwork from deeptictactoe.rl_agent import QAgent q = QAgent(approximator=NeuralNetwork(checkpoint=40), exploration_rate=0.) r = Randy() # evaluate the agent by letting it play against itself game = Game(q, q) result = game.play()['result'] print("Result Q-agent vs Q-agent:", result) # let the agent play against a Randy wins = 0 for i in range(1000): if i % 100 == 0: print("Simulating round {}".format(i)) game = Game(q, r) result = game.play()['result'] if result == 1: wins += 1 print("\nWins Q-agent vs Randy: {}".format(wins)) wins = 0 for i in range(1000): if i % 100 == 0: print("Simulating round {}".format(i)) game = Game(r, q) result = game.play()['result'] if result == 1: