def merge_with(fn, *dicts): """ Merge dictionaries and apply function to combined values A key may occur in more than one dict, and all values mapped from the key will be passed to the function as a list, such as fn([val1, val2, ...]). >>> merge_with(sum, {1: 1, 2: 2}, {1: 10, 2: 20}) {1: 11, 2: 22} >>> merge_with(first, {1: 1, 2: 2}, {2: 20, 3: 30}) # doctest: +SKIP {1: 1, 2: 2, 3: 30} See Also: merge """ if len(dicts) == 1 and not isinstance(dicts[0], dict): dicts = dicts[0] result = dict() for d in dicts: for k, v in d.items(): try: result[k].append(v) except: result[k] = [v] return dict((k, fn(v)) for k, v in result.items())
def __init__(self, session: Session, config, loop=None): self.logger = logging.getLogger(__name__) self.session = session self.config = config if loop is None: self._loop = asyncio.get_event_loop() else: self._loop = loop self._reader_task = None self._writer_task = None self._inflight_task = None self._reader_ready = asyncio.Event(loop=self._loop) self._writer_ready = asyncio.Event(loop=self._loop) self._inflight_ready = asyncio.Event(loop=self._loop) self._inflight_changed = asyncio.Condition(loop=self._loop) self._running = False self.session.local_address, self.session.local_port = self.session.writer.get_extra_info("sockname") self.incoming_queues = dict() for p in PacketType: self.incoming_queues[p] = asyncio.Queue() self.outgoing_queue = asyncio.Queue() self.inflight_messages = dict()
def test_bad_revparse(self): self.setupStep( bzr.Bzr(repourl='http://bzr.squid-cache.org/bzr/squid3/trunk', mode='incremental')) self.expectCommands( ExpectShell(workdir='wkdir', command=['bzr', '--version']) + 0, Expect('stat', dict(file='wkdir/.buildbot-patched', logEnviron=True)) + 1, Expect('stat', dict(file='wkdir/.bzr', logEnviron=True)) + 1, ExpectShell(workdir='wkdir', command=['bzr', 'checkout', 'http://bzr.squid-cache.org/bzr/squid3/trunk', '.']) + 0, ExpectShell(workdir='wkdir', command=['bzr', 'version-info', '--custom', "--template='{revno}"]) + ExpectShell.log('stdio', stdout='oiasdfj010laksjfd') + 0, ) self.expectOutcome(result=FAILURE, status_text=["updating"]) return self.runStep()
def _get_store_model(self, imsgstore): dct = {} props = dict( size=tags.PR_MESSAGE_SIZE_EXTENDED, guid=tags.PR_RECORD_KEY, lastLogon=tags.PR_LAST_LOGON_TIME, userEntryId=tags.PR_MAILBOX_OWNER_ENTRYID, ) storemodel = MapiObject(imsgstore, props) dct.update(dict( lastLogon=storemodel.lastLogon, size=storemodel.size, zarafaId=binguid_to_hexstr(storemodel.guid), )) if storemodel.userEntryId: imailuser = self.session.OpenEntry(storemodel.userEntryId, None, 0) usermodel = MapiObject(imailuser, dict(userid=tags.PR_ACCOUNT_W)) dct.update(dict( storeName=usermodel.userid, )) return Store(initial=dct)
def test_mode_full_timeout(self): self.setupStep( bzr.Bzr(repourl='http://bzr.squid-cache.org/bzr/squid3/trunk', mode='full', method='fresh', timeout=1)) self.expectCommands( ExpectShell(workdir='wkdir', timeout=1, command=['bzr', '--version']) + 0, Expect('stat', dict(file='wkdir/.buildbot-patched', logEnviron=True)) + 1, Expect('stat', dict(file='wkdir/.bzr', logEnviron=True)) + 0, ExpectShell(workdir='wkdir', timeout=1, command=['bzr', 'clean-tree', '--force']) + 0, ExpectShell(workdir='wkdir', timeout=1, command=['bzr', 'update']) + 0, ExpectShell(workdir='wkdir', timeout=1, command=['bzr', 'version-info', '--custom', "--template='{revno}"]) + ExpectShell.log('stdio', stdout='100') + 0, ) self.expectOutcome(result=SUCCESS, status_text=["update"]) self.expectProperty('got_revision', '100', 'Bzr') return self.runStep()
def delete_plugin(): """ Object delete handler """ app=request.args(0) plugin = request.args(1) plugin_name='plugin_'+plugin if 'nodelete' in request.vars: redirect(URL('design',args=app)) elif 'delete' in request.vars: try: for folder in ['models','views','controllers','static','modules']: path=os.path.join(apath(app,r=request),folder) for item in os.listdir(path): if item.rsplit('.',1)[0] == plugin_name: filename=os.path.join(path,item) if os.path.isdir(filename): shutil.rmtree(filename) else: os.unlink(filename) session.flash = T('plugin "%(plugin)s" deleted', dict(plugin=plugin)) except Exception: session.flash = T('unable to delete file plugin "%(plugin)s"', dict(plugin=plugin)) redirect(URL('design',args=request.args(0))) return dict(plugin=plugin)
def create_filetree(path=None, depth=0, max_depth=0): tree = None if max_depth == 0 or depth < max_depth: if path is None: path = os.getcwd() tree = dict(name=os.path.basename(path), children=[]) try: lst = os.listdir(path) except OSError: pass # ignore errors else: for name in lst: fn = os.path.join(path, name) if (os.path.isdir(fn) and re.match('^.*(Compiled)$', fn) is None): child = create_filetree(fn, depth + 1, max_depth) if child is not None: tree['children'].append(child) elif re.match('^.*\.(m|def|txt|csv)$', fn) is not None: tree['children'].append(dict(name=fn.replace( os.getcwd() + os.path.sep, ""))) return tree
def user(): if MULTI_USER_MODE: if not db(db.auth_user).count(): auth.settings.registration_requires_approval = False return dict(form=auth()) else: return dict(form=T("Disabled"))
def site(): """ Site handler """ myversion = request.env.web2py_version # Shortcut to make the elif statements more legible file_or_appurl = 'file' in request.vars or 'appurl' in request.vars if DEMO_MODE: pass elif request.vars.filename and not 'file' in request.vars: # create a new application appname = cleanpath(request.vars.filename).replace('.', '_') if app_create(appname, request): if MULTI_USER_MODE: db.app.insert(name=appname,owner=auth.user.id) session.flash = T('new application "%s" created', appname) redirect(URL('design',args=appname)) else: session.flash = \ T('unable to create application "%s" (it may exist already)', request.vars.filename) redirect(URL(r=request)) elif file_or_appurl and not request.vars.filename: # can't do anything without an app name msg = 'you must specify a name for the uploaded application' response.flash = T(msg) elif file_or_appurl and request.vars.filename: # fetch an application via URL or file upload f = None if request.vars.appurl is not '': try: f = urllib.urlopen(request.vars.appurl) except Exception, e: session.flash = DIV(T('Unable to download app because:'),PRE(str(e))) redirect(URL(r=request)) fname = request.vars.appurl elif request.vars.file is not '': f = request.vars.file.file fname = request.vars.file.filename if f: appname = cleanpath(request.vars.filename).replace('.', '_') installed = app_install(appname, f, request, fname, overwrite=request.vars.overwrite_check) if f and installed: msg = 'application %(appname)s installed with md5sum: %(digest)s' session.flash = T(msg, dict(appname=appname, digest=md5_hash(installed))) elif f and request.vars.overwrite_check: msg = 'unable to install application "%(appname)s"' session.flash = T(msg, dict(appname=request.vars.filename)) else: msg = 'unable to install application "%(appname)s"' session.flash = T(msg, dict(appname=request.vars.filename)) redirect(URL(r=request))
def main_interface(): cur = g.db.execute('select id, provider, num_affected, end_time, flagged from outages where start_time<? and (end_time is null or end_time>? ) and hidden==0 order by start_time desc', [datetime.datetime.now(), datetime.datetime.now()]) outages = [dict(id=row[0], provider=row[1], num_affected=row[2], end_time=row[3], flagged=row[4]) for row in cur.fetchall()] for outage in outages: cur = g.db.execute('select provider_ref_id from outage_texts where rowid==?', [outage['id']]) outage['provider_ref'] = cur.fetchone() cur = g.db.execute('select dslusers_id from outages_dslusers_rel where outages_id==?', [outage['id']]) dsluser_ids = '(' for row in cur.fetchall(): dsluser_ids += str(row[0]) + ', ' dsluser_ids = dsluser_ids.rstrip().rstrip(',') dsluser_ids += ')' cur = g.db.execute('select id, asid, account_name, phone_number, user_name from dslusers where id IN ' + dsluser_ids) outage['users'] = [dict(id=row[0], asid=row[1], account_name=row[2], phone_number=row[3], user_name=row[4]) for row in cur.fetchall()] cur = g.db.execute('select count(*) from outages_dslusers_rel where outages_id==?', [outage['id']]) outage['our_affected'] = cur.fetchone()[0] return render_template('base.html', queues=queues, notes=notes, outages=outages)
def _sourcedirIsUpdatable(self): myFileWriter = StringFileWriter() args = { 'workdir': self.build.path_module.join(self.workdir, 'CVS'), 'writer': myFileWriter, 'maxsize': None, 'blocksize': 32*1024, } cmd = buildstep.RemoteCommand('uploadFile', dict(slavesrc='Root', **args), ignore_updates=True) yield self.runCommand(cmd) if cmd.rc is not None and cmd.rc != 0: defer.returnValue(False) return if myFileWriter.buffer.strip() != self.cvsroot: defer.returnValue(False) return myFileWriter.buffer = "" cmd = buildstep.RemoteCommand('uploadFile', dict(slavesrc='Repository', **args), ignore_updates=True) yield self.runCommand(cmd) if cmd.rc is not None and cmd.rc != 0: defer.returnValue(False) return if myFileWriter.buffer.strip() != self.cvsmodule: defer.returnValue(False) return defer.returnValue(True)
def testCleanUp(self): class TestableTest(unittest2.TestCase): def testNothing(self): pass test = TestableTest('testNothing') self.assertEqual(test._cleanups, []) cleanups = [] def cleanup1(*args, **kwargs): cleanups.append((1, args, kwargs)) def cleanup2(*args, **kwargs): cleanups.append((2, args, kwargs)) test.addCleanup(cleanup1, 1, 2, 3, four='hello', five='goodbye') test.addCleanup(cleanup2) self.assertEqual(test._cleanups, [(cleanup1, (1, 2, 3), dict(four='hello', five='goodbye')), (cleanup2, (), {})]) result = test.doCleanups() self.assertTrue(result) self.assertEqual(cleanups, [(2, (), {}), (1, (1, 2, 3), dict(four='hello', five='goodbye'))])
def check(cmd, mf): m = mf.findNode('PyQt5') if m and not isinstance(m, MissingModule): try: # PyQt5 with sipconfig module, handled # by sip recipe import sipconfig return None except ImportError: pass # All imports are done from C code, hence not visible # for modulegraph # 1. Use of 'sip' # 2. Use of other modules, datafiles and C libraries # in the PyQt5 package. mf.import_hook('sip', m) if sys.version[0] != 2: return dict(packages=['PyQt5'], expected_missing_imports=set(['copy_reg', 'cStringIO', 'StringIO'])) else: return dict(packages=['PyQt5']) return None
def list_available(bitcoind): address_summary = dict() address_to_account = dict() for info in bitcoind.listreceivedbyaddress(0): address_to_account[info["address"]] = info["account"] unspent = bitcoind.listunspent(0) for output in unspent: # listunspent doesn't give addresses, so: rawtx = bitcoind.getrawtransaction(output['txid'], 1) vout = rawtx["vout"][output['vout']] pk = vout["scriptPubKey"] # This code only deals with ordinary pay-to-bitcoin-address # or pay-to-script-hash outputs right now; anything exotic is ignored. if pk["type"] != "pubkeyhash" and pk["type"] != "scripthash": continue address = pk["addresses"][0] if address in address_summary: address_summary[address]["total"] += vout["value"] address_summary[address]["outputs"].append(output) else: address_summary[address] = { "total" : vout["value"], "outputs" : [output], "account" : address_to_account.get(address, "") } return address_summary
def __init__(self, *args, **kwargs): """Constructor to resolve values for all Parameters. For example, the Task:: class MyTask(luigi.Task): count = luigi.IntParameter() can be instantiated as ``MyTask(count=10)``. """ params = self.get_params() param_values = self.get_param_values(params, args, kwargs) # Set all values on class instance for key, value in param_values: setattr(self, key, value) # Register args and kwargs as an attribute on the class. Might be useful self.param_args = tuple(value for key, value in param_values) self.param_kwargs = dict(param_values) # Build up task id task_id_parts = [] param_objs = dict(params) for param_name, param_value in param_values: if dict(params)[param_name].significant: task_id_parts.append('%s=%s' % (param_name, param_objs[param_name].serialize(param_value))) self.task_id = '%s(%s)' % (self.task_family, ', '.join(task_id_parts)) self.__hash = hash(self.task_id)
def _compute_graph(self): stack=[self.begin] index=dict() rindex=dict() count=Counter() graph=dict() def enter(state): n = count.get() index[state] = n rindex[n] = state enter(self.begin) while stack: I=stack.pop() table=dict() for X in self.grammar.symbols: transition=self.transition(I,X) if transition: table[X]=transition if transition not in index: enter(transition) stack.append(transition) graph[I]=table self._graph=graph self._index=index self._rindex=rindex
def __compute_alternative_params(self): # Copied directly from skopt transformed_bounds = np.array(self.__opt.space.transformed_bounds) est = clone(self.__opt.base_estimator) with warnings.catch_warnings(): warnings.simplefilter("ignore") est.fit(self.__opt.space.transform(self.__opt.Xi), self.__opt.yi) X = self.__opt.space.transform(self.__opt.space.rvs( n_samples=self.__opt.n_points, random_state=self.__opt.rng)) values = _gaussian_acquisition(X=X, model=est, y_opt=np.min(self.__opt.yi), acq_func='EI', acq_func_kwargs=dict(n_points=10000)) print('original point ei: %s' % np.min(values)) discount_width = .5 values = self.__discount_leased_params(X, values, discount_width) while np.min(values) > -1e-5 and discount_width > 1e-2: discount_width *= .9 values = _gaussian_acquisition(X=X, model=est, y_opt=np.min(self.__opt.yi), acq_func='EI', acq_func_kwargs=dict(n_points=10000)) values = self.__discount_leased_params(X, values, discount_width) next_x = X[np.argmin(values)] print('new point ei: %s' % np.min(values)) if not self.__opt.space.is_categorical: next_x = np.clip(next_x, transformed_bounds[:, 0], transformed_bounds[:, 1]) return self.__opt.space.inverse_transform(next_x.reshape((1, -1)))[0]
def edit_wiki_settings(node, auth, **kwargs): wiki_settings = node.get_addon('wiki') permissions = request.get_json().get('permission', None) if not wiki_settings: raise HTTPError(http.BAD_REQUEST, data=dict( message_short='Invalid request', message_long='Cannot change wiki settings without a wiki' )) if permissions == 'public': permissions = True elif permissions == 'private': permissions = False else: raise HTTPError(http.BAD_REQUEST, data=dict( message_short='Invalid request', message_long='Permissions flag used is incorrect.' )) try: wiki_settings.set_editing(permissions, auth, log=True) except NodeStateError as e: raise HTTPError(http.BAD_REQUEST, data=dict( message_short="Can't change privacy", message_long=e.message )) return { 'status': 'success', 'permissions': permissions, }
def test_height(self): prefix='thumb_' meta = dict(thumbnails=[dict(height=50, prefix=prefix, include=['*.jpg'])]) self._test_generic_thumbnails(meta) for fn in IMAGES: im = Image.open(self._deployed_image(prefix, fn)) assert im.size[1] == 50
def test_failed_overwrite(): table = Table.create('messages', schema=[ HashKey('id'), ], throughput={ 'read': 7, 'write': 3, }) data1 = {'id': '123', 'data': '678'} table.put_item(data=data1) data2 = {'id': '123', 'data': '345'} table.put_item(data=data2, overwrite=True) data3 = {'id': '123', 'data': '812'} table.put_item.when.called_with(data=data3).should.throw(ConditionalCheckFailedException) returned_item = table.lookup('123') dict(returned_item).should.equal(data2) data4 = {'id': '124', 'data': 812} table.put_item(data=data4) returned_item = table.lookup('124') dict(returned_item).should.equal(data4)
def test_item_add_and_describe_and_update(): table = create_table() data = { 'forum_name': 'LOLCat Forum', 'Body': 'http://url_to_lolcat.gif', 'SentBy': 'User A', } table.put_item(data=data) returned_item = table.get_item(forum_name="LOLCat Forum") returned_item.should_not.be.none dict(returned_item).should.equal({ 'forum_name': 'LOLCat Forum', 'Body': 'http://url_to_lolcat.gif', 'SentBy': 'User A', }) returned_item['SentBy'] = 'User B' returned_item.save(overwrite=True) returned_item = table.get_item( forum_name='LOLCat Forum' ) dict(returned_item).should.equal({ 'forum_name': 'LOLCat Forum', 'Body': 'http://url_to_lolcat.gif', 'SentBy': 'User B', })
def when_i_post_put_to_url_with_params(context, name, password,method, url, params = SKIP, admin = False, content_type = "application/x-www-form-urlencoded"): full_url = "%s%s" % (context.app_config.app_url_prefix,url) if admin is False else "%s%s" % (context.app_config.admin_url_prefix,url) h = httplib2.Http() h.add_credentials(name,password) # convert string params to dictionary form_params = {} if params is not SKIP: form_params = dict(urlparse.parse_qsl(params)) # handle both content = params.encode("utf-8") if content_type is "application/x-www-form-urlencoded": content = urllib.urlencode(form_params) if method in {"PUT","POST","PATCH"}: context.http_headers["content-type"] = content_type # encode http headers context.http_headers = dict((k.encode('ascii') if isinstance(k, unicode) else k, v.encode('ascii') if isinstance(v, unicode) else v) for k,v in context.http_headers.items()) context.resp, context.content = h.request(full_url.encode("utf-8"), method.encode("utf-8"), content, headers=context.http_headers) context.http_headers = {}
def close(self): """ Shut down the UnitManager, and all umgr components. """ # we do not cancel units at this point, in case any component or pilot # wants to continue to progress unit states, which should indeed be # independent from the umgr life cycle. if self._closed: return self._terminate.set() self.stop() self._rep.info('<<close unit manager') # we don't want any callback invokations during shutdown # FIXME: really? with self._cb_lock: self._callbacks = dict() for m in rpt.UMGR_METRICS: self._callbacks[m] = dict() self._log.info("Closed UnitManager %s." % self._uid) self._closed = True self._rep.ok('>>ok\n')
def test_existing_lcd_partial(self): commit1 = self._add_commit('Commit 1', ['file1']) commit2 = self._add_commit('Commit 2', ['file1', 'file2'], ['file2'], [commit1]) commit3 = self._add_commit('Commit 3', ['file1', 'file2', 'file3'], ['file3'], [commit2]) commit4 = self._add_commit('Commit 4', ['file1', 'file2', 'file3', 'file4'], ['file2', 'file4'], [commit3]) prev_lcd = M.repository.LastCommit( path='', commit_id=commit3._id, entries=[ dict( name='file1', commit_id=commit1._id), dict( name='file2', commit_id=commit2._id), dict( name='file3', commit_id=commit3._id), ], ) session(prev_lcd).flush() lcd = M.repository.LastCommit.get(commit4.tree) self.assertEqual(self.repo._commits[lcd.commit_id].message, commit4.message) self.assertEqual(lcd.path, '') self.assertEqual(len(lcd.entries), 4) self.assertEqual(lcd.by_name['file1'], commit1._id) self.assertEqual(lcd.by_name['file2'], commit4._id) self.assertEqual(lcd.by_name['file3'], commit3._id) self.assertEqual(lcd.by_name['file4'], commit4._id)
def testGetSetProperties(self): self.addEngine(4) dikt = dict(a=5, b='asdf', c=True, d=None, e=list(range(5))) d= self.multiengine.set_properties(dikt) d.addCallback(lambda r: self.multiengine.get_properties()) d.addCallback(lambda r: self.assertEquals(r, 4*[dikt])) d.addCallback(lambda r: self.multiengine.get_properties(('c',))) d.addCallback(lambda r: self.assertEquals(r, 4*[{'c': dikt['c']}])) d.addCallback(lambda r: self.multiengine.set_properties(dict(c=False))) d.addCallback(lambda r: self.multiengine.get_properties(('c', 'd'))) d.addCallback(lambda r: self.assertEquals(r, 4*[dict(c=False, d=None)])) #Non-blocking d.addCallback(lambda r: self.multiengine.set_properties(dikt, block=False)) d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True)) d.addCallback(lambda r: self.multiengine.get_properties(block=False)) d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True)) d.addCallback(lambda r: self.assertEquals(r, 4*[dikt])) d.addCallback(lambda r: self.multiengine.get_properties(('c',), block=False)) d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True)) d.addCallback(lambda r: self.assertEquals(r, 4*[{'c': dikt['c']}])) d.addCallback(lambda r: self.multiengine.set_properties(dict(c=False), block=False)) d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True)) d.addCallback(lambda r: self.multiengine.get_properties(('c', 'd'), block=False)) d.addCallback(lambda did: self.multiengine.get_pending_deferred(did, True)) d.addCallback(lambda r: self.assertEquals(r, 4*[dict(c=False, d=None)])) return d
def save(self, request, datastream_rev=None, visualization_rev=None): if datastream_rev: lifecycle = VisualizationLifeCycleManager(user=request.user) visualization_rev = lifecycle.create(datastream_rev, language=request.user.language, **self.cleaned_data) return dict( status='ok', revision_id=visualization_rev.id, messages=[ugettext('APP-VISUALIZATION-CREATEDSUCCESSFULLY-TEXT')] ) elif visualization_rev: lifecycle = VisualizationLifeCycleManager( user=request.user, visualization_revision_id=visualization_rev['visualization_revision_id'] ) visualization_rev = lifecycle.edit( language=request.auth_manager.language, changed_fields=self.changed_data, **self.cleaned_data ) return dict( status='ok', revision_id=visualization_rev.id, messages=[ugettext('APP-VISUALIZATION-CREATEDSUCCESSFULLY-TEXT')] )
def get_sortable_columns(self): """ Returns a dictionary of the sortable columns. Key is a model field name and value is sort column (for example - attribute). If `column_sortable_list` is set, will use it. Otherwise, will call `scaffold_sortable_columns` to get them from the model. """ self._sortable_joins = dict() if self.column_sortable_list is None: return self.scaffold_sortable_columns() else: result = dict() for c in self.column_sortable_list: if isinstance(c, tuple): column, path = self._get_field_with_path(c[1]) column_name = c[0] elif isinstance(c, InstrumentedAttribute): column, path = self._get_field_with_path(c) column_name = str(c) else: column, path = self._get_field_with_path(c) column_name = c result[column_name] = column if path: self._sortable_joins[column_name] = path return result
def test_ignore_multiple_matches(self): self.assertEqual( self.interpret_bootstrap_stderr([ dict( action_num=0, node_id='i-b659f519', path=('s3://bucket/tmp/logs/j-1EE0CL1O7FDXU/node/' 'i-b659f519/bootstrap-actions/1/stderr.gz'), ), dict( action_num=0, node_id='i-e647eb49', path=('s3://bucket/tmp/logs/j-1EE0CL1O7FDXU/node/' 'i-e647eb49/bootstrap-actions/1/stderr.gz'), ), ]), dict( errors=[dict( action_num=0, node_id='i-b659f519', task_error=dict( message='BOOM!\n', path=('s3://bucket/tmp/logs/j-1EE0CL1O7FDXU/node/' 'i-b659f519/bootstrap-actions/1/stderr.gz'), ), )], partial=True, ) ) self.mock_cat_log.called_once_with( self.mock_fs, ('s3://bucket/tmp/logs/j-1EE0CL1O7FDXU/node/' 'i-b659f519/bootstrap-actions/1/stderr.gz'))
def parse(self, basefile): # Find out possible skeleton entries by loading the entire # graph of resource references, and find resources that only # exist as objects. # # Note: if we used download_from_triplestore we know that this list # is clean -- we could just iterate the graph w/o filtering g = Graph() self.log.info("Parsing %s" % basefile) g.parse(self.store.downloaded_path(basefile), format="nt") self.log.info("Compiling object set") # create a uri -> True dict mapping -- maybe? objects = dict(zip([str(o).split("#")[0] for (s, p, o) in g], True)) self.log.info("Compiling subject set") subjects = dict(zip([str(s).split("#")[0] for (s, p, o) in g], True)) self.log.info("%s objects, %s subjects. Iterating through existing objects" % (len(objects), len(subjects))) for o in objects: if not o.startswith(self.config.url): continue if '9999:999' in o: continue if o in subjects: continue for repo in otherrepos: skelbase = repo.basefile_from_uri(repo) if skelbase: skel = repo.triples_from_uri(o) # need to impl with self.store.open_distilled(skelbase, "wb") as fp: fp.write(skel.serialize(format="pretty-xml")) self.log.info("Created skel for %s" % o)
def backup(self, context, instance, name, backup_type, rotation, extra_properties=None): self.extra_props_last_call = extra_properties props = dict(backup_type=backup_type, rotation=rotation) props.update(extra_properties or {}) return dict(id='123', status='ACTIVE', name=name, properties=props)
def apply_event(state: Dict[str, Any], event: Dict[str, Any], user_profile: UserProfile, client_gravatar: bool, include_subscribers: bool) -> None: if event['type'] == "message": state['max_message_id'] = max(state['max_message_id'], event['message']['id']) if 'raw_unread_msgs' in state: apply_unread_message_event( user_profile, state['raw_unread_msgs'], event['message'], event['flags'], ) if event['message']['type'] != "stream": if 'raw_recent_private_conversations' in state: # Handle maintaining the recent_private_conversations data structure. conversations = state['raw_recent_private_conversations'] recipient_id = get_recent_conversations_recipient_id( user_profile, event['message']['recipient_id'], event['message']["sender_id"]) if recipient_id not in conversations: conversations[recipient_id] = dict( user_ids=[user_dict['id'] for user_dict in event['message']['display_recipient'] if user_dict['id'] != user_profile.id] ) conversations[recipient_id]['max_message_id'] = event['message']['id'] return # Below, we handle maintaining first_message_id. for sub_dict in state.get('subscriptions', []): if event['message']['stream_id'] == sub_dict['stream_id']: if sub_dict['first_message_id'] is None: sub_dict['first_message_id'] = event['message']['id'] for stream_dict in state.get('streams', []): if event['message']['stream_id'] == stream_dict['stream_id']: if stream_dict['first_message_id'] is None: stream_dict['first_message_id'] = event['message']['id'] elif event['type'] == "hotspots": state['hotspots'] = event['hotspots'] elif event['type'] == "custom_profile_fields": state['custom_profile_fields'] = event['fields'] elif event['type'] == "pointer": state['pointer'] = max(state['pointer'], event['pointer']) elif event['type'] == "realm_user": person = event['person'] person_user_id = person['user_id'] if event['op'] == "add": person = copy.deepcopy(person) if client_gravatar: if 'gravatar.com' in person['avatar_url']: person['avatar_url'] = None person['is_active'] = True if not person['is_bot']: person['profile_data'] = {} state['raw_users'][person_user_id] = person elif event['op'] == "remove": state['raw_users'][person_user_id]['is_active'] = False elif event['op'] == 'update': is_me = (person_user_id == user_profile.id) if is_me: if ('avatar_url' in person and 'avatar_url' in state): state['avatar_source'] = person['avatar_source'] state['avatar_url'] = person['avatar_url'] state['avatar_url_medium'] = person['avatar_url_medium'] for field in ['is_admin', 'delivery_email', 'email', 'full_name']: if field in person and field in state: state[field] = person[field] # In the unlikely event that the current user # just changed to/from being an admin, we need # to add/remove the data on all bots in the # realm. This is ugly and probably better # solved by removing the all-realm-bots data # given to admin users from this flow. if ('is_admin' in person and 'realm_bots' in state): prev_state = state['raw_users'][user_profile.id] was_admin = prev_state['is_admin'] now_admin = person['is_admin'] if was_admin and not now_admin: state['realm_bots'] = [] if not was_admin and now_admin: state['realm_bots'] = get_owned_bot_dicts(user_profile) if client_gravatar and 'avatar_url' in person: # Respect the client_gravatar setting in the `users` data. if 'gravatar.com' in person['avatar_url']: person['avatar_url'] = None person['avatar_url_medium'] = None if person_user_id in state['raw_users']: p = state['raw_users'][person_user_id] for field in p: if field in person: p[field] = person[field] if 'custom_profile_field' in person: custom_field_id = person['custom_profile_field']['id'] custom_field_new_value = person['custom_profile_field']['value'] if 'rendered_value' in person['custom_profile_field']: p['profile_data'][custom_field_id] = { 'value': custom_field_new_value, 'rendered_value': person['custom_profile_field']['rendered_value'] } else: p['profile_data'][custom_field_id] = { 'value': custom_field_new_value } elif event['type'] == 'realm_bot': if event['op'] == 'add': state['realm_bots'].append(event['bot']) if event['op'] == 'remove': email = event['bot']['email'] for bot in state['realm_bots']: if bot['email'] == email: bot['is_active'] = False if event['op'] == 'delete': state['realm_bots'] = [item for item in state['realm_bots'] if item['email'] != event['bot']['email']] if event['op'] == 'update': for bot in state['realm_bots']: if bot['email'] == event['bot']['email']: if 'owner_id' in event['bot']: bot['owner'] = get_user_profile_by_id(event['bot']['owner_id']).email else: bot.update(event['bot']) elif event['type'] == 'stream': if event['op'] == 'create': for stream in event['streams']: if not stream['invite_only']: stream_data = copy.deepcopy(stream) if include_subscribers: stream_data['subscribers'] = [] stream_data['stream_weekly_traffic'] = None stream_data['is_old_stream'] = False stream_data['is_announcement_only'] = False # Add stream to never_subscribed (if not invite_only) state['never_subscribed'].append(stream_data) state['streams'].append(stream) state['streams'].sort(key=lambda elt: elt["name"]) if event['op'] == 'delete': deleted_stream_ids = {stream['stream_id'] for stream in event['streams']} state['streams'] = [s for s in state['streams'] if s['stream_id'] not in deleted_stream_ids] state['never_subscribed'] = [stream for stream in state['never_subscribed'] if stream['stream_id'] not in deleted_stream_ids] if event['op'] == 'update': # For legacy reasons, we call stream data 'subscriptions' in # the state var here, for the benefit of the JS code. for obj in state['subscriptions']: if obj['name'].lower() == event['name'].lower(): obj[event['property']] = event['value'] if event['property'] == "description": obj['rendered_description'] = event['rendered_description'] # Also update the pure streams data for stream in state['streams']: if stream['name'].lower() == event['name'].lower(): prop = event['property'] if prop in stream: stream[prop] = event['value'] if prop == 'description': stream['rendered_description'] = event['rendered_description'] elif event['op'] == "occupy": state['streams'] += event['streams'] elif event['op'] == "vacate": stream_ids = [s["stream_id"] for s in event['streams']] state['streams'] = [s for s in state['streams'] if s["stream_id"] not in stream_ids] elif event['type'] == 'default_streams': state['realm_default_streams'] = event['default_streams'] elif event['type'] == 'default_stream_groups': state['realm_default_stream_groups'] = event['default_stream_groups'] elif event['type'] == 'realm': if event['op'] == "update": field = 'realm_' + event['property'] state[field] = event['value'] if event['property'] == 'plan_type': # Then there are some extra fields that also need to be set. state['plan_includes_wide_organization_logo'] = event['value'] != Realm.LIMITED state['realm_upload_quota'] = event['extra_data']['upload_quota'] # Tricky interaction: Whether we can create streams can get changed here. if (field in ['realm_create_stream_policy', 'realm_waiting_period_threshold']) and 'can_create_streams' in state: state['can_create_streams'] = user_profile.can_create_streams() if (field in ['realm_invite_to_stream_policy', 'realm_waiting_period_threshold']) and 'can_subscribe_other_users' in state: state['can_subscribe_other_users'] = user_profile.can_subscribe_other_users() elif event['op'] == "update_dict": for key, value in event['data'].items(): state['realm_' + key] = value # It's a bit messy, but this is where we need to # update the state for whether password authentication # is enabled on this server. if key == 'authentication_methods': state['realm_password_auth_enabled'] = (value['Email'] or value['LDAP']) state['realm_email_auth_enabled'] = value['Email'] elif event['type'] == "subscription": if not include_subscribers and event['op'] in ['peer_add', 'peer_remove']: return if event['op'] in ["add"]: if not include_subscribers: # Avoid letting 'subscribers' entries end up in the list for i, sub in enumerate(event['subscriptions']): event['subscriptions'][i] = copy.deepcopy(event['subscriptions'][i]) del event['subscriptions'][i]['subscribers'] def name(sub: Dict[str, Any]) -> str: return sub['name'].lower() if event['op'] == "add": added_names = set(map(name, event["subscriptions"])) was_added = lambda s: name(s) in added_names # add the new subscriptions state['subscriptions'] += event['subscriptions'] # remove them from unsubscribed if they had been there state['unsubscribed'] = [s for s in state['unsubscribed'] if not was_added(s)] # remove them from never_subscribed if they had been there state['never_subscribed'] = [s for s in state['never_subscribed'] if not was_added(s)] elif event['op'] == "remove": removed_names = set(map(name, event["subscriptions"])) was_removed = lambda s: name(s) in removed_names # Find the subs we are affecting. removed_subs = list(filter(was_removed, state['subscriptions'])) # Remove our user from the subscribers of the removed subscriptions. if include_subscribers: for sub in removed_subs: sub['subscribers'] = [id for id in sub['subscribers'] if id != user_profile.id] # We must effectively copy the removed subscriptions from subscriptions to # unsubscribe, since we only have the name in our data structure. state['unsubscribed'] += removed_subs # Now filter out the removed subscriptions from subscriptions. state['subscriptions'] = [s for s in state['subscriptions'] if not was_removed(s)] elif event['op'] == 'update': for sub in state['subscriptions']: if sub['name'].lower() == event['name'].lower(): sub[event['property']] = event['value'] elif event['op'] == 'peer_add': user_id = event['user_id'] for sub in state['subscriptions']: if (sub['name'] in event['subscriptions'] and user_id not in sub['subscribers']): sub['subscribers'].append(user_id) for sub in state['never_subscribed']: if (sub['name'] in event['subscriptions'] and user_id not in sub['subscribers']): sub['subscribers'].append(user_id) elif event['op'] == 'peer_remove': user_id = event['user_id'] for sub in state['subscriptions']: if (sub['name'] in event['subscriptions'] and user_id in sub['subscribers']): sub['subscribers'].remove(user_id) elif event['type'] == "presence": # TODO: Add user_id to presence update events / state format! presence_user_profile = get_user(event['email'], user_profile.realm) state['presences'][event['email']] = UserPresence.get_status_dict_by_user( presence_user_profile)[event['email']] elif event['type'] == "update_message": # We don't return messages in /register, so we don't need to # do anything for content updates, but we may need to update # the unread_msgs data if the topic of an unread message changed. if TOPIC_NAME in event: stream_dict = state['raw_unread_msgs']['stream_dict'] topic = event[TOPIC_NAME] for message_id in event['message_ids']: if message_id in stream_dict: stream_dict[message_id]['topic'] = topic elif event['type'] == "delete_message": max_message = Message.objects.filter( usermessage__user_profile=user_profile).order_by('-id').first() if max_message: state['max_message_id'] = max_message.id else: state['max_message_id'] = -1 if 'raw_unread_msgs' in state: remove_id = event['message_id'] remove_message_id_from_unread_mgs(state['raw_unread_msgs'], remove_id) # The remainder of this block is about maintaining recent_private_conversations if 'raw_recent_private_conversations' not in state or event['message_type'] != 'private': return recipient_id = get_recent_conversations_recipient_id(user_profile, event['recipient_id'], event['sender_id']) # Ideally, we'd have test coverage for these two blocks. To # do that, we'll need a test where we delete not-the-latest # messages or delete a private message not in # recent_private_conversations. if recipient_id not in state['raw_recent_private_conversations']: # nocoverage return old_max_message_id = state['raw_recent_private_conversations'][recipient_id]['max_message_id'] if old_max_message_id != event['message_id']: # nocoverage return # OK, we just deleted what had been the max_message_id for # this recent conversation; we need to recompute that value # from scratch. Definitely don't need to re-query everything, # but this case is likely rare enough that it's reasonable to do so. state['raw_recent_private_conversations'] = \ get_recent_private_conversations(user_profile) elif event['type'] == "reaction": # The client will get the message with the reactions directly pass elif event['type'] == "submessage": # The client will get submessages with their messages pass elif event['type'] == 'typing': # Typing notification events are transient and thus ignored pass elif event['type'] == "attachment": # Attachment events are just for updating the "uploads" UI; # they are not sent directly. pass elif event['type'] == "update_message_flags": # We don't return messages in `/register`, so most flags we # can ignore, but we do need to update the unread_msgs data if # unread state is changed. if 'raw_unread_msgs' in state and event['flag'] == 'read' and event['operation'] == 'add': for remove_id in event['messages']: remove_message_id_from_unread_mgs(state['raw_unread_msgs'], remove_id) if event['flag'] == 'starred' and event['operation'] == 'add': state['starred_messages'] += event['messages'] if event['flag'] == 'starred' and event['operation'] == 'remove': state['starred_messages'] = [message for message in state['starred_messages'] if not (message in event['messages'])] elif event['type'] == "realm_domains": if event['op'] == 'add': state['realm_domains'].append(event['realm_domain']) elif event['op'] == 'change': for realm_domain in state['realm_domains']: if realm_domain['domain'] == event['realm_domain']['domain']: realm_domain['allow_subdomains'] = event['realm_domain']['allow_subdomains'] elif event['op'] == 'remove': state['realm_domains'] = [realm_domain for realm_domain in state['realm_domains'] if realm_domain['domain'] != event['domain']] elif event['type'] == "realm_emoji": state['realm_emoji'] = event['realm_emoji'] elif event['type'] == 'realm_export': # These realm export events are only available to # administrators, and aren't included in page_params. pass elif event['type'] == "alert_words": state['alert_words'] = event['alert_words'] elif event['type'] == "muted_topics": state['muted_topics'] = event["muted_topics"] elif event['type'] == "realm_filters": state['realm_filters'] = event["realm_filters"] elif event['type'] == "update_display_settings": assert event['setting_name'] in UserProfile.property_types state[event['setting_name']] = event['setting'] elif event['type'] == "update_global_notifications": assert event['notification_name'] in UserProfile.notification_setting_types state[event['notification_name']] = event['setting'] elif event['type'] == "invites_changed": pass elif event['type'] == "user_group": if event['op'] == 'add': state['realm_user_groups'].append(event['group']) state['realm_user_groups'].sort(key=lambda group: group['id']) elif event['op'] == 'update': for user_group in state['realm_user_groups']: if user_group['id'] == event['group_id']: user_group.update(event['data']) elif event['op'] == 'add_members': for user_group in state['realm_user_groups']: if user_group['id'] == event['group_id']: user_group['members'].extend(event['user_ids']) user_group['members'].sort() elif event['op'] == 'remove_members': for user_group in state['realm_user_groups']: if user_group['id'] == event['group_id']: members = set(user_group['members']) user_group['members'] = list(members - set(event['user_ids'])) user_group['members'].sort() elif event['op'] == 'remove': state['realm_user_groups'] = [ug for ug in state['realm_user_groups'] if ug['id'] != event['group_id']] elif event['type'] == 'user_status': user_id = event['user_id'] user_status = state['user_status'] away = event.get('away') status_text = event.get('status_text') if user_id not in user_status: user_status[user_id] = dict() if away is not None: if away: user_status[user_id]['away'] = True else: user_status[user_id].pop('away', None) if status_text is not None: if status_text == '': user_status[user_id].pop('status_text', None) else: user_status[user_id]['status_text'] = status_text if not user_status[user_id]: user_status.pop(user_id, None) state['user_status'] = user_status else: raise AssertionError("Unexpected event type %s" % (event['type'],))
def _wrap_send(func, instance, args, kwargs): """Trace the `Session.send` instance method""" # TODO[manu]: we already offer a way to provide the Global Tracer # and is ddtrace.tracer; it's used only inside our tests and can # be easily changed by providing a TracingTestCase that sets common # tracing functionalities. tracer = getattr(instance, 'datadog_tracer', ddtrace.tracer) # skip if tracing is not enabled if not tracer.enabled: return func(*args, **kwargs) request = kwargs.get('request') or args[0] if not request: return func(*args, **kwargs) # sanitize url of query parsed_uri = parse.urlparse(request.url) hostname = parsed_uri.hostname if parsed_uri.port: hostname = '{}:{}'.format(hostname, parsed_uri.port) sanitized_url = parse.urlunparse(( parsed_uri.scheme, parsed_uri.netloc, parsed_uri.path, parsed_uri.params, None, # drop parsed_uri.query parsed_uri.fragment)) with tracer.trace('requests.request', span_type=SpanTypes.HTTP) as span: # update the span service name before doing any action span.service = _extract_service_name(instance, span, hostname=hostname) # Configure trace search sample rate # DEV: analytics enabled on per-session basis cfg = config.get_from(instance) analytics_enabled = cfg.get('analytics_enabled') if analytics_enabled: span.set_tag(ANALYTICS_SAMPLE_RATE_KEY, cfg.get('analytics_sample_rate', True)) # propagate distributed tracing headers if cfg.get('distributed_tracing'): propagator = HTTPPropagator() propagator.inject(span.context, request.headers) # Storing request headers in the span store_request_headers(request.headers, span, config.requests) response = None try: response = func(*args, **kwargs) # Storing response headers in the span. Note that response.headers is not a dict, but an iterable # requests custom structure, that we convert to a dict if hasattr(response, 'headers'): store_response_headers(dict(response.headers), span, config.requests) return response finally: try: span.set_tag(http.METHOD, request.method.upper()) span.set_tag(http.URL, sanitized_url) if config.requests.trace_query_string: span.set_tag(http.QUERY_STRING, parsed_uri.query) if response is not None: span.set_tag(http.STATUS_CODE, response.status_code) # `span.error` must be an integer span.error = int(500 <= response.status_code) # Storing response headers in the span. # Note that response.headers is not a dict, but an iterable # requests custom structure, that we convert to a dict response_headers = dict(getattr(response, 'headers', {})) store_response_headers(response_headers, span, config.requests) except Exception: log.debug('requests: error adding tags', exc_info=True)
_KEYRANGE.fields_by_name['start_open'].containing_oneof = _KEYRANGE.oneofs_by_name['start_key_type'] _KEYRANGE.oneofs_by_name['end_key_type'].fields.append( _KEYRANGE.fields_by_name['end_closed']) _KEYRANGE.fields_by_name['end_closed'].containing_oneof = _KEYRANGE.oneofs_by_name['end_key_type'] _KEYRANGE.oneofs_by_name['end_key_type'].fields.append( _KEYRANGE.fields_by_name['end_open']) _KEYRANGE.fields_by_name['end_open'].containing_oneof = _KEYRANGE.oneofs_by_name['end_key_type'] _KEYSET.fields_by_name['keys'].message_type = google_dot_protobuf_dot_struct__pb2._LISTVALUE _KEYSET.fields_by_name['ranges'].message_type = _KEYRANGE DESCRIPTOR.message_types_by_name['KeyRange'] = _KEYRANGE DESCRIPTOR.message_types_by_name['KeySet'] = _KEYSET _sym_db.RegisterFileDescriptor(DESCRIPTOR) KeyRange = _reflection.GeneratedProtocolMessageType('KeyRange', (_message.Message,), dict( DESCRIPTOR = _KEYRANGE, __module__ = 'google.spanner.v1.keys_pb2' # @@protoc_insertion_point(class_scope:google.spanner.v1.KeyRange) )) _sym_db.RegisterMessage(KeyRange) KeySet = _reflection.GeneratedProtocolMessageType('KeySet', (_message.Message,), dict( DESCRIPTOR = _KEYSET, __module__ = 'google.spanner.v1.keys_pb2' # @@protoc_insertion_point(class_scope:google.spanner.v1.KeySet) )) _sym_db.RegisterMessage(KeySet) DESCRIPTOR.has_options = True DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\n\025com.google.spanner.v1B\tKeysProtoP\001Z8google.golang.org/genproto/googleapis/spanner/v1;spanner\252\002\027Google.Cloud.Spanner.V1')) try:
'LTU', 'LUX', 'LVA', 'MAC', 'MAR', 'MCO', 'MDA', 'MDG', 'MDV', 'MEX', 'MHL', 'MKD', 'MLI', 'MLT', 'MMR', 'MNG', 'MNP', 'MOZ', 'MRT', 'MSR', 'MTQ', 'MUS', 'MWI', 'MYS', 'MYT', 'NAM', 'NCL', 'NER', 'NFK', 'NGA', 'NIC', 'NIU', 'NLD', 'NOR', 'NPL', 'NRU', 'NZL', 'OMN', 'PAK', 'PAN', 'PCN', 'PER', 'PHL', 'PLW', 'PNG', 'POL', 'PRI', 'PRK', 'PRT', 'PRY', 'PSE', 'PYF', 'QAT', 'REU', 'ROU', 'RUS', 'RWA', 'SAU', 'SCG', 'SDN', 'SEN', 'SGP', 'SHN', 'SJM', 'SLB', 'SLE', 'SLV', 'SMR', 'SOM', 'SPM', 'STP', 'SUR', 'SVK', 'SVN', 'SWE', 'SWZ', 'SYC', 'SYR', 'TCA', 'TCD', 'TGO', 'THA', 'TJK', 'TKL', 'TKM', 'TLS', 'TON', 'TTO', 'TUN', 'TUR', 'TUV', 'TWN', 'TZA', 'UGA', 'UKR', 'URY', 'USA', 'UZB', 'VCT', 'VEN', 'VGB', 'VIR', 'VNM', 'VUT', 'WLF', 'WSM', 'YEM', 'ZAF', 'ZMB', 'ZWE', ] """ISO 3166 alpha-3 codes of countries used in ISIMIP_GPWV3_NATID_150AS""" NONISO_REGIONS = [ # Dummy region for numeric 0 (or empty string), sometimes used for oceans dict(name="", alpha_2="", alpha_3="", numeric="000"), dict(name="Akrotiri", alpha_2="XA", alpha_3="XXA", numeric="901"), dict(name="Baikonur", alpha_2="XB", alpha_3="XXB", numeric="902"), dict(name="Bajo Nuevo Bank", alpha_2="XJ", alpha_3="XXJ", numeric="903"), dict(name="Bir Tawil", alpha_2="XQ", alpha_3="XXQ", numeric="919"), dict(name="Brazilian I.", alpha_2="XE", alpha_3="XXE", numeric="909"), dict(name="Clipperton I.", alpha_2="XC", alpha_3="XXC", numeric="904"), dict(name="Coral Sea Is.", alpha_2="XO", alpha_3="XXO", numeric="905"), dict(name="Cyprus U.N. Buffer Zone", alpha_2="XU", alpha_3="XXU", numeric="906"), dict(name="Dhekelia", alpha_2="XD", alpha_3="XXD", numeric="907"), dict(name="Indian Ocean Ter.", alpha_2="XI", alpha_3="XXI", numeric="908"), # For Kosovo, we follow the iso3166 package and the statistical office of Canada: # https://www.statcan.gc.ca/eng/subjects/standard/sccai/2011/scountry-desc dict(name="Kosovo", alpha_2="XK", alpha_3="XKO", numeric="983"), dict(name="N. Cyprus", alpha_2="XY", alpha_3="XXY", numeric="910"), dict(name="Scarborough Reef", alpha_2="XS", alpha_3="XXS", numeric="912"),
def import_documents(request): def is_reserved_directory(doc): return doc['fields']['type'] == 'directory' and doc['fields'][ 'name'] in (Document2.HOME_DIR, Document2.TRASH_DIR) try: if request.FILES.get('documents'): documents = request.FILES['documents'].read() else: documents = json.loads(request.POST.get('documents')) documents = json.loads(documents) except ValueError as e: raise PopupException( _('Failed to import documents, the file does not contain valid JSON.' )) # Validate documents if not _is_import_valid(documents): raise PopupException( _('Failed to import documents, the file does not contain the expected JSON schema for Hue documents.' )) docs = [] uuids_map = dict((doc['fields']['uuid'], None) for doc in documents if not is_reserved_directory(doc)) for doc in documents: # Filter docs to import, ignoring reserved directories (home and Trash) and history docs if not is_reserved_directory(doc): # Remove any deprecated fields if 'tags' in doc['fields']: doc['fields'].pop('tags') # If doc is not owned by current user, make a copy of the document with current user as owner if doc['fields']['owner'][0] != request.user.username: doc = _copy_document_with_owner(doc, request.user, uuids_map) else: # Update existing doc or create new doc = _create_or_update_document_with_owner( doc, request.user, uuids_map) # For oozie docs replace dependent uuids with the newly created ones if doc['fields']['type'].startswith('oozie-'): doc = _update_imported_oozie_document(doc, uuids_map) # If the doc contains any history dependencies, ignore them # NOTE: this assumes that each dependency is exported as an array using the natural PK [uuid, version, is_history] deps_minus_history = [ dep for dep in doc['fields'].get('dependencies', []) if len(dep) >= 3 and not dep[2] ] doc['fields']['dependencies'] = deps_minus_history # Replace illegal characters if '/' in doc['fields']['name']: new_name = doc['fields']['name'].replace('/', '-') LOG.warn( "Found illegal slash in document named: %s, renaming to: %s." % (doc['fields']['name'], new_name)) doc['fields']['name'] = new_name # Set last modified date to now doc['fields']['last_modified'] = datetime.now().replace( microsecond=0).isoformat() docs.append(doc) f = tempfile.NamedTemporaryFile(mode='w+', suffix='.json') f.write(json.dumps(docs)) f.flush() stdout = string_io() try: with transaction.atomic( ): # We wrap both commands to commit loaddata & sync management.call_command('loaddata', f.name, verbosity=3, traceback=True, stdout=stdout) Document.objects.sync() if request.POST.get('redirect'): return redirect(request.POST.get('redirect')) else: return JsonResponse({ 'status': 0, 'message': stdout.getvalue(), 'count': len(documents), 'created_count': len([doc for doc in documents if doc['pk'] is None]), 'updated_count': len([doc for doc in documents if doc['pk'] is not None]), 'username': request.user.username, 'documents': [ dict([('name', doc['fields']['name']), ('uuid', doc['fields']['uuid']), ('type', doc['fields']['type']), ('owner', doc['fields']['owner'][0])]) for doc in docs ] }) except Exception as e: LOG.error('Failed to run loaddata command in import_documents:\n %s' % stdout.getvalue()) return JsonResponse({'status': -1, 'message': smart_str(e)}) finally: stdout.close()
def hopcroft_karp_matching(G, top_nodes=None): """Returns the maximum cardinality matching of the bipartite graph `G`. Parameters ---------- G : NetworkX graph Undirected bipartite graph top_nodes : container Container with all nodes in one bipartite node set. If not supplied it will be computed. But if more than one solution exists an exception will be raised. Returns ------- matches : dictionary The matching is returned as a dictionary, `matches`, such that ``matches[v] == w`` if node `v` is matched to node `w`. Unmatched nodes do not occur as a key in mate. Raises ------ AmbiguousSolution : Exception Raised if the input bipartite graph is disconnected and no container with all nodes in one bipartite set is provided. When determining the nodes in each bipartite set more than one valid solution is possible if the input graph is disconnected. Notes ----- This function is implemented with the `Hopcroft--Karp matching algorithm <https://en.wikipedia.org/wiki/Hopcroft%E2%80%93Karp_algorithm>`_ for bipartite graphs. See :mod:`bipartite documentation <networkx.algorithms.bipartite>` for further details on how bipartite graphs are handled in NetworkX. See Also -------- eppstein_matching References ---------- .. [1] John E. Hopcroft and Richard M. Karp. "An n^{5 / 2} Algorithm for Maximum Matchings in Bipartite Graphs" In: **SIAM Journal of Computing** 2.4 (1973), pp. 225--231. <https://doi.org/10.1137/0202019>. """ # First we define some auxiliary search functions. # # If you are a human reading these auxiliary search functions, the "global" # variables `leftmatches`, `rightmatches`, `distances`, etc. are defined # below the functions, so that they are initialized close to the initial # invocation of the search functions. def breadth_first_search(): for v in left: if leftmatches[v] is None: distances[v] = 0 queue.append(v) else: distances[v] = INFINITY distances[None] = INFINITY while queue: v = queue.popleft() if distances[v] < distances[None]: for u in G[v]: if distances[rightmatches[u]] is INFINITY: distances[rightmatches[u]] = distances[v] + 1 queue.append(rightmatches[u]) return distances[None] is not INFINITY def depth_first_search(v): if v is not None: for u in G[v]: if distances[rightmatches[u]] == distances[v] + 1: if depth_first_search(rightmatches[u]): rightmatches[u] = v leftmatches[v] = u return True distances[v] = INFINITY return False return True # Initialize the "global" variables that maintain state during the search. left, right = bipartite_sets(G, top_nodes) leftmatches = {v: None for v in left} rightmatches = {v: None for v in right} distances = {} queue = collections.deque() # Implementation note: this counter is incremented as pairs are matched but # it is currently not used elsewhere in the computation. num_matched_pairs = 0 while breadth_first_search(): for v in left: if leftmatches[v] is None: if depth_first_search(v): num_matched_pairs += 1 # Strip the entries matched to `None`. leftmatches = {k: v for k, v in leftmatches.items() if v is not None} rightmatches = {k: v for k, v in rightmatches.items() if v is not None} # At this point, the left matches and the right matches are inverses of one # another. In other words, # # leftmatches == {v, k for k, v in rightmatches.items()} # # Finally, we combine both the left matches and right matches. return dict(itertools.chain(leftmatches.items(), rightmatches.items()))
def select(self, location=None, channel=None, time=None, starttime=None, endtime=None, sampling_rate=None, minlatitude=None, maxlatitude=None, minlongitude=None, maxlongitude=None, latitude=None, longitude=None, minradius=None, maxradius=None): r""" Returns the :class:`Station` object with only the :class:`~obspy.core.inventory.channel.Channel`\ s that match the given criteria (e.g. all channels with ``channel="EHZ"``). .. warning:: The returned object is based on a shallow copy of the original object. That means that modifying any mutable child elements will also modify the original object (see https://docs.python.org/2/library/copy.html). Use :meth:`copy()` afterwards to make a new copy of the data in memory. .. rubric:: Example >>> from obspy import read_inventory, UTCDateTime >>> sta = read_inventory()[0][0] >>> t = UTCDateTime(2008, 7, 1, 12) >>> sta = sta.select(channel="[LB]HZ", time=t) >>> print(sta) # doctest: +NORMALIZE_WHITESPACE Station FUR (Fuerstenfeldbruck, Bavaria, GR-Net) Station Code: FUR Channel Count: None/None (Selected/Total) 2006-12-16T00:00:00.000000Z - Access: None Latitude: 48.16, Longitude: 11.28, Elevation: 565.0 m Available Channels: FUR..BHZ, FUR..LHZ The `location` and `channel` selection criteria may also contain UNIX style wildcards (e.g. ``*``, ``?``, ...; see :func:`~fnmatch.fnmatch`). :type location: str :param location: Potentially wildcarded location code. If not given, all location codes will be accepted. :type channel: str :param channel: Potentially wildcarded channel code. If not given, all channel codes will be accepted. :type time: :class:`~obspy.core.utcdatetime.UTCDateTime` :param time: Only include channels active at given point in time. :type starttime: :class:`~obspy.core.utcdatetime.UTCDateTime` :param starttime: Only include channels active at or after given point in time (i.e. channels ending before given time will not be shown). :type endtime: :class:`~obspy.core.utcdatetime.UTCDateTime` :param endtime: Only include channels active before or at given point in time (i.e. channels starting after given time will not be shown). :type sampling_rate: float :param sampling_rate: Only include channels whose sampling rate matches the given sampling rate, in Hz (within absolute tolerance of 1E-8 Hz and relative tolerance of 1E-5) :type minlatitude: float :param minlatitude: Only include channels with a latitude larger than the specified minimum. :type maxlatitude: float :param maxlatitude: Only include channels with a latitude smaller than the specified maximum. :type minlongitude: float :param minlongitude: Only include channels with a longitude larger than the specified minimum. :type maxlongitude: float :param maxlongitude: Only include channels with a longitude smaller than the specified maximum. :type latitude: float :param latitude: Specify the latitude to be used for a radius selection. :type longitude: float :param longitude: Specify the longitude to be used for a radius selection. :type minradius: float :param minradius: Only include channels within the specified minimum number of degrees from the geographic point defined by the latitude and longitude parameters. :type maxradius: float :param maxradius: Only include channels within the specified maximum number of degrees from the geographic point defined by the latitude and longitude parameters. """ channels = [] for cha in self.channels: # skip if any given criterion is not matched if location is not None: if not fnmatch.fnmatch(cha.location_code.upper(), location.upper()): continue if channel is not None: if not fnmatch.fnmatch(cha.code.upper(), channel.upper()): continue if sampling_rate is not None: if cha.sample_rate is None: msg = ("Omitting channel that has no sampling rate " "specified.") warnings.warn(msg) continue if not np.allclose(float(sampling_rate), cha.sample_rate, rtol=1E-5, atol=1E-8): continue if any([t is not None for t in (time, starttime, endtime)]): if not cha.is_active( time=time, starttime=starttime, endtime=endtime): continue geo_filters = dict(minlatitude=minlatitude, maxlatitude=maxlatitude, minlongitude=minlongitude, maxlongitude=maxlongitude, latitude=latitude, longitude=longitude, minradius=minradius, maxradius=maxradius) if any(value is not None for value in geo_filters.values()): if not inside_geobounds(cha, **geo_filters): continue channels.append(cha) sta = copy.copy(self) sta.channels = channels return sta
def diff(self, a=None, b=None, cached=False, flags=GIT_DIFF_NORMAL, context_lines=3, interhunk_lines=0): """ Show changes between the working tree and the index or a tree, changes between the index and a tree, changes between two trees, or changes between two blobs. Keyword arguments: cached use staged changes instead of workdir flag a GIT_DIFF_* constant context_lines the number of unchanged lines that define the boundary of a hunk (and to display before and after) interhunk_lines the maximum number of unchanged lines between hunk boundaries before the hunks will be merged into a one Examples:: # Changes in the working tree not yet staged for the next commit >>> diff() # Changes between the index and your last commit >>> diff(cached=True) # Changes in the working tree since your last commit >>> diff('HEAD') # Changes between commits >>> t0 = revparse_single('HEAD') >>> t1 = revparse_single('HEAD^') >>> diff(t0, t1) >>> diff('HEAD', 'HEAD^') # equivalent If you want to diff a tree against an empty tree, use the low level API (Tree.diff_to_tree()) directly. """ def whatever_to_tree_or_blob(obj): if obj is None: return None # If it's a string, then it has to be valid revspec if is_string(obj): obj = self.revparse_single(obj) # First we try to get to a blob try: obj = obj.peel(Blob) except Exception: pass # And if that failed, try to get a tree, raising a type # error if that still doesn't work try: obj = obj.peel(Tree) except Exception: raise TypeError('unexpected "%s"' % type(obj)) return obj a = whatever_to_tree_or_blob(a) b = whatever_to_tree_or_blob(b) opt_keys = ['flags', 'context_lines', 'interhunk_lines'] opt_values = [flags, context_lines, interhunk_lines] # Case 1: Diff tree to tree if isinstance(a, Tree) and isinstance(b, Tree): return a.diff_to_tree(b, **dict(zip(opt_keys, opt_values))) # Case 2: Index to workdir elif a is None and b is None: return self.index.diff_to_workdir(*opt_values) # Case 3: Diff tree to index or workdir elif isinstance(a, Tree) and b is None: if cached: return a.diff_to_index(self.index, *opt_values) else: return a.diff_to_workdir(*opt_values) # Case 4: Diff blob to blob if isinstance(a, Blob) and isinstance(b, Blob): raise NotImplementedError('git_diff_blob_to_blob()') raise ValueError("Only blobs and treeish can be diffed")
# Illustrates basic core components import dash import dash_core_components as dcc import dash_html_components as html app = dash.Dash(__name__) # initiate core components # . these are labels followed by the core component itself, assembled within a div # .. dropdowns dd_lbl = html.Label('Dropdown') dd = dcc.Dropdown( options=[ dict(label='New York City', value='NYC'), dict(label=u'Montréal', value='MTL'), dict(label='San Francisco', value='SF') ], value='MTL' ) # .. multi dropdowns mdd_lbl = html.Label('Multi-Dropdown') mdd = dcc.Dropdown( options=[ dict(label='New York City', value='NYC'), dict(label=u'Montréal', value='MTL'), dict(label='San Francisco', value='SF') ], value=['MTL', 'SF'],
def process_item(self, item, spider): self.sheet.insert(dict(item)) return item
def make_side_view(self, axis_name): scene = getattr(self, 'scene_%s' % axis_name) scene.scene.parallel_projection = True ipw_3d = getattr(self, 'ipw_3d_%s' % axis_name) # We create the image_plane_widgets in the side view using a # VTK dataset pointing to the data on the corresponding # image_plane_widget in the 3D view (it is returned by # ipw_3d._get_reslice_output()) side_src = ipw_3d.ipw._get_reslice_output() ipw = mlab.pipeline.image_plane_widget( side_src, plane_orientation='z_axes', vmin=self.data.min(), vmax=self.data.max(), figure=scene.mayavi_scene, name='Cut view %s' % axis_name, ) setattr(self, 'ipw_%s' % axis_name, ipw) # Extract the spacing of the side_src to convert coordinates # into indices spacing = side_src.spacing # Make left-clicking create a crosshair ipw.ipw.left_button_action = 0 x, y, z = self.position cursor = mlab.points3d( x, y, z, mode='axes', color=(0, 0, 0), scale_factor=2 * max(self.data.shape), figure=scene.mayavi_scene, name='Cursor view %s' % axis_name, ) self.cursors[axis_name] = cursor # Add a callback on the image plane widget interaction to # move the others this_axis_number = self._axis_names[axis_name] def move_view(obj, evt): # Disable rendering on all scene position = list(obj.GetCurrentCursorPosition() * spacing)[:2] position.insert(this_axis_number, self.position[this_axis_number]) # We need to special case y, as the view has been rotated. if axis_name is 'y': position = position[::-1] self.position = position ipw.ipw.add_observer('InteractionEvent', move_view) ipw.ipw.add_observer('StartInteractionEvent', move_view) # Center the image plane widget ipw.ipw.slice_position = 0.5 * self.data.shape[ self._axis_names[axis_name]] # 2D interaction: only pan and zoom scene.scene.interactor.interactor_style = \ tvtk.InteractorStyleImage() scene.scene.background = (0, 0, 0) # Some text: mlab.text(0.01, 0.8, axis_name, width=0.08) # Choose a view that makes sens views = dict(x=(0, 0), y=(90, 180), z=(0, 0)) mlab.view(views[axis_name][0], views[axis_name][1], focalpoint=0.5 * np.array(self.data.shape), figure=scene.mayavi_scene) scene.scene.camera.parallel_scale = 0.52 * np.mean(self.data.shape)
def main(): parser = argparse.ArgumentParser(description='Redirect streaming events as email notifications.') parser.add_argument( 'telemetryUrl', help='IP address or hostname of CVP or Telemetry', ) parser.add_argument( 'smtpServer', help='IP address or hostname of SMTP (email) server', ) parser.add_argument( 'sendToAddress', nargs='+', help='List of email recipients', ) parser.add_argument( '-cc', '--sendCcAddress', nargs='+', help='List of CC email recipients', ) parser.add_argument( '-s', '--subjectPrefix', default='[CloudVision Telemetry]', help='Text to prefix the Subject line', ) parser.add_argument( '-p', '--port', type=int, default=465, help='destination port on SMTP server', ) parser.add_argument( '--smtpUsername', help='SMTP (email) server username, if authentication is required. e.g.: [email protected]', ) parser.add_argument( '--smtpPassword', help='''SMTP (email) server password, if authentication is required. If omitted you will be prompted for it at startup''', ) parser.add_argument( '--noSmtpSsl', action='store_true', default=False, help='Flag to disable SSL SMTP connection', ) parser.add_argument( '--noTelemetrySsl', action='store_true', default=False, help='Flag to disable SSL websocket connection', ) parser.add_argument( '--telemetryUsername', help='Telemetry username, if authentication is required', ) parser.add_argument( '--telemetryPassword', help='''Telemetry password, if authentication is required. If omitted you will be prompted for it at startup''', ) parser.add_argument( '--noSslValidation', action='store_true', default=False, help='Disable validation of SSL certificates (inadvised; potentially dangerous)', ) parser.add_argument( '--verbose', action='store_true', default=False, help='Display additional info messages' ) cmd_args = parser.parse_args() passwords = dict() if cmd_args.smtpPassword: passwords['smtpPassword'] = cmd_args.smtpPassword elif cmd_args.smtpUsername: passwords['smtpPassword'] = get_password('Enter SMTP server password for {}'.format(cmd_args.smtpUsername)) if cmd_args.telemetryPassword: passwords['telemetryPassword'] = cmd_args.telemetryPassword elif not cmd_args.noTelemetrySsl: passwords['telemetryPassword'] = get_password('Enter Telemetry password for {}'.format(cmd_args.telemetryUrl)) logging_level = logging.DEBUG if cmd_args.verbose else logging.WARNING logging.basicConfig(level=logging_level) connection = TelemetryWs(cmd_args, passwords) try: ssl_options = None if cmd_args.noSslValidation: ssl_options = { 'check_hostname': False, 'cert_reqs': ssl.CERT_NONE, } connection.socket.run_forever(sslopt=ssl_options) except KeyboardInterrupt: connection.socket.close() exit()
def assign(self, approxs, squares, approxs_per_octave, gt_bboxes, gt_bboxes_ignore=None, gt_labels=None, iou_calculator=dict(type='BboxOverlaps2D')): """Assign gt to approxs. This method assign a gt bbox to each group of approxs (bboxes), each group of approxs is represent by a base approx (bbox) and will be assigned with -1, 0, or a positive number. -1 means don't care, 0 means negative sample, positive number is the index (1-based) of assigned gt. The assignment is done in following steps, the order matters. 1. assign every bbox to -1 2. use the max IoU of each group of approxs to assign 2. assign proposals whose iou with all gts < neg_iou_thr to 0 3. for each bbox, if the iou with its nearest gt >= pos_iou_thr, assign it to that bbox 4. for each gt bbox, assign its nearest proposals (may be more than one) to itself Args: approxs (Tensor): Bounding boxes to be assigned, shape(approxs_per_octave*n, 4). squares (Tensor): Base Bounding boxes to be assigned, shape(n, 4). approxs_per_octave (int): number of approxs per octave gt_bboxes (Tensor): Groundtruth boxes, shape (k, 4). gt_bboxes_ignore (Tensor, optional): Ground truth bboxes that are labelled as `ignored`, e.g., crowd boxes in COCO. gt_labels (Tensor, optional): Label of gt_bboxes, shape (k, ). Returns: :obj:`AssignResult`: The assign result. """ if squares.shape[0] == 0 or gt_bboxes.shape[0] == 0: raise ValueError('No gt or approxs') num_squares = squares.size(0) num_gts = gt_bboxes.size(0) # re-organize anchors by approxs_per_octave x num_squares approxs = torch.transpose( approxs.view(num_squares, approxs_per_octave, 4), 0, 1).contiguous().view(-1, 4) all_overlaps = self.iou_calculator(approxs, gt_bboxes) overlaps, _ = all_overlaps.view(approxs_per_octave, num_squares, num_gts).max(dim=0) overlaps = torch.transpose(overlaps, 0, 1) bboxes = squares[:, :4] if (self.ignore_iof_thr > 0) and (gt_bboxes_ignore is not None) and ( gt_bboxes_ignore.numel() > 0): if self.ignore_wrt_candidates: ignore_overlaps = self.iou_calculator( bboxes, gt_bboxes_ignore, mode='iof') ignore_max_overlaps, _ = ignore_overlaps.max(dim=1) else: ignore_overlaps = self.iou_calculator( gt_bboxes_ignore, bboxes, mode='iof') ignore_max_overlaps, _ = ignore_overlaps.max(dim=0) overlaps[:, ignore_max_overlaps > self.ignore_iof_thr] = -1 assign_result = self.assign_wrt_overlaps(overlaps, gt_labels) return assign_result
class VolumeSlicer(HasTraits): # The data to plot data = Array # The position of the view position = Array(shape=(3, )) # The 4 views displayed scene3d = Instance(MlabSceneModel, ()) scene_x = Instance(MlabSceneModel, ()) scene_y = Instance(MlabSceneModel, ()) scene_z = Instance(MlabSceneModel, ()) # The data source data_src = Instance(Source) # The image plane widgets of the 3D scene ipw_3d_x = Instance(PipelineBase) ipw_3d_y = Instance(PipelineBase) ipw_3d_z = Instance(PipelineBase) # The cursors on each view: cursors = Dict() disable_render = Bool _axis_names = dict(x=0, y=1, z=2) #--------------------------------------------------------------------------- # Object interface #--------------------------------------------------------------------------- def __init__(self, **traits): super(VolumeSlicer, self).__init__(**traits) # Force the creation of the image_plane_widgets: self.ipw_3d_x self.ipw_3d_y self.ipw_3d_z #--------------------------------------------------------------------------- # Default values #--------------------------------------------------------------------------- def _position_default(self): return 0.5 * np.array(self.data.shape) def _data_src_default(self): return mlab.pipeline.scalar_field( self.data, figure=self.scene3d.mayavi_scene, name='Data', ) def make_ipw_3d(self, axis_name): ipw = mlab.pipeline.image_plane_widget( self.data_src, figure=self.scene3d.mayavi_scene, plane_orientation='%s_axes' % axis_name, name='Cut %s' % axis_name) return ipw def _ipw_3d_x_default(self): return self.make_ipw_3d('x') def _ipw_3d_y_default(self): return self.make_ipw_3d('y') def _ipw_3d_z_default(self): return self.make_ipw_3d('z') #--------------------------------------------------------------------------- # Scene activation callbacks #--------------------------------------------------------------------------- @on_trait_change('scene3d.activated') def display_scene3d(self): outline = mlab.pipeline.outline( self.data_src, figure=self.scene3d.mayavi_scene, ) self.scene3d.mlab.view(40, 50) # Interaction properties can only be changed after the scene # has been created, and thus the interactor exists for ipw in (self.ipw_3d_x, self.ipw_3d_y, self.ipw_3d_z): ipw.ipw.interaction = 0 self.scene3d.scene.background = (0, 0, 0) # Keep the view always pointing up self.scene3d.scene.interactor.interactor_style = \ tvtk.InteractorStyleTerrain() self.update_position() def make_side_view(self, axis_name): scene = getattr(self, 'scene_%s' % axis_name) scene.scene.parallel_projection = True ipw_3d = getattr(self, 'ipw_3d_%s' % axis_name) # We create the image_plane_widgets in the side view using a # VTK dataset pointing to the data on the corresponding # image_plane_widget in the 3D view (it is returned by # ipw_3d._get_reslice_output()) side_src = ipw_3d.ipw._get_reslice_output() ipw = mlab.pipeline.image_plane_widget( side_src, plane_orientation='z_axes', vmin=self.data.min(), vmax=self.data.max(), figure=scene.mayavi_scene, name='Cut view %s' % axis_name, ) setattr(self, 'ipw_%s' % axis_name, ipw) # Extract the spacing of the side_src to convert coordinates # into indices spacing = side_src.spacing # Make left-clicking create a crosshair ipw.ipw.left_button_action = 0 x, y, z = self.position cursor = mlab.points3d( x, y, z, mode='axes', color=(0, 0, 0), scale_factor=2 * max(self.data.shape), figure=scene.mayavi_scene, name='Cursor view %s' % axis_name, ) self.cursors[axis_name] = cursor # Add a callback on the image plane widget interaction to # move the others this_axis_number = self._axis_names[axis_name] def move_view(obj, evt): # Disable rendering on all scene position = list(obj.GetCurrentCursorPosition() * spacing)[:2] position.insert(this_axis_number, self.position[this_axis_number]) # We need to special case y, as the view has been rotated. if axis_name is 'y': position = position[::-1] self.position = position ipw.ipw.add_observer('InteractionEvent', move_view) ipw.ipw.add_observer('StartInteractionEvent', move_view) # Center the image plane widget ipw.ipw.slice_position = 0.5 * self.data.shape[ self._axis_names[axis_name]] # 2D interaction: only pan and zoom scene.scene.interactor.interactor_style = \ tvtk.InteractorStyleImage() scene.scene.background = (0, 0, 0) # Some text: mlab.text(0.01, 0.8, axis_name, width=0.08) # Choose a view that makes sens views = dict(x=(0, 0), y=(90, 180), z=(0, 0)) mlab.view(views[axis_name][0], views[axis_name][1], focalpoint=0.5 * np.array(self.data.shape), figure=scene.mayavi_scene) scene.scene.camera.parallel_scale = 0.52 * np.mean(self.data.shape) @on_trait_change('scene_x.activated') def display_scene_x(self): return self.make_side_view('x') @on_trait_change('scene_y.activated') def display_scene_y(self): return self.make_side_view('y') @on_trait_change('scene_z.activated') def display_scene_z(self): return self.make_side_view('z') #--------------------------------------------------------------------------- # Traits callback #--------------------------------------------------------------------------- @on_trait_change('position') def update_position(self): """ Update the position of the cursors on each side view, as well as the image_plane_widgets in the 3D view. """ # First disable rendering in all scenes to avoid unecessary # renderings self.disable_render = True # For each axis, move image_plane_widget and the cursor in the # side view for axis_name, axis_number in self._axis_names.iteritems(): ipw3d = getattr(self, 'ipw_3d_%s' % axis_name) ipw3d.ipw.slice_position = self.position[axis_number] # Go from the 3D position, to the 2D coordinates in the # side view position2d = list(self.position) position2d.pop(axis_number) if axis_name is 'y': position2d = position2d[::-1] # Move the cursor # For the following to work, you need Mayavi 3.4.0, if you # have a less recent version, use 'x=[position2d[0]]' self.cursors[axis_name].mlab_source.set(x=position2d[0], y=position2d[1], z=0) # Finally re-enable rendering self.disable_render = False @on_trait_change('disable_render') def _render_enable(self): for scene in (self.scene3d, self.scene_x, self.scene_y, self.scene_z): scene.scene.disable_render = self.disable_render #--------------------------------------------------------------------------- # The layout of the dialog created #--------------------------------------------------------------------------- view = View( HGroup( Group( Item('scene_y', editor=SceneEditor(scene_class=Scene), height=250, width=300), Item('scene_z', editor=SceneEditor(scene_class=Scene), height=250, width=300), show_labels=False, ), Group( Item('scene_x', editor=SceneEditor(scene_class=Scene), height=250, width=300), Item('scene3d', editor=SceneEditor(scene_class=Scene), height=250, width=300), show_labels=False, ), ), resizable=True, title='Volume Slicer', )
# 测试eval()函数 s = "print('abcde')" eval(s) a = 10 b = 20 c = eval("a+b") print(c) dict1 = dict(a=10, b=20) d = eval("a+b", dict1) print(d)
def readCommand(argv): """ Processes the command used to run pacman from the command line. """ from optparse import OptionParser usageStr = """ USAGE: python pacman.py <options> EXAMPLES: (1) python pacman.py - starts an interactive game (2) python pacman.py --layout smallClassic --zoom 2 OR python pacman.py -l smallClassic -z 2 - starts an interactive game on a smaller board, zoomed in """ parser = OptionParser(usageStr) parser.add_option('-n', '--numGames', dest='numGames', type='int', help=default('the number of GAMES to play'), metavar='GAMES', default=1) parser.add_option( '-l', '--layout', dest='layout', help=default('the LAYOUT_FILE from which to load the map layout'), metavar='LAYOUT_FILE', default='mediumClassic') parser.add_option( '-p', '--pacman', dest='pacman', help=default('the agent TYPE in the pacmanAgents module to use'), metavar='TYPE', default='KeyboardAgent') parser.add_option('-t', '--textGraphics', action='store_true', dest='textGraphics', help='Display output as text only', default=False) parser.add_option('-q', '--quietTextGraphics', action='store_true', dest='quietGraphics', help='Generate minimal output and no graphics', default=False) parser.add_option( '-g', '--ghosts', dest='ghost', help=default('the ghost agent TYPE in the ghostAgents module to use'), metavar='TYPE', default='RandomGhost') parser.add_option('-k', '--numghosts', type='int', dest='numGhosts', help=default('The maximum number of ghosts to use'), default=4) parser.add_option('-z', '--zoom', type='float', dest='zoom', help=default('Zoom the size of the graphics window'), default=1.0) parser.add_option( '-f', '--fixRandomSeed', action='store_true', dest='fixRandomSeed', help='Fixes the random seed to always play the same game', default=False) parser.add_option( '-r', '--recordActions', action='store_true', dest='record', help= 'Writes game histories to a file (named by the time they were played)', default=False) parser.add_option('--replay', dest='gameToReplay', help='A recorded game file (pickle) to replay', default=None) parser.add_option( '-a', '--agentArgs', dest='agentArgs', help= 'Comma separated values sent to agent. e.g. "opt1=val1,opt2,opt3=val3"' ) parser.add_option( '-x', '--numTraining', dest='numTraining', type='int', help=default('How many episodes are training (suppresses output)'), default=0) parser.add_option( '--frameTime', dest='frameTime', type='float', help=default('Time to delay between frames; <0 means keyboard'), default=0.1) parser.add_option( '-c', '--catchExceptions', action='store_true', dest='catchExceptions', help='Turns on exception handling and timeouts during games', default=False) parser.add_option( '--timeout', dest='timeout', type='int', help=default( 'Maximum length of time an agent can spend computing in a single game' ), default=30) options, otherjunk = parser.parse_args(argv) if len(otherjunk) != 0: raise Exception('Command line input not understood: ' + str(otherjunk)) args = dict() # Fix the random seed if options.fixRandomSeed: random.seed('cs188') # Choose a layout args['layout'] = layout.getLayout(options.layout) if args['layout'] == None: raise Exception("The layout " + options.layout + " cannot be found") # Choose a Pacman agent noKeyboard = options.gameToReplay == None and (options.textGraphics or options.quietGraphics) pacmanType = loadAgent(options.pacman, noKeyboard) agentOpts = parseAgentArgs(options.agentArgs) if options.numTraining > 0: args['numTraining'] = options.numTraining if 'numTraining' not in agentOpts: agentOpts['numTraining'] = options.numTraining pacman = pacmanType(**agentOpts) # Instantiate Pacman with agentArgs args['pacman'] = pacman # Don't display training games if 'numTrain' in agentOpts: options.numQuiet = int(agentOpts['numTrain']) options.numIgnore = int(agentOpts['numTrain']) # Choose a ghost agent ghostType = loadAgent(options.ghost, noKeyboard) args['ghosts'] = [ghostType(i + 1) for i in range(options.numGhosts)] # Choose a display format if options.quietGraphics: import textDisplay args['display'] = textDisplay.NullGraphics() elif options.textGraphics: import textDisplay textDisplay.SLEEP_TIME = options.frameTime args['display'] = textDisplay.PacmanGraphics() else: import graphicsDisplay args['display'] = graphicsDisplay.PacmanGraphics( options.zoom, frameTime=options.frameTime) args['numGames'] = options.numGames args['record'] = options.record args['catchExceptions'] = options.catchExceptions args['timeout'] = options.timeout # Special case: recorded games don't use the runGames method or args structure if options.gameToReplay != None: print('Replaying recorded game %s.' % options.gameToReplay) import pickle f = open(options.gameToReplay, 'rb') try: recorded = pickle.load(f) finally: f.close() recorded['display'] = args['display'] replayGame(**recorded) sys.exit(0) return args
def handle_starttag(self, tag, attrs): href = dict(attrs).get("href") if href and tag == "a": self.urls.append(href)
variant = dict( algo_kwargs=dict( num_epochs=505, num_steps_per_epoch=1000, num_steps_per_eval=1000, tau=1e-2, batch_size=128, max_path_length=100, discount=0.99, # qf_learning_rate=1e-3, # policy_learning_rate=1e-4, ), env_kwargs=dict( include_puck=True, arm_range=0.1, use_big_red_puck=True, reward_params=dict( type="sparse", epsilon=0.2, # puck_reward_only=True, ), ), replay_kwargs=dict( fraction_goals_are_rollout_goals=0.2, fraction_goals_are_env_goals=0.5, ), algorithm='TD3', normalize=False, rdim=4, render=False, env=FullPusher2DEnv, use_env_goals=True, vae_paths=vae_paths, wrap_mujoco_env=True, do_state_based_exp=False, exploration_noise=0.1, )
#!/usr/bin/python3 """ Read a FASTQ file and remove redundancy For example when the base calling has some bugs I hope you have enough RAM """ import sys fastq = dict() linenum = 0 for i in open(sys.argv[1]): if linenum % 4 == 0: readid = i.rstrip() fastq[readid] = list() fastq[readid].append(i.rstrip()) linenum += 1 for i in fastq: print("\n".join(fastq[i]))
def fill_experiment_table(): """Populates the table that displays the results from previously ran experiments. The results are stored as yaml files in the directory that the global variable EXPERIMENT_SERIALIZATION_DIR points to. """ global data_table, table_source, table_data_df os.chdir(EXPERIMENT_SERIALIZATION_DIR) experiment_files = glob.glob("*.yml") names = [''] * len(experiment_files) Ls = [0] * len(experiment_files) reward_types = [''] * len(experiment_files) window_size = [''] * len(experiment_files) window_step = [''] * len(experiment_files) seq_or_con = [''] * len(experiment_files) for i, filename in enumerate(experiment_files): with open(filename, 'r') as ymlfile: experiment_data = yaml.safe_load(ymlfile) names[i] = filename Ls[i] = experiment_data['L'] reward_path = experiment_data['reward_path'] if 'threshold' in reward_path: reward_types[i] = 'threshold' + reward_path.split('threshold_')[ 1][0:3] elif 'top' in experiment_data['reward_path']: reward_types[i] = 'top' else: if 'pear' in reward_path: reward_types[i] = 'continous-pearson' elif 'MI_n1' in reward_path: reward_types[i] = 'continous-MI-normalized' else: reward_types[i] = 'continous-MI' window_size[i], window_step[i] = re.findall( r'\d+', reward_path)[0:2] seq_or_con[i] = reward_path.split('/')[-1][0:3] table_data = dict( names=names, seq_or_con=seq_or_con, Ls=Ls, reward_types=reward_types, window_size=window_size, window_step=window_step, ) table_data_df = pd.DataFrame(data=table_data) table_data_df = table_data_df.sort_values( ['seq_or_con', 'Ls', 'window_size', 'window_step', 'reward_types']) table_source = ColumnDataSource(data=table_data_df) table_source.selected.on_change('indices', load_experiment) columns = [ TableColumn(field="names", title="Experiment"), TableColumn(field="seq_or_con", title="Sequential/ concurrent"), TableColumn(field="Ls", title="L"), TableColumn(field="reward_types", title="Reward Type"), TableColumn(field='window_size', title="Window Size"), TableColumn(field='window_step', title="Window Step"), ] data_table = DataTable(source=table_source, columns=columns, height=200) data_table.sizing_mode = 'stretch_width' global pol_table, pol_source columns_pol_table = [ TableColumn(field="pol_name", title="Name"), TableColumn(field="overall_regret", title="Overall regret") ] pol_source = ColumnDataSource(dict(pol_name=[], overall_regret=[])) pol_source.selected.on_change('indices', plot_policy) pol_table = DataTable( source=pol_source, columns=columns_pol_table, height=300) pol_table.sizing_mode = 'stretch_width'
def print_row(title_, nutrients): rowformat = "{title:11} {calories:7} {carbs:6} {fat:6} {protein:6}" row = dict(title=title_) for nutrient in NUTRIENTS: row[nutrient] = decorated_nutrient(nutrients[nutrient]) print(rowformat.format(**row))
import sys data = np.zeros((1392,1080)) data[-1,:] = 1.e-5 field = xarray.Dataset({'rate': (['ny', 'nx'], data)}) # Choose where to save the results from this script output_file = 'INPUT/ice_outflow.nc' # Save file, making sure that time is the unlimited dimension. compress = True if compress: fileformat, zlib, complevel, area_dtype = 'NETCDF4', True, 1, 'f4' else: fileformat, zlib, complevel, area_dtype = 'NETCDF3_64BIT_OFFSET', None, None, 'd' comp = dict(zlib=zlib, complevel=complevel) encoding = {var: comp for var in field.data_vars} field.to_netcdf( output_file, format=fileformat, encoding=encoding ) # Delete the _FillValue attribute. # MOM will crash if this attribute is present. # I can't find how to get xarray to not write it. # Need to have NCO in your path or module load nco before running this script. # Get rid of all of them: #call(f'ncatted -h -O -a _FillValue,,d,, {output_file}', shell=True)
import monochromator import srs510 import serial import time import numpy configuration_text = open('grating_test.config.txt', 'r').read().split('\n') parameters = dict() for line in configuration_text: if (len(line) > 0): if line[0] != '#': l = line.split() try: parameters[l[0].upper()] = float(l[2]) print 'Added float ', parameters[ l[0].upper()], ' as entry for key ', l[0].upper() except: parameters[l[0].upper()] = l[2] print 'Added text variable ', parameters[ l[0].upper()], ' as entry for key ', l[0].upper() print "Grating Testing Program" print "Cross your fingers!" sim = False srs = srs510.SRS510(sim, parameters) a = srs.measure_const_SNR(10) ''' mono = monochromator.Monochromator(False, parameters)
ax.set_xlim(0.01, 0.99) ax.set_ylim(0, 2.0) #---------------------------------------- # plot the convolution ax = fig.add_subplot(223) ax.plot(x_w, y_w, '-k') ax.text(0.5, 0.95, "Convolution:\n" + r"$[D \ast W](x)$", ha='center', va='top', transform=ax.transAxes, bbox=dict(fc='w', ec='k', pad=8), zorder=2) ax.text(0.5, 0.05, (r'$[D \ast W](x)$' + r'$= \mathcal{F}^{-1}\{\mathcal{F}[D] \cdot \mathcal{F}[W]\}$'), ha='center', va='bottom', transform=ax.transAxes) for x_loc in (0.2, 0.8): y_loc = y_w[x_w <= x_loc][-1] ax.annotate('', (x_loc, y_loc), (x_loc, 2.0), zorder=1, arrowprops=dict(arrowstyle='->', color='gray', lw=2))
def _revise_resource_value(self, instance, files: list, user, action_type: str, importer_session_opts: typing.Optional[typing.Dict] = None): from geonode.upload.files import ALLOWED_EXTENSIONS session_opts = dict(importer_session_opts) if importer_session_opts is not None else {} spatial_files_type = get_spatial_files_dataset_type(ALLOWED_EXTENSIONS, files) if not spatial_files_type: raise Exception("No suitable Spatial Files avaialable for 'ALLOWED_EXTENSIONS' = {ALLOWED_EXTENSIONS}.") upload_session, _ = Upload.objects.get_or_create(resource=instance.get_real_instance().resourcebase_ptr, user=user) upload_session.resource = instance.get_real_instance().resourcebase_ptr upload_session.save() _name = instance.get_real_instance().name if not _name: _name = session_opts.get('name', None) or os.path.splitext(os.path.basename(spatial_files_type.base_file))[0] instance.get_real_instance().name = _name gs_dataset = None try: gs_dataset = gs_catalog.get_layer(_name) except Exception as e: logger.debug(e) _workspace = None _target_store = None if gs_dataset: _target_store = gs_dataset.resource.store.name if instance.get_real_instance().subtype == 'vector' else None _workspace = gs_dataset.resource.workspace.name if gs_dataset.resource.workspace else None if not _workspace: _workspace = session_opts.get('workspace', instance.get_real_instance().workspace) if not _workspace: _workspace = instance.get_real_instance().workspace or settings.DEFAULT_WORKSPACE if not _target_store: if instance.get_real_instance().subtype == 'vector' or spatial_files_type.dataset_type == 'vector': _dsname = ogc_server_settings.datastore_db['NAME'] _ds = create_geoserver_db_featurestore(store_name=_dsname, workspace=_workspace) if _ds: _target_store = session_opts.get('target_store', None) or _dsname # opening Import session for the selected layer import_session = gs_uploader.start_import( import_id=upload_session.id, name=_name, target_store=_target_store ) upload_session.set_processing_state(enumerations.STATE_PROCESSED) upload_session.import_id = import_session.id upload_session.name = _name upload_session.complete = True upload_session.processed = True upload_session.save() _gs_import_session_info = GeoServerImporterSessionInfo( upload_session=upload_session, import_session=import_session, spatial_files_type=spatial_files_type, dataset_name=None, workspace=_workspace, target_store=_target_store ) _local_files = [] _temporary_files = [] try: for _f in files: if os.path.exists(_f) and os.path.isfile(_f): _local_files.append(os.path.abspath(_f)) try: os.close(_f) except Exception: pass else: _suffix = os.path.splitext(os.path.basename(_f))[1] if len(os.path.splitext(os.path.basename(_f))) else None with tempfile.NamedTemporaryFile(mode="wb+", delete=False, dir=settings.MEDIA_ROOT, suffix=_suffix) as _tmp_file: _tmp_file.write(storage_manager.open(_f, 'rb+').read()) _tmp_file.seek(0) _tmp_file_name = f'{_tmp_file.name}' _local_files.append(os.path.abspath(_tmp_file_name)) _temporary_files.append(os.path.abspath(_tmp_file_name)) try: storage_manager.close(_f) except Exception: pass except Exception as e: logger.exception(e) if _local_files: try: import_session.upload_task(_local_files) task = import_session.tasks[0] # Changing layer name, mode and target task.layer.set_target_layer_name(_name) task.set_update_mode(action_type.upper()) task.set_target( store_name=_target_store, workspace=_workspace ) transforms = session_opts.get('transforms', None) if transforms: task.set_transforms(transforms) # Starting import process import_session.commit() import_session = import_session.reload() try: # Updating Resource with the files replaced if action_type.lower() == 'replace': updated_files_list = storage_manager.replace(instance, files) # Using update instead of save in order to avoid calling # side-effect function of the resource r = ResourceBase.objects.filter(id=instance.id) r.update(**updated_files_list) else: instance.files = files except Exception as e: logger.exception(e) _gs_import_session_info.import_session = import_session _gs_import_session_info.dataset_name = import_session.tasks[0].layer.name finally: for _f in _temporary_files: try: os.remove(_f) except Exception as e: logger.debug(e) return _gs_import_session_info
def shape_element(element, node_attr_fields=NODE_FIELDS, way_attr_fields=WAY_FIELDS, relation_attr_fields=RELATION_FIELDS, problem_chars=PROBLEMCHARS, default_tag_type='regular'): """Clean and shape node or way XML element to Python dict""" node_attribs = {} way_attribs = {} relation_attribs = {} tags = [] # Handle secondary tags the same way for node , way and relation elements way_nodes = [] relation_members = [] nd_position = 0 member_position = 0 for child in element : #======== # TAGS || #======== if child.tag == 'tag' : tags.append(dict()) key = child.attrib[ 'k' ] if 'k' in child.attrib else None if key and PROBLEMCHARS.search(key) : key = None if key and LOWER_COLON.search(key) : key = key.split(':') if key else None type_ = key[0] if key else None key = ':'.join(key[1:]) if key else None else : type_ = default_tag_type tags[len(tags)-1][ 'value' ] = child.attrib[ 'v' ] if 'v' in child.attrib else None tags[len(tags)-1][ 'id' ] = element.attrib[ 'id' ] if 'id' in element.attrib else None tags[len(tags)-1][ 'key' ] = key tags[len(tags)-1][ 'type' ] = type_ #============= # way_nodes || #============= elif child.tag == 'nd' : way_nodes.append(dict()) way_nodes[len(way_nodes)-1][ 'id' ] = element.attrib[ 'id' ] if 'id' in element.attrib else None way_nodes[len(way_nodes)-1][ 'node_id' ] = child.attrib[ 'ref' ] if 'ref' in child.attrib else None way_nodes[len(way_nodes)-1][ 'position' ] = nd_position nd_position += 1 #==================== # relation_members || #==================== elif child.tag == 'member' : relation_members.append(dict()) index = len(relation_members)-1 relation_members[index]['id'] = element.attrib['id'] if 'id' in element.attrib else None relation_members[index]['node_way_id'] = child.attrib['ref' ] if 'ref' in child.attrib else None relation_members[index]['role'] = child.attrib['role'] if 'role' in child.attrib else None relation_members[index]['type'] = child.attrib['type'] if 'type' in child.attrib else None relation_members[index]['position'] = member_position member_position += 1 #================== #========= # nodes || #========= if element.tag == 'node': for field in node_attr_fields : node_attribs[ field ] = element.attrib[ field ] if field in element.attrib else None return { 'node': node_attribs, 'node_tags': tags } #======== # WAYS || #======== elif element.tag == 'way': for field in way_attr_fields : way_attribs[ field ] = element.attrib[ field ] if field in element.attrib else None return { 'way': way_attribs, 'way_nodes': way_nodes, 'way_tags': tags } #============= # RELATIONS || #============= elif element.tag == 'relation': for field in relation_attr_fields : relation_attribs[ field ] = element.attrib[ field ] if field in element.attrib else None return { 'relation': relation_attribs, 'relation_members': relation_members, 'relation_tags': tags }
def raster_drill(layer, x, y, format_): """ Writes the information in the format provided by the user and reads some information from the geomet-climate yaml :param layer: layer name :param x: x coordinate :param y: y coordinate :param format_: output format (GeoJSON or CSV) :return: return the final file fo a given location """ from msc_pygeoapi.process.cccs import (GEOMET_CLIMATE_CONFIG, GEOMET_CLIMATE_BASEPATH, GEOMET_CLIMATE_BASEPATH_VRT, GEOMET_CLIMATE_EPSG) if GEOMET_CLIMATE_EPSG is not None: pyproj.set_datapath(GEOMET_CLIMATE_EPSG) else: raise Exception("Could not locate geomet-climate EPSG file.") LOGGER.info('start raster drilling') if format_ not in ['CSV', 'GeoJSON']: msg = 'Invalid format' LOGGER.error(msg) raise ValueError(msg) with open(GEOMET_CLIMATE_CONFIG, encoding='utf-8') as fh: cfg = yaml.load(fh, Loader=CLoader) data_basepath = GEOMET_CLIMATE_BASEPATH climate_model_path = cfg['layers'][layer]['climate_model']['basepath'] file_path = cfg['layers'][layer]['filepath'] inter_path = os.path.join(climate_model_path, file_path) if ('ABS' in layer or 'ANO' in layer and layer.startswith('CANGRD') is False): keys = [ 'Model', 'Variable', 'Scenario', 'Period', 'Type', 'Percentile' ] values = layer.replace('_', '.').split('.') layer_keys = dict(zip(keys, values)) file_name = cfg['layers'][layer]['filename'] elif 'TREND' not in layer and layer.startswith('CANGRD'): keys = ['Model', 'Type', 'Variable', 'Period'] values = layer.replace('_', '.').split('.') layer_keys = dict(zip(keys, values)) data_basepath = GEOMET_CLIMATE_BASEPATH_VRT climate_model_path = cfg['layers'][layer]['climate_model']['basepath'] file_path = cfg['layers'][layer]['filepath'] inter_path = os.path.join(climate_model_path, file_path) file_name = '{}.vrt'.format(cfg['layers'][layer]['filename']) elif layer.startswith('SPEI'): keys = ['Variable', 'Variation', 'Scenario', 'Period', 'Percentile'] values = layer.replace('-', '.').replace('_', '.').split('.') layer_keys = dict(zip(keys, values)) layer_keys['Type'] = 'ABS' file_name = cfg['layers'][layer]['filename'] elif layer.startswith('INDICES'): keys = ['Model', 'Variable', 'Scenario', 'Percentile'] values = layer.replace('_', '.').split('.') layer_keys = dict(zip(keys, values)) layer_keys['Type'] = 'ABS' file_name = cfg['layers'][layer]['filename'] else: msg = 'Not a valid or time enabled layer: {}'.format(layer) LOGGER.error(msg) raise ValueError(msg) srs = osr.SpatialReference() srs.ImportFromWkt(cfg['layers'][layer]['climate_model']['projection']) inProj = Proj(init='epsg:4326') outProj = Proj(srs.ExportToProj4()) _x, _y = transform(inProj, outProj, x, y) ds = os.path.join(data_basepath, inter_path, file_name) data = get_location_info(ds, _x, _y, cfg['layers'][layer], layer_keys) output = serialize(data, cfg['layers'][layer], format_, x, y) return output
from __future__ import division, print_function, absolute_import import numpy as np from random import shuffle from math import ceil import pickle import subprocess from .helpers import loadAudio from .conf import conf from config import ROOT_DIR library = dict() # Shuffle to arrays in unison def unison_shuffled_copies(a, b): assert len(a) == len(b) p = np.random.permutation(len(a)) return a[p], b[p] def loadData(): # reload audio object from file try: audio_file = open(ROOT_DIR + r'/resources/audio.pkl', 'rb') audio_matrix, classifications = pickle.load(audio_file) audio_file.close() return unison_shuffled_copies(audio_matrix, classifications) except: # If no pickle file
"Cross product of elements in a and elements in b." return [s+t for s in a for t in b] # Global variables ROWS = 'ABCDEFGHI' COLS = '123456789' BOXES = cross(ROWS, COLS) ROW_UNITS = [cross(r, COLS) for r in ROWS] COLUMN_UNITS = [cross(ROWS, c) for c in COLS] SQUARE_UNITS = [cross(rs, cs) for rs in ('ABC','DEF','GHI') for cs in ('123','456','789')] DIAGONALS = [[r+c for r, c in zip(ROWS, COLS)], [r+c for r, c in zip(ROWS[-1::-1], COLS)]] UNITLIST = ROW_UNITS + COLUMN_UNITS + SQUARE_UNITS + DIAGONALS UNITS = dict((s, [u for u in UNITLIST if s in u]) for s in BOXES) PEERS = dict((s, set(sum(UNITS[s],[]))-set([s])) for s in BOXES) def assign_value(values, box, value): """ Please use this function to update your values dictionary! Assigns a value to a given box. If it updates the board record it. """ values[box] = value if len(value) == 1: assignments.append(values.copy()) return values def naked_twins(values):
def main(imgfile, outfile, rows, cols, normalized): """Define regions of interest (ROIs) on an image, and return bounding boxes. Use number keys, 1 - 9, to select ROIs. ROI position can be adjusted with mouse or arrow keys. ROI size can be adjusted with mouse or shift+arrow keys. Assumes a grid of ROIs, but no constraints on ROI overlap. Bounding boxes are (minrow, mincol, maxrow, maxcol) to be consistent with skimage. Returned bounding boxes are returned in YAML format for easy parsing. See extractROI for a program that operates on ROIs. """ img = np.squeeze(io.imread(imgfile)) fig, main_ax = plt.subplots() plt.subplots_adjust(bottom=0.2) main_ax.imshow(img, cmap="gray") nplates = rows * cols irows, icols = img.shape selections = SelectorCollection() selections.outfile = outfile selections.rows = rows selections.cols = cols selections.normalized = normalized selections.colors = plt.cm.tab10(np.linspace(0, 1, nplates)) selections.selectors = \ [PersistentRectangleSelector( main_ax, selections.select_callback, drawtype='box', useblit=False, button=[1], minspanx=5, minspany=5, spancoords='data', rectprops = dict(facecolor=selections.colors[i],alpha=0.5), interactive=True) for i in range(nplates)] bboxes = subdivide_region((irows, icols), rows, cols) for i, bbox in enumerate(bboxes): minr, minc, maxr, maxc = bbox selections.selectors[i].extents = (minc, maxc, minr, maxr) selections.selectors[i].set_visible(True) selections.current = 0 selections.activate_current() ax_normalize = plt.axes([0.45, 0.05, 0.2, 0.075]) btn_normalize = Button(ax_normalize, "Normalize") btn_normalize.on_clicked(selections.normalize_and_update) ax_apply = plt.axes([0.7, 0.05, 0.2, 0.075]) btn_apply = Button(ax_apply, "Apply and Close") btn_apply.on_clicked(selections.write_geometry) plt.connect('key_press_event', selections.choose_selector) plt.sca(main_ax) plt.show()