def _select(sql, first, *args): """ 执行SQL,返回一个结果 或者多个结果组成的列表 :param sql: str :param first: boolean :param args: list :return: Dict instance """ global _db_ctx cursor, names = None, [] sql = sql.replace('?', '%s') logging.info('SQL: %s, ARGS: %s' % (sql, args)) try: cursor = _db_ctx.cursor() cursor.execute(sql, args) if cursor.description: names = [x[0] for x in cursor.description] if first: values = cursor.fetchone() if not values: return None return Dict(names, values) return [Dict(names, x) for x in cursor.fetchall()] finally: if cursor: cursor.close()
def group_files_sat(self, manifest): """ Group all the satellite files from downloaded manifest :param manifest: satellite manifest from retrieving :return result: groupped and cleanned manifest """ if not manifest: return manifest result = Dict({}) geore = re.compile(r'%s' % self.geo_prefix) pregeore = re.compile(r'%s' % self.pre_geo_prefix) firere = re.compile(r'%s' % self.fire_prefix) keys = np.array(list(manifest.keys())) labels = np.array([''.join(k.split('.')[1:3]) for k in keys]) indexes = duplicates(labels) for k,ind in indexes.items(): lenind = len(ind) if lenind != 2: logging.warning('group_files_sat: number of geo and fire granules %d different than 2' % lenind) if lenind < 2: logging.error('group_files_sat: geo or fire file is missing, number of granules %d different than 2' % lenind) continue geo = list(filter(geore.search,keys[ind])) pregeo = list(filter(pregeore.search,keys[ind])) fire = list(filter(firere.search,keys[ind])) if not geo: if pregeo: geok = pregeo[0] else: logging.error('group_files_sat: no geo data in the manifest') continue else: geok = geo[0] if fire: firek = fire[0] else: logging.error('group_files_sat: no fire data in the manifest') continue logging.info('group_files_sat: %s - geo %s and fire %s' % (k,geok,firek)) try: r = Dict({ 'time_start_iso' : manifest[geok]['time_start'], 'time_end_iso' : manifest[geok]['time_end'], 'geo_url' : manifest[geok]['url'], 'geo_local_path' : manifest[geok]['local_path'], 'geo_description' : manifest[geok]['dataset_id'], 'fire_url' : manifest[firek]['url'], 'fire_local_path' : manifest[firek]['local_path'], 'fire_description' : manifest[firek]['dataset_id'] }) result.update({k: r}) except Exception as e: logging.error('group_files_sat: when creating manifest with error %s' % str(e)) continue return result
def scan_dir(self, root): if not root.exists(): return [] st_mtime = root.stat().st_mtime if root in self.cache and st_mtime == self.cache[root][0]: entries = self.cache[root][1] else: entries = [] for item in root.iterdir(): if not item.exists(): continue if item.name.startswith('.'): continue path = item.relative_to(self.root) entries.append(Dict({ 'path': path, 'mtime': int(item.stat().st_mtime), 'size': item.stat().st_size, 'md5': self.get_md5(item), 'is_dir': item.is_dir(), })) entries.sort(key=lambda x: str(x.path).lower()) self.cache[root] = [st_mtime, entries] return entries
def get_metas_sat(self, bbox, time): """ Get all the meta data for all the necessary products :param bbox: polygon with the search bounding box :param time: time interval (init_time_datetime,final_time_datetime) :param maxg: max number of granules to process :return granules: dictionary with the metadata of all the products """ metas = Dict({}) metas.geo = self.search_api_sat(self.geo_prefix, bbox, time, collection=self.geo_collection_id) metas.fire = self.search_api_sat(self.fire_prefix, bbox, time, collection=self.fire_collection_id) metas.geo_nrt = self.search_api_sat( self.geo_nrt_prefix, bbox, time, collection=self.geo_nrt_collection_id) metas.fire_nrt = self.search_api_sat( self.fire_nrt_prefix, bbox, time, collection=self.fire_nrt_collection_id) return metas
def select(sql, select_first, *args): global _dbCntn sql = sql.replace('?', '%s') logger.info('SQL: %s, ARGS: %s' % (sql, args)) cursor = _dbCntn.cursor() cursor.execute(sql, args) result = None if cursor.description: names = [x[0] for x in cursor.description] if select_first: value = cursor.fetchone() if value: result = Dict(names, value) else: result = [Dict(names, x) for x in cursor.fetchall()] return result
def get_metas_sat(self, bbox, time, burned=False): """ Get all the meta data for all the necessary products :param bbox: polygon with the search bounding box :param time: time interval (init_time_datetime,final_time_datetime) :param maxg: max number of granules to process :return granules: dictionary with the metadata of all the products """ metas=Dict([]) metas.geo=self.search_api_sat(self.geo_prefix,bbox,time,self.version) if not metas.geo: if not self.pre_geo_prefix: logging.warning('any geolocation data matches the search.') return metas else: pre_geo=self.search_api_sat(self.pre_geo_prefix,bbox,time,self.version) metas.geo=self.search_archive_sat(self.geo_prefix,time,pre_geo) fire=self.search_api_sat(self.fire_prefix,bbox,time) if fire: # eliminate NRT products (duplicated) nlist=[m for m in fire if m['data_center']!='LANCEMODIS'] logging.info('eliminating %d NRT products (duplicated)' % int(abs(len(fire)-len(nlist)))) metas.fire=nlist else: metas.fire=self.search_archive_sat(self.fire_prefix,time,metas.geo) if burned: metas.ref=self.search_api_sat(self.ref_prefix,bbox,time) if not metas.ref: metas.ref=self.search_archive_sat(self.ref_prefix,time,metas.geo) return metas
def retrieve_data_sat(self, bounds, from_utc, to_utc): """ Retrieve satellite data in a bounding box coordinates and time interval :param bounds: polygon with the search bounding box :param time: time interval (init_time_iso,final_time_iso) :return data: dictonary with all the data """ if not osp.exists(osp.join(osp.expanduser('~'),'.netrc')): logging.warning('satellite acquisition can fail because some data centers require to have $HOME/.netrc specified from an existent Earthdata account') lonmin,lonmax,latmin,latmax = bounds bbox = [(lonmin,latmax),(lonmin,latmin),(lonmax,latmin),(lonmax,latmax),(lonmin,latmax)] time = (from_utc, to_utc) metas = self.get_metas_sat(bbox,time) logging.info('retrieve_data_sat: saved %s metas for %s satellite service' % (sum([len(m) for m in metas.items()]),self.prefix)) manifest = Dict({}) for k, meta in metas.items(): logging.info('downloading %s products' % k) for m in meta: id = osp.splitext(m['producer_granule_id'])[0] url = m['links'][0]['href'] dc = m['data_center'] m.update(self.download_sat(url,self.datacenter_to_appkey(dc))) if m: manifest.update({id: m}) group_manifest = self.group_files_sat(manifest) return group_manifest
def parse_example(example): features = Dict( tf.parse_single_example( serialized=example, features=dict(path=tf.FixedLenFeature([], dtype=tf.string), pitch=tf.FixedLenFeature([], dtype=tf.int64), source=tf.FixedLenFeature([], dtype=tf.int64)))) #Read the element waveform = tf.read_file(features.path) #Here we decode the audio a 16 bit obtaining 64000 dimension # will be scaled to -1.0 to 1.0 in float. waveform, _ = audio_ops.decode_wav(contents=waveform, desired_channels=1, desired_samples=64000) waveform = tf.squeeze(waveform) #to each element assign a label according its pitch label = index_table.lookup(features.pitch) label = tf.one_hot(label, len(pitches)) #cast pitch and source to int-32 pitch = tf.cast(features.pitch, tf.int32) source = tf.cast(features.source, tf.int32) return waveform, label, pitch, source
def __init__(self, worker_num=10, chunk_size=10000, log_interval=600, data_dir='data', log_dir='log'): self.chunk_size = chunk_size self.log_interval = log_interval self.urls = Queue() self.results = Queue() self.url_cache = Set() self.url_cache1 = Set() self.url_cache2 = Set() self.url_cache3 = Set() self.name_cache = Set() self.black_urls = Set() self.black_cache = Dict() self.chunk_num = 0 self.parser = HtmlParser(home='http://www.a-hospital.com/') self.last = 0 self.state = 1 if not os.path.exists(data_dir): os.mkdir(data_dir) if not os.path.exists(log_dir): os.mkdir(log_dir) self.data_dir = data_dir self.log_dir = log_dir self.writer = Thread(target=self._write) self.logger = Timer(log_interval, self._log) self.spiders = [Thread(target=self._scrap) for _ in range(worker_num)]
async def get(self): if self.app.options.auth: if self.current_user: token = self.args.token if self.current_user.admin and self.args.token else self.current_user.token docs = await self.query('share', {'token': token}) entries = [] for doc in docs: if doc.expired_at and doc.expired_at < datetime.datetime.now( ): await self.db.share.delete_one({'_id': doc._id}) else: entries.append( Dict({ 'path': Path(doc.name), 'mtime': doc.mtime, 'size': doc.size, 'is_dir': doc.is_dir, 'md5': doc.md5, 'key': doc._id, 'expired_at': doc.expired_at, 'shared': True, })) self.render('index.html', entries=entries, absolute=True) else: self.redirect(self.get_login_url()) else: self.redirect('/disk')
def parse_example(example): features = Dict( tf.parse_single_example( serialized=example, features=dict(path=tf.FixedLenFeature([], dtype=tf.string), pitch=tf.FixedLenFeature([], dtype=tf.int64), source=tf.FixedLenFeature([], dtype=tf.int64)))) waveform = tf.read_file(features.path) # Decode a 16-bit PCM WAV file to a float tensor. # The -32768 to 32767 signed 16-bit values # will be scaled to -1.0 to 1.0 in float. waveform, _ = audio_ops.decode_wav(contents=waveform, desired_channels=1, desired_samples=64000) waveform = tf.squeeze(waveform) label = index_table.lookup(features.pitch) label = tf.one_hot(label, len(pitches)) pitch = tf.cast(features.pitch, tf.int32) source = tf.cast(features.source, tf.int32) return waveform, label, pitch, source
def search_archive_sat(self, prod, time, geo_metas): """ Archive search of the different satellite granules :param prod: string of short name product, ex: 'VNP09' :param time: time interval (init_time_datetime,final_time_datetime) :param geo_metas: granules of the geolocation metadata :return metas: dictionary with the metadata of the search """ ids=['.'.join(gm['producer_granule_id'].split('.')[1:3]) for gm in geo_metas] # satellite ids in bounding box metas=[] delta=time[1]-time[0] nh=int(delta.total_seconds()/3600) dates=[time[0]+datetime.timedelta(seconds=3600*k) for k in range(1,nh+1)] fold=np.unique(['%d/%03d' % (date.timetuple().tm_year,date.timetuple().tm_yday) for date in dates]) urls=[self.url+'/'+prod+'/'+f for f in fold] for u in urls: js=requests.get(u+'.json').json() for j in js: arg=np.argwhere(np.array(ids)=='.'.join(j['name'].split('.')[1:3])) if arg.size: ar=arg[0][0] g=Dict(j) g.links=[{'href': u+'/'+g.name}] g.time_start=geo_metas[ar]['time_start'] g.time_end=geo_metas[ar]['time_end'] g.dataset_id='Archive downloaded: unknown dataset id' g.producer_granule_id=j['name'] metas.append(g) logging.info('Archive: %s gets %s hits in this range' % (self.prefix+prod,len(metas))) return metas
class Fig62Game(Game): """The game represented in [Fig. 6.2]. Serves as a simple test case. >>> g = Fig62Game() >>> minimax_decision('A', g) 'a1' >>> alphabeta_full_search('A', g) 'a1' >>> alphabeta_search('A', g) 'a1' """ succs = { 'A': [('a1', 'B'), ('a2', 'C'), ('a3', 'D')], 'B': [('b1', 'B1'), ('b2', 'B2'), ('b3', 'B3')], 'C': [('c1', 'C1'), ('c2', 'C2'), ('c3', 'C3')], 'D': [('d1', 'D1'), ('d2', 'D2'), ('d3', 'D3')] } utils = Dict(B1=3, B2=12, B3=8, C1=2, C2=4, C3=6, D1=14, D2=5, D3=2) initial = 'A' def successors(self, state): return self.succs.get(state, []) def utility(self, state, player): if player == 'MAX': return self.utils[state] else: return -self.utils[state] def terminal_test(self, state): return state not in ('A', 'B', 'C', 'D') def to_move(self, state): return if_(state in 'BCD', 'MIN', 'MAX')
def retrieve_metas(self, metas): """ Retrieve satellite data from CMR API metadata :return metas: dictonary with all the satellite data to retrieve :return manifest: dictonary with all the satellite data retrieved """ logging.info('retrieve_metas - downloading {} products'.format(self.id)) manifest = Dict({}) for p_id,meta in metas['fire'].items(): urls = [meta['url']] fire_meta = self.download_sat(urls) fire_meta.update(meta) local_path = fire_meta['local_path'] remote_size = int(fire_meta['file_size']) local_size = osp.getsize(local_path) if local_size != remote_size: logging.error('retrieve_metas - local file with size {} different than remote size {}'.format()) continue info_path = local_path + '.size' open(ensure_dir(info_path), 'w').write(str(local_size)) geo_meta = process_grid(self.ingest_dir,fire_meta) manifest.update({p_id: { 'time_start_iso' : fire_meta['time_start'], 'time_end_iso' : fire_meta['time_end'], 'geo_url' : fire_meta['url'], 'geo_local_path' : geo_meta['grid_path'], 'geo_description' : self.info, 'fire_url' : fire_meta['url'], 'fire_local_path' : fire_meta['local_path'], 'fire_description' : self.info }}) return manifest
def retrieve_metas(self, metas): """ Retrieve satellite data from CMR API metadata :return metas: dictonary with all the satellite data to retrieve :return manifest: dictonary with all the satellite data retrieved """ return Dict({})
def group_metas(self, metas): """ Group all the satellite metas before downloading to minimize number of files downloaded :param metas: satellite metadatas from API search :return result: groupped and cleanned metadata dictionary """ return Dict({})
def ner_collator(tokenizer, args): batchify_fn = lambda samples, fn=Dict({ 'input_ids': Pad(axis=0, pad_val=tokenizer.pad_token_id, dtype='int32'), # input 'token_type_ids': Pad(axis=0, pad_val=tokenizer.pad_token_type_id, dtype='int32'), # segment 'labels': Pad(axis=0, pad_val=args.ignore_label, dtype='int64') # label }): fn(samples) return batchify_fn
def qa_collator(tokenizer, args): train_batchify_fn = lambda samples, fn=Dict( { "input_ids": Pad(axis=0, pad_val=tokenizer.pad_token_id), "token_type_ids": Pad(axis=0, pad_val=tokenizer.pad_token_type_id), "start_positions": Stack(dtype="int64"), "end_positions": Stack(dtype="int64") }): fn(samples) return train_batchify_fn
def get_metas_sat(self, bounds, time): """ Get all the meta data for all the necessary products :param bounds: bounding box as (lon_min,lon_max,lat_min,lat_max) :param time: time interval (init_time_datetime,final_time_datetime) :param maxg: max number of granules to process :return metas: dictionary with the metadata of all the products """ return Dict({'fire': self.search_api_sat(None,None,time)})
def _get_session(session_id): """ get session from the runtime memory, which store as dictionary. Using lock to make thread-safe. :param session_id: the session id assigned to each request. :return: a Dict """ with _session_lock: if session_id not in _SESSIONS_WAREHOUSE: _SESSIONS_WAREHOUSE[session_id] = Dict() return _SESSIONS_WAREHOUSE[session_id]
def readMazeFromFile(fileName): matrix = {} i = 0 start = (0, 0) goal = (0, 0) with open(fileName) as f: start = tuple(map(int, f.readline().rstrip('\n').strip('()').split(','))) goal = tuple(map(int, f.readline().rstrip('\n').strip('()').split(','))) for line in f.readlines(): line = line.rstrip('\n') j = 0 for cost in line: if int(cost) != 0: matrix[(j, i)] = int(cost) j = j + 1 i = i + 1 graph = Dict() for point in matrix.keys(): entry = {point: {}} child1 = None child2 = None try: child1 = {(point[0], point[1] + 1): matrix[(point[0], point[1] + 1)]} except KeyError: pass try: child2 = {(point[0] + 1, point[1]): matrix[(point[0] + 1, point[1])]} except KeyError: pass if child1 is not None: entry[point].update(child1) if child2 is not None: entry[point].update(child2) graph.update(entry) undirGraph = UndirectedGraph(graph) undirGraph.locations = Dict() for i in undirGraph.nodes(): undirGraph.locations.update({i: i}) graphProblem = ManhattenGraphProblem(start, goal, undirGraph) return graphProblem
def input(self, **kw): """ 返回一个由传入的数据和从environ里取出的数据 组成的Dict对象,Dict对象的定义 见db模块 Get input as dict from request, fill dict using provided default value if key not exist. i = ctx.request.input(role='guest') i.role ==> 'guest' """ copy = Dict(**kw) raw = self._get_raw_input() for k, v in raw.iteritems(): copy[k] = v[0] if isinstance(v, list) else v return copy
async def get_info(self, name): info = Dict() if self.app.options.auth and not name.startswith('public'): doc = await self.db.share.find_one({ 'name': name, 'token': self.current_user.token }) info.share = True if doc else False if self.app.options.git and self.current_user.admin: doc = await self.db.files.find_one({'name': name}) info.git = True if doc else False return info
def input(self, **kw): """Get input as dict from request, fill dict using provided default value if key not exist. i = ctx.request.input(role='guest') i.role ==> 'guest' :param kw: :return: """ copy = Dict(**kw) raw = self._get_raw_input() for k, v in raw.iteritems(): copy[k] = v[0] if isinstance(v, list) else v return copy
def load_job_file(job_id): jobfile = osp.join(cfg['workspace_path'], job_id, 'job.json') logging.info('Loading job state from %s' % jobfile) try: js = Dict(json.load(open(jobfile, 'r'))) except IOError: logging.error('Cannot open %s' % jobfile) js = None except: logging.error('Cannot load %s' % jobfile) js = None # print(json.dumps(js, indent=4, separators=(',', ': '))) return js, jobfile
def base_select(sql, single, *args): """execute select SQL and fetch results.""" global db_context cursor = None sql = sql.replace('?', '%s') logging.info('SQL: %s, ARGS: %s' % (sql, args)) try: cursor = db_context.connection.cursor() cursor.execute(sql, args) if cursor.description: names = [x[0] for x in cursor.description] else: raise DateBaseError("No cursor description.") if single: values = cursor.fetchone() if not values: return None return Dict(names, values) return [Dict(names, x) for x in cursor.fetchall()] finally: if cursor: cursor.close()
def preview_tar(self, path): tf = tarfile.open(path) items = tf.getmembers() entries = [] for item in items: entries.append( Dict({ 'path': Path(item.name), 'mtime': item.mtime, 'size': item.size, 'is_dir': item.isdir(), })) self.render('index.html', entries=entries, absolute=True)
def transform_goes(d): """ Transform GOES mask data into similar to MODIS-VIIRS AF mask :param d: open NetCDF4 dataset """ fill = Dict({ (150., 151., 152., 153.): 3, (100.): 5, (15., 35.): 7, (14., 34.): 8, (10., 11., 12., 13., 30., 31., 32., 33.): 9 }) array = d.variables['Mask'][:] return fill_categories(array, fill)
def parse_fmda(self, args): """ Parse information inside the FMDA blob, if any. :param args: the forecast job argument dictionary """ if 'fuel_moisture_da' in args: fmda = args['fuel_moisture_da'] return Dict({ 'token': fmda['mesowest_token'], 'domains': fmda['domains'] }) else: return None
def _select(sql, first, *args): ' execute select SQL and return unique result or list results.' global _db_ctx, _db_convert cursor = None if _db_convert != '?': sql = sql.replace('?', _db_convert) _log('SQL: %s, ARGS: %s' % (sql, args)) start = time.time() try: cursor = _db_ctx.connection.cursor() cursor.execute(sql, args) if cursor.description: names = [x[0] for x in cursor.description] if first: values = cursor.fetchone() if not values: return None return Dict(names, values) return [Dict(names, x) for x in cursor.fetchall()] finally: if cursor: cursor.close() _profiling(start, sql)