def _select(sql, first, *args):
"""
执行SQL，返回一个结果 或者多个结果组成的列表
:param sql: str
:param first: boolean
:param args: list
:return: Dict instance
"""
global _db_ctx
cursor, names = None, []
sql = sql.replace('?', '%s')
logging.info('SQL: %s, ARGS: %s' % (sql, args))
try:
cursor = _db_ctx.cursor()
cursor.execute(sql, args)
if cursor.description:
names = [x[0] for x in cursor.description]
if first:
values = cursor.fetchone()
if not values:
return None
return Dict(names, values)
return [Dict(names, x) for x in cursor.fetchall()]
finally:
if cursor:
cursor.close()
def group_files_sat(self, manifest):
"""
Group all the satellite files from downloaded manifest
:param manifest: satellite manifest from retrieving
:return result: groupped and cleanned manifest
"""
if not manifest:
return manifest
result = Dict({})
geore = re.compile(r'%s' % self.geo_prefix)
pregeore = re.compile(r'%s' % self.pre_geo_prefix)
firere = re.compile(r'%s' % self.fire_prefix)
keys = np.array(list(manifest.keys()))
labels = np.array([''.join(k.split('.')[1:3]) for k in keys])
indexes = duplicates(labels)
for k,ind in indexes.items():
lenind = len(ind)
if lenind != 2:
logging.warning('group_files_sat: number of geo and fire granules %d different than 2' % lenind)
if lenind < 2:
logging.error('group_files_sat: geo or fire file is missing, number of granules %d different than 2' % lenind)
continue
geo = list(filter(geore.search,keys[ind]))
pregeo = list(filter(pregeore.search,keys[ind]))
fire = list(filter(firere.search,keys[ind]))
if not geo:
if pregeo:
geok = pregeo[0]
else:
logging.error('group_files_sat: no geo data in the manifest')
continue
else:
geok = geo[0]
if fire:
firek = fire[0]
else:
logging.error('group_files_sat: no fire data in the manifest')
continue
logging.info('group_files_sat: %s - geo %s and fire %s' % (k,geok,firek))
try:
r = Dict({
'time_start_iso' : manifest[geok]['time_start'],
'time_end_iso' : manifest[geok]['time_end'],
'geo_url' : manifest[geok]['url'],
'geo_local_path' : manifest[geok]['local_path'],
'geo_description' : manifest[geok]['dataset_id'],
'fire_url' : manifest[firek]['url'],
'fire_local_path' : manifest[firek]['local_path'],
'fire_description' : manifest[firek]['dataset_id']
})
result.update({k: r})
except Exception as e:
logging.error('group_files_sat: when creating manifest with error %s' % str(e))
continue
return result
def scan_dir(self, root):
if not root.exists():
return []
st_mtime = root.stat().st_mtime
if root in self.cache and st_mtime == self.cache[root][0]:
entries = self.cache[root][1]
else:
entries = []
for item in root.iterdir():
if not item.exists():
continue
if item.name.startswith('.'):
continue
path = item.relative_to(self.root)
entries.append(Dict({
'path': path,
'mtime': int(item.stat().st_mtime),
'size': item.stat().st_size,
'md5': self.get_md5(item),
'is_dir': item.is_dir(),
}))
entries.sort(key=lambda x: str(x.path).lower())
self.cache[root] = [st_mtime, entries]
return entries
def get_metas_sat(self, bbox, time):
"""
Get all the meta data for all the necessary products
:param bbox: polygon with the search bounding box
:param time: time interval (init_time_datetime,final_time_datetime)
:param maxg: max number of granules to process
:return granules: dictionary with the metadata of all the products
"""
metas = Dict({})
metas.geo = self.search_api_sat(self.geo_prefix,
bbox,
time,
collection=self.geo_collection_id)
metas.fire = self.search_api_sat(self.fire_prefix,
bbox,
time,
collection=self.fire_collection_id)
metas.geo_nrt = self.search_api_sat(
self.geo_nrt_prefix,
bbox,
time,
collection=self.geo_nrt_collection_id)
metas.fire_nrt = self.search_api_sat(
self.fire_nrt_prefix,
bbox,
time,
collection=self.fire_nrt_collection_id)
return metas
def select(sql, select_first, *args):
global _dbCntn
sql = sql.replace('?', '%s')
logger.info('SQL: %s, ARGS: %s' % (sql, args))
cursor = _dbCntn.cursor()
cursor.execute(sql, args)
result = None
if cursor.description:
names = [x[0] for x in cursor.description]
if select_first:
value = cursor.fetchone()
if value:
result = Dict(names, value)
else:
result = [Dict(names, x) for x in cursor.fetchall()]
return result
def get_metas_sat(self, bbox, time, burned=False):
"""
Get all the meta data for all the necessary products
:param bbox: polygon with the search bounding box
:param time: time interval (init_time_datetime,final_time_datetime)
:param maxg: max number of granules to process
:return granules: dictionary with the metadata of all the products
"""
metas=Dict([])
metas.geo=self.search_api_sat(self.geo_prefix,bbox,time,self.version)
if not metas.geo:
if not self.pre_geo_prefix:
logging.warning('any geolocation data matches the search.')
return metas
else:
pre_geo=self.search_api_sat(self.pre_geo_prefix,bbox,time,self.version)
metas.geo=self.search_archive_sat(self.geo_prefix,time,pre_geo)
fire=self.search_api_sat(self.fire_prefix,bbox,time)
if fire:
# eliminate NRT products (duplicated)
nlist=[m for m in fire if m['data_center']!='LANCEMODIS']
logging.info('eliminating %d NRT products (duplicated)' % int(abs(len(fire)-len(nlist))))
metas.fire=nlist
else:
metas.fire=self.search_archive_sat(self.fire_prefix,time,metas.geo)
if burned:
metas.ref=self.search_api_sat(self.ref_prefix,bbox,time)
if not metas.ref:
metas.ref=self.search_archive_sat(self.ref_prefix,time,metas.geo)
return metas
def retrieve_data_sat(self, bounds, from_utc, to_utc):
"""
Retrieve satellite data in a bounding box coordinates and time interval
:param bounds: polygon with the search bounding box
:param time: time interval (init_time_iso,final_time_iso)
:return data: dictonary with all the data
"""
if not osp.exists(osp.join(osp.expanduser('~'),'.netrc')):
logging.warning('satellite acquisition can fail because some data centers require to have $HOME/.netrc specified from an existent Earthdata account')
lonmin,lonmax,latmin,latmax = bounds
bbox = [(lonmin,latmax),(lonmin,latmin),(lonmax,latmin),(lonmax,latmax),(lonmin,latmax)]
time = (from_utc, to_utc)
metas = self.get_metas_sat(bbox,time)
logging.info('retrieve_data_sat: saved %s metas for %s satellite service' % (sum([len(m) for m in metas.items()]),self.prefix))
manifest = Dict({})
for k, meta in metas.items():
logging.info('downloading %s products' % k)
for m in meta:
id = osp.splitext(m['producer_granule_id'])[0]
url = m['links'][0]['href']
dc = m['data_center']
m.update(self.download_sat(url,self.datacenter_to_appkey(dc)))
if m:
manifest.update({id: m})
group_manifest = self.group_files_sat(manifest)
return group_manifest
def parse_example(example):
features = Dict(
tf.parse_single_example(
serialized=example,
features=dict(path=tf.FixedLenFeature([], dtype=tf.string),
pitch=tf.FixedLenFeature([], dtype=tf.int64),
source=tf.FixedLenFeature([], dtype=tf.int64))))
#Read the element
waveform = tf.read_file(features.path)
#Here we decode the audio a 16 bit obtaining 64000 dimension
# will be scaled to -1.0 to 1.0 in float.
waveform, _ = audio_ops.decode_wav(contents=waveform,
desired_channels=1,
desired_samples=64000)
waveform = tf.squeeze(waveform)
#to each element assign a label according its pitch
label = index_table.lookup(features.pitch)
label = tf.one_hot(label, len(pitches))
#cast pitch and source to int-32
pitch = tf.cast(features.pitch, tf.int32)
source = tf.cast(features.source, tf.int32)
return waveform, label, pitch, source
def __init__(self,
worker_num=10,
chunk_size=10000,
log_interval=600,
data_dir='data',
log_dir='log'):
self.chunk_size = chunk_size
self.log_interval = log_interval
self.urls = Queue()
self.results = Queue()
self.url_cache = Set()
self.url_cache1 = Set()
self.url_cache2 = Set()
self.url_cache3 = Set()
self.name_cache = Set()
self.black_urls = Set()
self.black_cache = Dict()
self.chunk_num = 0
self.parser = HtmlParser(home='http://www.a-hospital.com/')
self.last = 0
self.state = 1
if not os.path.exists(data_dir):
os.mkdir(data_dir)
if not os.path.exists(log_dir):
os.mkdir(log_dir)
self.data_dir = data_dir
self.log_dir = log_dir
self.writer = Thread(target=self._write)
self.logger = Timer(log_interval, self._log)
self.spiders = [Thread(target=self._scrap) for _ in range(worker_num)]
async def get(self):
if self.app.options.auth:
if self.current_user:
token = self.args.token if self.current_user.admin and self.args.token else self.current_user.token
docs = await self.query('share', {'token': token})
entries = []
for doc in docs:
if doc.expired_at and doc.expired_at < datetime.datetime.now(
):
await self.db.share.delete_one({'_id': doc._id})
else:
entries.append(
Dict({
'path': Path(doc.name),
'mtime': doc.mtime,
'size': doc.size,
'is_dir': doc.is_dir,
'md5': doc.md5,
'key': doc._id,
'expired_at': doc.expired_at,
'shared': True,
}))
self.render('index.html', entries=entries, absolute=True)
else:
self.redirect(self.get_login_url())
else:
self.redirect('/disk')
def parse_example(example):
features = Dict(
tf.parse_single_example(
serialized=example,
features=dict(path=tf.FixedLenFeature([], dtype=tf.string),
pitch=tf.FixedLenFeature([], dtype=tf.int64),
source=tf.FixedLenFeature([], dtype=tf.int64))))
waveform = tf.read_file(features.path)
# Decode a 16-bit PCM WAV file to a float tensor.
# The -32768 to 32767 signed 16-bit values
# will be scaled to -1.0 to 1.0 in float.
waveform, _ = audio_ops.decode_wav(contents=waveform,
desired_channels=1,
desired_samples=64000)
waveform = tf.squeeze(waveform)
label = index_table.lookup(features.pitch)
label = tf.one_hot(label, len(pitches))
pitch = tf.cast(features.pitch, tf.int32)
source = tf.cast(features.source, tf.int32)
return waveform, label, pitch, source
def search_archive_sat(self, prod, time, geo_metas):
"""
Archive search of the different satellite granules
:param prod: string of short name product, ex: 'VNP09'
:param time: time interval (init_time_datetime,final_time_datetime)
:param geo_metas: granules of the geolocation metadata
:return metas: dictionary with the metadata of the search
"""
ids=['.'.join(gm['producer_granule_id'].split('.')[1:3]) for gm in geo_metas] # satellite ids in bounding box
metas=[]
delta=time[1]-time[0]
nh=int(delta.total_seconds()/3600)
dates=[time[0]+datetime.timedelta(seconds=3600*k) for k in range(1,nh+1)]
fold=np.unique(['%d/%03d' % (date.timetuple().tm_year,date.timetuple().tm_yday) for date in dates])
urls=[self.url+'/'+prod+'/'+f for f in fold]
for u in urls:
js=requests.get(u+'.json').json()
for j in js:
arg=np.argwhere(np.array(ids)=='.'.join(j['name'].split('.')[1:3]))
if arg.size:
ar=arg[0][0]
g=Dict(j)
g.links=[{'href': u+'/'+g.name}]
g.time_start=geo_metas[ar]['time_start']
g.time_end=geo_metas[ar]['time_end']
g.dataset_id='Archive downloaded: unknown dataset id'
g.producer_granule_id=j['name']
metas.append(g)
logging.info('Archive: %s gets %s hits in this range' % (self.prefix+prod,len(metas)))
return metas
class Fig62Game(Game):
"""The game represented in [Fig. 6.2]. Serves as a simple test case.
>>> g = Fig62Game()
>>> minimax_decision('A', g)
'a1'
>>> alphabeta_full_search('A', g)
'a1'
>>> alphabeta_search('A', g)
'a1'
"""
succs = {
'A': [('a1', 'B'), ('a2', 'C'), ('a3', 'D')],
'B': [('b1', 'B1'), ('b2', 'B2'), ('b3', 'B3')],
'C': [('c1', 'C1'), ('c2', 'C2'), ('c3', 'C3')],
'D': [('d1', 'D1'), ('d2', 'D2'), ('d3', 'D3')]
}
utils = Dict(B1=3, B2=12, B3=8, C1=2, C2=4, C3=6, D1=14, D2=5, D3=2)
initial = 'A'
def successors(self, state):
return self.succs.get(state, [])
def utility(self, state, player):
if player == 'MAX':
return self.utils[state]
else:
return -self.utils[state]
def terminal_test(self, state):
return state not in ('A', 'B', 'C', 'D')
def to_move(self, state):
return if_(state in 'BCD', 'MIN', 'MAX')
def retrieve_metas(self, metas):
"""
Retrieve satellite data from CMR API metadata
:return metas: dictonary with all the satellite data to retrieve
:return manifest: dictonary with all the satellite data retrieved
"""
logging.info('retrieve_metas - downloading {} products'.format(self.id))
manifest = Dict({})
for p_id,meta in metas['fire'].items():
urls = [meta['url']]
fire_meta = self.download_sat(urls)
fire_meta.update(meta)
local_path = fire_meta['local_path']
remote_size = int(fire_meta['file_size'])
local_size = osp.getsize(local_path)
if local_size != remote_size:
logging.error('retrieve_metas - local file with size {} different than remote size {}'.format())
continue
info_path = local_path + '.size'
open(ensure_dir(info_path), 'w').write(str(local_size))
geo_meta = process_grid(self.ingest_dir,fire_meta)
manifest.update({p_id: {
'time_start_iso' : fire_meta['time_start'],
'time_end_iso' : fire_meta['time_end'],
'geo_url' : fire_meta['url'],
'geo_local_path' : geo_meta['grid_path'],
'geo_description' : self.info,
'fire_url' : fire_meta['url'],
'fire_local_path' : fire_meta['local_path'],
'fire_description' : self.info
}})
return manifest
def retrieve_metas(self, metas):
"""
Retrieve satellite data from CMR API metadata
:return metas: dictonary with all the satellite data to retrieve
:return manifest: dictonary with all the satellite data retrieved
"""
return Dict({})
def group_metas(self, metas):
"""
Group all the satellite metas before downloading to minimize number of files downloaded
:param metas: satellite metadatas from API search
:return result: groupped and cleanned metadata dictionary
"""
return Dict({})
def ner_collator(tokenizer, args):
batchify_fn = lambda samples, fn=Dict({
'input_ids': Pad(axis=0, pad_val=tokenizer.pad_token_id, dtype='int32'), # input
'token_type_ids': Pad(axis=0, pad_val=tokenizer.pad_token_type_id, dtype='int32'), # segment
'labels': Pad(axis=0, pad_val=args.ignore_label, dtype='int64') # label
}): fn(samples)
return batchify_fn
def qa_collator(tokenizer, args):
train_batchify_fn = lambda samples, fn=Dict(
{
"input_ids": Pad(axis=0, pad_val=tokenizer.pad_token_id),
"token_type_ids": Pad(axis=0, pad_val=tokenizer.pad_token_type_id),
"start_positions": Stack(dtype="int64"),
"end_positions": Stack(dtype="int64")
}): fn(samples)
return train_batchify_fn
def get_metas_sat(self, bounds, time):
"""
Get all the meta data for all the necessary products
:param bounds: bounding box as (lon_min,lon_max,lat_min,lat_max)
:param time: time interval (init_time_datetime,final_time_datetime)
:param maxg: max number of granules to process
:return metas: dictionary with the metadata of all the products
"""
return Dict({'fire': self.search_api_sat(None,None,time)})
def _get_session(session_id):
"""
get session from the runtime memory, which store as dictionary.
Using lock to make thread-safe.
:param session_id: the session id assigned to each request.
:return: a Dict
"""
with _session_lock:
if session_id not in _SESSIONS_WAREHOUSE:
_SESSIONS_WAREHOUSE[session_id] = Dict()
return _SESSIONS_WAREHOUSE[session_id]
def readMazeFromFile(fileName):
matrix = {}
i = 0
start = (0, 0)
goal = (0, 0)
with open(fileName) as f:
start = tuple(map(int, f.readline().rstrip('\n').strip('()').split(',')))
goal = tuple(map(int, f.readline().rstrip('\n').strip('()').split(',')))
for line in f.readlines():
line = line.rstrip('\n')
j = 0
for cost in line:
if int(cost) != 0:
matrix[(j, i)] = int(cost)
j = j + 1
i = i + 1
graph = Dict()
for point in matrix.keys():
entry = {point: {}}
child1 = None
child2 = None
try:
child1 = {(point[0], point[1] + 1): matrix[(point[0], point[1] + 1)]}
except KeyError:
pass
try:
child2 = {(point[0] + 1, point[1]): matrix[(point[0] + 1, point[1])]}
except KeyError:
pass
if child1 is not None:
entry[point].update(child1)
if child2 is not None:
entry[point].update(child2)
graph.update(entry)
undirGraph = UndirectedGraph(graph)
undirGraph.locations = Dict()
for i in undirGraph.nodes():
undirGraph.locations.update({i: i})
graphProblem = ManhattenGraphProblem(start, goal, undirGraph)
return graphProblem
def input(self, **kw):
"""
返回一个由传入的数据和从environ里取出的数据 组成的Dict对象，Dict对象的定义 见db模块
Get input as dict from request, fill dict using provided default value if key not exist.
i = ctx.request.input(role='guest')
i.role ==> 'guest'
"""
copy = Dict(**kw)
raw = self._get_raw_input()
for k, v in raw.iteritems():
copy[k] = v[0] if isinstance(v, list) else v
return copy
async def get_info(self, name):
info = Dict()
if self.app.options.auth and not name.startswith('public'):
doc = await self.db.share.find_one({
'name': name,
'token': self.current_user.token
})
info.share = True if doc else False
if self.app.options.git and self.current_user.admin:
doc = await self.db.files.find_one({'name': name})
info.git = True if doc else False
return info
def input(self, **kw):
"""Get input as dict from request, fill dict using provided default value if key not exist.
i = ctx.request.input(role='guest')
i.role ==> 'guest'
:param kw:
:return:
"""
copy = Dict(**kw)
raw = self._get_raw_input()
for k, v in raw.iteritems():
copy[k] = v[0] if isinstance(v, list) else v
return copy
def load_job_file(job_id):
jobfile = osp.join(cfg['workspace_path'], job_id, 'job.json')
logging.info('Loading job state from %s' % jobfile)
try:
js = Dict(json.load(open(jobfile, 'r')))
except IOError:
logging.error('Cannot open %s' % jobfile)
js = None
except:
logging.error('Cannot load %s' % jobfile)
js = None
# print(json.dumps(js, indent=4, separators=(',', ': ')))
return js, jobfile
def base_select(sql, single, *args):
"""execute select SQL and fetch results."""
global db_context
cursor = None
sql = sql.replace('?', '%s')
logging.info('SQL: %s, ARGS: %s' % (sql, args))
try:
cursor = db_context.connection.cursor()
cursor.execute(sql, args)
if cursor.description:
names = [x[0] for x in cursor.description]
else:
raise DateBaseError("No cursor description.")
if single:
values = cursor.fetchone()
if not values:
return None
return Dict(names, values)
return [Dict(names, x) for x in cursor.fetchall()]
finally:
if cursor:
cursor.close()
def preview_tar(self, path):
tf = tarfile.open(path)
items = tf.getmembers()
entries = []
for item in items:
entries.append(
Dict({
'path': Path(item.name),
'mtime': item.mtime,
'size': item.size,
'is_dir': item.isdir(),
}))
self.render('index.html', entries=entries, absolute=True)
def transform_goes(d):
"""
Transform GOES mask data into similar to MODIS-VIIRS AF mask
:param d: open NetCDF4 dataset
"""
fill = Dict({
(150., 151., 152., 153.): 3,
(100.): 5,
(15., 35.): 7,
(14., 34.): 8,
(10., 11., 12., 13., 30., 31., 32., 33.): 9
})
array = d.variables['Mask'][:]
return fill_categories(array, fill)
def parse_fmda(self, args):
"""
Parse information inside the FMDA blob, if any.
:param args: the forecast job argument dictionary
"""
if 'fuel_moisture_da' in args:
fmda = args['fuel_moisture_da']
return Dict({
'token': fmda['mesowest_token'],
'domains': fmda['domains']
})
else:
return None
def _select(sql, first, *args):
' execute select SQL and return unique result or list results.'
global _db_ctx, _db_convert
cursor = None
if _db_convert != '?':
sql = sql.replace('?', _db_convert)
_log('SQL: %s, ARGS: %s' % (sql, args))
start = time.time()
try:
cursor = _db_ctx.connection.cursor()
cursor.execute(sql, args)
if cursor.description:
names = [x[0] for x in cursor.description]
if first:
values = cursor.fetchone()
if not values:
return None
return Dict(names, values)
return [Dict(names, x) for x in cursor.fetchall()]
finally:
if cursor:
cursor.close()
_profiling(start, sql)
