def testing(ch, method, properties, body):
print(" [x] Received %r" % body)
ch.basic_ack(delivery_tag=method.delivery_tag)
server = parse_config_l('config.config')
download(server, body)
metadata(server, body)
downloadPolicy(server, body)
def bash(cmd, arg=None):
ALLOW_COMMAND = metadata('srce_command', _ALLOW_COMMAND)
PATH = metadata('srce_path', _PATH)
if cmd not in ALLOW_COMMAND.keys():
return '', 403
if arg:
if arg not in ALLOW_COMMAND[cmd]:
return '', 403
command = [f'{PATH}{cmd}', arg]
else:
command = f'{PATH}{cmd}'
emailNotify(auth.username, request.remote_addr, command)
proc = Popen(command, stdout=-1, stderr=-1)
try:
outs, errs = proc.communicate(timeout=30)
except TimeoutExpired:
return 'Process still running.'
return Response(f'Output:\n{outs.decode()}\n\nError:\n{errs.decode()}', mimetype='text/plain')
def _stat(self, path):
if '/' != path[0]:
path = '/' + path
if not self._query_path(path):
print(
"Invalid stat command: the file or directory does not exist.")
else:
info = self._query_metadata(path)
sample = metadata(info)
sample.display()
def new_thread(self, conn):
while True:
request = conn.recv(4096).decode()
if request == "#finished#":
conn.sendall("#finished#".encode())
break
print(f"server: {request}")
command, content = request.split(' -> ')
if command == 'insert':
sample = metadata(content)
if sample.path not in self.record:
self.record[sample.path] = [content, set()]
else:
self.record[sample.path][0] = content
elif command == 'query_path':
if content in self.record:
resp = self.record[content][0].split(', ')[2]
else:
resp = 'N'
conn.sendall(resp.encode())
elif command == 'add_dir':
dir_path, filename = content.split(':')
self.record[dir_path][1].add(filename)
elif command == 'query':
if content in self.record:
resp = ';;'.join(list(self.record[content][1]))
else:
resp = ''
if not resp:
resp = "##none##"
conn.sendall(resp.encode())
elif command == 'remove':
if content in self.record:
del self.record[content]
elif command == 'query_metadata':
resp = self.record[content][0]
conn.sendall(resp.encode())
elif command == 'distribution':
if self.record:
resp = sorted(self.record.keys())
resp = "\n".join(list(map(lambda x: "\t" + x, resp)))
else:
resp = "Empty"
conn.sendall(resp.encode())
else:
print("No corresponding response.")
print(self.record)
conn.close()
def commit(name, content):
GITHUB = metadata('feh_github', _GITHUB)
url = f"https://api.github.com/repos/{GITHUB['user']}/{GITHUB['repo']}/contents/{GITHUB['path']}/{name}.json"
data = {'message': name, 'content': content}
headers = {'Authorization': f"token {GITHUB['token']}"}
if PROXY:
proxies = {'https': f'http://{PROXY}'}
else:
proxies = None
return requests.put(url,
json=data,
headers=headers,
proxies=proxies,
timeout=10)
def emailNotify(user, ip, cmd):
SUBSCRIBE = metadata('srce_subscribe', _SUBSCRIBE)
msg = EmailMessage()
msg['Subject'] = f"SRCE Notification - {datetime.now().strftime('%Y%m%d %H:%M:%S')}"
msg['From'] = SUBSCRIBE['sender']
msg['To'] = SUBSCRIBE['subscriber']
if isinstance(cmd, list):
cmd = ' '.join(cmd)
msg.set_content(
f"{datetime.now().strftime('%Y/%m/%d-%H:%M:%S')}\nUser: {user}\nIP: {ip}\n\nCommand: {cmd}")
with SMTP(SUBSCRIBE['smtp_server'], SUBSCRIBE['smtp_server_port']) as s:
s.starttls()
s.login(SUBSCRIBE['sender'], SUBSCRIBE['password'])
s.send_message(msg)
def mail(feh: FEH_VotingGauntlet):
Round = {1: 'Round1', 2: 'Round2', 3: 'Final Round'}
timestamp = f"{feh.date.strftime(f'%Y%m%d')} {feh.hour}:00:00"
msg = EmailMessage()
msg['Subject'] = f'FEH 投票大戦第{feh.current_event}回 {Round[feh.current_round]} - {timestamp}'
content = '\n'.join(
[formatter(battle) for battle in feh.current_scoreboard])
msg.set_content(f'{content}\n\n{timestamp}')
SUBSCRIBE = metadata('feh_subscribe', _SUBSCRIBE)
msg['From'] = SUBSCRIBE['sender']
msg['To'] = SUBSCRIBE['subscriber']
with SMTP(SUBSCRIBE['smtp_server'], SUBSCRIBE['smtp_server_port']) as s:
s.starttls()
s.login(SUBSCRIBE['sender'], SUBSCRIBE['password'])
s.send_message(msg)
def _initialize_insert(self, line):
sample = metadata(line)
temp = sample.path.split("/")
pre_path = None
cur_index = -1
self._insert(sample)
while pre_path != '/':
pre_path = '/'.join(temp[:cur_index])
if pre_path == '':
pre_path = '/'
filename = temp[cur_index]
cur_index -= 1
if not self._query_path(pre_path):
self._create(pre_path, "yes")
self._add_to_dir(filename, pre_path)
def query(filter_or_pipeline=None,
projection=None,
sort=None,
limit=0,
mode='find'):
MONGO = metadata('feh_mongo', _MONGO)
if MONGO['username']:
username = quote_plus(MONGO['username'])
password = quote_plus(MONGO['password'])
URI = f"mongodb://{username}:{password}@{MONGO['server']}:{MONGO['port']}/{MONGO['database']}"
else:
URI = f"mongodb://{MONGO['server']}:{MONGO['port']}"
with MongoClient(URI) as client:
if mode == 'find':
return list(client[MONGO['database']][MONGO['collection']].find(
filter_or_pipeline, projection, sort=sort, limit=limit))
else:
return list(client[MONGO['database']][
MONGO['collection']].aggregate(filter_or_pipeline))
def isvalid(id_num):
"""
Check if a gutenberg book is an english textbook.
Args:
id_num: id of gutenberg book.
Returns:
a boolean which shows if id_num is id of an english text book
"""
try:
language = metadata(id_num)['language']
form = get_metadata('formaturi', id_num)
if 'en' not in language:
return False
form = ' '.join(form)
if re.search(r'\d+\.txt', form):
return True
return False
except:
return False
def mongo(feh: FEH_VotingGauntlet):
MONGO = metadata('feh_mongo', _MONGO)
if MONGO['username']:
username = quote_plus(MONGO['username'])
password = quote_plus(MONGO['password'])
URI = f"mongodb://{username}:{password}@{MONGO['server']}:{MONGO['port']}/{MONGO['database']}"
else:
URI = f"mongodb://{MONGO['server']}:{MONGO['port']}"
try:
with MongoClient(URI) as client:
collection = client[MONGO['database']][MONGO['collection']]
update = {
'event': feh.current_event,
'date': feh.date,
'hour': feh.hour
}
all_scoreboard = feh.scoreboard
for Round in all_scoreboard:
update['round'] = Round
for battle in all_scoreboard[Round]:
collection.update_one({'scoreboard': battle},
{'$setOnInsert': update}, True)
except:
pass
def to_numpy(raster, numpy_datatype=None):
"""
Wrapper for arcpy.RasterToNumpyArray with better metadata handling
This is just a wraper for the RasterToNumPyArray function within arcpy, but it also
extracts out all the spatial referencing information that will probably be needed
to save the raster after desired manipulations have been performed.
also see raster.from_numpy function in this module.
:param raster: Any raster supported by the arcpy.RasterToNumPyArray function
:param numpy_datatype: must be a string equal to any of the types listed at the following
address [http://docs.scipy.org/doc/numpy/user/basics.types.html]
for example: 'uint8' or 'int32' or 'float32'
:return numpy_rast: the numpy array version of the input raster
:return Metadata: a metadata object. see ``raster.metadata``
"""
# perform some checks to convert to supported data format
if not is_rast(raster):
try:
print(
"Raster '{0}' may not be supported, converting to tif".format(
raster))
tifraster = raster + ".tif"
if not os.path.exists(raster + ".tif"):
arcpy.CompositeBands_management(raster, tifraster)
raster = tifraster
except:
raise Exception("Raster type could not be recognized")
# read in the raster as a numpy array
numpy_rast = arcpy.RasterToNumPyArray(raster)
# build metadata for multi band raster
if len(numpy_rast.shape) == 3:
zs, ys, xs = numpy_rast.shape
meta = []
for i in range(zs):
bandpath = raster + "\\Band_{0}".format(i + 1)
meta.append(metadata(bandpath, xs, ys))
if numpy_datatype is None:
numpy_datatype = meta[0].numpy_datatype
# build metadata for single band raster
else:
ys, xs = numpy_rast.shape
meta = metadata(raster, xs, ys)
if numpy_datatype is None:
numpy_datatype = meta.numpy_datatype
numpy_rast = numpy_rast.astype(numpy_datatype)
# mask NoData values from the array
if 'float' in numpy_datatype:
numpy_rast[numpy_rast == meta.NoData_Value] = numpy.nan
numpy_rast = numpy.ma.masked_array(numpy_rast,
numpy.isnan(numpy_rast),
dtype=numpy_datatype)
elif 'int' in numpy_datatype: # (numpy.nan not supported by ints)
mask = numpy.zeros(numpy_rast.shape)
mask[numpy_rast != meta.NoData_Value] = False # do not mask
mask[numpy_rast == meta.NoData_Value] = True # mask
numpy_rast = numpy.ma.masked_array(numpy_rast,
mask,
dtype=numpy_datatype)
return numpy_rast, meta
def __init__(self, path, directory, relaxed=False, downsample=1):
self.path = path
self.directory = directory
self.metadata = metadata.metadata(path)
self.Evaluate(relaxed, downsample)
ARTWORK = utils.ARTWORK
ICON = utils.ICON
URL = utils.URL
SECTION = 100
SERIES = 200
EPISODE = 300
HOST = 400
DOWNLOAD = 500
MARKWATCHED = 600
MARKUNWATCHED = 601
AUTOPLAY = ADDON.getSetting('AUTOPLAY') == 'true'
import metadata
meta = metadata.metadata()
meta.SetDir(os.path.join(PROFILE, 'watched'))
meta.SetImageDir(os.path.join(PROFILE, 'images'))
def CheckVersion():
prev = ADDON.getSetting('VERSION')
curr = VERSION
if prev == curr:
return
ADDON.setSetting('VERSION', curr)
if curr == '1.0.17':
d = xbmcgui.Dialog()
fallback=587),
'password': config.get('email', 'PWD'),
'subscriber': config.get('email', 'SUBSCRIBER')
}
_MONGO = {
'server': config.get('mongodb', 'SERVER', fallback='localhost'),
'port': config.getint('mongodb', 'PORT', fallback=27017),
'database': config.get('mongodb', 'DATABASE', fallback='feh'),
'collection': config.get('mongodb', 'COLLECTION', fallback='feh'),
'username': config.get('mongodb', 'AUTH', fallback='feh'),
'password': config.get('mongodb', 'PASSWORD', fallback='feh')
}
if __name__ == '__main__':
SUBSCRIBE = metadata('feh_subscribe', _SUBSCRIBE)
MONGO = metadata('feh_mongo', _MONGO)
command = f"mongodump -h{MONGO['server']}:{MONGO['port']} -d{MONGO['database']} -c{MONGO['collection']} -u{MONGO['username']} -p{MONGO['password']} --gzip --archive"
attachment = BytesIO()
attachment.write(check_output(command, shell=True))
msg = EmailMessage()
msg['Subject'] = f'FEH Backup-{datetime.now():%Y%m%d}'
msg['From'] = SUBSCRIBE['sender']
msg['To'] = SUBSCRIBE['subscriber']
msg.add_attachment(attachment.getvalue(),
maintype='application',
subtype='octet-stream',
filename='database')
with SMTP(SUBSCRIBE['smtp_server'], SUBSCRIBE['smtp_server_port']) as s:
s.starttls()
s.login(SUBSCRIBE['sender'], SUBSCRIBE['password'])
def to_numpy(raster, numpy_datatype=None):
"""
Wrapper for arcpy.RasterToNumpyArray with better metadata handling
This is just a wraper for the RasterToNumPyArray function within arcpy, but it also
extracts out all the spatial referencing information that will probably be needed
to save the raster after desired manipulations have been performed.
also see raster.from_numpy function in this module.
inputs:
Raster Any raster supported by the arcpy.RasterToNumPyArray function
numpy_datatype must be a string equal to any of the types listed at the following
address [http://docs.scipy.org/doc/numpy/user/basics.types.html]
for example: 'uint8' or 'int32' or 'float32'
outputs:
numpy_rast the numpy array version of the input raster
Metadata An object with the following attributes.
.Xmin the left edge
.Ymin the bottom edge
.Xmax the right edge
.Ymax the top edge
.Xsize the number of columns
.Ysize the number of rows
.cellWidth resolution in x direction
.cellHeight resolution in y direction
.projection the projection information to give the raster
.NoData_Value the numerical value which represents NoData in this raster
Usage example:
call this function with " rast,Metadata = to_numpy(Raster) "
perform numpy manipulations as you please
then save the array with " raster.from_numpy(rast,Metadata,output) "
"""
# create a metadata object and assign attributes to it
# perform some checks to convert to supported data format
if not is_rast(raster):
try:
print(
"Raster '{0}' may not be supported, converting to tif".format(
raster))
tifraster = raster + ".tif"
if not os.path.exists(raster + ".tif"):
arcpy.CompositeBands_management(raster, tifraster)
raster = tifraster
except:
raise Exception("Raster type could not be recognized")
# read in the raster as a numpy array
numpy_rast = arcpy.RasterToNumPyArray(raster)
# build metadata for multi band raster
if len(numpy_rast.shape) == 3:
zs, ys, xs = numpy_rast.shape
meta = []
for i in range(zs):
bandpath = raster + "\\Band_{0}".format(i + 1)
meta.append(metadata(bandpath, xs, ys))
if numpy_datatype is None:
numpy_datatype = meta[0].numpy_datatype
# build metadata for single band raster
else:
ys, xs = numpy_rast.shape
meta = metadata(raster, xs, ys)
if numpy_datatype is None:
numpy_datatype = meta.numpy_datatype
numpy_rast = numpy_rast.astype(numpy_datatype)
# mask NoData values from the array
if 'float' in numpy_datatype:
numpy_rast[numpy_rast == meta.NoData_Value] = numpy.nan
numpy_rast = numpy.ma.masked_array(numpy_rast,
numpy.isnan(numpy_rast),
dtype=numpy_datatype)
elif 'int' in numpy_datatype: # (numpy.nan not supported by ints)
mask = numpy.zeros(numpy_rast.shape)
mask[numpy_rast != meta.NoData_Value] = False # do not mask
mask[numpy_rast == meta.NoData_Value] = True # mask
numpy_rast = numpy.ma.masked_array(numpy_rast,
mask,
dtype=numpy_datatype)
return numpy_rast, meta
def create_single_h5(
dirpath,
movepath_h5,
movepath_wf,
verbose=False
): #dirpath - path to directory which contains the strain data of the waveform
'''Generate Waveform HDF5 file from simulation data
dirpath(str) - simulation output directory with strain data as generated by MATLAB script
movepath_h5 (str) - Final path to store h5 file
movepath_wf (str) - Final path to store simulation output data
verbose (bool) - Turn on for debugging
'''
localsim_name = dirpath.split('/')[-1]
print(
"*(Create_Single_h5) >> Constructing h5 waveform for simulation - {} \n "
.format(localsim_name))
#Add path of directories to read data and save figures-
figdir = os.path.join(dirpath, "figures")
datadir = os.path.join(dirpath, "data")
strdir = os.path.join(datadir, "Strain")
#Check if the h5 file already exists
simname = simulation_name(dirpath)
h5check_path = glob.glob(os.path.join(movepath_h5, "*/%s.h5" % simname))
if len(h5check_path) > 0:
if os.path.exists(h5check_path[0]):
raise NameError(
"*(Create_Single_h5) >> h5file %s already exists. Please delete the old file before constructing the new one. Waveform getting moved to Failed Directory"
% simname)
#Create a dictionary to store all the modes
nr_strain_data = {}
lmax = ltemp = 0
# Safety Check
if not os.path.isfile(strdir + '/Strain_l2_m2.txt'):
error('*(Create_Single_h5) >> Strain Files Missing \n')
#The for loop ensures all the modes are included in this dictionary
for strfile in sorted(glob.glob(strdir + '/Strain*.txt')):
# NOTE: here we determine l and m from the filename. This assumes that each
# strain file has data for only one l and one m.
l = int((os.path.basename(strfile).split('_')[1])[1])
m = int((os.path.basename(strfile).split('_')[-1]).split('.')[0][1:])
# Find Lmax for metadata
ltemp = l
if ltemp > lmax:
lmax = ltemp
# Load the data
data_array = loadtxt(strfile, comments='#')
# Unpack the data array (get time, plus and cross )
time_uncrop, hp_uncrop, hx_uncrop = data_array[:,
0], data_array[:,
1], data_array[:,
2]
# Crop the data to remove the junk radiation
time_crop, hp, hx, crop_idx, junkrad_time = crop_data(
dirpath, time_uncrop, hp_uncrop, hx_uncrop)
time = time_crop
# Calculate the waveform amplitude and phase (Unwrapping phase - Request from NR-injection Infrastructure
A = abs(hp - 1.j * hx)
phase = -1.0 * unwrap(angle(hp - 1j * hx))
# NOTE: The NR-Injection infrastructure requests that phase decreases if m is positive. Here we multiply by -1 to enforce this.
phase *= -1
# Set all m=0 mode to 0:
# This is just temporary measure to fix the issue with m=0 modes. Better method would be to find the right tuning frequency by comparison with Quasi normal modes obtained from perturbation theory
if m == 0: #l==2 and m==0:
A *= 0.
phase *= 0.
# Create a dictionary that contains the NR data. This will be used as an input to the function that actually makes the h5 file.
nr_strain_data[(l, m)] = {'amp': A, 'phase': phase, 't': time}
# Compute the index of maximum of strain amplitude - all modes included
amp_sq = np.zeros(len(nr_strain_data[(2, 2)]['t']))
for key in nr_strain_data:
amp_sq = amp_sq + nr_strain_data[key]['amp']**2
if (np.all(amp_sq == 0)):
raise ValueError(
"*(Create_Single_h5) >> %s waveform is all zeros. Waveform getting moved to Failed Directory"
% simname)
amp_max_idx = np.where(amp_sq == np.amax(amp_sq))
amp22_max_idx = np.where(
nr_strain_data[(2, 2)]['amp'] == np.amax(nr_strain_data[(2,
2)]['amp']))
# Mean Center the data about max amplitude
for key in nr_strain_data:
time = nr_strain_data[key]['t'] - nr_strain_data[key]['t'][amp_max_idx]
nr_strain_data[key]['t'] = time
# Create plots of amplitude and phase for mean centered time
if key == (2, 2):
create_plot(time, nr_strain_data[key]['amp'], 'Time', 'h+', figdir,
'Amp_22_plot.png')
create_plot(time, nr_strain_data[key]['phase'], 'Time', 'Phase',
figdir, 'Phase_22_plot.png')
amp_lm = nr_strain_data[key][
'amp'] #t22 - t22[abs(h22)==np.amax(abs(h22))]
phase_lm = nr_strain_data[key][
'phase'] #unwrap(angle(hp_22 - 1j*hx_22))
##Testing Purpose Only - Save the file and compare with spline interpolant
#data = np.column_stack((time, amp_lm, phase_lm))
#hdr = "Time \t Amplitude \t Phase \n"
#
#timeshift_dir = os.path.join(strdir, "Strain_TimeShifted")
#if not os.path.exists(timeshift_dir): os.makedirs(timeshift_dir)
#l,m = key[0], key[1]
#strfile_lm = os.path.join(timeshift_dir,"Strain_{}{}_timeshift.asc".format(l,m))
#np.savetxt(strfile_lm, data, header=hdr, delimiter='\t')
# Compute the gravitational wave frequency
time_22 = nr_strain_data[(2, 2)]['t']
amp_22 = nr_strain_data[(
2, 2)]['amp'] #t22 - t22[abs(h22)==np.amax(abs(h22))]
phase_22 = nr_strain_data[(2,
2)]['phase'] #unwrap(angle(hp_22 - 1j*hx_22))
omega_22 = -1. * np.gradient(phase_22) / np.gradient(
time_22
) #(phase_uncrop[1:] - phase_uncrop[:-1])/(time_uncrop[1:]-time_uncrop[:-1])
create_plot(time_22, omega_22, 'Time', 'MOmega', figdir,
'Omega_22_plot.png')
# Compute inital GW frequency in code units (for (2,2) mode)
Mf_initial = omega_22[0]
if verbose:
print(
"*(Create_single_h5) >> Frequency of (2,2) mode after junk radiation - Momega = {} \n"
.format(Mf_initial))
# Check with frequency of ShiftTracker
omega_st = find_omega22_st(dirpath, junkrad_time - 75)
if verbose:
print(
'*(Create_single_h5) >> Gravitational Wave frequency from strain = %g/M, and from orbital frequency = %g/M \n'
% (Mf_initial, 2. * omega_st[0]))
# Compute the starting frquency of the waveform in SI units for 1Msun system
G = 6.67428e-11 # m^3/(kg s^2)
mass_sun = 1.98892e30 # kg
c = 2.99792458e8 # m/s
mass_sec = G * mass_sun / (c * c * c)
simulation_mass_sec = (
1.0) * mass_sec # NOTE that the 1.0 here is for 1solar mass
#simulation_mass_sec = 4.92549102554*10**-6
physical_freq_initial_at_1Msol = (Mf_initial /
(2. * np.pi)) / simulation_mass_sec
if verbose:
print(
'*(Create_single_h5) >> Gravitational Wave frequency from strain = %gHz, and from orbital frequency = %gHz \n'
% (physical_freq_initial_at_1Msol,
(2. * omega_st[0] / (2. * np.pi) / simulation_mass_sec)))
# Compute initial data
nr_initdata = initial_data(dirpath)
# Compute the nr_metadata
nr_meta_data = metadata(dirpath, junkrad_time, verbose=verbose)
nr_meta_data['Lmax'] = lmax
#nr_meta_data['Omega'] = Mf_initial
nr_meta_data['f_lower_at_1MSUN'] = physical_freq_initial_at_1Msol
# Create the h5 file and move to corresponding directory, move the waveform to completed directory
outpath = movepath_h5 + ('/%s.h5' % nr_meta_data['name'])
print('*(Create_single_h5) >> Creating HDF5 files:{}'.format(outpath))
nr2h5(nr_strain_data,
nr_meta_data,
nr_initdata,
output_path=outpath,
wfdir_path=dirpath,
verbose=True,
testsuite=False)
if nr_meta_data['simulation-type'] == 'non-spinning':
movepath_h5 = os.path.join(movepath_h5, 'NonSpinning')
movepath_wf = os.path.join(movepath_wf, 'NonSpinning')
elif nr_meta_data['simulation-type'] == 'aligned-spins':
movepath_h5 = os.path.join(movepath_h5, 'AlignedSpin')
movepath_wf = os.path.join(movepath_wf, 'AlignedSpin')
elif nr_meta_data['simulation-type'] == 'precessing':
movepath_h5 = os.path.join(movepath_h5, 'Precessing')
movepath_wf = os.path.join(movepath_wf, 'Precessing')
print(movepath_h5)
sh.move(dirpath, movepath_wf)
sh.move(outpath, movepath_h5)
def to_numpy(raster, numpy_datatype = None):
"""
Wrapper for arcpy.RasterToNumpyArray with better metadata handling
This is just a wraper for the RasterToNumPyArray function within arcpy, but it also
extracts out all the spatial referencing information that will probably be needed
to save the raster after desired manipulations have been performed.
also see raster.from_numpy function in this module.
inputs:
Raster Any raster supported by the arcpy.RasterToNumPyArray function
numpy_datatype must be a string equal to any of the types listed at the following
address [http://docs.scipy.org/doc/numpy/user/basics.types.html]
for example: 'uint8' or 'int32' or 'float32'
outputs:
numpy_rast the numpy array version of the input raster
Metadata An object with the following attributes.
.Xmin the left edge
.Ymin the bottom edge
.Xmax the right edge
.Ymax the top edge
.Xsize the number of columns
.Ysize the number of rows
.cellWidth resolution in x direction
.cellHeight resolution in y direction
.projection the projection information to give the raster
.NoData_Value the numerical value which represents NoData in this raster
Usage example:
call this function with " rast,Metadata = to_numpy(Raster) "
perform numpy manipulations as you please
then save the array with " raster.from_numpy(rast,Metadata,output) "
"""
# create a metadata object and assign attributes to it
# perform some checks to convert to supported data format
if not is_rast(raster):
try:
print("Raster '{0}' may not be supported, converting to tif".format(raster))
tifraster = raster + ".tif"
if not os.path.exists(raster + ".tif"):
arcpy.CompositeBands_management(raster, tifraster)
raster = tifraster
except:
raise Exception("Raster type could not be recognized")
# read in the raster as a numpy array
numpy_rast = arcpy.RasterToNumPyArray(raster)
# build metadata for multi band raster
if len(numpy_rast.shape) == 3:
zs, ys, xs = numpy_rast.shape
meta = []
for i in range(zs):
bandpath = raster + "\\Band_{0}".format(i+1)
meta.append(metadata(bandpath, xs, ys))
if numpy_datatype is None:
numpy_datatype = meta[0].numpy_datatype
# build metadata for single band raster
else:
ys, xs = numpy_rast.shape
meta = metadata(raster, xs, ys)
if numpy_datatype is None:
numpy_datatype = meta.numpy_datatype
numpy_rast = numpy_rast.astype(numpy_datatype)
# mask NoData values from the array
if 'float' in numpy_datatype:
numpy_rast[numpy_rast == meta.NoData_Value] = numpy.nan
numpy_rast = numpy.ma.masked_array(numpy_rast, numpy.isnan(numpy_rast),
dtype = numpy_datatype)
elif 'int' in numpy_datatype: # (numpy.nan not supported by ints)
mask = numpy.zeros(numpy_rast.shape)
mask[numpy_rast != meta.NoData_Value] = False # do not mask
mask[numpy_rast == meta.NoData_Value] = True # mask
numpy_rast = numpy.ma.masked_array(numpy_rast, mask,
dtype = numpy_datatype)
return numpy_rast, meta
URL = utils.URL
SECTION = 100
SERIES = 200
EPISODE = 300
HOST = 400
DOWNLOAD = 500
MARKWATCHED = 600
MARKUNWATCHED = 601
AUTOPLAY = ADDON.getSetting('AUTOPLAY') == 'true'
import metadata
meta = metadata.metadata()
meta.SetDir(os.path.join(PROFILE ,'watched'))
def CheckVersion():
prev = ADDON.getSetting('VERSION')
curr = VERSION
if prev == curr:
return
ADDON.setSetting('VERSION', curr)
if curr == '1.0.17':
d = xbmcgui.Dialog()
d.ok(TITLE + ' - ' + VERSION, 'Welcome to Watch Cartoon Online', 'Now with download feature.', '')
(
{}, {}, '{}', '{}'
);
'''
# Create computation log
query = query.format(metric_id, result_id, data_datetime, meta_name)
try:
db = connection.connect(query)
except Exception, err:
print 'METRIC_ID = ', metric_id
print 'DATA_DATETIME = ', data_datetime
print 'ERROR MESSAGE = ', err
# Create metadata for feature engineering
meta = metadata(metric_id, meta_name, sql_new)
try:
meta.dropTable()
meta.createTable()
except Exception, err:
print err
pass
if __name__ == '__main__':
metric_id = 6
print load(metric_id)
print 'sql:'
print collect(metric_id)
log_file_path = '/home/ubuntu/datasets/nasa_log_jul' # path to extract the log file
regex = "(.*) - - \[(.*)\] \"([A-Z]+) (.*)\" ([0-9]{3}) (-|[0-9]+)" # regex to get relevant information from a log entry
pattern = re.compile(regex)
FAMILIES = ['log_info','loca_info'] # column family names for server_logs_table
TABLE = 'server_logs' # table
LOG_COLS = ["log_info :host", "log_info:server_ts", "log_info:type", "log_info:url", "log_info:status", "log_info:bytes"] #columns to insert into server_logs
# hbase_client.truncate_table(TABLE, FAMILIES)
tgt_time_fmt = "%Y-%m-%d %H:%M:%S" # target date format
src_time_fmt = "%d/%b/%Y:%H:%M:%S %z" # source date format
denied_requests = [str(a) for a in range(400,500)]
start_time = "1995-07-01 00:00:00" # start time for server log entries
meta = metadata()
meta.initialize_timer(start_time=start_time)
window = meta.next_bound()
lower_b = time.mktime(time.strptime(start_time, tgt_time_fmt))
upper_b = next(window)
geo_ip = GeoIPUtil()
BATCH_LIMIT = 100
count = 1
batch_index = 1
fail_batch_count = 0
"""
Metadata operations for inserting one row of metadata
def metadata(_, value):
return value
app = create_app()
_BACKUP = {
'sender': 'SENDER',
'smtp_server': 'SMTP_SERVER',
'smtp_server_port': 587,
'password': '******',
'subscriber': 'SUBSCRIBER'
}
BACKUP = metadata('mybookmarks_backup', _BACKUP)
@click.group(invoke_without_command=True)
@click.pass_context
def cli(ctx):
if ctx.invoked_subcommand is None:
ctx.invoke(run)
@cli.command(short_help='Add User')
@click.argument('username')
def add(username):
db = sqlite3.connect(app.config['DATABASE'])
try:
db.executescript(
def find_key(key):
md = metadata()
return list(gen_dict_extract(key, md))
def get_pw(username):
ALLOW_USERS = metadata('srce_user', _ALLOW_USERS)
if username in ALLOW_USERS:
return ALLOW_USERS[username]
return None
import crossover as c
from metadata import metadata
from random import randint
md = metadata() #initializa meta data
population_count = 100 # even
def init():
population = []
total_fitness = 0
while i < population_count:
c = chromosome(md)
c.initializeRandomly()
total_fitness += c.fitness
population.append(c)
i+=1
return population, total_fitness
def reproduce(population, crossover):
new_population = []
total_fitness = 0
while i < population_count/2:
x, y = selection(population)
a = chromosome(md)
a.initalizeCrossover(x)
import smallCounty
import table
import metadata
import datetime
import pandas as pd
if __name__ == '__main__':
df_smallCounty = smallCounty.smallCounty(
"http://cmdweb.pcc.gov.tw/pccms/owa/guesmap.userinn")
df_smallCounty.to_csv(datetime.datetime.now().strftime("%Y%m%d_%H-%M-%S") +
"_smallCounty.csv",
encoding="utf_8_sig")
data_list1 = []
for tableLink, name in zip(df_smallCounty["table連結"],
df_smallCounty["鄉鎮"]):
print(tableLink, name)
data_list1.extend(table.table(tableLink))
df_table = table.getTableDF(data_list1)
df_table.to_csv(datetime.datetime.now().strftime("%Y%m%d_%H-%M-%S") +
"_engineering.csv",
encoding="utf_8_sig")
#df_table = pd.read_csv("20191002_09-51-09_engineering.csv")
data_list = []
for metadataLink in df_table["詳細連結"]:
data_list.append(metadata.metadata(metadataLink))
df_metadata = metadata.getMetadataDF(data_list)
df_metadata.to_csv(datetime.datetime.now().strftime("%Y%m%d_%H-%M-%S") +
"_metadata.csv",
encoding="utf_8_sig")
def configure(self, env):
import params
env.set_params(params)
metadata()
def configure(self, env): import params env.set_params(params) metadata()
def __init__(self,path,directory): self.path = path self.directory = directory self.metadata = metadata.metadata(path) self.Evaluate()
def to_numpy(raster, numpy_datatype = None):
"""
Wrapper for arcpy.RasterToNumpyArray with better metadata handling
This is just a wraper for the RasterToNumPyArray function within arcpy, but it also
extracts out all the spatial referencing information that will probably be needed
to save the raster after desired manipulations have been performed.
also see raster.from_numpy function in this module.
:param raster: Any raster supported by the arcpy.RasterToNumPyArray function
:param numpy_datatype: must be a string equal to any of the types listed at the following
address [http://docs.scipy.org/doc/numpy/user/basics.types.html]
for example: 'uint8' or 'int32' or 'float32'
:return numpy_rast: the numpy array version of the input raster
:return Metadata: a metadata object. see ``raster.metadata``
"""
# perform some checks to convert to supported data format
if not is_rast(raster):
try:
print("Raster '{0}' may not be supported, converting to tif".format(raster))
tifraster = raster + ".tif"
if not os.path.exists(raster + ".tif"):
arcpy.CompositeBands_management(raster, tifraster)
raster = tifraster
except:
raise Exception("Raster type could not be recognized")
# read in the raster as a numpy array
numpy_rast = arcpy.RasterToNumPyArray(raster)
# build metadata for multi band raster
if len(numpy_rast.shape) == 3:
zs, ys, xs = numpy_rast.shape
meta = []
for i in range(zs):
bandpath = raster + "\\Band_{0}".format(i+1)
meta.append(metadata(bandpath, xs, ys))
if numpy_datatype is None:
numpy_datatype = meta[0].numpy_datatype
# build metadata for single band raster
else:
ys, xs = numpy_rast.shape
meta = metadata(raster, xs, ys)
if numpy_datatype is None:
numpy_datatype = meta.numpy_datatype
numpy_rast = numpy_rast.astype(numpy_datatype)
# mask NoData values from the array
if 'float' in numpy_datatype:
numpy_rast[numpy_rast == meta.NoData_Value] = numpy.nan
numpy_rast = numpy.ma.masked_array(numpy_rast, numpy.isnan(numpy_rast),
dtype = numpy_datatype)
elif 'int' in numpy_datatype: # (numpy.nan not supported by ints)
mask = numpy.zeros(numpy_rast.shape)
mask[numpy_rast != meta.NoData_Value] = False # do not mask
mask[numpy_rast == meta.NoData_Value] = True # mask
numpy_rast = numpy.ma.masked_array(numpy_rast, mask,
dtype = numpy_datatype)
return numpy_rast, meta
def to_numpy(raster, numpy_datatype = None):
"""
Wrapper for arcpy.RasterToNumpyArray with better metadata handling
This is just a wraper for the RasterToNumPyArray function within arcpy, but it also
extracts out all the spatial referencing information that will probably be needed
to save the raster after desired manipulations have been performed.
also see raster.from_numpy function in this module.
inputs:
Raster Any raster supported by the arcpy.RasterToNumPyArray function
numpy_datatype must be a string equal to any of the types listed at the following
address [http://docs.scipy.org/doc/numpy/user/basics.types.html]
for example: 'uint8' or 'int32' or 'float32'
outputs:
numpy_rast the numpy array version of the input raster
Metadata An object with the following attributes.
.Xmin the left edge
.Ymin the bottom edge
.Xmax the right edge
.Ymax the top edge
.Xsize the number of columns
.Ysize the number of rows
.cellWidth resolution in x direction
.cellHeight resolution in y direction
.projection the projection information to give the raster
.NoData_Value the numerical value which represents NoData in this raster
Usage example:
call this function with " rast,Metadata = to_numpy(Raster) "
perform numpy manipulations as you please
then save the array with " raster.from_numpy(rast,Metadata,output) "
"""
# create a metadata object and assign attributes to it
# read in the raster as an array
if is_rast(raster):
numpy_rast = arcpy.RasterToNumPyArray(raster)
ys, xs = numpy_rast.shape
meta = metadata(raster, xs, ys)
if numpy_datatype is None:
numpy_datatype = meta.numpy_datatype
numpy_rast = numpy_rast.astype(numpy_datatype)
# mask NoData values from the array
if 'float' in numpy_datatype:
numpy_rast[numpy_rast == meta.NoData_Value] = numpy.nan
numpy_rast = numpy.ma.masked_array(numpy_rast, numpy.isnan(numpy_rast),
dtype = numpy_datatype)
elif 'int' in numpy_datatype: # (numpy.nan not supported by ints)
mask = numpy.zeros(numpy_rast.shape)
mask[numpy_rast != meta.NoData_Value] = False # do not mask
mask[numpy_rast == meta.NoData_Value] = True # mask
numpy_rast = numpy.ma.masked_array(numpy_rast, mask,
dtype = numpy_datatype)
else:
raise Exception("Raster '{0}'does not exist".format(raster))
return numpy_rast, meta
def configure(self, env, upgrade_type=None, config_dir=None):
import params
env.set_params(params)
install_atlas()
metadata('client')
def configure(self, env, upgrade_type=None, config_dir=None): import params env.set_params(params) metadata()
# Update analysisCount from all the runs
totalCount = 0
errCount = 0
for runId in runList:
for objName in objList:
for gesture in gestureList:
for gen in genderList:
runAnalysisCount[objName][gesture][gen] = 0
runCount = 0
run = dataDir + '/' + runId
objectReferenceId = run + '/' + 'object-reference.xml'
metadataId = run + '/' + 'metadata.xml'
metadataObj = metadata.metadata()
metadataObj.Read(metadataId)
copilot_gender = metadataObj.get_copilot_gender()
driver_gender = metadataObj.get_driver_gender()
gender = dict()
gender['driver'] = driver_gender
gender['copilot'] = copilot_gender
print runId, 'driver:', driver_gender, 'copilot:', copilot_gender
objWords, objAnnotations, objNotes = read_complex(objectReferenceId)
for objAnnotation in objAnnotations:
objLabel = objAnnotation.label
gesture = ''.join(objLabel.partition('(')[1:])
label = objLabel.partition('(')[0].strip()
