def testing(ch, method, properties, body): print(" [x] Received %r" % body) ch.basic_ack(delivery_tag=method.delivery_tag) server = parse_config_l('config.config') download(server, body) metadata(server, body) downloadPolicy(server, body)
def bash(cmd, arg=None): ALLOW_COMMAND = metadata('srce_command', _ALLOW_COMMAND) PATH = metadata('srce_path', _PATH) if cmd not in ALLOW_COMMAND.keys(): return '', 403 if arg: if arg not in ALLOW_COMMAND[cmd]: return '', 403 command = [f'{PATH}{cmd}', arg] else: command = f'{PATH}{cmd}' emailNotify(auth.username, request.remote_addr, command) proc = Popen(command, stdout=-1, stderr=-1) try: outs, errs = proc.communicate(timeout=30) except TimeoutExpired: return 'Process still running.' return Response(f'Output:\n{outs.decode()}\n\nError:\n{errs.decode()}', mimetype='text/plain')
def _stat(self, path): if '/' != path[0]: path = '/' + path if not self._query_path(path): print( "Invalid stat command: the file or directory does not exist.") else: info = self._query_metadata(path) sample = metadata(info) sample.display()
def new_thread(self, conn): while True: request = conn.recv(4096).decode() if request == "#finished#": conn.sendall("#finished#".encode()) break print(f"server: {request}") command, content = request.split(' -> ') if command == 'insert': sample = metadata(content) if sample.path not in self.record: self.record[sample.path] = [content, set()] else: self.record[sample.path][0] = content elif command == 'query_path': if content in self.record: resp = self.record[content][0].split(', ')[2] else: resp = 'N' conn.sendall(resp.encode()) elif command == 'add_dir': dir_path, filename = content.split(':') self.record[dir_path][1].add(filename) elif command == 'query': if content in self.record: resp = ';;'.join(list(self.record[content][1])) else: resp = '' if not resp: resp = "##none##" conn.sendall(resp.encode()) elif command == 'remove': if content in self.record: del self.record[content] elif command == 'query_metadata': resp = self.record[content][0] conn.sendall(resp.encode()) elif command == 'distribution': if self.record: resp = sorted(self.record.keys()) resp = "\n".join(list(map(lambda x: "\t" + x, resp))) else: resp = "Empty" conn.sendall(resp.encode()) else: print("No corresponding response.") print(self.record) conn.close()
def commit(name, content): GITHUB = metadata('feh_github', _GITHUB) url = f"https://api.github.com/repos/{GITHUB['user']}/{GITHUB['repo']}/contents/{GITHUB['path']}/{name}.json" data = {'message': name, 'content': content} headers = {'Authorization': f"token {GITHUB['token']}"} if PROXY: proxies = {'https': f'http://{PROXY}'} else: proxies = None return requests.put(url, json=data, headers=headers, proxies=proxies, timeout=10)
def emailNotify(user, ip, cmd): SUBSCRIBE = metadata('srce_subscribe', _SUBSCRIBE) msg = EmailMessage() msg['Subject'] = f"SRCE Notification - {datetime.now().strftime('%Y%m%d %H:%M:%S')}" msg['From'] = SUBSCRIBE['sender'] msg['To'] = SUBSCRIBE['subscriber'] if isinstance(cmd, list): cmd = ' '.join(cmd) msg.set_content( f"{datetime.now().strftime('%Y/%m/%d-%H:%M:%S')}\nUser: {user}\nIP: {ip}\n\nCommand: {cmd}") with SMTP(SUBSCRIBE['smtp_server'], SUBSCRIBE['smtp_server_port']) as s: s.starttls() s.login(SUBSCRIBE['sender'], SUBSCRIBE['password']) s.send_message(msg)
def mail(feh: FEH_VotingGauntlet): Round = {1: 'Round1', 2: 'Round2', 3: 'Final Round'} timestamp = f"{feh.date.strftime(f'%Y%m%d')} {feh.hour}:00:00" msg = EmailMessage() msg['Subject'] = f'FEH 投票大戦第{feh.current_event}回 {Round[feh.current_round]} - {timestamp}' content = '\n'.join( [formatter(battle) for battle in feh.current_scoreboard]) msg.set_content(f'{content}\n\n{timestamp}') SUBSCRIBE = metadata('feh_subscribe', _SUBSCRIBE) msg['From'] = SUBSCRIBE['sender'] msg['To'] = SUBSCRIBE['subscriber'] with SMTP(SUBSCRIBE['smtp_server'], SUBSCRIBE['smtp_server_port']) as s: s.starttls() s.login(SUBSCRIBE['sender'], SUBSCRIBE['password']) s.send_message(msg)
def _initialize_insert(self, line): sample = metadata(line) temp = sample.path.split("/") pre_path = None cur_index = -1 self._insert(sample) while pre_path != '/': pre_path = '/'.join(temp[:cur_index]) if pre_path == '': pre_path = '/' filename = temp[cur_index] cur_index -= 1 if not self._query_path(pre_path): self._create(pre_path, "yes") self._add_to_dir(filename, pre_path)
def query(filter_or_pipeline=None, projection=None, sort=None, limit=0, mode='find'): MONGO = metadata('feh_mongo', _MONGO) if MONGO['username']: username = quote_plus(MONGO['username']) password = quote_plus(MONGO['password']) URI = f"mongodb://{username}:{password}@{MONGO['server']}:{MONGO['port']}/{MONGO['database']}" else: URI = f"mongodb://{MONGO['server']}:{MONGO['port']}" with MongoClient(URI) as client: if mode == 'find': return list(client[MONGO['database']][MONGO['collection']].find( filter_or_pipeline, projection, sort=sort, limit=limit)) else: return list(client[MONGO['database']][ MONGO['collection']].aggregate(filter_or_pipeline))
def isvalid(id_num): """ Check if a gutenberg book is an english textbook. Args: id_num: id of gutenberg book. Returns: a boolean which shows if id_num is id of an english text book """ try: language = metadata(id_num)['language'] form = get_metadata('formaturi', id_num) if 'en' not in language: return False form = ' '.join(form) if re.search(r'\d+\.txt', form): return True return False except: return False
def mongo(feh: FEH_VotingGauntlet): MONGO = metadata('feh_mongo', _MONGO) if MONGO['username']: username = quote_plus(MONGO['username']) password = quote_plus(MONGO['password']) URI = f"mongodb://{username}:{password}@{MONGO['server']}:{MONGO['port']}/{MONGO['database']}" else: URI = f"mongodb://{MONGO['server']}:{MONGO['port']}" try: with MongoClient(URI) as client: collection = client[MONGO['database']][MONGO['collection']] update = { 'event': feh.current_event, 'date': feh.date, 'hour': feh.hour } all_scoreboard = feh.scoreboard for Round in all_scoreboard: update['round'] = Round for battle in all_scoreboard[Round]: collection.update_one({'scoreboard': battle}, {'$setOnInsert': update}, True) except: pass
def to_numpy(raster, numpy_datatype=None): """ Wrapper for arcpy.RasterToNumpyArray with better metadata handling This is just a wraper for the RasterToNumPyArray function within arcpy, but it also extracts out all the spatial referencing information that will probably be needed to save the raster after desired manipulations have been performed. also see raster.from_numpy function in this module. :param raster: Any raster supported by the arcpy.RasterToNumPyArray function :param numpy_datatype: must be a string equal to any of the types listed at the following address [http://docs.scipy.org/doc/numpy/user/basics.types.html] for example: 'uint8' or 'int32' or 'float32' :return numpy_rast: the numpy array version of the input raster :return Metadata: a metadata object. see ``raster.metadata`` """ # perform some checks to convert to supported data format if not is_rast(raster): try: print( "Raster '{0}' may not be supported, converting to tif".format( raster)) tifraster = raster + ".tif" if not os.path.exists(raster + ".tif"): arcpy.CompositeBands_management(raster, tifraster) raster = tifraster except: raise Exception("Raster type could not be recognized") # read in the raster as a numpy array numpy_rast = arcpy.RasterToNumPyArray(raster) # build metadata for multi band raster if len(numpy_rast.shape) == 3: zs, ys, xs = numpy_rast.shape meta = [] for i in range(zs): bandpath = raster + "\\Band_{0}".format(i + 1) meta.append(metadata(bandpath, xs, ys)) if numpy_datatype is None: numpy_datatype = meta[0].numpy_datatype # build metadata for single band raster else: ys, xs = numpy_rast.shape meta = metadata(raster, xs, ys) if numpy_datatype is None: numpy_datatype = meta.numpy_datatype numpy_rast = numpy_rast.astype(numpy_datatype) # mask NoData values from the array if 'float' in numpy_datatype: numpy_rast[numpy_rast == meta.NoData_Value] = numpy.nan numpy_rast = numpy.ma.masked_array(numpy_rast, numpy.isnan(numpy_rast), dtype=numpy_datatype) elif 'int' in numpy_datatype: # (numpy.nan not supported by ints) mask = numpy.zeros(numpy_rast.shape) mask[numpy_rast != meta.NoData_Value] = False # do not mask mask[numpy_rast == meta.NoData_Value] = True # mask numpy_rast = numpy.ma.masked_array(numpy_rast, mask, dtype=numpy_datatype) return numpy_rast, meta
def __init__(self, path, directory, relaxed=False, downsample=1): self.path = path self.directory = directory self.metadata = metadata.metadata(path) self.Evaluate(relaxed, downsample)
ARTWORK = utils.ARTWORK ICON = utils.ICON URL = utils.URL SECTION = 100 SERIES = 200 EPISODE = 300 HOST = 400 DOWNLOAD = 500 MARKWATCHED = 600 MARKUNWATCHED = 601 AUTOPLAY = ADDON.getSetting('AUTOPLAY') == 'true' import metadata meta = metadata.metadata() meta.SetDir(os.path.join(PROFILE, 'watched')) meta.SetImageDir(os.path.join(PROFILE, 'images')) def CheckVersion(): prev = ADDON.getSetting('VERSION') curr = VERSION if prev == curr: return ADDON.setSetting('VERSION', curr) if curr == '1.0.17': d = xbmcgui.Dialog()
fallback=587), 'password': config.get('email', 'PWD'), 'subscriber': config.get('email', 'SUBSCRIBER') } _MONGO = { 'server': config.get('mongodb', 'SERVER', fallback='localhost'), 'port': config.getint('mongodb', 'PORT', fallback=27017), 'database': config.get('mongodb', 'DATABASE', fallback='feh'), 'collection': config.get('mongodb', 'COLLECTION', fallback='feh'), 'username': config.get('mongodb', 'AUTH', fallback='feh'), 'password': config.get('mongodb', 'PASSWORD', fallback='feh') } if __name__ == '__main__': SUBSCRIBE = metadata('feh_subscribe', _SUBSCRIBE) MONGO = metadata('feh_mongo', _MONGO) command = f"mongodump -h{MONGO['server']}:{MONGO['port']} -d{MONGO['database']} -c{MONGO['collection']} -u{MONGO['username']} -p{MONGO['password']} --gzip --archive" attachment = BytesIO() attachment.write(check_output(command, shell=True)) msg = EmailMessage() msg['Subject'] = f'FEH Backup-{datetime.now():%Y%m%d}' msg['From'] = SUBSCRIBE['sender'] msg['To'] = SUBSCRIBE['subscriber'] msg.add_attachment(attachment.getvalue(), maintype='application', subtype='octet-stream', filename='database') with SMTP(SUBSCRIBE['smtp_server'], SUBSCRIBE['smtp_server_port']) as s: s.starttls() s.login(SUBSCRIBE['sender'], SUBSCRIBE['password'])
def to_numpy(raster, numpy_datatype=None): """ Wrapper for arcpy.RasterToNumpyArray with better metadata handling This is just a wraper for the RasterToNumPyArray function within arcpy, but it also extracts out all the spatial referencing information that will probably be needed to save the raster after desired manipulations have been performed. also see raster.from_numpy function in this module. inputs: Raster Any raster supported by the arcpy.RasterToNumPyArray function numpy_datatype must be a string equal to any of the types listed at the following address [http://docs.scipy.org/doc/numpy/user/basics.types.html] for example: 'uint8' or 'int32' or 'float32' outputs: numpy_rast the numpy array version of the input raster Metadata An object with the following attributes. .Xmin the left edge .Ymin the bottom edge .Xmax the right edge .Ymax the top edge .Xsize the number of columns .Ysize the number of rows .cellWidth resolution in x direction .cellHeight resolution in y direction .projection the projection information to give the raster .NoData_Value the numerical value which represents NoData in this raster Usage example: call this function with " rast,Metadata = to_numpy(Raster) " perform numpy manipulations as you please then save the array with " raster.from_numpy(rast,Metadata,output) " """ # create a metadata object and assign attributes to it # perform some checks to convert to supported data format if not is_rast(raster): try: print( "Raster '{0}' may not be supported, converting to tif".format( raster)) tifraster = raster + ".tif" if not os.path.exists(raster + ".tif"): arcpy.CompositeBands_management(raster, tifraster) raster = tifraster except: raise Exception("Raster type could not be recognized") # read in the raster as a numpy array numpy_rast = arcpy.RasterToNumPyArray(raster) # build metadata for multi band raster if len(numpy_rast.shape) == 3: zs, ys, xs = numpy_rast.shape meta = [] for i in range(zs): bandpath = raster + "\\Band_{0}".format(i + 1) meta.append(metadata(bandpath, xs, ys)) if numpy_datatype is None: numpy_datatype = meta[0].numpy_datatype # build metadata for single band raster else: ys, xs = numpy_rast.shape meta = metadata(raster, xs, ys) if numpy_datatype is None: numpy_datatype = meta.numpy_datatype numpy_rast = numpy_rast.astype(numpy_datatype) # mask NoData values from the array if 'float' in numpy_datatype: numpy_rast[numpy_rast == meta.NoData_Value] = numpy.nan numpy_rast = numpy.ma.masked_array(numpy_rast, numpy.isnan(numpy_rast), dtype=numpy_datatype) elif 'int' in numpy_datatype: # (numpy.nan not supported by ints) mask = numpy.zeros(numpy_rast.shape) mask[numpy_rast != meta.NoData_Value] = False # do not mask mask[numpy_rast == meta.NoData_Value] = True # mask numpy_rast = numpy.ma.masked_array(numpy_rast, mask, dtype=numpy_datatype) return numpy_rast, meta
def create_single_h5( dirpath, movepath_h5, movepath_wf, verbose=False ): #dirpath - path to directory which contains the strain data of the waveform '''Generate Waveform HDF5 file from simulation data dirpath(str) - simulation output directory with strain data as generated by MATLAB script movepath_h5 (str) - Final path to store h5 file movepath_wf (str) - Final path to store simulation output data verbose (bool) - Turn on for debugging ''' localsim_name = dirpath.split('/')[-1] print( "*(Create_Single_h5) >> Constructing h5 waveform for simulation - {} \n " .format(localsim_name)) #Add path of directories to read data and save figures- figdir = os.path.join(dirpath, "figures") datadir = os.path.join(dirpath, "data") strdir = os.path.join(datadir, "Strain") #Check if the h5 file already exists simname = simulation_name(dirpath) h5check_path = glob.glob(os.path.join(movepath_h5, "*/%s.h5" % simname)) if len(h5check_path) > 0: if os.path.exists(h5check_path[0]): raise NameError( "*(Create_Single_h5) >> h5file %s already exists. Please delete the old file before constructing the new one. Waveform getting moved to Failed Directory" % simname) #Create a dictionary to store all the modes nr_strain_data = {} lmax = ltemp = 0 # Safety Check if not os.path.isfile(strdir + '/Strain_l2_m2.txt'): error('*(Create_Single_h5) >> Strain Files Missing \n') #The for loop ensures all the modes are included in this dictionary for strfile in sorted(glob.glob(strdir + '/Strain*.txt')): # NOTE: here we determine l and m from the filename. This assumes that each # strain file has data for only one l and one m. l = int((os.path.basename(strfile).split('_')[1])[1]) m = int((os.path.basename(strfile).split('_')[-1]).split('.')[0][1:]) # Find Lmax for metadata ltemp = l if ltemp > lmax: lmax = ltemp # Load the data data_array = loadtxt(strfile, comments='#') # Unpack the data array (get time, plus and cross ) time_uncrop, hp_uncrop, hx_uncrop = data_array[:, 0], data_array[:, 1], data_array[:, 2] # Crop the data to remove the junk radiation time_crop, hp, hx, crop_idx, junkrad_time = crop_data( dirpath, time_uncrop, hp_uncrop, hx_uncrop) time = time_crop # Calculate the waveform amplitude and phase (Unwrapping phase - Request from NR-injection Infrastructure A = abs(hp - 1.j * hx) phase = -1.0 * unwrap(angle(hp - 1j * hx)) # NOTE: The NR-Injection infrastructure requests that phase decreases if m is positive. Here we multiply by -1 to enforce this. phase *= -1 # Set all m=0 mode to 0: # This is just temporary measure to fix the issue with m=0 modes. Better method would be to find the right tuning frequency by comparison with Quasi normal modes obtained from perturbation theory if m == 0: #l==2 and m==0: A *= 0. phase *= 0. # Create a dictionary that contains the NR data. This will be used as an input to the function that actually makes the h5 file. nr_strain_data[(l, m)] = {'amp': A, 'phase': phase, 't': time} # Compute the index of maximum of strain amplitude - all modes included amp_sq = np.zeros(len(nr_strain_data[(2, 2)]['t'])) for key in nr_strain_data: amp_sq = amp_sq + nr_strain_data[key]['amp']**2 if (np.all(amp_sq == 0)): raise ValueError( "*(Create_Single_h5) >> %s waveform is all zeros. Waveform getting moved to Failed Directory" % simname) amp_max_idx = np.where(amp_sq == np.amax(amp_sq)) amp22_max_idx = np.where( nr_strain_data[(2, 2)]['amp'] == np.amax(nr_strain_data[(2, 2)]['amp'])) # Mean Center the data about max amplitude for key in nr_strain_data: time = nr_strain_data[key]['t'] - nr_strain_data[key]['t'][amp_max_idx] nr_strain_data[key]['t'] = time # Create plots of amplitude and phase for mean centered time if key == (2, 2): create_plot(time, nr_strain_data[key]['amp'], 'Time', 'h+', figdir, 'Amp_22_plot.png') create_plot(time, nr_strain_data[key]['phase'], 'Time', 'Phase', figdir, 'Phase_22_plot.png') amp_lm = nr_strain_data[key][ 'amp'] #t22 - t22[abs(h22)==np.amax(abs(h22))] phase_lm = nr_strain_data[key][ 'phase'] #unwrap(angle(hp_22 - 1j*hx_22)) ##Testing Purpose Only - Save the file and compare with spline interpolant #data = np.column_stack((time, amp_lm, phase_lm)) #hdr = "Time \t Amplitude \t Phase \n" # #timeshift_dir = os.path.join(strdir, "Strain_TimeShifted") #if not os.path.exists(timeshift_dir): os.makedirs(timeshift_dir) #l,m = key[0], key[1] #strfile_lm = os.path.join(timeshift_dir,"Strain_{}{}_timeshift.asc".format(l,m)) #np.savetxt(strfile_lm, data, header=hdr, delimiter='\t') # Compute the gravitational wave frequency time_22 = nr_strain_data[(2, 2)]['t'] amp_22 = nr_strain_data[( 2, 2)]['amp'] #t22 - t22[abs(h22)==np.amax(abs(h22))] phase_22 = nr_strain_data[(2, 2)]['phase'] #unwrap(angle(hp_22 - 1j*hx_22)) omega_22 = -1. * np.gradient(phase_22) / np.gradient( time_22 ) #(phase_uncrop[1:] - phase_uncrop[:-1])/(time_uncrop[1:]-time_uncrop[:-1]) create_plot(time_22, omega_22, 'Time', 'MOmega', figdir, 'Omega_22_plot.png') # Compute inital GW frequency in code units (for (2,2) mode) Mf_initial = omega_22[0] if verbose: print( "*(Create_single_h5) >> Frequency of (2,2) mode after junk radiation - Momega = {} \n" .format(Mf_initial)) # Check with frequency of ShiftTracker omega_st = find_omega22_st(dirpath, junkrad_time - 75) if verbose: print( '*(Create_single_h5) >> Gravitational Wave frequency from strain = %g/M, and from orbital frequency = %g/M \n' % (Mf_initial, 2. * omega_st[0])) # Compute the starting frquency of the waveform in SI units for 1Msun system G = 6.67428e-11 # m^3/(kg s^2) mass_sun = 1.98892e30 # kg c = 2.99792458e8 # m/s mass_sec = G * mass_sun / (c * c * c) simulation_mass_sec = ( 1.0) * mass_sec # NOTE that the 1.0 here is for 1solar mass #simulation_mass_sec = 4.92549102554*10**-6 physical_freq_initial_at_1Msol = (Mf_initial / (2. * np.pi)) / simulation_mass_sec if verbose: print( '*(Create_single_h5) >> Gravitational Wave frequency from strain = %gHz, and from orbital frequency = %gHz \n' % (physical_freq_initial_at_1Msol, (2. * omega_st[0] / (2. * np.pi) / simulation_mass_sec))) # Compute initial data nr_initdata = initial_data(dirpath) # Compute the nr_metadata nr_meta_data = metadata(dirpath, junkrad_time, verbose=verbose) nr_meta_data['Lmax'] = lmax #nr_meta_data['Omega'] = Mf_initial nr_meta_data['f_lower_at_1MSUN'] = physical_freq_initial_at_1Msol # Create the h5 file and move to corresponding directory, move the waveform to completed directory outpath = movepath_h5 + ('/%s.h5' % nr_meta_data['name']) print('*(Create_single_h5) >> Creating HDF5 files:{}'.format(outpath)) nr2h5(nr_strain_data, nr_meta_data, nr_initdata, output_path=outpath, wfdir_path=dirpath, verbose=True, testsuite=False) if nr_meta_data['simulation-type'] == 'non-spinning': movepath_h5 = os.path.join(movepath_h5, 'NonSpinning') movepath_wf = os.path.join(movepath_wf, 'NonSpinning') elif nr_meta_data['simulation-type'] == 'aligned-spins': movepath_h5 = os.path.join(movepath_h5, 'AlignedSpin') movepath_wf = os.path.join(movepath_wf, 'AlignedSpin') elif nr_meta_data['simulation-type'] == 'precessing': movepath_h5 = os.path.join(movepath_h5, 'Precessing') movepath_wf = os.path.join(movepath_wf, 'Precessing') print(movepath_h5) sh.move(dirpath, movepath_wf) sh.move(outpath, movepath_h5)
def to_numpy(raster, numpy_datatype = None): """ Wrapper for arcpy.RasterToNumpyArray with better metadata handling This is just a wraper for the RasterToNumPyArray function within arcpy, but it also extracts out all the spatial referencing information that will probably be needed to save the raster after desired manipulations have been performed. also see raster.from_numpy function in this module. inputs: Raster Any raster supported by the arcpy.RasterToNumPyArray function numpy_datatype must be a string equal to any of the types listed at the following address [http://docs.scipy.org/doc/numpy/user/basics.types.html] for example: 'uint8' or 'int32' or 'float32' outputs: numpy_rast the numpy array version of the input raster Metadata An object with the following attributes. .Xmin the left edge .Ymin the bottom edge .Xmax the right edge .Ymax the top edge .Xsize the number of columns .Ysize the number of rows .cellWidth resolution in x direction .cellHeight resolution in y direction .projection the projection information to give the raster .NoData_Value the numerical value which represents NoData in this raster Usage example: call this function with " rast,Metadata = to_numpy(Raster) " perform numpy manipulations as you please then save the array with " raster.from_numpy(rast,Metadata,output) " """ # create a metadata object and assign attributes to it # perform some checks to convert to supported data format if not is_rast(raster): try: print("Raster '{0}' may not be supported, converting to tif".format(raster)) tifraster = raster + ".tif" if not os.path.exists(raster + ".tif"): arcpy.CompositeBands_management(raster, tifraster) raster = tifraster except: raise Exception("Raster type could not be recognized") # read in the raster as a numpy array numpy_rast = arcpy.RasterToNumPyArray(raster) # build metadata for multi band raster if len(numpy_rast.shape) == 3: zs, ys, xs = numpy_rast.shape meta = [] for i in range(zs): bandpath = raster + "\\Band_{0}".format(i+1) meta.append(metadata(bandpath, xs, ys)) if numpy_datatype is None: numpy_datatype = meta[0].numpy_datatype # build metadata for single band raster else: ys, xs = numpy_rast.shape meta = metadata(raster, xs, ys) if numpy_datatype is None: numpy_datatype = meta.numpy_datatype numpy_rast = numpy_rast.astype(numpy_datatype) # mask NoData values from the array if 'float' in numpy_datatype: numpy_rast[numpy_rast == meta.NoData_Value] = numpy.nan numpy_rast = numpy.ma.masked_array(numpy_rast, numpy.isnan(numpy_rast), dtype = numpy_datatype) elif 'int' in numpy_datatype: # (numpy.nan not supported by ints) mask = numpy.zeros(numpy_rast.shape) mask[numpy_rast != meta.NoData_Value] = False # do not mask mask[numpy_rast == meta.NoData_Value] = True # mask numpy_rast = numpy.ma.masked_array(numpy_rast, mask, dtype = numpy_datatype) return numpy_rast, meta
URL = utils.URL SECTION = 100 SERIES = 200 EPISODE = 300 HOST = 400 DOWNLOAD = 500 MARKWATCHED = 600 MARKUNWATCHED = 601 AUTOPLAY = ADDON.getSetting('AUTOPLAY') == 'true' import metadata meta = metadata.metadata() meta.SetDir(os.path.join(PROFILE ,'watched')) def CheckVersion(): prev = ADDON.getSetting('VERSION') curr = VERSION if prev == curr: return ADDON.setSetting('VERSION', curr) if curr == '1.0.17': d = xbmcgui.Dialog() d.ok(TITLE + ' - ' + VERSION, 'Welcome to Watch Cartoon Online', 'Now with download feature.', '')
( {}, {}, '{}', '{}' ); ''' # Create computation log query = query.format(metric_id, result_id, data_datetime, meta_name) try: db = connection.connect(query) except Exception, err: print 'METRIC_ID = ', metric_id print 'DATA_DATETIME = ', data_datetime print 'ERROR MESSAGE = ', err # Create metadata for feature engineering meta = metadata(metric_id, meta_name, sql_new) try: meta.dropTable() meta.createTable() except Exception, err: print err pass if __name__ == '__main__': metric_id = 6 print load(metric_id) print 'sql:' print collect(metric_id)
log_file_path = '/home/ubuntu/datasets/nasa_log_jul' # path to extract the log file regex = "(.*) - - \[(.*)\] \"([A-Z]+) (.*)\" ([0-9]{3}) (-|[0-9]+)" # regex to get relevant information from a log entry pattern = re.compile(regex) FAMILIES = ['log_info','loca_info'] # column family names for server_logs_table TABLE = 'server_logs' # table LOG_COLS = ["log_info :host", "log_info:server_ts", "log_info:type", "log_info:url", "log_info:status", "log_info:bytes"] #columns to insert into server_logs # hbase_client.truncate_table(TABLE, FAMILIES) tgt_time_fmt = "%Y-%m-%d %H:%M:%S" # target date format src_time_fmt = "%d/%b/%Y:%H:%M:%S %z" # source date format denied_requests = [str(a) for a in range(400,500)] start_time = "1995-07-01 00:00:00" # start time for server log entries meta = metadata() meta.initialize_timer(start_time=start_time) window = meta.next_bound() lower_b = time.mktime(time.strptime(start_time, tgt_time_fmt)) upper_b = next(window) geo_ip = GeoIPUtil() BATCH_LIMIT = 100 count = 1 batch_index = 1 fail_batch_count = 0 """ Metadata operations for inserting one row of metadata
def metadata(_, value): return value app = create_app() _BACKUP = { 'sender': 'SENDER', 'smtp_server': 'SMTP_SERVER', 'smtp_server_port': 587, 'password': '******', 'subscriber': 'SUBSCRIBER' } BACKUP = metadata('mybookmarks_backup', _BACKUP) @click.group(invoke_without_command=True) @click.pass_context def cli(ctx): if ctx.invoked_subcommand is None: ctx.invoke(run) @cli.command(short_help='Add User') @click.argument('username') def add(username): db = sqlite3.connect(app.config['DATABASE']) try: db.executescript(
def find_key(key): md = metadata() return list(gen_dict_extract(key, md))
def get_pw(username): ALLOW_USERS = metadata('srce_user', _ALLOW_USERS) if username in ALLOW_USERS: return ALLOW_USERS[username] return None
import crossover as c from metadata import metadata from random import randint md = metadata() #initializa meta data population_count = 100 # even def init(): population = [] total_fitness = 0 while i < population_count: c = chromosome(md) c.initializeRandomly() total_fitness += c.fitness population.append(c) i+=1 return population, total_fitness def reproduce(population, crossover): new_population = [] total_fitness = 0 while i < population_count/2: x, y = selection(population) a = chromosome(md) a.initalizeCrossover(x)
import smallCounty import table import metadata import datetime import pandas as pd if __name__ == '__main__': df_smallCounty = smallCounty.smallCounty( "http://cmdweb.pcc.gov.tw/pccms/owa/guesmap.userinn") df_smallCounty.to_csv(datetime.datetime.now().strftime("%Y%m%d_%H-%M-%S") + "_smallCounty.csv", encoding="utf_8_sig") data_list1 = [] for tableLink, name in zip(df_smallCounty["table連結"], df_smallCounty["鄉鎮"]): print(tableLink, name) data_list1.extend(table.table(tableLink)) df_table = table.getTableDF(data_list1) df_table.to_csv(datetime.datetime.now().strftime("%Y%m%d_%H-%M-%S") + "_engineering.csv", encoding="utf_8_sig") #df_table = pd.read_csv("20191002_09-51-09_engineering.csv") data_list = [] for metadataLink in df_table["詳細連結"]: data_list.append(metadata.metadata(metadataLink)) df_metadata = metadata.getMetadataDF(data_list) df_metadata.to_csv(datetime.datetime.now().strftime("%Y%m%d_%H-%M-%S") + "_metadata.csv", encoding="utf_8_sig")
def configure(self, env): import params env.set_params(params) metadata()
def __init__(self,path,directory): self.path = path self.directory = directory self.metadata = metadata.metadata(path) self.Evaluate()
def to_numpy(raster, numpy_datatype = None): """ Wrapper for arcpy.RasterToNumpyArray with better metadata handling This is just a wraper for the RasterToNumPyArray function within arcpy, but it also extracts out all the spatial referencing information that will probably be needed to save the raster after desired manipulations have been performed. also see raster.from_numpy function in this module. :param raster: Any raster supported by the arcpy.RasterToNumPyArray function :param numpy_datatype: must be a string equal to any of the types listed at the following address [http://docs.scipy.org/doc/numpy/user/basics.types.html] for example: 'uint8' or 'int32' or 'float32' :return numpy_rast: the numpy array version of the input raster :return Metadata: a metadata object. see ``raster.metadata`` """ # perform some checks to convert to supported data format if not is_rast(raster): try: print("Raster '{0}' may not be supported, converting to tif".format(raster)) tifraster = raster + ".tif" if not os.path.exists(raster + ".tif"): arcpy.CompositeBands_management(raster, tifraster) raster = tifraster except: raise Exception("Raster type could not be recognized") # read in the raster as a numpy array numpy_rast = arcpy.RasterToNumPyArray(raster) # build metadata for multi band raster if len(numpy_rast.shape) == 3: zs, ys, xs = numpy_rast.shape meta = [] for i in range(zs): bandpath = raster + "\\Band_{0}".format(i+1) meta.append(metadata(bandpath, xs, ys)) if numpy_datatype is None: numpy_datatype = meta[0].numpy_datatype # build metadata for single band raster else: ys, xs = numpy_rast.shape meta = metadata(raster, xs, ys) if numpy_datatype is None: numpy_datatype = meta.numpy_datatype numpy_rast = numpy_rast.astype(numpy_datatype) # mask NoData values from the array if 'float' in numpy_datatype: numpy_rast[numpy_rast == meta.NoData_Value] = numpy.nan numpy_rast = numpy.ma.masked_array(numpy_rast, numpy.isnan(numpy_rast), dtype = numpy_datatype) elif 'int' in numpy_datatype: # (numpy.nan not supported by ints) mask = numpy.zeros(numpy_rast.shape) mask[numpy_rast != meta.NoData_Value] = False # do not mask mask[numpy_rast == meta.NoData_Value] = True # mask numpy_rast = numpy.ma.masked_array(numpy_rast, mask, dtype = numpy_datatype) return numpy_rast, meta
def to_numpy(raster, numpy_datatype = None): """ Wrapper for arcpy.RasterToNumpyArray with better metadata handling This is just a wraper for the RasterToNumPyArray function within arcpy, but it also extracts out all the spatial referencing information that will probably be needed to save the raster after desired manipulations have been performed. also see raster.from_numpy function in this module. inputs: Raster Any raster supported by the arcpy.RasterToNumPyArray function numpy_datatype must be a string equal to any of the types listed at the following address [http://docs.scipy.org/doc/numpy/user/basics.types.html] for example: 'uint8' or 'int32' or 'float32' outputs: numpy_rast the numpy array version of the input raster Metadata An object with the following attributes. .Xmin the left edge .Ymin the bottom edge .Xmax the right edge .Ymax the top edge .Xsize the number of columns .Ysize the number of rows .cellWidth resolution in x direction .cellHeight resolution in y direction .projection the projection information to give the raster .NoData_Value the numerical value which represents NoData in this raster Usage example: call this function with " rast,Metadata = to_numpy(Raster) " perform numpy manipulations as you please then save the array with " raster.from_numpy(rast,Metadata,output) " """ # create a metadata object and assign attributes to it # read in the raster as an array if is_rast(raster): numpy_rast = arcpy.RasterToNumPyArray(raster) ys, xs = numpy_rast.shape meta = metadata(raster, xs, ys) if numpy_datatype is None: numpy_datatype = meta.numpy_datatype numpy_rast = numpy_rast.astype(numpy_datatype) # mask NoData values from the array if 'float' in numpy_datatype: numpy_rast[numpy_rast == meta.NoData_Value] = numpy.nan numpy_rast = numpy.ma.masked_array(numpy_rast, numpy.isnan(numpy_rast), dtype = numpy_datatype) elif 'int' in numpy_datatype: # (numpy.nan not supported by ints) mask = numpy.zeros(numpy_rast.shape) mask[numpy_rast != meta.NoData_Value] = False # do not mask mask[numpy_rast == meta.NoData_Value] = True # mask numpy_rast = numpy.ma.masked_array(numpy_rast, mask, dtype = numpy_datatype) else: raise Exception("Raster '{0}'does not exist".format(raster)) return numpy_rast, meta
def configure(self, env, upgrade_type=None, config_dir=None): import params env.set_params(params) install_atlas() metadata('client')
def configure(self, env, upgrade_type=None, config_dir=None): import params env.set_params(params) metadata()
# Update analysisCount from all the runs totalCount = 0 errCount = 0 for runId in runList: for objName in objList: for gesture in gestureList: for gen in genderList: runAnalysisCount[objName][gesture][gen] = 0 runCount = 0 run = dataDir + '/' + runId objectReferenceId = run + '/' + 'object-reference.xml' metadataId = run + '/' + 'metadata.xml' metadataObj = metadata.metadata() metadataObj.Read(metadataId) copilot_gender = metadataObj.get_copilot_gender() driver_gender = metadataObj.get_driver_gender() gender = dict() gender['driver'] = driver_gender gender['copilot'] = copilot_gender print runId, 'driver:', driver_gender, 'copilot:', copilot_gender objWords, objAnnotations, objNotes = read_complex(objectReferenceId) for objAnnotation in objAnnotations: objLabel = objAnnotation.label gesture = ''.join(objLabel.partition('(')[1:]) label = objLabel.partition('(')[0].strip()