def run_findlr_func(exp): config = get_config(experiment_name=exp.name, lr_schedule_type=exp.lr_schedule_type, lr=exp.lr, loss_types=exp.loss_types, loss_coefs=exp.loss_coefs, cat_weights=exp.cat_weights, attr_weights=exp.attr_weights, batch_size=exp.batch_size, num_epochs=exp.num_epochs, mixup_aug=exp.mixup_aug, mixup_aug_beta=exp.mixup_aug_beta) train_dataset = FashionDataset(config.dataset_path, 'training', h=config.h, w=config.w, mean=config.mean, train_pct=1., valid_pct=0.15) net = exp.net(train_dataset.num_classes, **exp.net_kwargs) optimizer = Adam(net.parameters(), weight_decay=0) device = torch.device("cuda:0") find_lr(net, optimizer, train_dataset, config) del net del optimizer del device torch.cuda.empty_cache()
def start(config, port=6000): app = App(port, **get_config(config)) app.start() print("Aplicação iniciada. Pressione h para ajuda.") while app.listening: ipt = input(">").split(" ", 1) cmd = ipt[0] data = ipt[1] if len(ipt) > 1 else None cmd = cmd.lower() if cmd in ["send", "s"]: to, msg = data.split(" ", 1) app.sendto(to, msg) elif cmd in ["messages", "m"]: print(app.socket.msg_queue.queue) elif cmd in ["generate", "g"]: app.generate_token() elif cmd in ["destroy", "d"]: app.destroy_token() elif cmd in ["help", "h"]: app.help() elif cmd in ["quit", "q"]: app.listening = False else: print("Comando desconhecido") print("Finalizando...") app.join() print("Aplicação finalizada")
def run_eval_experiment_func(exp): config = get_config(experiment_name=exp.name, lr_schedule_type=exp.lr_schedule_type, loss_types=exp.loss_types, loss_coefs=exp.loss_coefs, cat_weights=exp.cat_weights, attr_weights=exp.attr_weights, batch_size=1, num_epochs=exp.num_epochs) valid_dataset = FashionDataset(config.dataset_path, 'validation', h=config.h, w=config.w, mean=config.mean, train_pct=0.8, valid_pct=0.2) net = exp.net(valid_dataset.num_classes, **exp.net_kwargs) load_model(config, net, "last") device = torch.device("cuda") net.to(device) results = eval_model(net, valid_dataset, config, save_examples_flag=True, num_examples=50, save_path=os.path.join(config.experiment_dir, 'examples'))
def test_makes_all_files(temp_output_dir): input_directory = get_config()[INPUT_DIR_KEY] infiles = [x for x in os.listdir(input_directory) if x[0] == '@'] outfiles = os.listdir(temp_output_dir) assert (len(infiles) == len(outfiles))
def get_id_from_name(name: str, setting: str = 'DEFAULT') -> int: logger = py_logging.create_logger( 'get_id_from_name', '{}get_id_from_name.log'.format( os.path.dirname(os.path.realpath(__file__)) + os.sep)) config = get_config() html_text = requests.get(config[setting]['NameToIdBaseUrl'] + urllib.parse.quote_plus(name)).text logger.info("Sent request off to find the id for: {}".format(name)) soup = BeautifulSoup(html_text, 'lxml') possible_results = soup.find('table', {"class": "table"}) \ .find_all('tr') for possible_result in possible_results: holder = possible_result.find_all('td') possible_result = possible_result.find('td') if holder: if len(holder) > 1: holder = holder[1] if holder.getText().lower() == name.lower(): return possible_result.getText() return None
def test_TFIDF_model(): input_directory = get_config()[INPUT_DIR_KEY] infiles = [x for x in os.listdir(input_directory) if x[0]=='@'] test_file_path = os.path.join(input_directory, infiles[0]) with open(test_file_path, 'r') as file: in_data = json.loads(file.read()) df = pd.DataFrame(in_data, columns = ['tweet','date','embedding']) train, test = train_test_split(df['tweet'], test_size = 0.4, random_state = 1) # Now split the training data into simulated term-part (user's tweets) # and document part (other users' tweets). This just gets the array shapes # right. In non-unit test scenarios, 'context' must come from other users. train_doc, train_context = train_test_split(train, test_size = 0.5, random_state = 1) model = TFIDFModel(use_context=True) model.characterize(train_doc, train_context) sim_scores = model.similarity_score(test) assert(len(test)==len(sim_scores))
def run_pgs_rollout(config, **kwargs): if config is None: config = get_config() for name, val in kwargs.items(): setattr(config, name, val) config.log_path = config.exp_name pgs_simulate(config)
def __init__(self, event_q, shutdown_command='Stop'): self.logger = log_handler.get_log_handler('server_log.txt', 'info', 'door.EventHandler') self.logger.info('Starting: EventHandler') #Open config file config = get_config.get_config(CONFIG_FILE_NAME) self.event_logfile = config.get('Logging', 'event_logfile') self.event_queue = event_q self.shutdown_cmd = shutdown_command self.host = config.get('Logging', 'sql_server') self.user = config.get('Logging', 'sql_user') self.pwd = config.get('Logging', 'sql_pwd') self.db = config.get('Logging', 'sql_db') #self.mysql_conn = pymysql.connect(host=host, user=user, passwd=pwd, db=db) #self.mysql_cursor = self.mysql_conn.cursor() self.stop_event = threading.Event() #Start thread to process input events self.process_events_thread = threading.Thread( name='process_events', target=self.__process_events) self.process_events_thread.start()
def TweetHandler(self): """tweeting does not tweet from localhost""" self.response.out.write('<br/><br/>Tweeting<br/>') self.response.out.write('this info will be tweeted:<br/>') # oldest non-tweeted and prepared oldest_changeset = Changeset.all().order('created_at').filter('is_tweeted =', False).filter('is_prepared =', True).fetch(1) if not oldest_changeset: self.response.out.write('nothing to tweet') return else: c = oldest_changeset[0] config = get_config() # do not tweet from localhost if not 'localhost' in self.request.url: auth = tweepy.OAuthHandler(config["consumer_key"], config["consumer_secret"]) auth_data = OAuthAccessToken.all().filter('specifier =', config["twitter_username"]).fetch(1)[0] auth.set_access_token(auth_data.oauth_token, auth_data.oauth_token_secret) self.response.out.write('<br/>tweeting with oauth:<br/>') api = tweepy.API(auth) self.response.out.write("id: %d" % c.id) self.response.out.write("user: %s" % c.user) self.response.out.write("comment: %s" % c.comment) self.response.out.write("tweet: %s" % c.tweet) try: api.update_status(c.tweet) except tweepy.error.TweepError, e: self.response.out.write( 'failed: %s' % e.reason ) if "Status is a duplicate" in e.reason: c.is_tweeted = True c.put() return
def get(self): changesets = Changeset.all().order('-created_at').fetch(20) config = get_config() options = {'config': config, 'changesets': changesets} path = os.path.join(os.path.dirname(__file__), 'templates/rss.html') page = template.render(path,options) self.response.out.write(page)
async def neko(self, ctx, *args): """ .neko [tag] - shows image from nekos.life. ONLY 1 TAG! Available tags: 'tickle', 'classic', 'ngif', 'erofeet', 'meow', 'erok', 'poke', 'les', 'hololewd', 'lewdk', 'keta', 'feetg','nsfw_neko_gif', 'eroyuri', 'kiss', '8ball', 'kuni', 't**s', 'pussy_jpg', 'cum_jpg', 'pussy', 'lewdkemo','lizard', 'slap', 'lewd', 'cum', 'cuddle', 'spank', 'smallboobs', 'goose', 'Random_hentai_gif', 'avatar','fox_girl', 'nsfw_avatar', 'hug', 'gecg', 'boobs', 'pat', 'feet', 'smug', 'kemonomimi', 'solog', 'holo', 'wallpaper', 'bj', 'woof', 'yuri', 'trap', 'anal', 'baka', 'b*****b', 'holoero', 'feed', 'neko', 'gasm', 'hentai', 'futanari', 'ero', 'solo', 'waifu', 'pwankg', 'eron', 'erokemo' """ if ctx.message.server is not None: server = get_server(ctx.message.server) nsfw = get_channel(get_config()['token'], ctx.message.channel.id)['nsfw'] else: server = {'sfw': False, 'nsfw_channels': []} nsfw = True if ((not server['sfw']) and nsfw) or ctx.message.channel.id in server['nsfw_channels']: await self.client.send_typing( discord.Object(id=ctx.message.channel.id)) await self.neko_wrap(ctx, args) else: await self.client.add_reaction(ctx.message, '❌')
def train_lstm(x_array, y_array, vocab_size, embedding_dim, units, batch_size, epochs): lstm_model = lstm.BaseLine(vocab_size, embedding_dim, units, 128).build_model() checkpoint = tf.train.Checkpoint(optimizer=optimizer, lstm_model=lstm_model) train(x_array, y_array, batch_size, epochs, checkpoint, get_config.get_config()['model_data'], lstm_model)
def create_logfile_path(name, logdir=None): now = datetime.now().strftime('%Y%b%d_%H%m%S') logname = '{}_{}.log'.format(name, now) if not logdir: logdir = get_config("logdir") logfile = os.path.join(logdir, logname) return logfile
def run_pgs_eval(config, **kwargs): if config is None: config = get_config() for name, val in kwargs.items(): setattr(config, name, val) run_pgs_rollout(config) print('rollout done') run_compare(config) print('compare done')
def write_to_stat(self): stat_dir = get_config.get_config()["stat_dir"] if not os.path.exists(stat_dir): os.mkdir(stat_dir) stat_file = os.path.join(stat_dir, uni_util.get_file_name("stat_flow")) with open(stat_file, "w") as f_stat: f_stat.write(uni_util.get_crawl_time() + "\t" + self.module_name + "\t" + str(self.lines_num))
def load_test(path): x_array = [] with open(path) as f: f.readline() for line in f: x_array.append([int(i) for i in line.split(' ')]) x_array = tf.keras.preprocessing.sequence.pad_sequences( x_array, maxlen=get_config.get_config()['max_inp'], padding='post') return x_array
def get_defaults(transport_name): """Get defaults from config file""" json_cfg = get_config() return { 'host': json_cfg['host'], 'port': json_cfg['transports'][transport_name]['port'], 'login': json_cfg['transports'][transport_name]['login'], 'password': json_cfg['transports'][transport_name]['password'] }
def __init__(self): # GET CONNECT STRING -> TARGET DATABASE self.connect_string = get_config().target_conn() try: self.db_connection = psycopg2.connect(self.connect_string) self.db_cursor = self.db_connection.cursor() lw.log.debug("Conn2ected to target Database: " + self.connect_string) except Exception as e: lw.log.error("Error connecting do Database:" + str(e))
def __init__(self): """ Load the model and read the config """ self.conf = get_config.get_config() self.spark, self.sc = utilities.get_spark_session("classification_app") self.pipelineModel = PipelineModel.load(self.conf["model_path"]) self.label_list = self.pipelineModel.stages[3].labels
def temp_output_dir(tmpdir_factory): config = get_config() tmpdir = tmpdir_factory.mktemp('output') filter_tweets_from_directories(config[INPUT_DIR_KEY], tmpdir, config['regexp_tweet_filters'], int(config['min_tweet_characters'])) return tmpdir
def load_data(path): y_array = [] x_array = [] with open(path) as f: f.readline() for line in f: line = line.split('\t') y_array.append(int(line[0])) x_array.append([int(i) for i in line[1].split(' ')]) x_array = tf.keras.preprocessing.sequence.pad_sequences(x_array, maxlen=get_config.get_config()['max_inp'], padding='post') return x_array, np.array(y_array)
def color(data, template, logs_dir="/data2/ben/HE/data/result/color_logs/"): import tensorflow.compat.v1 as tf tf.reset_default_graph() sess = tf.Session() image_dist = [] is_train = False #print("FLAGS",tf.flags.FLAGS) #print("FLAGS.logs_dir==============",tf.flags.FLAGS.logs_dir) config = get_config(FLAGS, is_train) dist = DCGMM(sess, config, "DCGMM", is_train) db_tmpl = SampleProvider("Template_dataset", template, config.fileformat, config.image_options, is_train) mu_tmpl = 0 std_tmpl = 0 N = 0 while True: X = db_tmpl.DrawSample(config.batch_size) if len(X) == 0: break X_hsd = utils.RGB2HSD(X / 255.0) mu, std, gamma = dist.deploy(X_hsd) mu = np.asarray(mu) mu = np.swapaxes(mu, 1, 2) # -> dim: [ClustrNo x 1 x 3] std = np.asarray(std) std = np.swapaxes(std, 1, 2) # -> dim: [ClustrNo x 1 x 3] N = N + 1 mu_tmpl = (N - 1) / N * mu_tmpl + 1 / N * mu std_tmpl = (N - 1) / N * std_tmpl + 1 / N * std db = SampleProvider("Test_dataset", data, config.fileformat, config.image_options, is_train) while True: X = db.DrawSample(config.batch_size) if len(X) == 0: break X_hsd = utils.RGB2HSD(X / 255.0) mu, std, pi = dist.deploy(X_hsd) mu = np.asarray(mu) mu = np.swapaxes(mu, 1, 2) # -> dim: [ClustrNo x 1 x 3] std = np.asarray(std) std = np.swapaxes(std, 1, 2) # -> dim: [ClustrNo x 1 x 3] X_conv = image_dist_transform(X_hsd, mu, std, pi, mu_tmpl, std_tmpl, config.im_size, config.ClusterNo) image_dist.append(X_conv) # misc.imsave('/data2/ben/HE/data/result/color_tmp/color_norma.png', np.squeeze(X_conv)) return image_dist
def connect(): conn = None try: params = get_config() print('Connecting to the PostgreSQL database...') conn = psycopg2.connect(**params) except (Exception, psycopg2.DatabaseError) as error: print(error) sys.exit(1) print("Connection successfull!") return conn
def start(self): ''' This method starts MATS. ''' print 'Using ' + MatsController.__name__ + ' as controller.' print 'Loading config from "' + self.config_file + '"...', self.config = get_config(self.config_file) #get_config makes sure that the config makes sense. More details in get_config.py self.marionette_port = self.config['Marionette']['port'] print 'OK' print 'Starting Firefox/Nightly from "' + self.config['Firefox']['binary'] + '" with Marionette on port ' + str(self.marionette_port) + '.' self.FirefoxThread = FirefoxThread(self.config['Firefox']['binary'], self.marionette_port) self.FirefoxThread.start() print 'Creating controller' pid = self.FirefoxThread.getPID() # this function blocks until PID is available from FirefoxThread self.controller = MatsController(pid) print 'Starting controller' self.controller.start() self.controller.wait_for_ready() print 'Waiting for Marionette port to open (' + str(self.marionette_port) + ')' portReady = self.FirefoxThread.waitForMarionettePortOpenReady(self.config['Marionette']['port_timeout']) if portReady: print 'Marionette port open' else: print 'Error: timeout, shutting down MATS' self.controller.stop() self.FirefoxThread.stop() return #TODO: remove line below once https://bugzilla.mozilla.org/show_bug.cgi?id=753273 is fixed #sleep(10) try: print 'Starting Marionette' self.marionette = Marionette('localhost', self.marionette_port) #TODO: move starting session and navigation to separate methods print 'Starting session' sleep(5) #TODO temporary workaround for https://bugzilla.mozilla.org/show_bug.cgi?id=757078 self.marionette_session = self.marionette.start_session() print 'Navigating to ' + self.url print self.marionette.navigate(self.url) except Exception as e: print 'Error starting Marionette' fall(e) self.controller.stop() self.FirefoxThread.stop() print 'MATS up and running. Waiting until Firefox/Nightly stops.'
def get_twitter_auth_list(): twitter_auth_list = [] config = get_config() keys_list = config["keys"] for i in range(0, len(keys_list)): keys_arr = keys_list[i].strip().split(",") consumer_key = keys_arr[CONSUMER_KEY_INDEX] consumer_secret = keys_arr[CONSUMER_SECRET_INDEX] access_key = keys_arr[ACCESS_TOKEN_INDEX] access_secret = keys_arr[ACCESS_SECRET_INDEX] twitter_auth_list.append( Twitter(auth=OAuth(access_key, access_secret, consumer_key, consumer_secret))) return twitter_auth_list
def is_item_name_in_database(name: str, session_lock: multiprocessing.Lock) -> bool: config = get_config() engine = create_engine( '{}:///{}{}'.format( config['DEFAULT']['DatabaseType'], "C:\\Anthony\\Programs\\runescape-grand-exchange-data-analytics\\database_services\\database\\", config['DEFAULT']['DatabaseName'])) Session = sessionmaker(bind=engine) logger = py_logging.create_logger( "item_name_in_database", '{}in_database.log'.format(os.path.dirname(os.path.realpath(__file__)) + os.sep)) with session_lock: session = Session() count = session.query(Item).filter(Item.name == name).count() session.close() logger.debug("Number of {} in database: {}".format(name, count)) return bool(count)
def write(self, input_metric, input_tuple): lw.log.debug('Start metric processing') lw.log.debug("Write call started : metric=" + input_metric) self.metric = input_metric self.insert_tuple = input_tuple self.metric_conf = get_config().table_conf() self.metric_prefix = self.metric_conf[0] self.metric_suffix = self.metric_conf[1] SQL = "INSERT INTO " + self.metric_prefix + self.metric + self.metric_suffix + " (" keycount = 0 for key in self.insert_tuple[0].items(): if keycount > 0: SQL = SQL + ", " SQL = SQL + key[0] keycount = keycount + 1 SQL = SQL + ") VALUES (" keycount = 0 for key in self.insert_tuple[0].items(): if keycount > 0: SQL = SQL + ", " SQL = SQL + "%s" keycount = keycount + 1 SQL = SQL + ")" lw.log.debug('sql: ' + SQL) COUNTROWS = 0 for insert_dict in self.insert_tuple: VALUES = () COUNTROWS = COUNTROWS + 1 for value in insert_dict.items(): VALUES = VALUES + (value[1], ) lw.log.debug('INSERT_TUPLE= %s' % (VALUES, )) try: self.db_cursor.execute(SQL, VALUES) except Exception as e: lw.log.error('(Error inserting on: ' + input_metric + ')' + str(e)) try: self.db_connection.commit() lw.log.info("metric=" + input_metric + " rows=" + str(COUNTROWS)) except Exception as e: lw.log.error("metric=" + input_metric + " : " + str(e)) lw.log.debug('Write call ended : metric=' + input_metric)
async def e621(self, ctx, *args): """ .621 [tags] - sends image from e621.net. Number of tags shouldn't be more than 5. Tags must be separated by spaces. """ if ctx.message.server is not None: server = get_server(ctx.message.server) nsfw = get_channel(get_config()['token'], ctx.message.channel.id)['nsfw'] else: server = {'sfw': False, 'nsfw_channels': []} nsfw = True if ((not server['sfw']) and nsfw) or ctx.message.channel.id in server['nsfw_channels']: await self.client.send_typing( discord.Object(id=ctx.message.channel.id)) await self.e6_wrap(ctx, args) else: await self.client.add_reaction(ctx.message, '❌')
def LoadHandler(self): self.response.out.write('<br/><br/>Loading changesets<br/>') from xml.etree import cElementTree as etree config = get_config() # o,a for lat,lon # 1,2 for min,max o1,a1,o2,a2 = map(float,config['bbox'].split(',')) delta_o = o2 - o1 delta_a = a2 - a1 # load local data from localhost if DEBUG_LOAD_FROM_FILE: file_name = "data.osm" data = open(file_name,'r') else: url = "http://api.openstreetmap.org/api/0.6/changesets?bbox=%s" % config['bbox'] try: data = urllib2.urlopen(url) except urllib2.URLError, e: self.response.out.write( "error fetching url" ) return
def test_TF_model(): input_directory = get_config()[INPUT_DIR_KEY] infiles = [x for x in os.listdir(input_directory) if x[0]=='@'] test_file_path = os.path.join(input_directory, infiles[0]) with open(test_file_path, 'r') as file: in_data = json.loads(file.read()) df = pd.DataFrame(in_data, columns = ['tweet','date','embedding']) train, test = train_test_split(df['tweet'], test_size = 0.4, random_state = 1) model = TFIDFModel(use_context=False) model.characterize(train, None) sim_scores = model.similarity_score(test) assert(len(test)==len(sim_scores))
def get(self): changesets = Changeset.all().order('-created_at').fetch(20) counts = Count.all().order('-date').fetch(30) counts_list = [c.count for c in counts] counts_list.reverse() counts_list_str = ','.join(map(str,counts_list)) config = get_config() description = Description.get_or_insert('description').text options = { 'url': self.request.url, 'config': config, 'changesets': changesets, 'description': description, } if counts_list: options.update({ 'counts_list': counts_list_str, 'max_count': max(counts_list), 'average': sum(counts_list)/len(counts_list), }) path = os.path.join(os.path.dirname(__file__), 'templates/index.html') page = template.render(path,options) self.response.out.write(page)
def PrepareHandler(self): """prepare means: - find the oldest non-prepared - create bit.ly link if does not exist - create trimmed tweet text if the link exists""" config = get_config() self.response.out.write('<br/><br/>Preparing to tweet') # get oldest non-prepared changeset in the datastore oldest_changeset = Changeset.all().order('created_at').filter('is_prepared =', False).fetch(1) if oldest_changeset: c = oldest_changeset[0] self.response.out.write('<br/>There is non-prepared<br/>') self.response.out.write("id: %d<br/>" % c.id) self.response.out.write("created_at: %s<br/>" % c.created_at) self.response.out.write("comment: %s<br/>" % c.comment) self.response.out.write("user: %s<br/>" % c.user) # if no shortened link - create it if not c.link_url: long_url = "http://www.openstreetmap.org/browse/changeset/%d" % c.id bitly = BitLy(config["bitly_username"], config["bitly_key"]) short_url = bitly.short_url(long_url) if not short_url: self.response.out.write('could not shorten') return self.response.out.write(short_url) c.link_url = short_url if not c.tweet: if c.comment: tweet = "%s: %s %s" % (c.user, c.comment, c.link_url) else: tweet = "%s: %s" % (c.user, c.link_url) c.tweet = trim_to_tweet(tweet) if c.link_url and c.tweet: c.is_prepared = True self.response.out.write('tweet: %s' % c.tweet) c.put()
def __init__(self, sort_by): self.api_url = 'https://api.imgur.com/3/gallery/search/{}/'.format(sort_by) self.imgur_key = get_config('imlib.cfg', 'api_keys', 'imgur') self.headers = {'Authorization': 'Client-ID ' + self.imgur_key}
from class_changerate1 import DB from get_config import get_config data, tables = get_config() data.update({'table': "tabla6"}) #table = "tabla6" #db_params = {'host':"192.168.122.48", 'user':"******", 'passwd':"Sungard01!", 'database':"app1", 'table':table} obj = DB(host=data["host"], user=data["user"], passwd=data["password"], database=data["database"], table=data["table"]) obj.set_data_generation() #end = datetime.now().replace(microsecond=0) #print "Execution Time (Hr:Min:Sec)", end-start
#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ Takes user tweet files from 'raw_data_path', applies filters, and writes output to preprocessed_data_path. June, 2019 @author: Joshua Rubin """ from get_config import (get_config, create_dir_if_not_there) from tweetvalidator.data_processing import filter_tweets_from_directories # Pull-in filter settings from global configuration. config = get_config() create_dir_if_not_there(config['preprocessed_data_path']) filter_tweets_from_directories(config['raw_data_path'], config['preprocessed_data_path'], config['regexp_tweet_filters'], int(config['min_tweet_characters']) )
def eval_model(test_preds, model): """ Evaluate the ml model given the predictions and test data Args: test_preds - a list of transformed prediction data model - the ml pipelined model Returns: A confusion matrix, along with the precision, recall and F1 score of the currently trained model """ metrics = MulticlassMetrics(test_preds.select("prediction", "label").rdd) # Overall statistics precision = metrics.precision() recall = metrics.recall() f1Score = metrics.fMeasure() print("Confusion matrix") print(metrics.confusionMatrix()) print("Summary Stats") print("Precision = %s" % precision) print("Recall = %s" % recall) print("F1 Score = %s" % f1Score) if __name__ == "__main__": conf = get_config.get_config() spark, sc = utilities.get_spark_session("preprocessing") train_and_eval_model(conf, spark, sc) print("Done.")
def setUp(self): # case: check syntax yaml in config file self.print_info('\nGet Config file') self.conf = get_config.get_config() self.print_pass('Done')
#!/usr/bin/python from core.sg2_category import sg2_category as sg2c from core.database.sg2_database_utils import image_database from core.sg2_users import user as u import json import sys import get_config as gc cf = gc.get_config('config.dat') db = image_database(**cf['sg2']) def serach_img_from_database_by_index(index): img_table = 'sg2_image_info' rate_table = 'sg2_image_rate' image = {} img_val = db.get_table_row(img_table, "image_index='%s'"%index) if img_val == []: return image img_key = db.get_table_keys(img_table) for key, val in zip(img_key, img_val[0]): image[key[0]] = val return image if __name__== "__main__": index = sys.argv[1] print serach_img_from_database_by_index(index)
import logging import time from time import sleep, strftime THIS_DIR = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, THIS_DIR) import get_config import button_press_function #GLOBALS CONFIG_FILE_NAME = 'zone_controller.conf' CONFIG = get_config.get_config(CONFIG_FILE_NAME) LOGFILE = CONFIG.get('General', 'logfile') SERVER_IP = CONFIG.get('General', 'serverIP') SERVER_PORT = int(CONFIG.get('General', 'serverPort')) ARDUINO_ADDRESS = CONFIG.get('General', 'arduinoAddress') #Enable debugging DBG_ON = False logging.basicConfig(level=logging.DEBUG, filename=LOGFILE) #Commands for ZC ZC_CMD_GET = 'G' ZC_CMD_SET = 'S'
import pandas as pd from sklearn import linear_model import numpy as np import tweepy import requests import re import time from tweepy import OAuthHandler from get_config import get_config env = get_config() consumer_key = env.get("CONSUMER_KEY") consumer_secret = env.get("CONSUMER_SECRET") access_token = env.get("ACCESS_TOKEN") access_secret = env.get("ACCESS_TOKEN_SECRET") auth = OAuthHandler(consumer_key, consumer_secret) auth.set_access_token(access_token, access_secret) api = tweepy.API(auth) t = pd.read_csv("fetcher/top100.csv") # print t.t_id[:3] # 0 614807708575907840 # 1 618798114825220097 # 2 617840986006401024
sys.path.insert(0, join_path(dirname(__file__), "lib")) # extend sys.path # from demjson import decode as decode_json from django.utils.simplejson import loads as decode_json from google.appengine.api.urlfetch import fetch as urlfetch, GET, POST from google.appengine.ext import db from google.appengine.ext.webapp import RequestHandler, WSGIApplication # ------------------------------------------------------------------------------ # configuration -- SET THESE TO SUIT YOUR APP!! # ------------------------------------------------------------------------------ from get_config import get_config config = get_config() OAUTH_APP_SETTINGS = { "twitter": { "consumer_key": config["consumer_key"], "consumer_secret": config["consumer_secret"], "request_token_url": "https://twitter.com/oauth/request_token", "access_token_url": "https://twitter.com/oauth/access_token", "user_auth_url": "http://twitter.com/oauth/authorize", "default_api_prefix": "https://twitter.com", "default_api_suffix": ".json", }, "google": { "consumer_key": "", "consumer_secret": "", "request_token_url": "https://www.google.com/accounts/OAuthGetRequestToken",
def main(): """ Main function: calls the subfunction according to user input """ # read in the default settings from the configuration file global settings settings = get_config(default_config_file) # set the logging output i.e. to a File or the screen set_logging(settings) # get all parameters provided via cmd-line global input_params input_params = get_cmdline() # now that we know what dataset we need and where to find them, select the # correct reader for the requested dataset # first test if requested dataset does exist if settings.has_key('dataset.'+input_params['dataset']): reader = 'CF_' + input_params['dataset'] + '_Reader' else: err_msg = '[Error] -- ', now(), ' the requested dataset does not exist (is not configured)', input_params['dataset'] err_code = 3 handle_error(err_msg, err_code, settings) # call the reader module for the resepective dataset and process the data import dataset_reader attribute = getattr(dataset_reader, reader) f_read = attribute() #print 'READER: ', f_read #@@ # gets a listing of available DatasetSeries and their corresponding time-range base_flist, base_mask_flist, gfp_flist, gfpmask_flist = f_read.get_filelist(input_params, settings) # @@@@ ## processing-limits (max. filenumber to be used) here # @@@ if gfp_flist.__len__() > int(settings['general.def_maxfiles']): err_msg = '[Error] -- ', now(), ' the number of GFP products availabel (=', str(gfp_flist.__len__()).strip(),') for the selected time period is larger then the configured "def_maxfiles" of: ', settings['general.def_maxfiles'], '\n', 'Please select a shorter time-period.' err_code = 4 handle_error(err_msg, err_code, settings) # print the available input datasets: eg. during testing do_print_flist('BASE', base_flist) do_print_flist('BASE_Mask', base_mask_flist) do_print_flist('GFP', gfp_flist) do_print_flist('GFP_Mask', gfpmask_flist) lmsg = 'Dataset_listing - RUNTIME in sec: ', time.time() - startTime1 print_log(settings, lmsg) # create a temporarylocation under the provided settings['general.def_temp_dir'] to be used # for the temporary storage during processing temp_storage = tempfile.mkdtemp(prefix='cloudfree_',dir=settings['general.def_temp_dir']) if temp_storage[-1] != dsep: temp_storage = temp_storage+dsep if len(base_flist) >= 1: f_read.base_getcover(base_flist, input_params, settings, temp_storage, mask=False) if len(base_mask_flist) >= 1: f_read.base_getcover(base_mask_flist, input_params, settings, temp_storage, mask=True) lmsg = 'BASE dataset_download - RUNTIME in sec: ', time.time() - startTime1 #, '\n' print_log(settings, lmsg) # call the Processor module for the resepective dataset and process the data import dataset_processor cfprocessor = 'CF_' + input_params['dataset'] + '_Processor' attribute = getattr(dataset_processor, cfprocessor) f_proc = attribute() #print 'PROCESSOR: ', f_proc #@@ cf_result = f_proc.process_clouds_1(base_flist, base_mask_flist, gfp_flist, gfpmask_flist, input_params, settings, temp_storage, f_read) # copy results to output location and clean-up the temporary storage area do_cleanup_tmp(temp_storage, cf_result, input_params, settings) # if output_format and/or output_datatype has been set by the user -> translate resulting image(s) # using gdal_translate if change_output is True: cnv_output(cf_result, input_params, settings) # ---------- # for performance testing msg = 'Full Processing Runtime in sec: ', time.time() - startTime1, '\n' print_log(settings, msg) settings['logging.log_fsock'].close()