def __init__(self):
self.config = Config()
self.user = self.config.mysql_usr
self.passwd = self.config.mysql_pass
self.host = self.config.mysql_host
self.db = self.config.mysql_db
self.con = MySQLdb.connect (host=self.host, user=self.user, passwd=self.passwd, db=self.db, port=3306)
self.cursor = self.con.cursor()
def __init__(self, user=None, db=None):
from inputs.config import Config
self.config = Config()
self.host = self.config.host
self.client = MongoClient(self.host)
# self.db = self.client.(db)
# self.db = getattr(self.client, db)
self.db = self.client.japan
def __init__(self, db, kw, start, end):
# self.timer = datetime.datetime.now()
self.mongo = Mongo(db=db)
self.kwds = Keywords(keywords=kw)
self.lang = LanguageKit(keywords=self.kwds)
self.start = str(start)
self.end = str(end)
self.docs = [self.lang.FilterRetweets(t["text"]) for t in self.mongo.db["posts"].find(
{"lang":"en", 'timestamp':{"$gte": self.start, "$lt": self.end}}, {"text":1,"_id":0})]
# Spamfilter, remove redundant
self.docs = list(set(filter(None, self.docs)))
self.docs = self.lang.FilterSpam(self.docs)
self.docs = self.lang.SimpleSpamFilter(self.docs)
self.multicore = Multicore()
self.frequency = []
self.scores = []
self.trace = Config().out
self.output = Output(self.trace)
def __init__(self):
self.config = Config()
self.auth = tweepy.OAuthHandler(self.config.consumer_key,
self.config.consumer_secret)
self.auth.set_access_token(self.config.access_token_key,
self.config.access_token_secret)
#!/usr/bin/python
import sys, os, os.path
if __name__ == "__main__":
from inputs.config import Config
from inputs.dicts import Keywords
from twit.twitbot import TwitBot
from twit.search import SearchTwitterREST
from twit.stream import SearchTwitterStream
config = Config()
"""Handle command line args."""
db = sys.argv[1] if len(sys.argv) > 1 else config.db
kw = sys.argv[2:] if len(sys.argv) > 2 else [db]
"""Initialize Twitter bot."""
bot = TwitBot(user=config.user,
consumer_key=config.consumer_key,
consumer_secret=config.consumer_secret,
access_token_key=config.access_token_key,
access_token_secret=config.access_token_secret)
"""Initialize keywords and search."""
k = Keywords(keywords=kw).keywords
SearchTwitterStream(keywords=k, db=db)
def test_config(self):
bandindex = 8
confg = Config(self.sensor, bandindex)
confg.set_pixels(64, 64)
confg.set_geometry(30, 0, 0, 0)
confg.set_resoluztion(300.0, 300.0)
confg.set_surface("msf_%d" % self.bands_int[bandindex])
confg.set_aerosol_phases("mdlphs_%d" % self.bands_int[bandindex])
## get altitude grid from aerosol profile
aerosolp = SReader.aerosol_profile(taua=0.1)
z = aerosolp['z'].values #km
z = z * 1000 #convert it from km to m
# z_str = list(map(lambda x:"%.6E"%(x)),z) , z))
z_str = list(map(lambda x: "%.6E" % (x), z))
confg.set_altitude_grid(z_str)
## get aerosol extinction profile
referindex = 8 #set 865 as the reference
angstroms = AReader.get_angstroms(self.sensor, 10, referindex,
self.bands)
aprofile_s = ip.aersol(self.bands, angstroms, profile=aerosolp)
aerosol_extinct = aprofile_s['extinct_%d' %
self.bands_int[bandindex]].values
aerosol_extinct = aerosol_extinct * 0.001 #covert it from km^-1 into m^-1
taua_scaled = integrate.cumtrapz(aerosol_extinct, z, initial=0)[-1]
# print(taua_scaled)
assert (round(taua_scaled, 1) == 0.1)
confg.set_aerosol(map(lambda x: "%.6E" % (x), aerosol_extinct),
['0.9767' for i in range(aerosol_extinct.shape[0])],
38)
## get atmosphere profile
atm_profile = SReader.atm_profile()
ray_df, unit = ip.rayleight(atm_profile, self.bands)
ray_df.to_csv("../out/extinction_rayleigh_(%s).csv" % (unit))
z_ray = ray_df['z[km]'].values * 1e3 #convert from km to m
ray_extinct = ray_df['extinction_coe_%d' % self.bands_int[
bandindex]].values * 1e2 #convert from cm^-1 it into m^-1
f = inp.interp1d(z_ray, ray_extinct)
ray_extinct = f(z)
confg.set_rayleigh(map(lambda x: "%.6E" % (x), ray_extinct),
['1.000' for i in range(ray_extinct.shape[0])])
ray_scaled = integrate.cumtrapz(ray_extinct, z, initial=0)[-1]
# print(taua_scaled)
# assert (round(ray_scaled,1)==0.1)
print(ray_scaled)
## get temperature from atmosphere profile
temp_k = ray_df['t[K]'].values
f = inp.interp1d(z_ray, temp_k)
temp_k = f(z)
confg.set_tempprofile(map(lambda x: "%.6E" % (x), temp_k))
## get gas extinction profile
gas_profile = SReader.gas_profile()
no2_crosssect = SReader.no2_cross_section()
o3_crosssect = SReader.ozone_cross_section()
gas_abs, unit = ip.gas_absorption(bands=self.bands,
profile_df=gas_profile,
no2=no2_crosssect,
o3=o3_crosssect)
gas_abs.to_csv("../out/gas_abs_(%s).csv" % unit)
z_gas = gas_abs['Z'].values * 1e3 #convert from km to m
o3_extinct = gas_abs[
'abs_o3_%d' %
self.bands_int[bandindex]].values * 1e2 #convert it into m^-1
f = inp.interp1d(z_gas, o3_extinct)
o3_extinct = f(z)
no2_extinct = gas_abs[
'abs_no2_%d' %
self.bands_int[bandindex]].values * 1e2 #convert it into m^-1
f = inp.interp1d(z_gas, no2_extinct)
no2_extinct = f(z)
confg.set_gas([o3_extinct, no2_extinct])
confg.to_file("../out/config_%d" % (self.bands_int[bandindex]))
aero_phs = [0, 9, 20, 29, 50, 59, 70, 79]
resolutions = [30.0, 100.0, 500.0, 1500.0, 5000.0]
theta_s = [5, 10, 30, 50]
rho_diff = []
rho_diff_df = pd.read_csv("../database/s2_rho_diff.csv", index_col=0)
sensor = 'msia'
bands = AReader.get_bands(sensor)
bands_int = np.asarray(bands, dtype=int)
shape = "river"
columns = rows = 64
for bandindex, band in enumerate(rho_diff_df.columns[:-3]):
confg = Config(sensor, bandindex)
confg.set_pixels(rows, columns)
if not os.path.exists("../out/%d" % (bands_int[bandindex])):
os.mkdir("../out/%d" % (bands_int[bandindex]))
cur_dir = "../out/%d" % (bands_int[bandindex])
wavelenth = bands_int[bandindex]
diff_25, diff_75 = rho_diff_df.loc['25%'][band], rho_diff_df.loc['75%'][
band]
for taua_index, taua in enumerate(taua_nirls):
## get altitude grid from aerosol profile
aerosolp = SReader.aerosol_profile(taua=taua)
z = aerosolp['z'].values # km