def CollectFollowing(self, handle):
try:
self.following_stat = self.GetFollowingStats(handle)
self.pi("\n" + self.__notify.INFO + 'Attempting to collect the {} Users that {} is following'.format(self.following_stat, handle))
self.main = self.GetCursor("https://mobile.twitter.com/{}/following".format(handle))
self.__collection_following_pages.append("https://mobile.twitter.com/{}/following".format(handle))
while True:
if self.main is None and self.following_stat > 1:
return self.GetFollows(self.__collection_following_pages[0])
#self.__found_following
elif self.main is not None:
for item in self.main:
self.main = self.GetCursor(item)
self.__collection_following_pages.append(item)
if self.main is None: # Will surpress an NoneType error.
for obvious in self.__collection_following_pages:
self.appendage = self.GetFollows(obvious)
for trace in self.appendage:
self.__found_following.append(trace)
database.store().hoard_handles(self.__found_following)
return self.__found_following
else:
return None
except TypeError, e:
print e
exit(1)
self.pi(self.__notify.INFO + 'User Doesn\'t exist yet.')
exit(0)
def CollectFollowers(self, handle):
try:
self.followers_stat = self.GetFollowerStats(handle)
self.pi("\n" + self.__notify.INFO + 'Attempting to collect {} of {}\'s followers'.format(self.followers_stat ,handle))
self.main = self.GetCursor("https://mobile.twitter.com/{}/followers".format(handle))
self.__collection_followers_pages.append("https://mobile.twitter.com/{}/followers".format(handle))
while True:
if self.main is None and self.following_stat > 1:
return self.GetFollows(self.__collection_followers_pages[0])
elif self.main is not None:
for item in self.main:
self.main = self.GetCursor(item)
self.__collection_followers_pages.append(item)
if self.main is None: # Will /dev/null a NoneType error message.
for obvious in self.__collection_followers_pages:
self.appendage = self.GetFollows(obvious)
for trace in self.appendage:
self.__found_followers.append(trace)
database.store().hoard_handles(self.__found_followers)
return self.__found_followers
else:
return None
except TypeError:
self.pi(self.__notify.INFO + 'User Doesn\'t exist yet.')
exit(0)
def add_score(group_code, from_uin):
if not database.dic.has_key("score"):
database.dic["score"] = dict()
key = group_code + ":" + from_uin
if not database.dic["score"].has_key(key):
database.dic["score"][key] = 0
database.dic["score"][key] += 1
database.store("score")
def delete_learnt(key):
if not database.dic.has_key("chat"):
return u"Failed!"
if not database.dic["chat"].has_key(key):
return u"Failed!"
database.dic["chat"].pop(key)
database.store("chat")
return u"OK!"
def learn(key, value):
key = key.decode('utf-8')
value = value.encode('utf-8')
key = re.sub(u'[【】!?,！？，。]', u'', key)
if not database.dic.has_key("chat"):
database.dic["chat"] = dict()
if not database.dic["chat"].has_key(key):
database.dic["chat"][key] = []
if isinstance(database.dic["chat"][key], str) or isinstance(database.dic["chat"][key], unicode):
database.dic["chat"][key] = [ database.dic["chat"][key] ]
database.dic["chat"][key].append(value)
database.store("chat")
return u"Learn {0} => {1} OK!".format(key, value)
def GetEntireProfile(self, handle):
self.pi("\n{}Attempting to Generate {}'s Twitter Profile information.".format(self.__notify.INFO, handle))
self.response = self.GetRequest('https://mobile.twitter.com/{}?p=s'.format(handle))
if self.response.status_code == 404:
self.pi("{} 404: User not found!\n".format(ERROR))
exit(0)
# Location data.
self.profile_location_container = re.search(r'<div class="location">.*?</div>', self.response.content)
self.profile_location = self.profile_location_container.group()
self.location = self.profile_location.replace('<div class="location">', '').replace("</div>", '')
# Name data.
self.profile_name_container = re.search(r'<span class="screen-name">.*?</span>', self.response.content)
self.profile_name_block = self.profile_name_container.group()
self.profile_name = self.profile_name_block.replace('<span class="screen-name">', '').replace('</span>', '')
# Stat numbers data | Tweets | Following | Followers
self.stat_numbers = re.findall(r'<div class="statnum">.*?</div>', self.response.content)
self.total_tweets = self.stat_numbers[0].replace('<div class="statnum">', '').replace('</div>', "")
self.following_total = self.stat_numbers[1].replace('<div class="statnum">', '').replace('</div>', "")
self.followers_total = self.stat_numbers[2].replace('<div class="statnum">', '').replace('</div>', "")
# Get an identification number from a twitter handle.
self.id_number = str(self.GetIDFromHandle(handle))
self.profile_link = self.__notify.PL + handle
# Output information to console.
self.pi("" + self.__notify.TID + self.id_number)
self.pi("" + self.__notify.TWEETS + self.total_tweets)
self.pi("" + self.__notify.FOLLOWING + self.following_total)
self.pi("" + self.__notify.FOLLOWERS + self.followers_total)
self.pi("" + self.__notify.USERNAME + handle)
if self.location is not "":
self.pi(self.__notify.LOCATION + self.location)
self.pi(self.__notify.PROFILELINK + self.profile_link)
# self, ID, Username, Followers, Following, Tweets, PFL
database.store().profile2file(self.id_number, handle, self.followers_total, self.following_total, self.total_tweets, self.profile_link)
def CollectHandlesFromTrendStream(self):
trend = self.PickLatestTrend()
self.stream_location = 'https://mobile.twitter.com/search?q=%23{}'.format(trend)
self.stream_content = self.GetRequest(self.stream_location).content
self.pi(self.__notify.INFO + "Collecting users from the {} trend.".format(self.color.W + trend + self.color.N))
while True:
self.main_content = self.GetRefreshStreamCursor(self.stream_content)
if self.main_content != None:
self.refresh = self.__domain + self.main_content
self.stream_content = self.GetRequest(self.refresh).content
self.handles = self.GetUserNameFromStream(self.stream_content)
if self.handles != []:
database.store().hoard_handles(self.handles)
time.sleep(7) # Wait for more people to tweet about the current hashtag.
elif self.main_content == None:
self.pi(self.__notify.INFO + 'restarting the collection process ...')
self.CollectHandlesFromTrendStream() # Creates a vicious continuous loop.
def resnet_main(flags, model_function, input_function, shape=None):
"""Shared main loop for ResNet Models.
Args:
flags: FLAGS object that contains the params for running. See
ResnetArgParser for created flags.
model_function: the function that instantiates the Model and builds the
ops for train/eval. This will be passed directly into the estimator.
input_function: the function that processes the dataset and returns a
dataset that the estimator can train on. This will be wrapped with
all the relevant flags for running and passed to estimator.
shape: list of ints representing the shape of the images used for training.
This is only used if flags.export_dir is passed.
"""
# Using the Winograd non-fused algorithms provides a small performance boost.
os.environ['TF_ENABLE_WINOGRAD_NONFUSED'] = '1'
if flags.multi_gpu:
validate_batch_size_for_multi_gpu(flags.batch_size)
# There are two steps required if using multi-GPU: (1) wrap the model_fn,
# and (2) wrap the optimizer. The first happens here, and (2) happens
# in the model_fn itself when the optimizer is defined.
model_function = tf.contrib.estimator.replicate_model_fn(
model_function,
loss_reduction=tf.losses.Reduction.MEAN)
# Create session config based on values of inter_op_parallelism_threads and
# intra_op_parallelism_threads. Note that we default to having
# allow_soft_placement = True, which is required for multi-GPU and not
# harmful for other modes.
session_config = tf.ConfigProto(
inter_op_parallelism_threads=flags.inter_op_parallelism_threads,
intra_op_parallelism_threads=flags.intra_op_parallelism_threads,
allow_soft_placement=True)
# Set up a RunConfig to save checkpoint and set session config.
run_config = tf.estimator.RunConfig().replace(save_checkpoints_secs=1e9,
session_config=session_config)
classifier = tf.estimator.Estimator(
model_fn=model_function, model_dir=flags.model_dir, config=run_config,
params={
'resnet_size': flags.resnet_size,
'data_format': flags.data_format,
'batch_size': flags.batch_size,
'multi_gpu': flags.multi_gpu,
'version': flags.version,
})
if flags.build_database:
import database
data_path = os.path.join(flags.data_dir, 'predict.tfrecord')
def input_fn_pred():
return input_function(False, data_path, flags.batch_size,
flags.epochs_between_evals,
flags.num_parallel_calls, flags.multi_gpu)
result = classifier.predict(input_fn=input_fn_pred)
if flags.build_database:
results = []
for one in result:
results.append(one['logits'])
database.store(results)
return
if flags.benchmark_log_dir is not None:
benchmark_logger = logger.BenchmarkLogger(flags.benchmark_log_dir)
benchmark_logger.log_run_info("resnet")
else:
benchmark_logger = None
for _ in range(flags.train_epochs // flags.epochs_between_evals):
train_hooks = hooks_helper.get_train_hooks(
flags.hooks,
batch_size=flags.batch_size,
benchmark_log_dir=flags.benchmark_log_dir)
print('Starting a training cycle.')
def input_fn_train():
data_path = os.path.join(flags.data_dir, 'train.tfrecord')
return input_function(True, data_path, flags.batch_size,
flags.epochs_between_evals,
flags.num_parallel_calls, flags.multi_gpu)
classifier.train(input_fn=input_fn_train, hooks=train_hooks,
max_steps=flags.max_train_steps)
print('Starting to evaluate.')
# Evaluate the model and print results
def input_fn_eval():
data_path = os.path.join(flags.data_dir, 'test.tfrecord')
return input_function(False, data_path, flags.batch_size,
1, flags.num_parallel_calls, flags.multi_gpu)
# flags.max_train_steps is generally associated with testing and profiling.
# As a result it is frequently called with synthetic data, which will
# iterate forever. Passing steps=flags.max_train_steps allows the eval
# (which is generally unimportant in those circumstances) to terminate.
# Note that eval will run for max_train_steps each loop, regardless of the
# global_step count.
eval_results = classifier.evaluate(input_fn=input_fn_eval,
steps=flags.max_train_steps)
print(eval_results)
if benchmark_logger:
benchmark_logger.log_estimator_evaluation_result(eval_results)
if flags.export_dir is not None:
warn_on_multi_gpu_export(flags.multi_gpu)
# Exports a saved model for the given classifier.
input_receiver_fn = export.build_tensor_serving_input_receiver_fn(
shape, batch_size=flags.batch_size)
classifier.export_savedmodel(flags.export_dir, input_receiver_fn)
import time
from database import store
base_dir = '/sys/bus/w1/devices/'
def read_temp_raw():
device_folder = glob.glob(base_dir + '28*')[0]
device_file = device_folder + '/w1_slave'
f = open(device_file, 'r')
lines = f.readlines()
f.close()
return lines
def read_temp():
lines = read_temp_raw()
while lines[0].strip()[-3:] != 'YES':
time.sleep(0.2)
lines = read_temp_raw()
equals_pos = lines[1].find('t=')
if equals_pos != -1:
temp_string = lines[1][equals_pos + 2:]
temp_c = float(temp_string) / 1000.0
return temp_c
if __name__ == "__main__":
store(read_temp())
