def _connect_signals(self):
"""Connect callbacks to celery signals.
Since we are creating partials here, the weak arg must be False.
"""
# Beat
if self.options["show_beat"]:
beat_init.connect(
slack_beat_init(**self.options),
weak=False
)
# Celery
if self.options["show_startup"]:
celeryd_init.connect(
slack_celery_startup(**self.options),
weak=False
)
if self.options["show_shutdown"]:
worker_shutdown.connect(
slack_celery_shutdown(**self.options),
weak=False
)
# Task
task_prerun.connect(
slack_task_prerun(**self.options),
weak=False
)
def run(self, user_info_table, use_screenname=False):
worker_shutdown.connect(self.on_shutdown)
keystore = KeyStore()
#self.keystore.load_access_tokens_from_file()
self.access_tokens = keystore.acquire_access_tokens()
print "DENEME"
# If there is no available access_tokens
if not self.access_tokens:
self.logger.error("No access tokens. Sorry. Sending a message to retry 3 mins later."
"Hopefully someone else will pick it up, if it's not me.")
raise self.retry(countdown=3*60)
self.available_twitter_api_array = []
i = 1
for key_and_secret in self.access_tokens:
api = twitter.Api(keystore.app_consumer_key, keystore.app_consumer_secret, key_and_secret[0], key_and_secret[1], cache=None)
self.available_twitter_api_array.append(["(key %s)" % i, api])
i += 1
self.jobs = []
self.use_screenname = use_screenname
self.user_info_table = user_info_table
self.n_user_identifiers = len(self.user_info_table)
finished = False
while not finished:
while len(self.user_info_table) > 0:
# Assign stale API connections to the next job items.
# If there is a stale API connection, continue
if len(self.available_twitter_api_array) > 0:
(user, since_tweet_id, page_not_found) = self.user_info_table.pop()
user_identifier = self.get_user_identifier(user)
# (since_tweet_id, page_not_found, update_required) = get_userinfo(self.db_cursor, user_identifier)
if page_not_found == 1:
self.logger.info("Skipping " + user_identifier + " (we got page not found error before)")
# removed this because this decision is taken at drenaj_api
# elif not update_required:
# self.logger.info("Skipping " + user_identifier + " (not expired yet)")
else:
[token_owner_name, api] = self.available_twitter_api_array.pop()
t = TimelineHarvester(api, self.logger, user, since_tweet_id)
task_start_time = time.time()
t.start()
self.logger.info("Thread "+token_owner_name+" => "+user_identifier+" starting..")
self.logger.info("PROGRESS: " + str(len(self.user_info_table)) + "/"+str(self.n_user_identifiers))
self.jobs.append([t, user, task_start_time, api, token_owner_name])
else:
# No stale API connections found, break out of this loop.
break
if len(self.jobs) == 0:
finished = True
tmp_jobs = []
while len(self.jobs) > 0:
job = self.jobs.pop()
[t, user, task_start_time, api, token_owner_name] = job
user_identifier = self.get_user_identifier(user)
t.join(0.001)
if not t.isAlive():
time_elapsed = int(time.time()-task_start_time)
self.logger.info("Stopping thread "+user_identifier+" - (duration: "+str(time_elapsed)+" secs) - "+token_owner_name)
# update_userinfo(self.db_cursor, user_identifier, update_since_tweet_id, *result)
self.available_twitter_api_array.append([token_owner_name, api])
else:
tmp_jobs.append(job)
self.jobs = tmp_jobs
def run(self, user_info_table, use_screenname=False):
worker_shutdown.connect(self.on_shutdown)
print type(user_info_table)
print user_info_table
keystore = KeyStore()
#self.keystore.load_access_tokens_from_file()
self.access_tokens = keystore.acquire_access_tokens()
print "DENEME"
# If there is no available access_tokens
if not self.access_tokens:
self.logger.error(
"No access tokens. Sorry. Sending a message to retry 3 mins later."
"Hopefully someone else will pick it up, if it's not me.")
raise self.retry(countdown=3 * 60)
self.available_twitter_api_array = []
i = 1
for key_and_secret in self.access_tokens:
api = twitter.Api(keystore.app_consumer_key,
keystore.app_consumer_secret,
key_and_secret[0],
key_and_secret[1],
cache=None)
self.available_twitter_api_array.append(["(key %s)" % i, api])
i += 1
self.jobs = []
self.use_screenname = use_screenname
self.user_info_table = user_info_table
self.n_user_identifiers = len(self.user_info_table)
finished = False
while not finished:
while len(self.user_info_table) > 0:
# Assign stale API connections to the next job items.
# If there is a stale API connection, continue
if len(self.available_twitter_api_array) > 0:
(user, page_not_found) = self.user_info_table.pop()
user_identifier = self.get_user_identifier(user)
# (since_tweet_id, page_not_found, update_required) = get_userinfo(self.db_cursor, user_identifier)
if page_not_found == 1:
self.logger.info(
"Skipping " + user_identifier +
" (we got page not found error before)")
# removed this because this decision is taken at drenaj_api
# elif not update_required:
# self.logger.info("Skipping " + user_identifier + " (not expired yet)")
else:
[token_owner_name,
api] = self.available_twitter_api_array.pop()
t = UserInfoHarvester(api, self.logger, user)
task_start_time = time.time()
t.start()
self.logger.info("PROGRESS: " +
str(len(self.user_info_table)) + "/" +
str(self.n_user_identifiers))
self.logger.info("Thread User Info Harvester " +
token_owner_name + " => " +
user_identifier + " starting..")
self.jobs.append(
[t, user, task_start_time, api, token_owner_name])
else:
# No stale API connections found, break out of this loop.
break
if len(self.jobs) == 0:
finished = True
tmp_jobs = []
while len(self.jobs) > 0:
job = self.jobs.pop()
[t, user, task_start_time, api, token_owner_name] = job
user_identifier = self.get_user_identifier(user)
t.join(0.001)
if not t.isAlive():
time_elapsed = int(time.time() - task_start_time)
self.logger.info("Stopping thread " + user_identifier +
" - (duration: " + str(time_elapsed) +
" secs) - " + token_owner_name)
# update_userinfo(self.db_cursor, user_identifier, update_since_tweet_id, *result)
self.available_twitter_api_array.append(
[token_owner_name, api])
else:
tmp_jobs.append(job)
self.jobs = tmp_jobs
def __init__(self):
self.seqs = [0] *const.SEQ_MAX
self.__wid = 0
worker_shutdown.connect(self.release_wid)
# gracefully. Will be reimplemented using a custom `bootstep`
# tunnel.create_server(
# redis_server,
# local_node_port,
# ssh_password=ssh_password,
# # ssh_host_key=ssh_host_key
# )
tun = tunnel.BashTunnel(
redis_server,
local_node_port
)
celeryd_init.connect(tun.open)
worker_shutdown.connect(tun.close)
# ------------------------------------------------------------------------------
# DEFINE TASKS
# ------------------------------------------------------------------------------
@app.task
def directory(mypath):
return os.listdir(mypath)
@app.task
def generate(
dcd_filename,
steps,
def get_all_queues(self):
self.update_workers()
jobs = []
for q in self.queues:
jobs.extend(self.get_queue(q))
return jobs
def check_job(self, job):
self.update_workers()
for q in self.queues:
for j in self.get_queue(q):
if str(job) in j:
logging.debug("Found alive job %s in queue %s" % (j, q))
return True
return False
inspector = CeleryInspector()
def update_inspector(signal=None, sender=None, **kwargs):
logging.debug("Received signal %s, from sender %s" % (signal, sender))
inspector.flag_update()
worker_ready.connect(update_inspector)
worker_shutdown.connect(update_inspector)
