import time
import rediswq
import os
from playLib.log_logger import log_make_logger
global log
log=log_make_logger('WQ_DISPATCHER')
wq_listener():
host="redis-service"
q = rediswq.RedisWQ(name="job2", host="redis")
log.info("Worker with sessionID: " + q.sessionID())
log.info("Initial queue state: empty=" + str(q.empty()))
while not q.empty():
item = q.lease(lease_secs=10, block=True, timeout=2)
if item is not None:
itemstr = item.decode("utf-8")
print("Working on " + itemstr)
time.sleep(10) # Put your actual work here instead of sleep.
# later this will call wq_system
q.complete(item)
else:
log.info("Waiting for work")
log.info("Queue empty, exiting")
def wq_system(task):
print("rq_dispatch the TASK is %s" % task)
print(meta)
#with rio.open(outfile, 'w', **meta) as dst:
with rio.open(outfile,
'w',
**meta,
compress="DEFLATE",
tiled=True,
blockxsize=256,
blockysize=256) as dst:
dst.write(rst, 1)
return rst
log = log_make_logger('WORKERBEE')
log.info('starting whatever YAY!')
newitems = pickle.load(open("workitems_save.p", "rb"))
idx = my_work - 1
infile = newitems[idx]['infile']
temp = newitems[idx]['tempfile']
out = newitems[idx]['outfile']
geoj = newitems[idx]['geojson_file']
print(infile)
log.info(temp)
log.info(out)
log.info(geoj)
clip_by_geojson(infile, temp, geoj)
mid_prefix = '/'.join(bucket_file.split('/')[1:-1])
out_prefix = breed.out_prefix + '/' + mid_prefix
remote_file = 's3://' + breed.out_bucket + '/' + out_prefix + '/' + just_file
log.info("---"*25)
log.info("---"*25)
log.info(local_file)
log.info(remote_file)
s3cp(local_file, remote_file)
os.remove(local_file)
xml_from_file = 's3://' + bucket_file + '.aux.xml'
xml_to_file = remote_file + '.aux.xml'
s3cp(xml_from_file, xml_to_file)
log.info(xml_from_file)
log.info(xml_to_file)
# end of main_stuff
# ----------------------------------------------
if len(sys.argv) > 1:
global log
log=log_make_logger('COG Compressor')
log.info('starting cogification YAY!')
bucket_file = sys.argv[1]
main_stuff(bucket_file, log)
"""Now Consume those Right here"""
import time
from playLib.qbird import Qbird
from playLib.log_logger import log_make_logger
global log
log = log_make_logger('Q_BARFER')
def callback(ch, method, properties, body):
msg = " [x] Received %r" % body
print(msg)
log.info(msg)
cnt = 100
while cnt > 0:
try:
#qr=Qbird('localhost','tonyq')
qr = Qbird('rabbitmq', 'tonyq')
except:
time.sleep(3)
log.info('sleeping - waiting ... rabbitmq where are you?')
continue
break
qr.q_declare()
qr.q_set_callback(callback)
import os
import queue
import threading
import time
from playLib.log_logger import log_make_logger
global log
log = log_make_logger('TH_DISPATCHER')
class th_thread(threading.Thread):
def __init__(self, threadID, name, q, qlock):
threading.Thread.__init__(self)
self.threadID = threadID
self.name = name
self.q = q
self.qlock = qlock
self.exit_flag = 0
def run(self):
print("Starting " + self.name)
while not self.exit_flag:
self.qlock.acquire()
if not self.q.empty():
work = self.q.get()
self.qlock.release()
print("%s processing %s" % (self.name, work))
self._execute_work(work)
else:
"""Now Consume those Right here"""
import os
import time
from playLib.qbird import Qbird
from playLib.log_logger import log_make_logger
global log
log=log_make_logger('Q_SPAWNER')
def spawn_cmd(message):
msg = 'I will spawn thee: ', message
log.info(msg)
cmd = 'python3 ./' + message
log.info(cmd)
os.system(cmd)
def callback(ch, method, properties, body):
msg = body.decode('UTF-8')
print(msg)
log.info(msg)
spawn_cmd(msg)
ch.basic_ack(delivery_tag=method.delivery_tag)
cnt=100;
while cnt>0:
try:
#qr=Qbird('localhost','tonyq')
qr=Qbird('rabbitmq','tonyq')
bucket_file = '{}/{}{}'.format(bucket_name, prefix, numpy_file)
print(bucket_file)
ary = np_load_cloud(bucket_file)
amin = ary.min()
amax = ary.max()
print(amin, amax)
ary = normal(ary, amin, amax)
fname = make_fname(numpy_file)
print(fname)
pl.pl_create_png(fname, ary, cmap='magma', title=fname, subtitle=fname)
msg = '{} - created by fname with cmap = magma'.format(fname)
log.info(msg)
s3cp(fname)
os.remove(fname)
msg = '{} - removed/deleted'.format(fname)
log.info(msg)
bucket_name = "ga-et-data"
prefix = "Cloud_Veg_ET/Data/TMAX/"
numpy_file = "tmax_011_gw.tif.npy"
if len(sys.argv) > 1:
global log
log = log_make_logger('PNG Builder')
log.info('YAY!')
bucket_name, prefix, numpy_file = sys.argv[1].split(',')
main_stuff(bucket_name, prefix, numpy_file, log)