async def home(request): from producer import Producer pro = Producer() fids = ['40', '41', '43', '44', '45', '46', '47'] for fid in fids: pro.produce(fid=fid, page=2) return await file('home.html')
async def get_page(self, fid, page=2, objs=[]): from producer import Producer pro = Producer() keys = 'moxing_' + fid + '_' + str(page) res = rds.get(keys) if res: objs, page = json.loads(res.decode('utf-8')) pro.produce(fid=fid, page=page + 1) return objs, page objs, page = await self.get_page_list(fid=fid, page=page, start_page=page, objs=[]) pro.produce(fid=fid, page=page + 1) return objs, page
from producer import Producer import logging from pathlib import Path import json from confluent_kafka import avro def import_or_install(package): try: __import__(package) except ImportError: pip.main(['install', package]) import_or_install("pandas") key_schema = avro.load(f"{Path(__file__).parents[0]}/schemas/arrival_key.json") p = Producer("hello",key_schema,1,1) p.close() from confluent_kafka import Producer import socket conf = {'bootstrap.servers': "host1:9092,host2:9092", 'client.id': socket.gethostname()} producer = Producer(conf) producer.produce("dads", key="key", value="value") producer.
class BrainsphereModel: def __init__(self, functional_connectivity, patient_data, **kwargs): self.types = [ 'ConcentrationLinear', 'Constant', 'ConcentrationSigmoid', 'WeightedDegreeLinear', 'WeightedDegreeSigmoid' ] self.producer = Producer(self.types) self.params = self.producer.params for key, value in kwargs.items(): if key == "nodeCoordinates": self.nodeCoordinates = value elif key == "optimizer": self.optimizer = value elif key == "loss": self.loss = value elif key == "euclideanAdjacency": self.euclideanAdjacency = value elif key == "producer": self.producer = value elif key == "diffuser": self.diffuser = value elif key == "params": self.params.update(value) else: raise TypeError("Illegal Keyword '" + str(key) + "'") self.functionalConnectivity = functional_connectivity self.patientData = patient_data self.numNodes, _ = np.shape(functional_connectivity) self.loss = Loss("mse", self.patientData) self.lastloss = 0 self.reset() def reset(self): self.initializer = Initializer("braak1", self.numNodes, self.params) self.concentration = self.initializer.get() self.concentrationHistory = np.copy(self.concentration) self.producer = Producer(self.types) self.diffusor = Diffusor("euclidean", self.params, EuclideanAdjacency=self.euclideanAdjacency) def run(self): stop_concentration = 1100 timesteps = 2500 self.reset() deltaT = 0.0001 self.concentration += deltaT * ( self.producer.produce(params=self.params, concentration=self.concentration, connectivity=self.functionalConnectivity) + self.diffusor.diffuse(self.concentration)) self.concentrationHistory = np.append(self.concentrationHistory, self.concentration, axis=0) deltaConc = np.sum(self.concentrationHistory[1, :]) - np.sum( self.concentrationHistory[0, :]) if deltaConc <= 0.0: return 9999999 else: deltaT *= stop_concentration / timesteps / deltaConc while (np.sum(self.concentration) < stop_concentration) and (np.sum(deltaConc) > 0): deltaConc = deltaT * (self.producer.produce( params=self.params, concentration=self.concentration, connectivity=self.functionalConnectivity) + self.diffusor.diffuse(self.concentration)) self.concentration += deltaConc self.concentrationHistory = np.append(self.concentrationHistory, self.concentration, axis=0) # print(self.loss.get(self.concentrationHistory)) self.lastloss = self.loss.get(self.concentrationHistory) return self.lastloss def gradient(self, loss=None): if loss is None: loss = self.run() params_new = {} params_old = self.params.copy() deltaX = {} for key, value in params_old.items(): deltaX[key] = np.sign(np.random.randn()) * 0.01 params_new[key] = value + deltaX[key] self.params = params_new new_loss = self.run() grad = {} self.params = params_old for key, value in params_old.items(): grad[key] = (new_loss - loss) / (deltaX[key]) return grad def gradient4(self): loss = self.run() gradients = Parallel(n_jobs=4)(delayed(self.gradient)(loss) for i in range(4)) grad = {} for key in self.params: gradsum = 0 count = 0 for g in gradients: gradsum += g.get(key) count += 1.0 grad[key] = gradsum / count return grad
def ez_produce(name, queue, data, is_rpc=False, rpc_attempts=25): """ Send data to queue. """ if not (name and queue and data and isinstance(data, dict)): return name = name.upper() if not data: return False try: producer = Producer(host=os.getenv('RABBITMQ_HOST'), port=os.getenv('RABBITMQ_PORT', 5672), vhost=os.getenv('RABBITMQ_VHOST', '/'), username=os.getenv('RABBITMQ_%s_USER' % name), password=os.getenv('RABBITMQ_%s_PASS' % name), is_rpc=is_rpc, rpc_attempts=rpc_attempts) response = producer.produce(queue=os.getenv('RABBITMQ_%s_QUEUE' % name, queue), value=json.dumps(data)) # This MUST be ran through json.loads again... if response: response = json.loads(response) # Couldn't connect to rabbitmq, most likely except AttributeError as e: message = 'Your .env file is probably not set up correctly.' print({ 'MESSAGE': message, 'NAME': name, 'QUEUE': queue, 'RABBITMQ_HOST': os.getenv('RABBITMQ_HOST'), 'RABBITMQ_PORT': os.getenv('RABBITMQ_PORT', 5672), 'RABBITMQ_VHOST': os.getenv('RABBITMQ_VHOST', '/'), 'RABBITMQ_%s_USER': os.getenv('RABBITMQ_%s_USER' % name), 'RABBITMQ_%s_PASS': os.getenv('RABBITMQ_%s_PASS' % name), }) logger.write_log('%s_PRODUCER_ERROR_ATTR_ERROR' % name, message) return False # Type errors happen when data cannot be serialized to JSON except TypeError as e: logger.write_log('%s_PRODUCER_ERROR_TYPE_ERROR' % name, str(e)) return False # Any other exception except Exception as e: if is_rpc: raise e else: logger.write_log('%s_PRODUCER_ERROR' % name, str(e)) return False if not is_rpc: return True if response: return json.loads(response) if isinstance(response, str) else response
from os import getenv from socket import gethostname import logging from producer import Producer logging.basicConfig(level=logging.INFO) myproducer = Producer(kafka_host=getenv("KAFKA_HOST"), kafka_topic=getenv("KAFKA_TOPIC"), machine_identifier=gethostname()) myproducer.produce()