def main(): Queues = hlp.getData() for queue in Queues: print(queue) print("Customer") print(genCustomer(Queues[queue])) print("")
def threadedSS(connection: socket.socket): length = getLength(connection) allData = recv_all(connection, length) allData = pickle.loads(allData) needToVisit = allData['needToVisit'] url = allData['url'] print("Request: ", url) if len(needToVisit) == 0: # last stepping stone so get the file print("chainlist is empty") dataToSendBack = getData(allData['url']) dataToSendBack = pickle.dumps(dataToSendBack) sendDataHeader(connection, len(dataToSendBack)) print("Relaying file ...") connection.sendall(dataToSendBack) else: # get the next stepping stone chainfile = needToVisit addr, port = getAddrNextSteppingStone(chainfile) allData = pickle.dumps(allData) s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.connect((addr, port)) sendDataHeader(s, len(allData)) s.sendall(allData) length = getLength(s) print("waiting for file ...") data = recv_all(s, length) sendDataHeader(connection, len(data)) print("Relaying file ...") connection.sendall(data) connection.close() print("Goodbye!")
def main(): Queues = hlp.getData() for queue in Queues: print(queue) print("Service Rate") cardOrCashLiabil = hlp.TimeDifference(Queues[queue]) print serviceRateCard(Queues[queue][5], Queues[queue][7], cardOrCashLiabil)
def main(): Queues = hlp.getData() for queue in Queues: print(queue) print("Droput") dropProbs = get_dropout_probability_ranges(Queues[queue]) print(dropout(Queues[queue])) print(bernoulli_drop_out(dropProbs, 50)) print("")
def main(): #items = 30 #30 items infront of customer Queues = hlp.getData() for queue in Queues: print(queue) print("Arrival Rate") #rates = rateItems(Queues[queue]) #print(genArrivalRate(rates,queue)) print("")
def getUniqueRestraunts(self): df = helpers.getData() restaurants = Series(np.unique(df["Restaurant"].map(lambda x: x.strip()))) for ind in range(len(restaurants)): restaurants[ind] = restaurants[ind].title() restaurants = restaurants.sort_values() del df print(restaurants) return restaurants
def createPredictions(self, parameters, companies): bigDF = helpers.getData() day = parameters[0] rest = parameters[1] companies = companies predictions = appPredictions.getPredictions(bigDF, day, rest, companies) predList = [] for i in predictions: predList.append(i) del predictions return predList
def run(rank, world_size): os.environ["MASTER_ADDR"] = "localhost" os.environ["MASTER_PORT"] = "10638" dist_init(rank, world_size) os.environ["MASTER_PORT"] = "10639" dist.rpc.init_rpc(f"worker{rank}", rank=rank, world_size=world_size) initialize_model_parallel(1, world_size) model = getModel() data, target = getData()[0] loss_fn = getLossFun() device = torch.device("cuda", rank) if DEVICE == "cuda" else torch.device("cpu") model = fairscale.nn.Pipe( model, balance=[2, 1], style=fairscale.nn.Pipe.MultiProcess, worker_map={ 0: "worker0", 1: "worker1" }, # Needed to convert ranks to RPC worker names input_device=device, ).to(device) # define optimizer and loss function optimizer = optim.SGD(model.parameters(), lr=0.001) # zero the parameter gradients optimizer.zero_grad() # outputs and target need to be on the same device # forward step outputs = model(data.to(device)) # compute loss if rank == 1: loss = loss_fn(outputs.to(device), target.to(device)) # backward + optimize loss.backward() optimizer.step() else: model.back_helper(outputs) print(f"Finished Training Step on {rank}") del model
def run(rank, world_size): torch_pg.init_mpi() os.environ["MASTER_ADDR"] = "localhost" os.environ["MASTER_PORT"] = "10638" dist_init(rank, world_size) # FIXME (supports gloo) os.environ["MASTER_PORT"] = "10639" torch.distributed.rpc.init_rpc(f"worker{rank}", rank=rank, world_size=world_size) initialize_model_parallel(1, world_size, pipeline_backend="mpi") if rank == 1: # For RPC, all ranks other than 0 just need to call rpc.shutdown() torch.distributed.rpc.shutdown() return model = getModel() data, target = getData()[0] loss_fn = getLossFun() device = torch.device("cuda", rank) model = fairscale.nn.PipeRPCWrapper( model, balance=[2, 1], worker_map={ 0: "worker0", 1: "worker1" }, # Needed to convert ranks to RPC worker names input_device=device, ).to(device) # We can't directly access the model on each worker, so we need to call # foreach_worker with a callback to setup the optimizer model.foreach_worker(register_optimizer, {"lr": 0.001}, include_self=True) outputs = model(data.to(device)) loss = loss_fn(outputs.to(device), target.to(device)) loss.backward() # Same as earlier, use foreach_worker to step the optimizer on each rank model.foreach_worker(run_optimizer, include_self=True) print(f"Finished Training Step on {rank}") torch.distributed.rpc.shutdown() del model
def city(city): weather_data = getData(city) weather = { "city": city, "temp": round((weather_data["main"]["temp"] - 273.15), 1), "temp_feels": round((weather_data["main"]["feels_like"] - 273.15), 1), "temp_min": round((weather_data["main"]["temp_min"] - 273.15), 1), "temp_max": round((weather_data["main"]["temp_max"] - 273.15), 1), "pressure": weather_data["main"]["pressure"], "humidity": weather_data["main"]["humidity"], "wind": weather_data["wind"]["speed"], "cloudiness": weather_data["clouds"]["all"], "desc": weather_data["weather"][0]["main"], "icon": weather_data["weather"][0]["icon"] } return render_template("city.html", weather=weather)
def main(): # adding new city to the database if request.method == "POST": city = request.form.get("name") if city: city_to_add = Weather(name=city) if Weather.query.filter_by(name=city).first() is None: db.session.add(city_to_add) db.session.commit() # getting all cities cities = Weather.query.all() print("cities : ") print(cities) # if there are more than cities, delete the first one if len(cities) > 5: to_delete = Weather.query.first() db.session.delete(to_delete) db.session.commit() cities.pop(0) all_weather_data = [] alert = None for city in reversed(cities): weather_data = getData(city.name) if (weather_data["cod"] == '404'): alert = city.name to_delete = Weather.query.filter_by(name=city.name).first() db.session.delete(to_delete) db.session.commit() else: weather_to_add = { "city": weather_data["name"], "temp": round((weather_data["main"]["temp"] - 273.15), 1), "text": weather_data["weather"][0]["description"], "icon": weather_data["weather"][0]["icon"] } all_weather_data.append(weather_to_add) print(weather_data) return render_template("main.html", alert=alert, weather_list=all_weather_data)
from helpers import getData, getLossFun, getModel import torch import torch.optim as optim import fairscale DEVICE = "cuda" if torch.cuda.is_available() else "cpu" RANK = 0 # example model = getModel() data, target = getData()[0] loss_fn = getLossFun() model = fairscale.nn.Pipe(model, balance=[2, 1]) # define optimizer and loss function optimizer = optim.SGD(model.parameters(), lr=0.001) # zero the parameter gradients optimizer.zero_grad() device = torch.device("cuda", RANK) if DEVICE == "cuda" else torch.device("cpu") # outputs and target need to be on the same device # forward step outputs = model(data.to(device).requires_grad_()) # compute loss loss = loss_fn(outputs.to(device), target.to(device)) # backward + optimize
secondInopts=opt2) else: self.assertFileEqual(val, key.format(**outfiles), firstInopts=opt1, secondInopts=opt2) def __init__(self, *args, **kwargs): testly.TestCase.__init__(self, *args, **kwargs) members = {'__init__': __init__} proc = datarows[0].kwargs['proc'][len(klass) + 1:] proc = 'test' + proc dataprovider = 'dataProvider_' + proc members[dataprovider] = dataprovider_maker members[proc] = testit globals()['Test' + klass[0].upper() + klass[1:]] = type( klass, (testly.TestCase, ), members) DATAROWS = defaultdict(lambda: []) for data in getData(): klass = data.kwargs['proc'].split('.')[0] DATAROWS[klass].append(data) for key, val in DATAROWS.items(): TestClassFactory(key, val) if __name__ == '__main__': testly.main(verbosity=2)
from keras.models import Sequential from keras.optimizers import Adam from keras.utils import plot_model import helpers from helpers import INPUT_SHAPE gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.333) sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) dataset_dir_name, checkpoint_dir_name, stats_dir_name, model_name = helpers.getInput( ) data = helpers.calibrate(dataset_dir_name) training_data, testing_data = helpers.getData(data, 75) trainingImages = helpers.getMatrix(training_data) testingImages = helpers.getMatrix(testing_data) print("_" * 40) print("Image translation to matrix completed") print("_" * 40) now = datetime.now() date_time = now.strftime("%m/%d/%Y, %H:%M:%S") # NAME = f"{model_name}_{date_time}" tensorboard = TensorBoard(log_dir=f'logs/{model_name}') def build_model():
def train(rank: int, world_size: int, epochs: int, use_oss: bool): # DDP dist_init(rank, world_size) rank = torch.device("cpu") if DEVICE == "cpu" else rank # Problem statement model = getModel().to(rank) dataloader = getData(n_batches=1) loss_fn = getLossFun() optimizer: Optional[Union[OSS, torch.optim.SGD]] = None if not use_oss: optimizer = torch.optim.SGD(params=model.parameters(), lr=1e-4) else: base_optimizer = torch.optim.SGD base_optimizer_arguments = { "lr": 1e-4 } # any optimizer specific arguments, LR, momentum, etc... optimizer = OSS(params=model.parameters(), optim=base_optimizer, **base_optimizer_arguments) training_start = time.monotonic() # Any relevant training loop, nothing specific to OSS. For example: model.train() for _ in range(epochs): for (data, target) in dataloader: data, target = data.to(rank), target.to(rank) # Train model.zero_grad() outputs = model(data) loss = loss_fn(outputs, target) loss.backward() # if you want to clip the gradients / get the current max: max_norm = 1000.0 norm_type = 1 if not use_oss: _total_norm = torch.nn.utils.clip_grad_norm_( model.parameters(), max_norm, norm_type=norm_type) # type: ignore else: optimizer = cast(OSS, optimizer) _total_norm = optimizer.clip_grad_norm(max_norm, norm_type=norm_type) optimizer.step() print(f"Loss: {loss.item()}") training_end = time.monotonic() print( f"[{dist.get_rank()}] : Training done. {training_end-training_start:.2f} sec" ) if DEVICE == "cuda": max_memory = torch.cuda.max_memory_allocated(rank) print(f"[{dist.get_rank()}] : Peak memory {max_memory:.1f}MiB")
chart_files = [x.strip(' ') for x in chart_files] hawk_mode = str(os.getenv('HAWK_MODE')) print('Currently Running in {}'.format(hawk_mode)) tokens = helpers.plaidTokens() start_of_month = helpers.monthStart() chart_studio.tools.set_credentials_file(username=os.getenv('PLOTLY_USERNAME'), api_key=os.getenv('PLOTLY_API_KEY')) clear_file = helpers.clear_data_file() print(clear_file) encoded = base64.b64encode(open('templates/logo.png', "rb").read()) #Data from Plaid master_data = helpers.getData(hawk_mode, exclusions) data = dict(date=date.today().strftime('%m/%d/%y'), balance_chase=master_data['balance_chase'], chase_total=master_data['chase_total'], balance_schwab=master_data['balance_schwab'], schwab_total=master_data['schwab_total'], balance_cap_one=master_data['cap1_balance'], capone_total=master_data['cap1_total'], balance_great_lakes=master_data['lakes_balance'], greatlakes_total=master_data['lakes_total'], chart_pack={}, logo='data:image/png;base64,{}'.format(encoded.decode('utf8')), tables=[]) chart_links = [] #Update Guage Chart
from os import environ from dotenv import load_dotenv from sendgrid.helpers.mail import Mail from sendgrid import SendGridAPIClient import python_http_client import helpers load_dotenv() # keep_list = environ.get('KEEP_LIST').split(',') # keep_list = [x.strip(' ') for x in keep_list] new_transactions = helpers.frame_prep(helpers.getData(), environ.get('KEEP_LIST')) helpers.addTransaction(new_transactions) nt_styled = new_transactions.style.set_caption('New Transactions').set_table_styles(helpers.tableStyles()).set_table_attributes('border="1" class="dataframe table table-hover table-bordered"') html_data = helpers.jinjaTEST(nt_styled.render()) if environ.get('RUN_MODE') == 'Production': message = Mail( from_email=os.getenv('SENDGRID_MAIL'), to_emails=os.getenv('SENDGRID_MAIL'), subject='New Transaction Report', html_content=html_data) try: sg = SendGridAPIClient(os.environ.get('SENDGRID_API_KEY')) response = sg.send(message)