def main():
Queues = hlp.getData()
for queue in Queues:
print(queue)
print("Customer")
print(genCustomer(Queues[queue]))
print("")
def threadedSS(connection: socket.socket):
length = getLength(connection)
allData = recv_all(connection, length)
allData = pickle.loads(allData)
needToVisit = allData['needToVisit']
url = allData['url']
print("Request: ", url)
if len(needToVisit) == 0:
# last stepping stone so get the file
print("chainlist is empty")
dataToSendBack = getData(allData['url'])
dataToSendBack = pickle.dumps(dataToSendBack)
sendDataHeader(connection, len(dataToSendBack))
print("Relaying file ...")
connection.sendall(dataToSendBack)
else:
# get the next stepping stone
chainfile = needToVisit
addr, port = getAddrNextSteppingStone(chainfile)
allData = pickle.dumps(allData)
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((addr, port))
sendDataHeader(s, len(allData))
s.sendall(allData)
length = getLength(s)
print("waiting for file ...")
data = recv_all(s, length)
sendDataHeader(connection, len(data))
print("Relaying file ...")
connection.sendall(data)
connection.close()
print("Goodbye!")
def main():
Queues = hlp.getData()
for queue in Queues:
print(queue)
print("Service Rate")
cardOrCashLiabil = hlp.TimeDifference(Queues[queue])
print serviceRateCard(Queues[queue][5], Queues[queue][7],
cardOrCashLiabil)
def main():
Queues = hlp.getData()
for queue in Queues:
print(queue)
print("Droput")
dropProbs = get_dropout_probability_ranges(Queues[queue])
print(dropout(Queues[queue]))
print(bernoulli_drop_out(dropProbs, 50))
print("")
def main():
#items = 30 #30 items infront of customer
Queues = hlp.getData()
for queue in Queues:
print(queue)
print("Arrival Rate")
#rates = rateItems(Queues[queue])
#print(genArrivalRate(rates,queue))
print("")
def getUniqueRestraunts(self): df = helpers.getData() restaurants = Series(np.unique(df["Restaurant"].map(lambda x: x.strip()))) for ind in range(len(restaurants)): restaurants[ind] = restaurants[ind].title() restaurants = restaurants.sort_values() del df print(restaurants) return restaurants
def createPredictions(self, parameters, companies): bigDF = helpers.getData() day = parameters[0] rest = parameters[1] companies = companies predictions = appPredictions.getPredictions(bigDF, day, rest, companies) predList = [] for i in predictions: predList.append(i) del predictions return predList
def run(rank, world_size):
os.environ["MASTER_ADDR"] = "localhost"
os.environ["MASTER_PORT"] = "10638"
dist_init(rank, world_size)
os.environ["MASTER_PORT"] = "10639"
dist.rpc.init_rpc(f"worker{rank}", rank=rank, world_size=world_size)
initialize_model_parallel(1, world_size)
model = getModel()
data, target = getData()[0]
loss_fn = getLossFun()
device = torch.device("cuda",
rank) if DEVICE == "cuda" else torch.device("cpu")
model = fairscale.nn.Pipe(
model,
balance=[2, 1],
style=fairscale.nn.Pipe.MultiProcess,
worker_map={
0: "worker0",
1: "worker1"
}, # Needed to convert ranks to RPC worker names
input_device=device,
).to(device)
# define optimizer and loss function
optimizer = optim.SGD(model.parameters(), lr=0.001)
# zero the parameter gradients
optimizer.zero_grad()
# outputs and target need to be on the same device
# forward step
outputs = model(data.to(device))
# compute loss
if rank == 1:
loss = loss_fn(outputs.to(device), target.to(device))
# backward + optimize
loss.backward()
optimizer.step()
else:
model.back_helper(outputs)
print(f"Finished Training Step on {rank}")
del model
def run(rank, world_size):
torch_pg.init_mpi()
os.environ["MASTER_ADDR"] = "localhost"
os.environ["MASTER_PORT"] = "10638"
dist_init(rank, world_size) # FIXME (supports gloo)
os.environ["MASTER_PORT"] = "10639"
torch.distributed.rpc.init_rpc(f"worker{rank}",
rank=rank,
world_size=world_size)
initialize_model_parallel(1, world_size, pipeline_backend="mpi")
if rank == 1:
# For RPC, all ranks other than 0 just need to call rpc.shutdown()
torch.distributed.rpc.shutdown()
return
model = getModel()
data, target = getData()[0]
loss_fn = getLossFun()
device = torch.device("cuda", rank)
model = fairscale.nn.PipeRPCWrapper(
model,
balance=[2, 1],
worker_map={
0: "worker0",
1: "worker1"
}, # Needed to convert ranks to RPC worker names
input_device=device,
).to(device)
# We can't directly access the model on each worker, so we need to call
# foreach_worker with a callback to setup the optimizer
model.foreach_worker(register_optimizer, {"lr": 0.001}, include_self=True)
outputs = model(data.to(device))
loss = loss_fn(outputs.to(device), target.to(device))
loss.backward()
# Same as earlier, use foreach_worker to step the optimizer on each rank
model.foreach_worker(run_optimizer, include_self=True)
print(f"Finished Training Step on {rank}")
torch.distributed.rpc.shutdown()
del model
def city(city):
weather_data = getData(city)
weather = {
"city": city,
"temp": round((weather_data["main"]["temp"] - 273.15), 1),
"temp_feels": round((weather_data["main"]["feels_like"] - 273.15), 1),
"temp_min": round((weather_data["main"]["temp_min"] - 273.15), 1),
"temp_max": round((weather_data["main"]["temp_max"] - 273.15), 1),
"pressure": weather_data["main"]["pressure"],
"humidity": weather_data["main"]["humidity"],
"wind": weather_data["wind"]["speed"],
"cloudiness": weather_data["clouds"]["all"],
"desc": weather_data["weather"][0]["main"],
"icon": weather_data["weather"][0]["icon"]
}
return render_template("city.html", weather=weather)
def main():
# adding new city to the database
if request.method == "POST":
city = request.form.get("name")
if city:
city_to_add = Weather(name=city)
if Weather.query.filter_by(name=city).first() is None:
db.session.add(city_to_add)
db.session.commit()
# getting all cities
cities = Weather.query.all()
print("cities : ")
print(cities)
# if there are more than cities, delete the first one
if len(cities) > 5:
to_delete = Weather.query.first()
db.session.delete(to_delete)
db.session.commit()
cities.pop(0)
all_weather_data = []
alert = None
for city in reversed(cities):
weather_data = getData(city.name)
if (weather_data["cod"] == '404'):
alert = city.name
to_delete = Weather.query.filter_by(name=city.name).first()
db.session.delete(to_delete)
db.session.commit()
else:
weather_to_add = {
"city": weather_data["name"],
"temp": round((weather_data["main"]["temp"] - 273.15), 1),
"text": weather_data["weather"][0]["description"],
"icon": weather_data["weather"][0]["icon"]
}
all_weather_data.append(weather_to_add)
print(weather_data)
return render_template("main.html",
alert=alert,
weather_list=all_weather_data)
from helpers import getData, getLossFun, getModel
import torch
import torch.optim as optim
import fairscale
DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
RANK = 0 # example
model = getModel()
data, target = getData()[0]
loss_fn = getLossFun()
model = fairscale.nn.Pipe(model, balance=[2, 1])
# define optimizer and loss function
optimizer = optim.SGD(model.parameters(), lr=0.001)
# zero the parameter gradients
optimizer.zero_grad()
device = torch.device("cuda",
RANK) if DEVICE == "cuda" else torch.device("cpu")
# outputs and target need to be on the same device
# forward step
outputs = model(data.to(device).requires_grad_())
# compute loss
loss = loss_fn(outputs.to(device), target.to(device))
# backward + optimize
secondInopts=opt2)
else:
self.assertFileEqual(val,
key.format(**outfiles),
firstInopts=opt1,
secondInopts=opt2)
def __init__(self, *args, **kwargs):
testly.TestCase.__init__(self, *args, **kwargs)
members = {'__init__': __init__}
proc = datarows[0].kwargs['proc'][len(klass) + 1:]
proc = 'test' + proc
dataprovider = 'dataProvider_' + proc
members[dataprovider] = dataprovider_maker
members[proc] = testit
globals()['Test' + klass[0].upper() + klass[1:]] = type(
klass, (testly.TestCase, ), members)
DATAROWS = defaultdict(lambda: [])
for data in getData():
klass = data.kwargs['proc'].split('.')[0]
DATAROWS[klass].append(data)
for key, val in DATAROWS.items():
TestClassFactory(key, val)
if __name__ == '__main__':
testly.main(verbosity=2)
from keras.models import Sequential
from keras.optimizers import Adam
from keras.utils import plot_model
import helpers
from helpers import INPUT_SHAPE
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.333)
sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options))
dataset_dir_name, checkpoint_dir_name, stats_dir_name, model_name = helpers.getInput(
)
data = helpers.calibrate(dataset_dir_name)
training_data, testing_data = helpers.getData(data, 75)
trainingImages = helpers.getMatrix(training_data)
testingImages = helpers.getMatrix(testing_data)
print("_" * 40)
print("Image translation to matrix completed")
print("_" * 40)
now = datetime.now()
date_time = now.strftime("%m/%d/%Y, %H:%M:%S")
# NAME = f"{model_name}_{date_time}"
tensorboard = TensorBoard(log_dir=f'logs/{model_name}')
def build_model():
def train(rank: int, world_size: int, epochs: int, use_oss: bool):
# DDP
dist_init(rank, world_size)
rank = torch.device("cpu") if DEVICE == "cpu" else rank
# Problem statement
model = getModel().to(rank)
dataloader = getData(n_batches=1)
loss_fn = getLossFun()
optimizer: Optional[Union[OSS, torch.optim.SGD]] = None
if not use_oss:
optimizer = torch.optim.SGD(params=model.parameters(), lr=1e-4)
else:
base_optimizer = torch.optim.SGD
base_optimizer_arguments = {
"lr": 1e-4
} # any optimizer specific arguments, LR, momentum, etc...
optimizer = OSS(params=model.parameters(),
optim=base_optimizer,
**base_optimizer_arguments)
training_start = time.monotonic()
# Any relevant training loop, nothing specific to OSS. For example:
model.train()
for _ in range(epochs):
for (data, target) in dataloader:
data, target = data.to(rank), target.to(rank)
# Train
model.zero_grad()
outputs = model(data)
loss = loss_fn(outputs, target)
loss.backward()
# if you want to clip the gradients / get the current max:
max_norm = 1000.0
norm_type = 1
if not use_oss:
_total_norm = torch.nn.utils.clip_grad_norm_(
model.parameters(), max_norm,
norm_type=norm_type) # type: ignore
else:
optimizer = cast(OSS, optimizer)
_total_norm = optimizer.clip_grad_norm(max_norm,
norm_type=norm_type)
optimizer.step()
print(f"Loss: {loss.item()}")
training_end = time.monotonic()
print(
f"[{dist.get_rank()}] : Training done. {training_end-training_start:.2f} sec"
)
if DEVICE == "cuda":
max_memory = torch.cuda.max_memory_allocated(rank)
print(f"[{dist.get_rank()}] : Peak memory {max_memory:.1f}MiB")
chart_files = [x.strip(' ') for x in chart_files]
hawk_mode = str(os.getenv('HAWK_MODE'))
print('Currently Running in {}'.format(hawk_mode))
tokens = helpers.plaidTokens()
start_of_month = helpers.monthStart()
chart_studio.tools.set_credentials_file(username=os.getenv('PLOTLY_USERNAME'),
api_key=os.getenv('PLOTLY_API_KEY'))
clear_file = helpers.clear_data_file()
print(clear_file)
encoded = base64.b64encode(open('templates/logo.png', "rb").read())
#Data from Plaid
master_data = helpers.getData(hawk_mode, exclusions)
data = dict(date=date.today().strftime('%m/%d/%y'),
balance_chase=master_data['balance_chase'],
chase_total=master_data['chase_total'],
balance_schwab=master_data['balance_schwab'],
schwab_total=master_data['schwab_total'],
balance_cap_one=master_data['cap1_balance'],
capone_total=master_data['cap1_total'],
balance_great_lakes=master_data['lakes_balance'],
greatlakes_total=master_data['lakes_total'],
chart_pack={},
logo='data:image/png;base64,{}'.format(encoded.decode('utf8')),
tables=[])
chart_links = []
#Update Guage Chart
from os import environ
from dotenv import load_dotenv
from sendgrid.helpers.mail import Mail
from sendgrid import SendGridAPIClient
import python_http_client
import helpers
load_dotenv()
# keep_list = environ.get('KEEP_LIST').split(',')
# keep_list = [x.strip(' ') for x in keep_list]
new_transactions = helpers.frame_prep(helpers.getData(), environ.get('KEEP_LIST'))
helpers.addTransaction(new_transactions)
nt_styled = new_transactions.style.set_caption('New Transactions').set_table_styles(helpers.tableStyles()).set_table_attributes('border="1" class="dataframe table table-hover table-bordered"')
html_data = helpers.jinjaTEST(nt_styled.render())
if environ.get('RUN_MODE') == 'Production':
message = Mail(
from_email=os.getenv('SENDGRID_MAIL'),
to_emails=os.getenv('SENDGRID_MAIL'),
subject='New Transaction Report',
html_content=html_data)
try:
sg = SendGridAPIClient(os.environ.get('SENDGRID_API_KEY'))
response = sg.send(message)