def generate_report(task, db, directory):
date = dt.now()
date = date.strftime('%d-%m-%Y_%H-%M')
directory = os.path.join(directory, 'reports')
if not os.path.exists(directory):
os.makedirs(directory)
filename = "%s/%s.txt" %(directory, date)
with open(filename, 'w') as f:
f.write("\n======PROJECT PARAMS======\n")
for k, v in task.items():
if k not in ["action", "status","msg", "date", "creation_date", "_id"]:
if k == "date":
v = task[k].strftime('%d-%m-%Y@%H:%M:%S')
f.write(str(k)+": "+str(v)+"\n")
try:
f.write(str(k)+": "+str(v)+"\n")
except Exception:
pass
f.write(db.export_stats())
f.write("\n\n======HISTORY OF THE PROJECT======\n")
#date_list = [n.strftime('%d-%m-%Y %H-%M-%S') for n in task["date"]]
# status_list = list(zip(task["status"],task["msg"]))
# for msg in status_list:
# f.write("\n-"+str(msg))
logging.info("Your report is ready!\nCheck here: %s" %(filename))
return True
def get_dataloaders(
train_dir,
var_dir,
train_transform=None,
val_transform=None,
split=(0.5, 0.5),
batch_size=32,
*args, **kwargs):
"""
This function returns the train, val and test dataloaders.
"""
# create the datasets
train_ds = ImageFolder(root=train_dir, transform=train_transform)
val_ds = ImageFolder(root=var_dir, transform=val_transform)
# now we want to split the val_ds in validation and test
lengths = np.array(split) * len(val_ds)
lengths = lengths.astype(int)
left = len(val_ds) - lengths.sum()
# we need to add the different due to float approx to int
lengths[-1] += left
val_ds, test_ds = random_split(val_ds, lengths.tolist())
logging.info(f'Train samples={len(train_ds)}, Validation samples={len(val_ds)}, Test samples={len(test_ds)}')
train_dl = DataLoader(train_ds, batch_size=batch_size, shuffle=True, *args, **kwargs)
val_dl = DataLoader(val_ds, batch_size=batch_size, shuffle=False, *args, **kwargs)
test_dl = DataLoader(test_ds, batch_size=batch_size, shuffle=False, *args, **kwargs)
return train_dl, val_dl, test_dl
def write_to_mongo(df, database, collection, incremental_run):
"""
This method writes the data into MongoDB.
If incremental value is 1, then the data is appended to the database.
Else, overwritten.
Parameters:
-----------
df (Dataframe): The dataframe to be written to MongoDB.
database (string): The database in which we are going to write the data.
collection (string): The collection in which we are going to write the data.
incremental_run (int): Determines if data is overwrritten or appended
to the collection.
"""
try:
logging.info('Write to MongoDB in progress')
write_mode = "overwrite"
if incremental_run:
write_mode = "append"
df.write.format("mongo").mode(write_mode).option("database",database).\
option("collection", collection).save()
logging.info('Write to MongoDB completed successfully')
except Exception as e:
logging.error('Error in write_to_mongo() function: {0}'.format(e))
raise e
def buy_positions(api, stock_list, target_positions):
number_of_positions = len(api.list_positions())
positions_to_fill = target_positions - number_of_positions
if number_of_positions < target_positions:
cash_on_hand = float(api.get_account().cash)
potential_stocks_to_buys = stock_list[(stock_list['Buy'] == 'Yes') & (stock_list['Sell'] == '0')]
potential_stocks_to_buy = potential_stocks_to_buys.sort_values(by='50 day slope',ascending=False)
for stock in potential_stocks_to_buy.iterrows():
if positions_to_fill > 0:
if stock[1][10] <= (cash_on_hand/positions_to_fill) and number_of_positions < 5:
qty_to_buy = int((cash_on_hand/positions_to_fill)/(stock[1][10] * 1.05))
logging.info('Trying to buy {qty_to_buy} shares of {sym} stock for {price}'.format(qty_to_buy=qty_to_buy, sym=stock[1][0],price=(stock[1][10] * 1.05)))
make_order(api, 'buy', stock[1][0], qty_to_buy, 'limit', (stock[1][10] * 1.05))
#have to update the stock list so it wont be sold if bought today
stock_list.loc[stock_list['Symbol'] == stock[1][0], 'Sell'] = 'Just Bought'
number_of_positions += 1
positions_to_fill += -1
#needed to wait a little bit so the buy order could complete
time.sleep(10)
#needs to account for any orders already pending
pending_orders = api.list_orders()
cash_pending_orders = 0
for order in pending_orders:
if(order.side=='buy'):
cash_pending_orders += int(order.qty) * float(order.limit_price)
cash_on_hand = float(api.get_account().cash) - cash_pending_orders
return stock_list
def createIndex(esServer="localhost:9200",
indexName="test",
indexType="doctype",
mapping=None,
setting=None,
clear=False):
conn = ES(esServer)
if clear:
deleteIndex(esServer=esServer, indexName=indexName)
logging.info('Create index %s ...', indexName)
try:
if setting:
conn.indices.create_index(indexName, setting)
else:
conn.indices.create_index(indexName, None)
except exceptions.IndexAlreadyExistsException:
logging.warning('Index is already created: %s ...', indexName)
if mapping is not None:
logging.info('Put mapping...')
conn.indices.put_mapping(indexType, mapping, [indexName])
return True
def read_from_cassandra(incremental_run, keyspace, table):
"""
This method reads the data from cassandra based on the incremental_run
value.If incremental value is 1, then the data read is for the running
week and fetches the current day data from it. Else, fetches all the
data from cassandra.
Parameters:
-----------
incremental_run (int): Determines how data is to be read.
keyspace (string): Cassandra keyspace from which data is to be read.
table (string): Cassandra table inside the keyspace from which data is
to be read.
Returns
--------
df (Dataframe): The dataframe obtained after reading from Cassandra
"""
try:
logging.info('Read from_cassandra in progress')
column_names = ["event_time", "user_id"]
if incremental_run:
today = date.today()
next_day = today + timedelta(days=1)
today_starting_timestamp = datetime(today.year, today.month, \
today.day)
next_day_starting_timestamp = datetime(next_day.year, next_day.month,\
next_day.day)
#Set condition to fetch the current day data by pushing down the
#predicate to reduce the number of entries retrived from the database.
incremental_condition = \
(F.col("year") == year) & (F.col("week") == week_num) & \
(F.col("event_time") >= today_starting_timestamp) & \
(F.col("event_time") < next_day_starting_timestamp)
df=spark.read.format("org.apache.spark.sql.cassandra")\
.option("spark.cassandra.connection.port", "9042")\
.option("keyspace", keyspace)\
.option("table", table)\
.load()\
.select(column_names)\
.where(incremental_condition)
else:
df=spark.read.format("org.apache.spark.sql.cassandra")\
.option("spark.cassandra.connection.port", "9042")\
.option("keyspace", keyspace)\
.option("table", table)\
.load()\
.select(column_names)
logging.info('Dataframe loaded successfully')
return df
except Exception as e:
logging.error('Error in read_from_cassandra() function: {0}'.format(e))
raise e
def log_error_and_upload_manifests_to_s3(error, elasticsearch_docs):
logging.error("Exception caught while sending manifests to elasticsearch")
logging.exception(error)
logging.info("Uploading manifests to s3 fallback bucket")
s3_client.put_object(Bucket=MANIFEST_FALLBACK_BUCKET,
Key=os.path.join(f"s3-batch/manifests/{datetime.utcnow().strftime('%Y-%m-%d')}.json.gz"),
Body=gzip.compress(json.dumps(elasticsearch_docs).encode("utf-8")),
ACL="private")
def register():
global tokens_fn
tokens_fn = settings.config['dirs'].user_data_dir + "/twitter.toml"
session_setup()
logging.info("[twitter] writer registered.")
return publish
def main():
logging.info('Starting Up...')
first_of_day_trades(api, df)
schedule.every().day.at("09:32").do(first_of_day_trades, api, df)
schedule.every(10).minutes.do(during_day_check, api, df)
while True:
schedule.run_pending()
time.sleep(1)
def run(path="lineups/*.json"):
lineup_files = glob.glob(path)
number_of_lineups = len(lineup_files)
for idx, lineup_file in enumerate(lineup_files):
logging.info(f"PROCESSING LINEUP {idx+1}/{number_of_lineups}")
with open(lineup_file, "rb") as lf:
for line in lf.readlines():
lineup_data = json.loads(line)
handle_lineupdate(lineup_data)
def update_players(self, match: Match):
try:
white_pl = self.get_player_by_id(match.white_player_id)
black_pl = self.get_player_by_id(match.black_player_id)
white_pl.matches += 1
black_pl.matches += 1
white_pl.last_match_id = match.id
black_pl.last_match_id = match.id
white_pl.last_match_date = match.match_date
black_pl.last_match_date = match.match_date
res = match.result_code
w_score = int()
b_score = int()
if res == "1/2-1/2":
w_score = 0.5
b_score = 0.5
white_pl.draws += 1
black_pl.draws += 1
elif res == "1-0":
w_score = 1
b_score = 0
white_pl.wins += 1
black_pl.loses += 1
else:
w_score = 0
b_score = 1
white_pl.loses += 1
black_pl.wins += 1
white_pl.save(only=[
Player.matches, Player.last_match_id, Player.last_match_date,
Player.wins, Player.loses, Player.draws
])
black_pl.save(only=[
Player.matches, Player.last_match_id, Player.last_match_date,
Player.wins, Player.loses, Player.draws
])
self.update_player_elo(white_pl,
black_pl,
score_p1=w_score,
score_p2=b_score,
guild_id=match.guild_id)
logging.info(
f'[DB] Players:{white_pl.id}, {black_pl.id} has been updated')
return {
"white_player": model_to_dict(white_pl),
"black_player": model_to_dict(black_pl),
"match": match
}
except Exception as e:
logging.exception(
f'[DB] Error while updating players:{white_pl.id}, {black_pl.id} {e}'
)
def deleteIndex(esServer="localhost:9200", indexName="test"):
logging.info('Delete index %s ...', indexName)
conn = ES(esServer)
try:
conn.indices.delete_index(indexName)
return True
except:
logging.info('Cannot delete index %s', indexName)
return False
def exit_logic(current_indicators, current_balances):
# Unpack dictionary:
macd_current = current_indicators['macd_current']
# Calculate exit logic:
if macd_current <= 0:
place_exit_order(current_balances)
logging.info('Exit order placed.')
else:
logging.info('Exit conditons not met. Holding position.')
pass
def save_metadata(self, data_dir):
file_dir = os.path.join(data_dir, 'nea-%s' % self.api)
if not os.path.isdir(file_dir):
os.makedirs(file_dir)
logging.info('Created %s' % file_dir)
filename = '%s-metadata.csv' % (self.api)
self.metadata.to_csv(os.path.join(file_dir, filename),
header=False,
index=False)
def read_file(fname):
def read_it(encoding):
with open(fname, 'r', encoding=encoding) as f:
return f.read().splitlines()
try:
logging.info('Reading {}'.format(fname))
return read_it('utf-8')
except UnicodeDecodeError:
logging.debug('(error while reading file, trying latin-1)')
return read_it('latin-1')
def openIndex(esServer="http://localhost:9200", indexName="test"):
esIndex = esServer + '/' + indexName
logging.info('Open index %s ...', esIndex)
response = requests.post(esIndex + '/_open').text
logging.info(response)
if isOk(response):
return True
else:
return False
def main():
logging.info('Starting script...')
schedule.every().monday.at("09:30").do(daily_trading, symbols)
schedule.every().tuesday.at("09:30").do(daily_trading, symbols)
schedule.every().wednesday.at("09:30").do(daily_trading, symbols)
schedule.every().thursday.at("09:30").do(daily_trading, symbols)
schedule.every().friday.at("09:30").do(daily_trading, symbols)
schedule.every(5).minutes.do(during_day_check)
while True:
schedule.run_pending()
time.sleep(1)
def read_from_cassandra(incremental_run, keyspace, table):
"""
This method reads the data from cassandra based on the incremental_run value.
If incremental value is 1, then the data read is for the running week and fetches
the current day data from it. Else, fetches all the data from cassandra.
Parameters:
-----------
incremental_run (int): Determines how data is to be read.
keyspace (string): Cassandra keyspace from which data is to be read.
table (string): Cassandra table inside the keyspace from which data is to be read.
Returns
--------
df (Dataframe): The dataframe obtained after reading from Cassandra
"""
try:
logging.info('Read from_cassandra in progress')
column_names = ["event_time", "user_id"]
if incremental_run:
#Get current week number
today_date = datetime.date.today()
year, week_num, day_of_week = today_date.isocalendar()
#Set condition to fetch the current week data by pushing down the predicate to reduce the number of entries
#retrived from the database.
incremental_condition = (F.col("year") == year) & (F.col("week")
== week_num)
#Get the current week data from cassandra
df=spark.read.format("org.apache.spark.sql.cassandra")\
.option("keyspace", keyspace)\
.option("table", table)\
.load()\
.select(column_names)\
.where(incremental_condition)
#Filter and fetch the current day's data
df = df.filter(day(df.event_time) == today_date.day)
else:
#Reds the entire data from the cassandra
df=spark.read.format("org.apache.spark.sql.cassandra")\
.option("spark.cassandra.connection.port", "9042").option("keyspace", keyspace)\
.option("table", table)\
.load()\
.select(column_names)
logging.info('Dataframe loaded successfully')
return df
except Exception as e:
logging.error('Error in read_from_cassandra() function: {0}'.format(e))
raise e
def main():
logging.info('Starting Up...')
schedule.every().day.at("09:30").do(first_of_day_trades, api, df)
schedule.every().day.at("10:05").do(check_for_buys, api, df)
schedule.every().day.at("15:30").do(end_of_day_trades, api, df)
schedule.every(5).minutes.do(during_day_check, api, df)
while True:
schedule.run_pending()
time.sleep(1)
def save_report_s3(df):
conn = boto.connect_s3(AWSAccessKeyId, AWSSecretKey)
bucket = conn.get_bucket('algotradingreports')
todays_date = str(pd.Timestamp.today())[0:10]
string_df = df.to_csv(None)
file_df = bucket.new_key(
'reports/{today}_10avg_report.csv'.format(today=todays_date))
file_df.set_contents_from_string(string_df)
logging.info(
'{today} report saved to reports s3 bucket'.format(today=todays_date))
def place_landmarks(self, img, count) :
logging.info("place landmarks on image "+str(count))
rects = self.face_detection_dnn(img)
if len(rects) == 1 :
marks = self.detect_landmarks_cnn(img,rects[0])
return marks.ravel()
elif len(rects) >= 1:
logging.info("Detect more than 1 face on img number " +str(count) + " get default first face detect")
marks = self.detect_landmarks_cnn(img,rects[0])
return marks.ravel()
else :
return []
def save_data(self, data_dir):
file_dir = os.path.join(data_dir, 'nea-%s' % self.api)
if not os.path.isdir(file_dir):
os.makedirs(file_dir)
logging.info('Created %s' % file_dir)
filename = '%s-%s_%s.csv' % (self.start_date.strftime('%Y-%m-%d'),
self.end_date.strftime('%Y-%m-%d'),
self.api)
self.records.to_csv(os.path.join(file_dir, filename),
header=False,
index=False)
def run_driver_registration_demo(cls) -> None:
bangalore = [city for city in CITIES if city.name == 'Bangalore'][0]
address = AddressFactory.createAddress(bangalore)
rohit = UserRegistrationService.registerDriver(
'Rohit Singh',
address,
'*****@*****.**',
'8343923201',
'ABC434232',
3,
)
logging.info(f"Driver {rohit} successfully registered")
def read_file(fname):
def read_it(encoding):
with open(fname, 'r', encoding=encoding) as f:
return f.read().splitlines()
try:
logging.info('Reading {}'.format(fname))
return read_it('utf-8')
except UnicodeDecodeError:
logging.debug('(error while reading file, trying latin-1)')
return read_it('latin-1')
def start(self):
self.is_scraping = True
logging.info("[Worker] Starting worker")
while self.is_scraping:
work_start_time = time.time()
try:
self.reddit_scraper.scrape()
self.database.update_tables()
except Exception as e:
logging.exception(f"[Worker] Exception while scraping : {e}")
pass
logging.info(f"[Worker] Work done in {'{0:.3f}'.format(time.time()-work_start_time)}s")
time.sleep(self.scrape_interval)
def main(conn):
fifa_ids = pd.read_csv("fifa_ids.csv")["fifa_id"].values
logging.info(f"Inserting {len(fifa_ids)} players")
for fifa_id in fifa_ids:
player_obj = {}
player_df = get_latest_by_fifa_id(fifa_id, conn=conn)
if player_df.shape[0] == 0:
logging.warning(f"No player found for fifa id {fifa_id}")
player_obj["fifa_name"] = player_df["name"].item()
player_obj["fifa_id"] = fifa_id
insert(conn, **player_obj)
conn.commit()
def load_faction(self):
if self.PATH.exists():
self.parser.read(self.PATH)
self.name = self.parser.get('faction_meta', 'name')
self.faction_id = self.parser.get('faction_meta', 'faction_id')
self.points = self.parser.get('faction_meta', 'points')
#need logic to parse unit info here
else:
logging.info("Faction %s doesn't exist", self.faction_name)
logging.info("Creating Faction %s", self.faction_name)
self.create_faction()
def exit_logic(self, seconds_toCancel=60):
'''Cancel our outstanding stop-loss order and close our open position - hopefully for a large return :)'''
# Cancel stop loss order:
try:
stopLoss_id = self.api.get_open_orders().index[0]
self.api.cancel_open_order(stopLoss_id)
except:
logging.info('No stop loss order to cancel.')
# Create sell order:
sell_order = self.con.query_private(
'AddOrder', {
'pair': self.pair,
'type': 'sell',
'ordertype': 'limit',
'price': self.current_price,
'volume': self.crypto_on_hand,
'expiretm': f'+{seconds_toCancel}'
})
if len(sell_order['error']) == 0:
logging.info(
f'Placed order to sell {self.crypto_on_hand} shares at {self.current_price}.'
)
# Wait for it to fill or expire:
logging.info('Waiting for order to fill...')
while len(self.api.get_open_orders()) > 0:
time.sleep(3)
completed_order = self.api.get_closed_orders()[0].loc[
sell_order['result']['txid'][0]]
return sell_order, completed_order
else:
logging.info(f"Sell order error: {sell_order['error'][0]}.")
def episodes_json_to_html(self):
read_path = os.path.join(self.cache_directory(), 'episodes.json')
with open(read_path, 'r') as fp:
ep_json = json.loads(fp.read())
try:
html_episodes = ep_json['html']
except Exception as err:
logging.info('ep_json=%s', ep_json)
raise err
path = self.anime_html_filepath()
with open(path, 'w') as html_text:
html_text.write(html_episodes)
def check_if_running(dir_name):
to_process = False
currently_detecting = None
if os.path.isfile(log_path):
try:
with open(log_path, 'r') as f:
currently_detecting = json.load(f)
except json.decoder.JSONDecodeError:
logging.warning('Error loading currently detecting json')
pass
else:
logging.info('Loaded current runs')
if currently_detecting is not None:
if dir_name in currently_detecting:
start = datetime.strptime(currently_detecting[dir_name],
'%Y-%m-%d %H:%M:%S %z')
if (datetime.now(tz=timezone('Singapore')) -
start).total_seconds() / 60 > 60:
logging.info(
'Last run for %s started over 60 minutes ago, starting parallel run'
% dir_name)
to_process = True
currently_detecting[dir_name] = datetime.now(
tz=timezone('Singapore')).strftime('%Y-%m-%d %H:%M:%S %z')
else:
logging.info(
'Last run for %s within 60 mins still running, skipping for now'
% dir_name)
else:
logging.info('No current runs for %s detected' % dir_name)
to_process = True
currently_detecting[dir_name] = datetime.now(
tz=timezone('Singapore')).strftime('%Y-%m-%d %H:%M:%S %z')
else:
logging.info('No current run logs found, generating from scratch')
to_process = True
currently_detecting = {
dir_name:
datetime.now(
tz=timezone('Singapore')).strftime('%Y-%m-%d %H:%M:%S %z')
}
if to_process:
with open(log_path, 'w') as f:
json.dump(currently_detecting, f)
return to_process
def compile_detections(aggregated_df, dir_name):
metadata_path = os.path.join(metadata_dir, '%s_images.csv' % dir_name)
metadata_df = pd.read_csv(metadata_path,
names=[
'CameraID', 'Latitude', 'Longitude', 'Date',
'Time', 'Filename', 'Dimensions'
]).drop_duplicates()
combined = metadata_df.merge(aggregated_df,
left_on='Filename',
right_on='image',
how='left').drop(columns=['image'])
combined['Detections'].fillna(0, inplace=True)
logging.info('Combined DF shape: %s' % str(combined.shape))
if not os.path.isdir(aggregated_dir):
os.makedirs(aggregated_dir)
logging.info('Created %s' % aggregated_dir)
aggregated_path = os.path.join(aggregated_dir,
'%s_aggregated.csv.xz' % dir_name)
combined.to_csv(aggregated_path,
index=False,
header=False,
compression='xz')
logging.info('Saved aggregated data to %s' % aggregated_path)
upload_blob(bucket, aggregated_path,
'traffic-images-aggregated/%s_aggregated.csv.xz' % dir_name)
os.remove(aggregated_path)
logging.info('Deleted %s' % aggregated_path)
def generate_table(api):
data, records_count = datamall_load(apis[api]['url'])
logging.info('%s records found' % records_count)
compiled = []
for record_set in data:
for timestamp in record_set:
timestamp_abbr = timestamp.split(' ')[-1]
for record in record_set[timestamp]:
row = [timestamp_abbr]
for field in apis[api]['fields']:
row.append(record[field])
compiled.append(row)
return pd.DataFrame(compiled)
stream = StringIO()
writer = csv.writer(stream,
delimiter='\t',
escapechar='\\',
quotechar=None,
doublequote=False)
for clear_text_password in l:
t = (clear_text_password,)
writer.writerow(t)
stream.seek(0)
if args.keep:
dname = 'DEBUG_{}.csv'.format(os.path.basename(fname))
logging.debug('Dumping values to {}'.format(dname))
with open(dname, 'w', encoding='utf-8') as f:
f.write(stream.getvalue())
logging.debug('Importing queued passwords')
cur.copy_from(file=stream,
table='rainbows',
sep='\t',
columns=('clear_text_password',))
conn.commit()
logging.info('Added {} passwords'.format(len(l)))
cur.execute('deallocate all')
cur.close()
conn.close()
