def create_list_datetimes(self, start_date, end_date, nhours):
"""
Create a list of timesteps between start_date and end_date with a
time interval of nhours.
Add timesteps to database
"""
from datetime import datetime
from datetime import timedelta
if not (isinstance(start_date, datetime) and isinstance(end_date, datetime)):
message = (
"input start_date and end_date of function ",
"create_list_datetimes should be datetime.datetime objects",
)
logger.error(message)
raise IOError(message)
if not (isinstance(nhours, int)):
message = ("nhours input argement of create_list_datetimes should be ", "an integer number of hours")
logger.error(message)
raise IOError(message)
# check if the frequency between boundary conditions is not larger than the
# period between start and end date of the simulation
if (nhours * 3600) > (end_date - start_date).total_seconds():
message = (
"Interval of boundary conditions larger than timedelta ",
"between start_date and end_date in function ",
"database.create_list_datetimes",
)
logger.error(message)
raise IOError(message)
timesteps = utils.datetime_range(start_date, end_date, {"hours": nhours})
for idx, timestep in enumerate(timesteps):
self._add_timestep_to_db(timestep)
def create_list_datetimes(self, start_date, end_date, nhours):
'''
Create a list of timesteps between start_date and end_date with a
time interval of nhours.
Add timesteps to database
'''
from datetime import datetime
from datetime import timedelta
if not(isinstance(start_date, datetime) and isinstance(end_date, datetime)):
message = ('input start_date and end_date of function ',
'create_list_datetimes should be datetime.datetime objects')
logger.error(message)
raise IOError(message)
if not(isinstance(nhours, int)):
message = ('nhours input argement of create_list_datetimes should be ',
'an integer number of hours')
logger.error(message)
raise IOError(message)
# check if the frequency between boundary conditions is not larger than the
# period between start and end date of the simulation
if ((nhours*3600) > (end_date - start_date).total_seconds()):
message = ('Interval of boundary conditions larger than timedelta ',
'between start_date and end_date in function ',
'database.create_list_datetimes')
logger.error(message)
raise IOError(message)
timesteps = utils.datetime_range(start_date, end_date, {'hours':nhours})
for idx,timestep in enumerate(timesteps):
self._add_timestep_to_db(timestep)
def main():
path_file_base = os.path.join('bases', 'idka_base.csv')
# faz o download do csv no site da anbima
url = 'http://www.anbima.com.br/informacoes/idka/IDkA-down.asp'
name_download_folder = 'idka'
path_download = utils.prepare_download_folder(name_download_folder)
# verifica a última data disponível na base
today = datetime.now().date()
cal = utils.get_calendar()
ultima_data_base = cal.offset(today, -6)
dates_range = list(utils.datetime_range(start=ultima_data_base, end=today))
for dt_referencia in reversed(dates_range):
path_file = os.path.join(
path_download,
dt_referencia.strftime('%Y%m%d') + '_idka.csv')
download_file(url, dt_referencia, path_file)
utils.remove_zero_files(name_download_folder)
import_files(name_download_folder, path_file_base, ultima_data_base)
# organizar o arquivo base por dt_referencia
utils.generate_csv_base(path_file_base)
print("Arquivos baixados com sucesso e importados para a base de dados")
def main():
# apaga arquivos antigos
remove_old_files()
# verifica a última data disponível na base
name_file_base = 'debentures_base.csv'
path_file_base = os.path.join('bases', name_file_base)
# verifica a última data disponível na base
today = datetime.now().date()
cal = utils.get_calendar()
ultima_data_base = cal.offset(today, -6)
dates_range = list(utils.datetime_range(start=ultima_data_base, end=today))
# faz o download do csv no site da anbima
url = 'https://www.anbima.com.br/informacoes/merc-sec-debentures/arqs/db'
for dt_referencia in reversed(dates_range):
path_download = os.path.join('downloads')
if not os.path.exists(path_download):
os.makedirs(path_download)
path_download = os.path.join('downloads', 'debentures')
if not os.path.exists(path_download):
os.makedirs(path_download)
file_path = os.path.join(path_download,
dt_referencia.strftime('%y%m%d') + '.txt')
# faz o download do arquivo caso ele ainda não tiver sido baixado
if not os.path.exists(file_path):
download_file(url, dt_referencia, file_path)
print("Arquivos baixados com sucesso")
def main():
parser = OptionParser()
parser.add_option('-o',
dest='filename',
default='retweet_timeseries.csv',
help='output filename')
parser.add_option('--clusters',
dest='clusters',
default='data/clusters.json',
help='clusters filename')
parser.add_option('--start',
dest='start',
default='20110301',
help='start date (default=20110301)')
parser.add_option('--stop',
dest='stop',
default='20130401',
help='stop date (default=20130401)')
parser.add_option('--influencers',
dest='n',
type='int',
default=None,
help='maximum number of influencers (default=inf)')
parser.add_option('--mincount',
dest='mincount',
type='int',
default=1,
help='minimum number of retweet (default=1)')
options, args = parser.parse_args()
start_date = ymd_to_datetime(options.start)
stop_date = ymd_to_datetime(options.stop)
cluster_map = load_clusters(options.clusters)
clusters = list(set(cluster_map.values()))
days = [
d.strftime('%Y%m%d%H')
for d in datetime_range(start_date, stop_date, timedelta(hours=1))
]
count = {}
for c in clusters:
count[c] = {}
for d in days:
count[c][d] = 0
tweets = load_tweet(*args)
tweets = filter_by_datetime(tweets, start_date, stop_date)
for ruid, tss in group_by_influencer(tweets, options.n, options.mincount):
if ruid in cluster_map:
cluster = cluster_map[ruid]
for day, ts in group_by_hour(tss):
count[cluster][day] += len(list(ts))
writer = csv.writer(open(options.filename, 'w'))
writer.writerow(['date'] + clusters)
for d in days:
writer.writerow([d] + [count[c][d] for c in clusters])
def fill_gaps(time_begin, time_end, result):
# 如果这一分钟没有值，用上一分钟的值
step = datetime.timedelta(seconds=60)
first_minute = truncate_minute(time_begin)
first_exist = min(result)
last = result[first_exist]
if first_exist > first_minute:
logging.warning("data missing from %s to %s", first_minute,
first_exist)
for t in datetime_range(first_minute, time_end + step, step):
last = result.setdefault(t, last)
def main():
# apaga arquivos antigos
remove_old_files()
# verifica a última data disponível na base
name_file_base = 'ima_quadro_resumo_base.csv'
path_file_base = os.path.join('bases', name_file_base)
# ultima data base dispon[ivel
ultima_data_base = get_ultima_data_disponivel_base(path_file_base)
print('Última data base disponível:', ultima_data_base)
if (ultima_data_base is None):
ultima_data_base = datetime.date(2010, 11, 17)
carteiras = [
'irf-m', 'irf-m 1', 'irf-m 1+', 'ima-b', 'ima-b 5', 'ima-b 5+',
'ima-c', 'ima-s', 'ima-geral', 'ima-geral ex-c'
]
# faz o download do csv no site da anbima
url = 'http://www.anbima.com.br/informacoes/ima/ima-carteira-down.asp'
# verifica a última data disponível na base
today = datetime.now().date()
cal = utils.get_calendar()
ultima_data_base = cal.offset(today, -5)
dates_range = list(utils.datetime_range(start=ultima_data_base, end=today))
for dt_referencia in reversed(dates_range):
for carteira in carteiras:
path_download = os.path.join('downloads', carteira)
if not os.path.exists(path_download):
os.makedirs(path_download)
file_name = os.path.join(
path_download,
dt_referencia.strftime('%Y%m%d') + '_' + carteira + '.csv')
if utils.check_download(dt_referencia, file_name) is False:
break
# faz o download do arquivo caso ele ainda não tiver sido baixado
if not os.path.exists(file_name):
download_file_carteira(url, dt_referencia, file_name, carteira)
print("Arquivos baixados com sucesso e importados para a base de dados")
def main():
# apaga arquivos antigos
remove_old_files()
# verifica a última data disponível na base
name_file_base = 'ima_quadro_resumo_base.csv'
path_file_base = os.path.join('bases', name_file_base)
# ultima data base dispon[ivel
ultima_data_base = get_ultima_data_disponivel_base(path_file_base)
print('Última data base disponível:', ultima_data_base)
if (ultima_data_base is None):
ultima_data_base = datetime.date(2010, 11, 17)
# faz o download do csv no site da anbima
# lft
url = 'http://www.anbima.com.br/informacoes/merc-sec/arqs/ms'
# verifica a última data disponível na base
today = datetime.now().date()
cal = utils.get_calendar()
ultima_data_base = cal.offset(today, -5)
dates_range = list(utils.datetime_range(start=ultima_data_base, end=today))
path_download = os.path.join('downloads',
'titulos-publicos-merc-secundario')
if not os.path.exists(path_download):
os.makedirs(path_download)
for dt_referencia in reversed(dates_range):
file_path = os.path.join(
path_download,
dt_referencia.strftime('%Y%m%d') + '_ms_titulos-publicos.txt')
# verifica se o arquivo deve ser baixado
if not utils.check_download(dt_referencia, file_path):
continue
print(file_path)
# faz o download do arquivo caso ele ainda não tiver sido baixado
if not os.path.exists(file_path):
download_file(url, dt_referencia, file_path)
print("Arquivos baixados com sucesso")
def gamelogs():
gamelogs_query = {}
name = request.args.get('name')
if name:
gamelogs_query.update({'Player': name})
start = request.args.get('start')
end = request.args.get('end')
if start:
gamelogs_query.update(datetime_range(start, end))
active = request.args.get('active')
if not active:
gamelogs_list = query.query_games(gamelogs_query)
else:
gamelogs_list = query.query_games(gamelogs_query, active=True)
if gamelogs_list:
for gamelog_dict in gamelogs_list:
gamelog_dict.pop("_id", None)
return Response(json.dumps(gamelogs_list, default=json_util.default),
mimetype='application/json')
print('Fitting')
stepwise_model.fit(dataset)
print('Done!')
return stepwise_model
if __name__ == "__main__":
start_time_stamp = "2020-04-06T00:00:00"
end_time_stamp = "2020-04-21T00:00:00"
start = datetime.strptime(start_time_stamp, STR_FMT)
end = datetime.strptime(end_time_stamp, STR_FMT)
dts = [
dt.strftime(STR_FMT)
for dt in datetime_range(start, end, timedelta(hours=1))
]
df = read_data()
list_of_time_series = []
# df.to_csv('test.csv')
for i in set(df["CAMERA_ID"]):
list_of_time_series.append(df[df["CAMERA_ID"] == i].drop(
columns=["CAMERA_ID", "LAT", "LONG"]))
# list_of_models = []
# predictions = []
# for time_series in tqdm(list_of_time_series):
# model = train_ARIMA(time_series)
# list_of_models.append(model)
# predictions.append(model.predict(24))
# time_stamps = sorted(list(set(df.index)))
from keras_yolo.yolo import YOLO
from utils import datetime_range, get_data, save_image_from_url
STR_FMT = '%Y-%m-%dT%H:%M:%S'
if __name__ == "__main__":
start_time_stamp = "2020-03-18T20:00:00"
end_time_stamp = "2020-03-21T00:00:00"
refresh_rate = 60 * 60
start = datetime.strptime(start_time_stamp, STR_FMT)
end = datetime.strptime(end_time_stamp, STR_FMT)
dts = [
dt.strftime(STR_FMT)
for dt in datetime_range(start, end, timedelta(seconds=refresh_rate))
]
od = YOLO()
# main loop
with open(
os.path.join('results',
start_time_stamp + '_to_' + end_time_stamp + '.json'),
'w') as json_file:
result_dict = {}
for time_stamp in tqdm(dts):
with open(os.path.join('results', time_stamp + '.csv'), 'w') as f:
f.write('CAMERA_ID,LAT,LONG,NUM_OF_VEH\n')
result_dict[time_stamp] = {}
# print('Time stamp: ',time_stamp)
res = get_data(time_stamp)
