def render_csv_file(file_path, batch_name, data_alt_dict=None, output_path='./output/',
map_origin_ranges=None, map_tarjet_ranges=None):
u"""
Genera un lote de flames a partir de los valores de un archivo csv.
Existe la posibilidad de mapear la información de entrada de un rango de valores
a otro mediante los parámetros map_data, map_origin_ranges y map_tarjet_ranges.
Consultar función map_data en utils.py para más información.
:param file_path: ruta a archivo csv
:param batch_name: cadena
:param data_alt_dict: diccionario con var_names y param_names alternativos
:param output_path: directorio de salida
:param map_origin_ranges: lista de tuplas con rangos de origen de cada parámametro
:param map_tarjet_ranges: lista de tuplas con rangos de destino de cada parámametro
:return: éxito
"""
if data_alt_dict:
data_dict = data_alt_dict
else:
data_dict = {
'var_names': ['noise', 'spherical', 'sinusoidal'],
'param_names': [u'Grado Alcohólico', u'Acidez Total', u'Sulfuroso total'],
}
if map_origin_ranges:
data_dict['var_values'] = map_data(read_csv_data(file_path),
map_origin_ranges, map_tarjet_ranges)
else:
data_dict['var_values'] = read_csv_data(file_path)
create_batch(batch_name, data_dict, output_path=output_path)
def create_image(csv_path):
csv_data, _, _ = read_csv_data(csv_path)
plt.figure()
plt.plot(csv_data)
plt.axis('off')
plt.savefig('wind_data.png', bbox_inches='tight', dpi=500)
def get_redis(self, date=None):
"""
This function stores the data into redis instance
after reading and storing the csv data
"""
if date:
date = datetime.datetime.strptime(date, '%d-%m-%Y').date()
else:
date = datetime.date.today() - datetime.timedelta(days=2)
fdate = convert_date_to_bseurl_fmt(date)
bse_zip_url = get_bse_zip_url_for_fdate(fdate)
csv_file_url = read_zip_file(bse_zip_url, fdate, date)
data = read_csv_data(csv_file_url)
self.store_data(data, date)
def main():
print('hello')
size_tuple = (3, 2)
print(utils.random_array(size_tuple))
csv_path = 'MOCK_DATA.csv'
csv_path = resource_path(csv_path)
print("Reading data from %s" % csv_path)
print(utils.read_csv_data(csv_path)[:5])
def main():
print('hello')
size_tuple = (3, 2)
print(utils.random_array(size_tuple))
csv_files = ('MOCK_DATA.csv', 'MOCK_DATA2.csv')
csv_paths = [os.path.join('data', X) for X in csv_files]
csv_paths = [resource_path(X) for X in csv_paths]
for csv_path in csv_paths:
print("Reading data from %s" % csv_path)
print(utils.read_csv_data(csv_path)[:5])
def clean_caps_df(csv_data, present_vid_ids, present_vid_ids_csv):
vid_list = list(set([get_vid_ids(s) for s in present_vid_ids]))
assert len(vid_list) == len(present_vid_ids_csv)
df = csv_data.loc[((csv_data['VideoID'].isin(vid_list))
& (csv_data['Language'] == 'English'))
& csv_data['Description'].notnull()]
df.to_csv(config.MSVD_FINAL_CORPUS_PATH, index=False, encoding='utf-8')
df = utils.read_csv_data(config.MSVD_FINAL_CORPUS_PATH)
omitted_caps = []
punct_dict = get_punctuations()
translator = string.maketrans("", "")
df['Description'] = df.apply(lambda row: clean_caps(
row['Description'], punct_dict, translator, omitted_caps),
axis=1)
df = df.loc[df['Description'].notnull()]
df.to_csv(config.MSVD_FINAL_CORPUS_PATH, index=False, encoding='utf-8')
print("Non-ASCII captions omitted :" + str(len(omitted_caps)))
utils.write_list_to_file(config.MSVD_OMMITTED_CAPS_PATH, omitted_caps)
return df
def preproc_csv(fname, outfname):
csv_data = utils.read_csv_data(fname)
filtered_csv = csv_data.loc[csv_data['VideoID'].notnull()
& csv_data['Description'].notnull()]
filtered_csv.to_csv(outfname, index=False, encoding='utf-8')
for vid_caps in vid_caps_dict.items():
vid_id = vid_caps[0]
if vid_id[-4:] == ".avi":
vid_id = vid_id[:-4]
for seq_id in range(len(vid_caps[1])):
data_id = vid_id + "|" + str(seq_id)
data_ids.append(data_id)
utils.write_list_to_file(ids_save_path, data_ids)
if __name__ == '__main__':
has_ids_list = True
print("removing empty lines in original corpus...")
preproc_csv(config.MSVD_CSV_DATA_PATH, config.MSVD_PREPROC_CSV_DATA_PATH)
print("loading proccessed corpus...")
csv_data = utils.read_csv_data(config.MSVD_PREPROC_CSV_DATA_PATH)
print("reading video clips ids...")
if not has_ids_list:
vid_ids_list = utils.read_dir_files(
config.MSVD_VIDEO_DATA_PATH
) # read dataset vid ids from video clips directory
else:
vid_ids_list = utils.read_file_to_list(
config.DATA_DIR +
"present_vid_ids.txt") # read dataset vid ids from text file
assert len(vid_ids_list) == config.TOTAL_VIDS
print("filtering clips in df...")
present_vid_ids, missing_vid_ids, present_vid_ids_csv = filter_clips(
csv_data, vid_ids_list)
assert len(present_vid_ids) == config.TOTAL_VIDS
print("saving filtered df...")
x = x[1:]
y = y[1:]
ax.xaxis.set_major_formatter(mtick.FormatStrFormatter('%.0e'))
coeffs = np.polyfit(np.log(x), np.log(y), 1)
beta = -coeffs[0]
dimension = 1 + (3 - beta) / 2
print(beta)
print("The fractal dimension is", dimension)
plt.subplot(133)
plt.title("the Curve of log(power-spectrum) and log(frequency)")
plt.scatter(np.log(x), np.log(y), marker='o', s=10, c=list(range(len(x))))
# plt.plot(np.log(x), np.log(y), 'o', mfc='none')
plt.plot(np.log(x), np.polyval(coeffs, np.log(x)))
plt.xlabel('log freq')
plt.ylabel('log intensity')
plt.savefig("../pics/kyoto_wind.png")
plt.show()
if __name__ == '__main__':
# data of kyoto
filename = '../Kyoto_wind.csv'
wind_list, time_list, data_len = read_csv_data(filename=filename)
print(hfd.hfd(wind_list))
draw_spectrum(wind_list)
create_image(filename)
box_count()
def members():
return render_template('members.html',\
menus=MENUS,
members=read_json_data('members.json'),
alumni_phd=read_csv_data('alumni_phd.csv'),
alumni_ms=read_csv_data('alumni_ms.tsv', sep='\t'))
def members():
return render_template('members.html',\
menus=MENUS,
members=read_json_data('members.json'),
alumni_phd=read_csv_data('alumni_phd.csv'),
alumni_ms=read_csv_data('alumni_ms.tsv', sep='\t'))