def main():
global limit_step
global complex_messages
# configs
config = LoadConfigs('config.ini')
template_paths = GetConfigParameter(config, 'Templates', 'paths').split(',')
filemaxlines = int(GetConfigParameter(config, 'OutputFile', 'filemaxlines'))
delimiter = GetConfigParameter(config, 'OutputFile', 'delimiter')
quotechar = GetConfigParameter(config, 'OutputFile', 'quotechar')
outputdir = GetConfigParameter(config, 'OutputFile', 'outputdir')
analitics_path = outputdir + '/' + GetConfigParameter(config, 'Analitics', 'filename')
metrics = GetConfigParameter(config, 'Analitics', 'metrics').split(',') # add check if it is in query
limit_step = int(GetConfigParameter(config, 'ClickHouse', 'limit_step'))
clickhouse_host = GetConfigParameter(config, 'ClickHouse', 'host')
colums = GetConfigParameter(config, 'ClickHouse', 'colums').split(',')
date_start = sys.argv[1]
date_end = sys.argv[2]
complex_messages = []
total_count = limit_step*2
GetTemplatesFromCSV(template_paths)
client = ClickHouseClient(clickhouse_host, on_progress=OnProgress)
query_format = GenerateQueryFormat(config, colums)
filename = GenerateFileName(outputdir, date_start, date_end)
csvwriter = CSVWriter(filemaxlines, filename, delimiter, quotechar, u'\ufeff', colums + ['FullText', 'MessageType', 'Template'])
analitics = Analitics(analitics_path, metrics)
for single_date in DateRange(date_start, date_end):
date_str = single_date.strftime("%Y-%m-%d")
ColoredPrint('\n[Determiner]', 'Working on date(' + date_str + ')', bcolors.OKGREEN, bcolors.WARNING)
limit_count = 0
msg_count = 0
while limit_count < total_count:
query = query_format.format(date_str, limit_count, limit_step)
result = client.select(query, on_progress=OnProgress, send_progress_in_http_headers=1)
print()
data_len = len(result.data)
total_count = limit_count + data_len + 1
counter = Counter(data_len, 0.2)
for v in result.data:
message = Message(colums, v)
id, c_message = ToComplexMessage(message)
if c_message.isFullFiled():
operator_id = message.get('OperatorGroupId')
originator = message.get('OutgoingOriginator')
c_message.determineTemplate(GetTemplates(operator_id, originator))
c_message.writePartsToCsvFile(csvwriter)
analitics.addData(date_str[:-3], c_message.getParameters(metrics), c_message.getType(), c_message.getCount())
complex_messages.pop(id)
msg_count+=1
counter.step(bcolors.OKGREEN + '[' + date_str + '] ' + bcolors.ENDC + str(msg_count) + ' messages handled')
counter.lastTell(bcolors.OKGREEN + '[' + date_str + '] ' + bcolors.ENDC + str(msg_count) + ' messages handled')
del result
limit_count += limit_step
WriteDownAllRemainingData(csvwriter, analitics, metrics)
csvwriter.close()
SaveAnalitics(analitics, delimiter, quotechar)
def main():
tables = TableImporter()
writer = CSVWriter()
table = tables.get_table()
while table is not None:
writer.write(compute(table))
table = tables.get_table()
def do_write(self, filename):
"""
Writes the results into the specified file
usage: write <filename>
"""
print("Writing the contents to " + filename)
csvWriter = CSVWriter()
print("Writing the results")
print("Found %i results" % len(self.results))
try:
csvWriter.write_to_file(filename, self.results)
print("All results written to file succesfully!")
except PermissionError:
print("\nNo permission to write to file. Might it be accidently left open?")
def spawn_collectors(c):
root_url = "http://%s:%s/%s" % (c['hostname'], c['port'], c['base_url'])
pprint(c)
for data_type_name in c['data_type']:
data_type = c['data_type'][data_type_name]
if data_type['enable'] == False:
continue
filter_dict = data_type.get('filter', None)
for endpoint_name in data_type['endpoints']:
endpoint = data_type['endpoints'][endpoint_name]
url = root_url + endpoint['url']
interval = endpoint['interval']
prom_key = "%s:%s" % (data_type_name, endpoint_name)
labels = endpoint['labels']
values = endpoint['values']
print(data_type_name, endpoint_name)
pc = PrometheusClient(c['prometheus'], prom_key, labels, values)
csv = CSVWriter(c['csvwriter'], prom_key, labels + values)
if data_type_name == 'queue' and endpoint_name == 'config':
worker = QueueConfigCollector(c['dpid'], url, interval, pc,
csv, filter_dict)
worker.start()
continue
worker = Collector(c['dpid'], url, interval, pc, csv, filter_dict)
worker.start()
def verify():
jama = Jama()
csv = CSVWriter()
projects = jama.getProjects()
csv.write("projects.csv", projects)
project_ids = [project["id"] for project in projects]
item_type_ids = [item_type["id"] for item_type in jama.getItemTypes()]
for item_type_id in item_type_ids:
csv.write("{}.csv".format(item_type_id), jama.getItems(item_type_id))
relationships = []
for project_id in project_ids:
# if the relationships file gets too big you can create a file for each project's relationships
# instead of extending this list
relationships.extend(jama.getRelationships(project_id))
csv.write("relationships.csv", relationships)
# if the comments file gets too big you can split the list and scv.write() each half
csv.write("comments.csv", jama.getComments())
def main():
host = input('Hostname: ')
port = input('Port: ')
username = input('Username: '******'Password: '******'',
username,
password,
]
with paramiko.SSHClient() as ssh:
try:
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(host,
port=int(port),
username=username,
password=password)
except Exception as ex:
print(ex)
sys.exit(1)
session = ssh.invoke_shell()
dir_path = os.getcwd() + os.sep + time.strftime(
'%Y-%m-%d_%H:%M') + '_output'
if not os.path.exists(dir_path):
os.mkdir(dir_path)
csv_writer = CSVWriter(
dir_path, ['HOSTNAME', 'TELNET', 'SSH', 'VERSION', 'SERIALNUMBER'])
session_manager = SessionManager(session, auth_list, csv_writer,
dir_path)
try:
with open('hostnames.txt') as file:
hostnames = [hostname.rstrip('\n') for hostname in file]
except Exception as ex:
print(ex)
sys.exit(1)
with concurrent.futures.ThreadPoolExecutor() as executor:
executor.map(session_manager.manage, hostnames)
csv_writer.close()
def main():
degree = 60
tables = TableImporter()
writer = CSVWriter()
data_to_write = []
table = tables.get_table()
while table is not None:
x_velocity = compute_x_velocity(table)
initial_velocity = compute_initial_velocity(x_velocity, degree)
times = get_times(table)
ys = get_ys(table)
theory_ys = compute_y_at_times(initial_velocity, degree, times)
data_to_write.append(rms(ys, theory_ys))
table = tables.get_table()
writer.write(data_to_write)
def main():
usage = "usage: %prog [options]"
parser = OptionParser(usage)
parser.add_option("-i",
"--input",
dest="report",
help="NUnit XML report file name")
parser.add_option("-o",
"--output",
dest="filename",
help="Generated Excel file name without extension")
parser.add_option(
"-l",
"--light",
dest="light",
default=False,
action="store_true",
help=
"Activate light report generation (CSV files instead of Excel workbook)"
)
(options, args) = parser.parse_args()
# Exit if missing of invalid options
if not check_options(options):
parser.print_help()
sys.exit(1)
# Read NUnitTestAssembly from input report
assemblies = read_assemblies(options.report)
# Create the necessary generators
generators = create_generators(assemblies)
# Generate the appropriate report
writer = CSVWriter(generators) if options.light else ExcelWriter(
generators)
clean_filename(options)
writer.generate_report(options.filename)
print("[NUnit-Stats]: Bye")
def generate_sets(set_length=200, max_rows=10000, max_folders=1000):
writer = CSVWriter()
folder_name = writer.create_folder()
csv_name = writer.create_csv_file(folder_name)
reader = WikiReader(min_words_per_page=300)
file_name = f"{folder_name}/{csv_name}.csv"
for page_tokens in reader.get_and_reformat_pages(set_length):
if writer.count_rows(file_name) > max_rows:
if writer.count_csv_in_folder(folder_name) > max_folders:
folder_name = writer.create_folder()
csv_name = writer.create_csv_file(folder_name)
file_name = f"{folder_name}/{csv_name}.csv"
writer.add_rows(file_name,
[row for row in get_lines_separated(page_tokens)])
def main():
r = 0.038 / 2
d = 0.024
mark = 30
tables = TableImporter()
writer = CSVWriter()
table = tables.get_table()
writer.string_write(["theory", "exp", "error"])
while table is not None:
times = get_1d_data(table, 0)
ys = get_1d_data(table, 2)
h = get_h(mark)
valid_ys = filter_descending_movement(ys)
valid_ys_in_meter = change_to_meter(valid_ys)
start_point = (times[1], valid_ys_in_meter[1])
end_point = (times[len(valid_ys) - 2],
valid_ys_in_meter[len(valid_ys_in_meter) - 2])
exp_time = end_point[0] - start_point[0]
theory_time = theoretical_time(start_point, end_point, h, r, d)
error = (exp_time - theory_time) / theory_time * 100
writer.write([theory_time, exp_time, error])
table = tables.get_table()
async def get_artists_data(client):
queries_count = int(ARTISTS_NEEDED / 200)
offset = 0
csv_writer = CSVWriter()
for idx in range(queries_count):
artists_list = list()
resp = await client.artists_list(offset=offset+200*idx)
for artist in resp['obj']['data']:
artist_id = artist['chartmetric_artist_id']
fan_stats = await collect_fan_stats(artist_id, client)
artist.update(fan_stats)
tracks_helper = TracksCollector(artist_id, client)
tracks_list = await tracks_helper.tracks_list()
tracks = []
for track in tracks_list:
tmp_dict = dict(artist_id=artist_id)
if 'id' in track:
tmp_dict['track_id'] = track['id']
else:
tmp_dict['track_id'] = track['track_id']
tmp_dict['name'] = track['name']
if track['release_dates']:
tmp_dict['release_dates'] = ' '.join(
[d if d else '' for d in track['release_dates']])
tracks.append(tmp_dict)
csv_writer.write(tracks, 'artists_tracks')
artists_list.append(artist)
print(f'Took {offset+200*idx} offset')
csv_writer.write(artists_list, 'artists')
await client.close()
def main():
tables = TableImporter()
writer = CSVWriter()
table = tables.get_table(delimiter=",", header=5, footer=7)
error_rate_result = []
step = 3
while table is not None:
consistent_table = slice(table, 0, [15, 45])
average_rpm = average(get_column(consistent_table, 3))
w = rpm_to_rad_per_sec(average_rpm)
theory_rotation = compute_theory_rotation(step_to_length(step), w)
experiment_degree = linear_regression(get_column(consistent_table, 0),
get_column(consistent_table, 1))
experiment_rotation = degree_to_radian(experiment_degree)
error_rate_result.append(
(experiment_rotation - theory_rotation) / theory_rotation * 100)
table = tables.get_table(delimiter=",", header=5, footer=7)
writer.write(error_rate_result)
def main():
tables = TableImporter()
writer = CSVWriter()
result_times = []
result_degrees = []
table = tables.get_table()
while table is not None:
times = get_times(table)
ratio = compute_ratio(table)
atan = compute_atan(ratio)
positive_atan = atan_to_positive(atan)
accumulated_atan = accumulate_degree(positive_atan)
resulting_degree = set_starting_point_to_zero(accumulated_atan)
result_times.append(times)
result_degrees.append(resulting_degree)
table = tables.get_table()
result_to_write = [result_times[0]]
for i in range(len(result_times)):
result_to_write.append(result_degrees[i])
writer.write_2d(result_to_write)
def print_first_30_photos(preds):
csv_writer = CSVWriter('first_predictions.csv')
image_saver = ImageSaver('images/')
prediction_labels = np.argmax(preds, axis=1)
test_labels = np.argmax(dataset.test_labels, axis=1)
test_features = dataset.test_features
csv_writer.append_to_file(
['#', 'Paveikslėlis', 'Nuspėta klasė', 'Tikroji klasė'])
for index in range(30):
csv_writer.append_to_file([
index + 1, '', LABELS[prediction_labels[index]],
LABELS[test_labels[index]]
])
image_saver.plt.imshow(test_features[index])
image_saver.save_image(index)
from csv_reader import CSVReader
from csv_writer import CSVWriter
from trustpilot_url_encrypt import TrustPilotURLEncryption
from config import Config
if len(sys.argv) < 2:
print('Requires at least one filename')
exit()
for filename in sys.argv[1:]:
input_file = filename
output_file = filename.replace('.csv', '_links.csv')
print('reading', input_file)
config = Config('config.yaml')
reader = CSVReader(input_file)
writer = CSVWriter(output_file)
# write the header only once in the output file
write_header = True
# go through the source csv
for row in reader():
# build the record for trust pilot we get something like
record = {}
for field in ('name', 'email', 'ref'):
record[field] = row[config['fields'][field]]
url = TrustPilotURLEncryption.encrypt(record)
new_row = []
for key in reader.header:
new_row.append(row[key])
new_row.append(url)
if write_header:
write_header = False
statusFinder = PlayerStatusFinder(canonicalName(playerName),
position)
result = statusFinder.findStatus(currentYear, keepers,
oneYearAgoTransactions,
twoYearsAgoTransactions, None,
oneYearAgoDraft, twoYearsAgoDraft,
None)
if result is not None:
status, draftedYear, twoYearsAgoCost, oneYearAgoCost = result
players.append(
Player(playerName, position, status, draftedYear,
twoYearsAgoCost, oneYearAgoCost))
else:
raise Exception("Fail", "Missing player: " + playerName)
teams.append(Team(teamName, owner, players))
csvWriter = CSVWriter(teams, currentYear)
csvWriter.writeToCSV("out/status_" + str(currentYear) + ".csv")
while False:
player = input("Player?\n")
if player == "":
break
playerStatusFinder = PlayerStatusFinder(canonicalName(player))
status = playerStatusFinder.findStatus(currentYear, keepers,
oneYearAgoTransactions,
twoYearsAgoTransactions, None,
oneYearAgoDraft,
twoYearsAgoDraft, None)
statusRepresentation = playerStatusFinder.statusRepresentation(status)
print(statusRepresentation)
from loading import LHELoader
from csv_writer import CSVWriter
from ppjjww_wmuvm_process import *
if __name__=="__main__":
try: inpath = sys.argv[1]
except: logging.err("Input LHE file path expected!"); sys.exit(-1)
try: outpath = sys.argv[2]
except: logging.err("Output CSV file path expected!"); sys.exit(-1)
lhe = LHELoader(open(inpath))
csv = CSVWriter(open(outpath, "w"))
logging.info("Loading and parsing events...")
for i,eventlines in enumerate(lhe.yield_events()):
if i%10000==0: logging.dbg("%i events read..." % i)
particles = eventlines_to_particlesdict(eventlines, \
particles_naming_function = name_particles)
variables = calculate_dependent_variables(particles)
csv.write_dict(variables)
logging.info("events_counter = %i" % lhe.events_counter)
from ppjjww_wmuvm_process import *
if __name__ == "__main__":
try:
inpath = sys.argv[1]
except:
logging.err("Input LHE file path expected!")
sys.exit(-1)
try:
outpath = sys.argv[2]
except:
logging.err("Output CSV file path expected!")
sys.exit(-1)
lhe = LHELoader(open(inpath))
csv = CSVWriter(open(outpath, "w"))
logging.info("Loading and parsing events...")
for i, eventlines in enumerate(lhe.yield_events()):
if i % 10000 == 0: logging.dbg("%i events read..." % i)
particles = eventlines_to_particlesdict(eventlines, \
particles_naming_function = name_particles)
variables = calculate_dependent_variables(particles)
if len(variables) != 0:
csv.write_dict(variables)
logging.info("events_counter = %i" % lhe.events_counter)
type=str,
choices=['csv', 'tfrecords'],
required=True)
parser.add_argument("-n_splits", type=int, required=False, default=1)
parser.add_argument("-shuffle", action="store_true")
parser.add_argument("-stratify", action="store_true")
parser.add_argument("-group", type=str, required=False, help="group name")
parser.add_argument("-metadata", type=Path)
parser.add_argument("-path_column", type=str, default="path")
parser.add_argument("-label_column", type=str, default="label")
parser.add_argument("-seed", type=int, required=False)
args = parser.parse_args()
n_splits = args.n_splits
partitions = ImageDatasetBuilder(data_root=args.input_data_root,
n_splits=n_splits,
with_shuffle=args.shuffle,
with_stratify=args.stratify,
group=args.group,
metadata=args.metadata,
path_column=args.path_column,
label_column=args.label_column,
seed=args.seed).build()
if args.output_format == 'csv':
CSVWriter(n_splits=n_splits, output_dir=args.output_dir).write(partitions)
else:
ImageTFRecordsWriter(n_splits=n_splits,
output_dir=args.output_dir).write(partitions)
__author__ = "Janne"
import xml.etree.ElementTree as ET
from ieeeprocessor import IEEEProcessor
from csv_writer import CSVWriter
# Construct the tree by parsing the XML file
tree = ET.parse("searchresult.xml")
# Then, get the root of the tree
root = tree.getroot()
# Then, parse it to Entry
processor = IEEEProcessor()
processor.ProcessSearchResults(root)
print("Found %i entries" % len(processor.entries))
# Okay, now we need to process all the entries into a .csv file
# Initialize the csv writer
csvWriter = CSVWriter()
csvWriter.write_to_file("test.csv", processor.entries)
def main():
tables = TableImporter()
writer = CSVWriter()
table = tables.get_table()
while table is not None:
min_max_table = get_min_max(table, 2)
period = get_period(min_max_table)
amplitude = get_amplitude(min_max_table, 2)
writer.string_write(["period"], end=",")
writer.write(period)
writer.string_write(["amplitude"], end=",")
writer.write(amplitude)
writer.write([])
table = tables.get_table()
