def set_data(self):
with open(self.filename, 'w', newline='') as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=self.data[0].headers)
writer.writeheader()
for el in self.data:
writer.writerow(el.data)
def top_up(account, ammount):
### function tops up a selected account###
# shutil to merge temp into the old file
import shutil
import csv
filename = "bank.csv"
temp = "temp_bank.csv"
with open(filename, "r") as csvFile:
reader = csv.DictReader(csvFile)
with open(temp, "w") as temp:
fieldnames = ["acc_name", "balance"]
writer = csv.DictWriter(temp, fieldnames=fieldnames)
writer.writeheader()
for line in reader:
if line["acc_name"] == str(account):
line["balance"] = int(line["balance"]) + int(ammount)
writer.writerow(line)
else:
writer.writerow(line)
shutil.move(temp.name, filename)
def create_softwaresuggest_csv():
import csv
with open('/Users/muhammadaqib/Documents/softwaresuggest.csv',
mode='w') as csv_file:
fieldnames = ['saas_dir', 'app_name', 'app_category', 'domain']
writer = csv.DictWriter(csv_file, fieldnames=fieldnames)
writer.writeheader()
def append_dict_as_row(csv_file, hours_data, fields):
try:
with open(csv_file, 'a+', newline='') as csvfile:
writer = DictWriter(csvfile, fieldnames=fields)
writer.writeheader()
for data in hours_data:
writer.writerow(data)
except IOError:
print("I/O error")
def createExcelSheet(excel_file_path, headers):
# excel_file_path is the full file path, with extension, to the file to write.
# columns is an array of columns to add to the excel file.
# overwrite defines whether it should overwrite the file if it exists or not. Default is false.
with open(excel_file_path, 'w', newline='') as out_csv:
writer = csv.DictWriter(out_csv, headers)
writer.writeheader()
print("Creating excel file at: %s" % excel_file_path)
def create():
print("If a file with the same name already exists in the same directory, the old file is deleted.")
print("New file will be created in the same directory as this file.")
file_name = input("File name: ")
with open(file_name+".csv","w") as file:
headers = ["Name", "Price", "Release Date", "Type"]
writer = DictWriter(file, headers)
writer.writeheader()
print(file_name + " created")
file.close()
def create_getapp_csv():
import csv
with open('./getapp.csv', mode='w') as csv_file:
fieldnames = [
'saas_dir', 'app_name', 'app_category', 'domain', 'description',
'item_business_size', 'market', 'lang', 'ratings'
]
writer = csv.DictWriter(csv_file, fieldnames=fieldnames)
writer.writeheader()
def read_adc_channels(self, file, directory, nodeId, outputname, outputdir,
n_readings):
"""Start actual CANopen communication
This function contains an endless loop in which it is looped over all
ADC channels. Each value is read using
:meth:`read_sdo_can_thread` and written to its corresponding
"""
dev = AnalysisUtils().open_yaml_file(file=file, directory=directory)
# yaml file is needed to get the object dictionary items
dictionary_items = dev["Application"]["index_items"]
_adc_channels_reg = dev["adc_channels_reg"]["adc_channels"]
_adc_index = list(dev["adc_channels_reg"]["adc_index"])[0]
_channelItems = [int(channel) for channel in list(_adc_channels_reg)]
# Write header to the data
out_file_csv = AnalysisUtils().open_csv_file(outname=outputname,
directory=outputdir)
fieldnames = [
'Time', 'Channel', "nodeId", "ADCChannel", "ADCData",
"ADCDataConverted"
]
writer = csv.DictWriter(out_file_csv, fieldnames=fieldnames)
writer.writeheader()
csv_writer = csv.writer(
out_file_csv
) # , delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
monitoringTime = time.time()
for point in np.arange(0, n_readings):
# Read ADC channels
pbar = tqdm(total=len(_channelItems) + 1,
desc="ADC channels",
iterable=True)
for c in np.arange(len(_channelItems)):
channel = _channelItems[c]
subindex = channel - 2
data_point = self.read_sdo_can(nodeId, int(_adc_index, 16),
subindex, 1000)
ts = time.time()
elapsedtime = ts - monitoringTime
if data_point is not None:
adc_converted = Analysis().adc_conversion(
_adc_channels_reg[str(channel)], data_point)
adc_converted = round(adc_converted, 3)
csv_writer.writerow(
(str(round(elapsedtime,
1)), str(self.get_channel()), str(nodeId),
str(subindex), str(data_point), str(adc_converted)))
self.logger.info(
f'Got data for channel {channel}: = {adc_converted}')
pbar.update(1)
pbar.close()
self.logger.notice("ADC data are saved to %s%s" %
(outputdir, outputname))
def register_student(self, list_of_elem):
with open('student_info.csv', 'a+', newline='') as write_obj:
fieldnames = ['ID', 'Name', 'Amount', 'Amount Remaining']
writer = csv.DictWriter(write_obj,
fieldnames=fieldnames,
delimiter='|')
writer.writeheader()
writer.writerow({
'ID': list_of_elem[0],
'Name': list_of_elem[1],
'Amount': list_of_elem[2],
'Amount Remaining': list_of_elem[3]
})
def writer_csv(header, data, filename, option):
with open(filename, "w", newline="") as csvfile:
if option == "write":
wr = csv.writer(csvfile)
wr.writerow(header)
for x in data:
wr.writerow(x)
elif option == "update":
writer = csv.DictWriter(csvfile, fieldnames=header)
writer.writeheader()
writer.writerows(data)
else:
print("Option is not known")
def writeToCsv(data):
fields = [
'date',
'stock',
'Adj Close',
]
for v in data:
print(v)
fileName = config.STOCK_RESULTS
try:
with open(fileName, 'w') as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=fields)
writer.writeheader()
for d in data:
writer.writerow(d)
except Exception as e:
print(e)
def cronjob():
c = SelectedCountry.query.get(request.args.get('id'))
csv_columns = c.to_dict().keys()
cdict = c.to_dict()
csv_file = "countries.csv"
while True:
if not os.path.isfile('countries.csv'):
try:
with open(csv_file, 'w') as f:
writer = csv.DictWriter(f, fieldnames=csv_columns)
writer.writeheader()
except IOError:
print("I/O error")
with open(csv_file, 'a+', newline='') as f:
writer = csv.DictWriter(f, fieldnames=csv_columns)
writer.writerow(cdict)
#time.sleep(5)
time.sleep(86407)
return redirect(url_for('main.index'))
def delete_account(account):
###function deletes a selected account###
# returns erros message not_zero() if balance() is not 0
import shutil
import csv
filename = "bank.csv"
temp = "temp_bank.csv"
with open(filename, "r") as csvFile:
reader = csv.DictReader(csvFile)
with open(temp, "w") as temp:
fieldnames = ["acc_name", "balance"]
writer = csv.DictWriter(temp, fieldnames=fieldnames)
writer.writeheader()
for line in reader:
if line["acc_name"] == str(account):
next
else:
writer.writerow(line)
shutil.move(temp.name, filename)
def write_flows_in_new_format(self, flows, source_type, name):
path = os.path.join(self.DATA_DIR, source_type)
if not os.path.exists(path):
os.makedirs(path)
path = os.path.join(path, "%s.csv" % name)
with open(path, 'w', encoding='utf8') as output:
writer = DictWriter(output, fieldnames=self.fieldnames)
writer.writeheader()
new_flows = list(self._format_for_datapackage(flows, path))
if 'numrodeligne' in flows[0] or 'numerodeligne' in flows[0]:
nb_line_number_before = sum(
1 for f in flows
if f['numrodeligne'] != '' or f['numerodeligne'] != '')
nb_line_number_after = sum(
1 for f in new_flows
if 'line_number' in f and f['line_number'] != '')
if nb_line_number_before != nb_line_number_after:
print(nb_line_number_before)
print(nb_line_number_after)
exit(1)
writer.writerows(new_flows)
# path to be added in the datapackage resource
return path
def withdraw(account, ammount):
### function withfraws money from a selected account###
# return error message not_enought() if fund not sufficient
import shutil
import csv
filename = "bank.csv"
temp = "temp_bank.csv"
with open(filename, "r") as csvFile:
reader = csv.DictReader(csvFile)
with open(temp, "w") as temp:
fieldnames = ["acc_name", "balance"]
writer = csv.DictWriter(temp, fieldnames=fieldnames)
writer.writeheader()
for line in reader:
if line["acc_name"] == str(account):
line["balance"] = int(line["balance"]) - int(ammount)
writer.writerow(line)
else:
writer.writerow(line)
shutil.move(temp.name, filename)
def aggregate_data(read_path, write_path):
initial_dict = {
"num_jurisdictions": 0,
"active_registration": 0,
"election_participants": 0,
"eligible_voters_estimated": 0,
"jurisdictions_with_precinct_info": 0,
"jurisdictions_with_polling_place_info": 0,
"jurisdictions_with_poll_worker_count": 0,
"jurisdictions_with_age_info": 0,
"jurisdictions_with_difficulty_info": 0,
"registrants_in_jurisdictions_with_precinct_info": 0,
"registrants_in_jurisdictions_with_polling_place_info": 0,
"registrants_in_jurisdictions_with_poll_worker_info": 0,
"registrants_in_jurisdictions_with_poll_worker_age_info": 0,
"registrants_in_jurisdictions_with_difficulty_info": 0,
"participants_in_jurisdictions_with_precinct_info": 0,
"participants_in_jurisdictions_with_polling_place_info": 0,
"participants_in_jurisdictions_with_poll_worker_info": 0,
"participants_in_jurisdictions_with_poll_worker_age_info": 0,
"participants_in_jurisdictions_with_difficulty_info": 0,
"precincts": 0,
"polling_places": 0,
"poll_workers": 0,
"worker_age_group_1": 0,
"worker_age_group_2": 0,
"worker_age_group_3": 0,
"worker_age_group_4": 0,
"worker_age_group_5": 0,
"worker_age_group_6": 0
}
difficulties = set(val for val in difficulty_dict.values())
for difficulty in difficulties:
initial_dict[f"difficulty_{snakecase(difficulty)}"] = 0
data_by_state_and_year = {}
with open(read_path) as readfile:
reader = DictReader(readfile)
for row in reader:
year = row['year']
state = row['state']
data_by_state_and_year[year] = data_by_state_and_year.get(year, {})
data_by_year = data_by_state_and_year[year]
data_by_year[state] = data_by_year.get(state, initial_dict.copy())
d = data_by_year[state]
d["eligible_voters_estimated"] = voting_age_estimates[int(
year)][state]
d["num_jurisdictions"] += 1
registrants = float(get_or_0(row, "active_registration"))
participants = float(get_or_0(row, "election_participants"))
d["active_registration"] += registrants
d["election_participants"] += participants
if row.get("precincts"):
d["jurisdictions_with_precinct_info"] += 1
d["registrants_in_jurisdictions_with_precinct_info"] += registrants
d["participants_in_jurisdictions_with_precinct_info"] += participants
d["precincts"] += float(get_or_0(row, "precincts"))
if row.get("polling_places"):
d["jurisdictions_with_polling_place_info"] += 1
d["registrants_in_jurisdictions_with_polling_place_info"] += registrants
d["participants_in_jurisdictions_with_polling_place_info"] += participants
d["polling_places"] += float(get_or_0(row, "polling_places"))
if row.get("poll_workers"):
d["jurisdictions_with_poll_worker_count"] += 1
d["registrants_in_jurisdictions_with_poll_worker_info"] += registrants
d["participants_in_jurisdictions_with_poll_worker_info"] += participants
d["poll_workers"] += float(get_or_0(row, "poll_workers"))
if row.get("worker_age_group_1"):
d["jurisdictions_with_age_info"] += 1
d["registrants_in_jurisdictions_with_poll_worker_age_info"] += registrants
d["participants_in_jurisdictions_with_poll_worker_age_info"] += participants
d["worker_age_group_1"] += float(
get_or_0(row, "worker_age_group_1"))
d["worker_age_group_2"] += float(
get_or_0(row, "worker_age_group_2"))
d["worker_age_group_3"] += float(
get_or_0(row, "worker_age_group_3"))
d["worker_age_group_4"] += float(
get_or_0(row, "worker_age_group_4"))
d["worker_age_group_5"] += float(
get_or_0(row, "worker_age_group_5"))
d["worker_age_group_6"] += float(
get_or_0(row, "worker_age_group_6"))
if row.get("poll_worker_difficulty"):
d["jurisdictions_with_difficulty_info"] += 1
d["registrants_in_jurisdictions_with_difficulty_info"] += registrants
d["participants_in_jurisdictions_with_difficulty_info"] += participants
d[f"difficulty_{snakecase(row['poll_worker_difficulty'])}"] += 1
with open(write_path, "w") as writefile:
fieldnames = ["year", "state", *initial_dict.keys()]
writer = DictWriter(writefile, fieldnames=fieldnames)
writer.writeheader()
for year in data_by_state_and_year:
for state in data_by_state_and_year[year]:
writer.writerow({
"year": year,
"state": state,
**data_by_state_and_year[year][state]
})
def write_csv_file(data, file_path, header, way="w"):
with open(file_path, way) as f:
writer = DictWriter(f, header)
if way == "w" or (way == "a" and not exists(file_path)):
writer.writeheader()
writer.writerows(data)
def _process_data(self,
year,
udise_state_code,
udise_dist_code="none",
udise_block_code="none"):
"""This will process the data by loading the data file for a given
year, udise_state_code, udise_dist_code and udise_block_code.
Args:
year (TYPE): Education year of the report
udise_state_code (TYPE): UDISE state code
udise_dist_code (str, optional): UDISE district code
udise_block_code (str, optional): UDISE block code
Raises:
Exception: Throws an exception if the data file doesn't exist
"""
data_file = os.path.join(
self.data_folder,
DATA_FILE_NAME_FORMAT.format(year=year,
udise_state_code=udise_state_code,
udise_dist_code=udise_dist_code,
udise_block_code=udise_block_code))
data_rows = []
valid_columns = self.attribute_mapping.keys()
print("Processing {data_file}".format(data_file=data_file))
if path.exists(data_file):
json_data_file = open(data_file, "r")
json_data = json.loads(json_data_file.read())
json_data_file.close()
rows = json_data["rowValue"]
for row in rows:
for column in valid_columns:
# Process only if the data exists for that column
if column in row:
data_row = {}
data_row["Value"] = row[column]
data_row["Period"] = self._year_to_period(year)
data_row["LocationCode"] = row["location_code"]
data_row["LocationType"] = row["rpt_type"]
data_row["SocialCategory"] = SOCIAL_CATEGORY_MAPPING[
row["item_name"]]
attribute_map = self.attribute_mapping[column]
if "Gender" in attribute_map:
data_row["Gender"] = attribute_map["Gender"]
data_row["SchoolLevel"] = attribute_map["SchoolLevel"]
stat_var, variable_name = self._create_variable(
data_row)
data_row["StatisticalVariable"] = variable_name
data_rows.append(data_row)
# Write the final rows to CSV
write_header = False
if path.exists(self.csv_file_path) is False:
write_header = True
with open(self.csv_file_path, 'a') as file_object:
writer = csv.DictWriter(file_object,
extrasaction='ignore',
fieldnames=self.csv_headers)
if write_header:
writer.writeheader()
writer.writerows(data_rows)
file_object.close()
else:
raise Exception("Data file: {data_file} doesn't exist".format(
data_file=data_file))
def fetch_unified_messaging_page(url: str):
datax = {}
req = Request(url, headers=hdr)
html = urlopen(req, timeout=30)
soup = BeautifulSoup(html, features="lxml")
header_blocks = soup.find("header", {"class": "productBanner-alt"})
row_px = header_blocks.findAll("h1")
row_p = header_blocks.findAll("p")
datax[row_px[0].text.strip()] = row_p[0].text.strip()
container_fluid = soup.find("div", {"class": "container-fluid"})
list_features = []
row_title = container_fluid.findAll("h2")
for title in row_title:
datax[title.text.strip()] = ""
list_features.append(title.text.strip())
row_content = container_fluid.findAll("h3")
n = 0
for container in row_content:
if n == 0:
datax[list_features[0]] = container.text.strip()
if n == 1:
datax[list_features[1]] = container.text.strip()
if n == 2:
datax[list_features[2]] = container.text.strip()
if n == 3:
datax[list_features[4]] = container.text.strip()
n = n + 1
mini_dict = {}
sub_list_features = []
row_para = container_fluid.findAll("p")
for container in row_para:
data = container.findAll("span")[0].text.strip()
sub_list_features.append(data)
for span in container.findAll("span"):
span.decompose()
r = container.get_text().strip('\n').encode('utf-8')
r = str(r)
r = r.replace("b'", "")
r = r.replace("'", "")
r = r.strip()
mini_dict[data] = r
datax[list_features[3]] = mini_dict
print(datax)
# datax = [ast.literal_eval(i) for i in dict]
with open('datacrawler.csv', 'w') as csvfile:
fieldnames = ['Feature', 'Desc', 'Sub Features', 'Sub Desc']
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writeheader()
writer.writerows([
{
'Feature': row_px[0].text.strip(),
'Desc': datax[row_px[0].text.strip()].replace(",", "."),
'Sub Features': list_features[0],
'Sub Desc': datax[list_features[0]].replace(",", ".")
},
{
'Feature': row_px[0].text.strip(),
'Desc': datax[row_px[0].text.strip()].replace(",", "."),
'Sub Features': list_features[1],
'Sub Desc': datax[list_features[1]].replace(",", ".")
},
{
'Feature': row_px[0].text.strip(),
'Desc': datax[row_px[0].text.strip()].replace(",", "."),
'Sub Features': list_features[2],
'Sub Desc': datax[list_features[2]].replace(",", ".")
},
{
'Feature': row_px[0].text.strip(),
'Desc': datax[row_px[0].text.strip()].replace(",", "."),
'Sub Features': list_features[4],
'Sub Desc': datax[list_features[4]].replace(",", ".")
},
{
'Feature':
row_px[0].text.strip(),
'Desc':
datax[row_px[0].text.strip()].replace(",", "."),
'Sub Features':
sub_list_features[0],
'Sub Desc':
datax[list_features[3]][sub_list_features[0]].replace(
",", ".")
},
{
'Feature':
row_px[0].text.strip(),
'Desc':
datax[row_px[0].text.strip()].replace(",", "."),
'Sub Features':
sub_list_features[1],
'Sub Desc':
datax[list_features[3]][sub_list_features[1]].replace(
",", ".")
},
{
'Feature':
row_px[0].text.strip(),
'Desc':
datax[row_px[0].text.strip()].replace(",", "."),
'Sub Features':
sub_list_features[2],
'Sub Desc':
datax[list_features[3]][sub_list_features[2]].replace(
",", ".")
},
])
def run(staging_folder: str, input_list_file: str, output_file: str):
print("Starting Gamechanger EDA Metrics Pipeline")
# Load Extensions configuration files.
data_conf_filter = read_extension_conf()
directories = []
with open(input_list_file, "r") as file:
directories = file.readlines()
with open(output_file, 'w+', newline='') as outcsv:
writer = csv.DictWriter(outcsv,
fieldnames=[
"Directory", "Elasticsearch",
"Files Processed", "OCR", "PDS",
"PDS Quarantine", "SYN", "SYN Quarantine",
"Failed", "Failed (Text)", "Failed (PDF)"
])
writer.writeheader()
publish_audit = get_es_publisher(
staging_folder=staging_folder,
index_name=data_conf_filter['eda']['audit_index'],
alias=data_conf_filter['eda']['audit_index_alias'])
publish_es = get_es_publisher(
staging_folder=staging_folder,
index_name=data_conf_filter['eda']['eda_index'],
alias=data_conf_filter['eda']['eda_index_alias'])
for directory in directories:
print(f"-- {directory}")
# Items in ES
directory = directory.rstrip()
query_es_count = {
'query': {
'bool': {
'must': [{
'wildcard': {
'file_location_eda_ext.keyword': {
'value': directory + '*'
}
}
}]
}
}
}
response_item_in_es = publish_es.count(
index=data_conf_filter['eda']['eda_index'], body=query_es_count)
print("ES Count: : " + str(response_item_in_es['count']))
item_in_es_count = str(response_item_in_es['count'])
# Audit number of files processed
query_files_processed = {
'query': {
'wildcard': {
'gc_path_s': {
'value': directory + '*'
}
}
}
}
response_files_processed = publish_audit.count(
index=data_conf_filter['eda']['audit_index'],
body=query_files_processed)
print("Processed Count: : " + str(response_files_processed['count']))
items_processed_count = str(response_files_processed['count'])
# Audit number of files ocr
query_ocr_count = {
'query': {
'bool': {
'must': [{
'term': {
'is_ocr_b': {
'value': 'true'
}
}
}, {
'wildcard': {
'gc_path_s': {
'value': directory + '*'
}
}
}]
}
}
}
response_ocr = publish_audit.count(
index=data_conf_filter['eda']['audit_index'], body=query_ocr_count)
print("OCRed: : " + str(response_ocr['count']))
items_ocred_count = str(response_ocr['count'])
# files failed to complete processed
query_failed = {
'query': {
'bool': {
'must': [{
'term': {
'is_index_b': {
'value': 'false'
}
}
}, {
'wildcard': {
'gc_path_s': {
'value': directory + '*'
}
}
}]
}
}
}
response_failed = publish_audit.count(
index=data_conf_filter['eda']['audit_index'], body=query_failed)
print("Failed Count: : " + str(response_failed['count']))
items_failed_count = str(response_failed['count'])
# files failed to complete processed (text files)
query_failed_text = {
'query': {
'bool': {
'must': [{
'term': {
'is_index_b': {
'value': 'false'
}
}
}, {
'wildcard': {
'gc_path_s': {
'value': directory + '*.txt'
}
}
}]
}
}
}
response_failed_text = publish_audit.count(
index=data_conf_filter['eda']['audit_index'],
body=query_failed_text)
print("Failed Text Count: : " + str(response_failed_text['count']))
items_failed_text_count = str(response_failed_text['count'])
# files failed to complete processed (pdf files)
query_failed_pdf = {
'query': {
'bool': {
'must': [{
'term': {
'is_index_b': {
'value': 'false'
}
}
}, {
'wildcard': {
'gc_path_s': {
'value': directory + '*.pdf'
}
}
}]
}
}
}
response_failed_pdf = publish_audit.count(
index=data_conf_filter['eda']['audit_index'],
body=query_failed_pdf)
print("Failed PDF Count: : " + str(response_failed_pdf['count']))
items_failed_pdf_count = str(response_failed_pdf['count'])
# Metadata With PDS data
query_pds = {
'query': {
'bool': {
'must_not': [{
'term': {
'completed': {
'value': 'completed'
}
}
}],
'must': [{
'wildcard': {
'gc_path_s': {
'value': directory + '*'
}
}
}, {
'term': {
'metadata_type_s': {
'value': 'pds'
}
}
}, {
'term': {
'is_metadata_suc_b': {
'value': 'true'
}
}
}, {
'term': {
'is_supplementary_file_missing': {
'value': 'false'
}
}
}]
}
}
}
response_pds = publish_audit.count(
index=data_conf_filter['eda']['audit_index'], body=query_pds)
print("PDS Count: " + str(response_pds['count']))
items_pds_count = str(response_pds['count'])
# Metadata With PDS data quarantine
query_pds_quarantine = {
'query': {
'bool': {
'must_not': [{
'term': {
'completed': {
'value': 'completed'
}
}
}],
'must': [{
'wildcard': {
'gc_path_s': {
'value': directory + '*'
}
}
}, {
'term': {
'metadata_type_s': {
'value': 'pds'
}
}
}, {
'term': {
'is_metadata_suc_b': {
'value': 'false'
}
}
}, {
'term': {
'is_supplementary_file_missing': {
'value': 'true'
}
}
}]
}
}
}
response_pds_quarantine = publish_audit.count(
index=data_conf_filter['eda']['audit_index'],
body=query_pds_quarantine)
print("PDS Quarantine Count: " + str(response_pds_quarantine['count']))
items_pds_quarantine_count = str(response_pds_quarantine['count'])
# Metadata With SYN data
query_syn = {
'query': {
'bool': {
'must_not': [{
'term': {
'completed': {
'value': 'completed'
}
}
}],
'must': [{
'wildcard': {
'gc_path_s': {
'value': directory + '*'
}
}
}, {
'term': {
'metadata_type_s': {
'value': 'syn'
}
}
}, {
'term': {
'is_metadata_suc_b': {
'value': 'true'
}
}
}, {
'term': {
'is_supplementary_file_missing': {
'value': 'false'
}
}
}]
}
}
}
response_syn = publish_audit.count(
index=data_conf_filter['eda']['audit_index'], body=query_syn)
print("SYN Count: " + str(response_syn['count']))
items_syn_count = str(response_syn['count'])
# Metadata With SYN data quarantine
query_syn_quarantine = {
'query': {
'bool': {
'must_not': [{
'term': {
'completed': {
'value': 'completed'
}
}
}],
'must': [{
'wildcard': {
'gc_path_s': {
'value': directory + '*'
}
}
}, {
'term': {
'metadata_type_s': {
'value': 'syn'
}
}
}, {
'term': {
'is_metadata_suc_b': {
'value': 'false'
}
}
}, {
'term': {
'is_supplementary_file_missing': {
'value': 'true'
}
}
}]
}
}
}
response_syn_quarantine = publish_audit.count(
index=data_conf_filter['eda']['audit_index'],
body=query_syn_quarantine)
print("SYN Quarantine Count: " + str(response_syn_quarantine['count']))
items_syn_quarantine_count = str(response_syn_quarantine['count'])
row_contents = [
directory, item_in_es_count, items_processed_count,
items_ocred_count, items_pds_count, items_pds_quarantine_count,
items_syn_count, items_syn_quarantine_count, items_failed_count,
items_failed_text_count, items_failed_pdf_count
]
append_list_as_row(file_name=output_file, list_of_elem=row_contents)
from pathlib import Path
print('Running' if __name__ == '__main__' else 'Importing', Path(__file__).resolve())
from csv import DictReader, DictWriter, writer
from vignere_crypt import final_result
with open('Merge.csv', newline='') as file:
with open('Final.csv', 'w', newline="") as final:
reader = DictReader(file)
writer = DictWriter(final, fieldnames=[
'Name', 'Roll Number', 'DOB', 'Hash'])
writer.writeheader()
for row in reader:
dob = row['Date of Birth']
dob = dob.split("/")
dob = ['{:02d}'.format(int(x)) for x in dob]
dob[0], dob[1] = dob[1], dob[0]
roll_number = row['Roll Number']
final = '{}'.format(final_result(roll_number, dob))
dob[0], dob[1] = dob[1], dob[0]
writer.writerow({"Name": row['Name'], "DOB": "/".join(dob),
"Roll Number": row['Roll Number'], "Hash": final})
def getComputersforUsersFromCSV(usersCSV, outputCSV, username, password):
''' Function takes in CSV with user email addresses and runs a lookup for associated computers. Returns Dictionary of Computer info and exports to CSV '''
userList = []
notFound = []
with open (usersCSV, 'rU') as csvfile:
computerreader = csv.reader(csvfile, delimiter=',', quotechar='|')
next(computerreader, None)
for row in computerreader:
email = row[0].replace('"', '').strip()
userList.append(email)
print '\nAbout to run a computer lookup on the following email addresses: \n'
for email in userList:
print email
print '\nProcessing {} Email Addresses\n'.format(len(userList))
# Open output CSV for updating, Iterate through list of Email addresses and lookup associated computer details, write to CSV
csvHeaders = ['email','name','asset_tag','sn','mac_addr','jssID']
with open(outputCSV, 'w') as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=csvHeaders)
writer.writeheader()
for email in userList:
print '\nLooking up computers for {}...'.format(email)
dataDict = {}
reqStr = jss_api_base_url + '/computers/match/' + email
try:
r = apirequests.sendAPIRequest(reqStr, username, password, 'GET')
if r == -1:
return
#responseCode = r.code
baseXml = r.read()
#print baseXml
responseXml = etree.fromstring(baseXml)
# print 'All computers with ' + computerName + ' as part of the computer information:\n'
for computer in responseXml.findall('computer'):
name = computer.find('name').text
asset_tag = computer.find('asset_tag').text
sn = computer.find('serial_number').text
mac_addr = computer.find('mac_address').text
jssID = computer.find('id').text
# Create the Dictionary
dataDict.update({'email':email})
dataDict.update({'name':name})
dataDict.update({'asset_tag':asset_tag})
dataDict.update({'sn':sn})
dataDict.update({'mac_addr':mac_addr})
dataDict.update({'jssID':jssID})
if not dataDict:
print 'No computers found for {}, moving on...'.format(email)
notFound.append(email)
else:
# write row to CSV
writer.writerow(dataDict)
except UnicodeEncodeError as error:
print 'There was a problem parsing the data, likely an invalid character in one of the fields.\n'
print error
except AttributeError as error:
print 'There was a problem parsing the data, a required field may have a null value.\n'
print error
print '\nComputers CSV has been created at {}. All Done.'.format(outputCSV)
print '\nWas unable to find computers for the following user email addresses:\n'
for email in notFound:
print email
