def __init__(self):
self.cnfg = u.get_config(cfg='/Users/aliciadetelich/as_tools_config.yml')
self.header_row, self.csvfile = u.opencsv(self.cnfg['input_csv'])
self.rowcount = row_count = sum(1 for line in open(self.cnfg['input_csv']).readlines()) - 1
self.api_url = 'http://api.snaccooperative.org/'
self.headers = {'Content-type': 'application/json','Accept': 'text/plain'}
self.q = {'command': 'read','sameas': 'lcnaf_uri'}
def run_db_query():
config_file = utilities.get_config()
as_db = dbssh.DBConn(config_file=config_file)
open_query = open('get_unstructured_dates.sql', 'r', encoding='utf-8')
query_data = as_db.run_query(open_query.read())
q = query_data.values.tolist()
return q
def main():
home_dir = str(Path.home())
config_file = u.get_config(cfg=home_dir + '\\config.yml')
api_url, headers = u.login(url=config_file['api_url'], username=config_file['api_username'], password=config_file['api_password'])
print(f'Connected to {api_url}')
output_file = config_file['api_output_csv']
print('Welcome to the location barcode lookup tool. Scan barcodes below. Enter QUIT to exit.')
done = False
x = 0
while not done:
x += 1
barcode = input('Location barcode: ')
if barcode == 'QUIT':
break
else:
location_uri, data = search_locations(api_url, headers, barcode)
#will need to change this again so header row is not repeated
write_output(data, x, output_file)
print('Would you like to add containers to this location? Enter Y or N.')
decision = input('Y/N: ')
if decision == 'Y':
#this pulls the location URI for the barcode
barcode_list = process_barcodes(api_url, headers, location_uri)
search_and_update(api_url, headers, barcode_list, location_uri)
elif decision == 'N':
pass
print('Would you like to remove containers from this location? Enter Y or N.')
decision_2 = input('Y/N: ')
if decision_2 == 'Y':
unassociate_containers(api_url, headers, data)
elif decision_2 == 'N':
pass
def __init__(self, *sesh):
self.config_file = u.get_config(cfg=str(Path.home()) +
'/as_tools_config.yml')
self.api_url = self.config_file['api_url']
self.username = self.config_file['api_username']
self.password = self.config_file['api_password']
if not sesh:
self.sesh = as_session(api_url=self.api_url,
username=self.username,
password=self.password)
else:
self.sesh = sesh
self.schemas = self.get_schemas()
self.all_enumerations = self.get_dynamic_enums()
self.schema_exclusions = [
line.strip('\n') for line in open('fixtures/schema_exclusions.csv',
encoding='utf-8')
]
self.property_exclusions = [
line.strip('\n')
for line in open('fixtures/property_exclusions.csv',
encoding='utf-8')
]
self.jsonmodel_pattern = re.compile(
'(JSONModel)(\(:.*?\)\s)(uri|object|uri_or_object)')
def config_file_helper():
config_file = u.get_config(cfg='as_tools_config.yml')
with open('as_tools_config.yml', 'r', encoding='utf-8') as config_file:
cfg = json.load(config_file)
cfg[config_type] = value
with open('data/config.json', 'w', encoding='utf-8') as config_file:
json.dump(cfg, config_file)
return value
def run_client():
utilities.config = utilities.get_config()
config = utilities.config
utilities.client = TelegramClient("sessions_bot",
config["api_id"],
config["api_hash"],
loop=loop)
utilities.client.start()
utilities.load_plugins()
utilities.plugins.sort(key=sort_key)
utilities.public_plugins.sort(key=sort_key)
async def run(message, matches, chat_id, step, crons=None):
upd = ""
for line in runGitPull():
upd = upd + line.decode("utf-8")
if "Already" in upd:
return [message.reply("❏︙تم عاده تشغيل البوت وتم تحديث السورس")]
else:
utilities.config = utilities.get_config()
utilities.config["updateChat"] = message.chat_id
utilities.save_config()
await message.reply("❏︙جاري تحديث السورس رجاء الانتضار منفضلك")
restartBot()
return []
def main():
home_dir = str(Path.home())
config_file = u.get_config(cfg=home_dir + '/config.yml')
api_url, headers = u.login(url=config_file['api_url'],
username=config_file['api_username'],
password=config_file['api_password'])
output_file = config_file['api_output_csv']
#want to be able to do this more than once - ORRR just have this as a command line argument like the dupe detect
if len(sys.argv) > 1:
barcodes = sys.argv[1:]
for i, barcode in enumerate(barcodes):
data = search_locations(api_url, headers, barcode)
#need to fix this so that if there's already a header row don't create another one
write_output(data, i, output_file)
def __init__(self):
'''Need to test what happens if I change a value in the config file...'''
self.config_file = u.get_config(cfg='as_tools_config.yml')
self.api_url = self.config_file['api_url']
self.username = self.config_file['api_username']
self.password = self.config_file['api_password']
#is this what I want?
self.dirpath = u.setdirectory(self.config_file['backup_directory'])
#this can be changed, the csvdict function will need to be called again
self.csvfile = u.opencsvdict(self.config_file['input_csv'])
self.sesh, self.sesh_file = as_session(self.api_url, self.username,
self.password)
self.json_data = ASJsonData()
self.crud = ASCrud(self.config_file, self.sesh)
async def run(message, matches, chat_id, step, crons=None):
upd = ""
for line in runGitPull():
upd = upd + line.decode("utf-8")
if "Already" in upd:
return [message.reply("The source is up to date.")]
else:
utilities.config = utilities.get_config()
utilities.config["updateChat"] = message.chat_id
utilities.save_config()
await message.reply(
"The source has been updated,the bot will restart please wait."
)
restartBot()
return []
def __init__(self, sesh=None):
self.config_file = utes.get_config(cfg=str('config.yml')
self.api_url = self.config_file['api_url']
self.username = self.config_file['api_username']
self.password = self.config_file['api_password']
#is this what I want?
self.dirpath = utes.setdirectory(self.config_file['backup_directory'])
#this can be changed, the csvdict function will need to be called again
self.csvfile = utes.opencsvdict(self.config_file['input_csv'])
self.ead_3_transformation = self.config_file['ead_3_transformation']
self.ead_3_schema_path = self.config_file['ead_3_schema']
self.manifest_path = self.config_file['manifest_path']
self.transformation_errors = utes.openoutfile(self.config_file['transformation_errors'])
self.validation_errors = utes.openoutfile(self.config_file['validation_errors'])
self.saxon_path = self.config_file['saxon_path']
#self.ead_3_transformation = requests.get("https://raw.githubusercontent.com/YaleArchivesSpace/EAD3-to-PDF-UA/master/xslt-to-update-the-ASpace-export/yale.aspace_v2_to_yale_ead3.xsl").text
self.ead_3_schema = self.prep_schema_for_validation()
if sesh is None:
self.sesh = as_session(api_url=self.api_url, username=self.username, password=self.password)
else:
self.sesh = sesh
def log_subprocess_output(self, pipe):
for line in iter(pipe.readline, b''):
logging.warning(line)
def prep_schema_for_validation(self):
ead_3_schema_doc = etree.parse(self.ead_3_schema_path)
return etree.XMLSchema(ead_3_schema_doc)
def export_ead(self, row, ead3=True, get_ead=None):
'''Exports EAD files using a list of resource IDs as input.
Parameters:
row['resource_id']: The ID of the resource
row['repo_id']: The ID of the repository
Returns:
str: A string representation of the EAD response from the ArchivesSpace API.
'''
repo_id = row['repo_id']
resource_id = row['resource_id']
print(f'Exporting {resource_id}')
if ead3 == True:
get_ead = self.sesh.get(f"{self.api_url}/repositories/{repo_id}/resource_descriptions/{resource_id.strip()}.xml?include_unpublished=true&ead3=true", stream=True).text
elif ead3 == False:
get_ead = self.sesh.get(f"{self.api_url}/repositories/{repo_id}/resource_descriptions/{resource_id.strip()}.xml?include_unpublished=true", stream=True).text
print(f'{resource_id} exported. Writing to file.')
ead_file_path = f"{self.dirpath}/{resource_id}.xml"
with open(ead_file_path, 'a', encoding='utf-8') as outfile:
outfile.write(get_ead)
print(f'{resource_id} written to file: {ead_file_path}')
return ead_file_path
def transform_ead_3(self, ead_file_path):
'''Transforms EAD files using a user-defined XSLT file.'''
print(f'Transforming file: {ead_file_path}')
subprocess.run(["java", "-cp", f"{self.saxon_path}", "net.sf.saxon.Transform",
f"-s:{ead_file_path}",
f"-xsl:{self.ead_3_transformation}",
f"-o:{ead_file_path[:-4]}_out.xml"], stdout=self.transformation_errors, stderr=subprocess.STDOUT,
encoding='utf-8')
#if proc.stderr:
# self.log_subprocess_output(proc.stderr)
#this doesn't mean that it was successful...
print(f'Transformation finished: {ead_file_path}')
return f"{ead_file_path[:-4]}_out.xml"
#return open(f"{ead_file_path[:-4]}_out.xml", 'r', encoding='utf-8').read()
def validate_ead_3(self, ead_file_path):
print(f'Validating file: {ead_file_path}')
try:
#print(type(ead_file_path))
with open(ead_file_path, 'r', encoding='utf-8') as open_ead:
doc = etree.parse(open_ead)
try:
self.ead_3_schema.assertValid(doc)
#self.validation_errors.write(f'{ead_file_path} is valid')
logging.warning(f'\n\n{ead_file_path} is valid')
except etree.DocumentInvalid as err:
#self.validation_errors.write(f'Schema Validation Error: {ead_file_path}')
#self.validation_errors.write(traceback.format_exc())
#self.validation_errors.write(err.error_log)
logging.warning(f'\n\nSchema Validation Error: {ead_file_path}')
#logging.exception('Error: ')
logging.warning(err.error_log)
except Exception:
#self.validation_errors.write(f'Other validation error: {ead_file_path}')
logging.warning(f'\n\nOther validation error: {ead_file_path}')
logging.exception('Error: ')
#self.validation_errors.write(traceback.format_exc())
#this finds a problem with the file
except IOError:
#self.validation_errors.write(f'Invalid file: {ead_file_path}')
#self.validation_errors.write(traceback.format_exc())
logging.warning(f'\n\nInvalid file: {ead_file_path}')
logging.exception('Error: ')
#this finds syntax errors in XML
except etree.XMLSyntaxError as err:
#self.validation_errors.write(f'XML Syntax Error: {ead_file_path}')
#self.validation_errors.write(traceback.format_exc())
#self.validation_errors.write(err.error_log)
logging.warning(f'\n\nXML Syntax Error: {ead_file_path}')
logging.warning(err.error_log)
logging.exception('Error: ')
except Exception:
#self.validation_errors.write(f'Other validation error: {ead_file_path}')
#self.validation_errors.write(traceback.format_exc())
logging.warning(f'\n\nOther validation error: {ead_file_path}')
logging.exception('Error: ')
print(f'Validation complete: {ead_file_path}')
def export_transform_validate_ead3(self, row):
'''Runs export, transform, and validate EAD functions using a user-defined schema file.'''
ead_file_path = self.export_ead(row)
transformed_ead_path = self.transform_ead_3(ead_file_path)
validated_ead = self.validate_ead_3(transformed_ead_path)
if __name__ == "__main__":
error_log()
ead_utes = EADUtils()
logging.warning(f'''Starting logging for EAD export.
API URL: {ead_utes.api_url}
CSV Input: {ead_utes.config_file['input_csv']}
''')
for row in ead_utes.csvfile:
#instead of worrying about real time logging could just do a tqdm bar. The only problem is if it aborts somehow
ead_utes.export_transform_validate_ead3(row)
'''Standalone script for EAD3 transformations.
def __init__(self):
self.config_file = u.get_config(cfg='as_tools_config.yml')
self.dirpath = u.setdirectory(self.config_file['backup_directory'])
self.csvfile = u.opencsvdict(self.config_file['input_csv'])
self.dbconn = dbssh.DBConn(config_file=self.config_path)
self.query_data = ASQueries()
#local imports
from queries import ASQueries
from json_data import ASJsonData
from crud import ASCrud
from templates import ASTemplates
from aspace_run import ASpaceRun, as_session
import app_renderers as renderers
from app_layouts import serve_layout
import app_dropdown_values as dv
import aspace_tools_logging as atl
logger = atl.logging.getLogger(__name__)
config_file = u.get_config(cfg=str(Path.home()) + '/as_tools_config.yml')
app = dash.Dash(__name__)
app.config.suppress_callback_exceptions = True
app.layout = serve_layout
#############################
#### Query App Callbacks ####
#############################
#CAN'T I JUST PUT THE DIVIDER IN THE INPUT DIV???
@app.callback(Output('input_div',
'children'), [Input('selected_query_dropdown', 'value')],
[State('query_type_dropdown', 'value')])