def process_citation_url():
while True:
url = (yield)
if not dbm.is_attr_in_db('source_url', url, dbm.GAME_CITATION_TABLE):
click.echo('Extracting url {} through coroutine.'.format(url))
extractor = get_extractor_for_uri(url, get_url_source(url))
extractor.extract()
citation, extracted_options = extractor.create_citation()
citations.append((citation, NEW))
else:
click.echo('Found {} in local db.'.format(url))
citation = dbm.create_cite_ref_from_db(GAME_CITE_REF,
dbm.retrieve_attr_from_db(
'source_url',
url,
dbm.GAME_CITATION_TABLE)[0])
citations.append((citation, OLD))
def extract_uri(ctx, uri):
verbose = ctx.obj['VERBOSE']
no_prompts = ctx.obj['NO_PROMPTS']
source_name = get_uri_source_name(uri)
if source_name:
cond_print(verbose, "Starting extraction of {} source".format(source_name))
else:
click.echo("Could not find extractor for given uri:{}. Goodbye!".format(uri))
sys.exit(1)
try:
source = get_url_source(uri)
except SourceError as e:
click.echo(e.message)
sys.exit(1)
extractor = get_extractor_for_uri(uri, source)
cond_print(verbose, "Using {} for extraction".format(extractor.__class__.__name__))
# Check for duplicate entries, by url from source, this is needed since there might be a redirect from the
# input uri, like http -> https. Could check this in the extractor or tell the user that the url is changing.
# Though if the redirect always happens it wouldn't matter anyway, since the database retains the redirected url
cond_print(verbose, 'Checking for duplicates...')
if not no_prompts and has_potential_duplicates(source.url, 'source_uri', dbm.EXTRACTED_TABLE):
if settle_for_duplicate(source.url, 'source_uri', dbm.EXTRACTED_TABLE):
sys.exit(1)
cond_print(verbose, 'Validating URI...')
# Check if this is a game url for extraction
if not extractor.validate():
if no_prompts or not click.confirm('This doesn\'t appear to be a game related uri. Extract anyway?'):
sys.exit(1)
cond_print(verbose, 'Extracting URI...')
# These are separate since file downloads might rely on subprocesses
try:
extractor.extract()
except ExtractorError as e:
click.echo(e.message)
sys.exit(1)
# Block until extraction complete, needed for anything requiring sub-processes
while not extractor.extracted_info:
pass
extracted_info = extractor.extracted_info
# Create citation from extracted information
if no_prompts or click.confirm('Create citation from extracted data?'):
citation, extracted_options = extractor.create_citation()
if not no_prompts:
citation = get_citation_user_input(citation, extracted_options)
if citation.ref_type == GAME_CITE_REF:
alternate_citation = choose_game_citation(search_locally_with_citation(citation))
elif citation.ref_type == PERF_CITE_REF:
alternate_citation = choose_performance_citation(search_locally_with_citation(citation))
if not alternate_citation:
dbm.add_to_citation_table(citation, fts=True)
click.echo('Citation added to database.')
else:
dbm.add_to_citation_table(citation, fts=True)
if 'errors' not in extracted_info and dbm.add_to_extracted_table(extracted_info):
cond_print(verbose, "Extraction Successful!")
if not no_prompts:
summary_prompt(extracted_info)
else:
cond_print(verbose, "Extraction Failed!")
pprint.pprint(extracted_info)
def cite_performance(ctx, export, file_path, url, partial, schema_version):
verbose = ctx.obj['VERBOSE']
no_prompts = ctx.obj['NO_PROMPTS']
alternate_citation = None
# Make sure that only one input flag is used
if sum(map(lambda x: 1 if x else 0, [file_path, url, partial])) > 1:
click.echo('Please use only one option for citation source data.')
click.echo('Usage:\n\t{} --file_path PATH_TO_FILE\n\t{} --url URL\n\t{} --partial JSON_OBJECT'.format('cite_performance'))
sys.exit(1)
if not file_path and not url and not partial:
if no_prompts:
cond_print(verbose, 'Cannot create a blank citation with no-prompts flag active.')
cond_print(verbose, 'Creating a new citation...')
citation = get_citation_user_input(generate_cite_ref(PERF_CITE_REF, PERF_SCHEMA_VERSION))
cond_print(verbose, 'Searching for similar citations...')
alternate_citation = choose_performance_citation(search_locally_with_citation(citation))
if file_path:
extractor = get_extractor_for_file(os.path.expanduser(file_path))
try:
extractor.extract()
except ExtractorError as e:
click.echo(e.message)
sys.exit(1)
citation, extracted_options = extractor.create_citation()
if not no_prompts:
citation = get_citation_user_input(citation, extracted_options)
alternate_citation = search_locally_with_citation(citation)
elif url:
try:
source = get_url_source(url)
except SourceError as e:
click.echo(e.message)
sys.exit(1)
extractor = get_extractor_for_uri(url, source)
try:
extractor.extract()
except ExtractorError as e:
click.echo(e.message)
sys.exit(1)
# Block while waiting for extraction to finish, necessary for video downloads
while not extractor.extracted_info:
pass
citation, extracted_options = extractor.create_citation()
if not no_prompts:
citation = get_citation_user_input(citation, extracted_options)
alternate_citation = choose_game_citation(search_locally_with_citation(citation))
elif partial:
partial_json = json.loads(partial)
# Create citation based on partial description
citation = generate_cite_ref(PERF_CITE_REF, PERF_SCHEMA_VERSION, **partial_json['description'])
# Search locally based on partial description
if not no_prompts:
alternate_citation = choose_performance_citation(search_locally_with_citation(citation))
if alternate_citation:
citation = alternate_citation
else:
if citation:
dbm.add_to_citation_table(citation, fts=True)
if export and citation:
click.echo(citation.to_json_string())
def cite_game(ctx, file_path, directory, executable, url, title, partial, export, schema_version):
verbose = ctx.obj['VERBOSE']
no_prompts = ctx.obj['NO_PROMPTS']
alternate_citation = None
# Make sure that only one input flag is used
if sum(map(lambda x: 1 if x else 0, [file_path, url, title, partial])) > 1:
click.echo('Please use only one option for citation source data.')
click.echo('Usage:\n\tcite --file_path PATH_TO_FILE\n\tcite --url URL\n\tcite --title TITLE\n\tcite --partial JSON_OBJECT')
sys.exit(1)
# if no input flag
if not file_path and not url and not title and not partial and not directory:
# Make a brand new citation
if no_prompts:
cond_print(verbose, 'Cannot create a blank citation with no-prompts flag active.')
sys.exit(1)
cond_print('Creating new citation...\n', verbose)
citation = get_citation_user_input(generate_cite_ref(GAME_CITE_REF, schema_version))
cond_print('Searching for similar citations...\n', verbose)
# Check for similar citations
alternate_citation = choose_game_citation(search_locally_with_citation(citation))
# FILE PATH citation
if file_path:
extractor = get_extractor_for_file(os.path.expanduser(file_path))
try:
extractor.extract()
except ExtractorError as e:
click.echo(e.message)
sys.exit(1)
citation, extracted_options = extractor.create_citation()
if not no_prompts:
citation = get_citation_user_input(citation, extracted_options)
alternate_citation = choose_game_citation(search_locally_with_citation(citation))
# DIRECTORY citation
elif directory:
extractor = get_extractor_for_directory(os.path.abspath(os.path.expanduser(directory)))
alternate_citation = None # Needed to make sure name is present for check below
# Make sure this is a directory
if not extractor.validate():
click.echo('{} is not a valid directory.'.format(directory))
sys.exit(1)
# Add in the executable path if provided
options = {}
if executable:
options['main_executable'] = executable
# If there's additional information from a partial, load it
if partial:
partial_json = json.loads(partial)
citation = generate_cite_ref(GAME_CITE_REF, GAME_SCHEMA_VERSION, **partial_json['description'])
else:
citation = generate_cite_ref(GAME_CITE_REF, GAME_SCHEMA_VERSION, title=directory.split('/')[-1])
# Get input if prompts are allowed
if not no_prompts:
citation = get_citation_user_input(citation)
alternate_citation = choose_citation(search_locally_with_citation(citation))
# Extract all the files and paths
try:
extractor.extract(options=options)
except ExtractorError as e:
click.echo(e.message)
sys.exit(1)
# Add file_paths to game data store if this is a new citation
file_info = extractor.extracted_info['file_info']
if not alternate_citation:
for fd in file_info:
fd['game_uuid'] = citation['uuid']
dbm.insert_into_table(dbm.GAME_FILE_PATH_TABLE, fd.keys(), fd.values())
else:
# Clean up extracted data if alternate citation found (only for prompts call)
for fd in file_info:
shutil.rmtree(os.path.join(LOCAL_GAME_DATA_STORE, fd['source_data'], fd['file_path'].split('/')[-1]))
# URL citation
elif url:
try:
source = get_url_source(url)
except SourceError as e:
click.echo(e.message)
sys.exit(1)
extractor = get_extractor_for_uri(url, source)
try:
extractor.extract()
except ExtractorError as e:
click.echo(e.message)
sys.exit(1)
# Block if this is a link to a video or other extractor process
while not extractor.extracted_info:
pass
citation, extracted_options = extractor.create_citation()
if not no_prompts:
citation = get_citation_user_input(citation, extracted_options)
alternate_citation = choose_game_citation(search_locally_with_citation(citation))
# TITLE citation
elif title:
if no_prompts:
cond_print(verbose, 'Cannot do citation by title with no-prompts flag active.')
sys.exit(1)
# Try a local citation search
alternate_citation = choose_game_citation(search_locally_with_game_title(title))
citation = None
# If that didn't work try to search internet
# Current hard-coded limit as 10, may allow flag for future
if not alternate_citation:
citation = choose_game_citation(search_globally_with_game_title(title, limit=10))
# If that didn't work, make a new citation
if not citation and not alternate_citation:
citation = generate_cite_ref(GAME_CITE_REF,
GAME_SCHEMA_VERSION,
title=title)
# Edit the citation if needed
citation = get_citation_user_input(citation)
# PARTIAL citation
elif partial:
partial_json = json.loads(partial)
# Create citation based on partial description
citation = generate_cite_ref(GAME_CITE_REF, GAME_SCHEMA_VERSION, **partial_json['description'])
if not no_prompts:
# Search locally based on partial description
alternate_citation = choose_game_citation(search_locally_with_citation(citation))
# Search globally if that didn't work
if not alternate_citation:
citation = choose_game_citation(search_globally_with_game_partial(partial_json))
# If an alternate was found, don't do anything
if alternate_citation:
citation = alternate_citation
else:
# Add the new citation to the database
if citation:
dbm.add_to_citation_table(citation, fts=True)
if export and citation:
click.echo(citation.to_json_string())