def bst_apsharvest(dois="", recids="", query="", records="", new_mode="email",
update_mode="email", from_date="", until_date=None,
metadata="yes", fulltext="yes", hidden="yes", match="no"):
"""
Task to download APS metadata + fulltext given a list of arguments.
Operates in two ways:
1. Harvesting of new/updated metadata+fulltext from APS via REST API
This means that new records are being looked for at APS servers.
Active when from_date and until_date is given, in addition when
a DOI not already in the system is given.
If the value "last" is given to from_date the harvester will harvest
any new records since last run.
If match is set to "yes" the records harvested will be matched against
the database and split into "new" and "updated" records.
2. Attachment of fulltext only from APS for existing records
When the records to be processed already exists in the system, the
task only harvests the fulltext's themselves and attaches them
to the records.
Examples:
Get full update for existing records via record identifier:
>>> bst_apsharvest(recids="13,513,333")
Get full update for existing records via a search query and unhide fulltext:
>>> bst_apsharvest(query="find j prstab", hidden="no")
Get metadata only update for an existing doi:
>>> bst_apsharvest(dois="10.1103/PhysRevB.87.235401", fulltext="no")
Get fulltext only update for a record and append to record:
>>> bst_apsharvest(recids="11139", metadata="no", update_mode="append")
Get new records from APS, send update to holding pen and email new records
>>> bst_apsharvest(from_date="last", update_mode="o")
Get records from APS updated between given dates, insert new and correct
>>> bst_apsharvest(from_date="2013-06-03", until_date="2013-06-04",
new_mode="insert", update_mode="correct")
@param dois: comma-separated list of DOIs to download fulltext/metadata for.
@type dois: string
@param recids: comma-separated list of recids of record containing
a DOI to download fulltext for.
@type recids: string
@param query: an Invenio search query of records to download fulltext for.
@type query: string
@param records: get any records modified, created or both since last time
in the database to download fulltext for, can be either:
"new" - fetches all new records added
"modified" - fetches all modified records added
"both" - both of the above
@type records: string
@param new_mode: which mode should the fulltext files be submitted in:
"email" - does NOT run bibupload and sends an email instead. Default.
"insert" - inserts the records into the database
"append" - appends the fulltext to the existing attached files
"correct" - corrects existing attached fulltext files, or adds new
"replace" - replaces all attached files with new fulltext file
The fulltext is appended by default to new records.
@type mode: string
@param update_mode: which mode should the fulltext files be submitted in:
"email" - does NOT run bibupload and sends an email instead. Default.
"insert" - inserts the records into the database
"append" - appends the fulltext to the existing attached files
"correct" - corrects existing attached fulltext files, or adds new
"replace" - replaces all attached files with new fulltext file
The fulltext is appended by default to new records.
@type mode: string
@param from_date: ISO date for when to harvest records from. Ex. 2013-01-01
If the value is "last" it means to get records since last
harvest.
@type from_date: string
@param until_date: comma-separated list of DOIs to download fulltext/metadata for.
@type until_date: string
@param fulltext: should the record have fulltext attached? "yes" or "no"
@type fulltext: string
@param hidden: should the fulltext be hidden when attached? "yes" or "no"
@type hidden: string
@param match: should a simple match with the database be done? "yes" or "no"
@type match: string
"""
# This is the list of APSRecord objects to be harvested.
final_record_list = APSRecordList()
task_update_progress("Parsing input parameters")
# Validate modes
for mode in [new_mode, update_mode]:
if mode not in ("append", "a", "correct", "c", "o",
"replace", "r", "insert", "i", "email"):
raise Exception("Warning: given upload mode '%s' is not valid."
% (mode,))
# We hide fulltext by default
if hidden.lower() == "no":
hidden = False
else:
hidden = True
# We attach fulltext by default
if fulltext.lower() == "no":
fulltext = False
else:
fulltext = True
# We attach meta-data by default
if metadata.lower() == "no":
metadata = False
else:
metadata = True
# We do not match records by default
if match.lower() == "yes":
match = True
else:
match = False
if from_date:
# We get records from APS directly
new_harvest_date = None
perpage = 100
# Are we harvesting from last time or a specific date?
if from_date == "last":
dummy, harvest_from_date = fetch_last_updated(name="apsharvest_api_download")
# Keeping current time until completed harvest.
new_harvest_date = datetime.datetime.now()
else:
# Input from user. Validate date
try:
harvest_from_date = validate_date(from_date)
except ValueError, e:
write_message("Error parsing from_date, use (YYYY-MM-DD): %s" %
(str(e),),
stream=sys.stderr)
return 1
# Turn harvest_from_date back into a string (away from datetime object)
harvest_from_date = harvest_from_date.strftime("%Y-%m-%d")
status_message = "Checking for new records from APS from %s" % \
(harvest_from_date,)
if until_date:
# Input from user. Validate date
try:
validate_date(until_date)
except ValueError, e:
write_message("Error parsing until_date, use (YYYY-MM-DD): %s" %
(str(e),),
stream=sys.stderr)
return 1
status_message += " until %s" % (until_date,)
write_message("Performing a search query...")
# We are doing a search query, rg=0 allows the return of all results.
result = perform_request_search(p=query,
cc=CFG_APSHARVEST_SEARCH_COLLECTION,
of='id',
rg=0,
wl=0)
for recid in result:
final_record_list.append(APSRecord(recid))
if records in ("new", "modified", "both"):
write_message("Fetching records to update...")
# We fetch records from the database
last_recid, last_date = fetch_last_updated(name="apsharvest")
records_found = []
if records == "new":
records_found = get_all_new_records(since=last_date,
last_recid=last_recid)
elif records == "modified":
records_found = get_all_modified_records(since=last_date,
last_recid=last_recid)
elif records == "both":
records_found.extend(get_all_new_records(since=last_date,
last_recid=last_recid))
records_found.extend(get_all_modified_records(since=last_date,
last_recid=last_recid))
for recid, date in records_found:
final_record_list.append(APSRecord(recid, date=date))
def bst_apsharvest(dois="",
recids="",
query="",
records="",
new_mode="email",
update_mode="email",
from_date="",
until_date=None,
metadata="yes",
fulltext="yes",
hidden="yes",
match="no"):
"""
Task to download APS metadata + fulltext given a list of arguments.
Operates in two ways:
1. Harvesting of new/updated metadata+fulltext from APS via REST API
This means that new records are being looked for at APS servers.
Active when from_date and until_date is given, in addition when
a DOI not already in the system is given.
If the value "last" is given to from_date the harvester will harvest
any new records since last run.
If match is set to "yes" the records harvested will be matched against
the database and split into "new" and "updated" records.
2. Attachment of fulltext only from APS for existing records
When the records to be processed already exists in the system, the
task only harvests the fulltext's themselves and attaches them
to the records.
Examples:
Get full update for existing records via record identifier:
>>> bst_apsharvest(recids="13,513,333")
Get full update for existing records via a search query and unhide fulltext:
>>> bst_apsharvest(query="find j prstab", hidden="no")
Get metadata only update for an existing doi:
>>> bst_apsharvest(dois="10.1103/PhysRevB.87.235401", fulltext="no")
Get fulltext only update for a record and append to record:
>>> bst_apsharvest(recids="11139", metadata="no", update_mode="append")
Get new records from APS, send update to holding pen and email new records
>>> bst_apsharvest(from_date="last", update_mode="o")
Get records from APS updated between given dates, insert new and correct
>>> bst_apsharvest(from_date="2013-06-03", until_date="2013-06-04",
new_mode="insert", update_mode="correct")
@param dois: comma-separated list of DOIs to download fulltext/metadata for.
@type dois: string
@param recids: comma-separated list of recids of record containing
a DOI to download fulltext for.
@type recids: string
@param query: an Invenio search query of records to download fulltext for.
@type query: string
@param records: get any records modified, created or both since last time
in the database to download fulltext for, can be either:
"new" - fetches all new records added
"modified" - fetches all modified records added
"both" - both of the above
@type records: string
@param new_mode: which mode should the fulltext files be submitted in:
"email" - does NOT run bibupload and sends an email instead. Default.
"insert" - inserts the records into the database
"append" - appends the fulltext to the existing attached files
"correct" - corrects existing attached fulltext files, or adds new
"replace" - replaces all attached files with new fulltext file
The fulltext is appended by default to new records.
@type mode: string
@param update_mode: which mode should the fulltext files be submitted in:
"email" - does NOT run bibupload and sends an email instead. Default.
"insert" - inserts the records into the database
"append" - appends the fulltext to the existing attached files
"correct" - corrects existing attached fulltext files, or adds new
"replace" - replaces all attached files with new fulltext file
The fulltext is appended by default to new records.
@type mode: string
@param from_date: ISO date for when to harvest records from. Ex. 2013-01-01
If the value is "last" it means to get records since last
harvest.
@type from_date: string
@param until_date: comma-separated list of DOIs to download fulltext/metadata for.
@type until_date: string
@param fulltext: should the record have fulltext attached? "yes" or "no"
@type fulltext: string
@param hidden: should the fulltext be hidden when attached? "yes" or "no"
@type hidden: string
@param match: should a simple match with the database be done? "yes" or "no"
@type match: string
"""
# This is the list of APSRecord objects to be harvested.
final_record_list = APSRecordList()
task_update_progress("Parsing input parameters")
# Validate modes
for mode in [new_mode, update_mode]:
if mode not in ("append", "a", "correct", "c", "o", "replace", "r",
"insert", "i", "email"):
raise Exception("Warning: given upload mode '%s' is not valid." %
(mode, ))
# We hide fulltext by default
if hidden.lower() == "no":
hidden = False
else:
hidden = True
# We attach fulltext by default
if fulltext.lower() == "no":
fulltext = False
else:
fulltext = True
# We attach meta-data by default
if metadata.lower() == "no":
metadata = False
else:
metadata = True
# We do not match records by default
if match.lower() == "yes":
match = True
else:
match = False
if from_date:
# We get records from APS directly
new_harvest_date = None
perpage = 100
# Are we harvesting from last time or a specific date?
if from_date == "last":
dummy, harvest_from_date = fetch_last_updated(
name="apsharvest_api_download")
# Keeping current time until completed harvest.
new_harvest_date = datetime.datetime.now()
else:
# Input from user. Validate date
try:
harvest_from_date = validate_date(from_date)
except ValueError, e:
write_message("Error parsing from_date, use (YYYY-MM-DD): %s" %
(str(e), ),
stream=sys.stderr)
return 1
# Turn harvest_from_date back into a string (away from datetime object)
harvest_from_date = harvest_from_date.strftime("%Y-%m-%d")
status_message = "Checking for new records from APS from %s" % \
(harvest_from_date,)
if until_date:
# Input from user. Validate date
try:
validate_date(until_date)
except ValueError, e:
write_message(
"Error parsing until_date, use (YYYY-MM-DD): %s" %
(str(e), ),
stream=sys.stderr)
return 1
status_message += " until %s" % (until_date, )
# Input from user. Validate date
try:
validate_date(parameters.get("threshold_date"))
except ValueError, e:
write_message("Error parsing from_date, use (YYYY-MM-DD): %s" %
(str(e),),
stream=sys.stderr)
raise
if parameters.get("from_date"):
# We get records from APS directly
perpage = 100
# Are we harvesting from last time or a specific date?
if parameters.get("from_date") == "last":
dummy, harvest_from_date = fetch_last_updated(name="apsharvest_api_download")
# Keeping current time until completed harvest.
new_harvest_date = datetime.datetime.now()
else:
# Input from user. Validate date
try:
harvest_from_date = validate_date(parameters.get("from_date"))
except ValueError, e:
write_message("Error parsing from_date, use (YYYY-MM-DD): %s" %
(str(e),),
stream=sys.stderr)
raise
# If threshold is not given, set it to from_date - CFG_APSHARVEST_THRESHOLD_DAYS
if not parameters.get("threshold_date"):
# We are doing a search query, rg=0 allows the return of all results.
result = perform_request_search(
p=query,
cc=CFG_APSHARVEST_SEARCH_COLLECTION,
of='id',
rg=0,
wl=0)
for recid in result:
final_record_list.append(APSRecord(recid))
if records in ("new", "modified", "both"):
write_message("Fetching records to update...")
# We fetch records from the database
last_recid, last_date = fetch_last_updated(name="apsharvest")
records_found = []
if records == "new":
records_found = get_all_new_records(since=last_date,
last_recid=last_recid)
elif records == "modified":
records_found = get_all_modified_records(since=last_date,
last_recid=last_recid)
elif records == "both":
records_found.extend(
get_all_new_records(since=last_date,
last_recid=last_recid))
records_found.extend(
get_all_modified_records(since=last_date,
last_recid=last_recid))