def get_args():
utils.printer2("Getting arguments from command line")
argparser = argparse.ArgumentParser()
argparser.add_argument('-b','--UserSubmissionID', default='none', help = 'Enter the ID# of the batch you want to submit (e.g. -b 23)')
argparser.add_argument('-t','--test', help = 'Use this flag (no arguments) if you are NOT on a farm node and want to test the submission flag (-s)', action = 'store_true')
argparser.add_argument('-s','--submit', help = 'Use this flag (no arguments) if you want to submit the job', action = 'store_true')
argparser.add_argument('-w','--write_files', help = 'Use this flag (no arguments) if you want submission files to be written out to text files', action = 'store_true')
argparser.add_argument('-y','--scard_type', default='0', help = 'Enter scard type (e.g. -y 1 for submitting type 1 scards)')
argparser.add_argument(fs.debug_short,fs.debug_longdash, default = fs.debug_default,help = fs.debug_help)
argparser.add_argument('-l','--lite',help = "use -l or --lite to connect to sqlite DB, otherwise use MySQL DB", type=str, default=None)
argparser.add_argument('-o','--OutputDir', default='none', help = 'Enter full path of your desired output directory, e.g. /u/home/robertej')
argparser.add_argument('--test_database', action='store_true', default=False, help='Use testing database (MySQL)')
args = argparser.parse_args()
fs.DEBUG = getattr(args,fs.debug_long)
fs.use_mysql = False if args.lite else True
if not args.lite:
with open(fs.dirname+'/../msqlrw.txt','r') as myfile: #msql.txt is a file that contains two line: first line is username, second line is password
#This is a temporary fix, need to store the password information outside of github
login=myfile.read().replace('\n', ' ')
login_params = login.split()
fs.mysql_uname = login_params[0]
#Need to handle the case where the password is null
if len(login_params) < 2:
fs.mysql_psswrd = ""
else:
fs.mysql_psswrd = login_params[1]
return args
def GCard_Entry(UserSubmissionID,unixtimestamp,url_dir):
print("Gathering gcards from {0} ".format(url_dir))
if not 'http' in url_dir: #== fs.gcard_default:
utils.printer('Using gcard from /jlab/work')
gcard_text_db = url_dir
db_gcard_write(UserSubmissionID,unixtimestamp,gcard_text_db)
elif 'http' in url_dir:
utils.printer('Trying to download gcards from online repository')
if '.gcard' in url_dir:
utils.printer('Gcard URL name is: '+url_dir)
gcard_text = html_reader.html_reader(url_dir,'')[0]#This returns a tuple, we need the contents of the tuple
utils.printer2('HTML from gcard link is: {0}'.format(gcard_text))
gcard_text_db = gcard_text.replace('"',"'")
print("\t Gathered gcard '{0}'".format(url_dir))
db_gcard_write(UserSubmissionID,unixtimestamp,gcard_text_db)
else:
raw_html, gcard_urls = html_reader.html_reader(url_dir,fs.gcard_identifying_text)
if len(gcard_urls) == 0:
print("No gcard files found (they must end in '{0}'). Is the online repository correct?".format(fs.gcard_identifying_text))
exit()
else:
for url_ending in gcard_urls:
utils.printer('Gcard URL name is: '+url_ending)
gcard_text = html_reader.html_reader(url_dir+'/'+url_ending,'')[0]#This returns a tuple, we need the contents of the tuple
utils.printer2('HTML from gcard link is: {0}'.format(gcard_text))
gcard_text_db = gcard_text.replace('"',"'")
print("\t Gathered gcard '{0}'".format(url_ending))
db_gcard_write(UserSubmissionID,unixtimestamp,gcard_text_db)
# I don't think this block can ever be reached
else:
print('gcard not recognized as default option or online repository, please inspect scard')
exit()
def get_args_client():
utils.printer2("Getting arguments from command line")
argparser = argparse.ArgumentParser()
argparser.add_argument(
'scard',
help=
'relative path and name scard you want to submit, e.g. ../scard.txt',
nargs='?',
)
argparser.add_argument(fs.debug_short,
fs.debug_longdash,
default=fs.debug_default,
help=fs.debug_help)
argparser.add_argument(
'-l',
'--lite',
help="use -l or --lite to connect to sqlite DB, otherwise use MySQL DB",
action='store_true')
args = argparser.parse_args()
fs.DEBUG = getattr(args, fs.debug_long)
fs.use_mysql = not args.lite
if not args.lite:
with open(
fs.dirname + '/../msqlrw.txt', 'r'
) as myfile: #msql.txt is a file that contains two line: first line is username, second line is password
#This is a temporary fix, need to store the password information outside of github
login = myfile.read().replace('\n', ' ')
login_params = login.split()
fs.mysql_uname = login_params[0]
fs.mysql_psswrd = login_params[1]
return args
def GCard_Entry(BatchID, unixtimestamp, url_dir):
gcard_urls = Gather_Gcard_urls(url_dir)
for url_ending in gcard_urls:
response = urllib2.urlopen(url_dir + '/' + url_ending)
gcard_text = response.read()
utils.printer2('HTML from gcard link is: {}'.format(gcard_text))
gcard_text_db = gcard_text.replace('"', "'")
db_gcard_write(BatchID, unixtimestamp, gcard_text_db)
def Gather_Gcard_urls(url_dir):
gcard_urls = []
response = urllib2.urlopen(url_dir)
html = response.read().split(' ')
utils.printer2('HTML from webpage is: {}'.format(html))
matching_text = ".gcard"
second_qualifier = ">"
for section in html:
if matching_text in section:
sections = section.split('"')
for item in sections:
if (matching_text in item) and (second_qualifier not in item):
gcard_urls.append(item)
return gcard_urls
def Lund_Downloader(url_dir, lund_urls, lund_dir):
if len(lund_urls) == 0:
print(
"No Lund files found (they must end in '{0}'). Is the online repository correct?"
.format(fs.lund_identifying_text))
exit()
else:
for url_ending in lund_urls:
utils.printer('Lund URL name is: ' + url_ending)
lund_text = html_reader.html_reader(
url_dir + '/' + url_ending, ''
)[0] #This returns a tuple, we need the contents of the tuple
utils.printer2('HTML from lund link is: {0}'.format(lund_text))
lund_text_db = lund_text.replace(
'"', "'"
) #This isn't strictly needed but SQLite can't read " into data fields, only ' characters
print("\t Gathered lund file '{0}'".format(url_ending))
filename = lund_dir + "/" + url_ending
with open(filename, "a") as file:
file.write(lund_text_db)
def GCard_Entry(BatchID, unixtimestamp, url_dir):
print("Gathering gcards from {0} ".format(url_dir))
if url_dir == file_struct.gcard_default:
utils.printer('Using gcard from /jlab/work')
gcard_text_db = url_dir
db_gcard_write(BatchID, unixtimestamp, gcard_text_db)
elif 'https://' in url_dir:
utils.printer('Trying to download gcards from online repository')
raw_html, gcard_urls = html_reader.html_reader(
url_dir, file_struct.gcard_identifying_text)
for url_ending in gcard_urls:
utils.printer('Gcard URL name is: ' + url_ending)
gcard_text = html_reader.html_reader(
url_dir + '/' + url_ending, ''
)[0] #This returns a tuple, we need the contents of the tuple
utils.printer2('HTML from gcard link is: {0}'.format(gcard_text))
gcard_text_db = gcard_text.replace('"', "'")
print("\t Gathered gcard '{0}'".format(url_ending))
db_gcard_write(BatchID, unixtimestamp, gcard_text_db)
else:
print(
'gcard not recognized as default option or online repository, please inspect scard'
)
exit()
def handle_data(self, data):
utils.printer2("Encountered some data : {0}".format(data))
if data_identifyier in data:
urls.append(data)
def handle_endtag(self, tag):
utils.printer2("Encountered an end tag: {0}".format(tag))
def handle_starttag(self, tag, attrs):
utils.printer2("Encountered a start tag: {0}".format(tag))
def Lund_Entry(url_dir, target_dir):
""" Download or copy lund files and return the name of
the target directory.
Inputs:
-------
url_dir - A string containing the directory or path to lund file(s).
Returns:
--------
lund_dir - A string containing the name of the downloaded directory.
A few cases can occur:
1) One local file, extension will be .txt, .dat, or .lund and the string
will not contain http.
2) Several local files, no extension will be given. The string will not
contain http.
3) One web file, extension will be .txt, .dat, or .lund and the string
will contain http.
4) Many web files, no extension will be given. The string will contain http.
"""
lund_extensions = ['.dat', '.txt', '.lund']
lund_dir = target_dir
# A case used to work around not downloading for types 1/3
if url_dir == "no_download":
print('Not downloading files due to SCard type.')
return lund_dir
if os.path.exists(lund_dir):
print('Lund directory already exists, not downloading again.')
return lund_dir
# Create dir.
subprocess.call(['mkdir', '-p', lund_dir])
# Case 3/4
if 'http' in url_dir:
# Single web file
if any([ext in url_dir for ext in lund_extensions]):
lund_dir_unformatted = url_dir.split("/")
filename = lund_dir_unformatted[len(lund_dir_unformatted) - 1]
lund_text = html_reader.html_reader(url_dir, '')[
0] #This returns a tuple, we need the contents of the tuple
utils.printer2('HTML from lund link is: {0}'.format(lund_text))
lund_text_db = lund_text.replace(
'"', "'"
) #This isn't strictly needed but SQLite can't read " into data fields, only ' characters
print("\t Gathered lund file '{0}'".format(url_dir))
with open(lund_dir + "/" + filename, "a") as file:
file.write(lund_text_db)
# Web directory
else:
raw_html, lund_urls = html_reader.html_reader(
url_dir, fs.lund_identifying_text)
lund_dir_unformatted = url_dir.split("//")[1]
subprocess.call(['mkdir', '-p', lund_dir])
Lund_Downloader(url_dir, lund_urls, lund_dir)
# Case 1/2
else:
# Single local file
if any([ext in url_dir for ext in lund_extensions]):
subprocess.call(['cp', url_dir, lund_dir + '/'])
# Local directory, many files
else:
print('Downloading all files in {}'.format(url_dir))
lund_files = glob.glob(url_dir + '*')
print(lund_files)
for lf in lund_files:
if any([ext in lf for ext in lund_extensions]):
subprocess.call(['cp', lf, lund_dir + '/'])
return lund_dir