def upload_results(dir): try: db = database.Database('default', autocommit=False) if not os.path.exists(dir) or not os.listdir(dir): errormsg = 'ERROR: Results directory, %s, does not exist or is empty' % dir raise upload.UploadNonFatalError(errormsg) fitsfiles = glob.glob(os.path.join(dir, "*.fits")) data = datafile.autogen_dataobj(fitsfiles) version_number = JobUploader.get_version_number(dir) hdr = header.get_header(fitsfiles) print "\tHeader parsed." cands, tempdir = candidates.get_candidates(version_number, dir) print "\tPeriodicity candidates parsed." sp_cands = sp_candidates.get_spcandidates(version_number, dir) print "\tSingle pulse candidates parsed." for c in (cands + sp_cands): hdr.add_dependent(c) diags = diagnostics.get_diagnostics(data.obs_name, data.beam_id, \ data.obstype, \ version_number, \ dir) print "\tDiagnostics parsed." header_id = hdr.upload(db) for d in diags: d.upload(db) print "\tEverything uploaded and checked successfully. header_id=%d" % \ header_id except (upload.UploadNonFatalError): exceptionmsgs = traceback.format_exception(*sys.exc_info()) errormsg = "Error while checking results!\n" errormsg += "\tResults Dir: %s\n\n" % dir errormsg += "".join(exceptionmsgs) sys.stderr.write("Error while checking results!\n") sys.stderr.write("Database transaction will not be committed.\n") sys.stderr.write("\t%s" % exceptionmsgs[-1]) # Rolling back changes. db.rollback() except (database.DatabaseConnectionError, CornellFTP.CornellFTPTimeout,\ upload.UploadDeadlockError, database.DatabaseDeadlockError), e: # Connection error while uploading. We will try again later. sys.stderr.write(str(e)) sys.stderr.write("\tRolling back DB transaction and will re-try later.\n") # Rolling back changes. db.rollback()
def gen_crypto_bib(db, confs_years, expand_crossrefs): if expand_crossrefs == False: outname = "db/crypto_crossref.bib" else: outname = "db/crypto.bib" with open(outname, "w") as out: out.write(header.get_header(config, "gen.py", confs_years)) mybibtex.generator.bibtex_gen(out, db, expand_crossrefs=expand_crossrefs, include_crossrefs=not expand_crossrefs) out.write("\n") out.write("\n") with open("db/crypto_misc.bib") as fin: for line in fin: out.write(line)
def gen(out, abbrev, short=0): def write_abbrev(d, path=[]): val = get_value(d, short=short) if val != None: key = "".join(path) out.write("@string{{{} = {}{}}}\n".format( key, " "*(max(0, 32-len(key)-11)), val )) for k,v in d.iteritems(): if k in ["", "@0", "@1", "@2", "@3"]: continue write_abbrev(v, path+[k]) out.write(header.get_header(config, "gen.py")) write_abbrev(abbrev)
def upload_results(job_submit): """ Uploads Results for a given submit. Input: job_submit: A row from the job_submits table. Results from this job submission will be uploaded. Output: None """ print "Attempting to upload results" print "\tJob ID: %d, Job submission ID: %d" % \ (job_submit['job_id'], job_submit['id']) if debug.UPLOAD: upload.upload_timing_summary = {} starttime = time.time() try: # Connect to the DB db = database.Database('SPAN512', autocommit=False) # Prepare for upload dir = job_submit['output_dir'] if not os.path.exists(dir) or not os.listdir(dir): errormsg = 'ERROR: Results directory, %s, does not exist or is empty for job_id=%d' %\ (dir, job_submit['job_id']) raise upload.UploadNonFatalError(errormsg) fitsfiles = get_fitsfiles(job_submit) data = datafile.autogen_dataobj(fitsfiles) version_number = get_version_number(dir) if debug.UPLOAD: parsetime = time.time() # Upload results hdr = header.get_header(fitsfiles) print "\tHeader parsed." rat_inst_id_cache = ratings2.utils.RatingInstanceIDCache(dbname='nancay') cands, tempdir = candidates.get_candidates(version_number, dir, \ timestamp_mjd=data.timestamp_mjd, \ inst_cache=rat_inst_id_cache) print "\tPeriodicity candidates parsed." sp_cands = sp_candidates.get_spcandidates(version_number, dir, \ timestamp_mjd=data.timestamp_mjd) print "\tSingle pulse candidates parsed." for c in (cands + sp_cands): hdr.add_dependent(c) diags = diagnostics.get_diagnostics(data.obs_name, data.beam_id, \ data.obstype, \ version_number, \ dir) print "\tDiagnostics parsed." if debug.UPLOAD: upload.upload_timing_summary['Parsing'] = \ upload.upload_timing_summary.setdefault('Parsing', 0) + \ (time.time()-parsetime) # Perform the upload header_id = hdr.upload(db) for d in diags: d.upload(db) print "\tDB upload completed and checked successfully. header_id=%d" % \ header_id except (upload.UploadNonFatalError): # Parsing error caught. Job attempt has failed! exceptionmsgs = traceback.format_exception(*sys.exc_info()) errormsg = "Error while checking results!\n" errormsg += "\tJob ID: %d, Job submit ID: %d\n\n" % \ (job_submit['job_id'], job_submit['id']) errormsg += "".join(exceptionmsgs) sys.stderr.write("Error while checking results!\n") sys.stderr.write("Database transaction will not be committed.\n") sys.stderr.write("\t%s" % exceptionmsgs[-1]) queries = [] arglists = [] queries.append("UPDATE job_submits " \ "SET status='upload_failed', " \ "details=\"%s\", " \ "updated_at='%s' " \ "WHERE id=%d"%(errormsg.replace("\"","\'"), jobtracker.nowstr(), job_submit['id'])) queries.append("UPDATE jobs " \ "SET status='failed', " \ "details='Error while uploading results', " \ "updated_at='%s' " \ "WHERE id=%d"%(jobtracker.nowstr(), job_submit['job_id'])) jobtracker.query(queries) # Rolling back changes. db.rollback() except (database.DatabaseConnectionError, \ upload.UploadDeadlockError, database.DatabaseDeadlockError), e: # Connection error while uploading. We will try again later. sys.stderr.write(str(e)) sys.stderr.write("\tRolling back DB transaction and will re-try later.\n") # Rolling back changes. db.rollback()
def upload_results(job_submit): """ Uploads Results for a given submit. Input: job_submit: A row from the job_submits table. Results from this job submission will be uploaded. Output: None """ print "Attempting to upload results" print "\tJob ID: %d, Job submission ID: %d" % \ (job_submit['job_id'], job_submit['id']) if debug.UPLOAD: upload.upload_timing_summary = {} starttime = time.time() try: # Connect to the DB db = database.Database('default', autocommit=False) # Prepare for upload dir = job_submit['output_dir'] if not os.path.exists(dir) or not os.listdir(dir): errormsg = 'ERROR: Results directory, %s, does not exist or is empty for job_id=%d' %\ (dir, job_submit['job_id']) raise upload.UploadNonFatalError(errormsg) fitsfiles = get_fitsfiles(job_submit) data = datafile.autogen_dataobj(fitsfiles) version_number = get_version_number(dir) if debug.UPLOAD: parsetime = time.time() # Upload results hdr = header.get_header(fitsfiles) print "\tHeader parsed." cands = candidates.get_candidates(version_number, dir) print "\tPeriodicity candidates parsed." sp_cands = sp_candidates.get_spcandidates(version_number, dir) print "\tSingle pulse candidates parsed." for c in (cands + sp_cands): hdr.add_dependent(c) diags = diagnostics.get_diagnostics(data.obs_name, data.beam_id, \ data.obstype, \ version_number, \ dir) print "\tDiagnostics parsed." if debug.UPLOAD: upload.upload_timing_summary['Parsing'] = \ upload.upload_timing_summary.setdefault('Parsing', 0) + \ (time.time()-parsetime) # Perform the upload header_id = hdr.upload(db) for d in diags: d.upload(db) print "\tEverything uploaded and checked successfully. header_id=%d" % \ header_id except (upload.UploadNonFatalError): # Parsing error caught. Job attempt has failed! exceptionmsgs = traceback.format_exception(*sys.exc_info()) errormsg = "Error while checking results!\n" errormsg += "\tJob ID: %d, Job submit ID: %d\n\n" % \ (job_submit['job_id'], job_submit['id']) errormsg += "".join(exceptionmsgs) sys.stderr.write("Error while checking results!\n") sys.stderr.write("Database transaction will not be committed.\n") sys.stderr.write("\t%s" % exceptionmsgs[-1]) queries = [] arglists = [] queries.append("UPDATE job_submits " \ "SET status='upload_failed', " \ "details=?, " \ "updated_at=? " \ "WHERE id=?") arglists.append((errormsg, jobtracker.nowstr(), job_submit['id'])) queries.append("UPDATE jobs " \ "SET status='failed', " \ "details='Error while uploading results', " \ "updated_at=? " \ "WHERE id=?") arglists.append((jobtracker.nowstr(), job_submit['job_id'])) jobtracker.execute(queries, arglists) # Rolling back changes. db.rollback() except (database.DatabaseConnectionError, CornellFTP.CornellFTPTimeout,\ upload.UploadDeadlockError, database.DatabaseDeadlockError), e: # Connection error while uploading. We will try again later. sys.stderr.write(str(e)) sys.stderr.write( "\tRolling back DB transaction and will re-try later.\n") # Rolling back changes. db.rollback()
def upload_results(dir): try: db = database.Database('default', autocommit=False) if not os.path.exists(dir) or not os.listdir(dir): errormsg = 'ERROR: Results directory, %s, does not exist or is empty' % dir raise upload.UploadNonFatalError(errormsg) pdm_dir = os.path.join( dir, "zerodm") if config.upload.upload_zerodm_periodicity else dir sp_dir = os.path.join( dir, "zerodm") if config.upload.upload_zerodm_singlepulse else dir fitsfiles = glob.glob(os.path.join(dir, "*.fits")) data = datafile.autogen_dataobj(fitsfiles) version_number = JobUploader.get_version_number(dir) hdr = header.get_header(fitsfiles) print "\tHeader parsed." rat_inst_id_cache = ratings2.utils.RatingInstanceIDCache( dbname='common3') #!!!! #rat_inst_id_cache = ratings2.utils.RatingInstanceIDCache(dbname='MichellePalfaCands') #cands, tempdir = candidates.get_candidates(version_number, dir) cands, tempdir = candidates.get_candidates(version_number, pdm_dir, \ timestamp_mjd=data.timestamp_mjd, \ inst_cache=rat_inst_id_cache) print "\tPeriodicity candidates parsed." #sp_cands = sp_candidates.get_spcandidates(version_number, dir) sp_cands, tempdir_sp = sp_candidates.get_spcandidates(version_number, sp_dir, \ timestamp_mjd=data.timestamp_mjd, \ inst_cache=rat_inst_id_cache) print "\tSingle pulse candidates parsed." for c in (cands + sp_cands): hdr.add_dependent(c) diags = diagnostics.get_diagnostics(data.obs_name, data.beam_id, \ data.obstype, \ version_number, \ pdm_dir, sp_dir) print "\tDiagnostics parsed." header_id = hdr.upload(db) for d in diags: d.upload(db) print "\tEverything uploaded and checked successfully. header_id=%d" % \ header_id except (upload.UploadNonFatalError): exceptionmsgs = traceback.format_exception(*sys.exc_info()) errormsg = "Error while checking results!\n" errormsg += "\tResults Dir: %s\n\n" % dir errormsg += "".join(exceptionmsgs) sys.stderr.write("Error while checking results!\n") sys.stderr.write("Database transaction will not be committed.\n") sys.stderr.write("\t%s" % exceptionmsgs[-1]) # Rolling back changes. db.rollback() except (database.DatabaseConnectionError, CornellFTP.CornellFTPTimeout,\ upload.UploadDeadlockError, database.DatabaseDeadlockError), e: # Connection error while uploading. We will try again later. sys.stderr.write(str(e)) sys.stderr.write( "\tRolling back DB transaction and will re-try later.\n") # Rolling back changes. db.rollback()
if debug: pretty_print(output) #continue try: os.mkdir('output/%s_tps' % (settings.get('mefformset'))) os.mkdir('output/%s' % (settings.get('mefformset'))) except Exception: pass alist=[] with open('output/%s_tps/%s_output.xml' % (settings.get('mefformset'),mefform), 'w') as f: f.write(get_header(settings.get("mefformset"), mefform)) for each in output.getchildren(): if 'tps' in each.attrib: alist.append(each.attrib['tps'].split('.')[-1]) f.write('\t' + etree.tostring(each) + '\n') for ptfield in ptform_xml.xpath('//field[field_attributes/cid_mapping]'): if ptfield.attrib['id'] in alist: continue altid = (ptfield.xpath("./field_attributes/image/refined_image/alternate_field_id/@value") or [''])[0] cid = (ptfield.xpath("./field_attributes/cid_mapping/formml_field_id/@value") or [''])[0] f.write("\t<tps id='{0}' type='{1}' {2} {3}/>\n".format(ptfield.attrib['id'], (ptfield.xpath("./field_attributes/main/type/@value") or [''])[0], "" if not altid else "altid='%s' " % altid.upper(), "" if not cid else "cid='%s' " % cid ))
def upload_results(job_submit): """ Uploads Results for a given submit. Input: job_submit: A row from the job_submits table. Results from this job submission will be uploaded. Output: None """ print "Attempting to upload results" print "\tJob ID: %d, Job submission ID: %d\n\tOutput Dir: %s" % \ (job_submit['job_id'], job_submit['id'], job_submit['output_dir']) if debug.UPLOAD: upload.upload_timing_summary = {} starttime = time.time() try: # Connect to the DB db = database.Database('default', autocommit=False) # Prepare for upload dir = job_submit['output_dir'] # NEW Beluga - Untar the tarball import tarfile to_keep = os.listdir(job_submit['output_dir']) tarball = glob.glob(job_submit['output_dir'] + '/*00.tgz')[0] tar = tarfile.open(tarball, 'r:gz') tar.extractall(path=job_submit['output_dir']) tar.close() all_files = os.listdir(job_submit['output_dir']) to_del = set(all_files) - set(to_keep) if config.upload.upload_zerodm_periodicity or config.upload.upload_zerodm_singlepulse: to_keep_zerodm = os.listdir(job_submit['output_dir'] + '/zerodm') tarball = glob.glob(job_submit['output_dir'] + '/zerodm/*zerodm.tgz')[0] tar = tarfile.open(tarball, 'r:gz') tar.extractall(path=job_submit['output_dir'] + '/zerodm') tar.close() all_files_zerodm = os.listdir(job_submit['output_dir'] + '/zerodm') to_del_zerodm = set(all_files_zerodm) - set(to_keep_zerodm) pdm_dir = os.path.join( dir, "zerodm") if config.upload.upload_zerodm_periodicity else dir sp_dir = os.path.join( dir, "zerodm") if config.upload.upload_zerodm_singlepulse else dir if not os.path.exists(dir) or not os.listdir(dir): errormsg = 'ERROR: Results directory, %s, does not exist or is empty for job_id=%d' %\ (dir, job_submit['job_id']) raise upload.UploadNonFatalError(errormsg) elif len(os.listdir(dir)) == 1 and os.listdir(dir)[0] == 'zerodm' \ and not os.listdir(os.path.join(dir,os.listdir(dir)[0])): errormsg = 'ERROR: Results directory, %s, does not exist or is empty for job_id=%d' %\ (dir, job_submit['job_id']) raise upload.UploadNonFatalError(errormsg) fitsfiles = get_fitsfiles(job_submit) try: data = datafile.autogen_dataobj(fitsfiles) except ValueError: raise upload.UploadNonFatalError version_number = get_version_number(dir) if debug.UPLOAD: parsetime = time.time() # Upload results hdr = header.get_header(fitsfiles) print "\tHeader parsed." rat_inst_id_cache = ratings2.utils.RatingInstanceIDCache( dbname='common3') cands, tempdir = candidates.get_candidates(version_number, pdm_dir, \ timestamp_mjd=data.timestamp_mjd, \ inst_cache=rat_inst_id_cache) print "\tPeriodicity candidates parsed. (%d cands)" % len(cands) sp_cands, tempdir_sp = sp_candidates.get_spcandidates(version_number, sp_dir, \ timestamp_mjd=data.timestamp_mjd, \ inst_cache=rat_inst_id_cache) print "\tSingle pulse candidates parsed. (%d cands)" % len(sp_cands) diags = diagnostics.get_diagnostics(data.obs_name, data.beam_id, \ data.obstype, \ version_number, \ pdm_dir, sp_dir) print "\tDiagnostics parsed." for c in (cands + sp_cands): hdr.add_dependent(c) if debug.UPLOAD: upload.upload_timing_summary['Parsing'] = \ upload.upload_timing_summary.setdefault('Parsing', 0) + \ (time.time()-parsetime) # Perform the upload header_id = hdr.upload(db) print "Header ID: ", header_id for d in diags: d.upload(db) print "\tDB upload completed and checked successfully. header_id=%d" % \ header_id except (upload.UploadNonFatalError): # Parsing error caught. Job attempt has failed! exceptionmsgs = traceback.format_exception(*sys.exc_info()) errormsg = "Error while checking results!\n" errormsg += "\tJob ID: %d, Job submit ID: %d\n\n" % \ (job_submit['job_id'], job_submit['id']) errormsg += "".join(exceptionmsgs) sys.stderr.write("Error while checking results!\n") sys.stderr.write("Database transaction will not be committed.\n") sys.stderr.write("\t%s" % exceptionmsgs[-1]) queries = [] arglists = [] queries.append("UPDATE job_submits " \ "SET status='upload_failed', " \ "details=?, " \ "updated_at=? " \ "WHERE id=?") arglists.append((errormsg, jobtracker.nowstr(), job_submit['id'])) queries.append("UPDATE jobs " \ "SET status='failed', " \ "details='Error while uploading results', " \ "updated_at=? " \ "WHERE id=?") arglists.append((jobtracker.nowstr(), job_submit['job_id'])) jobtracker.execute(queries, arglists) # Rolling back changes. db.rollback() except (database.DatabaseConnectionError, ratings2.database.DatabaseConnectionError,\ CornellFTP.CornellFTPTimeout, upload.UploadDeadlockError,\ database.DatabaseDeadlockError), e: # Connection error while uploading. We will try again later. sys.stderr.write(str(e)) sys.stderr.write( "\tRolling back DB transaction and will re-try later.\n") # Rolling back changes. db.rollback()
import os import sys import header import preprocessor for i in range(1, len(sys.argv)): if "".join(list(reversed(sys.argv[1]))[:2]) == "c." and os.path.isfile( sys.argv[1]): f = open(sys.argv[i]) content = f.read().split("\n") res = header.get_header(sys.argv[i], i) res += preprocessor.format_preprocessor(content) print(sys.argv[i][:-2]) output = open(sys.argv[i][:-2] + "_normed.c", "w+") for i in res: output.write(i + "\n") print("\n".join(res)) else: print(sys.argv[1]) print("Warning: Not a valid file")
name = x[4] try: value = x[6] except IndexError: value = x[5] # path = x[2] cookie[name] = value return cookie d = {"sessionid": "f40247e2d924e43a7dab6b56", "currency": 3, "subtotal": 1, "fee": 2, "total": 3, "quantity": 1} url = 'https://steamcommunity.com/market/buylisting/726564084060013124' h = headers = {"Accept": "text/javascript, text/html, application/xml, text/xml, */*", "Accept-Encoding": "gzip,deflate,sdch", "Accept-Language": "en-US,en;q=0.8", "Host": "steamcommunity.com", "Referer": "http://steamcommunity.com/market/listings/730/P250%20|%20Boreal%20Forest%20%28Field-Tested%29", "Origin": "http://steamcommunity.com", "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:40.0) Gecko/20100101 Firefox/40.0" } r = requests.post(url, data=d, headers=get_header("http://steamcommunity.com/market/listings/730/P250%20|" "%20Boreal%20Forest%20%28Field-Tested%29"), cookies=cookie_cutter()) print(r.status_code) print(r.text)