def register_vehicle(conn):
while True:
serialno = common.read_string('Serial Number', 15)
if serialno == None:
return None
if not common.exists(conn, 'vehicle', 'serial_no', serialno):
break
print('The serial number of the vehicle already exists!')
maker = common.read_string('Maker', 20)
if maker == None:
return None
model = common.read_string('Model', 20)
if model == None:
return None
year = common.read_int('Year', 0, 9999)
if year == None:
return None
color = common.read_string('Color', 10)
if color == None:
return None
while True:
vtype = common.read_string('Type')
if vtype == None:
return None
if common.exists(conn, 'vehicle_type', 'type_id', vtype):
break
print('The selected vehicle type does not exists!')
try:
curs = conn.cursor()
curs.bindarraysize = 1
curs.setinputsizes(15,20,20,int,10,int)
curs.executemany('insert into vehicle values (:1,:2,:3,:4,:5,:6)',
[(serialno,maker,model,year,color,vtype)])
curs.close()
conn.commit()
return True
except cx_Oracle.DatabaseError as e:
error, = e.args
if error.code == 1:
print('Error: The serial number of the vehicle already exists!')
elif error.code == 2291:
print('Error: The selected vehicle type does not exists!')
else:
print('Unknown error', error.code,'!')
return False
def disc():
"""Download disc, disc set, and mission icons from the DJMAX site.
An icon will be skipped if it is determined that it already exists.
Existence is checked by a simple path check.
"""
all_id_urls = (url.id.star, url.id.pop, url.id.club, url.id.mission)
all_img_urls = (url.img.star, url.img.pop, url.img.club, url.img.mission)
all_img_dirs = (path.img.star, path.img.pop, path.img.club, path.img.mission)
all_keys = (site.key.star, site.key.pop, site.key.club, site.key.mission)
all_pages = (site.pages.star, site.pages.pop, site.pages.club, site.pages.mission)
all_suffixes = (1, 4, 1, 1) # image suffix ranges
for id_url, img_url, img_dir, key, end, suffix in zip(
all_id_urls, all_img_urls, all_img_dirs, all_keys, all_pages, all_suffixes
):
images = []
for page in range(1, end + 1):
for record in urlopen_json(id_url.format(page), "Retrieving image name"):
images.append(
(record[key["image"]] + (".png" if key == site.key.mission else ""), clean(record[key["name"]]))
)
for image in images:
unclean_name = image[0]
name, ext = image[0].rsplit(".", 1)
for i in range(1, suffix + 1):
clean_name = "{}_{}.{}".format(image[1], i, ext)
if exists(img_dir + clean_name):
continue
if key == site.key.pop:
unclean_name = "{}{}.{}".format(name[:-1], i, ext)
with open(img_dir + clean_name, "wb") as f:
f.write(urlopen_image(img_url.format(unclean_name)))
print('Wrote: "{}{}"'.format(img_dir, clean_name))
def stop(conf):
if (common.exists(conf['pidfile']) == True):
if (common.kill(int(open(conf['pidfile'], "r").read())) == True):
common.delete(conf['pidfile'])
return True
common.delete(conf['pidfile'])
return False
def register_person(conn):
while True:
sin = common.read_string('SIN', 15)
if sin == None:
return None
if not common.exists(conn, 'people', 'sin', sin):
break
print('A person with this social insurance number already exists!')
name = common.read_string('Name', 20)
if name == None:
return None
height = common.read_float('Height', 0)
if height == None:
return None
weight = common.read_float('Weight', 0)
if weight == None:
return None
eyecolor = common.read_string('Eye Color', 10)
if eyecolor == None:
return None
haircolor = common.read_string('Hair Color', 10)
if haircolor == None:
return None
addr = common.read_string('Address', 50)
if addr == None:
return None
while True:
gender = common.read_string('Gender (m/f)')
if gender == None:
return None
if gender == 'm' or gender == 'f':
break
print('Please select either \'m\' for male or \'f\' for female!')
if gender == None:
return None
birthday = common.read_date('Birthday')
if birthday == None:
return None
try:
curs = conn.cursor()
curs.bindarraysize = 1
curs.setinputsizes(15,40,float,float,10,10,50,1,8)
curs.executemany('insert into people values (:1,:2,:3,:4,:5,:6,:7,:8,to_date(:9,\'yyyymmdd\'))',
[(sin,name,height,weight,eyecolor,haircolor,addr,gender,format_date(birthday))])
curs.close()
conn.commit()
return True
except cx_Oracle.DatabaseError as e:
error, = e.args
if error.code == 1:
print('Error: A person with this social insurance number already exists!')
else:
print('Unknown error', error.code,'!')
return False
def pages():
"""Generate all ranking pages and the HTML index."""
def get_str(chart):
return chart["str"].upper()
def mk_cap(mode):
return mode.capitalize()
idx = index.read()
for name in set(key for mode in (game.mode.star, game.mode.pop) for key in idx[mode]):
tabs = []
clean_name = clean(name)
if exists(path.db.star + clean_name + ".json"): tabs.append(mk_cap(game.mode.star))
if exists(path.db.nm + clean_name + ".json"): tabs.append(get_str(game.chart.nm))
if exists(path.db.hd + clean_name + ".json"): tabs.append(get_str(game.chart.hd))
if exists(path.db.mx + clean_name + ".json"): tabs.append(get_str(game.chart.mx))
if exists(path.db.ex + clean_name + ".json"): tabs.append(get_str(game.chart.ex))
if len(tabs) > 0:
_page(tabs, name, "./images/disc")
for name in (key for key in idx[game.mode.club]):
if exists(path.db.club + clean(name) + ".json"):
_page([mk_cap(game.mode.club)], name, "./images/club")
for name in (key for key in idx[game.mode.mission]):
if exists(path.db.mission + clean(name) + ".json"):
_page([mk_cap(game.mode.mission)], name, "./images/mission")
_page([mk_cap(game.mode.star), get_str(game.chart.nm), get_str(game.chart.hd), get_str(game.chart.mx), mk_cap(game.mode.pop), mk_cap(game.mode.club), mk_cap(game.mode.mission)], "Master")
_index(idx)
def addSiteValidation(self, name, endpoint, identifier, address):
"""
Confirm no site data is empty
Confirm unique name for new site
"""
messages = self.siteValidation(name, endpoint, identifier, address)
messages += common.exists(name, self.sites)
return messages
def loadconf(conf):
if (common.exists("broxy.conf")):
path = open('broxy.conf','r')
lignes = path.readlines()
for lig in lignes:
sp = lig.split('#')[0]
sp = sp.split('=')
if len(sp[1]) > 1: conf[sp[0]] = sp[1].strip()
path.close()
def addBoxValidation(self, address, host, userName):
"""
Confirm no mailbox data is empty
Confirm unique address for new mailbox
"""
messages = self.boxValidation(address, host, userName)
messages += common.exists(address, self.mailboxes)
return messages
def updateBoxValidation(self, mailboxId, address, host, userName):
"""
Confirm no mailbox data is empty
Confirm new name does not exist
"""
messages = self.boxValidation(address, host, userName)
if address in self.mailboxes:
target = self.mailboxes[address]
if target[0] != mailboxId:
messages += common.exists(address, self.mailboxes)
return messages
def local_playlist(self):
self.playlist = list()
self.get_videolist()
video_locations = list()
for location in self.enabled_locations:
for path in self.local_locations:
path = os.path.join(c.download_folder, path)
if location in path:
if not "My Videos" in location:
for quality in c.get_quality():
qual_path = c.validatePath(
os.path.join(path, quality))
beta(qual_path)
if not c.exists(qual_path + "/"):
beta("Going to download a new video.")
local_videos.Downloader(mode=1).download()
return "restart"
data = {qual_path: location}
video_locations.append(data)
else:
data = {path: location}
video_locations.append(data)
if self.len_check(video_locations):
return "restart"
else:
found = 0
for item in video_locations:
for path in item.keys():
location = item[path]
for video in c.listdir(path, files=True):
beta("Found: {}".format(video))
found += 1
if not re.search(c.excludes, video):
video_path = c.validatePath(
os.path.join(path, video))
id = os.path.splitext(video)[0]
for item in self.videolist:
if item[0] in location:
for video in item[1]:
if id in video["id"]:
self.process_video_data(
item[0], [video], video_path)
if found == 0:
return "restart"
else:
return self.playlist
def start(conf):
if (common.exists(conf['pidfile']) == False
or os.path.getsize(conf['pidfile']) == 0):
pid = os.fork()
if (pid == 0):
os.setsid()
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind(('', int(conf['port'])))
s.listen(int(conf['maxconnection']))
while (1):
conn, addr = s.accept()
if (os.fork() == 0):
client(conn, addr, conf)
else:
open(conf['pidfile'], "w").write(str(pid))
print "Daemon started"
common.log(conf, "Daemon started")
else:
print >> sys.stderr, "Daemon is already running"
def icon():
"""Download DJ icons from the DJMAX site.
Scans the local database and downloads the necessary DJ icons. An icon will
be skipped if it is determined that it already exists. Existence is checked
by a simple path check.
"""
icons = set()
for db in (path.db.star, path.db.nm, path.db.hd, path.db.mx, path.db.ex, path.db.club, path.db.mission):
for record in ls(db):
with open(db + record, "rb") as f:
data = json.loads(f.read().decode())
icons.update([dj["djicon"] for dj in data["ranking"]])
for icon in icons:
if exists(path.img.icon + icon):
continue
with open(path.img.icon + icon, "wb") as f:
f.write(urlopen_image(url.img.icon.format(icon)))
print('Wrote: "{}{}"'.format(path.img.icon, icon))
def version_exists(dependency, resolved_version):
url_version = 'https://registry.npmjs.org/{0}/-/{1}-{2}.tgz'
url_version = url_version.format(dependency, sub_name(dependency), resolved_version)
return exists(url_version)
def register_licence(conn):
while True:
licenceno = common.read_string('Licence Number', 15)
if licenceno == None:
return None
if not common.exists(conn, 'drive_licence', 'licence_no', licenceno):
break
print('The licence number already exists!')
while True:
sin = common.read_string('SIN', 15)
if sin == None:
return None
if common.exists(conn, 'drive_licence', 'sin', sin):
print('The selected person already has a licence!')
continue
if common.exists(conn, 'people', 'sin', sin):
break
print('No person with this social insurance number exists!')
lclass = common.read_string('Class', 10)
if lclass == None:
return None
while True:
upload = common.read_string('Upload Picture (y/n)')
if upload == None:
return None
if upload == 'y':
break
if upload == 'n':
image_data = None
break
print('Please select either \'y\' for yes or \'n\' for no!')
if upload == 'y':
while True:
fimage = common.read_string_exact('Picture File')
if fimage == None:
return None
if not os.path.isfile(fimage):
print('File not found!')
continue
try:
pimage = open(fimage, 'rb')
image_data = pimage.read()
pimage.close()
break
except:
print('Failed to read image file!')
continue
issuing_date = common.read_date('Issuing Date')
if issuing_date == None:
return None
issuing_date = common.format_date(issuing_date)
expiring_date = common.read_date('Expiring Date')
if expiring_date == None:
return None
expiring_date = common.format_date(expiring_date)
try:
curs = conn.cursor()
curs.bindarraysize = 1
#curs.setinputsizes(15,15,cx_Oracle.LONG_BINARY,8,8)
#curs.executemany('insert into drive_licence values (:1,:2,:3,:4,:5)',
# [(licenceno,sin,lclass,image_data,issuing_date,expiring_date)])
curs.setinputsizes(15,15,10,cx_Oracle.LONG_BINARY,8,8)
curs.executemany('insert into drive_licence values (:1,:2,:3,:4,to_date(:5,\'yyyymmdd\'),to_date(:6,\'yyyymmdd\'))',
[(licenceno,sin,lclass,image_data,issuing_date,expiring_date)])
curs.close()
conn.commit()
return True
except cx_Oracle.DatabaseError as e:
error, = e.args
if type(error) == str:
print('Unknown error', error,'!')
elif error.code == 1:
print('Error: The licence number already exists or the person already has a licence!')
elif error.code == 2291:
print('Error: No person with this social insurance number exists!')
else:
print('Unknown error', error.code,'!')
return False
def add_ticket(conn):
while True:
print("Enter the SIN of the violator")
violator_no = common.read_string('SIN', 15)
if violator_no == None:
return None
if common.exists(conn, 'people', 'sin', violator_no):
break
print('A violator with this social insurance number does not exists!')
while True:
print("Enter the SIN of the Police Officer")
office_no = common.read_string('SIN', 15)
if office_no == None:
return None
if common.exists(conn, 'people', 'sin', office_no):
break
print('A police officer with this social insurance number does not exists!')
while True:
print("Enter the Vehicle ID of the automobile used by the violator")
vehicle_id = common.read_string('serial_no', 15)
if vehicle_id == None:
return None
if common.exists(conn, 'vehicle', 'serial_no', vehicle_id):
break
print('A vehicle with that serial number does not exist!')
while True:
vtype = find_string('Type',10)
if vtype == None:
return None
if common.exists(conn, 'ticket_type', 'vtype', vtype):
break
print('The selected ticket type does not exists!')
vdate = common.read_date('Date of violation')
if vdate== None:
return None
descriptions = common.read_string('Descriptions', 1024)
if descriptions == None:
return None
place = common.read_string('Place', 20)
if place == None:
return None
ticket_no = common.latest_int_key(conn,'ticket','ticket_no')
ticket_no = (ticket_no)+1
try:
curs = conn.cursor()
curs.bindarraysize = 1
curs.setinputsizes(int,15,15,15,10,8,20,1024)
curs.executemany('insert into ticket values (:1,:2,:3,:4,:5,to_date(:6,\'yyyymmdd\'),:7,:8)',[(ticket_no,violator_no,vehicle_id,office_no,vtype,format_date(vdate),place,descriptions)])
curs.execute('select * from ticket')
curs.close()
conn.commit()
return True
except cx_Oracle.DatabaseError as e:
error, = e.args
if error.code == 1:
print('Error: A person with this social insurance number already exists!')
else:
print('Unknown error', error.code,'!')
return False
def main(argv):
''' Main Entry Point '''
args = parseArguments(argv)
logging.basicConfig(level=args.log_level)
logging.info("%s v%s", __appname__, __version__)
logging.info(args)
batch = boto3.client('batch')
samples = readSamples(args.bamsheet)
if samples is False:
return -1
vcfFiles = listFiles(args.outputfolder_s3_path, suffix='.vcf')
genotypingJobs = []
for sample in samples:
sampleName = sample['name']
vcfFile = "%s/%s.vcf" % (args.outputfolder_s3_path, sampleName)
if vcfFile not in vcfFiles:
if not exists(sample['bam']):
logging.error("%s does not exist.", sample['bam'])
continue
logging.info("Genotyping %s", sampleName)
if sample['reference'] == 'hg19':
reference_s3_path = "s3://bmsrd-ngs-repo/cohort-matcher/hg19.tar.bz2"
targets_s3_path = "s3://bmsrd-ngs-repo/cohort-matcher/hg19.cohort-matcher.bed"
elif sample['reference'] == 'GRCh37ERCC':
reference_s3_path = "s3://bmsrd-ngs-repo/cohort-matcher/GRCh37ERCC.tar.bz2"
targets_s3_path = "s3://bmsrd-ngs-repo/cohort-matcher/GRCh37ERCC.cohort-matcher.bed"
elif sample['reference'] == 'hg38':
reference_s3_path = "s3://bmsrd-ngs-repo/cohort-matcher/hg38.tar.bz2"
targets_s3_path = "s3://bmsrd-ngs-repo/NGSCheckMate/SNP_GRCh38_hg38_wChr.sorted.bed"
elif sample['reference'] == 'GRCh38ERCC':
reference_s3_path = "s3://bmsrd-ngs-repo/cohort-matcher/GRCh38ERCC.tar.bz2"
targets_s3_path = "s3://bmsrd-ngs-repo/NGSCheckMate/SNP_GRCh38_hg38_woChr.sorted.bed"
else:
logging.warn("Skipping %s due to unknown reference",
sampleName)
continue
if args.dryRun:
logging.info("Would call batch.submit_job")
else:
response = batch.submit_job(
jobName='freebayes-%s' % sampleName,
jobQueue=args.job_queue,
jobDefinition=args.job_definition,
retryStrategy={'attempts': 2},
containerOverrides={
'memory':
args.memory,
'command': [
"--sample_name", sampleName, "--bamfile_s3_path",
sample['bam'], "--targets_s3_path",
targets_s3_path, "--reference_s3_path",
reference_s3_path, "--s3_output_folder_path",
args.outputfolder_s3_path
]
})
logging.debug(response)
jobId = response['jobId']
genotypingJobs.append(jobId)
logging.info("Submitted %s jobs", len(genotypingJobs))
completed_jobs = []
failed_jobs = []
for counter, jobid in enumerate(genotypingJobs):
status = ''
while status != 'SUCCEEDED' and status != 'FAILED':
logging.info("[%d/%d] Checking job %s", counter + 1,
len(genotypingJobs), jobid)
response = batch.describe_jobs(jobs=[jobid])
status = response['jobs'][0]['status']
logging.info("Job %s state is %s", jobid, status)
if status == 'SUCCEEDED':
completed_jobs.append(jobid)
elif status == 'FAILED':
failed_jobs.append(jobid)
else:
logging.info("Sleeping 60 secs")
time.sleep(60)
logging.info("Successed: %s", len(completed_jobs))
logging.info("Failed: %s", len(failed_jobs))
def hasPages(response):
return common.exists(response, 'page') and common.exists(response, 'perPage') and common.exists(response, 'count')
def add_owner(conn):
while True:
oid = common.read_string('Owner SIN', 15)
if oid == None:
return None
if common.exists(conn, 'people', 'sin', oid):
break
print('The social insurance number does not exists!')
while True:
vid = common.read_string('Vehicle Serial Number', 15)
if vid == None:
return None
if common.exists(conn, 'vehicle', 'serial_no', vid):
break
print('The serial number of the vehicle does not exists!')
if not common.exists2(conn, 'owner', [('vehicle_id',vid),('is_primary_owner', 'y')]):
print('This is the first owner of the selected vehicle and will automatically be set as primary.')
powner = 'y'
sys.stdin.readline()
else:
while True:
powner = common.read_string('Primary Owner (y/n)')
if powner == None:
return None
if powner == 'n':
break
if powner == 'y':
if common.exists2(conn, 'owner', [('vehicle_id',vid),('is_primary_owner', 'y')]):
print('There is already a primary owner for the vehicle.')
while True:
change = common.read_string('Change primary owner (y/n)')
if change == None:
return None
if change == 'y':
break
if change == 'n':
powner == 'n'
break
print('Please select either \'y\' for yes or \'n\' for no!')
break
print('Please select either \'y\' for yes or \'n\' for no!')
try:
curs = conn.cursor()
conn.begin()
if powner == 'y':
#curs.execute('begin update owner set is_primary_owner=\'n\' where vehicle_id=?; insert into owner values (?,?,?); end',(vid,oid,vid,powner))
curs.execute('begin update owner set is_primary_owner=\'n\' where vehicle_id=\''+vid+'\'; insert into owner values (\''+oid+'\',\''+vid+'\',\''+powner+'\'); end;')
else:
curs.bindarraysize = 1
curs.setinputsizes(15,15,1)
curs.executemany('insert into owner values (:1,:2,:3)',
[(oid,vid,powner)])
curs.close()
conn.commit()
return True
except cx_Oracle.DatabaseError as e:
conn.rollback()
error, = e.args
if error.code == 1:
print('Error: The selected person is already an owner of this vehicle!')
else:
print('Unknown error', error.code,'!')
return False