def delta_period(self, datas, code, timecode):
"""Consolidate datas by period
datas : Array(2)
- datetime
- value
code : string
timecode : string
Return : dict
"""
datas.sort()
old = datas[0]
result = {}
delta = 0
for data in datas:
if date.strftime(data[0], timecode) == date.strftime(old[0], timecode):
delta = delta + (data[1] - old[1])
key = "{}_{}".format(code, date.strftime(data[0], timecode))
result[key] = delta
else:
delta = delta + (data[1] - old[1])
key = "{}_{}".format(code, date.strftime(old[0], timecode))
result[key] = delta
delta = 0
old = data
return result
def generate_badge():
conn = get_conn(DB)
c = conn.cursor()
c.execute("SELECT * FROM samples ORDER BY DATE DESC LIMIT 1")
row = c.fetchone()
c.close()
date = dateparser.parse(row[0])
co2 = str(int(row[1]))
timestamp = date.strftime("%I:%M%p")
datestamp = date.strftime("%m/%d") if (datetime.now() - date).days > 0 else ""
print "Generating badge: %s - %s %s" % (co2, timestamp, datestamp)
image = Image.open("template.png")
draw = ImageDraw.ImageDraw(image)
font = ImageFont.truetype("arial.ttf", 10)
draw.setfont(font)
dw, dh = font.getsize(datestamp)
draw.text((30 - dw / 2, 78 - dh), datestamp)
tw, th = font.getsize(timestamp)
draw.text((30 - tw / 2, 77 - th - dh), timestamp)
font = ImageFont.truetype("arial.ttf", 26)
draw.setfont(font)
cw, ch = font.getsize(co2)
draw.text((30 - cw / 2, 8), co2)
image.save("webroot/badge.png")
def load_project_variants_from_vcf(project_id, vcf_files):
"""
Load any families and cohorts in this project that aren't loaded already
"""
print("Called load_project_variants_from_vcf on " + str(vcf_files))
print "Loading project %s" % project_id
print(date.strftime(datetime.now(), "%m/%d/%Y %H:%M:%S -- loading project: " + project_id + " - db.variants cache"))
project = Project.objects.get(project_id=project_id)
for vcf_file in vcf_files:
r = vcf.VCFReader(filename=vcf_file)
if "CSQ" in r.infos:
mall.get_annotator().add_preannotated_vcf_file(vcf_file)
else:
mall.get_annotator().add_vcf_file_to_annotator(vcf_file)
# batch load families by VCF file
print("project.families_by_vcf(): " + str(project.families_by_vcf()))
for vcf_file, families in project.families_by_vcf().items():
if vcf_file not in vcf_files:
print("Skipping %(vcf_file)s since its not in %(vcf_files)s" % locals())
continue
#families = [f for f in families if get_mall(project.project_id).variant_store.get_family_status(project_id, f.family_id) != 'loaded']
print("Loading families for VCF file: " + vcf_file)
for i in xrange(0, len(families), settings.FAMILY_LOAD_BATCH_SIZE):
#print(date.strftime(datetime.now(), "%m/%d/%Y %H:%M:%S -- loading project: " + project_id + " - families batch %d - %d families" % (i, len(families[i:i+settings.FAMILY_LOAD_BATCH_SIZE]))))
load_variants_for_family_list(project, families[i:i+settings.FAMILY_LOAD_BATCH_SIZE], vcf_file, mark_as_loaded=True)
print(date.strftime(datetime.now(), "%m/%d/%Y %H:%M:%S -- finished loading project: " + project_id))
def create_main_html(reqnums):
month_data = {}
lastmonth = 'none'
for i,row in reqnums.iterrows():
date = datetime.strptime(str(row[0]),'%Y-%m-%d %H:%M:%S.%f')
reqnum = trunc(row[1])
if date.strftime('%B_%Y') not in month_data.keys():
month_data[date.strftime('%B_%Y')] = [(date,reqnum)]
else:
month_data[date.strftime('%B_%Y')].append((date,reqnum))
tmp = []
iter = []
for month in month_data.keys():
iter.append(datetime.strptime(month,'%B_%Y'))
iter.sort()
for month in iter:
tmp.append(month.strftime('%B_%Y'))
iter = tmp
path = app.config["STATIC_PATH"]
archivefile = "report_archive_default.html"
archivepath = os.path.join(path,archivefile)
with open(archivepath,'w') as fh:
firsttime = 1
for month in reversed(iter):
if firsttime == 1:
fh.write("<div id='month'><h3>{month}</h3>\n".format(month=month_data[month][0][0].strftime('%B %Y')))
firsttime = 0
else:
fh.write("</div><br>\n<div id='month'><h3>{month}</h3>\n".format(month=month_data[month][0][0].strftime('%B %Y')))
for row in month_data[month]:
fh.write(" <button onclick=\"requestDoc('main','/static/reports/{rq}/report_{rq}.html')\">Report {rq}</button>\n".format(rq=row[1]))
fh.write('</div>')
def esPruneIndexes():
if options.output == 'syslog':
logger.addHandler(SysLogHandler(address=(options.sysloghostname, options.syslogport)))
else:
sh = logging.StreamHandler(sys.stderr)
sh.setFormatter(formatter)
logger.addHandler(sh)
logger.debug('started')
try:
es = pyes.ES((list('{0}'.format(s) for s in options.esservers)))
indices = es.indices.stats()['indices'].keys()
# do the pruning
for (index, dobackup, rotation, pruning) in zip(options.indices,
options.dobackup, options.rotation, options.pruning):
try:
if pruning != '0':
index_to_prune = index
if rotation == 'daily':
idate = date.strftime(datetime.utcnow()-timedelta(days=int(pruning)),'%Y%m%d')
index_to_prune += '-%s' % idate
elif rotation == 'monthly':
idate = date.strftime(datetime.utcnow()-timedelta(days=31*int(pruning)),'%Y%m')
index_to_prune += '-%s' % idate
if index_to_prune in indices:
logger.info('Deleting index: %s' % index_to_prune)
es.indices.delete_index(index_to_prune)
else:
logger.error('Error deleting index %s, index missing' % index_to_prune)
except Exception as e:
logger.error("Unhandled exception while deleting %s, terminating: %r" % (index_to_prune, e))
except Exception as e:
logger.error("Unhandled exception, terminating: %r"%e)
def writeCaption(date, headline):
#Generate the text of the Tumblr post based on the Guadian Headline
#headline = getGuardianHeadline(date + timedelta(1))
if len(headline) > 0:
title = headline['fields']['headline'].encode('utf_8')
if 'trailText' in headline['fields']:
standfirst = headline['fields']['trailText']
else:
standfirst = headline['fields']['standfirst']
standfirst = standfirst.encode('utf_8')
webUrl = headline['webUrl']
#[FIXME] Noticed that trailText can contain html code, it's usually <strong>, do I leave it?
else:
title = "Nothing of importance happened"
standfirst = ""
webUrl = "#"
formattedLink = '<a href="{link}">{headline}</a><br/>{standfirst}'.format(link=webUrl, headline=title, standfirst=standfirst)
caption ="<p>On this great day of {date} {earth}.</p> We note that:<br/>{link}"
tweet = "On {date} {earth}:"
earth = random.choice(config.EARTH_METAPHORS)
# Idea if no data for that date
#caption ="<p>The world was nowhere to be found on {date}</p>But we can note that:<br/>{link}"
#tweet = "Yesterday the world was nowhere to be found. {link}"
caption = caption.format(date=date.strftime(config.DATE_LONGFORM),link=formattedLink, earth=earth)
tweet = tweet.format(date=date.strftime(config.DATE_LONGFORM), earth=earth)
tweet = tweet + ' {link}'
return {'caption':caption, 'tweet':tweet}
def clearESCache():
es=esConnect(None)
indexes=es.indices.status()['indices']
# assums index names like events-YYYYMMDD etc.
# used to avoid operating on current indexes
dtNow = datetime.utcnow()
indexSuffix = date.strftime(dtNow, '%Y%m%d')
previousSuffix = date.strftime(dtNow - timedelta(days=1), '%Y%m%d')
for targetindex in sorted(indexes.keys()):
if indexSuffix not in targetindex and previousSuffix not in targetindex:
url = 'http://{0}/{1}/_stats'.format(random.choice(options.esservers), targetindex)
r = requests.get(url)
if r.status_code == 200:
indexstats = json.loads(r.text)
if indexstats['_all']['total']['search']['query_current'] == 0:
fielddata = indexstats['_all']['total']['fielddata']['memory_size_in_bytes']
if fielddata > 0:
logger.info('target: {0}: field data {1}'.format(targetindex, indexstats['_all']['total']['fielddata']['memory_size_in_bytes']))
clearurl = 'http://{0}/{1}/_cache/clear'.format(random.choice(options.esservers), targetindex)
clearRequest = requests.post(clearurl)
logger.info(clearRequest.text)
if options.conservative:
return
else:
logger.debug('{0}: <ignoring due to current search > field data {1}'.format(targetindex, indexstats['_all']['total']['fielddata']['memory_size_in_bytes']))
else:
logger.error('{0} returned {1}'.format(url, r.status_code))
def __source_v2_xls_to_csv_single(self, src_dir, dest_dir, date):
src_file = self.get_filename(src_dir, date, 'xls')
assert os.path.isfile(src_file)
if date >= datetime(2007, 4, 1).date() or date < datetime(2000, 9, 1).date():
return
book = xlrd.open_workbook(src_file)
sheet = book.sheet_by_index(0)
assert sheet.ncols is 21
assert sheet.cell(4, 0).value.strip() in ('Code & Name', 'CODE & NAME')
assert sheet.cell(4, 11).value.strip() in ('Code & Name', 'CODE & NAME')
assert sheet.cell(4, 8).value.strip() in ('PBR', 'в▐в╨вр')
assert sheet.cell(4, 19).value.strip() in ('PBR', 'в▐в╨вр')
dest_file = self.get_filename(dest_dir, date, 'csv')
fd = open(dest_file, 'w', newline='')
csv_writer = csv.writer(fd)
for r in self.__build_sheet_records(sheet, 0, 5):
r = [date.strftime('%Y-%m-%d')] + r
assert len(r) is 6
csv_writer.writerow(r)
self.LOGGER.debug('''%s => %s''' % (r, dest_file))
for r in self.__build_sheet_records(sheet, 11, 5):
r = [date.strftime('%Y-%m-%d')] + r
assert len(r) is 6
csv_writer.writerow(r)
self.LOGGER.debug('''%s => %s''' % (r, dest_file))
fd.close()
def write_org(out=ORG_FNAME):
a,b,c = read_index(FUTURE_FNAME)
lines = []
def w(s):
lines.append(s+"\n")
w('* Schedule')
for date, descr, time, location in parse_table(b):
date_str = date.strftime("%Y-%m-%d %a")
year = date.strftime("(%Y)")
parsed_time = parse_time(time)
if in_past(date, parsed_time):
continue
if parsed_time:
(start_hr, start_min), (end_hr, end_min) = parsed_time
if (start_hr, start_min) == (end_hr, end_min):
timestamp = "<%s %s:%s>" % (date_str, start_hr, start_min)
else:
timestamp = "<%s %s:%s>--<%s %s:%s>" % (
date_str, start_hr, start_min, date_str, end_hr, end_min)
else:
timestamp = "<%s>" % date_str
w("** %s (%s)" % (descr, location.replace(" %s" % year, "")))
w(" %s" % timestamp)
w("")
open(out, "w").writelines(lines)
def view_pregnancy(req, **flts):
filters = {
"period": default_period(req),
"location": default_location(req),
"province": default_province(req),
"district": default_district(req),
}
pregs = matching_reports(req, filters).filter(type=ReportType.objects.get(name="Pregnancy"))
lxn, crd = location_name(req), map_pointers(req, filters["location"], filters)
ans = [
{"label": "Total Pregnancy Reports", "id": "allpreg", "number": len(pregs)},
{"label": "High Risk Pregnancies", "id": "hrpreg", "number": len(fetch_high_risky_preg(pregs))},
{"label": "Pregnant Wives without Toilet", "id": "notoi", "number": len(fetch_without_toilet(pregs))},
{"label": "Pregnant Wives without Hand Washing", "id": "nohw", "number": len(fetch_without_hw(pregs))},
]
return render_to_response(
req,
"ubuzima/pregnancy.html",
{
"track": ans,
"filters": filters,
"start_date": date.strftime(filters["period"]["start"], "%d.%m.%Y"),
"end_date": date.strftime(filters["period"]["end"], "%d.%m.%Y"),
"locationname": lxn,
"coords": crd,
"postqn": (req.get_full_path().split("?", 2) + [""])[1],
},
)
def view_death(req, **flts):
filters = {
"period": default_period(req),
"location": default_location(req),
"province": default_province(req),
"district": default_district(req),
}
reps = matching_reports(req, filters)
chi_deaths = fetch_child_death(reps)
mo_deaths = fetch_maternal_death(reps)
lxn, crd = location_name(req), map_pointers(req, filters["location"], filters)
ans = [
{"label": "Maternal Death", "id": "matde", "number": len(mo_deaths)},
{"label": "Child Death", "id": "chide", "number": len(chi_deaths)},
]
return render_to_response(
req,
"ubuzima/death.html",
{
"track": ans,
"stat": {"matde": len(mo_deaths), "chide": len(chi_deaths)},
"filters": filters,
"start_date": date.strftime(filters["period"]["start"], "%d.%m.%Y"),
"end_date": date.strftime(filters["period"]["end"], "%d.%m.%Y"),
"locationname": lxn,
"coords": crd,
"postqn": (req.get_full_path().split("?", 2) + [""])[1],
},
)
def view_chihe(req, **flts):
filters = {
"period": default_period(req),
"location": default_location(req),
"province": default_province(req),
"district": default_district(req),
}
chihe_reps = (
matching_reports(req, filters).filter(type=ReportType.objects.get(name="Child Health")).select_related("fields")
)
vac_chihe_reps = fetch_vaccinated_stats(fetch_vaccinated_info(chihe_reps))
lxn, crd = location_name(req), map_pointers(req, filters["location"], filters)
ans = []
for v in vac_chihe_reps.keys():
ans.append({"label": "Children vaccinated with %s" % v, "id": "%s" % v, "number": len(vac_chihe_reps[v])})
ans.append({"label": "ALL Children vaccinated ", "id": "all", "number": len(fetch_vaccinated_info(chihe_reps))})
return render_to_response(
req,
"ubuzima/chihe.html",
{
"track": ans,
"filters": filters,
"start_date": date.strftime(filters["period"]["start"], "%d.%m.%Y"),
"end_date": date.strftime(filters["period"]["end"], "%d.%m.%Y"),
"locationname": lxn,
"coords": crd,
"postqn": (req.get_full_path().split("?", 2) + [""])[1],
},
)
def view_stats(req, **flts):
filters = {
"period": default_period(req),
"location": default_location(req),
"province": default_province(req),
"district": default_district(req),
}
track = get_stats_track(req, filters)
hindics = health_indicators(req, filters)
stt = filters["period"]["start"]
fin = filters["period"]["end"]
lox, lxn, crd = 0, location_name(req), map_pointers(req, filters["location"], filters)
if req.REQUEST.has_key("location") and req.REQUEST["location"] != "0":
lox = int(req.REQUEST["location"])
lxn = Location.objects.get(id=lox)
return render_to_response(
req,
"ubuzima/stats.html",
{
"track": track,
"filters": filters,
"hindics": paginated(req, hindics, prefix="hind"),
"start_date": date.strftime(filters["period"]["start"], "%d.%m.%Y"),
"end_date": date.strftime(filters["period"]["end"], "%d.%m.%Y"),
"coords": crd,
"location": lox,
"locationname": lxn,
"chosen": the_chosen(req.REQUEST),
"important": get_important_stats(req, filters),
"targets": HealthTarget.objects.all(),
"postqn": (req.get_full_path().split("?", 2) + [""])[1],
},
)
def reports(request):
clinics = Provider.objects.values('clinic__id','clinic__name').distinct()
zones = Case.objects.order_by("zone").values('zone', 'zone__name').distinct()
p = Case.objects.order_by("zone").values('provider', 'provider__user__first_name', 'provider__user__last_name', 'zone').distinct()
providers = []
for provider in p:
tmp = {}
tmp['id'] = provider['provider']
tmp['name'] = provider['provider__user__last_name'] + " " + provider['provider__user__first_name']
tmp['zone'] = provider['zone']
providers.append(tmp)
now = datetime.today()
first = Case.objects.order_by('created_at')[:1][0]
date = first.created_at
months=[]
while ((now - date).days > 0):
months.append({'id': date.strftime("%m%Y"), 'label': date.strftime("%B %Y"), 'date': date})
date = next_month(date)
context = {
"app": app,
"clinics": clinics,
"providers": providers,
"zones": zones,
"months": months
}
return as_html(request, "reports/reports.html", context)
def format_date(x, pos=None):
date = dates_list[int(x)]
if date.month % 2 == 0:
return ''
if date.month == 1:
return date.strftime('%b %Y')
else: return date.strftime('%b')
def main():
parser = OptionParser()
parser.add_option("-d", "--db", dest="db_file", help="SMS DB File [default: %default]")
parser.add_option("-s", "--start", dest="starttime", help="DD-MM-YY date to start at [default: %default]")
parser.add_option("-e", "--end", dest="endtime", help="DD-MM-YY date to end at [default: %default]")
parser.add_option("-n", "--number", dest="number", help="Phone # to search with - must match AddressBook.app format exactly, in quotes - e.g.: \"(888) 555-1212\"")
parser.add_option("-c", "--chunksize", dest="chunksize", type="int", help="What size chunk to break the period into, in seconds [default: %default (1 hour = 3600 seconds)]")
parser.add_option("-o", "--output", dest="output", help="What kind of output format? - Console, CSV, TSV [default: %default]")
parser.add_option("-D", "--dateformat", dest="dateformat", help="stftime style date format [default: %default] (e.g. 'Wed 11AM')")
parser.set_defaults(db_file="sms.db")
parser.set_defaults(starttime=date.strftime(date.today() + timedelta(days=-7), "%d-%m-%y"))
parser.set_defaults(endtime=date.strftime(date.today(), "%d-%m-%y"))
parser.set_defaults(chunksize=43200)
parser.set_defaults(output="Console")
parser.set_defaults(dateformat="%m/%d/%y - %p")
(options, args) = parser.parse_args()
if not options.number:
print "Missing phone number, option REQUIRED."
print "Please provide the '-n' or '--number' argument. See '-h' for help."
sys.exit(2)
# Collect all the SMS data
data = runsearch(options.db_file, options.starttime, options.endtime, options.number, options.chunksize)
# Pass SMS raw data to be formatted for the user-specified output
output(data, options.output, options.dateformat)
# Exit normally
sys.exit()
def text2elasticsearch(court, case, date):
match_filename = date.strftime('%Y-%m-%d')+'_'+court+'_'+case+'_*'
#print match_filename
for file_name in os.listdir(data_path):
if fnmatch.fnmatch(file_name, match_filename):
#utf8_text = open(data_path+file_name, 'r').read().replace('\r','').decode('utf8')
#print file_name
f = open(data_path+file_name, 'r')
line = f.readline().strip(' \t\n\r').decode('utf8')
#utf8_id = replace_chinese_numbers(line)
first_digit_in_string = re.search('\d', line)
if first_digit_in_string is None:
line = f.readline().strip(' \t\n\r').decode('utf8')
first_digit_in_string = re.search('\d', line)
if first_digit_in_string is None:
utf8_id = file_name+'_chineseNumberFailure'
failed_list.write(utf8_id+'\n')
else:
utf8_id = line[first_digit_in_string.start():]
utf8_court = courts[court].decode('utf8')
utf8_case = cases[case].decode('utf8')
utf8_text = open(data_path+file_name, 'r').read().decode('utf8')
doc = {
'court':utf8_court,
'case':utf8_case,
'date':date.strftime('%Y-%m-%d'),
'text':utf8_text
}
#print doc['text']
#es.index(index = 'lawper', doc_type = 'raw_text', id = utf8_id, body = doc)
es.index(index = 'lawper', doc_type = 'crash', id = utf8_id, body = doc)
print file_name, utf8_id
def generate_formats(date):
year_full = date.strftime("%Y")
year_short = date.strftime("%y")
month_full = date.strftime("%m")
month_short = month_full.lstrip("0")
day_full = date.strftime("%d")
day_short = day_full.lstrip("0")
date_formats = ["Mdyy", "MMdyy", "Mddyy", "MMddyy", "Mdyyyy", "MMdyyyy", "Mddyyyy", "MMddyyyy", "dMyy", "dMMyy", "ddMyy", "ddMMyy", "dMyyyy", "dMMyyyy", "ddMyyyy", "ddMMyyyy"]
formatted_dates = []
for df in date_formats:
cp = deepcopy(df)
cp = cp.replace("MM", month_full)
cp = cp.replace("M", month_short)
cp = cp.replace("yyyy", year_full)
cp = cp.replace("yy", year_short)
cp = cp.replace("dd", day_full)
cp = cp.replace("d", day_short)
formatted_dates.append(cp)
return formatted_dates
def esPruneIndexes():
logger.debug('started')
try:
es = ElasticsearchClient((list('{0}'.format(s) for s in options.esservers)))
indices = es.get_indices()
# do the pruning
for (index, dobackup, rotation, pruning) in zip(options.indices, options.dobackup, options.rotation, options.pruning):
try:
if pruning != '0':
index_to_prune = index
if rotation == 'daily':
idate = date.strftime(toUTC(datetime.now()) - timedelta(days=int(pruning)), '%Y%m%d')
index_to_prune += '-%s' % idate
elif rotation == 'monthly':
idate = date.strftime(datetime.utcnow() - timedelta(days=31 * int(pruning)), '%Y%m')
index_to_prune += '-%s' % idate
if index_to_prune in indices:
logger.debug('Deleting index: %s' % index_to_prune)
es.delete_index(index_to_prune, True)
else:
logger.error('Error deleting index %s, index missing' % index_to_prune)
except Exception as e:
logger.error("Unhandled exception while deleting %s, terminating: %r" % (index_to_prune, e))
except Exception as e:
logger.error("Unhandled exception, terminating: %r" % e)
def main():
if options.output=='syslog':
logger.addHandler(SysLogHandler(address=(options.sysloghostname,options.syslogport)))
else:
sh=logging.StreamHandler(sys.stderr)
sh.setFormatter(formatter)
logger.addHandler(sh)
logger.debug('started')
#logger.debug(options)
try:
es=pyes.ES((list('{0}'.format(s) for s in options.esservers)))
boto.connect_cloudtrail(aws_access_key_id=options.aws_access_key_id,aws_secret_access_key=options.aws_secret_access_key)
#capture the time we start running so next time we catch any files created while we run.
lastrun=toUTC(datetime.now()).isoformat()
#in case we don't archive files..only look at today and yesterday's files.
yesterday=date.strftime(datetime.utcnow()-timedelta(days=1),'%Y/%m/%d')
today = date.strftime(datetime.utcnow(),'%Y/%m/%d')
for region in boto.cloudtrail.regions():
logger.debug('connecting to AWS region {0}'.format(region.name))
ct=boto.cloudtrail.connect_to_region(region.name,aws_access_key_id=options.aws_access_key_id,aws_secret_access_key=options.aws_secret_access_key)
trails=ct.describe_trails()['trailList']
for trail in trails:
s3 = boto.connect_s3(aws_access_key_id=options.aws_access_key_id,aws_secret_access_key=options.aws_secret_access_key)
ctbucket=s3.get_bucket(trail['S3BucketName'])
#ctbucket.get_all_keys()
filelist=list()
for bfile in ctbucket.list():
if 'CloudTrail' in bfile.key and 'json' in bfile.key:
if today in bfile.key or yesterday in bfile.key:
filelist.append(bfile.key)
else:
if options.purge: #delete old files so we don't try to keep reading them.
s3file=ctbucket.get_key(afile)
s3file.delete()
for afile in filelist:
s3file=ctbucket.get_key(afile)
logger.debug('{0} {1}'.format(afile,s3file.last_modified))
if toUTC(s3file.last_modified)>options.lastrun:
compressedData=s3file.read()
databuf=StringIO(compressedData)
f=gzip.GzipFile(fileobj=databuf)
jlog=json.loads(f.read())
try:
for r in jlog['Records']:
r['utctimestamp']=toUTC(r['eventTime']).isoformat()
jbody=json.dumps(r)
res=es.index(index='events',doc_type='cloudtrail',doc=jbody)
#logger.debug(res)
except Exception as e:
logger.error('Error handling log record {0} {1}'.format(r, e))
continue
setConfig('lastrun',lastrun,options.configfile)
except boto.exception.NoAuthHandlerFound:
logger.error("No auth handler found, check your credentials")
except Exception as e:
logger.error("Unhandled exception, terminating: %r"%e)
def generate_graph_object(select_lists):
"""Generates moderation and subscription data objects
to be sent to dashboard for graph generation.
"""
sub_objects = {}
mod_objects = {}
today = datetime.today().date()
dates = []
for i in range(0, 31):
dates.append(today + timedelta(days=-i))
for date in dates:
mod_count = TaskCalender.objects.filter(list_id__in=select_lists).filter(log_type="moderation").filter(on_date=date).count()
sub_count = TaskCalender.objects.filter(list_id__in=select_lists).filter(log_type="subscription").filter(on_date=date).count()
if mod_count == 0:
mod_objects[date.strftime("%Y-%m-%d")] = 0
elif mod_count > 0:
mod_objects[date.strftime("%Y-%m-%d")] = mod_count
if sub_count == 0:
sub_objects[date.strftime("%Y-%m-%d")] = 0
elif sub_count > 0:
sub_objects[date.strftime("%Y-%m-%d")] = sub_count
# Sort The Data according to occurence date
sub_objects = collections.OrderedDict(sorted(sub_objects.items()))
mod_objects = collections.OrderedDict(sorted(mod_objects.items()))
return sub_objects, mod_objects
def tree_read(tile, date = datetime.date(2001,1,1)):
date = datetime.date(date.year,1,1)
src = treeRaw + tile + '/'
flnm = src + 'MOD44B.A' + date.strftime('%Y') + '065.' + tile + '.005.hdf'
dst = treePath + tile + '/'
flnm_n = dst + 'MOD44B.' + date.strftime('%Y') + tile + '.npy'
if os.path.isfile(flnm_n):
return np.load(flnm_n)
if not os.path.exists(dst):
os.makedirs(dst)
print 'Transferring treecover data'
if os.path.isfile(flnm):
temp = gdal.Open('HDF4_EOS:EOS_GRID:"' + flnm + '":MOD44B_250m_GRID:Percent_Tree_Cover', gdal.GA_ReadOnly).ReadAsArray().astype(np.float)
temp[temp>100] = np.nan
tree = rebin(temp, [2400,2400])
#np.save(flnm_n, tree)
else:
myfile = open(tile + 'treemissing.txt', 'a')
myfile.write(flnm + '; occurred for 0 times.\n')
myfile.close
tree = np.empty([2400,2400])
tree[:] = np.nan
return tree
def load_project_datastore(project_id, vcf_files=None, start_from_chrom=None, end_with_chrom=None):
"""
Load this project into the project datastore
Which allows queries over all variants in a project
"""
print(date.strftime(datetime.now(), "%m/%d/%Y %H:%M:%S -- starting load_project_datastore: " + project_id))
project = Project.objects.get(project_id=project_id)
get_project_datastore(project_id).delete_project_store(project_id)
get_project_datastore(project_id).add_project(project_id)
for vcf_file in sorted(project.get_all_vcf_files(), key=lambda v:v.path()):
vcf_file_path = vcf_file.path()
if vcf_files is not None and vcf_file_path not in vcf_files:
print("Skipping - %(vcf_file_path)s is not in %(vcf_files)s" % locals())
project_indiv_ids = [i.indiv_id for i in project.get_individuals()]
vcf_ids = vcf_file.sample_id_list()
indiv_id_list = [i for i in project_indiv_ids if i in vcf_ids]
get_project_datastore(project_id).add_variants_to_project_from_vcf(
vcf_file.file_handle(),
project_id,
indiv_id_list=indiv_id_list,
start_from_chrom=start_from_chrom,
end_with_chrom=end_with_chrom
)
get_project_datastore(project_id).set_project_collection_to_loaded(project_id)
print(date.strftime(datetime.now(), "%m/%d/%Y %H:%M:%S -- load_project_datastore: " + project_id + " is done!"))
def esPruneIndexes():
es = pyes.ES((list('{0}'.format(s) for s in options.esservers)))
indexes = es.indices.stats()['indices'].keys()
# print('[*]\tcurrent indexes: {0}'.format(indexes))
# set index names events-YYYYMMDD, alerts-YYYYMM, etc.
dtNow = datetime.utcnow()
targetSuffix=date.strftime(dtNow- timedelta(days=options.days),'%Y%m%d')
# rotate daily
eventsIndexName = 'events-{0}'.format(targetSuffix)
# rotate monthly
targetSuffix = date.strftime(dtNow- timedelta(days=options.months*30), '%Y%m')
alertsIndexName = 'alerts-{0}'.format(targetSuffix)
correlationsIndexName = 'correlations-{0}'.format(targetSuffix)
print('[*]\tlooking for old indexes: {0},{1},{2}'.format(
eventsIndexName,
alertsIndexName,
correlationsIndexName))
if eventsIndexName in indexes:
print('[*]\tdeleting: {0}'.format(eventsIndexName))
es.indices.delete_index(eventsIndexName)
if alertsIndexName in indexes:
print('[*]\tdeleting: {0}'.format(alertsIndexName))
es.indices.delete_index(alertsIndexName)
if correlationsIndexName in indexes:
print('[*]\tdeleting: {0}'.format(correlationsIndexName))
es.indices.delete_index(correlationsIndexName)
def load_project_variants_from_vcf(project_id, vcf_files, mark_as_loaded=True, start_from_chrom=None, end_with_chrom=None):
"""
Load any families and cohorts in this project that aren't loaded already
Args:
project_id: the project id as a string
vcf_files: a list of one or more vcf file paths
"""
print("Called load_project_variants_from_vcf on " + str(vcf_files))
print(date.strftime(datetime.now(), "%m/%d/%Y %H:%M:%S -- loading project: " + project_id + " - db.variants cache"))
project = Project.objects.get(project_id=project_id)
for vcf_file in vcf_files:
r = vcf.VCFReader(filename=vcf_file)
if "CSQ" not in r.infos:
raise ValueError("VEP annotations not found in VCF: " + vcf_file)
if vcf_file in vcf_files:
mall.get_annotator().add_preannotated_vcf_file(vcf_file)
# batch load families by VCF file
print("project.families_by_vcf(): " + str(project.families_by_vcf()))
for vcf_file, families in project.families_by_vcf().items():
if vcf_file not in vcf_files:
print("Skipping %(vcf_file)s since its not in %(vcf_files)s" % locals())
continue
#families = [f for f in families if get_mall(project.project_id).variant_store.get_family_status(project_id, f.family_id) != 'loaded']
print("Loading families for VCF file: " + vcf_file)
for i in xrange(0, len(families), settings.FAMILY_LOAD_BATCH_SIZE):
#print(date.strftime(datetime.now(), "%m/%d/%Y %H:%M:%S -- loading project: " + project_id + " - families batch %d - %d families" % (i, len(families[i:i+settings.FAMILY_LOAD_BATCH_SIZE]))))
load_variants_for_family_list(project, families[i:i+settings.FAMILY_LOAD_BATCH_SIZE], vcf_file, mark_as_loaded=mark_as_loaded, start_from_chrom=start_from_chrom, end_with_chrom=end_with_chrom)
print(date.strftime(datetime.now(), "%m/%d/%Y %H:%M:%S -- finished loading project: " + project_id))
def get_id_period (self, date):
from_iso_dt, to_iso_dt = util.inc_dt(date.strftime(util.ISO8601_DATE), util.ISO8601_DATE, self.PERIOD_TYPE)
from_dt = util.get_dt(from_iso_dt, util.ISO8601_DATE)
to_dt = util.get_dt(to_iso_dt, util.ISO8601_DATE)
response = self.br.open(self.search_url)
fields = {}
date = date.strftime(self.request_date_format)
date_parts = date.split('/')
#fields [self.date_from_field['day']] = date_parts[0]
fields [self.date_field_from['month']] = date_parts[1]
fields [self.date_field_from['year']] = date_parts[2]
util.setup_form(self.br, self.search_form, fields )
if self.DEBUG: print self.br.form
response = util.submit_form(self.br)
final_result = []
if response:
html = response.read()
url = response.geturl()
result = scrapemark.scrape(self.scrape_ids, html, url)
if result and result.get('records'):
self.clean_ids(result['records'])
final_result.extend(result['records'])
if final_result:
return final_result, from_dt, to_dt
else:
return [], None, None # monthly scraper - so empty result is always invalid
def MonthDelta(date, months):
month = int(date.strftime("%m")) - months
while month <= 0:
month += 12
year = int(date.strftime("%Y")) - 1
date = date.replace(year=year)
return date.replace(month=month)
def load_project_variants(project_id, force_annotations=False, ignore_csq_in_vcf=False):
"""
Load any families and cohorts in this project that aren't loaded already
"""
print "Loading project %s" % project_id
print(date.strftime(datetime.now(), "%m/%d/%Y %H:%M:%S -- loading project: " + project_id + " - db.variants cache"))
project = Project.objects.get(project_id=project_id)
for vcf_obj in project.get_all_vcf_files():
r = vcf.VCFReader(filename=vcf_obj.path())
if not ignore_csq_in_vcf and "CSQ" not in r.infos:
raise ValueError("VEP annotations not found in VCF: " + vcf_file)
mall.get_annotator().add_preannotated_vcf_file(vcf_obj.path(), force=force_annotations)
# batch load families by VCF file
for vcf_file, families in project.families_by_vcf().items():
families = [f for f in families if get_mall(project.project_id).variant_store.get_family_status(project_id, f.family_id) != 'loaded']
for i in xrange(0, len(families), settings.FAMILY_LOAD_BATCH_SIZE):
print(date.strftime(datetime.now(), "%m/%d/%Y %H:%M:%S -- loading project: " + project_id + " - families batch %d - %d families" % (i, len(families[i:i+settings.FAMILY_LOAD_BATCH_SIZE])) ))
load_variants_for_family_list(project, families[i:i+settings.FAMILY_LOAD_BATCH_SIZE], vcf_file)
# now load cohorts
load_cohorts(project_id)
def send_email(score):
msg = Message(
'New score submitted by {name} at {datetime} for a {round} on {date}: {score}'.format(
name=score.archer.get_name(), date=date.strftime(score.date, '%d/%m/%Y'), score=score.score,
round=score.round.name, datetime=datetime.strftime(datetime.now(), '%d/%m/%Y %H:%M:%S')),
recipients=app.config['MAIL_RECORDS']
)
msg.html = '{name} has submitted a score:<br/><br/>' \
'{round} on {date} as {category}<br/>' \
'Score: {score}, hits: {hits}, golds: {golds}, Xs: {xs}<br/><br/>' \
'CSV format:<br/>' \
'<pre>{name_csv},{cat_csv},{bow_csv},{round_csv},{date_csv},{event_csv},{score_csv},{hits_csv},' \
'{golds_csv},{xs_csv},</pre><br/>' \
'To approve the score click <a href="{approve}">here</a> or to reject click ' \
'<a href="{reject}">this link</a><br/><br/>' \
'This email was automatically generated, please don\'t reply.' \
.format(name=score.archer.get_name(), round=score.round.name, date=date.strftime(score.date, '%d/%m/%Y'),
category=score.category, score=score.score, hits=score.num_hits, golds=score.num_golds,
xs=score.num_xs or 'N/A', name_csv=score.archer.get_name(),
cat_csv=condense_category(score.archer.gender, score.category), bow_csv=score.bow.name,
round_csv=score.round.name, date_csv=date.strftime(score.date, '%d/%m/%Y'), event_csv=score.event.name,
score_csv=score.score, hits_csv=score.num_hits, golds_csv=score.num_golds, xs_csv=score.num_xs or '',
approve=app.config['SITE_URL'] + url_for('admin.approve_score', score_id=score.id),
reject=app.config['SITE_URL'] + url_for('admin.reject_score', score_id=score.id))
thread = Thread(target=send_async_email, args=[msg])
thread.start()
def cover_read(tile, date = datetime.date(2001,1,1), forced = False, detect = False):
date = datetime.date(date.year,1,1)
src = coverRaw + date.strftime('%Y') + '.01.01/'
if not os.path.exists(src): #date out of bound
return False
dst = coverPath + tile + '/'
flnm_n = dst + 'MCD12Q1.' + date.strftime('%Y') + tile + '.npy' #file name at destination
if not forced and not detect: #if not foreced updated then return previous stored data
if os.path.isfile(flnm_n):
return np.load(flnm_n)
flnm = src + 'MCD12Q1.A' + date.strftime('%Y') + '001.' + tile + '.051.hdf'
if os.path.isfile(flnm):
if detect:
return True
cover = gdal.Open('HDF4_EOS:EOS_GRID:"' + flnm + '":MOD12Q1:Land_Cover_Type_1', gdal.GA_ReadOnly).ReadAsArray().astype(np.int)
cover[cover>16] = len(veg_dict) #unclassified as 17
if not os.path.exists(dst): #new directory
os.makedirs(dst)
np.save(flnm_n, cover)
return cover
if cover_read(tile, date = date + relativedelta(years=1), detect = detect) is not False: #tile exis in next year
myfile = open(tile + 'covermissing.txt', 'a')
myfile.write(flnm + '\n')
myfile.close
return cover_read(tile, date = date + relativedelta(years=1), detect = detect)
def format_date(date): # date = datetime object.
return date.strftime('%Y-%m-%d %H:%M:%S')
def generate_preventive_check(self, contracts):
issue_obj = self.env['project.issue']
invoice_obj = self.env['account.invoice']
account_payment_term_obj = self.env['account.payment.term']
currency_obj = self.env['res.currency']
ctx = dict(self._context)
invoices_list = []
for contract in contracts:
account_id = contract.product_id.property_account_income.id or contract.product_id.categ_id.property_account_income_categ.id
if contract.invoice_partner_type == 'branch':
for branch in contract.branch_ids:
if branch.property_product_pricelist:
if contract.pricelist_id.currency_id.id != branch.property_product_pricelist.currency_id.id:
import_currency_rate = contract.pricelist_id.currency_id.get_exchange_rate(
branch.property_product_pricelist.currency_id,
date.strftime(date.today(), "%Y-%m-%d"))[0]
else:
import_currency_rate = 1
else:
if contract.pricelist_id.currency_id.id != contract.company_id.currency_id.id:
import_currency_rate = contract.pricelist_id.currency_id.get_exchange_rate(
contract.company_id.currency_id,
date.strftime(date.today(), "%Y-%m-%d"))[0]
else:
import_currency_rate = 1
date_due = False
if branch.property_payment_term:
pterm_list = account_payment_term_obj.compute(
branch.property_payment_term.id,
value=1,
date_ref=time.strftime('%Y-%m-%d'))
if pterm_list:
pterm_list = [line[0] for line in pterm_list]
pterm_list.sort()
date_due = pterm_list[-1]
invoice = {
'partner_id':
branch.id,
'company_id':
contract.company_id.id,
'payment_term':
branch.property_payment_term.id,
'account_id':
branch.property_account_receivable.id,
'name':
contract.name,
'currency_id':
branch.property_product_pricelist.currency_id.id
or contract.company_id.currency_id.id,
'fiscal_position':
branch.property_account_position.id,
'date_due':
date_due
}
ctx = dict(self._context)
ctx['lang'] = branch.lang
ctx['force_company'] = contract.company_id.id
ctx['company_id'] = contract.company_id.id
self = self.with_context(ctx)
inv = invoice_obj.create(invoice)
invoices_list.append(inv.id)
invoice_line = {
'product_id':
contract.product_id.id,
'name':
contract.name,
'quantity':
1,
'price_unit':
contract.amount_preventive_check *
import_currency_rate,
'discount':
contract.to_invoice.factor,
'account_analytic_id':
contract.id,
'invoice_line_tax_id':
[(6, 0,
[tax.id for tax in contract.product_id.taxes_id])],
'account_id':
account_id
}
inv.write({'invoice_line': [(0, 0, invoice_line)]})
elif contract.invoice_partner_type == 'customer':
date_due = False
if contract.partner_id.property_product_pricelist:
if contract.pricelist_id.currency_id.id != contract.partner_id.property_product_pricelist.currency_id.id:
import_currency_rate = contract.pricelist_id.currency_id.get_exchange_rate(
contract.partner_id.
property_product_pricelist.currency_id,
date.strftime(date.today(), "%Y-%m-%d"))[0]
else:
import_currency_rate = 1
else:
if contract.pricelist_id.currency_id.id != contract.company_id.currency_id.id:
import_currency_rate = contract.pricelist_id.currency_id.get_exchange_rate(
contract.company_id.currency_id,
date.strftime(date.today(), "%Y-%m-%d"))[0]
else:
import_currency_rate = 1
if contract.partner_id.property_payment_term:
pterm_list = account_payment_term_obj.compute(
contract.partner_id.property_payment_term.id,
value=1,
date_ref=time.strftime('%Y-%m-%d'))
if pterm_list:
pterm_list = [line[0] for line in pterm_list]
pterm_list.sort()
date_due = pterm_list[-1]
invoice = {
'partner_id':
contract.partner_id.id,
'company_id':
contract.company_id.id,
'payment_term':
contract.partner_id.property_payment_term.id,
'account_id':
contract.partner_id.property_account_receivable.id,
'name':
contract.name,
'currency_id':
contract.company_id.currency_id.id,
'fiscal_position':
contract.partner_id.property_account_position.id,
'date_due':
date_due
}
ctx = dict(self._context)
ctx['lang'] = contract.partner_id.lang
ctx['force_company'] = contract.company_id.id
ctx['company_id'] = contract.company_id.id
self = self.with_context(ctx)
inv = invoice_obj.create(invoice)
invoices_list.append(inv.id)
invoice_line = {
'product_id':
contract.product_id.id,
'name':
contract.name,
'quantity':
1,
'price_unit':
contract.amount_preventive_check * import_currency_rate,
'discount':
contract.to_invoice.factor,
'account_analytic_id':
contract.id,
'invoice_line_tax_id':
[(6, 0, [tax.id for tax in contract.product_id.taxes_id])],
'account_id':
account_id
}
inv.write({'invoice_line': [(0, 0, invoice_line)]})
for issue in issue_obj.search([('stage_id.closed', '=', True),
('sale_order_id', '=', False),
('invoice_ids', '=', False),
('analytic_account_id', '=',
contract.id)]):
inv.write({'issue_ids': [(4, issue.id)]})
return invoices_list
def get_weeknr_from_date(self, date):
return date.strftime("%V")
def _event_name(self):
if self.otio_timeline.name:
return self.otio_timeline.name
return date.strftime(date.today(), "%m-%e-%y")
def _date_to_api_string(date: date) -> str: # pylint: disable=W0621
"""Convert a date object to a YYYYMMDD string expected by the API."""
return date.strftime("%Y%m%d")
def fmt(self, date):
return date.strftime("%Y%m%d"+self.time)
def format_only_date(date): # date = datetime object.
return date.strftime('%Y-%m-%d')
def main():
expiry = date.strftime(date.today() + relativedelta(years=1), "%Y-%m-%d")
module = AnsibleModule(argument_spec=dict(
birthday=dict(default=None, type='str'),
city=dict(default=None, type='str'),
country=dict(default=None, type='str'),
department_number=dict(default=None,
type='str',
aliases=['departmentNumber']),
description=dict(default=None, type='str'),
display_name=dict(default=None, type='str', aliases=['displayName']),
email=dict(default=[''], type='list'),
employee_number=dict(default=None,
type='str',
aliases=['employeeNumber']),
employee_type=dict(default=None, type='str', aliases=['employeeType']),
firstname=dict(default=None, type='str'),
gecos=dict(default=None, type='str'),
groups=dict(default=[], type='list'),
home_share=dict(default=None, type='str', aliases=['homeShare']),
home_share_path=dict(default=None,
type='str',
aliases=['homeSharePath']),
home_telephone_number=dict(default=[],
type='list',
aliases=['homeTelephoneNumber']),
homedrive=dict(default=None, type='str'),
lastname=dict(default=None, type='str'),
mail_alternative_address=dict(default=[],
type='list',
aliases=['mailAlternativeAddress']),
mail_home_server=dict(default=None,
type='str',
aliases=['mailHomeServer']),
mail_primary_address=dict(default=None,
type='str',
aliases=['mailPrimaryAddress']),
mobile_telephone_number=dict(default=[],
type='list',
aliases=['mobileTelephoneNumber']),
organisation=dict(default=None, type='str'),
overridePWHistory=dict(default=False,
type='bool',
aliases=['override_pw_history']),
overridePWLength=dict(default=False,
type='bool',
aliases=['override_pw_length']),
pager_telephonenumber=dict(default=[],
type='list',
aliases=['pagerTelephonenumber']),
password=dict(default=None, type='str', no_log=True),
phone=dict(default=[], type='list'),
postcode=dict(default=None, type='str'),
primary_group=dict(default=None, type='str', aliases=['primaryGroup']),
profilepath=dict(default=None, type='str'),
pwd_change_next_login=dict(default=None,
type='str',
choices=['0', '1'],
aliases=['pwdChangeNextLogin']),
room_number=dict(default=None, type='str', aliases=['roomNumber']),
samba_privileges=dict(default=[],
type='list',
aliases=['sambaPrivileges']),
samba_user_workstations=dict(default=[],
type='list',
aliases=['sambaUserWorkstations']),
sambahome=dict(default=None, type='str'),
scriptpath=dict(default=None, type='str'),
secretary=dict(default=[], type='list'),
serviceprovider=dict(default=[''], type='list'),
shell=dict(default='/bin/bash', type='str'),
street=dict(default=None, type='str'),
title=dict(default=None, type='str'),
unixhome=dict(default=None, type='str'),
userexpiry=dict(default=expiry, type='str'),
username=dict(required=True, aliases=['name'], type='str'),
position=dict(default='', type='str'),
ou=dict(default='', type='str'),
subpath=dict(default='cn=users', type='str'),
state=dict(default='present',
choices=['present', 'absent'],
type='str')),
supports_check_mode=True,
required_if=([
('state', 'present',
['firstname', 'lastname', 'password'])
]))
username = module.params['username']
position = module.params['position']
ou = module.params['ou']
subpath = module.params['subpath']
state = module.params['state']
changed = False
users = list(
ldap_search('(&(objectClass=posixAccount)(uid={}))'.format(username),
attr=['uid']))
if position != '':
container = position
else:
if ou != '':
ou = 'ou={},'.format(ou)
if subpath != '':
subpath = '{},'.format(subpath)
container = '{}{}{}'.format(subpath, ou, base_dn())
user_dn = 'uid={},{}'.format(username, container)
exists = bool(len(users))
if state == 'present':
try:
if not exists:
obj = umc_module_for_add('users/user', container)
else:
obj = umc_module_for_edit('users/user', user_dn)
if module.params['displayName'] is None:
module.params['displayName'] = '{} {}'.format(
module.params['firstname'], module.params['lastname'])
if module.params['unixhome'] is None:
module.params['unixhome'] = '/home/{}'.format(
module.params['username'])
for k in obj.keys():
if (k != 'password' and k != 'groups'
and k != 'overridePWHistory' and k in module.params
and module.params[k] is not None):
obj[k] = module.params[k]
# handle some special values
obj['e-mail'] = module.params['email']
password = module.params['password']
if obj['password'] is None:
obj['password'] = password
else:
old_password = obj['password'].split('}', 2)[1]
if crypt.crypt(password, old_password) != old_password:
obj['overridePWHistory'] = module.params[
'overridePWHistory']
obj['overridePWLength'] = module.params['overridePWLength']
obj['password'] = password
diff = obj.diff()
if exists:
for k in obj.keys():
if obj.hasChanged(k):
changed = True
else:
changed = True
if not module.check_mode:
if not exists:
obj.create()
elif changed:
obj.modify()
except:
module.fail_json(
msg="Creating/editing user {} in {} failed".format(
username, container))
try:
groups = module.params['groups']
if groups:
filter = '(&(objectClass=posixGroup)(|(cn={})))'.format(
')(cn='.join(groups))
group_dns = list(ldap_search(filter, attr=['dn']))
for dn in group_dns:
grp = umc_module_for_edit('groups/group', dn[0])
if user_dn not in grp['users']:
grp['users'].append(user_dn)
if not module.check_mode:
grp.modify()
changed = True
except:
module.fail_json(
msg="Adding groups to user {} failed".format(username))
if state == 'absent' and exists:
try:
obj = umc_module_for_edit('users/user', user_dn)
if not module.check_mode:
obj.remove()
changed = True
except:
module.fail_json(msg="Removing user {} failed".format(username))
module.exit_json(changed=changed,
username=username,
diff=diff,
container=container)
def get_harvest_day(date, day_name):
if day_name == "monday":
return []
# print date
try:
response = urllib.request.urlopen(URL)
# + BY_DAYS[date.weekday()] + ".html")
output = response.read()
except urllib.error.URLError:
return []
soup = BeautifulSoup(output)
# print soup
# print day_name
div = soup.findAll("div", attrs={"id": day_name})[0]
# print div
table = div.find('table') # .findAll("tbody")[0]
# print table
rows = table.findAll('tr')[1:]
days_programs = []
for row in rows:
cols = row.findAll('td')
ptime = datetime.datetime.combine(
date,
datetime.datetime.strptime(
cols[0].text.replace(" AM", "AM").replace(" PM", "PM"),
"%I:%M%p").time())
if ptime.date() > date:
continue
time_str = ptime.strftime("%H%M")
link = cols[2].text
days_programs.append({
"date": date.strftime("%Y-%m-%d"),
"starts": time_str,
"starts_datetime": ptime,
"program_name": link
})
length = len(days_programs)
for index, prog in enumerate(days_programs):
if index == length - 1:
next_prog = {
"starts_datetime":
datetime.datetime.combine(date + datetime.timedelta(days=1),
datetime.datetime.min.time())
}
else:
next_prog = days_programs[index + 1]
if prog["program_name"] == "prev":
prog["program_name"] = days_programs[index - 1]["program_name"]
duration = next_prog["starts_datetime"] - prog["starts_datetime"]
durmins = int(total_secs(duration) / 60)
# print "this: " + str(prog["starts_datetime"]) + ", next: " + str(next_prog["starts_datetime"])
# print durmins
days_programs[index]["duration"] = durmins
for prog in days_programs:
del prog["starts_datetime"]
return days_programs
def get_time(self):
date = time.strftime(DEFAULT_SERVER_DATETIME_FORMAT)
date = datetime.strptime(date, "%Y-%m-%d %H:%M:%S")
date = date + timedelta(hours=4)
date = date.strftime("%d-%m-%Y %H:%M:%S")
return date
if not path.exists('data/' + datet + '.csv'):
wget.download(url, out='data/')
os.rename('data/csv', 'data/' + datet + '.csv')
print('')
data = {}
for country in countries:
dates = []
cases = []
deaths = []
data[country] = {}
with open('data/' + datet + '.csv', "rt",
encoding="iso-8859-2") as csv_file:
csv_reader = csv.reader(csv_file, delimiter=',')
for row in csv_reader:
if country == row[6]:
date = datetime.strptime(row[0], '%d/%m/%Y')
dates.append(date.strftime('%m/%d/%Y'))
cases.append(row[4])
deaths.append(row[5])
data[country]['DATE'] = dates[::-1]
data[country]['CUM_CASES'] = np.cumsum([int(ns) for ns in cases[::-1]])
data[country]['CUM_DEATHS'] = np.cumsum(
[int(ns) for ns in deaths[::-1]])
build_plot(data, countries, 'Cases', vis, dpi)
build_plot(data, countries, 'Deaths', vis, dpi)
build_plot(data, countries, 'Mortality', vis, dpi)
print('Done!')
# and then creates a drawing canvas, draw, to which we can draw text and graphics
img = Image.new('P', (inky_display.WIDTH, inky_display.HEIGHT))
draw = ImageDraw.Draw(img)
# import our fonts
tempFont = ImageFont.truetype('/home/pi/weatherDisplay/fonts/Aller_Bd.ttf', 22)
dayFont = ImageFont.truetype('/home/pi/weatherDisplay/fonts/Roboto-Black.ttf', 18)
iconFont = ImageFont.truetype('/home/pi/weatherDisplay/fonts/Roboto-Medium.ttf', 16)
dateFont = ImageFont.truetype('/home/pi/weatherDisplay/fonts/Roboto-Bold.ttf', 14)
font = ImageFont.truetype('/home/pi/weatherDisplay/fonts/Roboto-Regular.ttf', 12)
smallFont = ImageFont.truetype('/home/pi/weatherDisplay/fonts/ElecSign.ttf', 8)
smallestFont = ImageFont.truetype('/home/pi/weatherDisplay/fonts/ElecSign.ttf', 7)
# define weekday text
weekday = date.today()
day_Name = date.strftime(weekday, '%A')
day_month = date.strftime(weekday, '%-d %B')
# format the summary texts for today
currentCondFormatted = textwrap.fill(upcoming_conditions, 19, max_lines=4)
# draw today's name on left side
draw.text((3, 3), day_Name, inky_display.BLACK, dayFont)
# draw today's date on left side below today's name
dayDate = day_month
draw.text((3, 25), dayDate, inky_display.BLACK, dateFont)
#draw current temperature to right of day name and date. C and F
draw.text((105, 8), temp, inky_display.BLACK, tempFont)
draw.text((105, 34), tempF, inky_display.BLACK, font)
def to_isoz_date_string(value: date) -> str:
return date.strftime(value, "%Y-%m-%d")
def date_str(date):
return date.strftime("%Y-%m-%d")
hsVerificationSocket.bind("tcp://192.168.100.43:%s" % verificationPort)
while True:
if interfaceSocket.poll(100) != 0:
dbCommand = interfaceSocket.recv()
print dbCommand
if dbCommand == 'startHyperSpec':
acquisitionStopped = False
stopped = False
print "dbCommand = startHyperSpec"
interfaceSocket.send("startHyperSpec Received - Wait 9 Seconds")
if HyperSpecAcqStarted == False: #run HyperSpecAcq and verify
print "Acquisition Script Called"
date = datetime.now()
currTime = date.strftime("%Y%m%d%H%M%S")
home = "E:\\HS_stdout\\"
iiFolderName = "stdOut_2i_%s" % currTime
iiFileName = "%s.txt" % iiFolderName
iiPath = home + iiFolderName
nirFolderName = "stdOut_NIR_%s" % currTime
nirFileName = "%s.txt" % nirFolderName
nirPath = home + nirFolderName
if not os.path.exists(iiPath):
os.makedirs(iiPath)
if not os.path.exists(nirPath):
os.makedirs(nirPath)
iiFile = open("%s\\%s" % (iiPath, iiFileName), 'a+')
nirFile = open("%s\\%s" % (nirPath, nirFileName), 'a+')
iiAcquisition = subprocess.Popen(
"E:\\ResononAPI_2.2_Beta\\bin\\2iHyperSpecAcquisitionV2.6.exe",
def parsedate(date):
return date.strftime('%A')
for i in invertors:
invertorID = i
os.chdir("/home/vishnu/Documents/icer/" + invertorID)
print(invertorID)
#( date,month,year); GENERATE DATES .CSV FORMAT
start = datetime.datetime.strptime("01-01-2018", "%d-%m-%Y")
end = datetime.datetime.strptime("2-1-2018",
"%d-%m-%Y") # One day after the end date
date_generated = [
start + datetime.timedelta(days=x)
for x in range(0, (end - start).days)
]
g = []
for date in date_generated:
start_date = (date.strftime("%d"))
end_date = (date.strftime("%d"))
month = (date.strftime("%m"))
year = (date.strftime("%Y"))
filename = (str(year) + str(month) + str(start_date)) + ".csv"
df = pd.DataFrame()
dg = pd.read_csv(filename)
df = df.append(dg, ignore_index=True)
df.insert(0, 'Timeadjusted', 0) # df.insert(idx,col_name,value)
df['Timeadjusted'] = pd.to_datetime(df.Time) - pd.Timedelta(hours=8,
minutes=30)
#Identify and drop NAN Values
tt = df[df['Timeadjusted'].isnull()]
print(tt)
df = df.dropna()
# prints dropped df
stock_dict[item["ticker"]][1] + pos_neg[1] * score,
stock_dict[item["ticker"]][2] + score
]
print("finalizing data...")
output = []
date = date.today() - timedelta(days=NUM_DAYS_AGO)
for key, val in stock_dict.items():
#limit min number of ppl per stock
if (val[2] > 3):
#if symbol isn't registered in yahoo finance, skip
try:
data = round(DataReader(key, 'yahoo', date)['Adj Close'][0], 2)
output.append([
key,
int((val[0] / val[2]) * 100) / 100,
int((val[1] / val[2]) * 100) / 100, data, val[2]
])
if (len(output) % 25 == 0):
print(f"Created {len(output)} entries...")
except:
continue
print(f"Created {len(output)} entries")
output = sorted(output, key=lambda x: x[4], reverse=True)
output_file = open('reddit_data.txt', 'a')
formatted_date = date.strftime("%m/%d/%y")
print(formatted_date, sep="\n", file=output_file)
print(*output, sep="\n", file=output_file)
print(f"Total time elapsed: {round((time.time() - start_time)/60, 2)} minutes")
def add_variants_to_project_from_vcf(self,
vcf_file,
project_id,
indiv_id_list=None,
start_from_chrom=None,
end_with_chrom=None):
"""
This is how variants are loaded
"""
chrom_list = list(map(str, range(1, 23))) + ['X', 'Y']
chrom_list_start_index = 0
if start_from_chrom:
chrom_list_start_index = chrom_list.index(
start_from_chrom.replace("chr", "").upper())
chrom_list_end_index = len(chrom_list)
if end_with_chrom:
chrom_list_end_index = chrom_list.index(
end_with_chrom.replace("chr", "").upper())
chromosomes_to_include = set(
chrom_list[chrom_list_start_index:chrom_list_end_index])
#tabix_file = pysam.TabixFile(vcf_file)
#vcf_iter = tabix_file.header
#for chrom in chrom_list[chrom_list_start_index:chrom_list_end_index]:
# print("Will load chrom: " + chrom)
# vcf_iter = itertools.chain(vcf_iter, tabix_file.fetch(chrom))
project_collection = self._get_project_collection(project_id)
reference_populations = self._annotator.reference_population_slugs + self._custom_populations_map.get(
project_id)
for counter, variant in enumerate(
vcf_stuff.iterate_vcf(vcf_file,
genotypes=True,
indiv_id_list=indiv_id_list)):
if (start_from_chrom or end_with_chrom) and variant.chr.replace(
"chr", "") not in chromosomes_to_include:
continue
if variant.alt == "*":
#print("Skipping GATK 3.4 * alt allele: " + str(variant.unique_tuple()))
continue
if counter % 2000 == 0:
print(
date.strftime(datetime.now(), "%m/%d/%Y %H:%M:%S") +
"-- inserting variant %d %s:%s-%s-%s (%0.1f%% done with %s) "
% (counter, variant.chr, variant.pos, variant.ref,
variant.alt, 100 * variant.pos /
CHROMOSOME_SIZES[variant.chr.replace("chr", "")],
variant.chr))
variant_dict = project_collection.find_one({
'xpos': variant.xpos,
'ref': variant.ref,
'alt': variant.alt
})
if not variant_dict:
variant_dict = variant.toJSON()
try:
annotation = self._annotator.get_annotation(
variant.xpos,
variant.ref,
variant.alt,
populations=reference_populations)
except ValueError, e:
sys.stderr.write("WARNING: " + str(e) + "\n")
continue
_add_index_fields_to_variant(variant_dict, annotation)
else:
for indiv_id, genotype in variant.get_genotypes():
if genotype.num_alt != 0:
variant_dict['genotypes'][indiv_id] = genotype._asdict(
)
project_collection.save(variant_dict)
2], true_dates[last_date_idx, 1],
true_dates[last_date_idx,
0]) # obtengo la fecha de fin
all_ticks = numpy.linspace(
date2num(start_date), date2num(end_date),
test_size) # obtengo un arreglo con todos los valores numericos de fechas
tick_spacing = test_size if test_size <= 60 else 30
date_format = "%m/%d" if test_size <= 60 else "%y/%m"
# major_ticks = numpy.arange(date2num(start_date), date2num(end_date), tick_spacing) # obtengo un arreglo con los valores de fecha que quiero mostrar
major_ticks = numpy.linspace(
date2num(start_date), date2num(end_date), tick_spacing
) # obtengo un arreglo con los valores de fecha que quiero mostrar
major_tick_labels = [
date.strftime(date_format) for date in num2date(major_ticks)
]
# PLOTEO DE LA DEMANDA DE SALIDA NORMALIZADA JUNTO CON LA PORCION DE INCUMBENCIA DE LOS DATOS ORIGINALES -----------------------------------------
true_net_demand = test_demand
predicted_net_demand = predicted
plot_w_xticks(all_ticks, major_ticks, major_tick_labels,
[(true_net_demand, 'b-o'), (predicted_net_demand, 'r-o')])
axes = plt.gca()
#axes.set_ylim([0, 1]) # seteo limite en el eje y entre 0 y 1
plt.grid()
plt.show()
# PLOTEO EL ERROR COMETIDO ----------------------------------------------------------------------------------------------------------
denormalized_true_net_demand = de_normalize_dataset(test_demand.copy(),
DIFF_DEMAND)
def fmt_date(date):
return "%s, week %d, %s" % (date.isoformat(), date.isocalendar()[1],
date.strftime("%A, %B"))
class MongoDatastore(datastore.Datastore):
def __init__(self,
db,
annotator,
custom_population_store=None,
custom_populations_map=None):
self._db = db
self._annotator = annotator
self._custom_population_store = custom_population_store
self._custom_populations_map = custom_populations_map
if self._custom_populations_map is None:
self._custom_populations_map = {}
def _make_db_query(self, genotype_filter=None, variant_filter=None):
"""
Caller specifies filters to get_variants, but they are evaluated later.
Here, we just inspect those filters and see what heuristics we can apply to avoid a full table scan,
Query here must return a superset of the true get_variants results
Note that the full annotation isn't stored, so use the fields added by _add_index_fields_to_variant
"""
db_query = {}
# genotype filter
if genotype_filter is not None:
_add_genotype_filter_to_variant_query(db_query, genotype_filter)
if variant_filter:
if variant_filter.locations:
location_ranges = []
for xstart, xend in variant_filter.locations:
location_ranges.append({
'$and': [{
'xpos': {
'$gte': xstart
}
}, {
'xpos': {
'$lte': xend
}
}]
})
db_query['$or'] = location_ranges
if variant_filter.so_annotations:
db_query['db_tags'] = {'$in': variant_filter.so_annotations}
if variant_filter.genes:
db_query['db_gene_ids'] = {'$in': variant_filter.genes}
if variant_filter.ref_freqs:
for population, freq in variant_filter.ref_freqs:
if population in self._annotator.reference_population_slugs:
db_query['db_freqs.' + population] = {'$lte': freq}
return db_query
#
# Variant search
#
def get_variants(self,
project_id,
family_id,
genotype_filter=None,
variant_filter=None):
db_query = self._make_db_query(genotype_filter, variant_filter)
sys.stderr.write("%s\n" % str(db_query))
collection = self._get_family_collection(project_id, family_id)
if not collection:
print(
"Error: mongodb collection not found for project %s family %s "
% (project_id, family_id))
return
counters = OrderedDict([('returned_by_query', 0),
('passes_variant_filter', 0)])
for i, variant_dict in enumerate(
collection.find({
'$and': [{
k: v
} for k, v in db_query.items()]
}).sort('xpos').limit(MONGO_QUERY_RESULTS_LIMIT + 5)):
if i >= MONGO_QUERY_RESULTS_LIMIT:
raise Exception(
"ERROR: this search exceeded the %s variant result size limit. Please set additional filters and try again."
% MONGO_QUERY_RESULTS_LIMIT)
variant = Variant.fromJSON(variant_dict)
self.add_annotations_to_variant(variant, project_id)
counters["returned_by_query"] += 1
if passes_variant_filter(variant, variant_filter)[0]:
counters["passes_variant_filter"] += 1
yield variant
for k, v in counters.items():
sys.stderr.write(" %s: %s\n" % (k, v))
def get_variants_in_gene(self,
project_id,
family_id,
gene_id,
genotype_filter=None,
variant_filter=None):
if variant_filter is None:
modified_variant_filter = VariantFilter()
else:
modified_variant_filter = copy.deepcopy(variant_filter)
modified_variant_filter.add_gene(gene_id)
db_query = self._make_db_query(genotype_filter,
modified_variant_filter)
collection = self._get_family_collection(project_id, family_id)
if not collection:
return
# we have to collect list in memory here because mongo can't sort on xpos,
# as result size can get too big.
# need to find a better way to do this.
variants = []
for variant_dict in collection.find(db_query).hint([
('db_gene_ids', pymongo.ASCENDING), ('xpos', pymongo.ASCENDING)
]):
variant = Variant.fromJSON(variant_dict)
self.add_annotations_to_variant(variant, project_id)
if passes_variant_filter(variant, modified_variant_filter):
variants.append(variant)
variants = sorted(variants, key=lambda v: v.unique_tuple())
for v in variants:
yield v
def get_variants_in_range(self, project_id, family_id, xpos_start,
xpos_end):
collection = self._get_family_collection(project_id, family_id)
if not collection:
raise ValueError("Family not found: " + str(family_id))
for i, variant_dict in enumerate(
collection.find({
'$and': [{
'xpos': {
'$gte': xpos_start
}
}, {
'xpos': {
'$lte': xpos_end
}
}]
}).limit(MONGO_QUERY_RESULTS_LIMIT + 5)):
if i > MONGO_QUERY_RESULTS_LIMIT:
raise Exception(
"ERROR: this search exceeded the %s variant result size limit. Please set additional filters and try again."
% MONGO_QUERY_RESULTS_LIMIT)
variant = Variant.fromJSON(variant_dict)
self.add_annotations_to_variant(variant, project_id)
yield variant
def get_single_variant(self, project_id, family_id, xpos, ref, alt):
collection = self._get_family_collection(project_id, family_id)
if not collection:
return None
variant_dict = collection.find_one({
'xpos': xpos,
'ref': ref,
'alt': alt
})
if variant_dict:
variant = Variant.fromJSON(variant_dict)
self.add_annotations_to_variant(variant, project_id)
return variant
else:
return None
def get_variants_cohort(self, project_id, cohort_id, variant_filter=None):
db_query = self._make_db_query(None, variant_filter)
collection = self._get_family_collection(project_id, cohort_id)
for i, variant in enumerate(
collection.find(db_query).sort('xpos').limit(
MONGO_QUERY_RESULTS_LIMIT + 5)):
if i > MONGO_QUERY_RESULTS_LIMIT:
raise Exception(
"ERROR: this search exceeded the %s variant result size limit. Please set additional filters and try again."
% MONGO_QUERY_RESULTS_LIMIT)
yield Variant.fromJSON(variant)
def get_single_variant_cohort(self, project_id, cohort_id, xpos, ref, alt):
collection = self._get_family_collection(project_id, cohort_id)
variant = collection.find_one({'xpos': xpos, 'ref': ref, 'alt': alt})
return Variant.fromJSON(variant)
#
# New sample stuff
#
def get_all_individuals(self):
"""
List of all individuals in the datastore
Items are (project_id, indiv_id) tuples
"""
return [(i['project_id'], i['indiv_id'])
for i in self._db.individuals.find()]
def get_all_families(self):
"""
List of all families in the datastore
Items are (project_id, family_id) tuples
"""
return [(i['project_id'], i['family_id'])
for i in self._db.families.find()]
def individual_exists(self, project_id, indiv_id):
return self._db.individuals.find_one({
'project_id': project_id,
'indiv_id': indiv_id
}) is not None
def add_individual(self, project_id, indiv_id):
if self.individual_exists(project_id, indiv_id):
raise Exception("Indiv (%s, %s) already exists" %
(project_id, indiv_id))
indiv = {
'project_id': project_id,
'indiv_id': indiv_id,
}
self._db.individuals.save(indiv)
def get_individuals(self, project_id):
return [
i['indiv_id']
for i in self._db.individuals.find({'project_id': project_id})
]
def family_exists(self, project_id, family_id):
return self._db.families.find_one({
'project_id': project_id,
'family_id': family_id
}) is not None
def get_individuals_for_family(self, project_id, family_id):
return self._db.families.find_one({
'project_id': project_id,
'family_id': family_id
})['individuals']
def get_family_status(self, project_id, family_id):
family_doc = self._db.families.find_one({
'project_id': project_id,
'family_id': family_id
})
if not family_doc:
return None
return family_doc['status']
def get_family_statuses(self, family_list):
ret = {f: None for f in family_list}
by_project = defaultdict(list)
for project_id, family_id in family_list:
by_project[project_id].append(family_id)
for project_id, family_id_list in by_project.items():
for family_doc in self._db.families.find({
'project_id': project_id,
'family_id': {
'$in': family_id_list
}
}):
ret[(project_id,
family_doc['family_id'])] = family_doc['status']
return ret
def _get_family_info(self, project_id, family_id=None):
if family_id is None:
return [
family_info for family_info in self._db.families.find(
{'project_id': project_id})
]
else:
return self._db.families.find_one({
'project_id': project_id,
'family_id': family_id
})
def _get_family_collection(self, project_id, family_id):
family_info = self._get_family_info(project_id, family_id)
if not family_info:
return None
return self._db[family_info['coll_name']]
#
# Variant loading
# Unique to mongo datastore, not part of protocol
#
def _add_family_info(self, project_id, family_id, individuals):
"""
Add all the background info about this family
We try to keep this as simple as possible - just IDs
After this is run, variants are ready to be loaded
"""
if self.family_exists(project_id, family_id):
#raise Exception("Family (%s, %s) already exists" % (project_id, family_id))
return
for indiv_id in individuals:
if not self.individual_exists(project_id, indiv_id):
self.add_individual(project_id, indiv_id)
family_coll_name = "family_%s_%s" % (slugify(
project_id, separator='_'), slugify(family_id, separator='_'))
family = {
'project_id': project_id,
'family_id': family_id,
'individuals': individuals,
'coll_name': family_coll_name,
'status': 'loading'
}
family_collection = self._db[family_coll_name]
self._index_family_collection(family_collection)
self._db.families.save(family)
def add_family(self, project_id, family_id, individuals):
"""
Add new family
Adds individuals if they don't exist yet
Phenotypes and pedigrees aren't stored, just which individuals
"""
self._add_family_info(project_id, family_id, individuals)
def add_family_set(self, family_list):
"""
Add a set of families from the same VCF file
family_list is just a list of dicts with keys of project_id, family_id, individuals
"""
for fam_info in family_list:
self._add_family_info(fam_info['project_id'],
fam_info['family_id'],
fam_info['individuals'])
def load_family_set(self,
vcf_file_path,
family_list,
reference_populations=None,
vcf_id_map=None,
mark_as_loaded=True,
start_from_chrom=None,
end_with_chrom=None):
"""
Load a set of families from the same VCF file
family_list is a list of (project_id, family_id) tuples
"""
family_info_list = [
self._get_family_info(f[0], f[1]) for f in family_list
]
self._load_variants_for_family_set(
family_info_list,
vcf_file_path,
reference_populations=reference_populations,
vcf_id_map=vcf_id_map,
start_from_chrom=start_from_chrom,
end_with_chrom=end_with_chrom,
)
if mark_as_loaded:
for family in family_info_list:
self._finalize_family_load(family['project_id'],
family['family_id'])
def _load_variants_for_family_set(self,
family_info_list,
vcf_file_path,
reference_populations=None,
vcf_id_map=None,
start_from_chrom=None,
end_with_chrom=None):
"""
Load variants for a set of families, assuming all come from the same VCF file
Added after load_variants_for_family to speed up loading - goal is to
only iterate the VCF once. Here's how it works:
for each raw variant:
annotate
for each family:
extract family variant from full variant
update variant inheritance
if family variant is relevant for family:
add to collection
"""
self._add_vcf_file_for_family_set(
family_info_list,
vcf_file_path,
reference_populations=reference_populations,
vcf_id_map=vcf_id_map,
start_from_chrom=start_from_chrom,
end_with_chrom=end_with_chrom,
)
def _add_vcf_file_for_family_set(self,
family_info_list,
vcf_file_path,
reference_populations=None,
vcf_id_map=None,
start_from_chrom=None,
end_with_chrom=None):
collections = {
f['family_id']: self._db[f['coll_name']]
for f in family_info_list
}
#for collection in collections.values():
# collection.drop_indexes()
indiv_id_list = [i for f in family_info_list for i in f['individuals']]
number_of_families = len(family_info_list)
sys.stderr.write(
"Loading variants for %(number_of_families)d families %(family_info_list)s from %(vcf_file_path)s\n"
% locals())
for family in family_info_list:
print("Indexing family: " + str(family))
collection = collections[family['family_id']]
collection.ensure_index([('xpos', 1), ('ref', 1), ('alt', 1)])
# check whether some of the variants for this chromosome has been loaded already
# if yes, start from the last loaded variant, and not from the beginning
if "_chr" in vcf_file_path or ".chr" in vcf_file_path:
# if the VCF files are split by chromosome (eg. for WGS projects), check within the chromosome
vcf_file = compressed_file(vcf_file_path)
variant = next(
vcf_stuff.iterate_vcf(vcf_file,
genotypes=False,
indiv_id_list=indiv_id_list,
vcf_id_map=vcf_id_map))
print(vcf_file_path + " - chromsome: " + str(variant.chr))
vcf_file.close()
position_per_chrom = {}
for chrom in range(1, 24):
position_per_chrom[chrom] = defaultdict(int)
for family in family_info_list: #variants = collections[family['family_id']].find().sort([('xpos',-1)]).limit(1)
variants = list(collections[family['family_id']].find({
'$and': [{
'xpos': {
'$gte': chrom * 1e9
}
}, {
'xpos': {
'$lt': (chrom + 1) * 1e9
}
}]
}).sort([('xpos', -1)]).limit(1))
if len(variants) > 0:
position_per_chrom[chrom][family[
'family_id']] = variants[0]['xpos'] - chrom * 1e9
else:
position_per_chrom[chrom][family['family_id']] = 0
for chrom in range(1, 24):
position_per_chrom[chrom] = min(
position_per_chrom[chrom].values()
) # get the smallest last-loaded variant position for this chromosome across all families
chr_idx = int(variant.xpos / 1e9)
start_from_pos = int(position_per_chrom[chr_idx])
print("Start from: %s - %s (%0.1f%% done)" %
(chr_idx, start_from_pos, 100. * start_from_pos /
CHROMOSOME_SIZES[variant.chr.replace("chr", "")]))
tabix_file = pysam.TabixFile(vcf_file_path)
vcf_iter = itertools.chain(
tabix_file.header,
tabix_file.fetch(variant.chr.replace("chr", ""),
start_from_pos, int(2.5e8)))
elif start_from_chrom or end_with_chrom:
if start_from_chrom:
print("Start chrom: chr%s" % start_from_chrom)
if end_with_chrom:
print("End chrom: chr%s" % end_with_chrom)
chrom_list = list(map(str, range(1, 23))) + ['X', 'Y']
chrom_list_start_index = 0
if start_from_chrom:
chrom_list_start_index = chrom_list.index(
start_from_chrom.replace("chr", "").upper())
chrom_list_end_index = len(chrom_list)
if end_with_chrom:
chrom_list_end_index = chrom_list.index(
end_with_chrom.replace("chr", "").upper())
tabix_file = pysam.TabixFile(vcf_file_path)
vcf_iter = tabix_file.header
for chrom in chrom_list[
chrom_list_start_index:chrom_list_end_index + 1]:
print("Will load chrom: " + chrom)
try:
vcf_iter = itertools.chain(vcf_iter,
tabix_file.fetch(chrom))
except ValueError as e:
print("WARNING: " + str(e))
else:
vcf_iter = vcf_file = compressed_file(vcf_file_path)
# TODO handle case where it's one vcf file, not split by chromosome
size = os.path.getsize(vcf_file_path)
#progress = get_progressbar(size, 'Loading VCF: {}'.format(vcf_file_path))
def insert_all_variants_in_buffer(buff, collections_dict):
for family_id in buff:
if len(buff[family_id]) == 0: # defensive programming
raise ValueError(
"%s has zero variants to insert. Should not be in buff."
% family_id)
while len(buff) > 0:
# choose a random family for which to insert a variant from among families that still have variants to insert
family_id = random.choice(buff.keys())
# pop a variant off the list for this family, and insert it
family_variant_dict_to_insert = buff[family_id].pop()
c = collections_dict[family_id]
c.insert(family_variant_dict_to_insert)
if len(buff[family_id]) == 0:
del buff[
family_id] # if no more variants for this family, delete it
vcf_rows_counter = 0
variants_buffered_counter = 0
family_id_to_variant_list = defaultdict(
list) # will accumulate variants to be inserted all at once
for variant in vcf_stuff.iterate_vcf(vcf_iter,
genotypes=True,
indiv_id_list=indiv_id_list,
vcf_id_map=vcf_id_map):
if variant.alt == "*":
#print("Skipping GATK 3.4 * alt allele: " + str(variant.unique_tuple()))
continue
try:
annotation = self._annotator.get_annotation(
variant.xpos,
variant.ref,
variant.alt,
populations=reference_populations)
except ValueError, e:
sys.stderr.write("WARNING: " + str(e) + "\n")
continue
vcf_rows_counter += 1
for family in family_info_list:
# TODO: can we move this inside the if relevant clause below?
try:
family_variant = variant.make_copy(
restrict_to_genotypes=family['individuals'])
family_variant_dict = family_variant.toJSON()
_add_index_fields_to_variant(family_variant_dict,
annotation)
if xbrowse_utils.is_variant_relevant_for_individuals(
family_variant, family['individuals']):
collection = collections[family['family_id']]
if not collection.find_one({
'xpos': family_variant.xpos,
'ref': family_variant.ref,
'alt': family_variant.alt
}):
family_id_to_variant_list[family[
'family_id']].append(family_variant_dict)
variants_buffered_counter += 1
except Exception, e:
sys.stderr.write(
"ERROR: on variant %s, family: %s - %s\n" %
(variant.toJSON(), family, e))
if variants_buffered_counter > 2000:
print(
date.strftime(datetime.now(), "%m/%d/%Y %H:%M:%S") +
"-- %s:%s-%s-%s (%0.1f%% done) - inserting %d family-variants from %d vcf rows into %s families"
% (variant.chr, variant.pos, variant.ref, variant.alt,
100 * variant.pos /
CHROMOSOME_SIZES[variant.chr.replace("chr", "")],
variants_buffered_counter, vcf_rows_counter,
len(family_id_to_variant_list)))
insert_all_variants_in_buffer(family_id_to_variant_list,
collections)
assert len(family_id_to_variant_list) == 0
vcf_rows_counter = 0
variants_buffered_counter = 0
#TIMEZONE = 1
sun = Sun()
def daterange(start_date, end_date):
for n in range(int ((end_date - start_date).days)+1):
yield start_date + timedelta(n)
#start_date = date(2019, 1, 1)
#end_date = date(2019, 12, 31)
start_date = date(2020, 11, 1)
end_date = date(2020, 12, 31)
date = datetime.now()
gy = int(date.strftime("%Y"))
gm = int(date.strftime("%m"))
gd = int(date.strftime("%d"))
print "BEGIN:VCALENDAR"
print "PRODID:calendar-201901"
print "VERSION:2.0"
for single_date in daterange(start_date, end_date):
#print single_date.strftime("%Y-%m-%d")
dy = int(single_date.strftime("%Y"))
dm = int(single_date.strftime("%m"))
dd = int(single_date.strftime("%d"))
sunrise = sun.getSunriseTime( coords, single_date )['decimal']# + TIMEZONE
#!/usr/bin/python3
import sys, getopt
import os
from datetime import date, datetime
from src.rss_wordcloud_generator import RssWordcloudGenerator
# Generating word clouds
day = date.today()
try:
opts, args = getopt.getopt(sys.argv[1:], "hd:", ["day="])
except getopt.GetoptError:
print('-d <day>')
sys.exit(1)
for opt, arg in opts:
if opt in ("-d", "--day"):
day = arg
date = datetime.strptime(day, '%Y-%m-%d')
gen = RssWordcloudGenerator(today=date.strftime("%Y-%m-%d"))
gen.run()
def create_invoice_lines_issues(self,issues,invoice_dict,line_detailed,is_warranty):
account_obj=self.env['account.analytic.account']
invoice_obj=self.env['account.invoice']
user = self.env['res.users'].browse(self._uid)
total_expenses=0.0
count_lines=1
count_lines_products=1
first_line_product=0
invoices_list=[]
cont_invoice=0
limit_lines=user.company_id.maximum_invoice_lines
if line_detailed==True:
for issue in issues:
if issue.timesheet_ids or issue.expense_line_ids and cont_invoice==0:
cont_invoice=1
break
if cont_invoice==1 or line_detailed==False:
inv=invoice_obj.create(invoice_dict)
invoices_list.append(inv.id)
for issue in issues:
for timesheet in issue.timesheet_ids:
for account_line in timesheet.line_id:
if not account_line.invoice_id:
total_timesheet=0.0
quantity,total_timesheet=account_obj._get_invoice_price(account_line.account_id,account_line.date,timesheet.start_time,timesheet.end_time,issue.product_id.id,issue.categ_id.id,account_line.unit_amount,timesheet.service_type,timesheet.employee_id.id,account_line.to_invoice.id)
if is_warranty==False:
if issue.partner_id and issue.branch_id:
if issue.branch_id.property_product_pricelist:
if account_line.account_id.pricelist_id.currency_id.id != issue.branch_id.property_product_pricelist.currency_id.id:
import_currency_rate=account_line.account_id.pricelist_id.currency_id.get_exchange_rate(issue.branch_id.property_product_pricelist.currency_id,date.strftime(date.today(), "%Y-%m-%d"))[0]
else:
import_currency_rate = 1
else:
if account_line.account_id.pricelist_id.currency_id.id != account_line.account_id.company_id.currency_id.id:
import_currency_rate=account_line.account_id.pricelist_id.currency_id.get_exchange_rate(account_line.account_id.company_id.currency_id,date.strftime(date.today(), "%Y-%m-%d"))[0]
else:
import_currency_rate = 1
elif issue.partner_id and not issue.branch_id:
if issue.partner_id.property_product_pricelist:
if account_line.account_id.pricelist_id.currency_id.id != issue.partner_id.property_product_pricelist.currency_id.id:
import_currency_rate=account_line.account_id.pricelist_id.currency_id.get_exchange_rate(issue.partner_id.property_product_pricelist.currency_id,date.strftime(date.today(), "%Y-%m-%d"))[0]
else:
import_currency_rate = 1
else:
if account_line.account_id.pricelist_id.currency_id.id != account_line.account_id.company_id.currency_id.id:
import_currency_rate=account_line.account_id.pricelist_id.currency_id.get_exchange_rate(account_line.account_id.company_id.currency_id,date.strftime(date.today(), "%Y-%m-%d"))[0]
else:
import_currency_rate = 1
elif is_warranty==True:
if issue.product_id.manufacturer.property_product_pricelist_purchase:
if account_line.account_id.pricelist_id.currency_id.id != issue.product_id.manufacturer.property_product_pricelist_purchase.currency_id.id:
import_currency_rate=account_line.account_id.pricelist_id.currency_id.get_exchange_rate(issue.product_id.manufacturer.property_product_pricelist_purchase.currency_id,date.strftime(date.today(), "%Y-%m-%d"))[0]
else:
import_currency_rate = 1
else:
if account_line.account_id.pricelist_id.currency_id.id != account_line.account_id.company_id.currency_id.id:
import_currency_rate=account_line.account_id.pricelist_id.currency_id.get_exchange_rate(account_line.account_id.company_id.currency_id,date.strftime(date.today(), "%Y-%m-%d"))[0]
else:
import_currency_rate = 1
if (timesheet.end_time-timesheet.start_time!=0 or total_timesheet!=0):
invoice_line={
'product_id':account_line.product_id.id ,
'name': account_line.product_id.description+'-'+issue.product_id.description,
'real_quantity':timesheet.end_time-timesheet.start_time,
'quantity':quantity,
'price_unit':total_timesheet*import_currency_rate,
'uos_id':account_line.product_id.uom_id.id,
'discount':account_line.account_id.to_invoice.factor,
'account_analytic_id':account_line.account_id.id,
'price_subtotal':total_timesheet*quantity,
'invoice_line_tax_id':[(6, 0, [tax.id for tax in account_line.product_id.taxes_id])],
'account_id':account_line.product_id.property_account_income.id or account_line.product_id.categ_id.property_account_income_categ.id
}
if timesheet.ticket_number:
invoice_line['reference']='R#'+timesheet.ticket_number
if issue.warranty=='seller':
invoice_line['price_unit']=0
if count_lines<=limit_lines or limit_lines==0 or limit_lines==-1:
inv.write({'invoice_line':[(0,0,invoice_line)]})
if issue.issue_number not in inv.origin:
inv.write({'origin':inv.origin + issue.issue_number +'-'})
inv.write({'issue_ids':[(4,issue.id)]})
count_lines+=1
else:
if issue.issue_number not in inv.origin:
inv.write({'origin':inv.origin + issue.issue_number})
inv=invoice_obj.create(invoice_dict)
invoices_list.append(inv.id)
count_lines=1
inv.write({'invoice_line':[(0,0,invoice_line)]})
if issue.issue_number not in inv.origin:
inv.write({'origin':inv.origin + issue.issue_number +'-'})
inv.write({'issue_ids':[(4,issue.id)]})
count_lines+=1
account_line.write({'invoice_id':inv.id})
for backorder in issue.backorder_ids:
if backorder.delivery_note_id and backorder.delivery_note_id.state=='done' and backorder.picking_type_id.code=='outgoing' and backorder.invoice_state!='invoiced' and backorder.state=='done':
for delivery_note_lines in backorder.delivery_note_id.note_lines:
if is_warranty==False:
if issue.partner_id and issue.branch_id:
if issue.branch_id.property_product_pricelist:
if delivery_note_lines.note_id.currency_id.id != issue.branch_id.property_product_pricelist.currency_id.id:
import_currency_rate=delivery_note_lines.note_id.currency_id.get_exchange_rate(issue.branch_id.property_product_pricelist.currency_id,date.strftime(date.today(), "%Y-%m-%d"))[0]
else:
import_currency_rate = 1
else:
if delivery_note_lines.note_id.currency_id.id != issue.analytic_account_id.company_id.currency_id.id:
import_currency_rate=delivery_note_lines.note_id.currency_id.get_exchange_rate(issue.analytic_account_id.company_id.currency_id,date.strftime(date.today(), "%Y-%m-%d"))[0]
else:
import_currency_rate = 1
elif issue.partner_id and not issue.branch_id:
if issue.partner_id.property_product_pricelist:
if delivery_note_lines.note_id.currency_id.id != issue.partner_id.property_product_pricelist.currency_id.id:
import_currency_rate=delivery_note_lines.note_id.currency_id.get_exchange_rate(issue.partner_id.property_product_pricelist.currency_id,date.strftime(date.today(), "%Y-%m-%d"))[0]
else:
import_currency_rate = 1
else:
if delivery_note_lines.note_id.currency_id.id != issue.analytic_account_id.company_id.currency_id.id:
import_currency_rate=delivery_note_lines.note_id.currency_id.get_exchange_rate(issue.analytic_account_id.company_id.currency_id,date.strftime(date.today(), "%Y-%m-%d"))[0]
else:
import_currency_rate = 1
elif is_warranty==True:
if issue.product_id.manufacturer.property_product_pricelist_purchase:
if delivery_note_lines.note_id.currency_id.id != issue.product_id.manufacturer.property_product_pricelist_purchase.currency_id.id:
import_currency_rate=delivery_note_lines.note_id.currency_id.get_exchange_rate(issue.product_id.manufacturer.property_product_pricelist_purchase.currency_id,date.strftime(date.today(), "%Y-%m-%d"))[0]
else:
import_currency_rate = 1
else:
if delivery_note_lines.note_id.currency_id.id !=issue.analytic_account_id.company_id.currency_id.id:
import_currency_rate=delivery_note_lines.note_id.currency_id.get_exchange_rate(issue.analytic_account_id.company_id.currency_id,date.strftime(date.today(), "%Y-%m-%d"))[0]
else:
import_currency_rate = 1
if delivery_note_lines.note_id.delivery_note_origin.name:
origin=delivery_note_lines.note_id.delivery_note_origin.name
else:
origin=delivery_note_lines.note_id.name
invoice_line={
'product_id':delivery_note_lines.product_id.id,
'name': issue.product_id.description +'-'+delivery_note_lines.product_id.description,
'quantity':delivery_note_lines.quantity,
'real_quantity':delivery_note_lines.quantity,
'uos_id':delivery_note_lines.product_uos.id,
'price_unit':delivery_note_lines.price_unit*import_currency_rate,
'discount':delivery_note_lines.discount,
'invoice_line_tax_id':[(6, 0, [tax.id for tax in delivery_note_lines.taxes_id])],
'account_analytic_id':issue.analytic_account_id.id,
'reference':'NE#'+ origin,
'account_id':delivery_note_lines.product_id.property_account_income.id or delivery_note_lines.product_id.categ_id.property_account_income_categ.id
}
if issue.warranty=='seller':
invoice_line['price_unit']=0
if line_detailed==True:
if first_line_product==0:
inv_prod=invoice_obj.create(invoice_dict)
if issue.issue_number not in inv_prod.origin:
inv_prod.write({'origin':inv_prod.origin + issue.issue_number +'-'})
invoices_list.append(inv_prod.id)
first_line_product+=1
if count_lines_products<=limit_lines or limit_lines==0 or limit_lines==-1:
if issue.issue_number not in inv_prod.origin:
inv_prod.write({'origin':inv_prod.origin + issue.issue_number +'-'})
inv_prod.write({'invoice_line':[(0,0,invoice_line)]})
inv_prod.write({'issue_ids':[(4,issue.id)]})
backorder.delivery_note_id.write({'invoice_ids':[(4,inv_prod.id)]})
count_lines_products+=1
else:
if issue.issue_number not in inv_prod.origin:
inv_prod.write({'origin':inv_prod.origin + issue.issue_number})
inv_prod=invoice_obj.create(invoice_dict)
invoices_list.append(inv_prod.id)
count_lines_products=0
if issue.issue_number not in inv_prod.origin:
inv_prod.write({'origin':inv_prod.origin + issue.issue_number +'-'})
inv_prod.write({'invoice_line':[(0,0,invoice_line)]})
inv_prod.write({'issue_ids':[(4,issue.id)]})
backorder.delivery_note_id.write({'invoice_ids':[(4,inv_prod.id)]})
count_lines_products+=1
else:
if count_lines<=limit_lines or limit_lines==0 or limit_lines==-1:
if issue.issue_number not in inv.origin:
inv.write({'origin':inv.origin + issue.issue_number +'-'})
inv.write({'invoice_line':[(0,0,invoice_line)]})
inv.write({'issue_ids':[(4,issue.id)]})
backorder.delivery_note_id.write({'invoice_ids':[(4,inv.id)]})
count_lines+=1
else:
if issue.issue_number not in inv.origin:
inv.write({'origin':inv.origin + issue.issue_number})
inv=invoice_obj.create(invoice_dict)
invoices_list.append(inv.id)
count_lines=1
if issue.issue_number not in inv.origin:
inv.write({'origin':inv.origin + issue.issue_number +'-'})
inv.write({'invoice_line':[(0,0,invoice_line)]})
inv.write({'issue_ids':[(4,issue.id)]})
backorder.delivery_note_id.write({'invoice_ids':[(4,inv.id)]})
count_lines+=1
backorder.write({'invoice_state':'invoiced'})
backorder.move_lines.write({'invoice_state':'invoiced'})
backorder.delivery_note_id.write({'state':'invoiced'})
for expense_line in issue.expense_line_ids:
if expense_line.expense_id.state=='done' or expense_line.expense_id.state=='paid':
for move_lines in expense_line.expense_id.account_move_id.line_id:
for lines in move_lines.analytic_lines:
if lines.account_id==expense_line.analytic_account and lines.name==expense_line.name and lines.unit_amount==expense_line.unit_quantity and (lines.amount*-1/lines.unit_amount)==expense_line.unit_amount and not lines.invoice_id:
lines.write({'invoice_id':inv.id})
return invoices_list
url = 'https://finance.yahoo.com/quote/%s/history?p=%s' % (stock_code,
stock_code)
r = requests.get(url)
m = re.findall('"HistoricalPriceStore":{"prices":(.*?),"isPending"',
r.text)
if m:
quotes = json.loads(m[0])
quotes = quotes[::-1]
return [item for item in quotes if not 'type' in item]
quotes = retrieve_quotes_historical('KO')
list1 = []
for i in range(len(quotes)):
x = date.fromtimestamp(quotes[i]['date'])
y = date.strftime(x, '%Y-%m-%d')
list1.append(y)
quotesdf_ori = pd.DataFrame(quotes, index=list1)
# quotesdf_m = quotesdf_ori.drop(['unadjclose'], axis = 1)
quotesdf = quotesdf_ori.drop(['date'], axis=1)
listtemp = []
for i in range(len(quotesdf)):
temp = time.strptime(quotesdf.index[i], "%Y-%m-%d")
listtemp.append(temp.tm_mon)
tempdf = quotesdf.copy()
tempdf['month'] = listtemp
pl.figure(figsize=(8, 6), dpi=50)
closeMeansKO = tempdf.groupby('month').close.mean()
x = closeMeansKO.index
y = closeMeansKO.values
plt.plot(x, y, 'o')
def convert_date(date):
return date.strftime("%Y%m%d")
def init(conffile):
fp = open(conffile, "r")
global BACKUP_VOLS
BACKUP_VOLS = json.load(fp, "utf-8")
for (volume, conf) in BACKUP_VOLS.items():
daylist = savedays[volume] = []
# last n days
for c in range(conf['days'] - 1, -1, -1):
daylist.append(today - timedelta(days=c))
# last n weeks (get mondays)
monday = today - timedelta(days=today.isoweekday() - 1)
daylist.append(monday)
for c in range(conf['weeks'] - 1, 0, -1):
daylist.append(monday - timedelta(days=c * 7))
# last n months (first monday of month)
for c in range(conf['months'] - 1, -1, -1):
year = today.year
month = today.month - c
if month <= 0:
year = year - 1
month = 12 + month
firstmonday = datetime(year, month, 1).date()
if firstmonday.isoweekday() != 1:
firstmonday = firstmonday + timedelta(
days=7 - firstmonday.isoweekday() + 1)
daylist.append(firstmonday)
#daylist.append(datetime(today.year, today.month - c, 1).date())
snapshots_data = local('ec2-describe-snapshots', capture=True).split('\n')
snapshots_data = [
tuple(l.split('\t'))[:9] for l in snapshots_data
if l.startswith('SNAPSHOT')
]
for (_, snapshot, volume, status, datestr, progress, _, _,
_) in snapshots_data:
snapshotdate = dateutil.parser.parse(datestr).date()
if volume in BACKUP_VOLS:
if snapshotdate == today:
hastoday[volume] = {
'status': status,
'snapshot': snapshot,
'progress': progress.replace('%', '')
}
if volume not in snapshots:
snapshots[volume] = []
snapshots[volume].append((snapshot, status, snapshotdate))
# sort by date
for snapshotlist in snapshots.values():
snapshotlist.sort(key=lambda x: x[2], reverse=True)
for volume in BACKUP_VOLS.keys():
if volume not in snapshots:
snapshots[volume] = []
print "VOLUME\tSNAPSHOT\tSTATUS\tDATE\tDESC"
for (volume, snapshotlist) in snapshots.items():
for (snapshot, status, date) in snapshotlist:
datestr = date.strftime('%Y-%m-%d')
print "%s\t%s\t%s\t%s\t%s" % (volume, snapshot, status, datestr,
BACKUP_VOLS[volume]['comment'])
def setLastLoggedIn(date):
date_str = date.strftime("%Y-%m-%d %H:%M:%S")
config.plugins.iptvplayer.shortcut_last_loggin.value = date_str
config.plugins.iptvplayer.shortcut_last_loggin.save()
def get_range_dates(startdate, date_range):
date_ranges = []
for i in range(date_range):
startdate += datetime.timedelta(days=1)
date_ranges.append(date.strftime(startdate, '%m/%d/%y'))
return date_ranges
